plugin-agent-orchestrator 1.1.8 → 1.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (495) hide show
  1. package/LICENSE +201 -0
  2. package/README.md +207 -204
  3. package/client-v2.d.ts +2 -2
  4. package/client-v2.js +1 -1
  5. package/client.d.ts +2 -2
  6. package/client.js +1 -1
  7. package/dist/client/117.6c69b6a87751dd18.js +10 -0
  8. package/dist/client/253.52f62e1bc0e9520a.js +10 -0
  9. package/dist/client/325.8f7f509bdb35e142.js +10 -0
  10. package/dist/client/404.86593f98c9768ed5.js +10 -0
  11. package/dist/client/453.fa2d169576262213.js +10 -0
  12. package/dist/client/53.567db3ae078650c6.js +10 -0
  13. package/dist/client/619.01713281e5e951d3.js +10 -0
  14. package/dist/client/652.69da1598312b3a3f.js +10 -0
  15. package/dist/client/773.648965568ff476f9.js +10 -0
  16. package/dist/client/981.37e7aeecdb1ef6fa.js +10 -0
  17. package/dist/client/993.ac89f3852990918d.js +10 -0
  18. package/dist/client/index.js +1 -1
  19. package/dist/client-v2/117.c982d79ff68738f5.js +10 -0
  20. package/dist/client-v2/214.94c4257bf2838577.js +10 -0
  21. package/dist/client-v2/253.8e19956d2a44dfb8.js +10 -0
  22. package/dist/client-v2/264.80e8e358c8c6c2e5.js +10 -0
  23. package/dist/client-v2/325.fafe5a6e83ba9cb3.js +10 -0
  24. package/dist/client-v2/404.ab032b79c95dab6f.js +10 -0
  25. package/dist/client-v2/{41.3deb9e57a0f64fc4.js → 41.cd5610fcf7446127.js} +1 -1
  26. package/dist/client-v2/453.42c6d70628d56cc2.js +10 -0
  27. package/dist/client-v2/{53.349fff8db80b19d4.js → 53.37c08089a648f89a.js} +1 -1
  28. package/dist/client-v2/{619.0a2144aaffefe018.js → 619.6e9618550101e819.js} +1 -1
  29. package/dist/client-v2/652.eeba4e0c38367321.js +10 -0
  30. package/dist/client-v2/{70.ffff2997c10fd4b0.js → 70.332698e53eb59677.js} +1 -1
  31. package/dist/client-v2/773.4de2ba5acb07318c.js +10 -0
  32. package/dist/client-v2/981.76060f466e6f7c11.js +10 -0
  33. package/dist/client-v2/993.f8cdb5172566ce75.js +10 -0
  34. package/dist/client-v2/index.js +1 -1
  35. package/dist/externalVersion.js +13 -7
  36. package/dist/locale/en-US.json +308 -1
  37. package/dist/locale/vi-VN.json +5 -1
  38. package/dist/locale/zh-CN.json +308 -1
  39. package/dist/node_modules/acorn/LICENSE +21 -0
  40. package/dist/node_modules/acorn/bin/acorn +4 -0
  41. package/dist/node_modules/acorn/dist/acorn.d.mts +857 -0
  42. package/dist/node_modules/acorn/dist/acorn.d.ts +857 -0
  43. package/dist/node_modules/acorn/dist/acorn.js +1 -0
  44. package/dist/node_modules/acorn/dist/bin.js +90 -0
  45. package/dist/node_modules/acorn/package.json +1 -0
  46. package/dist/node_modules/acorn-walk/LICENSE +21 -0
  47. package/dist/node_modules/acorn-walk/dist/walk.d.mts +177 -0
  48. package/dist/node_modules/acorn-walk/dist/walk.d.ts +177 -0
  49. package/dist/node_modules/acorn-walk/dist/walk.js +1 -0
  50. package/dist/node_modules/acorn-walk/node_modules/.bin/acorn +15 -0
  51. package/dist/node_modules/acorn-walk/node_modules/.bin/acorn.cmd +7 -0
  52. package/dist/node_modules/acorn-walk/node_modules/acorn/bin/acorn +4 -0
  53. package/dist/node_modules/acorn-walk/node_modules/acorn/dist/acorn.d.mts +866 -0
  54. package/dist/node_modules/acorn-walk/node_modules/acorn/dist/acorn.d.ts +866 -0
  55. package/dist/node_modules/acorn-walk/node_modules/acorn/dist/acorn.js +6174 -0
  56. package/dist/node_modules/acorn-walk/node_modules/acorn/dist/bin.js +90 -0
  57. package/dist/node_modules/acorn-walk/node_modules/acorn/package.json +50 -0
  58. package/dist/node_modules/acorn-walk/package.json +1 -0
  59. package/dist/node_modules/adm-zip/adm-zip.js +1 -1
  60. package/dist/node_modules/adm-zip/package.json +1 -1
  61. package/dist/node_modules/ajv/package.json +1 -1
  62. package/dist/node_modules/zod/index.cjs +1 -1
  63. package/dist/node_modules/zod/package.json +1 -1
  64. package/dist/server/collections/agent-auto-approval-checkpoints.js +47 -0
  65. package/dist/server/collections/agent-execution-spans.js +148 -9
  66. package/dist/server/collections/agent-governance-audit-events.js +47 -0
  67. package/dist/server/collections/agent-governance-policies.js +47 -0
  68. package/dist/server/collections/agent-governance-policy-versions.js +48 -0
  69. package/dist/server/collections/agent-governance-prompt-bindings.js +50 -0
  70. package/dist/server/collections/agent-hierarchy-invocations.js +59 -0
  71. package/dist/server/collections/agent-loop-runs.js +23 -5
  72. package/dist/server/collections/agent-routing-graph-locks.js +40 -0
  73. package/dist/server/collections/agent-session-auto-approvals.js +45 -0
  74. package/dist/server/collections/orchestrator-config.js +17 -0
  75. package/dist/server/migrations/20261110000000-add-usage-pricing-snapshots.js +117 -0
  76. package/dist/server/migrations/20261110010000-add-usage-department-attribution.js +54 -0
  77. package/dist/server/migrations/20261119000000-widen-native-employee-identifiers.js +59 -0
  78. package/dist/server/plugin.js +169 -33
  79. package/dist/server/resources/agent-auto-approval.js +80 -0
  80. package/dist/server/resources/agent-governance.js +192 -0
  81. package/dist/server/resources/agent-harness-profile.js +11 -3
  82. package/dist/server/resources/agent-monitor.js +109 -339
  83. package/dist/server/resources/agent-usage.js +423 -0
  84. package/dist/server/resources/hierarchical-routing.js +60 -0
  85. package/dist/server/resources/native-monitor-format.js +179 -0
  86. package/dist/server/resources/tracing.js +35 -12
  87. package/dist/server/services/AgentGovernanceService.js +500 -0
  88. package/dist/server/services/AutoApprovalCheckpoint.js +133 -0
  89. package/dist/server/services/CodeValidator.js +8 -4
  90. package/dist/server/services/DepartmentAttribution.js +65 -0
  91. package/dist/server/services/ExecutionSpanService.js +96 -3
  92. package/dist/server/services/GovernanceAdmission.js +271 -0
  93. package/dist/server/services/HarnessCapabilities.js +118 -0
  94. package/dist/server/services/HarnessCompiler.js +3 -0
  95. package/dist/server/services/HarnessSchema.js +13 -1
  96. package/dist/server/services/HierarchicalAgentRuntime.js +925 -0
  97. package/dist/server/services/HierarchyEnrollmentGuard.js +216 -0
  98. package/dist/server/services/HierarchyInvocationJournal.js +157 -0
  99. package/dist/server/services/HierarchyRequestCancellation.js +150 -0
  100. package/dist/server/services/LoopBudgetService.js +29 -11
  101. package/dist/server/services/LoopRunRepository.js +16 -7
  102. package/dist/server/services/LoopRunStateMachine.js +20 -9
  103. package/dist/server/services/LoopTriggerService.js +1 -1
  104. package/dist/server/services/LoopWorkerService.js +71 -37
  105. package/dist/server/services/ManagedPromptProtection.js +99 -0
  106. package/dist/server/services/NativeAgentMonitorSyncService.js +584 -0
  107. package/dist/server/services/NativeSubAgentObserver.js +141 -508
  108. package/dist/server/services/NativeUsageAttribution.js +97 -0
  109. package/dist/server/services/PluginAiRuntimeAdapter.js +107 -9
  110. package/dist/server/services/RoutingBindingRepository.js +156 -0
  111. package/dist/server/services/RoutingBindingValidation.js +161 -0
  112. package/dist/server/services/SandboxRunner.js +19 -0
  113. package/dist/server/services/SessionAutoApproval.js +173 -0
  114. package/dist/server/services/TokenTracker.js +787 -104
  115. package/dist/server/services/TracingRetentionPolicy.js +66 -0
  116. package/dist/server/services/VerificationService.js +20 -10
  117. package/dist/server/skill-hub/plugin.js +17 -0
  118. package/dist/server/skill-hub/sandbox-config.json +6 -6
  119. package/dist/server/skill-hub/services/RegistrySkillInstallationService.js +2 -2
  120. package/dist/server/skill-hub/services/registry-node-package.js +191 -0
  121. package/dist/server/skill-hub/tasks/SkillExecutionTask.js +21 -6
  122. package/dist/server/tools/skill-execute.js +1 -0
  123. package/dist/server/utils/ensure-indexes.js +15 -5
  124. package/package.json +66 -53
  125. package/server.d.ts +2 -2
  126. package/server.js +1 -1
  127. package/src/client/AgentRunsTab.tsx +2 -2
  128. package/src/client/ApprovalsTab.tsx +2 -2
  129. package/src/client/HarnessProfilesTab.tsx +2 -2
  130. package/src/client/OrchestratorSettings.tsx +152 -152
  131. package/src/client/SkillPages.tsx +34 -0
  132. package/src/client/__tests__/embedded-settings.test.tsx +225 -0
  133. package/src/client/hooks/useRunEventStream.ts +76 -76
  134. package/src/client/index.tsx +2 -2
  135. package/src/client/plugin.tsx +71 -33
  136. package/src/client/skill-hub/components/ExecutionHistory.tsx +200 -200
  137. package/src/client/skill-hub/components/GitSkillImport.tsx +562 -562
  138. package/src/client/skill-hub/components/LoopSettings.tsx +331 -331
  139. package/src/client/skill-hub/components/SkillEditor.tsx +460 -460
  140. package/src/client/skill-hub/components/SkillManager.tsx +194 -194
  141. package/src/client/skill-hub/components/SkillMetrics.tsx +162 -162
  142. package/src/client/skill-hub/components/SkillTestPanel.tsx +141 -141
  143. package/src/client/skill-hub/index.tsx +6 -6
  144. package/src/client/skill-hub/locale.ts +16 -16
  145. package/src/client/skill-hub/tools/InteractionSchemasProvider.tsx +147 -132
  146. package/src/client/skill-hub/tools/SkillHubCard.tsx +110 -104
  147. package/src/client/skill-hub/tools/__tests__/skill-hub-card.test.tsx +255 -0
  148. package/src/client/skill-hub/tools/loopTemplates.ts +52 -52
  149. package/src/client/skill-hub/tools/registerSkillLoopCards.ts +80 -71
  150. package/src/client/skill-hub/utils/jsonFields.ts +41 -41
  151. package/src/client-v2/__tests__/embedded-settings.test.tsx +202 -0
  152. package/src/client-v2/__tests__/hierarchical-routing-settings.test.tsx +350 -0
  153. package/src/client-v2/auto-approval/AutoApprovalBridge.tsx +513 -0
  154. package/src/client-v2/auto-approval/AutoApprovalProvider.tsx +399 -0
  155. package/src/client-v2/auto-approval/__tests__/session-auto-approval.test.tsx +1676 -0
  156. package/src/client-v2/components/AIEmployeeSelect.tsx +47 -47
  157. package/src/client-v2/components/AIEmployeesContext.tsx +97 -97
  158. package/src/client-v2/components/AgentBindingsTab.tsx +541 -269
  159. package/src/client-v2/components/AgentRunsTab.tsx +85 -83
  160. package/src/client-v2/components/ApprovalsTab.tsx +337 -337
  161. package/src/client-v2/components/EmployeeGovernanceTab.tsx +695 -0
  162. package/src/client-v2/components/HarnessProfilesTab.tsx +700 -541
  163. package/src/client-v2/components/HarnessSettingsForm.tsx +221 -0
  164. package/src/client-v2/components/KnowledgeAccessTab.tsx +116 -116
  165. package/src/client-v2/components/RetrievalTraceTab.tsx +162 -162
  166. package/src/client-v2/components/UsageGovernanceTab.tsx +379 -0
  167. package/src/client-v2/index.tsx +1 -1
  168. package/src/client-v2/pages/AgentBindingsPage.tsx +15 -13
  169. package/src/client-v2/pages/ApprovalsPage.tsx +12 -10
  170. package/src/client-v2/pages/EmployeeGovernancePage.tsx +15 -0
  171. package/src/client-v2/pages/ExecutionHistoryPage.tsx +12 -10
  172. package/src/client-v2/pages/HarnessProfilesPage.tsx +12 -10
  173. package/src/client-v2/pages/KnowledgeAccessPage.tsx +12 -10
  174. package/src/client-v2/pages/LoopSettingsPage.tsx +12 -10
  175. package/src/client-v2/pages/MemoryInspectorPage.tsx +15 -13
  176. package/src/client-v2/pages/RetrievalTracePage.tsx +12 -10
  177. package/src/client-v2/pages/RulesPage.tsx +15 -13
  178. package/src/client-v2/pages/SkillDefinitionsPage.tsx +12 -10
  179. package/src/client-v2/pages/SkillMetricsPage.tsx +12 -10
  180. package/src/client-v2/pages/TracingPage.tsx +15 -13
  181. package/src/client-v2/pages/UsageGovernancePage.tsx +15 -0
  182. package/src/client-v2/plugin.tsx +134 -111
  183. package/src/client-v2/skill-hub/components/ExecutionHistory.tsx +196 -196
  184. package/src/client-v2/skill-hub/components/FileLinkList.tsx +37 -37
  185. package/src/client-v2/skill-hub/components/GitSkillImport.tsx +539 -539
  186. package/src/client-v2/skill-hub/components/LoopSettings.tsx +331 -331
  187. package/src/client-v2/skill-hub/components/SkillEditor.tsx +453 -453
  188. package/src/client-v2/skill-hub/components/SkillManager.tsx +191 -174
  189. package/src/client-v2/skill-hub/components/SkillMetrics.tsx +162 -162
  190. package/src/client-v2/skill-hub/components/SkillTestPanel.tsx +135 -135
  191. package/src/client-v2/skill-hub/locale.ts +13 -13
  192. package/src/client-v2/skill-hub/tools/InteractionSchemasProvider.tsx +62 -0
  193. package/src/client-v2/skill-hub/tools/SkillHubCard.tsx +104 -0
  194. package/src/client-v2/skill-hub/tools/__tests__/interactions.test.ts +126 -0
  195. package/src/client-v2/skill-hub/tools/interactions.ts +137 -0
  196. package/src/client-v2/skill-hub/tools/loopTemplates.ts +52 -52
  197. package/src/client-v2/skill-hub/utils/jsonFields.ts +41 -41
  198. package/src/client-v2/utils/jsonFields.ts +41 -41
  199. package/src/index.ts +2 -2
  200. package/src/locale/en-US.json +308 -1
  201. package/src/locale/vi-VN.json +5 -1
  202. package/src/locale/zh-CN.json +308 -1
  203. package/src/server/__tests__/agent-execution-context.test.ts +18 -18
  204. package/src/server/__tests__/agent-governance.integration.test.ts +385 -0
  205. package/src/server/__tests__/agent-knowledge-insights.test.ts +123 -123
  206. package/src/server/__tests__/agent-monitor-acl.integration.test.ts +267 -0
  207. package/src/server/__tests__/agent-registry-service.test.ts +147 -147
  208. package/src/server/__tests__/agent-runtime-lifecycle.test.ts +46 -46
  209. package/src/server/__tests__/agent-usage-report.integration.test.ts +335 -0
  210. package/src/server/__tests__/auto-approval-checkpoint.integration.test.ts +295 -0
  211. package/src/server/__tests__/code-validator.test.ts +63 -63
  212. package/src/server/__tests__/external-rag-search.test.ts +107 -107
  213. package/src/server/__tests__/harness-capabilities.test.ts +88 -0
  214. package/src/server/__tests__/harness-compiler.test.ts +278 -214
  215. package/src/server/__tests__/harness-profile-service.test.ts +167 -167
  216. package/src/server/__tests__/harness-runtime-policy.test.ts +293 -293
  217. package/src/server/__tests__/harness-tool-plan.test.ts +175 -172
  218. package/src/server/__tests__/hierarchical-routing.integration.test.ts +2318 -0
  219. package/src/server/__tests__/hierarchy-invocation-journal.integration.test.ts +119 -0
  220. package/src/server/__tests__/input-schema-validator.test.ts +26 -26
  221. package/src/server/__tests__/loop-approval-decision.test.ts +624 -512
  222. package/src/server/__tests__/loop-approval-expiry.test.ts +258 -258
  223. package/src/server/__tests__/loop-budget-service.test.ts +29 -0
  224. package/src/server/__tests__/loop-pattern-schema.test.ts +73 -73
  225. package/src/server/__tests__/loop-run-repository.test.ts +25 -0
  226. package/src/server/__tests__/loop-run-state-machine.test.ts +101 -101
  227. package/src/server/__tests__/loop-trigger-service.test.ts +185 -148
  228. package/src/server/__tests__/loop-worker-claim.test.ts +110 -10
  229. package/src/server/__tests__/loop-worker-execution.test.ts +375 -317
  230. package/src/server/__tests__/native-agent-monitor-sync.test.ts +343 -0
  231. package/src/server/__tests__/native-hierarchy-probe.integration.test.ts +527 -0
  232. package/src/server/__tests__/native-sub-agent-observer.test.ts +463 -662
  233. package/src/server/__tests__/native-usage-attribution.integration.test.ts +168 -0
  234. package/src/server/__tests__/plugin-ai-runtime-adapter.test.ts +403 -0
  235. package/src/server/__tests__/registry-node-package.test.ts +299 -0
  236. package/src/server/__tests__/routing-binding-graph.integration.test.ts +217 -0
  237. package/src/server/__tests__/routing-binding-validation.test.ts +256 -0
  238. package/src/server/__tests__/session-auto-approval.integration.test.ts +329 -0
  239. package/src/server/__tests__/skill-access-service.test.ts +64 -64
  240. package/src/server/__tests__/skill-execute.test.ts +81 -70
  241. package/src/server/__tests__/skill-execution-task-policy.test.ts +140 -140
  242. package/src/server/__tests__/skill-review-guard.test.ts +171 -0
  243. package/src/server/__tests__/skill-sandbox-worker-mode.test.ts +15 -15
  244. package/src/server/__tests__/skill-settings.test.ts +63 -63
  245. package/src/server/__tests__/skill-tool-name.test.ts +23 -23
  246. package/src/server/__tests__/token-tracker.test.ts +632 -0
  247. package/src/server/__tests__/tool-loop-detection.test.ts +276 -276
  248. package/src/server/__tests__/tracing-retention-policy.test.ts +30 -0
  249. package/src/server/__tests__/usage-ledger.integration.test.ts +187 -0
  250. package/src/server/collections/agent-auto-approval-checkpoints.ts +18 -0
  251. package/src/server/collections/agent-execution-spans.ts +386 -245
  252. package/src/server/collections/agent-governance-audit-events.ts +19 -0
  253. package/src/server/collections/agent-governance-policies.ts +19 -0
  254. package/src/server/collections/agent-governance-policy-versions.ts +20 -0
  255. package/src/server/collections/agent-governance-prompt-bindings.ts +22 -0
  256. package/src/server/collections/agent-hierarchy-invocations.ts +30 -0
  257. package/src/server/collections/agent-loop-action-approvals.ts +39 -39
  258. package/src/server/collections/agent-loop-runs.ts +389 -369
  259. package/src/server/collections/agent-loop-steps.ts +220 -220
  260. package/src/server/collections/agent-memory-contexts.ts +95 -95
  261. package/src/server/collections/agent-routing-graph-locks.ts +11 -0
  262. package/src/server/collections/agent-session-auto-approvals.ts +16 -0
  263. package/src/server/collections/orchestrator-config.ts +113 -87
  264. package/src/server/collections/orchestrator-logs.ts +116 -116
  265. package/src/server/collections/skill-definitions.ts +159 -159
  266. package/src/server/collections/skill-executions.ts +154 -154
  267. package/src/server/collections/skill-loop-configs.ts +65 -65
  268. package/src/server/collections/skill-metrics.ts +36 -36
  269. package/src/server/collections/skill-registry-installations.ts +41 -41
  270. package/src/server/collections/skill-worker-configs.ts +86 -86
  271. package/src/server/index.ts +1 -1
  272. package/src/server/migrations/20260423000000-add-progress-fields.ts +50 -50
  273. package/src/server/migrations/20260425000000-add-interaction-schema.ts +37 -37
  274. package/src/server/migrations/20260427000000-add-tracing-detail-fields.ts +41 -41
  275. package/src/server/migrations/20260427000001-change-other-json-to-text.ts +61 -61
  276. package/src/server/migrations/20260524001000-add-plan-approval-and-harness-profiles.ts +145 -145
  277. package/src/server/migrations/20260601000000-add-token-fields.ts +89 -89
  278. package/src/server/migrations/20260615000000-normalize-ai-employee-tool-bindings.ts +39 -39
  279. package/src/server/migrations/20260621000000-native-policy-profile-defaults.ts +190 -190
  280. package/src/server/migrations/20260727000000-remove-unused-knowledge-scopes.ts +25 -25
  281. package/src/server/migrations/20260727010000-backfill-skill-tool-names.ts +45 -45
  282. package/src/server/migrations/20261110000000-add-usage-pricing-snapshots.ts +110 -0
  283. package/src/server/migrations/20261110010000-add-usage-department-attribution.ts +28 -0
  284. package/src/server/migrations/20261119000000-widen-native-employee-identifiers.ts +37 -0
  285. package/src/server/plugin.ts +724 -573
  286. package/src/server/resources/agent-auto-approval.ts +51 -0
  287. package/src/server/resources/agent-governance.ts +161 -0
  288. package/src/server/resources/agent-harness-profile.ts +103 -94
  289. package/src/server/resources/agent-knowledge-insights.ts +306 -306
  290. package/src/server/resources/agent-loop-approval.ts +84 -84
  291. package/src/server/resources/agent-loop-events-stream.ts +108 -108
  292. package/src/server/resources/agent-loop.ts +252 -252
  293. package/src/server/resources/agent-monitor.ts +208 -482
  294. package/src/server/resources/agent-usage.ts +413 -0
  295. package/src/server/resources/hierarchical-routing.ts +39 -0
  296. package/src/server/resources/native-monitor-format.ts +150 -0
  297. package/src/server/resources/tracing.ts +296 -267
  298. package/src/server/services/AgentExecutionContext.ts +23 -23
  299. package/src/server/services/AgentGovernanceService.ts +541 -0
  300. package/src/server/services/AgentMemoryContextService.ts +344 -344
  301. package/src/server/services/AgentRegistryService.ts +220 -220
  302. package/src/server/services/AgentRuntimeLifecycle.ts +64 -64
  303. package/src/server/services/AutoApprovalCheckpoint.ts +140 -0
  304. package/src/server/services/CodeValidator.ts +257 -238
  305. package/src/server/services/DepartmentAttribution.ts +47 -0
  306. package/src/server/services/DurableCircuitBreakerService.ts +235 -235
  307. package/src/server/services/ExecutionSpanService.ts +271 -146
  308. package/src/server/services/FileManager.ts +159 -159
  309. package/src/server/services/GovernanceAdmission.ts +314 -0
  310. package/src/server/services/HarnessCapabilities.ts +112 -0
  311. package/src/server/services/HarnessCompiler.ts +182 -170
  312. package/src/server/services/HarnessProfileService.ts +235 -235
  313. package/src/server/services/HarnessSchema.ts +224 -194
  314. package/src/server/services/HierarchicalAgentRuntime.ts +1052 -0
  315. package/src/server/services/HierarchyEnrollmentGuard.ts +211 -0
  316. package/src/server/services/HierarchyInvocationJournal.ts +172 -0
  317. package/src/server/services/HierarchyRequestCancellation.ts +132 -0
  318. package/src/server/services/InputSchemaValidator.ts +44 -44
  319. package/src/server/services/LoopBudgetService.ts +325 -271
  320. package/src/server/services/LoopControlService.ts +120 -120
  321. package/src/server/services/LoopPatternSchema.ts +231 -231
  322. package/src/server/services/LoopPatternService.ts +171 -171
  323. package/src/server/services/LoopRunRepository.ts +343 -332
  324. package/src/server/services/LoopRunStateMachine.ts +538 -525
  325. package/src/server/services/LoopSchedulerService.ts +195 -195
  326. package/src/server/services/LoopTriggerService.ts +162 -159
  327. package/src/server/services/LoopWorkerService.ts +975 -912
  328. package/src/server/services/ManagedPromptProtection.ts +81 -0
  329. package/src/server/services/NativeAgentMonitorSyncService.ts +684 -0
  330. package/src/server/services/NativeSubAgentObserver.ts +324 -757
  331. package/src/server/services/NativeUsageAttribution.ts +91 -0
  332. package/src/server/services/PathLockService.ts +317 -317
  333. package/src/server/services/PluginAiRuntimeAdapter.ts +518 -387
  334. package/src/server/services/RoutingBindingRepository.ts +146 -0
  335. package/src/server/services/RoutingBindingValidation.ts +161 -0
  336. package/src/server/services/RunEventBus.ts +58 -58
  337. package/src/server/services/SandboxRunner.ts +526 -492
  338. package/src/server/services/SessionAutoApproval.ts +167 -0
  339. package/src/server/services/SkillAccessService.ts +175 -175
  340. package/src/server/services/SkillManager.ts +585 -585
  341. package/src/server/services/SkillRepositoryService.ts +236 -236
  342. package/src/server/services/TokenTracker.ts +1151 -287
  343. package/src/server/services/ToolLoopDetectionService.ts +210 -210
  344. package/src/server/services/TracingRetentionPolicy.ts +49 -0
  345. package/src/server/services/VerificationSchema.ts +31 -31
  346. package/src/server/services/VerificationService.ts +325 -313
  347. package/src/server/services/WorkerEnvManager.ts +198 -198
  348. package/src/server/services/__tests__/SkillRepositoryService.test.ts +33 -33
  349. package/src/server/services/harness-runtime-policy.ts +134 -134
  350. package/src/server/services/pythonGuardTemplate.ts +87 -87
  351. package/src/server/services/sandboxCapabilities.ts +348 -348
  352. package/src/server/skill-hub/actions/__tests__/git-import.test.ts +563 -563
  353. package/src/server/skill-hub/actions/git-import.ts +936 -936
  354. package/src/server/skill-hub/mcp/McpController.ts +107 -107
  355. package/src/server/skill-hub/middlewares/git-import-request-method-policy.ts +51 -51
  356. package/src/server/skill-hub/plugin.ts +1272 -1232
  357. package/src/server/skill-hub/sandbox-config.json +6 -6
  358. package/src/server/skill-hub/services/RegistrySkillInstallationService.ts +291 -281
  359. package/src/server/skill-hub/services/__tests__/RegistrySkillInstallationService.test.ts +295 -295
  360. package/src/server/skill-hub/services/registry-node-package.ts +191 -0
  361. package/src/server/skill-hub/tasks/SkillExecutionTask.ts +597 -568
  362. package/src/server/skill-hub/utils/json-fields.ts +57 -57
  363. package/src/server/tools/delegate-task.ts +30 -30
  364. package/src/server/tools/external-rag-search.ts +329 -329
  365. package/src/server/tools/skill-execute.ts +226 -216
  366. package/src/server/types.ts +21 -21
  367. package/src/server/utils/ai-manager.ts +24 -24
  368. package/src/server/utils/ctx-utils.ts +132 -132
  369. package/src/server/utils/ensure-indexes.ts +113 -96
  370. package/src/server/utils/ha-lock.ts +80 -80
  371. package/src/server/utils/loop-pattern-context.ts +84 -84
  372. package/src/server/utils/record-utils.ts +19 -19
  373. package/src/server/utils/skill-settings.ts +135 -135
  374. package/src/server/utils/skill-tool-name.ts +58 -58
  375. package/dist/client-v2/117.7565cff57c39318a.js +0 -10
  376. package/dist/client-v2/214.106635011859dede.js +0 -10
  377. package/dist/client-v2/253.59577b40d9875be0.js +0 -10
  378. package/dist/client-v2/264.7ed8bee71fa5935c.js +0 -10
  379. package/dist/client-v2/418.41ed200e7d6da332.js +0 -10
  380. package/dist/client-v2/794.a67a2614405ba627.js +0 -10
  381. package/dist/client-v2/981.e4a1f41cfc94a856.js +0 -10
  382. package/dist/client-v2/993.b0688fdc5ccb000a.js +0 -10
  383. package/dist/node_modules/simple-git/dist/cjs/index.js +0 -7399
  384. package/dist/node_modules/simple-git/dist/esm/index.js +0 -4745
  385. package/dist/node_modules/simple-git/dist/esm/package.json +0 -3
  386. package/dist/node_modules/simple-git/dist/src/lib/api.d.ts +0 -13
  387. package/dist/node_modules/simple-git/dist/src/lib/args/log-format.d.ts +0 -9
  388. package/dist/node_modules/simple-git/dist/src/lib/errors/git-construct-error.d.ts +0 -15
  389. package/dist/node_modules/simple-git/dist/src/lib/errors/git-error.d.ts +0 -30
  390. package/dist/node_modules/simple-git/dist/src/lib/errors/git-plugin-error.d.ts +0 -7
  391. package/dist/node_modules/simple-git/dist/src/lib/errors/git-response-error.d.ts +0 -32
  392. package/dist/node_modules/simple-git/dist/src/lib/errors/task-configuration-error.d.ts +0 -12
  393. package/dist/node_modules/simple-git/dist/src/lib/git-factory.d.ts +0 -15
  394. package/dist/node_modules/simple-git/dist/src/lib/git-logger.d.ts +0 -21
  395. package/dist/node_modules/simple-git/dist/src/lib/parsers/parse-branch-delete.d.ts +0 -5
  396. package/dist/node_modules/simple-git/dist/src/lib/parsers/parse-branch.d.ts +0 -2
  397. package/dist/node_modules/simple-git/dist/src/lib/parsers/parse-commit.d.ts +0 -2
  398. package/dist/node_modules/simple-git/dist/src/lib/parsers/parse-diff-summary.d.ts +0 -3
  399. package/dist/node_modules/simple-git/dist/src/lib/parsers/parse-fetch.d.ts +0 -2
  400. package/dist/node_modules/simple-git/dist/src/lib/parsers/parse-list-log-summary.d.ts +0 -6
  401. package/dist/node_modules/simple-git/dist/src/lib/parsers/parse-merge.d.ts +0 -11
  402. package/dist/node_modules/simple-git/dist/src/lib/parsers/parse-move.d.ts +0 -2
  403. package/dist/node_modules/simple-git/dist/src/lib/parsers/parse-pull.d.ts +0 -6
  404. package/dist/node_modules/simple-git/dist/src/lib/parsers/parse-push.d.ts +0 -4
  405. package/dist/node_modules/simple-git/dist/src/lib/parsers/parse-remote-messages.d.ts +0 -5
  406. package/dist/node_modules/simple-git/dist/src/lib/parsers/parse-remote-objects.d.ts +0 -3
  407. package/dist/node_modules/simple-git/dist/src/lib/plugins/abort-plugin.d.ts +0 -3
  408. package/dist/node_modules/simple-git/dist/src/lib/plugins/block-unsafe-operations-plugin.d.ts +0 -3
  409. package/dist/node_modules/simple-git/dist/src/lib/plugins/command-config-prefixing-plugin.d.ts +0 -2
  410. package/dist/node_modules/simple-git/dist/src/lib/plugins/completion-detection.plugin.d.ts +0 -3
  411. package/dist/node_modules/simple-git/dist/src/lib/plugins/custom-binary.plugin.d.ts +0 -3
  412. package/dist/node_modules/simple-git/dist/src/lib/plugins/error-detection.plugin.d.ts +0 -7
  413. package/dist/node_modules/simple-git/dist/src/lib/plugins/index.d.ts +0 -11
  414. package/dist/node_modules/simple-git/dist/src/lib/plugins/plugin-store.d.ts +0 -11
  415. package/dist/node_modules/simple-git/dist/src/lib/plugins/progress-monitor-plugin.d.ts +0 -3
  416. package/dist/node_modules/simple-git/dist/src/lib/plugins/simple-git-plugin.d.ts +0 -48
  417. package/dist/node_modules/simple-git/dist/src/lib/plugins/spawn-options-plugin.d.ts +0 -3
  418. package/dist/node_modules/simple-git/dist/src/lib/plugins/suffix-paths.plugin.d.ts +0 -2
  419. package/dist/node_modules/simple-git/dist/src/lib/plugins/timout-plugin.d.ts +0 -3
  420. package/dist/node_modules/simple-git/dist/src/lib/responses/BranchDeleteSummary.d.ts +0 -12
  421. package/dist/node_modules/simple-git/dist/src/lib/responses/BranchSummary.d.ts +0 -14
  422. package/dist/node_modules/simple-git/dist/src/lib/responses/CheckIgnore.d.ts +0 -4
  423. package/dist/node_modules/simple-git/dist/src/lib/responses/CleanSummary.d.ts +0 -9
  424. package/dist/node_modules/simple-git/dist/src/lib/responses/ConfigList.d.ts +0 -13
  425. package/dist/node_modules/simple-git/dist/src/lib/responses/DiffSummary.d.ts +0 -10
  426. package/dist/node_modules/simple-git/dist/src/lib/responses/FileStatusSummary.d.ts +0 -9
  427. package/dist/node_modules/simple-git/dist/src/lib/responses/GetRemoteSummary.d.ts +0 -11
  428. package/dist/node_modules/simple-git/dist/src/lib/responses/InitSummary.d.ts +0 -9
  429. package/dist/node_modules/simple-git/dist/src/lib/responses/MergeSummary.d.ts +0 -16
  430. package/dist/node_modules/simple-git/dist/src/lib/responses/PullSummary.d.ts +0 -25
  431. package/dist/node_modules/simple-git/dist/src/lib/responses/StatusSummary.d.ts +0 -19
  432. package/dist/node_modules/simple-git/dist/src/lib/responses/TagList.d.ts +0 -7
  433. package/dist/node_modules/simple-git/dist/src/lib/runners/git-executor-chain.d.ts +0 -25
  434. package/dist/node_modules/simple-git/dist/src/lib/runners/git-executor.d.ts +0 -14
  435. package/dist/node_modules/simple-git/dist/src/lib/runners/promise-wrapped.d.ts +0 -2
  436. package/dist/node_modules/simple-git/dist/src/lib/runners/scheduler.d.ts +0 -11
  437. package/dist/node_modules/simple-git/dist/src/lib/runners/tasks-pending-queue.d.ts +0 -23
  438. package/dist/node_modules/simple-git/dist/src/lib/simple-git-api.d.ts +0 -20
  439. package/dist/node_modules/simple-git/dist/src/lib/task-callback.d.ts +0 -2
  440. package/dist/node_modules/simple-git/dist/src/lib/tasks/apply-patch.d.ts +0 -3
  441. package/dist/node_modules/simple-git/dist/src/lib/tasks/branch.d.ts +0 -7
  442. package/dist/node_modules/simple-git/dist/src/lib/tasks/change-working-directory.d.ts +0 -2
  443. package/dist/node_modules/simple-git/dist/src/lib/tasks/check-ignore.d.ts +0 -2
  444. package/dist/node_modules/simple-git/dist/src/lib/tasks/check-is-repo.d.ts +0 -9
  445. package/dist/node_modules/simple-git/dist/src/lib/tasks/checkout.d.ts +0 -2
  446. package/dist/node_modules/simple-git/dist/src/lib/tasks/clean.d.ts +0 -25
  447. package/dist/node_modules/simple-git/dist/src/lib/tasks/clone.d.ts +0 -9
  448. package/dist/node_modules/simple-git/dist/src/lib/tasks/commit.d.ts +0 -4
  449. package/dist/node_modules/simple-git/dist/src/lib/tasks/config.d.ts +0 -8
  450. package/dist/node_modules/simple-git/dist/src/lib/tasks/count-objects.d.ts +0 -12
  451. package/dist/node_modules/simple-git/dist/src/lib/tasks/diff-name-status.d.ts +0 -12
  452. package/dist/node_modules/simple-git/dist/src/lib/tasks/diff.d.ts +0 -5
  453. package/dist/node_modules/simple-git/dist/src/lib/tasks/fetch.d.ts +0 -4
  454. package/dist/node_modules/simple-git/dist/src/lib/tasks/first-commit.d.ts +0 -2
  455. package/dist/node_modules/simple-git/dist/src/lib/tasks/grep.d.ts +0 -12
  456. package/dist/node_modules/simple-git/dist/src/lib/tasks/hash-object.d.ts +0 -5
  457. package/dist/node_modules/simple-git/dist/src/lib/tasks/init.d.ts +0 -3
  458. package/dist/node_modules/simple-git/dist/src/lib/tasks/log.d.ts +0 -32
  459. package/dist/node_modules/simple-git/dist/src/lib/tasks/merge.d.ts +0 -4
  460. package/dist/node_modules/simple-git/dist/src/lib/tasks/move.d.ts +0 -3
  461. package/dist/node_modules/simple-git/dist/src/lib/tasks/pull.d.ts +0 -3
  462. package/dist/node_modules/simple-git/dist/src/lib/tasks/push.d.ts +0 -9
  463. package/dist/node_modules/simple-git/dist/src/lib/tasks/remote.d.ts +0 -8
  464. package/dist/node_modules/simple-git/dist/src/lib/tasks/reset.d.ts +0 -11
  465. package/dist/node_modules/simple-git/dist/src/lib/tasks/show.d.ts +0 -2
  466. package/dist/node_modules/simple-git/dist/src/lib/tasks/stash-list.d.ts +0 -4
  467. package/dist/node_modules/simple-git/dist/src/lib/tasks/status.d.ts +0 -3
  468. package/dist/node_modules/simple-git/dist/src/lib/tasks/sub-module.d.ts +0 -5
  469. package/dist/node_modules/simple-git/dist/src/lib/tasks/tag.d.ts +0 -18
  470. package/dist/node_modules/simple-git/dist/src/lib/tasks/task.d.ts +0 -14
  471. package/dist/node_modules/simple-git/dist/src/lib/tasks/version.d.ts +0 -9
  472. package/dist/node_modules/simple-git/dist/src/lib/types/handlers.d.ts +0 -21
  473. package/dist/node_modules/simple-git/dist/src/lib/types/index.d.ts +0 -136
  474. package/dist/node_modules/simple-git/dist/src/lib/types/tasks.d.ts +0 -19
  475. package/dist/node_modules/simple-git/dist/src/lib/utils/argument-filters.d.ts +0 -14
  476. package/dist/node_modules/simple-git/dist/src/lib/utils/exit-codes.d.ts +0 -10
  477. package/dist/node_modules/simple-git/dist/src/lib/utils/git-output-streams.d.ts +0 -7
  478. package/dist/node_modules/simple-git/dist/src/lib/utils/index.d.ts +0 -8
  479. package/dist/node_modules/simple-git/dist/src/lib/utils/line-parser.d.ts +0 -15
  480. package/dist/node_modules/simple-git/dist/src/lib/utils/simple-git-options.d.ts +0 -2
  481. package/dist/node_modules/simple-git/dist/src/lib/utils/task-options.d.ts +0 -13
  482. package/dist/node_modules/simple-git/dist/src/lib/utils/task-parser.d.ts +0 -5
  483. package/dist/node_modules/simple-git/dist/src/lib/utils/util.d.ts +0 -47
  484. package/dist/node_modules/simple-git/dist/typings/errors.d.ts +0 -5
  485. package/dist/node_modules/simple-git/dist/typings/index.d.ts +0 -14
  486. package/dist/node_modules/simple-git/dist/typings/response.d.ts +0 -556
  487. package/dist/node_modules/simple-git/dist/typings/simple-git.d.ts +0 -1033
  488. package/dist/node_modules/simple-git/dist/typings/types.d.ts +0 -22
  489. package/dist/node_modules/simple-git/node_modules/debug/package.json +0 -64
  490. package/dist/node_modules/simple-git/node_modules/debug/src/browser.js +0 -272
  491. package/dist/node_modules/simple-git/node_modules/debug/src/common.js +0 -292
  492. package/dist/node_modules/simple-git/node_modules/debug/src/index.js +0 -10
  493. package/dist/node_modules/simple-git/node_modules/debug/src/node.js +0 -263
  494. package/dist/node_modules/simple-git/package.json +0 -1
  495. package/dist/node_modules/simple-git/promise.js +0 -17
@@ -1,912 +1,975 @@
1
- import { createHash, randomUUID } from 'node:crypto';
2
- import type { Database, Model } from '@nocobase/database';
3
- import type { Application } from '@nocobase/server';
4
- import { DurableCircuitBreakerService } from './DurableCircuitBreakerService';
5
- import type { CompiledHarness } from './HarnessCompiler';
6
- import { LoopBudgetService } from './LoopBudgetService';
7
- import { LoopControlService } from './LoopControlService';
8
- import { loopPatternPolicySchema } from './LoopPatternSchema';
9
- import type { LoopPatternPolicy } from './LoopPatternSchema';
10
- import { LoopRunStateMachine } from './LoopRunStateMachine';
11
- import type { ClaimedLoopRun } from './LoopRunStateMachine';
12
- import { PathLockService } from './PathLockService';
13
- import { PluginAiRuntimeAdapter } from './PluginAiRuntimeAdapter';
14
- import type { InvocationOutcome, ToolCallDecision } from './PluginAiRuntimeAdapter';
15
- import { ToolLoopDetectionService, toolLoopNotice, toolLoopReason } from './ToolLoopDetectionService';
16
- import { TokenTracker } from './TokenTracker';
17
- import { VerificationService } from './VerificationService';
18
- import { read } from '../utils/record-utils';
19
- import { asObject } from '../utils/ctx-utils';
20
-
21
- const POLL_INTERVAL_MS = 3_000;
22
- const LEASE_MS = 120_000;
23
- const HEARTBEAT_MS = 30_000;
24
- const PATH_LOCK_TTL_MS = 900_000;
25
-
26
- const LEADER_SYSTEM_MESSAGE =
27
- 'You lead an autonomous run. Break the goal into concrete work for your makers, then finish with a concise plan of what must be built and which evidence will prove it. You do not modify anything yourself.';
28
- const MAKER_SYSTEM_MESSAGE =
29
- 'You are a maker on an autonomous run. Do the work assigned by the leader using only the tools you were granted, and report exactly what you changed and the evidence for it.';
30
-
31
- export type LoopWorkerSyncMessage = {
32
- type: 'loop-run-abort-requested';
33
- runId: number;
34
- };
35
-
36
- export function loopWorkerAbortMessage(runId: number): LoopWorkerSyncMessage {
37
- return { type: 'loop-run-abort-requested', runId };
38
- }
39
-
40
- type ResumeContext = {
41
- role: 'leader' | 'maker';
42
- username: string;
43
- sessionId: string;
44
- messageId: string;
45
- makerReports: string[];
46
- makerQueue: string[];
47
- };
48
-
49
- export type RunProgress = {
50
- currentPhase: 'leader' | 'maker' | 'verifying' | 'complete';
51
- totalPhases: number;
52
- completedPhases: number;
53
- currentStepTitle: string;
54
- currentRole: string;
55
- estimatedMsRemaining: number | null;
56
- startedAt: string;
57
- updatedAt: string;
58
- };
59
-
60
- function buildProgress(
61
- phase: RunProgress['currentPhase'],
62
- totalPhases: number,
63
- completedPhases: number,
64
- stepTitle: string,
65
- role: string,
66
- startedAt: Date,
67
- ): RunProgress {
68
- return {
69
- currentPhase: phase,
70
- totalPhases,
71
- completedPhases,
72
- currentStepTitle: stepTitle,
73
- currentRole: role,
74
- estimatedMsRemaining: null,
75
- startedAt: startedAt.toISOString(),
76
- updatedAt: new Date().toISOString(),
77
- };
78
- }
79
-
80
- type RunSnapshot = {
81
- id: number;
82
- patternId: number;
83
- goal: string;
84
- autonomyLevel: 'L1' | 'L2' | 'L3';
85
- policy: LoopPatternPolicy;
86
- leaderUsername: string;
87
- makerUsernames: string[];
88
- verifierUsername: string;
89
- leaderHarness: CompiledHarness;
90
- makerHarnesses: Record<string, { effective: CompiledHarness }>;
91
- verifierHarness: CompiledHarness;
92
- repositoryKey: string;
93
- actingOn: string[];
94
- userId?: number;
95
- approvalStatus: string;
96
- resumeContext: ResumeContext | null;
97
- };
98
-
99
- function asStringArray(value: unknown): string[] {
100
- return Array.isArray(value) ? value.map((item) => String(item)).filter(Boolean) : [];
101
- }
102
-
103
- function autonomyLevel(value: unknown): 'L1' | 'L2' | 'L3' {
104
- return value === 'L2' || value === 'L3' ? value : 'L1';
105
- }
106
-
107
- function harnessFrom(value: unknown): CompiledHarness {
108
- const snapshot = asObject(value);
109
- const effective = asObject(snapshot.effective);
110
- if (!effective.tools) throw new Error('The run harness snapshot is missing its compiled settings.');
111
- return effective as unknown as CompiledHarness;
112
- }
113
-
114
- function resumeContextFrom(value: unknown): ResumeContext | null {
115
- const context = asObject(value);
116
- if (!context) return null;
117
- const role = context.role === 'leader' || context.role === 'maker' ? context.role : null;
118
- const username = typeof context.username === 'string' ? context.username : '';
119
- const sessionId = typeof context.sessionId === 'string' ? context.sessionId : '';
120
- const messageId = typeof context.messageId === 'string' ? context.messageId : '';
121
- if (!role || !username || !sessionId || !messageId) return null;
122
- return {
123
- role,
124
- username,
125
- sessionId,
126
- messageId,
127
- makerReports: asStringArray(context.makerReports),
128
- makerQueue: asStringArray(context.makerQueue),
129
- };
130
- }
131
-
132
- function inputHash(value: unknown) {
133
- return createHash('sha256')
134
- .update(JSON.stringify(value ?? null))
135
- .digest('hex');
136
- }
137
-
138
- export class LoopWorkerService {
139
- private readonly control: LoopControlService;
140
- private readonly stateMachine: LoopRunStateMachine;
141
- private readonly budgets: LoopBudgetService;
142
- private readonly circuits: DurableCircuitBreakerService;
143
- private readonly pathLocks: PathLockService;
144
- private readonly runtime: PluginAiRuntimeAdapter;
145
- private readonly verification: VerificationService;
146
- private readonly loopGuard: ToolLoopDetectionService;
147
- private readonly tokenTracker: TokenTracker;
148
- private readonly workerId: string;
149
- private readonly active = new Map<number, AbortController>();
150
- private readonly heldPathLocks = new Map<number, string>();
151
- private timer: ReturnType<typeof setTimeout> | null = null;
152
- private running = false;
153
- private draining: Promise<void> | null = null;
154
-
155
- constructor(
156
- private readonly app: Application,
157
- private readonly database: Database,
158
- ) {
159
- const distributedLock = {
160
- runExclusive: <T>(key: string, operation: () => Promise<T>, ttl?: number) =>
161
- this.app.lockManager.runExclusive(key, operation, ttl),
162
- };
163
- this.control = new LoopControlService(database);
164
- this.stateMachine = new LoopRunStateMachine(database);
165
- this.budgets = new LoopBudgetService(database, distributedLock);
166
- this.circuits = new DurableCircuitBreakerService(database, distributedLock);
167
- this.pathLocks = new PathLockService(database, distributedLock);
168
- this.runtime = new PluginAiRuntimeAdapter(app);
169
- this.tokenTracker = new TokenTracker({ app, db: database });
170
- this.verification = new VerificationService(database, this.runtime, this.stateMachine, this.tokenTracker);
171
- this.loopGuard = new ToolLoopDetectionService(database);
172
- this.workerId = `${app.name}:${process.pid}:${randomUUID()}`;
173
- }
174
-
175
- readonly start = () => {
176
- // Multi-instance deployments dedicate a process to background work; only that process
177
- // should claim runs, otherwise every web node competes for the same leases.
178
- if (this.running || !this.app.serving('agent-loop:worker')) return;
179
- this.running = true;
180
- this.scheduleNextPoll();
181
- };
182
-
183
- async stop() {
184
- this.running = false;
185
- if (this.timer) clearTimeout(this.timer);
186
- this.timer = null;
187
- for (const controller of this.active.values()) controller.abort();
188
- if (this.draining) await this.draining.catch(() => undefined);
189
- }
190
-
191
- abortRun(runId: number) {
192
- this.active.get(runId)?.abort();
193
- }
194
-
195
- // A run is only active on the node that claimed it, and `sendSyncMessage` publishes with
196
- // `skipSelf`, so pause/cancel aborts locally and broadcasts for whichever other node holds it.
197
- async handleSyncMessage(message: unknown) {
198
- if (!message || typeof message !== 'object') return;
199
- const value = message as Partial<LoopWorkerSyncMessage>;
200
- if (value.type !== 'loop-run-abort-requested') return;
201
- const runId = Number(value.runId);
202
- if (!Number.isSafeInteger(runId) || runId <= 0) return;
203
- this.abortRun(runId);
204
- }
205
-
206
- private scheduleNextPoll() {
207
- if (!this.running) return;
208
- this.timer = setTimeout(() => {
209
- this.draining = this.poll()
210
- .catch((error) => {
211
- this.app.logger.error('[AgentOrchestrator] Loop worker poll failed.', { error });
212
- })
213
- .finally(() => this.scheduleNextPoll());
214
- }, POLL_INTERVAL_MS);
215
- }
216
-
217
- private async poll() {
218
- if (!this.running) return;
219
- await this.stateMachine.expireOverdueApprovals(new Date()).catch((error) => {
220
- this.app.logger.error('[AgentOrchestrator] Approval expiry sweep failed.', { error });
221
- });
222
- const claimed = await this.stateMachine.claimNext(this.workerId, LEASE_MS);
223
- if (!claimed) return;
224
- await this.execute(claimed);
225
- }
226
-
227
- private async updateProgress(runId: number, progress: RunProgress) {
228
- await this.database
229
- .getRepository('agentLoopRuns')
230
- .update({
231
- filterByTk: runId,
232
- values: { progress },
233
- })
234
- .catch((error: unknown) => {
235
- this.app.logger?.warn?.(`[AgentOrchestrator] Failed to update progress for run ${runId}`, { error });
236
- });
237
- }
238
-
239
- private async execute(claimed: ClaimedLoopRun) {
240
- const runId = Number(claimed.run.id);
241
- const controller = new AbortController();
242
- this.active.set(runId, controller);
243
-
244
- // The path lock TTL is much longer than the run lease, but a run can outlive it. Renewing both
245
- // on the same beat keeps them expiring together, so a run can never keep executing on paths
246
- // whose lock another run is already free to take.
247
- const heartbeat = setInterval(() => {
248
- this.heartbeat(runId, claimed.leaseToken).catch((error) => {
249
- this.app.logger.warn(`[AgentOrchestrator] Loop run ${runId} lost its worker lease.`, { error });
250
- controller.abort();
251
- });
252
- }, HEARTBEAT_MS);
253
-
254
- let snapshot: RunSnapshot | null = null;
255
- try {
256
- snapshot = this.snapshotFrom(claimed.run);
257
- await this.runToCompletion(snapshot, claimed.leaseToken, controller.signal);
258
- } catch (error) {
259
- await this.failRun(runId, claimed.leaseToken, snapshot, error);
260
- } finally {
261
- clearInterval(heartbeat);
262
- this.active.delete(runId);
263
- const repositoryKey = this.heldPathLocks.get(runId);
264
- this.heldPathLocks.delete(runId);
265
- if (repositoryKey) {
266
- // The lease token is the lock owner. Passing it stops a worker whose lease was already
267
- // reclaimed from releasing a lock the new owner of the same run now holds.
268
- await this.pathLocks
269
- .release(runId, repositoryKey, claimed.leaseToken)
270
- .catch((error) => this.app.logger.warn(`[AgentOrchestrator] Path lock release failed.`, { error }));
271
- }
272
- await this.stateMachine.releaseLease(runId, claimed.leaseToken).catch(() => undefined);
273
- }
274
- }
275
-
276
- private async heartbeat(runId: number, leaseToken: string) {
277
- await this.stateMachine.renewLease(runId, leaseToken, LEASE_MS);
278
- const repositoryKey = this.heldPathLocks.get(runId);
279
- if (repositoryKey) await this.pathLocks.renew(runId, repositoryKey, PATH_LOCK_TTL_MS, leaseToken);
280
- }
281
-
282
- private async runToCompletion(snapshot: RunSnapshot, leaseToken: string, signal: AbortSignal) {
283
- await this.control.assertCanExecute();
284
- const scopeKey = `pattern:${snapshot.patternId}`;
285
- await this.circuits.authorizeAttempt({
286
- patternId: snapshot.patternId,
287
- scopeKey,
288
- policy: snapshot.policy.circuit,
289
- });
290
-
291
- if (!(await this.acquirePaths(snapshot, leaseToken))) return;
292
-
293
- // A run requeued from a pause while its approvals are still pending has nothing to execute:
294
- // the decisions never arrived, so it parks again instead of restarting from the leader.
295
- if (snapshot.resumeContext && snapshot.approvalStatus === 'pending') {
296
- await this.stateMachine.transition({
297
- runId: snapshot.id,
298
- to: 'waiting_approval',
299
- actorType: 'worker',
300
- actorIdentity: this.workerId,
301
- leaseToken,
302
- eventType: 'run_waiting_approval',
303
- title: 'Waiting for action approval',
304
- content: 'Run requeued while its approval decisions are still pending.',
305
- correlationKey: `approval-repark:${snapshot.id}`,
306
- values: { approvalStatus: 'pending' },
307
- });
308
- return;
309
- }
310
-
311
- await this.stateMachine.transition({
312
- runId: snapshot.id,
313
- to: 'running',
314
- actorType: 'worker',
315
- actorIdentity: this.workerId,
316
- leaseToken,
317
- eventType: 'run_started',
318
- title: snapshot.resumeContext ? 'Run resumed' : 'Run started',
319
- content: snapshot.resumeContext
320
- ? `${snapshot.resumeContext.username} resumed after approval decisions.`
321
- : `Leader ${snapshot.leaderUsername} began working on the goal.`,
322
- correlationKey: `started:${snapshot.id}`,
323
- values: { currentRole: snapshot.resumeContext ? snapshot.resumeContext.role : 'leader' },
324
- });
325
-
326
- // Loops recorded before this claim (for example before a park for approval) keep warning the
327
- // prompts composed later in the run.
328
- const loopNotices = await this.loopGuard.existingNotices(snapshot.id);
329
- const totalPhases = 1 + snapshot.makerUsernames.length + 1;
330
- const runStartedAt = new Date();
331
-
332
- let makerReports: string[];
333
- let makerQueue: string[];
334
-
335
- if (snapshot.resumeContext) {
336
- const resume = snapshot.resumeContext;
337
- const decisions = await this.consumeDecisions(snapshot.id);
338
- const resumed = await this.resumeRole(snapshot, resume, decisions, signal);
339
-
340
- const resumeGuard = await this.enforceToolLoopGuard(
341
- snapshot,
342
- leaseToken,
343
- { role: resume.role, username: resume.username },
344
- loopNotices,
345
- );
346
- if (resumeGuard === 'halt') return;
347
-
348
- // A resumed conversation can hit another approval gate; park with the same role context so
349
- // the next resume continues exactly where this one stopped.
350
- if (resumed.interrupted.length) {
351
- await this.parkForApproval(snapshot, resumed, leaseToken, {
352
- role: resume.role,
353
- username: resume.username,
354
- makerReports: resume.makerReports,
355
- makerQueue: resume.makerQueue,
356
- });
357
- return;
358
- }
359
- if (resume.role === 'leader') {
360
- makerReports = [resumed.content];
361
- makerQueue = [...snapshot.makerUsernames];
362
- } else {
363
- makerReports = [...resume.makerReports, `# ${resume.username}\n${resumed.content}`];
364
- makerQueue = [...resume.makerQueue];
365
- }
366
- } else {
367
- const leader = await this.invokeRole(snapshot, {
368
- role: 'leader',
369
- username: snapshot.leaderUsername,
370
- harness: snapshot.leaderHarness,
371
- systemMessage: LEADER_SYSTEM_MESSAGE,
372
- prompt: snapshot.goal,
373
- leaseToken,
374
- signal,
375
- });
376
-
377
- const leaderGuard = await this.enforceToolLoopGuard(
378
- snapshot,
379
- leaseToken,
380
- { role: 'leader', username: snapshot.leaderUsername },
381
- loopNotices,
382
- );
383
- if (leaderGuard === 'halt') return;
384
- await this.updateProgress(
385
- snapshot.id,
386
- buildProgress('leader', totalPhases, 1, 'Leader plan complete', 'leader', runStartedAt),
387
- );
388
-
389
- // plugin-ai stops the graph on an approval-gated tool. Approval is a human decision, so the
390
- // run parks here instead of the worker deciding on the operator's behalf.
391
- if (leader.interrupted.length) {
392
- await this.parkForApproval(snapshot, leader, leaseToken, {
393
- role: 'leader',
394
- username: snapshot.leaderUsername,
395
- makerReports: [],
396
- makerQueue: snapshot.makerUsernames,
397
- });
398
- return;
399
- }
400
- makerReports = [leader.content];
401
- makerQueue = [...snapshot.makerUsernames];
402
- }
403
-
404
- // Each maker executes with its own compiled harness snapshot. Reusing one harness would apply
405
- // the wrong per-employee tool grants, so the snapshot is looked up by username and a maker with
406
- // no snapshot is a compile error, not a silent skip.
407
- while (makerQueue.length) {
408
- const username = makerQueue[0];
409
- const harness = snapshot.makerHarnesses[username]?.effective;
410
- if (!harness) {
411
- throw new Error(`The claimed run has no harness snapshot for maker "${username}".`);
412
- }
413
- const maker = await this.invokeRole(snapshot, {
414
- role: 'maker',
415
- username,
416
- harness,
417
- systemMessage: MAKER_SYSTEM_MESSAGE,
418
- prompt: [
419
- `Goal: ${snapshot.goal}`,
420
- '',
421
- "Leader's plan:",
422
- makerReports[0] || '(the leader produced no textual plan)',
423
- ...loopNotices,
424
- ].join('\n'),
425
- leaseToken,
426
- signal,
427
- });
428
- const makerGuard = await this.enforceToolLoopGuard(
429
- snapshot,
430
- leaseToken,
431
- { role: 'maker', username },
432
- loopNotices,
433
- );
434
- if (makerGuard === 'halt') return;
435
- if (maker.interrupted.length) {
436
- await this.parkForApproval(snapshot, maker, leaseToken, {
437
- role: 'maker',
438
- username,
439
- makerReports,
440
- makerQueue: makerQueue.slice(1),
441
- });
442
- return;
443
- }
444
- makerReports.push(`# ${username}\n${maker.content}`);
445
- const completedMakers = snapshot.makerUsernames.length - makerQueue.length + 1;
446
- await this.updateProgress(
447
- snapshot.id,
448
- buildProgress('maker', totalPhases, 1 + completedMakers, `Maker "${username}" complete`, 'maker', runStartedAt),
449
- );
450
- makerQueue = makerQueue.slice(1);
451
- }
452
- const makerSummary = makerReports.join('\n\n');
453
-
454
- await this.stateMachine.transition({
455
- runId: snapshot.id,
456
- to: 'verifying',
457
- actorType: 'worker',
458
- actorIdentity: this.workerId,
459
- leaseToken,
460
- eventType: 'run_verifying',
461
- title: 'Verification started',
462
- content: `Verifier ${snapshot.verifierUsername} is reviewing the reported result.`,
463
- correlationKey: `verifying:${snapshot.id}`,
464
- values: { currentRole: 'verifier', finalAnswer: makerSummary },
465
- });
466
- await this.updateProgress(
467
- snapshot.id,
468
- buildProgress(
469
- 'verifying',
470
- totalPhases,
471
- totalPhases - 1,
472
- `Verifier "${snapshot.verifierUsername}" reviewing`,
473
- 'verifier',
474
- runStartedAt,
475
- ),
476
- );
477
-
478
- await this.budgets.reserve({
479
- runId: snapshot.id,
480
- patternId: snapshot.patternId,
481
- policy: snapshot.policy,
482
- delta: { verifications: 1 },
483
- });
484
- const outcome = await this.verification.verifyAndFinalize({
485
- runId: snapshot.id,
486
- goal: snapshot.goal,
487
- autonomyLevel: snapshot.autonomyLevel,
488
- verifierUsername: snapshot.verifierUsername,
489
- leaderUsername: snapshot.leaderUsername,
490
- makerUsernames: snapshot.makerUsernames,
491
- verifierHarness: snapshot.verifierHarness,
492
- policy: snapshot.policy,
493
- makerSummary,
494
- actingOn: snapshot.actingOn,
495
- userId: snapshot.userId,
496
- workerId: this.workerId,
497
- leaseToken,
498
- loopNotices,
499
- signal,
500
- });
501
-
502
- // The tool loop guard already moved the run to blocked or waiting_human; the circuit breaker
503
- // tracks failed attempts, not behavioral stops, so it must not see this as a failure.
504
- if (outcome.finalStatus === 'halted') return;
505
-
506
- if (outcome.finalStatus === 'failed') {
507
- await this.circuits.recordFailure({
508
- patternId: snapshot.patternId,
509
- scopeKey: `pattern:${snapshot.patternId}`,
510
- policy: snapshot.policy.circuit,
511
- error: new Error(`Verifier rejected run ${snapshot.id}.`),
512
- lastRunId: snapshot.id,
513
- });
514
- return;
515
- }
516
- await this.circuits.recordSuccess(snapshot.patternId, `pattern:${snapshot.patternId}`, snapshot.id);
517
- await this.updateProgress(
518
- snapshot.id,
519
- buildProgress(
520
- 'complete',
521
- totalPhases,
522
- totalPhases,
523
- 'Run completed successfully',
524
- snapshot.currentRole || 'verifier',
525
- runStartedAt,
526
- ),
527
- );
528
- }
529
-
530
- private async acquirePaths(snapshot: RunSnapshot, leaseToken: string) {
531
- if (!snapshot.repositoryKey || !snapshot.actingOn.length) return true;
532
- const lock = await this.pathLocks.acquire({
533
- runId: snapshot.id,
534
- repositoryKey: snapshot.repositoryKey,
535
- owner: leaseToken,
536
- paths: snapshot.actingOn,
537
- ttlMs: PATH_LOCK_TTL_MS,
538
- });
539
- if (lock.acquired) {
540
- // Recorded before any work starts so the heartbeat renews this lock for as long as the run
541
- // executes, and the release in `execute` frees exactly the lock this worker took.
542
- this.heldPathLocks.set(snapshot.id, snapshot.repositoryKey);
543
- return true;
544
- }
545
-
546
- const deadlocked = lock.deadlocked;
547
- await this.stateMachine.transition({
548
- runId: snapshot.id,
549
- to: deadlocked ? 'blocked' : 'waiting_lock',
550
- actorType: 'worker',
551
- actorIdentity: this.workerId,
552
- leaseToken,
553
- eventType: deadlocked ? 'run_blocked' : 'run_waiting_lock',
554
- title: deadlocked ? 'Run blocked by a lock cycle' : 'Waiting for repository paths',
555
- content: `Blocked by run(s): ${lock.blockers.join(', ')}.`,
556
- correlationKey: `lock:${snapshot.id}:${lock.blockers.join('-')}`,
557
- values: deadlocked ? { blockedReason: 'Path lock wait graph contains a cycle.' } : {},
558
- });
559
- return false;
560
- }
561
-
562
- private async invokeRole(
563
- snapshot: RunSnapshot,
564
- input: {
565
- role: 'leader' | 'maker';
566
- username: string;
567
- harness: CompiledHarness;
568
- systemMessage: string;
569
- prompt: string;
570
- leaseToken: string;
571
- signal: AbortSignal;
572
- },
573
- ) {
574
- await this.budgets.reserve({
575
- runId: snapshot.id,
576
- patternId: snapshot.patternId,
577
- policy: snapshot.policy,
578
- delta: { invocations: 1 },
579
- });
580
- const sessionId = await this.runtime.createConversation({
581
- username: input.username,
582
- userId: snapshot.userId,
583
- runId: snapshot.id,
584
- title: snapshot.goal.slice(0, 200),
585
- });
586
- const step = await this.database.getRepository('agentLoopSteps').create({
587
- values: {
588
- runId: snapshot.id,
589
- runtimeVersion: 'control-plane-v2',
590
- sequence: await this.nextSequence(snapshot.id),
591
- role: input.role,
592
- kind: 'invocation',
593
- type: 'reasoning',
594
- employeeUsername: input.username,
595
- sessionId,
596
- title: `${input.role} invocation`,
597
- inputHash: inputHash(input.prompt),
598
- status: 'running',
599
- startedAt: new Date(),
600
- createdAt: new Date(),
601
- },
602
- });
603
-
604
- const outcome = await this.runtime.invoke({
605
- username: input.username,
606
- sessionId,
607
- userId: snapshot.userId,
608
- systemMessage: input.systemMessage,
609
- harness: input.harness,
610
- prompt: input.prompt,
611
- signal: input.signal,
612
- });
613
-
614
- // Track actual token usage from the LLM response; fall back to text-length estimation
615
- await this.tokenTracker.trackActualUsage(
616
- read(step, 'id'),
617
- outcome.tokenUsage ?? null,
618
- input.prompt,
619
- outcome.content,
620
- snapshot.id,
621
- );
622
-
623
- await this.database.getRepository('agentLoopSteps').update({
624
- filterByTk: read(step, 'id'),
625
- values: {
626
- status: outcome.interrupted.length ? 'waiting_user' : 'succeeded',
627
- output: { content: outcome.content },
628
- endedAt: new Date(),
629
- },
630
- });
631
- await this.database.getRepository('agentLoopRuns').update({
632
- filterByTk: snapshot.id,
633
- values: { sessionId: outcome.sessionId, messageId: outcome.messageId },
634
- });
635
- return outcome;
636
- }
637
-
638
- private async resumeRole(
639
- snapshot: RunSnapshot,
640
- resume: ResumeContext,
641
- decisions: Array<{ toolCallId: string; decision: ToolCallDecision }>,
642
- signal: AbortSignal,
643
- ) {
644
- await this.budgets.reserve({
645
- runId: snapshot.id,
646
- patternId: snapshot.patternId,
647
- policy: snapshot.policy,
648
- delta: { invocations: 1 },
649
- });
650
- const harness =
651
- resume.role === 'leader' ? snapshot.leaderHarness : snapshot.makerHarnesses[resume.username]?.effective;
652
- if (!harness) {
653
- throw new Error(`The claimed run has no harness snapshot for ${resume.role} "${resume.username}".`);
654
- }
655
- const step = await this.database.getRepository('agentLoopSteps').create({
656
- values: {
657
- runId: snapshot.id,
658
- runtimeVersion: 'control-plane-v2',
659
- sequence: await this.nextSequence(snapshot.id),
660
- role: resume.role,
661
- kind: 'invocation',
662
- type: 'resume',
663
- employeeUsername: resume.username,
664
- sessionId: resume.sessionId,
665
- title: `${resume.role} resume after approval`,
666
- inputHash: inputHash(decisions),
667
- status: 'running',
668
- startedAt: new Date(),
669
- createdAt: new Date(),
670
- },
671
- });
672
-
673
- const outcome = await this.runtime.resume({
674
- username: resume.username,
675
- sessionId: resume.sessionId,
676
- messageId: resume.messageId,
677
- userId: snapshot.userId,
678
- systemMessage: resume.role === 'leader' ? LEADER_SYSTEM_MESSAGE : MAKER_SYSTEM_MESSAGE,
679
- harness,
680
- decisions,
681
- signal,
682
- });
683
-
684
- // Track actual token usage from the LLM response; fall back to text-length estimation
685
- await this.tokenTracker.trackActualUsage(
686
- read(step, 'id'),
687
- outcome.tokenUsage ?? null,
688
- resume.role === 'leader' ? LEADER_SYSTEM_MESSAGE : MAKER_SYSTEM_MESSAGE,
689
- outcome.content,
690
- snapshot.id,
691
- );
692
-
693
- await this.database.getRepository('agentLoopSteps').update({
694
- filterByTk: read(step, 'id'),
695
- values: {
696
- status: outcome.interrupted.length ? 'waiting_user' : 'succeeded',
697
- output: { content: outcome.content },
698
- endedAt: new Date(),
699
- },
700
- });
701
- await this.database.getRepository('agentLoopRuns').update({
702
- filterByTk: snapshot.id,
703
- values: { sessionId: outcome.sessionId, messageId: outcome.messageId },
704
- });
705
- return outcome;
706
- }
707
-
708
- // Run-level scan over every tool call issued so far, evaluated at each pass boundary. A warning
709
- // is injected into the prompts composed later in the run; block and escalate halt it at once.
710
- private async enforceToolLoopGuard(
711
- snapshot: RunSnapshot,
712
- leaseToken: string,
713
- pass: { role: 'leader' | 'maker'; username: string },
714
- loopNotices: string[],
715
- ): Promise<'continue' | 'halt'> {
716
- const finding = await this.loopGuard.scanRun(snapshot.id, snapshot.policy);
717
- if (!finding) return 'continue';
718
-
719
- await this.loopGuard.recordFinding(
720
- {
721
- runId: snapshot.id,
722
- role: pass.role,
723
- username: pass.username,
724
- runStatus: 'running',
725
- actorType: 'worker',
726
- actorIdentity: this.workerId,
727
- },
728
- finding,
729
- );
730
-
731
- if (finding.level === 'warn') {
732
- const notice = toolLoopNotice(finding.toolName, finding.count);
733
- if (!loopNotices.includes(notice)) loopNotices.push(notice);
734
- return 'continue';
735
- }
736
-
737
- const reason = toolLoopReason(finding);
738
- await this.stateMachine.transition({
739
- runId: snapshot.id,
740
- to: finding.level === 'block' ? 'blocked' : 'waiting_human',
741
- actorType: 'worker',
742
- actorIdentity: this.workerId,
743
- leaseToken,
744
- eventType: finding.level === 'block' ? 'run_blocked' : 'run_escalated',
745
- title: finding.level === 'block' ? 'Run blocked by tool loop detection' : 'Run escalated by tool loop detection',
746
- content: reason,
747
- correlationKey: `tool-loop-${finding.level}:${snapshot.id}:${finding.signature}`,
748
- values: finding.level === 'block' ? { blockedReason: reason } : { escalationReason: reason },
749
- });
750
- return 'halt';
751
- }
752
-
753
- // Decisions are consumed exactly once: rows are marked consumed so a retry or a second resume
754
- // can never replay a human decision against a fresh conversation.
755
- private async consumeDecisions(runId: number) {
756
- const repository = this.database.getRepository('agentLoopActionApprovals');
757
- const approvals = await repository.find({ filter: { runId } });
758
- const decisions: Array<{ toolCallId: string; decision: ToolCallDecision }> = [];
759
- const now = new Date();
760
- for (const approval of approvals) {
761
- const status = String(read(approval, 'status'));
762
- if (status !== 'approved' && status !== 'rejected') continue;
763
- if (read(approval, 'consumedAt')) continue;
764
- const editedInput = read(approval, 'editedInput');
765
- const decision: ToolCallDecision =
766
- status === 'approved'
767
- ? editedInput
768
- ? { type: 'edit', editedAction: { name: String(read(approval, 'toolName')), args: editedInput } }
769
- : { type: 'approve' }
770
- : { type: 'reject', message: String(read(approval, 'decisionNote') || '') || undefined };
771
- decisions.push({ toolCallId: String(read(approval, 'toolCallId')), decision });
772
- await repository.update({ filterByTk: Number(read(approval, 'id')), values: { consumedAt: now } });
773
- }
774
- if (!decisions.length) throw new Error('The resumed run has no recorded approval decisions.');
775
- return decisions;
776
- }
777
-
778
- private async parkForApproval(
779
- snapshot: RunSnapshot,
780
- outcome: InvocationOutcome,
781
- leaseToken: string,
782
- context: { role: 'leader' | 'maker'; username: string; makerReports: string[]; makerQueue: string[] },
783
- ) {
784
- const now = new Date();
785
- const expiresAt = new Date(now.getTime() + snapshot.policy.actions.approvalTimeoutMs);
786
- const assignedToId = snapshot.policy.actions.approvalAssigneeIds[0] || snapshot.userId || null;
787
- const harness =
788
- context.role === 'leader' ? snapshot.leaderHarness : snapshot.makerHarnesses[context.username]?.effective;
789
- const escalatable = new Set(harness?.tools.escalate ?? []);
790
- for (const call of outcome.interrupted) {
791
- // An escalation is a request for authority the harness never granted, widened for exactly
792
- // one call; a plain tool_call approval gates a tool the harness allows but asks about.
793
- const escalation = escalatable.has(call.toolName);
794
- await this.database.getRepository('agentLoopActionApprovals').create({
795
- values: {
796
- runId: snapshot.id,
797
- toolCallId: call.toolCallId,
798
- toolName: call.toolName,
799
- actionType: escalation ? 'escalation' : 'tool_call',
800
- proposedInput: call.args,
801
- inputHash: inputHash(call.args),
802
- paths: snapshot.actingOn,
803
- policyDecision: { decision: 'ask', interruptId: call.interruptId },
804
- reason: escalation
805
- ? `Tool "${call.toolName}" is not granted by this run's harness. Approving widens authority for this single call only.`
806
- : `Tool "${call.toolName}" requires human approval under this pattern's policy.`,
807
- status: 'pending',
808
- assignedToId,
809
- requestedById: snapshot.userId || null,
810
- requestedAt: now,
811
- expiresAt,
812
- createdAt: now,
813
- },
814
- });
815
- }
816
- await this.database.getRepository('agentLoopRuns').update({
817
- filterByTk: snapshot.id,
818
- values: {
819
- // The resume point is durable: which role stopped, in which conversation, with which
820
- // reports already collected and which makers still queued. Without it a resumed run
821
- // would restart from the leader and redo every completed invocation.
822
- resumeContext: {
823
- role: context.role,
824
- username: context.username,
825
- sessionId: outcome.sessionId,
826
- messageId: outcome.messageId,
827
- makerReports: context.makerReports,
828
- makerQueue: context.makerQueue,
829
- },
830
- },
831
- });
832
- await this.stateMachine.transition({
833
- runId: snapshot.id,
834
- to: 'waiting_approval',
835
- actorType: 'worker',
836
- actorIdentity: this.workerId,
837
- leaseToken,
838
- eventType: 'run_waiting_approval',
839
- title: 'Waiting for action approval',
840
- content: `${outcome.interrupted.length} tool call(s) require approval.`,
841
- correlationKey: `approval:${snapshot.id}:${outcome.messageId}`,
842
- values: { approvalStatus: 'pending', messageId: outcome.messageId },
843
- });
844
- }
845
-
846
- private async failRun(runId: number, leaseToken: string, snapshot: RunSnapshot | null, error: unknown) {
847
- const message = error instanceof Error ? error.message : String(error);
848
- try {
849
- if (snapshot) {
850
- await this.circuits.recordFailure({
851
- patternId: snapshot.patternId,
852
- scopeKey: `pattern:${snapshot.patternId}`,
853
- policy: snapshot.policy.circuit,
854
- error,
855
- lastRunId: runId,
856
- });
857
- }
858
- await this.stateMachine.transition({
859
- runId,
860
- to: 'failed',
861
- actorType: 'worker',
862
- actorIdentity: this.workerId,
863
- leaseToken,
864
- eventType: 'run_failed',
865
- title: 'Run failed',
866
- content: message,
867
- correlationKey: `failed:${runId}`,
868
- values: { summary: message },
869
- });
870
- } catch (transitionError) {
871
- this.app.logger.error(`[AgentOrchestrator] Loop run ${runId} could not be marked failed.`, {
872
- error: transitionError,
873
- cause: message,
874
- });
875
- }
876
- }
877
-
878
- private async nextSequence(runId: number) {
879
- return this.database.getRepository('agentLoopSteps').count({ filter: { runId } });
880
- }
881
-
882
- private snapshotFrom(run: Record<string, unknown>): RunSnapshot {
883
- const patternId = Number(run.patternId);
884
- if (!Number.isSafeInteger(patternId) || patternId <= 0) {
885
- throw new Error('The claimed run has no pattern snapshot.');
886
- }
887
- const roleBindings = asObject(run.roleBindingsSnapshot);
888
- const makerSnapshots = asObject(run.makerHarnessSnapshot);
889
- const approvalStatus = String(run.approvalStatus || 'none');
890
- return {
891
- id: Number(run.id),
892
- patternId,
893
- goal: String(run.goal || ''),
894
- autonomyLevel: autonomyLevel(run.autonomyLevel),
895
- policy: loopPatternPolicySchema.parse(run.policySnapshot),
896
- leaderUsername: String(roleBindings.leader || run.leaderUsername || ''),
897
- makerUsernames: asStringArray(roleBindings.makers),
898
- verifierUsername: String(roleBindings.verifier || run.verifierUsername || ''),
899
- leaderHarness: harnessFrom(run.leaderHarnessSnapshot),
900
- makerHarnesses: makerSnapshots as Record<string, { effective: CompiledHarness }>,
901
- verifierHarness: harnessFrom(run.verifierHarnessSnapshot),
902
- repositoryKey: String(run.repositoryKey || ''),
903
- actingOn: asStringArray(run.actingOn),
904
- userId: Number(run.userId) || undefined,
905
- approvalStatus,
906
- // Only a live approval state may resume. Stale context left over from an expired window or a
907
- // cleared retry would otherwise replay an old conversation against fresh approvals.
908
- resumeContext:
909
- approvalStatus === 'decided' || approvalStatus === 'pending' ? resumeContextFrom(run.resumeContext) : null,
910
- };
911
- }
912
- }
1
+ // SPDX-License-Identifier: Apache-2.0
2
+
3
+ import { createHash, randomUUID } from 'node:crypto';
4
+ import type { Database, Model } from '@nocobase/database';
5
+ import type { Application } from '@nocobase/server';
6
+ import { DurableCircuitBreakerService } from './DurableCircuitBreakerService';
7
+ import type { CompiledHarness } from './HarnessCompiler';
8
+ import { LoopBudgetService } from './LoopBudgetService';
9
+ import { LoopControlService } from './LoopControlService';
10
+ import { loopPatternPolicySchema } from './LoopPatternSchema';
11
+ import type { LoopPatternPolicy } from './LoopPatternSchema';
12
+ import { LoopRunStateMachine } from './LoopRunStateMachine';
13
+ import type { ClaimedLoopRun } from './LoopRunStateMachine';
14
+ import { PathLockService } from './PathLockService';
15
+ import { PluginAiRuntimeAdapter } from './PluginAiRuntimeAdapter';
16
+ import type { InvocationOutcome, ToolCallDecision } from './PluginAiRuntimeAdapter';
17
+ import { ToolLoopDetectionService, toolLoopNotice, toolLoopReason } from './ToolLoopDetectionService';
18
+ import { TokenTracker } from './TokenTracker';
19
+ import { VerificationService } from './VerificationService';
20
+ import { read } from '../utils/record-utils';
21
+ import { asObject } from '../utils/ctx-utils';
22
+
23
+ const POLL_INTERVAL_MS = 3_000;
24
+ const LEASE_MS = 120_000;
25
+ const HEARTBEAT_MS = 30_000;
26
+ const PATH_LOCK_TTL_MS = 900_000;
27
+
28
+ const LEADER_SYSTEM_MESSAGE =
29
+ 'You lead an autonomous run. Break the goal into concrete work for your makers, then finish with a concise plan of what must be built and which evidence will prove it. You do not modify anything yourself.';
30
+ const MAKER_SYSTEM_MESSAGE =
31
+ 'You are a maker on an autonomous run. Do the work assigned by the leader using only the tools you were granted, and report exactly what you changed and the evidence for it.';
32
+
33
+ export type LoopWorkerSyncMessage = {
34
+ type: 'loop-run-abort-requested';
35
+ runId: number;
36
+ };
37
+
38
+ export function loopWorkerAbortMessage(runId: number): LoopWorkerSyncMessage {
39
+ return { type: 'loop-run-abort-requested', runId };
40
+ }
41
+
42
+ type ResumeContext = {
43
+ role: 'leader' | 'maker';
44
+ username: string;
45
+ sessionId: string;
46
+ messageId: string;
47
+ makerReports: string[];
48
+ makerQueue: string[];
49
+ };
50
+
51
+ export type RunProgress = {
52
+ currentPhase: 'leader' | 'maker' | 'verifying' | 'complete';
53
+ totalPhases: number;
54
+ completedPhases: number;
55
+ currentStepTitle: string;
56
+ currentRole: string;
57
+ estimatedMsRemaining: number | null;
58
+ startedAt: string;
59
+ updatedAt: string;
60
+ };
61
+
62
+ function buildProgress(
63
+ phase: RunProgress['currentPhase'],
64
+ totalPhases: number,
65
+ completedPhases: number,
66
+ stepTitle: string,
67
+ role: string,
68
+ startedAt: Date,
69
+ ): RunProgress {
70
+ return {
71
+ currentPhase: phase,
72
+ totalPhases,
73
+ completedPhases,
74
+ currentStepTitle: stepTitle,
75
+ currentRole: role,
76
+ estimatedMsRemaining: null,
77
+ startedAt: startedAt.toISOString(),
78
+ updatedAt: new Date().toISOString(),
79
+ };
80
+ }
81
+
82
+ type RunSnapshot = {
83
+ id: number;
84
+ rootRunId: string;
85
+ patternId: number;
86
+ goal: string;
87
+ autonomyLevel: 'L1' | 'L2' | 'L3';
88
+ policy: LoopPatternPolicy;
89
+ leaderUsername: string;
90
+ makerUsernames: string[];
91
+ verifierUsername: string;
92
+ leaderHarness: CompiledHarness;
93
+ makerHarnesses: Record<string, { effective: CompiledHarness }>;
94
+ verifierHarness: CompiledHarness;
95
+ repositoryKey: string;
96
+ actingOn: string[];
97
+ userId?: number;
98
+ approvalStatus: string;
99
+ resumeContext: ResumeContext | null;
100
+ verificationRecoverySummary: string | null;
101
+ };
102
+
103
+ function asStringArray(value: unknown): string[] {
104
+ return Array.isArray(value) ? value.map((item) => String(item)).filter(Boolean) : [];
105
+ }
106
+
107
+ function autonomyLevel(value: unknown): 'L1' | 'L2' | 'L3' {
108
+ return value === 'L2' || value === 'L3' ? value : 'L1';
109
+ }
110
+
111
+ function harnessFrom(value: unknown): CompiledHarness {
112
+ const snapshot = asObject(value);
113
+ const effective = asObject(snapshot.effective);
114
+ if (!effective.tools) throw new Error('The run harness snapshot is missing its compiled settings.');
115
+ return effective as unknown as CompiledHarness;
116
+ }
117
+
118
+ function resumeContextFrom(value: unknown): ResumeContext | null {
119
+ const context = asObject(value);
120
+ if (!context) return null;
121
+ const role = context.role === 'leader' || context.role === 'maker' ? context.role : null;
122
+ const username = typeof context.username === 'string' ? context.username : '';
123
+ const sessionId = typeof context.sessionId === 'string' ? context.sessionId : '';
124
+ const messageId = typeof context.messageId === 'string' ? context.messageId : '';
125
+ if (!role || !username || !sessionId || !messageId) return null;
126
+ return {
127
+ role,
128
+ username,
129
+ sessionId,
130
+ messageId,
131
+ makerReports: asStringArray(context.makerReports),
132
+ makerQueue: asStringArray(context.makerQueue),
133
+ };
134
+ }
135
+
136
+ function inputHash(value: unknown) {
137
+ return createHash('sha256')
138
+ .update(JSON.stringify(value ?? null))
139
+ .digest('hex');
140
+ }
141
+
142
+ export class LoopWorkerService {
143
+ private readonly control: LoopControlService;
144
+ private readonly stateMachine: LoopRunStateMachine;
145
+ private readonly budgets: LoopBudgetService;
146
+ private readonly circuits: DurableCircuitBreakerService;
147
+ private readonly pathLocks: PathLockService;
148
+ private readonly runtime: PluginAiRuntimeAdapter;
149
+ private readonly verification: VerificationService;
150
+ private readonly loopGuard: ToolLoopDetectionService;
151
+ private readonly tokenTracker: TokenTracker;
152
+ private readonly workerId: string;
153
+ private readonly active = new Map<number, AbortController>();
154
+ private readonly heldPathLocks = new Map<number, string>();
155
+ private timer: ReturnType<typeof setTimeout> | null = null;
156
+ private running = false;
157
+ private draining: Promise<void> | null = null;
158
+
159
+ constructor(
160
+ private readonly app: Application,
161
+ private readonly database: Database,
162
+ ) {
163
+ const distributedLock = {
164
+ runExclusive: <T>(key: string, operation: () => Promise<T>, ttl?: number) =>
165
+ this.app.lockManager.runExclusive(key, operation, ttl),
166
+ };
167
+ this.control = new LoopControlService(database);
168
+ this.stateMachine = new LoopRunStateMachine(database);
169
+ this.budgets = new LoopBudgetService(database, distributedLock);
170
+ this.circuits = new DurableCircuitBreakerService(database, distributedLock);
171
+ this.pathLocks = new PathLockService(database, distributedLock);
172
+ this.runtime = new PluginAiRuntimeAdapter(app);
173
+ this.tokenTracker = new TokenTracker({ app, db: database });
174
+ this.verification = new VerificationService(database, this.runtime, this.stateMachine, this.tokenTracker);
175
+ this.loopGuard = new ToolLoopDetectionService(database);
176
+ this.workerId = `${app.name}:${process.pid}:${randomUUID()}`;
177
+ }
178
+
179
+ readonly start = () => {
180
+ // Multi-instance deployments dedicate a process to background work; only that process
181
+ // should claim runs, otherwise every web node competes for the same leases.
182
+ if (this.running || !this.app.serving('agent-loop:worker')) return;
183
+ this.running = true;
184
+ this.scheduleNextPoll();
185
+ };
186
+
187
+ async stop() {
188
+ this.running = false;
189
+ if (this.timer) clearTimeout(this.timer);
190
+ this.timer = null;
191
+ for (const controller of this.active.values()) controller.abort();
192
+ if (this.draining) await this.draining.catch(() => undefined);
193
+ }
194
+
195
+ abortRun(runId: number) {
196
+ this.active.get(runId)?.abort();
197
+ }
198
+
199
+ // A run is only active on the node that claimed it, and `sendSyncMessage` publishes with
200
+ // `skipSelf`, so pause/cancel aborts locally and broadcasts for whichever other node holds it.
201
+ async handleSyncMessage(message: unknown) {
202
+ if (!message || typeof message !== 'object') return;
203
+ const value = message as Partial<LoopWorkerSyncMessage>;
204
+ if (value.type !== 'loop-run-abort-requested') return;
205
+ const runId = Number(value.runId);
206
+ if (!Number.isSafeInteger(runId) || runId <= 0) return;
207
+ this.abortRun(runId);
208
+ }
209
+
210
+ private scheduleNextPoll() {
211
+ if (!this.running) return;
212
+ this.timer = setTimeout(() => {
213
+ this.draining = this.poll()
214
+ .catch((error) => {
215
+ this.app.logger.error('[AgentOrchestrator] Loop worker poll failed.', { error });
216
+ })
217
+ .finally(() => this.scheduleNextPoll());
218
+ }, POLL_INTERVAL_MS);
219
+ }
220
+
221
+ private async poll() {
222
+ if (!this.running) return;
223
+ await this.stateMachine.expireOverdueApprovals(new Date()).catch((error) => {
224
+ this.app.logger.error('[AgentOrchestrator] Approval expiry sweep failed.', { error });
225
+ });
226
+ const claimed = await this.stateMachine.claimNext(this.workerId, LEASE_MS);
227
+ if (!claimed) return;
228
+ await this.execute(claimed);
229
+ }
230
+
231
+ private async updateProgress(runId: number, progress: RunProgress) {
232
+ await this.database
233
+ .getRepository('agentLoopRuns')
234
+ .update({
235
+ filterByTk: runId,
236
+ values: { progress },
237
+ })
238
+ .catch((error: unknown) => {
239
+ this.app.logger?.warn?.(`[AgentOrchestrator] Failed to update progress for run ${runId}`, { error });
240
+ });
241
+ }
242
+
243
+ private async execute(claimed: ClaimedLoopRun) {
244
+ const runId = Number(claimed.run.id);
245
+ const controller = new AbortController();
246
+ this.active.set(runId, controller);
247
+
248
+ // The path lock TTL is much longer than the run lease, but a run can outlive it. Renewing both
249
+ // on the same beat keeps them expiring together, so a run can never keep executing on paths
250
+ // whose lock another run is already free to take.
251
+ const heartbeat = setInterval(() => {
252
+ this.heartbeat(runId, claimed.leaseToken).catch((error) => {
253
+ this.app.logger.warn(`[AgentOrchestrator] Loop run ${runId} lost its worker lease.`, { error });
254
+ controller.abort();
255
+ });
256
+ }, HEARTBEAT_MS);
257
+
258
+ let snapshot: RunSnapshot | null = null;
259
+ try {
260
+ snapshot = this.snapshotFrom(claimed.run);
261
+ await this.runToCompletion(snapshot, claimed.leaseToken, controller.signal);
262
+ } catch (error) {
263
+ await this.failRun(runId, claimed.leaseToken, snapshot, error);
264
+ } finally {
265
+ clearInterval(heartbeat);
266
+ this.active.delete(runId);
267
+ const repositoryKey = this.heldPathLocks.get(runId);
268
+ this.heldPathLocks.delete(runId);
269
+ if (repositoryKey) {
270
+ // The lease token is the lock owner. Passing it stops a worker whose lease was already
271
+ // reclaimed from releasing a lock the new owner of the same run now holds.
272
+ await this.pathLocks
273
+ .release(runId, repositoryKey, claimed.leaseToken)
274
+ .catch((error) => this.app.logger.warn(`[AgentOrchestrator] Path lock release failed.`, { error }));
275
+ }
276
+ await this.stateMachine.releaseLease(runId, claimed.leaseToken).catch(() => undefined);
277
+ }
278
+ }
279
+
280
+ private async heartbeat(runId: number, leaseToken: string) {
281
+ await this.stateMachine.renewLease(runId, leaseToken, LEASE_MS);
282
+ const repositoryKey = this.heldPathLocks.get(runId);
283
+ if (repositoryKey) await this.pathLocks.renew(runId, repositoryKey, PATH_LOCK_TTL_MS, leaseToken);
284
+ }
285
+
286
+ private async runToCompletion(snapshot: RunSnapshot, leaseToken: string, signal: AbortSignal) {
287
+ await this.control.assertCanExecute();
288
+ const scopeKey = `pattern:${snapshot.patternId}`;
289
+ await this.circuits.authorizeAttempt({
290
+ patternId: snapshot.patternId,
291
+ scopeKey,
292
+ policy: snapshot.policy.circuit,
293
+ });
294
+
295
+ if (!(await this.acquirePaths(snapshot, leaseToken))) return;
296
+
297
+ // A run requeued from a pause while its approvals are still pending has nothing to execute:
298
+ // the decisions never arrived, so it parks again instead of restarting from the leader.
299
+ if (snapshot.resumeContext && snapshot.approvalStatus === 'pending') {
300
+ await this.stateMachine.transition({
301
+ runId: snapshot.id,
302
+ to: 'waiting_approval',
303
+ actorType: 'worker',
304
+ actorIdentity: this.workerId,
305
+ leaseToken,
306
+ eventType: 'run_waiting_approval',
307
+ title: 'Waiting for action approval',
308
+ content: 'Run requeued while its approval decisions are still pending.',
309
+ correlationKey: `approval-repark:${leaseToken}`,
310
+ values: { approvalStatus: 'pending' },
311
+ });
312
+ return;
313
+ }
314
+
315
+ await this.stateMachine.transition({
316
+ runId: snapshot.id,
317
+ to: 'running',
318
+ actorType: 'worker',
319
+ actorIdentity: this.workerId,
320
+ leaseToken,
321
+ eventType: 'run_started',
322
+ title: snapshot.verificationRecoverySummary
323
+ ? 'Verification recovery started'
324
+ : snapshot.resumeContext
325
+ ? 'Run resumed'
326
+ : 'Run started',
327
+ content: snapshot.verificationRecoverySummary
328
+ ? 'Recovering verification using the saved maker summary.'
329
+ : snapshot.resumeContext
330
+ ? `${snapshot.resumeContext.username} resumed after approval decisions.`
331
+ : `Leader ${snapshot.leaderUsername} began working on the goal.`,
332
+ correlationKey: `started:${leaseToken}`,
333
+ values: {
334
+ currentRole: snapshot.verificationRecoverySummary
335
+ ? 'verifier'
336
+ : snapshot.resumeContext
337
+ ? snapshot.resumeContext.role
338
+ : 'leader',
339
+ },
340
+ });
341
+
342
+ // Loops recorded before this claim (for example before a park for approval) keep warning the
343
+ // prompts composed later in the run.
344
+ const loopNotices = await this.loopGuard.existingNotices(snapshot.id);
345
+ const totalPhases = 1 + snapshot.makerUsernames.length + 1;
346
+ const runStartedAt = new Date();
347
+
348
+ if (snapshot.verificationRecoverySummary) {
349
+ return this.finishVerification(
350
+ snapshot,
351
+ leaseToken,
352
+ signal,
353
+ loopNotices,
354
+ totalPhases,
355
+ runStartedAt,
356
+ snapshot.verificationRecoverySummary,
357
+ );
358
+ }
359
+
360
+ let makerReports: string[];
361
+ let makerQueue: string[];
362
+
363
+ if (snapshot.resumeContext) {
364
+ const resume = snapshot.resumeContext;
365
+ const decisions = await this.consumeDecisions(snapshot.id);
366
+ const resumed = await this.resumeRole(snapshot, resume, decisions, signal);
367
+
368
+ const resumeGuard = await this.enforceToolLoopGuard(
369
+ snapshot,
370
+ leaseToken,
371
+ { role: resume.role, username: resume.username },
372
+ loopNotices,
373
+ );
374
+ if (resumeGuard === 'halt') return;
375
+
376
+ // A resumed conversation can hit another approval gate; park with the same role context so
377
+ // the next resume continues exactly where this one stopped.
378
+ if (resumed.interrupted.length) {
379
+ await this.parkForApproval(snapshot, resumed, leaseToken, {
380
+ role: resume.role,
381
+ username: resume.username,
382
+ makerReports: resume.makerReports,
383
+ makerQueue: resume.makerQueue,
384
+ });
385
+ return;
386
+ }
387
+ if (resume.role === 'leader') {
388
+ makerReports = [resumed.content];
389
+ makerQueue = [...snapshot.makerUsernames];
390
+ } else {
391
+ makerReports = [...resume.makerReports, `# ${resume.username}\n${resumed.content}`];
392
+ makerQueue = [...resume.makerQueue];
393
+ }
394
+ } else {
395
+ const leader = await this.invokeRole(snapshot, {
396
+ role: 'leader',
397
+ username: snapshot.leaderUsername,
398
+ harness: snapshot.leaderHarness,
399
+ systemMessage: LEADER_SYSTEM_MESSAGE,
400
+ prompt: snapshot.goal,
401
+ leaseToken,
402
+ signal,
403
+ });
404
+
405
+ const leaderGuard = await this.enforceToolLoopGuard(
406
+ snapshot,
407
+ leaseToken,
408
+ { role: 'leader', username: snapshot.leaderUsername },
409
+ loopNotices,
410
+ );
411
+ if (leaderGuard === 'halt') return;
412
+ await this.updateProgress(
413
+ snapshot.id,
414
+ buildProgress('leader', totalPhases, 1, 'Leader plan complete', 'leader', runStartedAt),
415
+ );
416
+
417
+ // plugin-ai stops the graph on an approval-gated tool. Approval is a human decision, so the
418
+ // run parks here instead of the worker deciding on the operator's behalf.
419
+ if (leader.interrupted.length) {
420
+ await this.parkForApproval(snapshot, leader, leaseToken, {
421
+ role: 'leader',
422
+ username: snapshot.leaderUsername,
423
+ makerReports: [],
424
+ makerQueue: snapshot.makerUsernames,
425
+ });
426
+ return;
427
+ }
428
+ makerReports = [leader.content];
429
+ makerQueue = [...snapshot.makerUsernames];
430
+ }
431
+
432
+ // Each maker executes with its own compiled harness snapshot. Reusing one harness would apply
433
+ // the wrong per-employee tool grants, so the snapshot is looked up by username and a maker with
434
+ // no snapshot is a compile error, not a silent skip.
435
+ while (makerQueue.length) {
436
+ const username = makerQueue[0];
437
+ const harness = snapshot.makerHarnesses[username]?.effective;
438
+ if (!harness) {
439
+ throw new Error(`The claimed run has no harness snapshot for maker "${username}".`);
440
+ }
441
+ const maker = await this.invokeRole(snapshot, {
442
+ role: 'maker',
443
+ username,
444
+ harness,
445
+ systemMessage: MAKER_SYSTEM_MESSAGE,
446
+ prompt: [
447
+ `Goal: ${snapshot.goal}`,
448
+ '',
449
+ "Leader's plan:",
450
+ makerReports[0] || '(the leader produced no textual plan)',
451
+ ...loopNotices,
452
+ ].join('\n'),
453
+ leaseToken,
454
+ signal,
455
+ });
456
+ const makerGuard = await this.enforceToolLoopGuard(
457
+ snapshot,
458
+ leaseToken,
459
+ { role: 'maker', username },
460
+ loopNotices,
461
+ );
462
+ if (makerGuard === 'halt') return;
463
+ if (maker.interrupted.length) {
464
+ await this.parkForApproval(snapshot, maker, leaseToken, {
465
+ role: 'maker',
466
+ username,
467
+ makerReports,
468
+ makerQueue: makerQueue.slice(1),
469
+ });
470
+ return;
471
+ }
472
+ makerReports.push(`# ${username}\n${maker.content}`);
473
+ const completedMakers = snapshot.makerUsernames.length - makerQueue.length + 1;
474
+ await this.updateProgress(
475
+ snapshot.id,
476
+ buildProgress('maker', totalPhases, 1 + completedMakers, `Maker "${username}" complete`, 'maker', runStartedAt),
477
+ );
478
+ makerQueue = makerQueue.slice(1);
479
+ }
480
+ const makerSummary = makerReports.join('\n\n');
481
+
482
+ await this.finishVerification(snapshot, leaseToken, signal, loopNotices, totalPhases, runStartedAt, makerSummary);
483
+ }
484
+
485
+ private async finishVerification(
486
+ snapshot: RunSnapshot,
487
+ leaseToken: string,
488
+ signal: AbortSignal,
489
+ loopNotices: string[],
490
+ totalPhases: number,
491
+ runStartedAt: Date,
492
+ makerSummary: string,
493
+ ) {
494
+ await this.stateMachine.transition({
495
+ runId: snapshot.id,
496
+ to: 'verifying',
497
+ actorType: 'worker',
498
+ actorIdentity: this.workerId,
499
+ leaseToken,
500
+ eventType: 'run_verifying',
501
+ title: 'Verification started',
502
+ content: `Verifier ${snapshot.verifierUsername} is reviewing the reported result.`,
503
+ correlationKey: `verifying:${leaseToken}`,
504
+ values: {
505
+ currentRole: 'verifier',
506
+ finalAnswer: makerSummary,
507
+ ...(snapshot.verificationRecoverySummary ? { resumeContext: null } : {}),
508
+ },
509
+ });
510
+ await this.updateProgress(
511
+ snapshot.id,
512
+ buildProgress(
513
+ 'verifying',
514
+ totalPhases,
515
+ totalPhases - 1,
516
+ `Verifier "${snapshot.verifierUsername}" reviewing`,
517
+ 'verifier',
518
+ runStartedAt,
519
+ ),
520
+ );
521
+
522
+ await this.budgets.reserve({
523
+ runId: snapshot.id,
524
+ patternId: snapshot.patternId,
525
+ policy: snapshot.policy,
526
+ delta: { verifications: 1 },
527
+ });
528
+ const outcome = await this.verification.verifyAndFinalize({
529
+ runId: snapshot.id,
530
+ rootRunId: snapshot.rootRunId,
531
+ goal: snapshot.goal,
532
+ autonomyLevel: snapshot.autonomyLevel,
533
+ verifierUsername: snapshot.verifierUsername,
534
+ leaderUsername: snapshot.leaderUsername,
535
+ makerUsernames: snapshot.makerUsernames,
536
+ verifierHarness: snapshot.verifierHarness,
537
+ policy: snapshot.policy,
538
+ makerSummary,
539
+ actingOn: snapshot.actingOn,
540
+ userId: snapshot.userId,
541
+ workerId: this.workerId,
542
+ leaseToken,
543
+ loopNotices,
544
+ signal,
545
+ });
546
+
547
+ // The tool loop guard already moved the run to blocked or waiting_human; the circuit breaker
548
+ // tracks failed attempts, not behavioral stops, so it must not see this as a failure.
549
+ if (outcome.finalStatus === 'halted') return;
550
+
551
+ if (outcome.finalStatus === 'failed') {
552
+ await this.circuits.recordFailure({
553
+ patternId: snapshot.patternId,
554
+ scopeKey: `pattern:${snapshot.patternId}`,
555
+ policy: snapshot.policy.circuit,
556
+ error: new Error(`Verifier rejected run ${snapshot.id}.`),
557
+ lastRunId: snapshot.id,
558
+ });
559
+ return;
560
+ }
561
+ await this.circuits.recordSuccess(snapshot.patternId, `pattern:${snapshot.patternId}`, snapshot.id);
562
+ await this.updateProgress(
563
+ snapshot.id,
564
+ buildProgress('complete', totalPhases, totalPhases, 'Run completed successfully', 'verifier', runStartedAt),
565
+ );
566
+ }
567
+
568
+ private async acquirePaths(snapshot: RunSnapshot, leaseToken: string) {
569
+ if (!snapshot.repositoryKey || !snapshot.actingOn.length) return true;
570
+ const lock = await this.pathLocks.acquire({
571
+ runId: snapshot.id,
572
+ repositoryKey: snapshot.repositoryKey,
573
+ owner: leaseToken,
574
+ paths: snapshot.actingOn,
575
+ ttlMs: PATH_LOCK_TTL_MS,
576
+ });
577
+ if (lock.acquired === true) {
578
+ // Recorded before any work starts so the heartbeat renews this lock for as long as the run
579
+ // executes, and the release in `execute` frees exactly the lock this worker took.
580
+ this.heldPathLocks.set(snapshot.id, snapshot.repositoryKey);
581
+ return true;
582
+ }
583
+
584
+ const deadlocked = lock.deadlocked;
585
+ await this.stateMachine.transition({
586
+ runId: snapshot.id,
587
+ to: deadlocked ? 'blocked' : 'waiting_lock',
588
+ actorType: 'worker',
589
+ actorIdentity: this.workerId,
590
+ leaseToken,
591
+ eventType: deadlocked ? 'run_blocked' : 'run_waiting_lock',
592
+ title: deadlocked ? 'Run blocked by a lock cycle' : 'Waiting for repository paths',
593
+ content: `Blocked by run(s): ${lock.blockers.join(', ')}.`,
594
+ correlationKey: `lock:${leaseToken}`,
595
+ values: deadlocked ? { blockedReason: 'Path lock wait graph contains a cycle.' } : {},
596
+ });
597
+ return false;
598
+ }
599
+
600
+ private async invokeRole(
601
+ snapshot: RunSnapshot,
602
+ input: {
603
+ role: 'leader' | 'maker';
604
+ username: string;
605
+ harness: CompiledHarness;
606
+ systemMessage: string;
607
+ prompt: string;
608
+ leaseToken: string;
609
+ signal: AbortSignal;
610
+ },
611
+ ) {
612
+ await this.budgets.reserve({
613
+ runId: snapshot.id,
614
+ patternId: snapshot.patternId,
615
+ policy: snapshot.policy,
616
+ delta: { invocations: 1 },
617
+ });
618
+ const sessionId = await this.runtime.createConversation({
619
+ username: input.username,
620
+ userId: snapshot.userId,
621
+ runId: snapshot.id,
622
+ title: snapshot.goal.slice(0, 200),
623
+ });
624
+ const step = await this.database.getRepository('agentLoopSteps').create({
625
+ values: {
626
+ runId: snapshot.id,
627
+ runtimeVersion: 'control-plane-v2',
628
+ sequence: await this.nextSequence(snapshot.id),
629
+ role: input.role,
630
+ kind: 'invocation',
631
+ type: 'reasoning',
632
+ employeeUsername: input.username,
633
+ sessionId,
634
+ title: `${input.role} invocation`,
635
+ inputHash: inputHash(input.prompt),
636
+ status: 'running',
637
+ startedAt: new Date(),
638
+ createdAt: new Date(),
639
+ },
640
+ });
641
+
642
+ const outcome = await this.runtime.invoke({
643
+ username: input.username,
644
+ sessionId,
645
+ userId: snapshot.userId,
646
+ systemMessage: input.systemMessage,
647
+ harness: input.harness,
648
+ prompt: input.prompt,
649
+ signal: input.signal,
650
+ });
651
+
652
+ const stepId = read(step, 'id');
653
+ if (typeof stepId !== 'string' && typeof stepId !== 'number') throw new Error('Invocation step has no identifier.');
654
+ await this.tokenTracker.trackLoopInvocationUsage({
655
+ runId: snapshot.id,
656
+ rootRunId: snapshot.rootRunId,
657
+ stepId,
658
+ sessionId: outcome.sessionId,
659
+ messageId: outcome.messageId,
660
+ usage: outcome.tokenUsage,
661
+ input: input.prompt,
662
+ output: outcome.content,
663
+ role: input.role,
664
+ employeeUsername: input.username,
665
+ userId: snapshot.userId,
666
+ });
667
+
668
+ await this.database.getRepository('agentLoopSteps').update({
669
+ filterByTk: stepId,
670
+ values: {
671
+ status: outcome.interrupted.length ? 'waiting_user' : 'succeeded',
672
+ output: { content: outcome.content },
673
+ endedAt: new Date(),
674
+ },
675
+ });
676
+ await this.database.getRepository('agentLoopRuns').update({
677
+ filterByTk: snapshot.id,
678
+ values: { sessionId: outcome.sessionId, messageId: outcome.messageId },
679
+ });
680
+ return outcome;
681
+ }
682
+
683
+ private async resumeRole(
684
+ snapshot: RunSnapshot,
685
+ resume: ResumeContext,
686
+ decisions: Array<{ toolCallId: string; decision: ToolCallDecision }>,
687
+ signal: AbortSignal,
688
+ ) {
689
+ await this.budgets.reserve({
690
+ runId: snapshot.id,
691
+ patternId: snapshot.patternId,
692
+ policy: snapshot.policy,
693
+ delta: { invocations: 1 },
694
+ });
695
+ const harness =
696
+ resume.role === 'leader' ? snapshot.leaderHarness : snapshot.makerHarnesses[resume.username]?.effective;
697
+ if (!harness) {
698
+ throw new Error(`The claimed run has no harness snapshot for ${resume.role} "${resume.username}".`);
699
+ }
700
+ const step = await this.database.getRepository('agentLoopSteps').create({
701
+ values: {
702
+ runId: snapshot.id,
703
+ runtimeVersion: 'control-plane-v2',
704
+ sequence: await this.nextSequence(snapshot.id),
705
+ role: resume.role,
706
+ kind: 'invocation',
707
+ type: 'resume',
708
+ employeeUsername: resume.username,
709
+ sessionId: resume.sessionId,
710
+ title: `${resume.role} resume after approval`,
711
+ inputHash: inputHash(decisions),
712
+ status: 'running',
713
+ startedAt: new Date(),
714
+ createdAt: new Date(),
715
+ },
716
+ });
717
+
718
+ const outcome = await this.runtime.resume({
719
+ username: resume.username,
720
+ sessionId: resume.sessionId,
721
+ messageId: resume.messageId,
722
+ userId: snapshot.userId,
723
+ systemMessage: resume.role === 'leader' ? LEADER_SYSTEM_MESSAGE : MAKER_SYSTEM_MESSAGE,
724
+ harness,
725
+ decisions,
726
+ signal,
727
+ });
728
+
729
+ const stepId = read(step, 'id');
730
+ if (typeof stepId !== 'string' && typeof stepId !== 'number') throw new Error('Resume step has no identifier.');
731
+ await this.tokenTracker.trackLoopInvocationUsage({
732
+ runId: snapshot.id,
733
+ rootRunId: snapshot.rootRunId,
734
+ stepId,
735
+ sessionId: outcome.sessionId,
736
+ messageId: outcome.messageId,
737
+ usage: outcome.tokenUsage,
738
+ input: resume.role === 'leader' ? LEADER_SYSTEM_MESSAGE : MAKER_SYSTEM_MESSAGE,
739
+ output: outcome.content,
740
+ role: resume.role,
741
+ employeeUsername: resume.username,
742
+ userId: snapshot.userId,
743
+ });
744
+
745
+ await this.database.getRepository('agentLoopSteps').update({
746
+ filterByTk: stepId,
747
+ values: {
748
+ status: outcome.interrupted.length ? 'waiting_user' : 'succeeded',
749
+ output: { content: outcome.content },
750
+ endedAt: new Date(),
751
+ },
752
+ });
753
+ await this.database.getRepository('agentLoopRuns').update({
754
+ filterByTk: snapshot.id,
755
+ values: { sessionId: outcome.sessionId, messageId: outcome.messageId },
756
+ });
757
+ return outcome;
758
+ }
759
+
760
+ // Run-level scan over every tool call issued so far, evaluated at each pass boundary. A warning
761
+ // is injected into the prompts composed later in the run; block and escalate halt it at once.
762
+ private async enforceToolLoopGuard(
763
+ snapshot: RunSnapshot,
764
+ leaseToken: string,
765
+ pass: { role: 'leader' | 'maker'; username: string },
766
+ loopNotices: string[],
767
+ ): Promise<'continue' | 'halt'> {
768
+ const finding = await this.loopGuard.scanRun(snapshot.id, snapshot.policy);
769
+ if (!finding) return 'continue';
770
+
771
+ await this.loopGuard.recordFinding(
772
+ {
773
+ runId: snapshot.id,
774
+ role: pass.role,
775
+ username: pass.username,
776
+ runStatus: 'running',
777
+ actorType: 'worker',
778
+ actorIdentity: this.workerId,
779
+ },
780
+ finding,
781
+ );
782
+
783
+ if (finding.level === 'warn') {
784
+ const notice = toolLoopNotice(finding.toolName, finding.count);
785
+ if (!loopNotices.includes(notice)) loopNotices.push(notice);
786
+ return 'continue';
787
+ }
788
+
789
+ const reason = toolLoopReason(finding);
790
+ await this.stateMachine.transition({
791
+ runId: snapshot.id,
792
+ to: finding.level === 'block' ? 'blocked' : 'waiting_human',
793
+ actorType: 'worker',
794
+ actorIdentity: this.workerId,
795
+ leaseToken,
796
+ eventType: finding.level === 'block' ? 'run_blocked' : 'run_escalated',
797
+ title: finding.level === 'block' ? 'Run blocked by tool loop detection' : 'Run escalated by tool loop detection',
798
+ content: reason,
799
+ correlationKey: `tool-loop-${finding.level}:${leaseToken}`,
800
+ values: finding.level === 'block' ? { blockedReason: reason } : { escalationReason: reason },
801
+ });
802
+ return 'halt';
803
+ }
804
+
805
+ // Decisions are consumed exactly once: rows are marked consumed so a retry or a second resume
806
+ // can never replay a human decision against a fresh conversation.
807
+ private async consumeDecisions(runId: number) {
808
+ const repository = this.database.getRepository('agentLoopActionApprovals');
809
+ const approvals = await repository.find({ filter: { runId } });
810
+ const decisions: Array<{ toolCallId: string; decision: ToolCallDecision }> = [];
811
+ const now = new Date();
812
+ for (const approval of approvals) {
813
+ const status = String(read(approval, 'status'));
814
+ if (status !== 'approved' && status !== 'rejected') continue;
815
+ if (read(approval, 'consumedAt')) continue;
816
+ const editedInput = read(approval, 'editedInput');
817
+ const decision: ToolCallDecision =
818
+ status === 'approved'
819
+ ? editedInput
820
+ ? { type: 'edit', editedAction: { name: String(read(approval, 'toolName')), args: editedInput } }
821
+ : { type: 'approve' }
822
+ : { type: 'reject', message: String(read(approval, 'decisionNote') || '') || undefined };
823
+ decisions.push({ toolCallId: String(read(approval, 'toolCallId')), decision });
824
+ await repository.update({ filterByTk: Number(read(approval, 'id')), values: { consumedAt: now } });
825
+ }
826
+ if (!decisions.length) throw new Error('The resumed run has no recorded approval decisions.');
827
+ return decisions;
828
+ }
829
+
830
+ private async parkForApproval(
831
+ snapshot: RunSnapshot,
832
+ outcome: InvocationOutcome,
833
+ leaseToken: string,
834
+ context: { role: 'leader' | 'maker'; username: string; makerReports: string[]; makerQueue: string[] },
835
+ ) {
836
+ const now = new Date();
837
+ const expiresAt = new Date(now.getTime() + snapshot.policy.actions.approvalTimeoutMs);
838
+ const assignedToId = snapshot.policy.actions.approvalAssigneeIds[0] || snapshot.userId || null;
839
+ const harness =
840
+ context.role === 'leader' ? snapshot.leaderHarness : snapshot.makerHarnesses[context.username]?.effective;
841
+ const escalatable = new Set(harness?.tools.escalate ?? []);
842
+ for (const call of outcome.interrupted) {
843
+ // An escalation is a request for authority the harness never granted, widened for exactly
844
+ // one call; a plain tool_call approval gates a tool the harness allows but asks about.
845
+ const escalation = escalatable.has(call.toolName);
846
+ await this.database.getRepository('agentLoopActionApprovals').create({
847
+ values: {
848
+ runId: snapshot.id,
849
+ toolCallId: call.toolCallId,
850
+ toolName: call.toolName,
851
+ actionType: escalation ? 'escalation' : 'tool_call',
852
+ proposedInput: call.args,
853
+ inputHash: inputHash(call.args),
854
+ paths: snapshot.actingOn,
855
+ policyDecision: { decision: 'ask', interruptId: call.interruptId },
856
+ reason: escalation
857
+ ? `Tool "${call.toolName}" is not granted by this run's harness. Approving widens authority for this single call only.`
858
+ : `Tool "${call.toolName}" requires human approval under this pattern's policy.`,
859
+ status: 'pending',
860
+ assignedToId,
861
+ requestedById: snapshot.userId || null,
862
+ requestedAt: now,
863
+ expiresAt,
864
+ createdAt: now,
865
+ },
866
+ });
867
+ }
868
+ await this.database.getRepository('agentLoopRuns').update({
869
+ filterByTk: snapshot.id,
870
+ values: {
871
+ // The resume point is durable: which role stopped, in which conversation, with which
872
+ // reports already collected and which makers still queued. Without it a resumed run
873
+ // would restart from the leader and redo every completed invocation.
874
+ resumeContext: {
875
+ role: context.role,
876
+ username: context.username,
877
+ sessionId: outcome.sessionId,
878
+ messageId: outcome.messageId,
879
+ makerReports: context.makerReports,
880
+ makerQueue: context.makerQueue,
881
+ },
882
+ },
883
+ });
884
+ await this.stateMachine.transition({
885
+ runId: snapshot.id,
886
+ to: 'waiting_approval',
887
+ actorType: 'worker',
888
+ actorIdentity: this.workerId,
889
+ leaseToken,
890
+ eventType: 'run_waiting_approval',
891
+ title: 'Waiting for action approval',
892
+ content: `${outcome.interrupted.length} tool call(s) require approval.`,
893
+ correlationKey: `approval:${leaseToken}`,
894
+ values: { approvalStatus: 'pending', messageId: outcome.messageId },
895
+ });
896
+ }
897
+
898
+ private async failRun(runId: number, leaseToken: string, snapshot: RunSnapshot | null, error: unknown) {
899
+ const message = error instanceof Error ? error.message : String(error);
900
+ try {
901
+ if (snapshot) {
902
+ await this.circuits.recordFailure({
903
+ patternId: snapshot.patternId,
904
+ scopeKey: `pattern:${snapshot.patternId}`,
905
+ policy: snapshot.policy.circuit,
906
+ error,
907
+ lastRunId: runId,
908
+ });
909
+ }
910
+ await this.stateMachine.transition({
911
+ runId,
912
+ to: 'failed',
913
+ actorType: 'worker',
914
+ actorIdentity: this.workerId,
915
+ leaseToken,
916
+ eventType: 'run_failed',
917
+ title: 'Run failed',
918
+ content: message,
919
+ correlationKey: `failed:${leaseToken}`,
920
+ values: { summary: message },
921
+ });
922
+ } catch (transitionError) {
923
+ this.app.logger.error(`[AgentOrchestrator] Loop run ${runId} could not be marked failed.`, {
924
+ error: transitionError,
925
+ cause: message,
926
+ });
927
+ }
928
+ }
929
+
930
+ private async nextSequence(runId: number) {
931
+ return this.database.getRepository('agentLoopSteps').count({ filter: { runId } });
932
+ }
933
+
934
+ private snapshotFrom(run: Record<string, unknown>): RunSnapshot {
935
+ const patternId = Number(run.patternId);
936
+ if (!Number.isSafeInteger(patternId) || patternId <= 0) {
937
+ throw new Error('The claimed run has no pattern snapshot.');
938
+ }
939
+ const roleBindings = asObject(run.roleBindingsSnapshot);
940
+ const makerSnapshots = asObject(run.makerHarnessSnapshot);
941
+ const approvalStatus = String(run.approvalStatus || 'none');
942
+ const recoveryContext = asObject(run.resumeContext);
943
+ const verificationRecoverySummary =
944
+ recoveryContext.kind === 'verification_lease_recovery' &&
945
+ typeof recoveryContext.makerSummary === 'string' &&
946
+ recoveryContext.makerSummary.trim()
947
+ ? recoveryContext.makerSummary
948
+ : null;
949
+ return {
950
+ id: Number(run.id),
951
+ rootRunId: String(run.rootRunId || ''),
952
+ patternId,
953
+ goal: String(run.goal || ''),
954
+ autonomyLevel: autonomyLevel(run.autonomyLevel),
955
+ policy: loopPatternPolicySchema.parse(run.policySnapshot),
956
+ leaderUsername: String(roleBindings.leader || run.leaderUsername || ''),
957
+ makerUsernames: asStringArray(roleBindings.makers),
958
+ verifierUsername: String(roleBindings.verifier || run.verifierUsername || ''),
959
+ leaderHarness: harnessFrom(run.leaderHarnessSnapshot),
960
+ makerHarnesses: makerSnapshots as Record<string, { effective: CompiledHarness }>,
961
+ verifierHarness: harnessFrom(run.verifierHarnessSnapshot),
962
+ repositoryKey: String(run.repositoryKey || ''),
963
+ actingOn: asStringArray(run.actingOn),
964
+ userId: Number(run.userId) || undefined,
965
+ approvalStatus,
966
+ verificationRecoverySummary,
967
+ // Only a live approval state may resume. Stale context left over from an expired window or a
968
+ // cleared retry would otherwise replay an old conversation against fresh approvals.
969
+ resumeContext:
970
+ !verificationRecoverySummary && (approvalStatus === 'decided' || approvalStatus === 'pending')
971
+ ? resumeContextFrom(run.resumeContext)
972
+ : null,
973
+ };
974
+ }
975
+ }