plugin-agent-orchestrator 1.1.2 → 1.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (173) hide show
  1. package/dist/client/index.js +1 -1
  2. package/dist/client-v2/214.b063f33ba044a808.js +10 -0
  3. package/dist/client-v2/253.59577b40d9875be0.js +10 -0
  4. package/dist/client-v2/264.7ed8bee71fa5935c.js +10 -0
  5. package/dist/client-v2/{41.ba6e080cc0488143.js → 41.3deb9e57a0f64fc4.js} +1 -1
  6. package/dist/client-v2/418.58fa350f5496efd3.js +10 -0
  7. package/dist/client-v2/{53.7c29c28043e9112d.js → 53.9a10455323390f14.js} +1 -1
  8. package/dist/client-v2/619.0a2144aaffefe018.js +10 -0
  9. package/dist/client-v2/{70.bda9518881c05360.js → 70.ffff2997c10fd4b0.js} +1 -1
  10. package/dist/client-v2/{794.9bbb8d3a61b2bdf2.js → 794.a67a2614405ba627.js} +1 -1
  11. package/dist/client-v2/925.e5925bf21ed68e7a.js +10 -0
  12. package/dist/client-v2/{993.82778b6fd6db3cb1.js → 993.b0688fdc5ccb000a.js} +1 -1
  13. package/dist/client-v2/index.js +1 -1
  14. package/dist/externalVersion.js +8 -6
  15. package/dist/locale/en-US.json +48 -28
  16. package/dist/locale/vi-VN.json +48 -28
  17. package/dist/locale/zh-CN.json +48 -28
  18. package/dist/server/collections/agent-execution-spans.js +47 -0
  19. package/dist/server/collections/agent-harness-profile-versions.js +56 -0
  20. package/dist/server/collections/agent-harness-profiles.js +12 -0
  21. package/dist/server/collections/agent-loop-action-approvals.js +68 -0
  22. package/dist/server/collections/agent-loop-artifacts.js +55 -0
  23. package/dist/server/collections/agent-loop-circuit-states.js +53 -0
  24. package/dist/server/collections/agent-loop-control-settings.js +51 -0
  25. package/dist/server/collections/agent-loop-events.js +27 -0
  26. package/dist/server/collections/agent-loop-path-locks.js +57 -0
  27. package/dist/server/collections/agent-loop-patterns.js +72 -0
  28. package/dist/server/collections/agent-loop-runs.js +163 -2
  29. package/dist/server/collections/agent-loop-steps.js +69 -0
  30. package/dist/server/collections/agent-loop-usage-buckets.js +52 -0
  31. package/dist/server/collections/agent-loop-worktrees.js +55 -0
  32. package/dist/server/migrations/{20260427000000-change-packages-to-text.js → 20260427000002-change-packages-to-text.js} +2 -0
  33. package/dist/server/migrations/20260429000000-add-llm-fields.js +3 -2
  34. package/dist/server/migrations/{20260429000000-fix-inputargs-json-to-text.js → 20260429000001-fix-inputargs-json-to-text.js} +14 -6
  35. package/dist/server/migrations/20260503000000-add-orchestrator-trace-fields.js +2 -1
  36. package/dist/server/migrations/20260524000000-add-agent-loop-fields-to-skill-executions.js +2 -1
  37. package/dist/server/migrations/20260803010000-backfill-loop-control-plane-data.js +262 -0
  38. package/dist/server/plugin.js +52 -150
  39. package/dist/server/resources/agent-loop-approval.js +83 -0
  40. package/dist/server/resources/agent-loop-artifact.js +86 -0
  41. package/dist/server/resources/agent-loop-events-stream.js +130 -0
  42. package/dist/server/resources/agent-loop.js +181 -139
  43. package/dist/server/resources/resource-helpers.js +63 -0
  44. package/dist/server/resources/tracing.js +121 -107
  45. package/dist/server/services/DurableCircuitBreakerService.js +229 -0
  46. package/dist/server/services/HarnessCompiler.js +142 -0
  47. package/dist/server/services/HarnessProfileService.js +168 -0
  48. package/dist/server/services/HarnessSchema.js +167 -0
  49. package/dist/server/services/LoopBudgetService.js +269 -0
  50. package/dist/server/services/LoopControlService.js +139 -0
  51. package/dist/server/services/LoopPatternSchema.js +180 -0
  52. package/dist/server/services/LoopPatternService.js +132 -0
  53. package/dist/server/services/LoopRunRepository.js +205 -0
  54. package/dist/server/services/LoopRunStateMachine.js +389 -0
  55. package/dist/server/services/LoopSchedulerService.js +193 -0
  56. package/dist/server/services/LoopTriggerService.js +128 -0
  57. package/dist/server/services/LoopWorkerService.js +461 -0
  58. package/dist/server/services/PathLockService.js +288 -0
  59. package/dist/server/services/PluginAiRuntimeAdapter.js +215 -0
  60. package/dist/server/services/VerificationSchema.js +59 -0
  61. package/dist/server/services/VerificationService.js +169 -0
  62. package/dist/server/skill-hub/actions/git-import.js +13 -7
  63. package/dist/server/skill-hub/services/RegistrySkillSnapshotService.js +1 -5
  64. package/dist/server/utils/ensure-indexes.js +111 -0
  65. package/dist/server/utils/loop-pattern-context.js +56 -0
  66. package/package.json +2 -2
  67. package/src/client/OrchestratorSettings.tsx +44 -0
  68. package/src/client-v2/components/AgentRunsTab.tsx +576 -494
  69. package/src/client-v2/components/HarnessProfilesTab.tsx +13 -8
  70. package/src/client-v2/components/MemoryInspectorTab.tsx +19 -3
  71. package/src/client-v2/components/TracingTab.tsx +515 -432
  72. package/src/client-v2/hooks/useApiRequest.ts +141 -121
  73. package/src/client-v2/skill-hub/components/SkillMetrics.tsx +8 -6
  74. package/src/locale/en-US.json +52 -28
  75. package/src/locale/vi-VN.json +52 -28
  76. package/src/locale/zh-CN.json +52 -28
  77. package/src/server/__tests__/circuit-breaker.test.ts +225 -146
  78. package/src/server/__tests__/harness-compiler.test.ts +131 -0
  79. package/src/server/__tests__/harness-profile-service.test.ts +116 -0
  80. package/src/server/__tests__/harness-tool-plan.test.ts +148 -0
  81. package/src/server/__tests__/loop-budget-service.test.ts +255 -0
  82. package/src/server/__tests__/loop-control-plane-backfill.test.ts +266 -0
  83. package/src/server/__tests__/loop-pattern-schema.test.ts +56 -0
  84. package/src/server/__tests__/loop-pattern-service.test.ts +148 -0
  85. package/src/server/__tests__/loop-run-state-machine.test.ts +90 -0
  86. package/src/server/__tests__/loop-worker-claim.test.ts +341 -0
  87. package/src/server/__tests__/loop-worker-execution.test.ts +236 -0
  88. package/src/server/__tests__/path-lock-service.test.ts +254 -0
  89. package/src/server/__tests__/smoke.test.ts +62 -21
  90. package/src/server/collections/agent-execution-spans.ts +47 -0
  91. package/src/server/collections/agent-harness-profile-versions.ts +26 -0
  92. package/src/server/collections/agent-harness-profiles.ts +12 -0
  93. package/src/server/collections/agent-loop-action-approvals.ts +38 -0
  94. package/src/server/collections/agent-loop-artifacts.ts +25 -0
  95. package/src/server/collections/agent-loop-circuit-states.ts +23 -0
  96. package/src/server/collections/agent-loop-control-settings.ts +21 -0
  97. package/src/server/collections/agent-loop-events.ts +27 -0
  98. package/src/server/collections/agent-loop-path-locks.ts +27 -0
  99. package/src/server/collections/agent-loop-patterns.ts +42 -0
  100. package/src/server/collections/agent-loop-runs.ts +163 -2
  101. package/src/server/collections/agent-loop-steps.ts +69 -0
  102. package/src/server/collections/agent-loop-usage-buckets.ts +22 -0
  103. package/src/server/collections/agent-loop-worktrees.ts +25 -0
  104. package/src/server/migrations/{20260427000000-change-packages-to-text.ts → 20260427000002-change-packages-to-text.ts} +3 -0
  105. package/src/server/migrations/20260429000000-add-llm-fields.ts +3 -2
  106. package/src/server/migrations/{20260429000000-fix-inputargs-json-to-text.ts → 20260429000001-fix-inputargs-json-to-text.ts} +13 -6
  107. package/src/server/migrations/20260503000000-add-orchestrator-trace-fields.ts +2 -1
  108. package/src/server/migrations/20260524000000-add-agent-loop-fields-to-skill-executions.ts +2 -1
  109. package/src/server/migrations/20260803010000-backfill-loop-control-plane-data.ts +293 -0
  110. package/src/server/plugin.ts +53 -167
  111. package/src/server/resources/agent-loop-approval.ts +52 -0
  112. package/src/server/resources/agent-loop-artifact.ts +56 -0
  113. package/src/server/resources/agent-loop-events-stream.ts +105 -0
  114. package/src/server/resources/agent-loop.ts +202 -143
  115. package/src/server/resources/agent-monitor.ts +482 -482
  116. package/src/server/resources/resource-helpers.ts +33 -0
  117. package/src/server/resources/tracing.ts +145 -137
  118. package/src/server/services/DurableCircuitBreakerService.ts +239 -0
  119. package/src/server/services/HarnessCompiler.ts +133 -0
  120. package/src/server/services/HarnessProfileService.ts +157 -0
  121. package/src/server/services/HarnessSchema.ts +163 -0
  122. package/src/server/services/LoopBudgetService.ts +275 -0
  123. package/src/server/services/LoopControlService.ts +124 -0
  124. package/src/server/services/LoopPatternSchema.ts +167 -0
  125. package/src/server/services/LoopPatternService.ts +143 -0
  126. package/src/server/services/LoopRunRepository.ts +237 -0
  127. package/src/server/services/LoopRunStateMachine.ts +432 -0
  128. package/src/server/services/LoopSchedulerService.ts +199 -0
  129. package/src/server/services/LoopTriggerService.ts +115 -0
  130. package/src/server/services/LoopWorkerService.ts +522 -0
  131. package/src/server/services/PathLockService.ts +321 -0
  132. package/src/server/services/PluginAiRuntimeAdapter.ts +315 -0
  133. package/src/server/services/VerificationSchema.ts +31 -0
  134. package/src/server/services/VerificationService.ts +185 -0
  135. package/src/server/skill-hub/actions/__tests__/git-import.test.ts +62 -0
  136. package/src/server/skill-hub/actions/git-import.ts +9 -3
  137. package/src/server/skill-hub/services/RegistrySkillSnapshotService.ts +1 -5
  138. package/src/server/skill-hub/services/__tests__/RegistrySkillSnapshotService.test.ts +1 -3
  139. package/src/server/utils/ensure-indexes.ts +96 -0
  140. package/src/server/utils/logging.ts +0 -1
  141. package/src/server/utils/loop-pattern-context.ts +20 -0
  142. package/dist/client-v2/214.79650a549273f163.js +0 -10
  143. package/dist/client-v2/253.9f0537713cee9e6c.js +0 -10
  144. package/dist/client-v2/264.718a107e43fc163c.js +0 -10
  145. package/dist/client-v2/418.ce8f5b2b8a229d1d.js +0 -10
  146. package/dist/client-v2/619.bd3c5698b40705c3.js +0 -10
  147. package/dist/client-v2/925.f5370de8f6632d65.js +0 -10
  148. package/dist/server/services/AgentHarness.js +0 -81
  149. package/dist/server/services/AgentLoopController.js +0 -1023
  150. package/dist/server/services/AgentLoopRepository.js +0 -192
  151. package/dist/server/services/AgentLoopService.js +0 -142
  152. package/dist/server/services/AgentPlanValidator.js +0 -99
  153. package/dist/server/services/AgentPlannerService.js +0 -99
  154. package/dist/server/services/CircuitBreaker.js +0 -125
  155. package/dist/server/services/ContextAggregator.js +0 -201
  156. package/dist/server/services/PlanningPromptService.js +0 -95
  157. package/dist/server/tools/agent-loop.js +0 -373
  158. package/dist/server/tools/orchestrator-plan.js +0 -254
  159. package/src/server/__tests__/agent-loop-controller.test.ts +0 -375
  160. package/src/server/__tests__/context-aggregator.test.ts +0 -222
  161. package/src/server/__tests__/parallel-execution.test.ts +0 -318
  162. package/src/server/__tests__/planning-prompt-service.test.ts +0 -23
  163. package/src/server/services/AgentHarness.ts +0 -67
  164. package/src/server/services/AgentLoopController.ts +0 -1263
  165. package/src/server/services/AgentLoopRepository.ts +0 -178
  166. package/src/server/services/AgentLoopService.ts +0 -177
  167. package/src/server/services/AgentPlanValidator.ts +0 -73
  168. package/src/server/services/AgentPlannerService.ts +0 -80
  169. package/src/server/services/CircuitBreaker.ts +0 -126
  170. package/src/server/services/ContextAggregator.ts +0 -239
  171. package/src/server/services/PlanningPromptService.ts +0 -72
  172. package/src/server/tools/agent-loop.ts +0 -360
  173. package/src/server/tools/orchestrator-plan.ts +0 -255
@@ -0,0 +1,115 @@
1
+ import { randomUUID } from 'node:crypto';
2
+ import type { Database, Model } from '@nocobase/database';
3
+ import { LoopControlService } from './LoopControlService';
4
+ import { LoopPatternService } from './LoopPatternService';
5
+
6
+ type TriggerType = 'manual' | 'cron' | 'event';
7
+
8
+ function read(record: Model | Record<string, unknown>, key: string) {
9
+ const model = record as Model & { get?: (name: string) => unknown };
10
+ return typeof model.get === 'function' ? model.get(key) : (record as Record<string, unknown>)[key];
11
+ }
12
+
13
+ function plain(record: Model | Record<string, unknown>) {
14
+ return typeof (record as Model).toJSON === 'function'
15
+ ? ((record as Model).toJSON() as Record<string, unknown>)
16
+ : (record as Record<string, unknown>);
17
+ }
18
+
19
+ function isUniqueConstraintError(error: unknown) {
20
+ return error instanceof Error && error.name === 'SequelizeUniqueConstraintError';
21
+ }
22
+
23
+ export class LoopTriggerService {
24
+ private readonly control: LoopControlService;
25
+
26
+ constructor(
27
+ private readonly database: Database,
28
+ private readonly patterns: LoopPatternService,
29
+ ) {
30
+ this.control = new LoopControlService(database);
31
+ }
32
+
33
+ async enqueue(input: {
34
+ patternId: number;
35
+ triggerType: TriggerType;
36
+ triggerKey: string;
37
+ triggerPayload?: Record<string, unknown>;
38
+ userId?: number;
39
+ goal?: string;
40
+ perRunHarness?: unknown;
41
+ }) {
42
+ const triggerKey = input.triggerKey.trim();
43
+ if (!triggerKey) throw new Error('Trigger key is required.');
44
+ await this.control.assertCanEnqueue();
45
+ const compiled = await this.patterns.compile(input.patternId, input.perRunHarness);
46
+ if (compiled.pattern.triggerType !== input.triggerType && input.triggerType !== 'manual') {
47
+ throw new Error(
48
+ `Pattern ${compiled.pattern.key} expects ${compiled.pattern.triggerType} triggers, not ${input.triggerType}.`,
49
+ );
50
+ }
51
+ const goal = (input.goal || compiled.pattern.goalTemplate).trim();
52
+ if (!goal) throw new Error('Loop run goal is required.');
53
+
54
+ try {
55
+ return await this.database.sequelize.transaction(async (transaction) => {
56
+ await this.control.assertCanEnqueue(transaction);
57
+ const now = new Date();
58
+ const run = await this.database.getRepository('agentLoopRuns').create({
59
+ values: {
60
+ rootRunId: `loop_${randomUUID()}`,
61
+ runtimeVersion: 'control-plane-v2',
62
+ recordMode: 'observed-execution',
63
+ patternId: input.patternId,
64
+ triggerType: input.triggerType,
65
+ triggerKey,
66
+ triggerPayload: input.triggerPayload || {},
67
+ autonomyLevel: compiled.pattern.autonomyLevel,
68
+ roleBindingsSnapshot: compiled.roleBindings,
69
+ leaderHarnessSnapshot: compiled.leaderHarness,
70
+ makerHarnessSnapshot: compiled.makerHarnesses,
71
+ verifierHarnessSnapshot: compiled.verifierHarness,
72
+ policySnapshot: compiled.policy,
73
+ leaderUsername: compiled.roleBindings.leader,
74
+ verifierUsername: compiled.roleBindings.verifier,
75
+ goal,
76
+ status: 'queued',
77
+ repositoryKey: compiled.pattern.repositoryKey,
78
+ repositoryRoot: compiled.pattern.repositoryRoot,
79
+ baseRef: compiled.pattern.baseRef,
80
+ actingOn: [...compiled.pattern.actingOn],
81
+ userId: input.userId || null,
82
+ createdAt: now,
83
+ updatedAt: now,
84
+ },
85
+ transaction,
86
+ });
87
+ const runId = Number(read(run, 'id'));
88
+ await this.database.getRepository('agentLoopEvents').create({
89
+ values: {
90
+ runId,
91
+ type: 'run_queued',
92
+ title: 'Run queued',
93
+ content: `Pattern ${compiled.pattern.key} accepted ${input.triggerType} trigger ${triggerKey}.`,
94
+ status: 'queued',
95
+ payload: { patternKey: compiled.pattern.key, triggerType: input.triggerType },
96
+ actorType: input.triggerType === 'manual' ? 'user' : 'system',
97
+ actorIdentity: input.userId ? String(input.userId) : input.triggerType,
98
+ correlationKey: `queued:${triggerKey}`,
99
+ userId: input.userId || null,
100
+ createdAt: now,
101
+ },
102
+ transaction,
103
+ });
104
+ return { created: true, run: plain(run) };
105
+ });
106
+ } catch (error) {
107
+ if (!isUniqueConstraintError(error)) throw error;
108
+ const existing = await this.database.getRepository('agentLoopRuns').findOne({
109
+ filter: { patternId: input.patternId, triggerKey },
110
+ });
111
+ if (!existing) throw error;
112
+ return { created: false, run: plain(existing) };
113
+ }
114
+ }
115
+ }
@@ -0,0 +1,522 @@
1
+ import { createHash, randomUUID } from 'node:crypto';
2
+ import type { Database, Model } from '@nocobase/database';
3
+ import type { Application } from '@nocobase/server';
4
+ import { DurableCircuitBreakerService } from './DurableCircuitBreakerService';
5
+ import type { CompiledHarness } from './HarnessCompiler';
6
+ import { LoopBudgetService } from './LoopBudgetService';
7
+ import { LoopControlService } from './LoopControlService';
8
+ import { loopPatternPolicySchema } from './LoopPatternSchema';
9
+ import type { LoopPatternPolicy } from './LoopPatternSchema';
10
+ import { LoopRunStateMachine } from './LoopRunStateMachine';
11
+ import type { ClaimedLoopRun } from './LoopRunStateMachine';
12
+ import { PathLockService } from './PathLockService';
13
+ import { PluginAiRuntimeAdapter } from './PluginAiRuntimeAdapter';
14
+ import type { InvocationOutcome } from './PluginAiRuntimeAdapter';
15
+ import { VerificationService } from './VerificationService';
16
+
17
+ const POLL_INTERVAL_MS = 3_000;
18
+ const LEASE_MS = 120_000;
19
+ const HEARTBEAT_MS = 30_000;
20
+ const PATH_LOCK_TTL_MS = 900_000;
21
+
22
+ export type LoopWorkerSyncMessage = {
23
+ type: 'loop-run-abort-requested';
24
+ runId: number;
25
+ };
26
+
27
+ export function loopWorkerAbortMessage(runId: number): LoopWorkerSyncMessage {
28
+ return { type: 'loop-run-abort-requested', runId };
29
+ }
30
+
31
+ type RunSnapshot = {
32
+ id: number;
33
+ patternId: number;
34
+ goal: string;
35
+ autonomyLevel: 'L1' | 'L2' | 'L3';
36
+ policy: LoopPatternPolicy;
37
+ leaderUsername: string;
38
+ makerUsernames: string[];
39
+ verifierUsername: string;
40
+ leaderHarness: CompiledHarness;
41
+ makerHarnesses: Record<string, { effective: CompiledHarness }>;
42
+ verifierHarness: CompiledHarness;
43
+ repositoryKey: string;
44
+ actingOn: string[];
45
+ userId?: number;
46
+ };
47
+
48
+ function read(record: Model | Record<string, unknown>, key: string) {
49
+ const model = record as Model & { get?: (name: string) => unknown };
50
+ return typeof model.get === 'function' ? model.get(key) : (record as Record<string, unknown>)[key];
51
+ }
52
+
53
+ function asObject(value: unknown): Record<string, unknown> {
54
+ return value && typeof value === 'object' && !Array.isArray(value) ? (value as Record<string, unknown>) : {};
55
+ }
56
+
57
+ function asStringArray(value: unknown): string[] {
58
+ return Array.isArray(value) ? value.map((item) => String(item)).filter(Boolean) : [];
59
+ }
60
+
61
+ function autonomyLevel(value: unknown): 'L1' | 'L2' | 'L3' {
62
+ return value === 'L2' || value === 'L3' ? value : 'L1';
63
+ }
64
+
65
+ function harnessFrom(value: unknown): CompiledHarness {
66
+ const snapshot = asObject(value);
67
+ const effective = asObject(snapshot.effective);
68
+ if (!effective.tools) throw new Error('The run harness snapshot is missing its compiled settings.');
69
+ return effective as unknown as CompiledHarness;
70
+ }
71
+
72
+ function inputHash(value: unknown) {
73
+ return createHash('sha256')
74
+ .update(JSON.stringify(value ?? null))
75
+ .digest('hex');
76
+ }
77
+
78
+ export class LoopWorkerService {
79
+ private readonly control: LoopControlService;
80
+ private readonly stateMachine: LoopRunStateMachine;
81
+ private readonly budgets: LoopBudgetService;
82
+ private readonly circuits: DurableCircuitBreakerService;
83
+ private readonly pathLocks: PathLockService;
84
+ private readonly runtime: PluginAiRuntimeAdapter;
85
+ private readonly verification: VerificationService;
86
+ private readonly workerId: string;
87
+ private readonly active = new Map<number, AbortController>();
88
+ private readonly heldPathLocks = new Map<number, string>();
89
+ private timer: ReturnType<typeof setTimeout> | null = null;
90
+ private running = false;
91
+ private draining: Promise<void> | null = null;
92
+
93
+ constructor(
94
+ private readonly app: Application,
95
+ private readonly database: Database,
96
+ ) {
97
+ const distributedLock = {
98
+ runExclusive: <T>(key: string, operation: () => Promise<T>, ttl?: number) =>
99
+ this.app.lockManager.runExclusive(key, operation, ttl),
100
+ };
101
+ this.control = new LoopControlService(database);
102
+ this.stateMachine = new LoopRunStateMachine(database);
103
+ this.budgets = new LoopBudgetService(database, distributedLock);
104
+ this.circuits = new DurableCircuitBreakerService(database, distributedLock);
105
+ this.pathLocks = new PathLockService(database, distributedLock);
106
+ this.runtime = new PluginAiRuntimeAdapter(app);
107
+ this.verification = new VerificationService(database, this.runtime, this.stateMachine);
108
+ this.workerId = `${app.name}:${process.pid}:${randomUUID()}`;
109
+ }
110
+
111
+ readonly start = () => {
112
+ // Multi-instance deployments dedicate a process to background work; only that process
113
+ // should claim runs, otherwise every web node competes for the same leases.
114
+ if (this.running || !this.app.serving('agent-loop:worker')) return;
115
+ this.running = true;
116
+ this.scheduleNextPoll();
117
+ };
118
+
119
+ async stop() {
120
+ this.running = false;
121
+ if (this.timer) clearTimeout(this.timer);
122
+ this.timer = null;
123
+ for (const controller of this.active.values()) controller.abort();
124
+ if (this.draining) await this.draining.catch(() => undefined);
125
+ }
126
+
127
+ abortRun(runId: number) {
128
+ this.active.get(runId)?.abort();
129
+ }
130
+
131
+ // A run is only active on the node that claimed it, and `sendSyncMessage` publishes with
132
+ // `skipSelf`, so pause/cancel aborts locally and broadcasts for whichever other node holds it.
133
+ async handleSyncMessage(message: unknown) {
134
+ if (!message || typeof message !== 'object') return;
135
+ const value = message as Partial<LoopWorkerSyncMessage>;
136
+ if (value.type !== 'loop-run-abort-requested') return;
137
+ const runId = Number(value.runId);
138
+ if (!Number.isSafeInteger(runId) || runId <= 0) return;
139
+ this.abortRun(runId);
140
+ }
141
+
142
+ private scheduleNextPoll() {
143
+ if (!this.running) return;
144
+ this.timer = setTimeout(() => {
145
+ this.draining = this.poll()
146
+ .catch((error) => {
147
+ this.app.logger.error('[AgentOrchestrator] Loop worker poll failed.', { error });
148
+ })
149
+ .finally(() => this.scheduleNextPoll());
150
+ }, POLL_INTERVAL_MS);
151
+ }
152
+
153
+ private async poll() {
154
+ if (!this.running) return;
155
+ const claimed = await this.stateMachine.claimNext(this.workerId, LEASE_MS);
156
+ if (!claimed) return;
157
+ await this.execute(claimed);
158
+ }
159
+
160
+ private async execute(claimed: ClaimedLoopRun) {
161
+ const runId = Number(claimed.run.id);
162
+ const controller = new AbortController();
163
+ this.active.set(runId, controller);
164
+
165
+ // The path lock TTL is much longer than the run lease, but a run can outlive it. Renewing both
166
+ // on the same beat keeps them expiring together, so a run can never keep executing on paths
167
+ // whose lock another run is already free to take.
168
+ const heartbeat = setInterval(() => {
169
+ this.heartbeat(runId, claimed.leaseToken).catch((error) => {
170
+ this.app.logger.warn(`[AgentOrchestrator] Loop run ${runId} lost its worker lease.`, { error });
171
+ controller.abort();
172
+ });
173
+ }, HEARTBEAT_MS);
174
+
175
+ let snapshot: RunSnapshot | null = null;
176
+ try {
177
+ snapshot = this.snapshotFrom(claimed.run);
178
+ await this.runToCompletion(snapshot, claimed.leaseToken, controller.signal);
179
+ } catch (error) {
180
+ await this.failRun(runId, claimed.leaseToken, snapshot, error);
181
+ } finally {
182
+ clearInterval(heartbeat);
183
+ this.active.delete(runId);
184
+ const repositoryKey = this.heldPathLocks.get(runId);
185
+ this.heldPathLocks.delete(runId);
186
+ if (repositoryKey) {
187
+ // The lease token is the lock owner. Passing it stops a worker whose lease was already
188
+ // reclaimed from releasing a lock the new owner of the same run now holds.
189
+ await this.pathLocks
190
+ .release(runId, repositoryKey, claimed.leaseToken)
191
+ .catch((error) => this.app.logger.warn(`[AgentOrchestrator] Path lock release failed.`, { error }));
192
+ }
193
+ await this.stateMachine.releaseLease(runId, claimed.leaseToken).catch(() => undefined);
194
+ }
195
+ }
196
+
197
+ private async heartbeat(runId: number, leaseToken: string) {
198
+ await this.stateMachine.renewLease(runId, leaseToken, LEASE_MS);
199
+ const repositoryKey = this.heldPathLocks.get(runId);
200
+ if (repositoryKey) await this.pathLocks.renew(runId, repositoryKey, PATH_LOCK_TTL_MS, leaseToken);
201
+ }
202
+
203
+ private async runToCompletion(snapshot: RunSnapshot, leaseToken: string, signal: AbortSignal) {
204
+ await this.control.assertCanExecute();
205
+ const scopeKey = `pattern:${snapshot.patternId}`;
206
+ await this.circuits.authorizeAttempt({
207
+ patternId: snapshot.patternId,
208
+ scopeKey,
209
+ policy: snapshot.policy.circuit,
210
+ });
211
+
212
+ if (!(await this.acquirePaths(snapshot, leaseToken))) return;
213
+
214
+ await this.stateMachine.transition({
215
+ runId: snapshot.id,
216
+ to: 'running',
217
+ actorType: 'worker',
218
+ actorIdentity: this.workerId,
219
+ leaseToken,
220
+ eventType: 'run_started',
221
+ title: 'Run started',
222
+ content: `Leader ${snapshot.leaderUsername} began working on the goal.`,
223
+ correlationKey: `started:${snapshot.id}`,
224
+ values: { currentRole: 'leader' },
225
+ });
226
+
227
+ const leader = await this.invokeRole(snapshot, {
228
+ role: 'leader',
229
+ username: snapshot.leaderUsername,
230
+ harness: snapshot.leaderHarness,
231
+ systemMessage:
232
+ 'You lead an autonomous run. Break the goal into concrete work for your makers, then finish with a concise plan of what must be built and which evidence will prove it. You do not modify anything yourself.',
233
+ prompt: snapshot.goal,
234
+ leaseToken,
235
+ signal,
236
+ });
237
+
238
+ // plugin-ai stops the graph on an approval-gated tool. Approval is a human decision, so the
239
+ // run parks here instead of the worker deciding on the operator's behalf.
240
+ if (leader.interrupted.length) {
241
+ await this.parkForApproval(snapshot, leader, leaseToken);
242
+ return;
243
+ }
244
+
245
+ // Each maker executes with its own compiled harness snapshot. Reusing one harness would apply
246
+ // the wrong per-employee tool grants, so the snapshot is looked up by username and a maker with
247
+ // no snapshot is a compile error, not a silent skip.
248
+ const makerReports: string[] = [leader.content];
249
+ for (const username of snapshot.makerUsernames) {
250
+ const harness = snapshot.makerHarnesses[username]?.effective;
251
+ if (!harness) {
252
+ throw new Error(`The claimed run has no harness snapshot for maker "${username}".`);
253
+ }
254
+ const maker = await this.invokeRole(snapshot, {
255
+ role: 'maker',
256
+ username,
257
+ harness,
258
+ systemMessage:
259
+ 'You are a maker on an autonomous run. Do the work assigned by the leader using only the tools you were granted, and report exactly what you changed and the evidence for it.',
260
+ prompt: [
261
+ `Goal: ${snapshot.goal}`,
262
+ '',
263
+ "Leader's plan:",
264
+ leader.content || '(the leader produced no textual plan)',
265
+ ].join('\n'),
266
+ leaseToken,
267
+ signal,
268
+ });
269
+ if (maker.interrupted.length) {
270
+ await this.parkForApproval(snapshot, maker, leaseToken);
271
+ return;
272
+ }
273
+ makerReports.push(`# ${username}\n${maker.content}`);
274
+ }
275
+ const makerSummary = makerReports.join('\n\n');
276
+
277
+ await this.stateMachine.transition({
278
+ runId: snapshot.id,
279
+ to: 'verifying',
280
+ actorType: 'worker',
281
+ actorIdentity: this.workerId,
282
+ leaseToken,
283
+ eventType: 'run_verifying',
284
+ title: 'Verification started',
285
+ content: `Verifier ${snapshot.verifierUsername} is reviewing the reported result.`,
286
+ correlationKey: `verifying:${snapshot.id}`,
287
+ values: { currentRole: 'verifier', finalAnswer: makerSummary },
288
+ });
289
+
290
+ await this.budgets.reserve({
291
+ runId: snapshot.id,
292
+ patternId: snapshot.patternId,
293
+ policy: snapshot.policy,
294
+ delta: { verifications: 1 },
295
+ });
296
+ const outcome = await this.verification.verifyAndFinalize({
297
+ runId: snapshot.id,
298
+ goal: snapshot.goal,
299
+ autonomyLevel: snapshot.autonomyLevel,
300
+ verifierUsername: snapshot.verifierUsername,
301
+ leaderUsername: snapshot.leaderUsername,
302
+ makerUsernames: snapshot.makerUsernames,
303
+ verifierHarness: snapshot.verifierHarness,
304
+ policy: snapshot.policy,
305
+ makerSummary,
306
+ userId: snapshot.userId,
307
+ workerId: this.workerId,
308
+ leaseToken,
309
+ signal,
310
+ });
311
+
312
+ if (outcome.finalStatus === 'failed') {
313
+ await this.circuits.recordFailure({
314
+ patternId: snapshot.patternId,
315
+ scopeKey: `pattern:${snapshot.patternId}`,
316
+ policy: snapshot.policy.circuit,
317
+ error: new Error(`Verifier rejected run ${snapshot.id}.`),
318
+ lastRunId: snapshot.id,
319
+ });
320
+ return;
321
+ }
322
+ await this.circuits.recordSuccess(snapshot.patternId, `pattern:${snapshot.patternId}`, snapshot.id);
323
+ }
324
+
325
+ private async acquirePaths(snapshot: RunSnapshot, leaseToken: string) {
326
+ if (!snapshot.repositoryKey || !snapshot.actingOn.length) return true;
327
+ const lock = await this.pathLocks.acquire({
328
+ runId: snapshot.id,
329
+ repositoryKey: snapshot.repositoryKey,
330
+ owner: leaseToken,
331
+ paths: snapshot.actingOn,
332
+ ttlMs: PATH_LOCK_TTL_MS,
333
+ });
334
+ if (lock.acquired) {
335
+ // Recorded before any work starts so the heartbeat renews this lock for as long as the run
336
+ // executes, and the release in `execute` frees exactly the lock this worker took.
337
+ this.heldPathLocks.set(snapshot.id, snapshot.repositoryKey);
338
+ return true;
339
+ }
340
+
341
+ const deadlocked = lock.deadlocked;
342
+ await this.stateMachine.transition({
343
+ runId: snapshot.id,
344
+ to: deadlocked ? 'blocked' : 'waiting_lock',
345
+ actorType: 'worker',
346
+ actorIdentity: this.workerId,
347
+ leaseToken,
348
+ eventType: deadlocked ? 'run_blocked' : 'run_waiting_lock',
349
+ title: deadlocked ? 'Run blocked by a lock cycle' : 'Waiting for repository paths',
350
+ content: `Blocked by run(s): ${lock.blockers.join(', ')}.`,
351
+ correlationKey: `lock:${snapshot.id}:${lock.blockers.join('-')}`,
352
+ values: deadlocked ? { blockedReason: 'Path lock wait graph contains a cycle.' } : {},
353
+ });
354
+ return false;
355
+ }
356
+
357
+ private async invokeRole(
358
+ snapshot: RunSnapshot,
359
+ input: {
360
+ role: 'leader' | 'maker';
361
+ username: string;
362
+ harness: CompiledHarness;
363
+ systemMessage: string;
364
+ prompt: string;
365
+ leaseToken: string;
366
+ signal: AbortSignal;
367
+ },
368
+ ) {
369
+ await this.budgets.reserve({
370
+ runId: snapshot.id,
371
+ patternId: snapshot.patternId,
372
+ policy: snapshot.policy,
373
+ delta: { invocations: 1 },
374
+ });
375
+ const sessionId = await this.runtime.createConversation({
376
+ username: input.username,
377
+ userId: snapshot.userId,
378
+ runId: snapshot.id,
379
+ title: snapshot.goal.slice(0, 200),
380
+ });
381
+ const step = await this.database.getRepository('agentLoopSteps').create({
382
+ values: {
383
+ runId: snapshot.id,
384
+ runtimeVersion: 'control-plane-v2',
385
+ sequence: await this.nextSequence(snapshot.id),
386
+ role: input.role,
387
+ kind: 'invocation',
388
+ type: 'reasoning',
389
+ employeeUsername: input.username,
390
+ title: `${input.role} invocation`,
391
+ inputHash: inputHash(input.prompt),
392
+ status: 'running',
393
+ startedAt: new Date(),
394
+ createdAt: new Date(),
395
+ },
396
+ });
397
+
398
+ const outcome = await this.runtime.invoke({
399
+ username: input.username,
400
+ sessionId,
401
+ userId: snapshot.userId,
402
+ systemMessage: input.systemMessage,
403
+ harness: input.harness,
404
+ prompt: input.prompt,
405
+ signal: input.signal,
406
+ });
407
+
408
+ await this.database.getRepository('agentLoopSteps').update({
409
+ filterByTk: read(step, 'id'),
410
+ values: {
411
+ status: outcome.interrupted.length ? 'waiting_user' : 'succeeded',
412
+ output: { content: outcome.content },
413
+ endedAt: new Date(),
414
+ },
415
+ });
416
+ await this.database.getRepository('agentLoopRuns').update({
417
+ filterByTk: snapshot.id,
418
+ values: { sessionId: outcome.sessionId, messageId: outcome.messageId },
419
+ });
420
+ return outcome;
421
+ }
422
+
423
+ private async parkForApproval(snapshot: RunSnapshot, outcome: InvocationOutcome, leaseToken: string) {
424
+ const now = new Date();
425
+ const expiresAt = new Date(now.getTime() + snapshot.policy.actions.approvalTimeoutMs);
426
+ const assignedToId = snapshot.policy.actions.approvalAssigneeIds[0] || snapshot.userId || null;
427
+ for (const call of outcome.interrupted) {
428
+ await this.database.getRepository('agentLoopActionApprovals').create({
429
+ values: {
430
+ runId: snapshot.id,
431
+ toolCallId: call.toolCallId,
432
+ toolName: call.toolName,
433
+ proposedInput: call.args,
434
+ inputHash: inputHash(call.args),
435
+ paths: snapshot.actingOn,
436
+ policyDecision: { decision: 'ask', interruptId: call.interruptId },
437
+ reason: `Tool "${call.toolName}" requires human approval under this pattern's policy.`,
438
+ status: 'pending',
439
+ assignedToId,
440
+ requestedById: snapshot.userId || null,
441
+ requestedAt: now,
442
+ expiresAt,
443
+ createdAt: now,
444
+ },
445
+ });
446
+ }
447
+ await this.stateMachine.transition({
448
+ runId: snapshot.id,
449
+ to: 'waiting_approval',
450
+ actorType: 'worker',
451
+ actorIdentity: this.workerId,
452
+ leaseToken,
453
+ eventType: 'run_waiting_approval',
454
+ title: 'Waiting for action approval',
455
+ content: `${outcome.interrupted.length} tool call(s) require approval.`,
456
+ correlationKey: `approval:${snapshot.id}:${outcome.messageId}`,
457
+ values: { approvalStatus: 'pending', messageId: outcome.messageId },
458
+ });
459
+ }
460
+
461
+ private async failRun(runId: number, leaseToken: string, snapshot: RunSnapshot | null, error: unknown) {
462
+ const message = error instanceof Error ? error.message : String(error);
463
+ try {
464
+ if (snapshot) {
465
+ await this.circuits.recordFailure({
466
+ patternId: snapshot.patternId,
467
+ scopeKey: `pattern:${snapshot.patternId}`,
468
+ policy: snapshot.policy.circuit,
469
+ error,
470
+ lastRunId: runId,
471
+ });
472
+ }
473
+ await this.stateMachine.transition({
474
+ runId,
475
+ to: 'failed',
476
+ actorType: 'worker',
477
+ actorIdentity: this.workerId,
478
+ leaseToken,
479
+ eventType: 'run_failed',
480
+ title: 'Run failed',
481
+ content: message,
482
+ correlationKey: `failed:${runId}`,
483
+ values: { summary: message },
484
+ });
485
+ } catch (transitionError) {
486
+ this.app.logger.error(`[AgentOrchestrator] Loop run ${runId} could not be marked failed.`, {
487
+ error: transitionError,
488
+ cause: message,
489
+ });
490
+ }
491
+ }
492
+
493
+ private async nextSequence(runId: number) {
494
+ const steps = await this.database.getRepository('agentLoopSteps').find({ filter: { runId } });
495
+ return steps.length;
496
+ }
497
+
498
+ private snapshotFrom(run: Record<string, unknown>): RunSnapshot {
499
+ const patternId = Number(run.patternId);
500
+ if (!Number.isSafeInteger(patternId) || patternId <= 0) {
501
+ throw new Error('The claimed run has no pattern snapshot.');
502
+ }
503
+ const roleBindings = asObject(run.roleBindingsSnapshot);
504
+ const makerSnapshots = asObject(run.makerHarnessSnapshot);
505
+ return {
506
+ id: Number(run.id),
507
+ patternId,
508
+ goal: String(run.goal || ''),
509
+ autonomyLevel: autonomyLevel(run.autonomyLevel),
510
+ policy: loopPatternPolicySchema.parse(run.policySnapshot),
511
+ leaderUsername: String(roleBindings.leader || run.leaderUsername || ''),
512
+ makerUsernames: asStringArray(roleBindings.makers),
513
+ verifierUsername: String(roleBindings.verifier || run.verifierUsername || ''),
514
+ leaderHarness: harnessFrom(run.leaderHarnessSnapshot),
515
+ makerHarnesses: makerSnapshots as Record<string, { effective: CompiledHarness }>,
516
+ verifierHarness: harnessFrom(run.verifierHarnessSnapshot),
517
+ repositoryKey: String(run.repositoryKey || ''),
518
+ actingOn: asStringArray(run.actingOn),
519
+ userId: Number(run.userId) || undefined,
520
+ };
521
+ }
522
+ }