plugin-agent-orchestrator 1.1.4 → 1.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (148) hide show
  1. package/README.md +204 -114
  2. package/dist/client/index.js +1 -1
  3. package/dist/client-v2/117.7565cff57c39318a.js +10 -0
  4. package/dist/client-v2/214.106635011859dede.js +10 -0
  5. package/dist/client-v2/418.41ed200e7d6da332.js +10 -0
  6. package/dist/client-v2/53.349fff8db80b19d4.js +10 -0
  7. package/dist/client-v2/981.e4a1f41cfc94a856.js +10 -0
  8. package/dist/client-v2/index.js +1 -1
  9. package/dist/externalVersion.js +7 -8
  10. package/dist/locale/en-US.json +226 -170
  11. package/dist/locale/vi-VN.json +226 -170
  12. package/dist/locale/zh-CN.json +226 -170
  13. package/dist/server/collections/agent-execution-spans.js +8 -0
  14. package/dist/server/collections/agent-loop-action-approvals.js +1 -0
  15. package/dist/server/collections/agent-loop-runs.js +7 -1
  16. package/dist/server/collections/agent-loop-steps.js +7 -0
  17. package/dist/server/collections/skill-executions.js +8 -0
  18. package/dist/server/plugin.js +3 -1
  19. package/dist/server/resources/agent-harness-profile.js +117 -0
  20. package/dist/server/resources/agent-knowledge-insights.js +1 -3
  21. package/dist/server/resources/agent-loop-approval.js +25 -0
  22. package/dist/server/resources/agent-loop.js +3 -1
  23. package/dist/server/services/AgentMemoryContextService.js +34 -0
  24. package/dist/server/services/HarnessCompiler.js +35 -2
  25. package/dist/server/services/HarnessProfileService.js +74 -9
  26. package/dist/server/services/HarnessSchema.js +30 -3
  27. package/dist/server/services/LoopPatternSchema.js +16 -1
  28. package/dist/server/services/LoopRunRepository.js +80 -0
  29. package/dist/server/services/LoopRunStateMachine.js +92 -3
  30. package/dist/server/services/LoopTriggerService.js +37 -0
  31. package/dist/server/services/LoopWorkerService.js +261 -23
  32. package/dist/server/services/NativeSubAgentObserver.js +165 -23
  33. package/dist/server/services/PluginAiRuntimeAdapter.js +16 -5
  34. package/dist/server/services/ToolLoopDetectionService.js +210 -0
  35. package/dist/server/services/VerificationService.js +66 -0
  36. package/dist/server/services/harness-runtime-policy.js +139 -0
  37. package/dist/server/skill-hub/plugin.js +36 -3
  38. package/dist/server/skill-hub/tasks/SkillExecutionTask.js +29 -0
  39. package/package.json +53 -53
  40. package/src/client/ApprovalsTab.tsx +2 -0
  41. package/src/client/OrchestratorSettings.tsx +11 -0
  42. package/src/client/skill-hub/components/GitSkillImport.tsx +3 -3
  43. package/src/client/skill-hub/components/SkillMetrics.tsx +15 -10
  44. package/src/client/skill-hub/index.tsx +0 -52
  45. package/src/client-v2/components/AgentRunsTab.tsx +576 -576
  46. package/src/client-v2/components/ApprovalsTab.tsx +337 -0
  47. package/src/client-v2/components/HarnessProfilesTab.tsx +385 -105
  48. package/src/client-v2/components/MemoryInspectorTab.tsx +141 -141
  49. package/src/client-v2/components/TracingTab.tsx +515 -515
  50. package/src/client-v2/hooks/useApiRequest.ts +141 -141
  51. package/src/client-v2/pages/ApprovalsPage.tsx +10 -0
  52. package/src/client-v2/plugin.tsx +6 -6
  53. package/src/client-v2/skill-hub/components/SkillMetrics.tsx +5 -2
  54. package/src/locale/en-US.json +226 -174
  55. package/src/locale/vi-VN.json +226 -174
  56. package/src/locale/zh-CN.json +226 -174
  57. package/src/server/__tests__/circuit-breaker.test.ts +248 -248
  58. package/src/server/__tests__/harness-compiler.test.ts +83 -0
  59. package/src/server/__tests__/harness-profile-service.test.ts +51 -0
  60. package/src/server/__tests__/harness-runtime-policy.test.ts +293 -0
  61. package/src/server/__tests__/harness-tool-plan.test.ts +24 -0
  62. package/src/server/__tests__/loop-approval-decision.test.ts +512 -0
  63. package/src/server/__tests__/loop-approval-expiry.test.ts +258 -0
  64. package/src/server/__tests__/loop-budget-service.test.ts +255 -255
  65. package/src/server/__tests__/loop-control-plane-backfill.test.ts +266 -266
  66. package/src/server/__tests__/loop-pattern-schema.test.ts +18 -1
  67. package/src/server/__tests__/loop-pattern-service.test.ts +148 -148
  68. package/src/server/__tests__/loop-run-state-machine.test.ts +11 -0
  69. package/src/server/__tests__/loop-trigger-service.test.ts +148 -0
  70. package/src/server/__tests__/loop-worker-claim.test.ts +341 -341
  71. package/src/server/__tests__/loop-worker-execution.test.ts +82 -1
  72. package/src/server/__tests__/native-sub-agent-observer.test.ts +288 -4
  73. package/src/server/__tests__/path-lock-service.test.ts +254 -254
  74. package/src/server/__tests__/skill-execution-task-policy.test.ts +140 -0
  75. package/src/server/__tests__/smoke.test.ts +162 -162
  76. package/src/server/__tests__/tool-loop-detection.test.ts +275 -0
  77. package/src/server/collections/agent-execution-spans.ts +8 -0
  78. package/src/server/collections/agent-harness-profile-versions.ts +26 -26
  79. package/src/server/collections/agent-harness-profiles.ts +71 -71
  80. package/src/server/collections/agent-loop-action-approvals.ts +1 -0
  81. package/src/server/collections/agent-loop-artifacts.ts +25 -25
  82. package/src/server/collections/agent-loop-circuit-states.ts +23 -23
  83. package/src/server/collections/agent-loop-control-settings.ts +21 -21
  84. package/src/server/collections/agent-loop-events.ts +98 -98
  85. package/src/server/collections/agent-loop-path-locks.ts +27 -27
  86. package/src/server/collections/agent-loop-patterns.ts +42 -42
  87. package/src/server/collections/agent-loop-runs.ts +7 -1
  88. package/src/server/collections/agent-loop-steps.ts +7 -0
  89. package/src/server/collections/agent-loop-usage-buckets.ts +22 -22
  90. package/src/server/collections/agent-loop-worktrees.ts +25 -25
  91. package/src/server/collections/skill-executions.ts +8 -0
  92. package/src/server/migrations/20260427000002-change-packages-to-text.ts +52 -52
  93. package/src/server/migrations/20260429000000-add-llm-fields.ts +47 -47
  94. package/src/server/migrations/20260429000001-fix-inputargs-json-to-text.ts +45 -45
  95. package/src/server/migrations/20260503000000-add-orchestrator-trace-fields.ts +33 -33
  96. package/src/server/migrations/20260524000000-add-agent-loop-fields-to-skill-executions.ts +31 -31
  97. package/src/server/migrations/20260803010000-backfill-loop-control-plane-data.ts +293 -293
  98. package/src/server/plugin.ts +3 -1
  99. package/src/server/resources/agent-harness-profile.ts +94 -0
  100. package/src/server/resources/agent-knowledge-insights.ts +5 -3
  101. package/src/server/resources/agent-loop-approval.ts +32 -0
  102. package/src/server/resources/agent-loop-artifact.ts +56 -56
  103. package/src/server/resources/agent-loop-events-stream.ts +105 -105
  104. package/src/server/resources/agent-loop.ts +3 -1
  105. package/src/server/resources/resource-helpers.ts +33 -33
  106. package/src/server/resources/tracing.ts +266 -266
  107. package/src/server/services/AgentMemoryContextService.ts +44 -0
  108. package/src/server/services/DurableCircuitBreakerService.ts +239 -239
  109. package/src/server/services/ExecutionSpanService.ts +2 -0
  110. package/src/server/services/HarnessCompiler.ts +37 -0
  111. package/src/server/services/HarnessProfileService.ts +92 -10
  112. package/src/server/services/HarnessSchema.ts +31 -0
  113. package/src/server/services/LoopBudgetService.ts +275 -275
  114. package/src/server/services/LoopControlService.ts +124 -124
  115. package/src/server/services/LoopPatternSchema.ts +19 -1
  116. package/src/server/services/LoopPatternService.ts +143 -143
  117. package/src/server/services/LoopRunRepository.ts +99 -0
  118. package/src/server/services/LoopRunStateMachine.ts +100 -3
  119. package/src/server/services/LoopSchedulerService.ts +199 -199
  120. package/src/server/services/LoopTriggerService.ts +42 -0
  121. package/src/server/services/LoopWorkerService.ts +327 -27
  122. package/src/server/services/NativeSubAgentObserver.ts +186 -21
  123. package/src/server/services/PathLockService.ts +321 -321
  124. package/src/server/services/PluginAiRuntimeAdapter.ts +24 -7
  125. package/src/server/services/ToolLoopDetectionService.ts +213 -0
  126. package/src/server/services/VerificationSchema.ts +31 -31
  127. package/src/server/services/VerificationService.ts +81 -2
  128. package/src/server/services/harness-runtime-policy.ts +141 -0
  129. package/src/server/skill-hub/plugin.ts +41 -3
  130. package/src/server/skill-hub/services/RegistrySkillSnapshotService.ts +235 -235
  131. package/src/server/skill-hub/services/__tests__/RegistrySkillSnapshotService.test.ts +61 -61
  132. package/src/server/skill-hub/tasks/SkillExecutionTask.ts +43 -2
  133. package/src/server/utils/ensure-indexes.ts +96 -96
  134. package/src/server/utils/logging.ts +62 -62
  135. package/src/server/utils/loop-pattern-context.ts +20 -20
  136. package/dist/client-v2/214.b063f33ba044a808.js +0 -10
  137. package/dist/client-v2/373.f5d5292e53c4e832.js +0 -10
  138. package/dist/client-v2/418.58fa350f5496efd3.js +0 -10
  139. package/dist/client-v2/53.9a10455323390f14.js +0 -10
  140. package/dist/client-v2/677.a991ce0250ff5c77.js +0 -10
  141. package/dist/client-v2/925.e5925bf21ed68e7a.js +0 -10
  142. package/src/client/AIEmployeeSelect.tsx +0 -47
  143. package/src/client/AIEmployeesContext.tsx +0 -102
  144. package/src/client/RulesTab.tsx +0 -2
  145. package/src/client/TracingTab.tsx +0 -436
  146. package/src/client/skill-hub/components/ExecutionProgress.tsx +0 -51
  147. package/src/client-v2/components/RulesTab.tsx +0 -2
  148. package/src/client-v2/pages/AgentRunsPage.tsx +0 -13
@@ -1,239 +1,239 @@
1
- import { createHash } from 'node:crypto';
2
- import type { Database, Model } from '@nocobase/database';
3
- import type { LoopPatternPolicy } from './LoopPatternSchema';
4
-
5
- type DistributedLock = {
6
- runExclusive<T>(key: string, operation: () => Promise<T>, ttl?: number): Promise<T>;
7
- };
8
-
9
- type CircuitSnapshot = {
10
- id: number;
11
- state: 'closed' | 'open' | 'half_open';
12
- attempts: number;
13
- consecutiveFailures: number;
14
- errorSignature: string;
15
- repeatedErrorCount: number;
16
- retryAt: Date | null;
17
- };
18
-
19
- function read(record: Model | Record<string, unknown>, key: string) {
20
- const model = record as Model & { get?: (name: string) => unknown };
21
- return typeof model.get === 'function' ? model.get(key) : (record as Record<string, unknown>)[key];
22
- }
23
-
24
- function count(value: unknown, label: string) {
25
- if (value === null || value === undefined) return 0;
26
- const parsed = Number(value);
27
- if (!Number.isSafeInteger(parsed) || parsed < 0) throw new Error(`${label} counter is invalid.`);
28
- return parsed;
29
- }
30
-
31
- function dateOrNull(value: unknown) {
32
- if (!value) return null;
33
- const date = value instanceof Date ? value : new Date(String(value));
34
- return Number.isNaN(date.getTime()) ? null : date;
35
- }
36
-
37
- function snapshot(record: Model | Record<string, unknown>): CircuitSnapshot {
38
- const id = Number(read(record, 'id'));
39
- const state = read(record, 'state');
40
- if (!Number.isSafeInteger(id) || id <= 0) throw new Error('Circuit breaker state has an invalid identifier.');
41
- if (state !== 'closed' && state !== 'open' && state !== 'half_open') {
42
- throw new Error('Circuit breaker state is invalid.');
43
- }
44
- return {
45
- id,
46
- state,
47
- attempts: count(read(record, 'attempts'), 'Circuit attempt'),
48
- consecutiveFailures: count(read(record, 'consecutiveFailures'), 'Circuit consecutive failure'),
49
- errorSignature: typeof read(record, 'errorSignature') === 'string' ? String(read(record, 'errorSignature')) : '',
50
- repeatedErrorCount: count(read(record, 'repeatedErrorCount'), 'Circuit repeated error'),
51
- retryAt: dateOrNull(read(record, 'retryAt')),
52
- };
53
- }
54
-
55
- function normalizeScopeKey(scopeKey: string) {
56
- const normalized = scopeKey.trim();
57
- if (!normalized) throw new Error('Circuit breaker scope key is required.');
58
- if (normalized.length > 300) throw new Error('Circuit breaker scope key cannot exceed 300 characters.');
59
- return normalized;
60
- }
61
-
62
- export function normalizeErrorSignature(error: unknown) {
63
- const raw = error instanceof Error ? `${error.name}:${error.message}` : String(error);
64
- const normalized = raw
65
- .toLowerCase()
66
- .replace(/[a-z]:[\\/][^\s"']+/gi, '<path>')
67
- .replace(/\/(?:[^\s/]+\/)+[^\s"']*/g, '<path>')
68
- .replace(/\b[0-9a-f]{8}-[0-9a-f-]{27,}\b/gi, '<uuid>')
69
- .replace(/\b\d+\b/g, '<n>')
70
- .replace(/\s+/g, ' ')
71
- .trim();
72
- return createHash('sha256')
73
- .update(normalized || 'unknown-error')
74
- .digest('hex');
75
- }
76
-
77
- export class DurableCircuitBreakerService {
78
- constructor(
79
- private readonly database: Database,
80
- private readonly distributedLock: DistributedLock,
81
- ) {}
82
-
83
- async authorizeAttempt(input: {
84
- patternId: number;
85
- scopeKey: string;
86
- policy: LoopPatternPolicy['circuit'];
87
- now?: Date;
88
- }) {
89
- const scopeKey = normalizeScopeKey(input.scopeKey);
90
- const now = input.now || new Date();
91
- return this.withScopeLock(input.patternId, scopeKey, async () => {
92
- const decision = await this.database.sequelize.transaction(async (transaction) => {
93
- const repository = this.database.getRepository('agentLoopCircuitStates');
94
- let record = await repository.findOne({
95
- filter: { patternId: input.patternId, scopeKey },
96
- transaction,
97
- lock: transaction.LOCK.UPDATE,
98
- });
99
- if (!record) {
100
- record = await repository.create({
101
- values: {
102
- patternId: input.patternId,
103
- scopeKey,
104
- state: 'closed',
105
- attempts: 0,
106
- consecutiveFailures: 0,
107
- repeatedErrorCount: 0,
108
- },
109
- transaction,
110
- });
111
- }
112
- const current = snapshot(record);
113
- if (current.state === 'half_open') {
114
- return { allowed: false as const, reason: 'probe' as const, retryAt: current.retryAt };
115
- }
116
- if (current.state === 'open' && (!current.retryAt || current.retryAt.getTime() > now.getTime())) {
117
- return { allowed: false as const, reason: 'open' as const, retryAt: current.retryAt };
118
- }
119
- if (current.state === 'closed' && current.attempts >= input.policy.maxAttempts) {
120
- const retryAt = new Date(now.getTime() + input.policy.cooldownMs);
121
- await repository.update({
122
- filterByTk: current.id,
123
- values: { state: 'open', openedAt: now, retryAt },
124
- transaction,
125
- });
126
- return { allowed: false as const, reason: 'attempts' as const, retryAt };
127
- }
128
- const nextState = current.state === 'open' ? 'half_open' : 'closed';
129
- await repository.update({
130
- filterByTk: current.id,
131
- values: { state: nextState, attempts: current.attempts + 1 },
132
- transaction,
133
- });
134
- return { allowed: true as const, state: nextState, attempt: current.attempts + 1 };
135
- });
136
- if (decision.allowed) return decision;
137
- if (decision.reason === 'probe') throw new Error('Circuit breaker probe is already in progress.');
138
- if (decision.reason === 'attempts') throw new Error('Circuit breaker maximum attempts reached.');
139
- throw new Error(`Circuit breaker is open until ${decision.retryAt?.toISOString() || 'manual reset'}.`);
140
- });
141
- }
142
-
143
- async recordSuccess(patternId: number, scopeKey: string, lastRunId?: number) {
144
- const normalizedScope = normalizeScopeKey(scopeKey);
145
- return this.withScopeLock(patternId, normalizedScope, async () =>
146
- this.database.sequelize.transaction(async (transaction) => {
147
- const repository = this.database.getRepository('agentLoopCircuitStates');
148
- const record = await repository.findOne({
149
- filter: { patternId, scopeKey: normalizedScope },
150
- transaction,
151
- lock: transaction.LOCK.UPDATE,
152
- });
153
- if (!record) return;
154
- await repository.update({
155
- filterByTk: read(record, 'id'),
156
- values: {
157
- state: 'closed',
158
- attempts: 0,
159
- consecutiveFailures: 0,
160
- errorSignature: null,
161
- repeatedErrorCount: 0,
162
- lastRunId: lastRunId || null,
163
- openedAt: null,
164
- retryAt: null,
165
- },
166
- transaction,
167
- });
168
- }),
169
- );
170
- }
171
-
172
- async recordFailure(input: {
173
- patternId: number;
174
- scopeKey: string;
175
- policy: LoopPatternPolicy['circuit'];
176
- error: unknown;
177
- lastRunId?: number;
178
- now?: Date;
179
- }) {
180
- const scopeKey = normalizeScopeKey(input.scopeKey);
181
- const signature = normalizeErrorSignature(input.error);
182
- const now = input.now || new Date();
183
- return this.withScopeLock(input.patternId, scopeKey, async () =>
184
- this.database.sequelize.transaction(async (transaction) => {
185
- const repository = this.database.getRepository('agentLoopCircuitStates');
186
- let record = await repository.findOne({
187
- filter: { patternId: input.patternId, scopeKey },
188
- transaction,
189
- lock: transaction.LOCK.UPDATE,
190
- });
191
- if (!record) {
192
- record = await repository.create({
193
- values: {
194
- patternId: input.patternId,
195
- scopeKey,
196
- state: 'closed',
197
- attempts: 1,
198
- consecutiveFailures: 0,
199
- repeatedErrorCount: 0,
200
- },
201
- transaction,
202
- });
203
- }
204
- const current = snapshot(record);
205
- const consecutiveFailures = current.consecutiveFailures + 1;
206
- const repeatedErrorCount = current.errorSignature === signature ? current.repeatedErrorCount + 1 : 1;
207
- const shouldOpen =
208
- current.state === 'half_open' ||
209
- current.attempts >= input.policy.maxAttempts ||
210
- consecutiveFailures >= input.policy.maxConsecutiveFailures ||
211
- repeatedErrorCount >= input.policy.maxRepeatedError;
212
- await repository.update({
213
- filterByTk: current.id,
214
- values: {
215
- state: shouldOpen ? 'open' : 'closed',
216
- consecutiveFailures,
217
- errorSignature: signature,
218
- repeatedErrorCount,
219
- lastRunId: input.lastRunId || null,
220
- openedAt: shouldOpen ? now : null,
221
- retryAt: shouldOpen ? new Date(now.getTime() + input.policy.cooldownMs) : null,
222
- },
223
- transaction,
224
- });
225
- return { open: shouldOpen, signature, consecutiveFailures, repeatedErrorCount };
226
- }),
227
- );
228
- }
229
-
230
- private withScopeLock<T>(patternId: number, scopeKey: string, operation: () => Promise<T>) {
231
- const normalizedScope = scopeKey.trim();
232
- if (!normalizedScope) throw new Error('Circuit breaker scope key is required.');
233
- return this.distributedLock.runExclusive(
234
- `agent-loop:circuit:${patternId}:${createHash('sha256').update(normalizedScope).digest('hex')}`,
235
- operation,
236
- 30_000,
237
- );
238
- }
239
- }
1
+ import { createHash } from 'node:crypto';
2
+ import type { Database, Model } from '@nocobase/database';
3
+ import type { LoopPatternPolicy } from './LoopPatternSchema';
4
+
5
+ type DistributedLock = {
6
+ runExclusive<T>(key: string, operation: () => Promise<T>, ttl?: number): Promise<T>;
7
+ };
8
+
9
+ type CircuitSnapshot = {
10
+ id: number;
11
+ state: 'closed' | 'open' | 'half_open';
12
+ attempts: number;
13
+ consecutiveFailures: number;
14
+ errorSignature: string;
15
+ repeatedErrorCount: number;
16
+ retryAt: Date | null;
17
+ };
18
+
19
+ function read(record: Model | Record<string, unknown>, key: string) {
20
+ const model = record as Model & { get?: (name: string) => unknown };
21
+ return typeof model.get === 'function' ? model.get(key) : (record as Record<string, unknown>)[key];
22
+ }
23
+
24
+ function count(value: unknown, label: string) {
25
+ if (value === null || value === undefined) return 0;
26
+ const parsed = Number(value);
27
+ if (!Number.isSafeInteger(parsed) || parsed < 0) throw new Error(`${label} counter is invalid.`);
28
+ return parsed;
29
+ }
30
+
31
+ function dateOrNull(value: unknown) {
32
+ if (!value) return null;
33
+ const date = value instanceof Date ? value : new Date(String(value));
34
+ return Number.isNaN(date.getTime()) ? null : date;
35
+ }
36
+
37
+ function snapshot(record: Model | Record<string, unknown>): CircuitSnapshot {
38
+ const id = Number(read(record, 'id'));
39
+ const state = read(record, 'state');
40
+ if (!Number.isSafeInteger(id) || id <= 0) throw new Error('Circuit breaker state has an invalid identifier.');
41
+ if (state !== 'closed' && state !== 'open' && state !== 'half_open') {
42
+ throw new Error('Circuit breaker state is invalid.');
43
+ }
44
+ return {
45
+ id,
46
+ state,
47
+ attempts: count(read(record, 'attempts'), 'Circuit attempt'),
48
+ consecutiveFailures: count(read(record, 'consecutiveFailures'), 'Circuit consecutive failure'),
49
+ errorSignature: typeof read(record, 'errorSignature') === 'string' ? String(read(record, 'errorSignature')) : '',
50
+ repeatedErrorCount: count(read(record, 'repeatedErrorCount'), 'Circuit repeated error'),
51
+ retryAt: dateOrNull(read(record, 'retryAt')),
52
+ };
53
+ }
54
+
55
+ function normalizeScopeKey(scopeKey: string) {
56
+ const normalized = scopeKey.trim();
57
+ if (!normalized) throw new Error('Circuit breaker scope key is required.');
58
+ if (normalized.length > 300) throw new Error('Circuit breaker scope key cannot exceed 300 characters.');
59
+ return normalized;
60
+ }
61
+
62
+ export function normalizeErrorSignature(error: unknown) {
63
+ const raw = error instanceof Error ? `${error.name}:${error.message}` : String(error);
64
+ const normalized = raw
65
+ .toLowerCase()
66
+ .replace(/[a-z]:[\\/][^\s"']+/gi, '<path>')
67
+ .replace(/\/(?:[^\s/]+\/)+[^\s"']*/g, '<path>')
68
+ .replace(/\b[0-9a-f]{8}-[0-9a-f-]{27,}\b/gi, '<uuid>')
69
+ .replace(/\b\d+\b/g, '<n>')
70
+ .replace(/\s+/g, ' ')
71
+ .trim();
72
+ return createHash('sha256')
73
+ .update(normalized || 'unknown-error')
74
+ .digest('hex');
75
+ }
76
+
77
+ export class DurableCircuitBreakerService {
78
+ constructor(
79
+ private readonly database: Database,
80
+ private readonly distributedLock: DistributedLock,
81
+ ) {}
82
+
83
+ async authorizeAttempt(input: {
84
+ patternId: number;
85
+ scopeKey: string;
86
+ policy: LoopPatternPolicy['circuit'];
87
+ now?: Date;
88
+ }) {
89
+ const scopeKey = normalizeScopeKey(input.scopeKey);
90
+ const now = input.now || new Date();
91
+ return this.withScopeLock(input.patternId, scopeKey, async () => {
92
+ const decision = await this.database.sequelize.transaction(async (transaction) => {
93
+ const repository = this.database.getRepository('agentLoopCircuitStates');
94
+ let record = await repository.findOne({
95
+ filter: { patternId: input.patternId, scopeKey },
96
+ transaction,
97
+ lock: transaction.LOCK.UPDATE,
98
+ });
99
+ if (!record) {
100
+ record = await repository.create({
101
+ values: {
102
+ patternId: input.patternId,
103
+ scopeKey,
104
+ state: 'closed',
105
+ attempts: 0,
106
+ consecutiveFailures: 0,
107
+ repeatedErrorCount: 0,
108
+ },
109
+ transaction,
110
+ });
111
+ }
112
+ const current = snapshot(record);
113
+ if (current.state === 'half_open') {
114
+ return { allowed: false as const, reason: 'probe' as const, retryAt: current.retryAt };
115
+ }
116
+ if (current.state === 'open' && (!current.retryAt || current.retryAt.getTime() > now.getTime())) {
117
+ return { allowed: false as const, reason: 'open' as const, retryAt: current.retryAt };
118
+ }
119
+ if (current.state === 'closed' && current.attempts >= input.policy.maxAttempts) {
120
+ const retryAt = new Date(now.getTime() + input.policy.cooldownMs);
121
+ await repository.update({
122
+ filterByTk: current.id,
123
+ values: { state: 'open', openedAt: now, retryAt },
124
+ transaction,
125
+ });
126
+ return { allowed: false as const, reason: 'attempts' as const, retryAt };
127
+ }
128
+ const nextState = current.state === 'open' ? 'half_open' : 'closed';
129
+ await repository.update({
130
+ filterByTk: current.id,
131
+ values: { state: nextState, attempts: current.attempts + 1 },
132
+ transaction,
133
+ });
134
+ return { allowed: true as const, state: nextState, attempt: current.attempts + 1 };
135
+ });
136
+ if (decision.allowed) return decision;
137
+ if (decision.reason === 'probe') throw new Error('Circuit breaker probe is already in progress.');
138
+ if (decision.reason === 'attempts') throw new Error('Circuit breaker maximum attempts reached.');
139
+ throw new Error(`Circuit breaker is open until ${decision.retryAt?.toISOString() || 'manual reset'}.`);
140
+ });
141
+ }
142
+
143
+ async recordSuccess(patternId: number, scopeKey: string, lastRunId?: number) {
144
+ const normalizedScope = normalizeScopeKey(scopeKey);
145
+ return this.withScopeLock(patternId, normalizedScope, async () =>
146
+ this.database.sequelize.transaction(async (transaction) => {
147
+ const repository = this.database.getRepository('agentLoopCircuitStates');
148
+ const record = await repository.findOne({
149
+ filter: { patternId, scopeKey: normalizedScope },
150
+ transaction,
151
+ lock: transaction.LOCK.UPDATE,
152
+ });
153
+ if (!record) return;
154
+ await repository.update({
155
+ filterByTk: read(record, 'id'),
156
+ values: {
157
+ state: 'closed',
158
+ attempts: 0,
159
+ consecutiveFailures: 0,
160
+ errorSignature: null,
161
+ repeatedErrorCount: 0,
162
+ lastRunId: lastRunId || null,
163
+ openedAt: null,
164
+ retryAt: null,
165
+ },
166
+ transaction,
167
+ });
168
+ }),
169
+ );
170
+ }
171
+
172
+ async recordFailure(input: {
173
+ patternId: number;
174
+ scopeKey: string;
175
+ policy: LoopPatternPolicy['circuit'];
176
+ error: unknown;
177
+ lastRunId?: number;
178
+ now?: Date;
179
+ }) {
180
+ const scopeKey = normalizeScopeKey(input.scopeKey);
181
+ const signature = normalizeErrorSignature(input.error);
182
+ const now = input.now || new Date();
183
+ return this.withScopeLock(input.patternId, scopeKey, async () =>
184
+ this.database.sequelize.transaction(async (transaction) => {
185
+ const repository = this.database.getRepository('agentLoopCircuitStates');
186
+ let record = await repository.findOne({
187
+ filter: { patternId: input.patternId, scopeKey },
188
+ transaction,
189
+ lock: transaction.LOCK.UPDATE,
190
+ });
191
+ if (!record) {
192
+ record = await repository.create({
193
+ values: {
194
+ patternId: input.patternId,
195
+ scopeKey,
196
+ state: 'closed',
197
+ attempts: 1,
198
+ consecutiveFailures: 0,
199
+ repeatedErrorCount: 0,
200
+ },
201
+ transaction,
202
+ });
203
+ }
204
+ const current = snapshot(record);
205
+ const consecutiveFailures = current.consecutiveFailures + 1;
206
+ const repeatedErrorCount = current.errorSignature === signature ? current.repeatedErrorCount + 1 : 1;
207
+ const shouldOpen =
208
+ current.state === 'half_open' ||
209
+ current.attempts >= input.policy.maxAttempts ||
210
+ consecutiveFailures >= input.policy.maxConsecutiveFailures ||
211
+ repeatedErrorCount >= input.policy.maxRepeatedError;
212
+ await repository.update({
213
+ filterByTk: current.id,
214
+ values: {
215
+ state: shouldOpen ? 'open' : 'closed',
216
+ consecutiveFailures,
217
+ errorSignature: signature,
218
+ repeatedErrorCount,
219
+ lastRunId: input.lastRunId || null,
220
+ openedAt: shouldOpen ? now : null,
221
+ retryAt: shouldOpen ? new Date(now.getTime() + input.policy.cooldownMs) : null,
222
+ },
223
+ transaction,
224
+ });
225
+ return { open: shouldOpen, signature, consecutiveFailures, repeatedErrorCount };
226
+ }),
227
+ );
228
+ }
229
+
230
+ private withScopeLock<T>(patternId: number, scopeKey: string, operation: () => Promise<T>) {
231
+ const normalizedScope = scopeKey.trim();
232
+ if (!normalizedScope) throw new Error('Circuit breaker scope key is required.');
233
+ return this.distributedLock.runExclusive(
234
+ `agent-loop:circuit:${patternId}:${createHash('sha256').update(normalizedScope).digest('hex')}`,
235
+ operation,
236
+ 30_000,
237
+ );
238
+ }
239
+ }
@@ -19,6 +19,8 @@ export type OrchestratorTraceContext = {
19
19
  type SpanValues = {
20
20
  rootRunId: string;
21
21
  parentSpanId?: string;
22
+ depth?: number;
23
+ agentLoopRunId?: string | number;
22
24
  source?: string;
23
25
  parentSessionId?: string;
24
26
  subSessionId?: string;
@@ -54,6 +54,30 @@ function strictestNetwork(values: HarnessSettings['isolation']['networkAccess'][
54
54
  );
55
55
  }
56
56
 
57
+ const sharingRank: Record<HarnessSettings['observability']['sharing'], number> = {
58
+ full: 0,
59
+ 'feedback-only': 1,
60
+ disabled: 2,
61
+ };
62
+
63
+ function strictestSharing(values: HarnessSettings['observability']['sharing'][]) {
64
+ return values.reduce<HarnessSettings['observability']['sharing']>(
65
+ (strictest, value) => (sharingRank[value] > sharingRank[strictest] ? value : strictest),
66
+ 'full',
67
+ );
68
+ }
69
+
70
+ // Per-tool caps compile like every other limit: the smallest value across layers wins.
71
+ function mergeTimeouts(settings: HarnessSettings[]) {
72
+ const timeouts: Record<string, number> = {};
73
+ for (const layer of settings) {
74
+ for (const [name, timeoutMs] of Object.entries(layer.tools.timeouts)) {
75
+ timeouts[name] = timeouts[name] === undefined ? timeoutMs : Math.min(timeouts[name], timeoutMs);
76
+ }
77
+ }
78
+ return timeouts;
79
+ }
80
+
57
81
  export function compileHarness(layers: HarnessLayer[]): CompiledHarness {
58
82
  if (!layers.length) {
59
83
  throw new Error('At least one harness layer is required.');
@@ -78,8 +102,12 @@ export function compileHarness(layers: HarnessLayer[]): CompiledHarness {
78
102
  allow: intersectRestrictions(settings.map((layer) => layer.tools.allow)),
79
103
  ask: union(settings.map((layer) => layer.tools.ask)),
80
104
  deny: union(settings.map((layer) => layer.tools.deny)),
105
+ // Escalatable tools widen authority per approved call, so a child may only inherit the
106
+ // escalation surface every layer agrees on — same semantics as `allow`.
107
+ escalate: intersectRestrictions(settings.map((layer) => layer.tools.escalate)),
81
108
  effects,
82
109
  trustedPreHandlerTools: intersectRestrictions(settings.map((layer) => layer.tools.trustedPreHandlerTools)),
110
+ timeouts: mergeTimeouts(settings),
83
111
  },
84
112
  memory: {
85
113
  enabled: settings.every((layer) => layer.memory.enabled),
@@ -103,6 +131,11 @@ export function compileHarness(layers: HarnessLayer[]): CompiledHarness {
103
131
  maxTotalTokens: minimumNullable(settings.map((layer) => layer.limits.maxTotalTokens)),
104
132
  maxCost: minimumNullable(settings.map((layer) => layer.limits.maxCost)),
105
133
  },
134
+ context: {
135
+ spill: {
136
+ maxInlineBytes: minimumNullable(settings.map((layer) => layer.context.spill.maxInlineBytes)),
137
+ },
138
+ },
106
139
  isolation: {
107
140
  mode: settings.some((layer) => layer.isolation.mode === 'worktree') ? 'worktree' : 'none',
108
141
  requireWorktree: settings.some((layer) => layer.isolation.requireWorktree),
@@ -114,6 +147,10 @@ export function compileHarness(layers: HarnessLayer[]): CompiledHarness {
114
147
  tracingRetentionDays: Math.min(...settings.map((layer) => layer.observability.tracingRetentionDays)),
115
148
  captureInputs: settings.every((layer) => layer.observability.captureInputs),
116
149
  captureOutputs: settings.every((layer) => layer.observability.captureOutputs),
150
+ // Any layer demanding redaction wins; any layer restricting sharing wins. Telemetry safety
151
+ // is union-of-concerns, not intersection.
152
+ redactSecrets: settings.some((layer) => layer.observability.redactSecrets),
153
+ sharing: strictestSharing(settings.map((layer) => layer.observability.sharing)),
117
154
  },
118
155
  };
119
156
  }