plugin-agent-orchestrator 1.1.4 → 1.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +204 -114
- package/dist/client/index.js +1 -1
- package/dist/client-v2/117.7565cff57c39318a.js +10 -0
- package/dist/client-v2/214.106635011859dede.js +10 -0
- package/dist/client-v2/418.41ed200e7d6da332.js +10 -0
- package/dist/client-v2/53.349fff8db80b19d4.js +10 -0
- package/dist/client-v2/981.e4a1f41cfc94a856.js +10 -0
- package/dist/client-v2/index.js +1 -1
- package/dist/externalVersion.js +7 -8
- package/dist/locale/en-US.json +226 -170
- package/dist/locale/vi-VN.json +226 -170
- package/dist/locale/zh-CN.json +226 -170
- package/dist/server/collections/agent-execution-spans.js +8 -0
- package/dist/server/collections/agent-loop-action-approvals.js +1 -0
- package/dist/server/collections/agent-loop-runs.js +7 -1
- package/dist/server/collections/agent-loop-steps.js +7 -0
- package/dist/server/collections/skill-executions.js +8 -0
- package/dist/server/plugin.js +3 -1
- package/dist/server/resources/agent-harness-profile.js +117 -0
- package/dist/server/resources/agent-knowledge-insights.js +1 -3
- package/dist/server/resources/agent-loop-approval.js +25 -0
- package/dist/server/resources/agent-loop.js +3 -1
- package/dist/server/services/AgentMemoryContextService.js +34 -0
- package/dist/server/services/HarnessCompiler.js +35 -2
- package/dist/server/services/HarnessProfileService.js +74 -9
- package/dist/server/services/HarnessSchema.js +30 -3
- package/dist/server/services/LoopPatternSchema.js +16 -1
- package/dist/server/services/LoopRunRepository.js +80 -0
- package/dist/server/services/LoopRunStateMachine.js +92 -3
- package/dist/server/services/LoopTriggerService.js +37 -0
- package/dist/server/services/LoopWorkerService.js +261 -23
- package/dist/server/services/NativeSubAgentObserver.js +165 -23
- package/dist/server/services/PluginAiRuntimeAdapter.js +16 -5
- package/dist/server/services/ToolLoopDetectionService.js +210 -0
- package/dist/server/services/VerificationService.js +66 -0
- package/dist/server/services/harness-runtime-policy.js +139 -0
- package/dist/server/skill-hub/plugin.js +36 -3
- package/dist/server/skill-hub/tasks/SkillExecutionTask.js +29 -0
- package/package.json +53 -53
- package/src/client/ApprovalsTab.tsx +2 -0
- package/src/client/OrchestratorSettings.tsx +11 -0
- package/src/client/skill-hub/components/GitSkillImport.tsx +3 -3
- package/src/client/skill-hub/components/SkillMetrics.tsx +15 -10
- package/src/client/skill-hub/index.tsx +0 -52
- package/src/client-v2/components/AgentRunsTab.tsx +576 -576
- package/src/client-v2/components/ApprovalsTab.tsx +337 -0
- package/src/client-v2/components/HarnessProfilesTab.tsx +385 -105
- package/src/client-v2/components/MemoryInspectorTab.tsx +141 -141
- package/src/client-v2/components/TracingTab.tsx +515 -515
- package/src/client-v2/hooks/useApiRequest.ts +141 -141
- package/src/client-v2/pages/ApprovalsPage.tsx +10 -0
- package/src/client-v2/plugin.tsx +6 -6
- package/src/client-v2/skill-hub/components/SkillMetrics.tsx +5 -2
- package/src/locale/en-US.json +226 -174
- package/src/locale/vi-VN.json +226 -174
- package/src/locale/zh-CN.json +226 -174
- package/src/server/__tests__/circuit-breaker.test.ts +248 -248
- package/src/server/__tests__/harness-compiler.test.ts +83 -0
- package/src/server/__tests__/harness-profile-service.test.ts +51 -0
- package/src/server/__tests__/harness-runtime-policy.test.ts +293 -0
- package/src/server/__tests__/harness-tool-plan.test.ts +24 -0
- package/src/server/__tests__/loop-approval-decision.test.ts +512 -0
- package/src/server/__tests__/loop-approval-expiry.test.ts +258 -0
- package/src/server/__tests__/loop-budget-service.test.ts +255 -255
- package/src/server/__tests__/loop-control-plane-backfill.test.ts +266 -266
- package/src/server/__tests__/loop-pattern-schema.test.ts +18 -1
- package/src/server/__tests__/loop-pattern-service.test.ts +148 -148
- package/src/server/__tests__/loop-run-state-machine.test.ts +11 -0
- package/src/server/__tests__/loop-trigger-service.test.ts +148 -0
- package/src/server/__tests__/loop-worker-claim.test.ts +341 -341
- package/src/server/__tests__/loop-worker-execution.test.ts +82 -1
- package/src/server/__tests__/native-sub-agent-observer.test.ts +288 -4
- package/src/server/__tests__/path-lock-service.test.ts +254 -254
- package/src/server/__tests__/skill-execution-task-policy.test.ts +140 -0
- package/src/server/__tests__/smoke.test.ts +162 -162
- package/src/server/__tests__/tool-loop-detection.test.ts +275 -0
- package/src/server/collections/agent-execution-spans.ts +8 -0
- package/src/server/collections/agent-harness-profile-versions.ts +26 -26
- package/src/server/collections/agent-harness-profiles.ts +71 -71
- package/src/server/collections/agent-loop-action-approvals.ts +1 -0
- package/src/server/collections/agent-loop-artifacts.ts +25 -25
- package/src/server/collections/agent-loop-circuit-states.ts +23 -23
- package/src/server/collections/agent-loop-control-settings.ts +21 -21
- package/src/server/collections/agent-loop-events.ts +98 -98
- package/src/server/collections/agent-loop-path-locks.ts +27 -27
- package/src/server/collections/agent-loop-patterns.ts +42 -42
- package/src/server/collections/agent-loop-runs.ts +7 -1
- package/src/server/collections/agent-loop-steps.ts +7 -0
- package/src/server/collections/agent-loop-usage-buckets.ts +22 -22
- package/src/server/collections/agent-loop-worktrees.ts +25 -25
- package/src/server/collections/skill-executions.ts +8 -0
- package/src/server/migrations/20260427000002-change-packages-to-text.ts +52 -52
- package/src/server/migrations/20260429000000-add-llm-fields.ts +47 -47
- package/src/server/migrations/20260429000001-fix-inputargs-json-to-text.ts +45 -45
- package/src/server/migrations/20260503000000-add-orchestrator-trace-fields.ts +33 -33
- package/src/server/migrations/20260524000000-add-agent-loop-fields-to-skill-executions.ts +31 -31
- package/src/server/migrations/20260803010000-backfill-loop-control-plane-data.ts +293 -293
- package/src/server/plugin.ts +3 -1
- package/src/server/resources/agent-harness-profile.ts +94 -0
- package/src/server/resources/agent-knowledge-insights.ts +5 -3
- package/src/server/resources/agent-loop-approval.ts +32 -0
- package/src/server/resources/agent-loop-artifact.ts +56 -56
- package/src/server/resources/agent-loop-events-stream.ts +105 -105
- package/src/server/resources/agent-loop.ts +3 -1
- package/src/server/resources/resource-helpers.ts +33 -33
- package/src/server/resources/tracing.ts +266 -266
- package/src/server/services/AgentMemoryContextService.ts +44 -0
- package/src/server/services/DurableCircuitBreakerService.ts +239 -239
- package/src/server/services/ExecutionSpanService.ts +2 -0
- package/src/server/services/HarnessCompiler.ts +37 -0
- package/src/server/services/HarnessProfileService.ts +92 -10
- package/src/server/services/HarnessSchema.ts +31 -0
- package/src/server/services/LoopBudgetService.ts +275 -275
- package/src/server/services/LoopControlService.ts +124 -124
- package/src/server/services/LoopPatternSchema.ts +19 -1
- package/src/server/services/LoopPatternService.ts +143 -143
- package/src/server/services/LoopRunRepository.ts +99 -0
- package/src/server/services/LoopRunStateMachine.ts +100 -3
- package/src/server/services/LoopSchedulerService.ts +199 -199
- package/src/server/services/LoopTriggerService.ts +42 -0
- package/src/server/services/LoopWorkerService.ts +327 -27
- package/src/server/services/NativeSubAgentObserver.ts +186 -21
- package/src/server/services/PathLockService.ts +321 -321
- package/src/server/services/PluginAiRuntimeAdapter.ts +24 -7
- package/src/server/services/ToolLoopDetectionService.ts +213 -0
- package/src/server/services/VerificationSchema.ts +31 -31
- package/src/server/services/VerificationService.ts +81 -2
- package/src/server/services/harness-runtime-policy.ts +141 -0
- package/src/server/skill-hub/plugin.ts +41 -3
- package/src/server/skill-hub/services/RegistrySkillSnapshotService.ts +235 -235
- package/src/server/skill-hub/services/__tests__/RegistrySkillSnapshotService.test.ts +61 -61
- package/src/server/skill-hub/tasks/SkillExecutionTask.ts +43 -2
- package/src/server/utils/ensure-indexes.ts +96 -96
- package/src/server/utils/logging.ts +62 -62
- package/src/server/utils/loop-pattern-context.ts +20 -20
- package/dist/client-v2/214.b063f33ba044a808.js +0 -10
- package/dist/client-v2/373.f5d5292e53c4e832.js +0 -10
- package/dist/client-v2/418.58fa350f5496efd3.js +0 -10
- package/dist/client-v2/53.9a10455323390f14.js +0 -10
- package/dist/client-v2/677.a991ce0250ff5c77.js +0 -10
- package/dist/client-v2/925.e5925bf21ed68e7a.js +0 -10
- package/src/client/AIEmployeeSelect.tsx +0 -47
- package/src/client/AIEmployeesContext.tsx +0 -102
- package/src/client/RulesTab.tsx +0 -2
- package/src/client/TracingTab.tsx +0 -436
- package/src/client/skill-hub/components/ExecutionProgress.tsx +0 -51
- package/src/client-v2/components/RulesTab.tsx +0 -2
- package/src/client-v2/pages/AgentRunsPage.tsx +0 -13
|
@@ -1,239 +1,239 @@
|
|
|
1
|
-
import { createHash } from 'node:crypto';
|
|
2
|
-
import type { Database, Model } from '@nocobase/database';
|
|
3
|
-
import type { LoopPatternPolicy } from './LoopPatternSchema';
|
|
4
|
-
|
|
5
|
-
type DistributedLock = {
|
|
6
|
-
runExclusive<T>(key: string, operation: () => Promise<T>, ttl?: number): Promise<T>;
|
|
7
|
-
};
|
|
8
|
-
|
|
9
|
-
type CircuitSnapshot = {
|
|
10
|
-
id: number;
|
|
11
|
-
state: 'closed' | 'open' | 'half_open';
|
|
12
|
-
attempts: number;
|
|
13
|
-
consecutiveFailures: number;
|
|
14
|
-
errorSignature: string;
|
|
15
|
-
repeatedErrorCount: number;
|
|
16
|
-
retryAt: Date | null;
|
|
17
|
-
};
|
|
18
|
-
|
|
19
|
-
function read(record: Model | Record<string, unknown>, key: string) {
|
|
20
|
-
const model = record as Model & { get?: (name: string) => unknown };
|
|
21
|
-
return typeof model.get === 'function' ? model.get(key) : (record as Record<string, unknown>)[key];
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
function count(value: unknown, label: string) {
|
|
25
|
-
if (value === null || value === undefined) return 0;
|
|
26
|
-
const parsed = Number(value);
|
|
27
|
-
if (!Number.isSafeInteger(parsed) || parsed < 0) throw new Error(`${label} counter is invalid.`);
|
|
28
|
-
return parsed;
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
function dateOrNull(value: unknown) {
|
|
32
|
-
if (!value) return null;
|
|
33
|
-
const date = value instanceof Date ? value : new Date(String(value));
|
|
34
|
-
return Number.isNaN(date.getTime()) ? null : date;
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
function snapshot(record: Model | Record<string, unknown>): CircuitSnapshot {
|
|
38
|
-
const id = Number(read(record, 'id'));
|
|
39
|
-
const state = read(record, 'state');
|
|
40
|
-
if (!Number.isSafeInteger(id) || id <= 0) throw new Error('Circuit breaker state has an invalid identifier.');
|
|
41
|
-
if (state !== 'closed' && state !== 'open' && state !== 'half_open') {
|
|
42
|
-
throw new Error('Circuit breaker state is invalid.');
|
|
43
|
-
}
|
|
44
|
-
return {
|
|
45
|
-
id,
|
|
46
|
-
state,
|
|
47
|
-
attempts: count(read(record, 'attempts'), 'Circuit attempt'),
|
|
48
|
-
consecutiveFailures: count(read(record, 'consecutiveFailures'), 'Circuit consecutive failure'),
|
|
49
|
-
errorSignature: typeof read(record, 'errorSignature') === 'string' ? String(read(record, 'errorSignature')) : '',
|
|
50
|
-
repeatedErrorCount: count(read(record, 'repeatedErrorCount'), 'Circuit repeated error'),
|
|
51
|
-
retryAt: dateOrNull(read(record, 'retryAt')),
|
|
52
|
-
};
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
function normalizeScopeKey(scopeKey: string) {
|
|
56
|
-
const normalized = scopeKey.trim();
|
|
57
|
-
if (!normalized) throw new Error('Circuit breaker scope key is required.');
|
|
58
|
-
if (normalized.length > 300) throw new Error('Circuit breaker scope key cannot exceed 300 characters.');
|
|
59
|
-
return normalized;
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
export function normalizeErrorSignature(error: unknown) {
|
|
63
|
-
const raw = error instanceof Error ? `${error.name}:${error.message}` : String(error);
|
|
64
|
-
const normalized = raw
|
|
65
|
-
.toLowerCase()
|
|
66
|
-
.replace(/[a-z]:[\\/][^\s"']+/gi, '<path>')
|
|
67
|
-
.replace(/\/(?:[^\s/]+\/)+[^\s"']*/g, '<path>')
|
|
68
|
-
.replace(/\b[0-9a-f]{8}-[0-9a-f-]{27,}\b/gi, '<uuid>')
|
|
69
|
-
.replace(/\b\d+\b/g, '<n>')
|
|
70
|
-
.replace(/\s+/g, ' ')
|
|
71
|
-
.trim();
|
|
72
|
-
return createHash('sha256')
|
|
73
|
-
.update(normalized || 'unknown-error')
|
|
74
|
-
.digest('hex');
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
export class DurableCircuitBreakerService {
|
|
78
|
-
constructor(
|
|
79
|
-
private readonly database: Database,
|
|
80
|
-
private readonly distributedLock: DistributedLock,
|
|
81
|
-
) {}
|
|
82
|
-
|
|
83
|
-
async authorizeAttempt(input: {
|
|
84
|
-
patternId: number;
|
|
85
|
-
scopeKey: string;
|
|
86
|
-
policy: LoopPatternPolicy['circuit'];
|
|
87
|
-
now?: Date;
|
|
88
|
-
}) {
|
|
89
|
-
const scopeKey = normalizeScopeKey(input.scopeKey);
|
|
90
|
-
const now = input.now || new Date();
|
|
91
|
-
return this.withScopeLock(input.patternId, scopeKey, async () => {
|
|
92
|
-
const decision = await this.database.sequelize.transaction(async (transaction) => {
|
|
93
|
-
const repository = this.database.getRepository('agentLoopCircuitStates');
|
|
94
|
-
let record = await repository.findOne({
|
|
95
|
-
filter: { patternId: input.patternId, scopeKey },
|
|
96
|
-
transaction,
|
|
97
|
-
lock: transaction.LOCK.UPDATE,
|
|
98
|
-
});
|
|
99
|
-
if (!record) {
|
|
100
|
-
record = await repository.create({
|
|
101
|
-
values: {
|
|
102
|
-
patternId: input.patternId,
|
|
103
|
-
scopeKey,
|
|
104
|
-
state: 'closed',
|
|
105
|
-
attempts: 0,
|
|
106
|
-
consecutiveFailures: 0,
|
|
107
|
-
repeatedErrorCount: 0,
|
|
108
|
-
},
|
|
109
|
-
transaction,
|
|
110
|
-
});
|
|
111
|
-
}
|
|
112
|
-
const current = snapshot(record);
|
|
113
|
-
if (current.state === 'half_open') {
|
|
114
|
-
return { allowed: false as const, reason: 'probe' as const, retryAt: current.retryAt };
|
|
115
|
-
}
|
|
116
|
-
if (current.state === 'open' && (!current.retryAt || current.retryAt.getTime() > now.getTime())) {
|
|
117
|
-
return { allowed: false as const, reason: 'open' as const, retryAt: current.retryAt };
|
|
118
|
-
}
|
|
119
|
-
if (current.state === 'closed' && current.attempts >= input.policy.maxAttempts) {
|
|
120
|
-
const retryAt = new Date(now.getTime() + input.policy.cooldownMs);
|
|
121
|
-
await repository.update({
|
|
122
|
-
filterByTk: current.id,
|
|
123
|
-
values: { state: 'open', openedAt: now, retryAt },
|
|
124
|
-
transaction,
|
|
125
|
-
});
|
|
126
|
-
return { allowed: false as const, reason: 'attempts' as const, retryAt };
|
|
127
|
-
}
|
|
128
|
-
const nextState = current.state === 'open' ? 'half_open' : 'closed';
|
|
129
|
-
await repository.update({
|
|
130
|
-
filterByTk: current.id,
|
|
131
|
-
values: { state: nextState, attempts: current.attempts + 1 },
|
|
132
|
-
transaction,
|
|
133
|
-
});
|
|
134
|
-
return { allowed: true as const, state: nextState, attempt: current.attempts + 1 };
|
|
135
|
-
});
|
|
136
|
-
if (decision.allowed) return decision;
|
|
137
|
-
if (decision.reason === 'probe') throw new Error('Circuit breaker probe is already in progress.');
|
|
138
|
-
if (decision.reason === 'attempts') throw new Error('Circuit breaker maximum attempts reached.');
|
|
139
|
-
throw new Error(`Circuit breaker is open until ${decision.retryAt?.toISOString() || 'manual reset'}.`);
|
|
140
|
-
});
|
|
141
|
-
}
|
|
142
|
-
|
|
143
|
-
async recordSuccess(patternId: number, scopeKey: string, lastRunId?: number) {
|
|
144
|
-
const normalizedScope = normalizeScopeKey(scopeKey);
|
|
145
|
-
return this.withScopeLock(patternId, normalizedScope, async () =>
|
|
146
|
-
this.database.sequelize.transaction(async (transaction) => {
|
|
147
|
-
const repository = this.database.getRepository('agentLoopCircuitStates');
|
|
148
|
-
const record = await repository.findOne({
|
|
149
|
-
filter: { patternId, scopeKey: normalizedScope },
|
|
150
|
-
transaction,
|
|
151
|
-
lock: transaction.LOCK.UPDATE,
|
|
152
|
-
});
|
|
153
|
-
if (!record) return;
|
|
154
|
-
await repository.update({
|
|
155
|
-
filterByTk: read(record, 'id'),
|
|
156
|
-
values: {
|
|
157
|
-
state: 'closed',
|
|
158
|
-
attempts: 0,
|
|
159
|
-
consecutiveFailures: 0,
|
|
160
|
-
errorSignature: null,
|
|
161
|
-
repeatedErrorCount: 0,
|
|
162
|
-
lastRunId: lastRunId || null,
|
|
163
|
-
openedAt: null,
|
|
164
|
-
retryAt: null,
|
|
165
|
-
},
|
|
166
|
-
transaction,
|
|
167
|
-
});
|
|
168
|
-
}),
|
|
169
|
-
);
|
|
170
|
-
}
|
|
171
|
-
|
|
172
|
-
async recordFailure(input: {
|
|
173
|
-
patternId: number;
|
|
174
|
-
scopeKey: string;
|
|
175
|
-
policy: LoopPatternPolicy['circuit'];
|
|
176
|
-
error: unknown;
|
|
177
|
-
lastRunId?: number;
|
|
178
|
-
now?: Date;
|
|
179
|
-
}) {
|
|
180
|
-
const scopeKey = normalizeScopeKey(input.scopeKey);
|
|
181
|
-
const signature = normalizeErrorSignature(input.error);
|
|
182
|
-
const now = input.now || new Date();
|
|
183
|
-
return this.withScopeLock(input.patternId, scopeKey, async () =>
|
|
184
|
-
this.database.sequelize.transaction(async (transaction) => {
|
|
185
|
-
const repository = this.database.getRepository('agentLoopCircuitStates');
|
|
186
|
-
let record = await repository.findOne({
|
|
187
|
-
filter: { patternId: input.patternId, scopeKey },
|
|
188
|
-
transaction,
|
|
189
|
-
lock: transaction.LOCK.UPDATE,
|
|
190
|
-
});
|
|
191
|
-
if (!record) {
|
|
192
|
-
record = await repository.create({
|
|
193
|
-
values: {
|
|
194
|
-
patternId: input.patternId,
|
|
195
|
-
scopeKey,
|
|
196
|
-
state: 'closed',
|
|
197
|
-
attempts: 1,
|
|
198
|
-
consecutiveFailures: 0,
|
|
199
|
-
repeatedErrorCount: 0,
|
|
200
|
-
},
|
|
201
|
-
transaction,
|
|
202
|
-
});
|
|
203
|
-
}
|
|
204
|
-
const current = snapshot(record);
|
|
205
|
-
const consecutiveFailures = current.consecutiveFailures + 1;
|
|
206
|
-
const repeatedErrorCount = current.errorSignature === signature ? current.repeatedErrorCount + 1 : 1;
|
|
207
|
-
const shouldOpen =
|
|
208
|
-
current.state === 'half_open' ||
|
|
209
|
-
current.attempts >= input.policy.maxAttempts ||
|
|
210
|
-
consecutiveFailures >= input.policy.maxConsecutiveFailures ||
|
|
211
|
-
repeatedErrorCount >= input.policy.maxRepeatedError;
|
|
212
|
-
await repository.update({
|
|
213
|
-
filterByTk: current.id,
|
|
214
|
-
values: {
|
|
215
|
-
state: shouldOpen ? 'open' : 'closed',
|
|
216
|
-
consecutiveFailures,
|
|
217
|
-
errorSignature: signature,
|
|
218
|
-
repeatedErrorCount,
|
|
219
|
-
lastRunId: input.lastRunId || null,
|
|
220
|
-
openedAt: shouldOpen ? now : null,
|
|
221
|
-
retryAt: shouldOpen ? new Date(now.getTime() + input.policy.cooldownMs) : null,
|
|
222
|
-
},
|
|
223
|
-
transaction,
|
|
224
|
-
});
|
|
225
|
-
return { open: shouldOpen, signature, consecutiveFailures, repeatedErrorCount };
|
|
226
|
-
}),
|
|
227
|
-
);
|
|
228
|
-
}
|
|
229
|
-
|
|
230
|
-
private withScopeLock<T>(patternId: number, scopeKey: string, operation: () => Promise<T>) {
|
|
231
|
-
const normalizedScope = scopeKey.trim();
|
|
232
|
-
if (!normalizedScope) throw new Error('Circuit breaker scope key is required.');
|
|
233
|
-
return this.distributedLock.runExclusive(
|
|
234
|
-
`agent-loop:circuit:${patternId}:${createHash('sha256').update(normalizedScope).digest('hex')}`,
|
|
235
|
-
operation,
|
|
236
|
-
30_000,
|
|
237
|
-
);
|
|
238
|
-
}
|
|
239
|
-
}
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import type { Database, Model } from '@nocobase/database';
|
|
3
|
+
import type { LoopPatternPolicy } from './LoopPatternSchema';
|
|
4
|
+
|
|
5
|
+
type DistributedLock = {
|
|
6
|
+
runExclusive<T>(key: string, operation: () => Promise<T>, ttl?: number): Promise<T>;
|
|
7
|
+
};
|
|
8
|
+
|
|
9
|
+
type CircuitSnapshot = {
|
|
10
|
+
id: number;
|
|
11
|
+
state: 'closed' | 'open' | 'half_open';
|
|
12
|
+
attempts: number;
|
|
13
|
+
consecutiveFailures: number;
|
|
14
|
+
errorSignature: string;
|
|
15
|
+
repeatedErrorCount: number;
|
|
16
|
+
retryAt: Date | null;
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
function read(record: Model | Record<string, unknown>, key: string) {
|
|
20
|
+
const model = record as Model & { get?: (name: string) => unknown };
|
|
21
|
+
return typeof model.get === 'function' ? model.get(key) : (record as Record<string, unknown>)[key];
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function count(value: unknown, label: string) {
|
|
25
|
+
if (value === null || value === undefined) return 0;
|
|
26
|
+
const parsed = Number(value);
|
|
27
|
+
if (!Number.isSafeInteger(parsed) || parsed < 0) throw new Error(`${label} counter is invalid.`);
|
|
28
|
+
return parsed;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function dateOrNull(value: unknown) {
|
|
32
|
+
if (!value) return null;
|
|
33
|
+
const date = value instanceof Date ? value : new Date(String(value));
|
|
34
|
+
return Number.isNaN(date.getTime()) ? null : date;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function snapshot(record: Model | Record<string, unknown>): CircuitSnapshot {
|
|
38
|
+
const id = Number(read(record, 'id'));
|
|
39
|
+
const state = read(record, 'state');
|
|
40
|
+
if (!Number.isSafeInteger(id) || id <= 0) throw new Error('Circuit breaker state has an invalid identifier.');
|
|
41
|
+
if (state !== 'closed' && state !== 'open' && state !== 'half_open') {
|
|
42
|
+
throw new Error('Circuit breaker state is invalid.');
|
|
43
|
+
}
|
|
44
|
+
return {
|
|
45
|
+
id,
|
|
46
|
+
state,
|
|
47
|
+
attempts: count(read(record, 'attempts'), 'Circuit attempt'),
|
|
48
|
+
consecutiveFailures: count(read(record, 'consecutiveFailures'), 'Circuit consecutive failure'),
|
|
49
|
+
errorSignature: typeof read(record, 'errorSignature') === 'string' ? String(read(record, 'errorSignature')) : '',
|
|
50
|
+
repeatedErrorCount: count(read(record, 'repeatedErrorCount'), 'Circuit repeated error'),
|
|
51
|
+
retryAt: dateOrNull(read(record, 'retryAt')),
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function normalizeScopeKey(scopeKey: string) {
|
|
56
|
+
const normalized = scopeKey.trim();
|
|
57
|
+
if (!normalized) throw new Error('Circuit breaker scope key is required.');
|
|
58
|
+
if (normalized.length > 300) throw new Error('Circuit breaker scope key cannot exceed 300 characters.');
|
|
59
|
+
return normalized;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export function normalizeErrorSignature(error: unknown) {
|
|
63
|
+
const raw = error instanceof Error ? `${error.name}:${error.message}` : String(error);
|
|
64
|
+
const normalized = raw
|
|
65
|
+
.toLowerCase()
|
|
66
|
+
.replace(/[a-z]:[\\/][^\s"']+/gi, '<path>')
|
|
67
|
+
.replace(/\/(?:[^\s/]+\/)+[^\s"']*/g, '<path>')
|
|
68
|
+
.replace(/\b[0-9a-f]{8}-[0-9a-f-]{27,}\b/gi, '<uuid>')
|
|
69
|
+
.replace(/\b\d+\b/g, '<n>')
|
|
70
|
+
.replace(/\s+/g, ' ')
|
|
71
|
+
.trim();
|
|
72
|
+
return createHash('sha256')
|
|
73
|
+
.update(normalized || 'unknown-error')
|
|
74
|
+
.digest('hex');
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
export class DurableCircuitBreakerService {
|
|
78
|
+
constructor(
|
|
79
|
+
private readonly database: Database,
|
|
80
|
+
private readonly distributedLock: DistributedLock,
|
|
81
|
+
) {}
|
|
82
|
+
|
|
83
|
+
async authorizeAttempt(input: {
|
|
84
|
+
patternId: number;
|
|
85
|
+
scopeKey: string;
|
|
86
|
+
policy: LoopPatternPolicy['circuit'];
|
|
87
|
+
now?: Date;
|
|
88
|
+
}) {
|
|
89
|
+
const scopeKey = normalizeScopeKey(input.scopeKey);
|
|
90
|
+
const now = input.now || new Date();
|
|
91
|
+
return this.withScopeLock(input.patternId, scopeKey, async () => {
|
|
92
|
+
const decision = await this.database.sequelize.transaction(async (transaction) => {
|
|
93
|
+
const repository = this.database.getRepository('agentLoopCircuitStates');
|
|
94
|
+
let record = await repository.findOne({
|
|
95
|
+
filter: { patternId: input.patternId, scopeKey },
|
|
96
|
+
transaction,
|
|
97
|
+
lock: transaction.LOCK.UPDATE,
|
|
98
|
+
});
|
|
99
|
+
if (!record) {
|
|
100
|
+
record = await repository.create({
|
|
101
|
+
values: {
|
|
102
|
+
patternId: input.patternId,
|
|
103
|
+
scopeKey,
|
|
104
|
+
state: 'closed',
|
|
105
|
+
attempts: 0,
|
|
106
|
+
consecutiveFailures: 0,
|
|
107
|
+
repeatedErrorCount: 0,
|
|
108
|
+
},
|
|
109
|
+
transaction,
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
const current = snapshot(record);
|
|
113
|
+
if (current.state === 'half_open') {
|
|
114
|
+
return { allowed: false as const, reason: 'probe' as const, retryAt: current.retryAt };
|
|
115
|
+
}
|
|
116
|
+
if (current.state === 'open' && (!current.retryAt || current.retryAt.getTime() > now.getTime())) {
|
|
117
|
+
return { allowed: false as const, reason: 'open' as const, retryAt: current.retryAt };
|
|
118
|
+
}
|
|
119
|
+
if (current.state === 'closed' && current.attempts >= input.policy.maxAttempts) {
|
|
120
|
+
const retryAt = new Date(now.getTime() + input.policy.cooldownMs);
|
|
121
|
+
await repository.update({
|
|
122
|
+
filterByTk: current.id,
|
|
123
|
+
values: { state: 'open', openedAt: now, retryAt },
|
|
124
|
+
transaction,
|
|
125
|
+
});
|
|
126
|
+
return { allowed: false as const, reason: 'attempts' as const, retryAt };
|
|
127
|
+
}
|
|
128
|
+
const nextState = current.state === 'open' ? 'half_open' : 'closed';
|
|
129
|
+
await repository.update({
|
|
130
|
+
filterByTk: current.id,
|
|
131
|
+
values: { state: nextState, attempts: current.attempts + 1 },
|
|
132
|
+
transaction,
|
|
133
|
+
});
|
|
134
|
+
return { allowed: true as const, state: nextState, attempt: current.attempts + 1 };
|
|
135
|
+
});
|
|
136
|
+
if (decision.allowed) return decision;
|
|
137
|
+
if (decision.reason === 'probe') throw new Error('Circuit breaker probe is already in progress.');
|
|
138
|
+
if (decision.reason === 'attempts') throw new Error('Circuit breaker maximum attempts reached.');
|
|
139
|
+
throw new Error(`Circuit breaker is open until ${decision.retryAt?.toISOString() || 'manual reset'}.`);
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
async recordSuccess(patternId: number, scopeKey: string, lastRunId?: number) {
|
|
144
|
+
const normalizedScope = normalizeScopeKey(scopeKey);
|
|
145
|
+
return this.withScopeLock(patternId, normalizedScope, async () =>
|
|
146
|
+
this.database.sequelize.transaction(async (transaction) => {
|
|
147
|
+
const repository = this.database.getRepository('agentLoopCircuitStates');
|
|
148
|
+
const record = await repository.findOne({
|
|
149
|
+
filter: { patternId, scopeKey: normalizedScope },
|
|
150
|
+
transaction,
|
|
151
|
+
lock: transaction.LOCK.UPDATE,
|
|
152
|
+
});
|
|
153
|
+
if (!record) return;
|
|
154
|
+
await repository.update({
|
|
155
|
+
filterByTk: read(record, 'id'),
|
|
156
|
+
values: {
|
|
157
|
+
state: 'closed',
|
|
158
|
+
attempts: 0,
|
|
159
|
+
consecutiveFailures: 0,
|
|
160
|
+
errorSignature: null,
|
|
161
|
+
repeatedErrorCount: 0,
|
|
162
|
+
lastRunId: lastRunId || null,
|
|
163
|
+
openedAt: null,
|
|
164
|
+
retryAt: null,
|
|
165
|
+
},
|
|
166
|
+
transaction,
|
|
167
|
+
});
|
|
168
|
+
}),
|
|
169
|
+
);
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
async recordFailure(input: {
|
|
173
|
+
patternId: number;
|
|
174
|
+
scopeKey: string;
|
|
175
|
+
policy: LoopPatternPolicy['circuit'];
|
|
176
|
+
error: unknown;
|
|
177
|
+
lastRunId?: number;
|
|
178
|
+
now?: Date;
|
|
179
|
+
}) {
|
|
180
|
+
const scopeKey = normalizeScopeKey(input.scopeKey);
|
|
181
|
+
const signature = normalizeErrorSignature(input.error);
|
|
182
|
+
const now = input.now || new Date();
|
|
183
|
+
return this.withScopeLock(input.patternId, scopeKey, async () =>
|
|
184
|
+
this.database.sequelize.transaction(async (transaction) => {
|
|
185
|
+
const repository = this.database.getRepository('agentLoopCircuitStates');
|
|
186
|
+
let record = await repository.findOne({
|
|
187
|
+
filter: { patternId: input.patternId, scopeKey },
|
|
188
|
+
transaction,
|
|
189
|
+
lock: transaction.LOCK.UPDATE,
|
|
190
|
+
});
|
|
191
|
+
if (!record) {
|
|
192
|
+
record = await repository.create({
|
|
193
|
+
values: {
|
|
194
|
+
patternId: input.patternId,
|
|
195
|
+
scopeKey,
|
|
196
|
+
state: 'closed',
|
|
197
|
+
attempts: 1,
|
|
198
|
+
consecutiveFailures: 0,
|
|
199
|
+
repeatedErrorCount: 0,
|
|
200
|
+
},
|
|
201
|
+
transaction,
|
|
202
|
+
});
|
|
203
|
+
}
|
|
204
|
+
const current = snapshot(record);
|
|
205
|
+
const consecutiveFailures = current.consecutiveFailures + 1;
|
|
206
|
+
const repeatedErrorCount = current.errorSignature === signature ? current.repeatedErrorCount + 1 : 1;
|
|
207
|
+
const shouldOpen =
|
|
208
|
+
current.state === 'half_open' ||
|
|
209
|
+
current.attempts >= input.policy.maxAttempts ||
|
|
210
|
+
consecutiveFailures >= input.policy.maxConsecutiveFailures ||
|
|
211
|
+
repeatedErrorCount >= input.policy.maxRepeatedError;
|
|
212
|
+
await repository.update({
|
|
213
|
+
filterByTk: current.id,
|
|
214
|
+
values: {
|
|
215
|
+
state: shouldOpen ? 'open' : 'closed',
|
|
216
|
+
consecutiveFailures,
|
|
217
|
+
errorSignature: signature,
|
|
218
|
+
repeatedErrorCount,
|
|
219
|
+
lastRunId: input.lastRunId || null,
|
|
220
|
+
openedAt: shouldOpen ? now : null,
|
|
221
|
+
retryAt: shouldOpen ? new Date(now.getTime() + input.policy.cooldownMs) : null,
|
|
222
|
+
},
|
|
223
|
+
transaction,
|
|
224
|
+
});
|
|
225
|
+
return { open: shouldOpen, signature, consecutiveFailures, repeatedErrorCount };
|
|
226
|
+
}),
|
|
227
|
+
);
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
private withScopeLock<T>(patternId: number, scopeKey: string, operation: () => Promise<T>) {
|
|
231
|
+
const normalizedScope = scopeKey.trim();
|
|
232
|
+
if (!normalizedScope) throw new Error('Circuit breaker scope key is required.');
|
|
233
|
+
return this.distributedLock.runExclusive(
|
|
234
|
+
`agent-loop:circuit:${patternId}:${createHash('sha256').update(normalizedScope).digest('hex')}`,
|
|
235
|
+
operation,
|
|
236
|
+
30_000,
|
|
237
|
+
);
|
|
238
|
+
}
|
|
239
|
+
}
|
|
@@ -54,6 +54,30 @@ function strictestNetwork(values: HarnessSettings['isolation']['networkAccess'][
|
|
|
54
54
|
);
|
|
55
55
|
}
|
|
56
56
|
|
|
57
|
+
const sharingRank: Record<HarnessSettings['observability']['sharing'], number> = {
|
|
58
|
+
full: 0,
|
|
59
|
+
'feedback-only': 1,
|
|
60
|
+
disabled: 2,
|
|
61
|
+
};
|
|
62
|
+
|
|
63
|
+
function strictestSharing(values: HarnessSettings['observability']['sharing'][]) {
|
|
64
|
+
return values.reduce<HarnessSettings['observability']['sharing']>(
|
|
65
|
+
(strictest, value) => (sharingRank[value] > sharingRank[strictest] ? value : strictest),
|
|
66
|
+
'full',
|
|
67
|
+
);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
// Per-tool caps compile like every other limit: the smallest value across layers wins.
|
|
71
|
+
function mergeTimeouts(settings: HarnessSettings[]) {
|
|
72
|
+
const timeouts: Record<string, number> = {};
|
|
73
|
+
for (const layer of settings) {
|
|
74
|
+
for (const [name, timeoutMs] of Object.entries(layer.tools.timeouts)) {
|
|
75
|
+
timeouts[name] = timeouts[name] === undefined ? timeoutMs : Math.min(timeouts[name], timeoutMs);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
return timeouts;
|
|
79
|
+
}
|
|
80
|
+
|
|
57
81
|
export function compileHarness(layers: HarnessLayer[]): CompiledHarness {
|
|
58
82
|
if (!layers.length) {
|
|
59
83
|
throw new Error('At least one harness layer is required.');
|
|
@@ -78,8 +102,12 @@ export function compileHarness(layers: HarnessLayer[]): CompiledHarness {
|
|
|
78
102
|
allow: intersectRestrictions(settings.map((layer) => layer.tools.allow)),
|
|
79
103
|
ask: union(settings.map((layer) => layer.tools.ask)),
|
|
80
104
|
deny: union(settings.map((layer) => layer.tools.deny)),
|
|
105
|
+
// Escalatable tools widen authority per approved call, so a child may only inherit the
|
|
106
|
+
// escalation surface every layer agrees on — same semantics as `allow`.
|
|
107
|
+
escalate: intersectRestrictions(settings.map((layer) => layer.tools.escalate)),
|
|
81
108
|
effects,
|
|
82
109
|
trustedPreHandlerTools: intersectRestrictions(settings.map((layer) => layer.tools.trustedPreHandlerTools)),
|
|
110
|
+
timeouts: mergeTimeouts(settings),
|
|
83
111
|
},
|
|
84
112
|
memory: {
|
|
85
113
|
enabled: settings.every((layer) => layer.memory.enabled),
|
|
@@ -103,6 +131,11 @@ export function compileHarness(layers: HarnessLayer[]): CompiledHarness {
|
|
|
103
131
|
maxTotalTokens: minimumNullable(settings.map((layer) => layer.limits.maxTotalTokens)),
|
|
104
132
|
maxCost: minimumNullable(settings.map((layer) => layer.limits.maxCost)),
|
|
105
133
|
},
|
|
134
|
+
context: {
|
|
135
|
+
spill: {
|
|
136
|
+
maxInlineBytes: minimumNullable(settings.map((layer) => layer.context.spill.maxInlineBytes)),
|
|
137
|
+
},
|
|
138
|
+
},
|
|
106
139
|
isolation: {
|
|
107
140
|
mode: settings.some((layer) => layer.isolation.mode === 'worktree') ? 'worktree' : 'none',
|
|
108
141
|
requireWorktree: settings.some((layer) => layer.isolation.requireWorktree),
|
|
@@ -114,6 +147,10 @@ export function compileHarness(layers: HarnessLayer[]): CompiledHarness {
|
|
|
114
147
|
tracingRetentionDays: Math.min(...settings.map((layer) => layer.observability.tracingRetentionDays)),
|
|
115
148
|
captureInputs: settings.every((layer) => layer.observability.captureInputs),
|
|
116
149
|
captureOutputs: settings.every((layer) => layer.observability.captureOutputs),
|
|
150
|
+
// Any layer demanding redaction wins; any layer restricting sharing wins. Telemetry safety
|
|
151
|
+
// is union-of-concerns, not intersection.
|
|
152
|
+
redactSecrets: settings.some((layer) => layer.observability.redactSecrets),
|
|
153
|
+
sharing: strictestSharing(settings.map((layer) => layer.observability.sharing)),
|
|
117
154
|
},
|
|
118
155
|
};
|
|
119
156
|
}
|