plugin-agent-orchestrator 1.1.4 → 1.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +204 -114
- package/dist/client/index.js +1 -1
- package/dist/client-v2/117.7565cff57c39318a.js +10 -0
- package/dist/client-v2/214.106635011859dede.js +10 -0
- package/dist/client-v2/418.41ed200e7d6da332.js +10 -0
- package/dist/client-v2/53.349fff8db80b19d4.js +10 -0
- package/dist/client-v2/981.e4a1f41cfc94a856.js +10 -0
- package/dist/client-v2/index.js +1 -1
- package/dist/externalVersion.js +7 -8
- package/dist/locale/en-US.json +226 -170
- package/dist/locale/vi-VN.json +226 -170
- package/dist/locale/zh-CN.json +226 -170
- package/dist/server/collections/agent-execution-spans.js +8 -0
- package/dist/server/collections/agent-loop-action-approvals.js +1 -0
- package/dist/server/collections/agent-loop-runs.js +7 -1
- package/dist/server/collections/agent-loop-steps.js +7 -0
- package/dist/server/collections/skill-executions.js +8 -0
- package/dist/server/plugin.js +3 -1
- package/dist/server/resources/agent-harness-profile.js +117 -0
- package/dist/server/resources/agent-knowledge-insights.js +1 -3
- package/dist/server/resources/agent-loop-approval.js +25 -0
- package/dist/server/resources/agent-loop.js +3 -1
- package/dist/server/services/AgentMemoryContextService.js +34 -0
- package/dist/server/services/HarnessCompiler.js +35 -2
- package/dist/server/services/HarnessProfileService.js +74 -9
- package/dist/server/services/HarnessSchema.js +30 -3
- package/dist/server/services/LoopPatternSchema.js +16 -1
- package/dist/server/services/LoopRunRepository.js +80 -0
- package/dist/server/services/LoopRunStateMachine.js +92 -3
- package/dist/server/services/LoopTriggerService.js +37 -0
- package/dist/server/services/LoopWorkerService.js +261 -23
- package/dist/server/services/NativeSubAgentObserver.js +165 -23
- package/dist/server/services/PluginAiRuntimeAdapter.js +16 -5
- package/dist/server/services/ToolLoopDetectionService.js +210 -0
- package/dist/server/services/VerificationService.js +66 -0
- package/dist/server/services/harness-runtime-policy.js +139 -0
- package/dist/server/skill-hub/plugin.js +36 -3
- package/dist/server/skill-hub/tasks/SkillExecutionTask.js +29 -0
- package/package.json +53 -53
- package/src/client/ApprovalsTab.tsx +2 -0
- package/src/client/OrchestratorSettings.tsx +11 -0
- package/src/client/skill-hub/components/GitSkillImport.tsx +3 -3
- package/src/client/skill-hub/components/SkillMetrics.tsx +15 -10
- package/src/client/skill-hub/index.tsx +0 -52
- package/src/client-v2/components/AgentRunsTab.tsx +576 -576
- package/src/client-v2/components/ApprovalsTab.tsx +337 -0
- package/src/client-v2/components/HarnessProfilesTab.tsx +385 -105
- package/src/client-v2/components/MemoryInspectorTab.tsx +141 -141
- package/src/client-v2/components/TracingTab.tsx +515 -515
- package/src/client-v2/hooks/useApiRequest.ts +141 -141
- package/src/client-v2/pages/ApprovalsPage.tsx +10 -0
- package/src/client-v2/plugin.tsx +6 -6
- package/src/client-v2/skill-hub/components/SkillMetrics.tsx +5 -2
- package/src/locale/en-US.json +226 -174
- package/src/locale/vi-VN.json +226 -174
- package/src/locale/zh-CN.json +226 -174
- package/src/server/__tests__/circuit-breaker.test.ts +248 -248
- package/src/server/__tests__/harness-compiler.test.ts +83 -0
- package/src/server/__tests__/harness-profile-service.test.ts +51 -0
- package/src/server/__tests__/harness-runtime-policy.test.ts +293 -0
- package/src/server/__tests__/harness-tool-plan.test.ts +24 -0
- package/src/server/__tests__/loop-approval-decision.test.ts +512 -0
- package/src/server/__tests__/loop-approval-expiry.test.ts +258 -0
- package/src/server/__tests__/loop-budget-service.test.ts +255 -255
- package/src/server/__tests__/loop-control-plane-backfill.test.ts +266 -266
- package/src/server/__tests__/loop-pattern-schema.test.ts +18 -1
- package/src/server/__tests__/loop-pattern-service.test.ts +148 -148
- package/src/server/__tests__/loop-run-state-machine.test.ts +11 -0
- package/src/server/__tests__/loop-trigger-service.test.ts +148 -0
- package/src/server/__tests__/loop-worker-claim.test.ts +341 -341
- package/src/server/__tests__/loop-worker-execution.test.ts +82 -1
- package/src/server/__tests__/native-sub-agent-observer.test.ts +288 -4
- package/src/server/__tests__/path-lock-service.test.ts +254 -254
- package/src/server/__tests__/skill-execution-task-policy.test.ts +140 -0
- package/src/server/__tests__/smoke.test.ts +162 -162
- package/src/server/__tests__/tool-loop-detection.test.ts +275 -0
- package/src/server/collections/agent-execution-spans.ts +8 -0
- package/src/server/collections/agent-harness-profile-versions.ts +26 -26
- package/src/server/collections/agent-harness-profiles.ts +71 -71
- package/src/server/collections/agent-loop-action-approvals.ts +1 -0
- package/src/server/collections/agent-loop-artifacts.ts +25 -25
- package/src/server/collections/agent-loop-circuit-states.ts +23 -23
- package/src/server/collections/agent-loop-control-settings.ts +21 -21
- package/src/server/collections/agent-loop-events.ts +98 -98
- package/src/server/collections/agent-loop-path-locks.ts +27 -27
- package/src/server/collections/agent-loop-patterns.ts +42 -42
- package/src/server/collections/agent-loop-runs.ts +7 -1
- package/src/server/collections/agent-loop-steps.ts +7 -0
- package/src/server/collections/agent-loop-usage-buckets.ts +22 -22
- package/src/server/collections/agent-loop-worktrees.ts +25 -25
- package/src/server/collections/skill-executions.ts +8 -0
- package/src/server/migrations/20260427000002-change-packages-to-text.ts +52 -52
- package/src/server/migrations/20260429000000-add-llm-fields.ts +47 -47
- package/src/server/migrations/20260429000001-fix-inputargs-json-to-text.ts +45 -45
- package/src/server/migrations/20260503000000-add-orchestrator-trace-fields.ts +33 -33
- package/src/server/migrations/20260524000000-add-agent-loop-fields-to-skill-executions.ts +31 -31
- package/src/server/migrations/20260803010000-backfill-loop-control-plane-data.ts +293 -293
- package/src/server/plugin.ts +3 -1
- package/src/server/resources/agent-harness-profile.ts +94 -0
- package/src/server/resources/agent-knowledge-insights.ts +5 -3
- package/src/server/resources/agent-loop-approval.ts +32 -0
- package/src/server/resources/agent-loop-artifact.ts +56 -56
- package/src/server/resources/agent-loop-events-stream.ts +105 -105
- package/src/server/resources/agent-loop.ts +3 -1
- package/src/server/resources/resource-helpers.ts +33 -33
- package/src/server/resources/tracing.ts +266 -266
- package/src/server/services/AgentMemoryContextService.ts +44 -0
- package/src/server/services/DurableCircuitBreakerService.ts +239 -239
- package/src/server/services/ExecutionSpanService.ts +2 -0
- package/src/server/services/HarnessCompiler.ts +37 -0
- package/src/server/services/HarnessProfileService.ts +92 -10
- package/src/server/services/HarnessSchema.ts +31 -0
- package/src/server/services/LoopBudgetService.ts +275 -275
- package/src/server/services/LoopControlService.ts +124 -124
- package/src/server/services/LoopPatternSchema.ts +19 -1
- package/src/server/services/LoopPatternService.ts +143 -143
- package/src/server/services/LoopRunRepository.ts +99 -0
- package/src/server/services/LoopRunStateMachine.ts +100 -3
- package/src/server/services/LoopSchedulerService.ts +199 -199
- package/src/server/services/LoopTriggerService.ts +42 -0
- package/src/server/services/LoopWorkerService.ts +327 -27
- package/src/server/services/NativeSubAgentObserver.ts +186 -21
- package/src/server/services/PathLockService.ts +321 -321
- package/src/server/services/PluginAiRuntimeAdapter.ts +24 -7
- package/src/server/services/ToolLoopDetectionService.ts +213 -0
- package/src/server/services/VerificationSchema.ts +31 -31
- package/src/server/services/VerificationService.ts +81 -2
- package/src/server/services/harness-runtime-policy.ts +141 -0
- package/src/server/skill-hub/plugin.ts +41 -3
- package/src/server/skill-hub/services/RegistrySkillSnapshotService.ts +235 -235
- package/src/server/skill-hub/services/__tests__/RegistrySkillSnapshotService.test.ts +61 -61
- package/src/server/skill-hub/tasks/SkillExecutionTask.ts +43 -2
- package/src/server/utils/ensure-indexes.ts +96 -96
- package/src/server/utils/logging.ts +62 -62
- package/src/server/utils/loop-pattern-context.ts +20 -20
- package/dist/client-v2/214.b063f33ba044a808.js +0 -10
- package/dist/client-v2/373.f5d5292e53c4e832.js +0 -10
- package/dist/client-v2/418.58fa350f5496efd3.js +0 -10
- package/dist/client-v2/53.9a10455323390f14.js +0 -10
- package/dist/client-v2/677.a991ce0250ff5c77.js +0 -10
- package/dist/client-v2/925.e5925bf21ed68e7a.js +0 -10
- package/src/client/AIEmployeeSelect.tsx +0 -47
- package/src/client/AIEmployeesContext.tsx +0 -102
- package/src/client/RulesTab.tsx +0 -2
- package/src/client/TracingTab.tsx +0 -436
- package/src/client/skill-hub/components/ExecutionProgress.tsx +0 -51
- package/src/client-v2/components/RulesTab.tsx +0 -2
- package/src/client-v2/pages/AgentRunsPage.tsx +0 -13
|
@@ -1,255 +1,255 @@
|
|
|
1
|
-
import type { Database } from '@nocobase/database';
|
|
2
|
-
import { describe, expect, it } from 'vitest';
|
|
3
|
-
import { LoopBudgetService } from '../services/LoopBudgetService';
|
|
4
|
-
import { loopPatternPolicySchema } from '../services/LoopPatternSchema';
|
|
5
|
-
|
|
6
|
-
type Row = Record<string, unknown>;
|
|
7
|
-
|
|
8
|
-
function matches(row: Row, filter: Row) {
|
|
9
|
-
return Object.entries(filter).every(([key, value]) => row[key] === value);
|
|
10
|
-
}
|
|
11
|
-
|
|
12
|
-
class MemoryRepository {
|
|
13
|
-
private nextId = 1;
|
|
14
|
-
|
|
15
|
-
constructor(readonly rows: Row[] = []) {
|
|
16
|
-
this.resetSequence();
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
async find(options: Row = {}) {
|
|
20
|
-
const filter = (options.filter as Row | undefined) || {};
|
|
21
|
-
return this.rows.filter((row) => matches(row, filter));
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
async findOne(options: Row) {
|
|
25
|
-
if (options.filterByTk !== undefined) return this.rows.find((row) => row.id === options.filterByTk) || null;
|
|
26
|
-
return this.rows.find((row) => matches(row, (options.filter as Row | undefined) || {})) || null;
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
async create(options: Row) {
|
|
30
|
-
const row = { id: this.nextId++, ...((options.values as Row | undefined) || {}) };
|
|
31
|
-
this.rows.push(row);
|
|
32
|
-
return row;
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
async update(options: Row) {
|
|
36
|
-
const values = (options.values as Row | undefined) || {};
|
|
37
|
-
for (const row of this.rows) {
|
|
38
|
-
if (options.filterByTk !== undefined && row.id === options.filterByTk) Object.assign(row, values);
|
|
39
|
-
}
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
restore(rows: Row[]) {
|
|
43
|
-
this.rows.splice(0, this.rows.length, ...structuredClone(rows));
|
|
44
|
-
this.resetSequence();
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
private resetSequence() {
|
|
48
|
-
this.nextId = this.rows.reduce((maximum, row) => Math.max(maximum, Number(row.id) || 0), 0) + 1;
|
|
49
|
-
}
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
class SerialLock {
|
|
53
|
-
readonly keys: string[] = [];
|
|
54
|
-
private readonly tails = new Map<string, Promise<void>>();
|
|
55
|
-
|
|
56
|
-
async runExclusive<T>(key: string, operation: () => Promise<T>) {
|
|
57
|
-
this.keys.push(key);
|
|
58
|
-
const previous = this.tails.get(key) || Promise.resolve();
|
|
59
|
-
let release = () => {};
|
|
60
|
-
const pending = new Promise<void>((resolve) => {
|
|
61
|
-
release = resolve;
|
|
62
|
-
});
|
|
63
|
-
const tail = previous.then(() => pending);
|
|
64
|
-
this.tails.set(key, tail);
|
|
65
|
-
await previous;
|
|
66
|
-
try {
|
|
67
|
-
return await operation();
|
|
68
|
-
} finally {
|
|
69
|
-
release();
|
|
70
|
-
if (this.tails.get(key) === tail) this.tails.delete(key);
|
|
71
|
-
}
|
|
72
|
-
}
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
function run(id: number, patternId: number, values: Row = {}) {
|
|
76
|
-
return {
|
|
77
|
-
id,
|
|
78
|
-
patternId,
|
|
79
|
-
runtimeVersion: 'control-plane-v2',
|
|
80
|
-
invocationCount: 0,
|
|
81
|
-
toolCallCount: 0,
|
|
82
|
-
delegationCount: 0,
|
|
83
|
-
verificationCount: 0,
|
|
84
|
-
totalInputTokens: 0,
|
|
85
|
-
totalOutputTokens: 0,
|
|
86
|
-
totalTokens: 0,
|
|
87
|
-
totalCost: 0,
|
|
88
|
-
...values,
|
|
89
|
-
};
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
function createService(input: { runs?: Row[]; buckets?: Row[]; control?: Row } = {}) {
|
|
93
|
-
const repositories = new Map<string, MemoryRepository>([
|
|
94
|
-
['agentLoopRuns', new MemoryRepository(input.runs || [run(1, 1)])],
|
|
95
|
-
['agentLoopUsageBuckets', new MemoryRepository(input.buckets || [])],
|
|
96
|
-
[
|
|
97
|
-
'agentLoopControlSettings',
|
|
98
|
-
new MemoryRepository([
|
|
99
|
-
{
|
|
100
|
-
id: 1,
|
|
101
|
-
key: 'global',
|
|
102
|
-
state: 'running',
|
|
103
|
-
dailyMaxTokens: null,
|
|
104
|
-
dailyMaxCost: null,
|
|
105
|
-
...(input.control || {}),
|
|
106
|
-
},
|
|
107
|
-
]),
|
|
108
|
-
],
|
|
109
|
-
]);
|
|
110
|
-
const transaction = { LOCK: { UPDATE: 'UPDATE' } };
|
|
111
|
-
const database = {
|
|
112
|
-
getRepository(name: string) {
|
|
113
|
-
const repository = repositories.get(name);
|
|
114
|
-
if (!repository) throw new Error(`Unexpected repository ${name}`);
|
|
115
|
-
return repository;
|
|
116
|
-
},
|
|
117
|
-
sequelize: {
|
|
118
|
-
async transaction<T>(callback: (value: typeof transaction) => Promise<T>) {
|
|
119
|
-
const before = new Map(
|
|
120
|
-
Array.from(repositories, ([name, repository]) => [name, structuredClone(repository.rows)]),
|
|
121
|
-
);
|
|
122
|
-
try {
|
|
123
|
-
return await callback(transaction);
|
|
124
|
-
} catch (error) {
|
|
125
|
-
for (const [name, rows] of before) repositories.get(name)?.restore(rows);
|
|
126
|
-
throw error;
|
|
127
|
-
}
|
|
128
|
-
},
|
|
129
|
-
},
|
|
130
|
-
};
|
|
131
|
-
const lock = new SerialLock();
|
|
132
|
-
return {
|
|
133
|
-
lock,
|
|
134
|
-
repositories,
|
|
135
|
-
service: new LoopBudgetService(database as unknown as Database, lock),
|
|
136
|
-
};
|
|
137
|
-
}
|
|
138
|
-
|
|
139
|
-
function policy(input: Record<string, unknown> = {}) {
|
|
140
|
-
return loopPatternPolicySchema.parse(input);
|
|
141
|
-
}
|
|
142
|
-
|
|
143
|
-
const now = new Date('2026-08-03T12:00:00.000Z');
|
|
144
|
-
|
|
145
|
-
describe('LoopBudgetService', () => {
|
|
146
|
-
it('reserves run and daily usage atomically without undercounting token totals', async () => {
|
|
147
|
-
const { lock, repositories, service } = createService();
|
|
148
|
-
const result = await service.reserve({
|
|
149
|
-
runId: 1,
|
|
150
|
-
patternId: 1,
|
|
151
|
-
policy: policy(),
|
|
152
|
-
delta: { invocations: 1, toolCalls: 2, inputTokens: 30, outputTokens: 20, totalTokens: 10, cost: 0.25 },
|
|
153
|
-
now,
|
|
154
|
-
});
|
|
155
|
-
|
|
156
|
-
expect(result).toMatchObject({
|
|
157
|
-
bucketDate: '2026-08-03',
|
|
158
|
-
reserved: { invocations: 1, toolCalls: 2, inputTokens: 30, outputTokens: 20, totalTokens: 50, cost: 0.25 },
|
|
159
|
-
});
|
|
160
|
-
expect(repositories.get('agentLoopRuns')?.rows[0]).toMatchObject({
|
|
161
|
-
invocationCount: 1,
|
|
162
|
-
toolCallCount: 2,
|
|
163
|
-
totalInputTokens: 30,
|
|
164
|
-
totalOutputTokens: 20,
|
|
165
|
-
totalTokens: 50,
|
|
166
|
-
totalCost: 0.25,
|
|
167
|
-
});
|
|
168
|
-
expect(repositories.get('agentLoopUsageBuckets')?.rows[0]).toMatchObject({
|
|
169
|
-
patternId: 1,
|
|
170
|
-
bucketDate: '2026-08-03',
|
|
171
|
-
invocationCount: 1,
|
|
172
|
-
toolCallCount: 2,
|
|
173
|
-
totalTokens: 50,
|
|
174
|
-
totalCost: 0.25,
|
|
175
|
-
});
|
|
176
|
-
expect(lock.keys).toEqual(['agent-loop:budget:global:2026-08-03']);
|
|
177
|
-
});
|
|
178
|
-
|
|
179
|
-
it('rolls back both rows when a per-run or pattern-daily limit rejects the reservation', async () => {
|
|
180
|
-
const { repositories, service } = createService({
|
|
181
|
-
runs: [run(1, 1, { invocationCount: 1, totalTokens: 5 })],
|
|
182
|
-
buckets: [{ id: 1, patternId: 1, bucketDate: '2026-08-03', invocationCount: 1, totalTokens: 5 }],
|
|
183
|
-
});
|
|
184
|
-
const runPolicy = policy({ perRun: { maxInvocations: 1 } });
|
|
185
|
-
await expect(
|
|
186
|
-
service.reserve({ runId: 1, patternId: 1, policy: runPolicy, delta: { invocations: 1 }, now }),
|
|
187
|
-
).rejects.toThrow('Per-run invocation budget exceeded');
|
|
188
|
-
|
|
189
|
-
const dailyPolicy = policy({ daily: { maxTokens: 5 } });
|
|
190
|
-
await expect(
|
|
191
|
-
service.reserve({ runId: 1, patternId: 1, policy: dailyPolicy, delta: { totalTokens: 1 }, now }),
|
|
192
|
-
).rejects.toThrow('Daily token budget exceeded');
|
|
193
|
-
|
|
194
|
-
expect(repositories.get('agentLoopRuns')?.rows[0]).toMatchObject({ invocationCount: 1, totalTokens: 5 });
|
|
195
|
-
expect(repositories.get('agentLoopUsageBuckets')?.rows[0]).toMatchObject({ invocationCount: 1, totalTokens: 5 });
|
|
196
|
-
});
|
|
197
|
-
|
|
198
|
-
it('enforces global daily caps across different patterns', async () => {
|
|
199
|
-
const { repositories, service } = createService({
|
|
200
|
-
runs: [run(1, 1), run(2, 2)],
|
|
201
|
-
buckets: [
|
|
202
|
-
{ id: 1, patternId: 1, bucketDate: '2026-08-03', totalTokens: 60, totalCost: 1 },
|
|
203
|
-
{ id: 2, patternId: 2, bucketDate: '2026-08-03', totalTokens: 30, totalCost: 1 },
|
|
204
|
-
],
|
|
205
|
-
control: { dailyMaxTokens: 100, dailyMaxCost: 2.5 },
|
|
206
|
-
});
|
|
207
|
-
|
|
208
|
-
await expect(
|
|
209
|
-
service.reserve({ runId: 2, patternId: 2, policy: policy(), delta: { totalTokens: 11 }, now }),
|
|
210
|
-
).rejects.toThrow('Global daily token budget exceeded');
|
|
211
|
-
await expect(
|
|
212
|
-
service.reserve({ runId: 2, patternId: 2, policy: policy(), delta: { cost: 0.6 }, now }),
|
|
213
|
-
).rejects.toThrow('Global daily cost budget exceeded');
|
|
214
|
-
|
|
215
|
-
expect(repositories.get('agentLoopRuns')?.rows[1]).toMatchObject({ totalTokens: 0, totalCost: 0 });
|
|
216
|
-
expect(repositories.get('agentLoopUsageBuckets')?.rows[1]).toMatchObject({ totalTokens: 30, totalCost: 1 });
|
|
217
|
-
});
|
|
218
|
-
|
|
219
|
-
it('serializes racing reservations so only one can cross the remaining budget boundary', async () => {
|
|
220
|
-
const { repositories, service } = createService();
|
|
221
|
-
const limited = policy({ perRun: { maxTokens: 10 }, daily: { maxTokens: 10 } });
|
|
222
|
-
const requests = await Promise.allSettled([
|
|
223
|
-
service.reserve({ runId: 1, patternId: 1, policy: limited, delta: { totalTokens: 6 }, now }),
|
|
224
|
-
service.reserve({ runId: 1, patternId: 1, policy: limited, delta: { totalTokens: 6 }, now }),
|
|
225
|
-
]);
|
|
226
|
-
|
|
227
|
-
expect(requests.filter((request) => request.status === 'fulfilled')).toHaveLength(1);
|
|
228
|
-
expect(requests.filter((request) => request.status === 'rejected')).toHaveLength(1);
|
|
229
|
-
expect(repositories.get('agentLoopRuns')?.rows[0].totalTokens).toBe(6);
|
|
230
|
-
expect(repositories.get('agentLoopUsageBuckets')?.rows[0].totalTokens).toBe(6);
|
|
231
|
-
});
|
|
232
|
-
|
|
233
|
-
it('fails closed for invalid deltas, historical runs, mismatched patterns, and paused control', async () => {
|
|
234
|
-
const invalid = createService();
|
|
235
|
-
await expect(
|
|
236
|
-
invalid.service.reserve({ runId: 1, patternId: 1, policy: policy(), delta: { toolCalls: 0.5 }, now }),
|
|
237
|
-
).rejects.toThrow('nonnegative integer');
|
|
238
|
-
expect(invalid.repositories.get('agentLoopUsageBuckets')?.rows).toHaveLength(0);
|
|
239
|
-
|
|
240
|
-
const historical = createService({ runs: [run(1, 1, { runtimeVersion: 'legacy-plan-v1' })] });
|
|
241
|
-
await expect(
|
|
242
|
-
historical.service.reserve({ runId: 1, patternId: 1, policy: policy(), delta: { invocations: 1 }, now }),
|
|
243
|
-
).rejects.toThrow('Historical plan-era');
|
|
244
|
-
|
|
245
|
-
const mismatched = createService();
|
|
246
|
-
await expect(
|
|
247
|
-
mismatched.service.reserve({ runId: 1, patternId: 2, policy: policy(), delta: { invocations: 1 }, now }),
|
|
248
|
-
).rejects.toThrow('does not belong to pattern');
|
|
249
|
-
|
|
250
|
-
const paused = createService({ control: { state: 'paused', reason: 'Maintenance' } });
|
|
251
|
-
await expect(
|
|
252
|
-
paused.service.reserve({ runId: 1, patternId: 1, policy: policy(), delta: { invocations: 1 }, now }),
|
|
253
|
-
).rejects.toThrow('Maintenance');
|
|
254
|
-
});
|
|
255
|
-
});
|
|
1
|
+
import type { Database } from '@nocobase/database';
|
|
2
|
+
import { describe, expect, it } from 'vitest';
|
|
3
|
+
import { LoopBudgetService } from '../services/LoopBudgetService';
|
|
4
|
+
import { loopPatternPolicySchema } from '../services/LoopPatternSchema';
|
|
5
|
+
|
|
6
|
+
type Row = Record<string, unknown>;
|
|
7
|
+
|
|
8
|
+
function matches(row: Row, filter: Row) {
|
|
9
|
+
return Object.entries(filter).every(([key, value]) => row[key] === value);
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
class MemoryRepository {
|
|
13
|
+
private nextId = 1;
|
|
14
|
+
|
|
15
|
+
constructor(readonly rows: Row[] = []) {
|
|
16
|
+
this.resetSequence();
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
async find(options: Row = {}) {
|
|
20
|
+
const filter = (options.filter as Row | undefined) || {};
|
|
21
|
+
return this.rows.filter((row) => matches(row, filter));
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
async findOne(options: Row) {
|
|
25
|
+
if (options.filterByTk !== undefined) return this.rows.find((row) => row.id === options.filterByTk) || null;
|
|
26
|
+
return this.rows.find((row) => matches(row, (options.filter as Row | undefined) || {})) || null;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
async create(options: Row) {
|
|
30
|
+
const row = { id: this.nextId++, ...((options.values as Row | undefined) || {}) };
|
|
31
|
+
this.rows.push(row);
|
|
32
|
+
return row;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
async update(options: Row) {
|
|
36
|
+
const values = (options.values as Row | undefined) || {};
|
|
37
|
+
for (const row of this.rows) {
|
|
38
|
+
if (options.filterByTk !== undefined && row.id === options.filterByTk) Object.assign(row, values);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
restore(rows: Row[]) {
|
|
43
|
+
this.rows.splice(0, this.rows.length, ...structuredClone(rows));
|
|
44
|
+
this.resetSequence();
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
private resetSequence() {
|
|
48
|
+
this.nextId = this.rows.reduce((maximum, row) => Math.max(maximum, Number(row.id) || 0), 0) + 1;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
class SerialLock {
|
|
53
|
+
readonly keys: string[] = [];
|
|
54
|
+
private readonly tails = new Map<string, Promise<void>>();
|
|
55
|
+
|
|
56
|
+
async runExclusive<T>(key: string, operation: () => Promise<T>) {
|
|
57
|
+
this.keys.push(key);
|
|
58
|
+
const previous = this.tails.get(key) || Promise.resolve();
|
|
59
|
+
let release = () => {};
|
|
60
|
+
const pending = new Promise<void>((resolve) => {
|
|
61
|
+
release = resolve;
|
|
62
|
+
});
|
|
63
|
+
const tail = previous.then(() => pending);
|
|
64
|
+
this.tails.set(key, tail);
|
|
65
|
+
await previous;
|
|
66
|
+
try {
|
|
67
|
+
return await operation();
|
|
68
|
+
} finally {
|
|
69
|
+
release();
|
|
70
|
+
if (this.tails.get(key) === tail) this.tails.delete(key);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function run(id: number, patternId: number, values: Row = {}) {
|
|
76
|
+
return {
|
|
77
|
+
id,
|
|
78
|
+
patternId,
|
|
79
|
+
runtimeVersion: 'control-plane-v2',
|
|
80
|
+
invocationCount: 0,
|
|
81
|
+
toolCallCount: 0,
|
|
82
|
+
delegationCount: 0,
|
|
83
|
+
verificationCount: 0,
|
|
84
|
+
totalInputTokens: 0,
|
|
85
|
+
totalOutputTokens: 0,
|
|
86
|
+
totalTokens: 0,
|
|
87
|
+
totalCost: 0,
|
|
88
|
+
...values,
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function createService(input: { runs?: Row[]; buckets?: Row[]; control?: Row } = {}) {
|
|
93
|
+
const repositories = new Map<string, MemoryRepository>([
|
|
94
|
+
['agentLoopRuns', new MemoryRepository(input.runs || [run(1, 1)])],
|
|
95
|
+
['agentLoopUsageBuckets', new MemoryRepository(input.buckets || [])],
|
|
96
|
+
[
|
|
97
|
+
'agentLoopControlSettings',
|
|
98
|
+
new MemoryRepository([
|
|
99
|
+
{
|
|
100
|
+
id: 1,
|
|
101
|
+
key: 'global',
|
|
102
|
+
state: 'running',
|
|
103
|
+
dailyMaxTokens: null,
|
|
104
|
+
dailyMaxCost: null,
|
|
105
|
+
...(input.control || {}),
|
|
106
|
+
},
|
|
107
|
+
]),
|
|
108
|
+
],
|
|
109
|
+
]);
|
|
110
|
+
const transaction = { LOCK: { UPDATE: 'UPDATE' } };
|
|
111
|
+
const database = {
|
|
112
|
+
getRepository(name: string) {
|
|
113
|
+
const repository = repositories.get(name);
|
|
114
|
+
if (!repository) throw new Error(`Unexpected repository ${name}`);
|
|
115
|
+
return repository;
|
|
116
|
+
},
|
|
117
|
+
sequelize: {
|
|
118
|
+
async transaction<T>(callback: (value: typeof transaction) => Promise<T>) {
|
|
119
|
+
const before = new Map(
|
|
120
|
+
Array.from(repositories, ([name, repository]) => [name, structuredClone(repository.rows)]),
|
|
121
|
+
);
|
|
122
|
+
try {
|
|
123
|
+
return await callback(transaction);
|
|
124
|
+
} catch (error) {
|
|
125
|
+
for (const [name, rows] of before) repositories.get(name)?.restore(rows);
|
|
126
|
+
throw error;
|
|
127
|
+
}
|
|
128
|
+
},
|
|
129
|
+
},
|
|
130
|
+
};
|
|
131
|
+
const lock = new SerialLock();
|
|
132
|
+
return {
|
|
133
|
+
lock,
|
|
134
|
+
repositories,
|
|
135
|
+
service: new LoopBudgetService(database as unknown as Database, lock),
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
function policy(input: Record<string, unknown> = {}) {
|
|
140
|
+
return loopPatternPolicySchema.parse(input);
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
const now = new Date('2026-08-03T12:00:00.000Z');
|
|
144
|
+
|
|
145
|
+
describe('LoopBudgetService', () => {
|
|
146
|
+
it('reserves run and daily usage atomically without undercounting token totals', async () => {
|
|
147
|
+
const { lock, repositories, service } = createService();
|
|
148
|
+
const result = await service.reserve({
|
|
149
|
+
runId: 1,
|
|
150
|
+
patternId: 1,
|
|
151
|
+
policy: policy(),
|
|
152
|
+
delta: { invocations: 1, toolCalls: 2, inputTokens: 30, outputTokens: 20, totalTokens: 10, cost: 0.25 },
|
|
153
|
+
now,
|
|
154
|
+
});
|
|
155
|
+
|
|
156
|
+
expect(result).toMatchObject({
|
|
157
|
+
bucketDate: '2026-08-03',
|
|
158
|
+
reserved: { invocations: 1, toolCalls: 2, inputTokens: 30, outputTokens: 20, totalTokens: 50, cost: 0.25 },
|
|
159
|
+
});
|
|
160
|
+
expect(repositories.get('agentLoopRuns')?.rows[0]).toMatchObject({
|
|
161
|
+
invocationCount: 1,
|
|
162
|
+
toolCallCount: 2,
|
|
163
|
+
totalInputTokens: 30,
|
|
164
|
+
totalOutputTokens: 20,
|
|
165
|
+
totalTokens: 50,
|
|
166
|
+
totalCost: 0.25,
|
|
167
|
+
});
|
|
168
|
+
expect(repositories.get('agentLoopUsageBuckets')?.rows[0]).toMatchObject({
|
|
169
|
+
patternId: 1,
|
|
170
|
+
bucketDate: '2026-08-03',
|
|
171
|
+
invocationCount: 1,
|
|
172
|
+
toolCallCount: 2,
|
|
173
|
+
totalTokens: 50,
|
|
174
|
+
totalCost: 0.25,
|
|
175
|
+
});
|
|
176
|
+
expect(lock.keys).toEqual(['agent-loop:budget:global:2026-08-03']);
|
|
177
|
+
});
|
|
178
|
+
|
|
179
|
+
it('rolls back both rows when a per-run or pattern-daily limit rejects the reservation', async () => {
|
|
180
|
+
const { repositories, service } = createService({
|
|
181
|
+
runs: [run(1, 1, { invocationCount: 1, totalTokens: 5 })],
|
|
182
|
+
buckets: [{ id: 1, patternId: 1, bucketDate: '2026-08-03', invocationCount: 1, totalTokens: 5 }],
|
|
183
|
+
});
|
|
184
|
+
const runPolicy = policy({ perRun: { maxInvocations: 1 } });
|
|
185
|
+
await expect(
|
|
186
|
+
service.reserve({ runId: 1, patternId: 1, policy: runPolicy, delta: { invocations: 1 }, now }),
|
|
187
|
+
).rejects.toThrow('Per-run invocation budget exceeded');
|
|
188
|
+
|
|
189
|
+
const dailyPolicy = policy({ daily: { maxTokens: 5 } });
|
|
190
|
+
await expect(
|
|
191
|
+
service.reserve({ runId: 1, patternId: 1, policy: dailyPolicy, delta: { totalTokens: 1 }, now }),
|
|
192
|
+
).rejects.toThrow('Daily token budget exceeded');
|
|
193
|
+
|
|
194
|
+
expect(repositories.get('agentLoopRuns')?.rows[0]).toMatchObject({ invocationCount: 1, totalTokens: 5 });
|
|
195
|
+
expect(repositories.get('agentLoopUsageBuckets')?.rows[0]).toMatchObject({ invocationCount: 1, totalTokens: 5 });
|
|
196
|
+
});
|
|
197
|
+
|
|
198
|
+
it('enforces global daily caps across different patterns', async () => {
|
|
199
|
+
const { repositories, service } = createService({
|
|
200
|
+
runs: [run(1, 1), run(2, 2)],
|
|
201
|
+
buckets: [
|
|
202
|
+
{ id: 1, patternId: 1, bucketDate: '2026-08-03', totalTokens: 60, totalCost: 1 },
|
|
203
|
+
{ id: 2, patternId: 2, bucketDate: '2026-08-03', totalTokens: 30, totalCost: 1 },
|
|
204
|
+
],
|
|
205
|
+
control: { dailyMaxTokens: 100, dailyMaxCost: 2.5 },
|
|
206
|
+
});
|
|
207
|
+
|
|
208
|
+
await expect(
|
|
209
|
+
service.reserve({ runId: 2, patternId: 2, policy: policy(), delta: { totalTokens: 11 }, now }),
|
|
210
|
+
).rejects.toThrow('Global daily token budget exceeded');
|
|
211
|
+
await expect(
|
|
212
|
+
service.reserve({ runId: 2, patternId: 2, policy: policy(), delta: { cost: 0.6 }, now }),
|
|
213
|
+
).rejects.toThrow('Global daily cost budget exceeded');
|
|
214
|
+
|
|
215
|
+
expect(repositories.get('agentLoopRuns')?.rows[1]).toMatchObject({ totalTokens: 0, totalCost: 0 });
|
|
216
|
+
expect(repositories.get('agentLoopUsageBuckets')?.rows[1]).toMatchObject({ totalTokens: 30, totalCost: 1 });
|
|
217
|
+
});
|
|
218
|
+
|
|
219
|
+
it('serializes racing reservations so only one can cross the remaining budget boundary', async () => {
|
|
220
|
+
const { repositories, service } = createService();
|
|
221
|
+
const limited = policy({ perRun: { maxTokens: 10 }, daily: { maxTokens: 10 } });
|
|
222
|
+
const requests = await Promise.allSettled([
|
|
223
|
+
service.reserve({ runId: 1, patternId: 1, policy: limited, delta: { totalTokens: 6 }, now }),
|
|
224
|
+
service.reserve({ runId: 1, patternId: 1, policy: limited, delta: { totalTokens: 6 }, now }),
|
|
225
|
+
]);
|
|
226
|
+
|
|
227
|
+
expect(requests.filter((request) => request.status === 'fulfilled')).toHaveLength(1);
|
|
228
|
+
expect(requests.filter((request) => request.status === 'rejected')).toHaveLength(1);
|
|
229
|
+
expect(repositories.get('agentLoopRuns')?.rows[0].totalTokens).toBe(6);
|
|
230
|
+
expect(repositories.get('agentLoopUsageBuckets')?.rows[0].totalTokens).toBe(6);
|
|
231
|
+
});
|
|
232
|
+
|
|
233
|
+
it('fails closed for invalid deltas, historical runs, mismatched patterns, and paused control', async () => {
|
|
234
|
+
const invalid = createService();
|
|
235
|
+
await expect(
|
|
236
|
+
invalid.service.reserve({ runId: 1, patternId: 1, policy: policy(), delta: { toolCalls: 0.5 }, now }),
|
|
237
|
+
).rejects.toThrow('nonnegative integer');
|
|
238
|
+
expect(invalid.repositories.get('agentLoopUsageBuckets')?.rows).toHaveLength(0);
|
|
239
|
+
|
|
240
|
+
const historical = createService({ runs: [run(1, 1, { runtimeVersion: 'legacy-plan-v1' })] });
|
|
241
|
+
await expect(
|
|
242
|
+
historical.service.reserve({ runId: 1, patternId: 1, policy: policy(), delta: { invocations: 1 }, now }),
|
|
243
|
+
).rejects.toThrow('Historical plan-era');
|
|
244
|
+
|
|
245
|
+
const mismatched = createService();
|
|
246
|
+
await expect(
|
|
247
|
+
mismatched.service.reserve({ runId: 1, patternId: 2, policy: policy(), delta: { invocations: 1 }, now }),
|
|
248
|
+
).rejects.toThrow('does not belong to pattern');
|
|
249
|
+
|
|
250
|
+
const paused = createService({ control: { state: 'paused', reason: 'Maintenance' } });
|
|
251
|
+
await expect(
|
|
252
|
+
paused.service.reserve({ runId: 1, patternId: 1, policy: policy(), delta: { invocations: 1 }, now }),
|
|
253
|
+
).rejects.toThrow('Maintenance');
|
|
254
|
+
});
|
|
255
|
+
});
|