plugin-agent-orchestrator 1.1.4 → 1.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +204 -114
- package/dist/client/index.js +1 -1
- package/dist/client-v2/117.7565cff57c39318a.js +10 -0
- package/dist/client-v2/214.106635011859dede.js +10 -0
- package/dist/client-v2/418.41ed200e7d6da332.js +10 -0
- package/dist/client-v2/53.349fff8db80b19d4.js +10 -0
- package/dist/client-v2/981.e4a1f41cfc94a856.js +10 -0
- package/dist/client-v2/index.js +1 -1
- package/dist/externalVersion.js +7 -8
- package/dist/locale/en-US.json +226 -170
- package/dist/locale/vi-VN.json +226 -170
- package/dist/locale/zh-CN.json +226 -170
- package/dist/server/collections/agent-execution-spans.js +8 -0
- package/dist/server/collections/agent-loop-action-approvals.js +1 -0
- package/dist/server/collections/agent-loop-runs.js +7 -1
- package/dist/server/collections/agent-loop-steps.js +7 -0
- package/dist/server/collections/skill-executions.js +8 -0
- package/dist/server/plugin.js +3 -1
- package/dist/server/resources/agent-harness-profile.js +117 -0
- package/dist/server/resources/agent-knowledge-insights.js +1 -3
- package/dist/server/resources/agent-loop-approval.js +25 -0
- package/dist/server/resources/agent-loop.js +3 -1
- package/dist/server/services/AgentMemoryContextService.js +34 -0
- package/dist/server/services/HarnessCompiler.js +35 -2
- package/dist/server/services/HarnessProfileService.js +74 -9
- package/dist/server/services/HarnessSchema.js +30 -3
- package/dist/server/services/LoopPatternSchema.js +16 -1
- package/dist/server/services/LoopRunRepository.js +80 -0
- package/dist/server/services/LoopRunStateMachine.js +92 -3
- package/dist/server/services/LoopTriggerService.js +37 -0
- package/dist/server/services/LoopWorkerService.js +261 -23
- package/dist/server/services/NativeSubAgentObserver.js +165 -23
- package/dist/server/services/PluginAiRuntimeAdapter.js +16 -5
- package/dist/server/services/ToolLoopDetectionService.js +210 -0
- package/dist/server/services/VerificationService.js +66 -0
- package/dist/server/services/harness-runtime-policy.js +139 -0
- package/dist/server/skill-hub/plugin.js +36 -3
- package/dist/server/skill-hub/tasks/SkillExecutionTask.js +29 -0
- package/package.json +53 -53
- package/src/client/ApprovalsTab.tsx +2 -0
- package/src/client/OrchestratorSettings.tsx +11 -0
- package/src/client/skill-hub/components/GitSkillImport.tsx +3 -3
- package/src/client/skill-hub/components/SkillMetrics.tsx +15 -10
- package/src/client/skill-hub/index.tsx +0 -52
- package/src/client-v2/components/AgentRunsTab.tsx +576 -576
- package/src/client-v2/components/ApprovalsTab.tsx +337 -0
- package/src/client-v2/components/HarnessProfilesTab.tsx +385 -105
- package/src/client-v2/components/MemoryInspectorTab.tsx +141 -141
- package/src/client-v2/components/TracingTab.tsx +515 -515
- package/src/client-v2/hooks/useApiRequest.ts +141 -141
- package/src/client-v2/pages/ApprovalsPage.tsx +10 -0
- package/src/client-v2/plugin.tsx +6 -6
- package/src/client-v2/skill-hub/components/SkillMetrics.tsx +5 -2
- package/src/locale/en-US.json +226 -174
- package/src/locale/vi-VN.json +226 -174
- package/src/locale/zh-CN.json +226 -174
- package/src/server/__tests__/circuit-breaker.test.ts +248 -248
- package/src/server/__tests__/harness-compiler.test.ts +83 -0
- package/src/server/__tests__/harness-profile-service.test.ts +51 -0
- package/src/server/__tests__/harness-runtime-policy.test.ts +293 -0
- package/src/server/__tests__/harness-tool-plan.test.ts +24 -0
- package/src/server/__tests__/loop-approval-decision.test.ts +512 -0
- package/src/server/__tests__/loop-approval-expiry.test.ts +258 -0
- package/src/server/__tests__/loop-budget-service.test.ts +255 -255
- package/src/server/__tests__/loop-control-plane-backfill.test.ts +266 -266
- package/src/server/__tests__/loop-pattern-schema.test.ts +18 -1
- package/src/server/__tests__/loop-pattern-service.test.ts +148 -148
- package/src/server/__tests__/loop-run-state-machine.test.ts +11 -0
- package/src/server/__tests__/loop-trigger-service.test.ts +148 -0
- package/src/server/__tests__/loop-worker-claim.test.ts +341 -341
- package/src/server/__tests__/loop-worker-execution.test.ts +82 -1
- package/src/server/__tests__/native-sub-agent-observer.test.ts +288 -4
- package/src/server/__tests__/path-lock-service.test.ts +254 -254
- package/src/server/__tests__/skill-execution-task-policy.test.ts +140 -0
- package/src/server/__tests__/smoke.test.ts +162 -162
- package/src/server/__tests__/tool-loop-detection.test.ts +275 -0
- package/src/server/collections/agent-execution-spans.ts +8 -0
- package/src/server/collections/agent-harness-profile-versions.ts +26 -26
- package/src/server/collections/agent-harness-profiles.ts +71 -71
- package/src/server/collections/agent-loop-action-approvals.ts +1 -0
- package/src/server/collections/agent-loop-artifacts.ts +25 -25
- package/src/server/collections/agent-loop-circuit-states.ts +23 -23
- package/src/server/collections/agent-loop-control-settings.ts +21 -21
- package/src/server/collections/agent-loop-events.ts +98 -98
- package/src/server/collections/agent-loop-path-locks.ts +27 -27
- package/src/server/collections/agent-loop-patterns.ts +42 -42
- package/src/server/collections/agent-loop-runs.ts +7 -1
- package/src/server/collections/agent-loop-steps.ts +7 -0
- package/src/server/collections/agent-loop-usage-buckets.ts +22 -22
- package/src/server/collections/agent-loop-worktrees.ts +25 -25
- package/src/server/collections/skill-executions.ts +8 -0
- package/src/server/migrations/20260427000002-change-packages-to-text.ts +52 -52
- package/src/server/migrations/20260429000000-add-llm-fields.ts +47 -47
- package/src/server/migrations/20260429000001-fix-inputargs-json-to-text.ts +45 -45
- package/src/server/migrations/20260503000000-add-orchestrator-trace-fields.ts +33 -33
- package/src/server/migrations/20260524000000-add-agent-loop-fields-to-skill-executions.ts +31 -31
- package/src/server/migrations/20260803010000-backfill-loop-control-plane-data.ts +293 -293
- package/src/server/plugin.ts +3 -1
- package/src/server/resources/agent-harness-profile.ts +94 -0
- package/src/server/resources/agent-knowledge-insights.ts +5 -3
- package/src/server/resources/agent-loop-approval.ts +32 -0
- package/src/server/resources/agent-loop-artifact.ts +56 -56
- package/src/server/resources/agent-loop-events-stream.ts +105 -105
- package/src/server/resources/agent-loop.ts +3 -1
- package/src/server/resources/resource-helpers.ts +33 -33
- package/src/server/resources/tracing.ts +266 -266
- package/src/server/services/AgentMemoryContextService.ts +44 -0
- package/src/server/services/DurableCircuitBreakerService.ts +239 -239
- package/src/server/services/ExecutionSpanService.ts +2 -0
- package/src/server/services/HarnessCompiler.ts +37 -0
- package/src/server/services/HarnessProfileService.ts +92 -10
- package/src/server/services/HarnessSchema.ts +31 -0
- package/src/server/services/LoopBudgetService.ts +275 -275
- package/src/server/services/LoopControlService.ts +124 -124
- package/src/server/services/LoopPatternSchema.ts +19 -1
- package/src/server/services/LoopPatternService.ts +143 -143
- package/src/server/services/LoopRunRepository.ts +99 -0
- package/src/server/services/LoopRunStateMachine.ts +100 -3
- package/src/server/services/LoopSchedulerService.ts +199 -199
- package/src/server/services/LoopTriggerService.ts +42 -0
- package/src/server/services/LoopWorkerService.ts +327 -27
- package/src/server/services/NativeSubAgentObserver.ts +186 -21
- package/src/server/services/PathLockService.ts +321 -321
- package/src/server/services/PluginAiRuntimeAdapter.ts +24 -7
- package/src/server/services/ToolLoopDetectionService.ts +213 -0
- package/src/server/services/VerificationSchema.ts +31 -31
- package/src/server/services/VerificationService.ts +81 -2
- package/src/server/services/harness-runtime-policy.ts +141 -0
- package/src/server/skill-hub/plugin.ts +41 -3
- package/src/server/skill-hub/services/RegistrySkillSnapshotService.ts +235 -235
- package/src/server/skill-hub/services/__tests__/RegistrySkillSnapshotService.test.ts +61 -61
- package/src/server/skill-hub/tasks/SkillExecutionTask.ts +43 -2
- package/src/server/utils/ensure-indexes.ts +96 -96
- package/src/server/utils/logging.ts +62 -62
- package/src/server/utils/loop-pattern-context.ts +20 -20
- package/dist/client-v2/214.b063f33ba044a808.js +0 -10
- package/dist/client-v2/373.f5d5292e53c4e832.js +0 -10
- package/dist/client-v2/418.58fa350f5496efd3.js +0 -10
- package/dist/client-v2/53.9a10455323390f14.js +0 -10
- package/dist/client-v2/677.a991ce0250ff5c77.js +0 -10
- package/dist/client-v2/925.e5925bf21ed68e7a.js +0 -10
- package/src/client/AIEmployeeSelect.tsx +0 -47
- package/src/client/AIEmployeesContext.tsx +0 -102
- package/src/client/RulesTab.tsx +0 -2
- package/src/client/TracingTab.tsx +0 -436
- package/src/client/skill-hub/components/ExecutionProgress.tsx +0 -51
- package/src/client-v2/components/RulesTab.tsx +0 -2
- package/src/client-v2/pages/AgentRunsPage.tsx +0 -13
|
@@ -1,341 +1,341 @@
|
|
|
1
|
-
import type { Database } from '@nocobase/database';
|
|
2
|
-
import { describe, expect, it } from 'vitest';
|
|
3
|
-
import { loopPatternPolicySchema } from '../services/LoopPatternSchema';
|
|
4
|
-
import { LoopRunStateMachine } from '../services/LoopRunStateMachine';
|
|
5
|
-
|
|
6
|
-
type Row = Record<string, unknown>;
|
|
7
|
-
|
|
8
|
-
function matchesValue(actual: unknown, expected: unknown) {
|
|
9
|
-
if (expected && typeof expected === 'object' && !Array.isArray(expected)) {
|
|
10
|
-
const operators = expected as Record<string, unknown>;
|
|
11
|
-
if (Array.isArray(operators.$in)) return operators.$in.includes(actual);
|
|
12
|
-
}
|
|
13
|
-
return actual === expected;
|
|
14
|
-
}
|
|
15
|
-
|
|
16
|
-
function matches(row: Row, filter: Row) {
|
|
17
|
-
return Object.entries(filter).every(([key, value]) => matchesValue(row[key], value));
|
|
18
|
-
}
|
|
19
|
-
|
|
20
|
-
class MemoryRepository {
|
|
21
|
-
private nextId = 1;
|
|
22
|
-
|
|
23
|
-
constructor(readonly rows: Row[] = []) {
|
|
24
|
-
this.resetSequence();
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
async find(options: Row = {}) {
|
|
28
|
-
const filter = (options.filter as Row | undefined) || {};
|
|
29
|
-
const rows = this.rows.filter((row) => matches(row, filter));
|
|
30
|
-
const sort = (options.sort as string[] | undefined) || [];
|
|
31
|
-
rows.sort((left, right) => {
|
|
32
|
-
for (const field of sort) {
|
|
33
|
-
const descending = field.startsWith('-');
|
|
34
|
-
const key = descending ? field.slice(1) : field;
|
|
35
|
-
const leftValue = left[key];
|
|
36
|
-
const rightValue = right[key];
|
|
37
|
-
if (leftValue === rightValue) continue;
|
|
38
|
-
const result = String(leftValue) < String(rightValue) ? -1 : 1;
|
|
39
|
-
return descending ? -result : result;
|
|
40
|
-
}
|
|
41
|
-
return 0;
|
|
42
|
-
});
|
|
43
|
-
return rows.slice(0, Number(options.limit) || rows.length);
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
async findOne(options: Row) {
|
|
47
|
-
if (options.filterByTk !== undefined) return this.rows.find((row) => row.id === options.filterByTk) || null;
|
|
48
|
-
return (await this.find(options))[0] || null;
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
async count(options: Row = {}) {
|
|
52
|
-
return (await this.find(options)).length;
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
async create(options: Row) {
|
|
56
|
-
const row = { id: this.nextId++, ...((options.values as Row | undefined) || {}) };
|
|
57
|
-
this.rows.push(row);
|
|
58
|
-
return row;
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
async update(options: Row) {
|
|
62
|
-
const values = (options.values as Row | undefined) || {};
|
|
63
|
-
for (const row of this.rows) {
|
|
64
|
-
if (options.filterByTk !== undefined && row.id === options.filterByTk) Object.assign(row, values);
|
|
65
|
-
}
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
restore(rows: Row[]) {
|
|
69
|
-
this.rows.splice(0, this.rows.length, ...structuredClone(rows));
|
|
70
|
-
this.resetSequence();
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
private resetSequence() {
|
|
74
|
-
this.nextId = this.rows.reduce((maximum, row) => Math.max(maximum, Number(row.id) || 0), 0) + 1;
|
|
75
|
-
}
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
class TransactionQueue {
|
|
79
|
-
private tail = Promise.resolve();
|
|
80
|
-
|
|
81
|
-
async run<T>(operation: () => Promise<T>) {
|
|
82
|
-
let release = () => {};
|
|
83
|
-
const previous = this.tail;
|
|
84
|
-
this.tail = new Promise<void>((resolve) => {
|
|
85
|
-
release = resolve;
|
|
86
|
-
});
|
|
87
|
-
await previous;
|
|
88
|
-
try {
|
|
89
|
-
return await operation();
|
|
90
|
-
} finally {
|
|
91
|
-
release();
|
|
92
|
-
}
|
|
93
|
-
}
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
function run(id: number, patternId: number, status: string, values: Row = {}) {
|
|
97
|
-
return {
|
|
98
|
-
id,
|
|
99
|
-
rootRunId: `loop-${id}`,
|
|
100
|
-
runtimeVersion: 'control-plane-v2',
|
|
101
|
-
patternId,
|
|
102
|
-
status,
|
|
103
|
-
policySnapshot: loopPatternPolicySchema.parse({ maxConcurrency: 1 }),
|
|
104
|
-
createdAt: new Date(`2026-08-03T00:00:${String(id).padStart(2, '0')}.000Z`),
|
|
105
|
-
lockedBy: null,
|
|
106
|
-
lockedUntil: null,
|
|
107
|
-
...values,
|
|
108
|
-
};
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
function createStateMachine(input: { runs: Row[]; patterns?: Row[]; globalMaxConcurrency?: number }) {
|
|
112
|
-
const repositories = new Map<string, MemoryRepository>([
|
|
113
|
-
['agentLoopRuns', new MemoryRepository(input.runs)],
|
|
114
|
-
[
|
|
115
|
-
'agentLoopPatterns',
|
|
116
|
-
new MemoryRepository(
|
|
117
|
-
input.patterns || [
|
|
118
|
-
{ id: 1, enabled: true },
|
|
119
|
-
{ id: 2, enabled: true },
|
|
120
|
-
{ id: 3, enabled: true },
|
|
121
|
-
],
|
|
122
|
-
),
|
|
123
|
-
],
|
|
124
|
-
['agentLoopEvents', new MemoryRepository()],
|
|
125
|
-
[
|
|
126
|
-
'agentLoopControlSettings',
|
|
127
|
-
new MemoryRepository([
|
|
128
|
-
{
|
|
129
|
-
id: 1,
|
|
130
|
-
key: 'global',
|
|
131
|
-
state: 'running',
|
|
132
|
-
acceptNewRuns: true,
|
|
133
|
-
globalMaxConcurrency: input.globalMaxConcurrency || 5,
|
|
134
|
-
dailyMaxTokens: null,
|
|
135
|
-
dailyMaxCost: null,
|
|
136
|
-
},
|
|
137
|
-
]),
|
|
138
|
-
],
|
|
139
|
-
]);
|
|
140
|
-
const transaction = { LOCK: { UPDATE: 'UPDATE' } };
|
|
141
|
-
const queue = new TransactionQueue();
|
|
142
|
-
const database = {
|
|
143
|
-
getRepository(name: string) {
|
|
144
|
-
const repository = repositories.get(name);
|
|
145
|
-
if (!repository) throw new Error(`Unexpected repository ${name}`);
|
|
146
|
-
return repository;
|
|
147
|
-
},
|
|
148
|
-
sequelize: {
|
|
149
|
-
transaction<T>(callback: (value: typeof transaction) => Promise<T>) {
|
|
150
|
-
return queue.run(async () => {
|
|
151
|
-
const before = new Map(
|
|
152
|
-
Array.from(repositories, ([name, repository]) => [name, structuredClone(repository.rows)]),
|
|
153
|
-
);
|
|
154
|
-
try {
|
|
155
|
-
return await callback(transaction);
|
|
156
|
-
} catch (error) {
|
|
157
|
-
for (const [name, rows] of before) repositories.get(name)?.restore(rows);
|
|
158
|
-
throw error;
|
|
159
|
-
}
|
|
160
|
-
});
|
|
161
|
-
},
|
|
162
|
-
},
|
|
163
|
-
};
|
|
164
|
-
return {
|
|
165
|
-
repositories,
|
|
166
|
-
stateMachine: new LoopRunStateMachine(database as unknown as Database),
|
|
167
|
-
};
|
|
168
|
-
}
|
|
169
|
-
|
|
170
|
-
describe('LoopRunStateMachine worker leases', () => {
|
|
171
|
-
it('allows only one of two workers to claim a queued run', async () => {
|
|
172
|
-
const { repositories, stateMachine } = createStateMachine({ runs: [run(1, 1, 'queued')] });
|
|
173
|
-
const claims = await Promise.all([
|
|
174
|
-
stateMachine.claimNext('worker-a', 30_000),
|
|
175
|
-
stateMachine.claimNext('worker-b', 30_000),
|
|
176
|
-
]);
|
|
177
|
-
|
|
178
|
-
expect(claims.filter(Boolean)).toHaveLength(1);
|
|
179
|
-
expect(repositories.get('agentLoopRuns')?.rows[0]).toMatchObject({ status: 'preparing' });
|
|
180
|
-
expect(String(repositories.get('agentLoopRuns')?.rows[0].lockedBy)).toMatch(/^worker-[ab]:/);
|
|
181
|
-
expect(repositories.get('agentLoopEvents')?.rows).toHaveLength(1);
|
|
182
|
-
expect(repositories.get('agentLoopEvents')?.rows[0]).toMatchObject({ type: 'run_claimed', status: 'preparing' });
|
|
183
|
-
});
|
|
184
|
-
|
|
185
|
-
it('blocks malformed snapshots and skips a pattern at capacity without hiding another pattern', async () => {
|
|
186
|
-
const { repositories, stateMachine } = createStateMachine({
|
|
187
|
-
runs: [
|
|
188
|
-
run(1, 3, 'queued', { policySnapshot: { maxConcurrency: 0 } }),
|
|
189
|
-
run(2, 1, 'running'),
|
|
190
|
-
run(3, 1, 'queued'),
|
|
191
|
-
run(4, 2, 'queued'),
|
|
192
|
-
],
|
|
193
|
-
});
|
|
194
|
-
|
|
195
|
-
const claimed = await stateMachine.claimNext('worker-a', 30_000);
|
|
196
|
-
|
|
197
|
-
expect(claimed?.run.id).toBe(4);
|
|
198
|
-
expect(repositories.get('agentLoopRuns')?.rows[0]).toMatchObject({
|
|
199
|
-
status: 'blocked',
|
|
200
|
-
blockedReason: 'Run policy snapshot is invalid.',
|
|
201
|
-
});
|
|
202
|
-
expect(repositories.get('agentLoopRuns')?.rows[2].status).toBe('queued');
|
|
203
|
-
expect(repositories.get('agentLoopRuns')?.rows[3].status).toBe('preparing');
|
|
204
|
-
expect(repositories.get('agentLoopEvents')?.rows).toMatchObject([
|
|
205
|
-
{ runId: 1, type: 'run_blocked' },
|
|
206
|
-
{ runId: 4, type: 'run_claimed' },
|
|
207
|
-
]);
|
|
208
|
-
});
|
|
209
|
-
|
|
210
|
-
it('enforces global concurrency before claiming another run', async () => {
|
|
211
|
-
const { repositories, stateMachine } = createStateMachine({
|
|
212
|
-
runs: [run(1, 1, 'running'), run(2, 2, 'queued')],
|
|
213
|
-
globalMaxConcurrency: 1,
|
|
214
|
-
});
|
|
215
|
-
|
|
216
|
-
await expect(stateMachine.claimNext('worker-a', 30_000)).resolves.toBeNull();
|
|
217
|
-
expect(repositories.get('agentLoopRuns')?.rows[1].status).toBe('queued');
|
|
218
|
-
expect(repositories.get('agentLoopEvents')?.rows).toHaveLength(0);
|
|
219
|
-
});
|
|
220
|
-
|
|
221
|
-
it('rejects stale worker transitions and accepts the current unexpired lease', async () => {
|
|
222
|
-
const { repositories, stateMachine } = createStateMachine({ runs: [run(1, 1, 'queued')] });
|
|
223
|
-
const claimed = await stateMachine.claimNext('worker-a', 30_000);
|
|
224
|
-
if (!claimed) throw new Error('Expected the queued run to be claimed.');
|
|
225
|
-
|
|
226
|
-
const transition = {
|
|
227
|
-
runId: 1,
|
|
228
|
-
to: 'running' as const,
|
|
229
|
-
actorType: 'worker',
|
|
230
|
-
actorIdentity: 'worker-a',
|
|
231
|
-
};
|
|
232
|
-
await expect(stateMachine.transition(transition)).rejects.toThrow('no longer owned');
|
|
233
|
-
await expect(stateMachine.transition({ ...transition, leaseToken: 'worker-b:wrong' })).rejects.toThrow(
|
|
234
|
-
'no longer owned',
|
|
235
|
-
);
|
|
236
|
-
|
|
237
|
-
const expiredRunRow = repositories.get('agentLoopRuns')?.rows[0];
|
|
238
|
-
if (!expiredRunRow) throw new Error('Expected claimed run.');
|
|
239
|
-
expiredRunRow.lockedUntil = new Date(Date.now() - 1_000);
|
|
240
|
-
await expect(stateMachine.transition({ ...transition, leaseToken: claimed.leaseToken })).rejects.toThrow('expired');
|
|
241
|
-
|
|
242
|
-
const currentRunRow = repositories.get('agentLoopRuns')?.rows[0];
|
|
243
|
-
if (!currentRunRow) throw new Error('Expected claimed run after rollback.');
|
|
244
|
-
currentRunRow.lockedUntil = new Date(Date.now() + 30_000);
|
|
245
|
-
await expect(stateMachine.transition({ ...transition, leaseToken: claimed.leaseToken })).resolves.toMatchObject({
|
|
246
|
-
from: 'preparing',
|
|
247
|
-
to: 'running',
|
|
248
|
-
});
|
|
249
|
-
expect(currentRunRow.status).toBe('running');
|
|
250
|
-
expect(repositories.get('agentLoopEvents')?.rows).toMatchObject([
|
|
251
|
-
{ type: 'run_claimed', status: 'preparing' },
|
|
252
|
-
{ type: 'run_status_changed', status: 'running' },
|
|
253
|
-
]);
|
|
254
|
-
});
|
|
255
|
-
|
|
256
|
-
it('renews and releases only the matching worker lease', async () => {
|
|
257
|
-
const { repositories, stateMachine } = createStateMachine({ runs: [run(1, 1, 'queued')] });
|
|
258
|
-
const claimed = await stateMachine.claimNext('worker-a', 30_000);
|
|
259
|
-
if (!claimed) throw new Error('Expected the queued run to be claimed.');
|
|
260
|
-
|
|
261
|
-
await expect(stateMachine.renewLease(1, 'worker-b:wrong', 30_000)).rejects.toThrow('no longer owned');
|
|
262
|
-
const renewedUntil = await stateMachine.renewLease(1, claimed.leaseToken, 30_000);
|
|
263
|
-
expect(renewedUntil.getTime()).toBeGreaterThan(Date.now());
|
|
264
|
-
await expect(stateMachine.releaseLease(1, 'worker-b:wrong')).rejects.toThrow('no longer owned');
|
|
265
|
-
await expect(stateMachine.releaseLease(1, claimed.leaseToken)).resolves.toBe(true);
|
|
266
|
-
expect(repositories.get('agentLoopRuns')?.rows[0]).toMatchObject({ lockedBy: null, lockedUntil: null });
|
|
267
|
-
});
|
|
268
|
-
|
|
269
|
-
it('reclaims a crashed run with an expired lease and frees its concurrency slot', async () => {
|
|
270
|
-
const { repositories, stateMachine } = createStateMachine({
|
|
271
|
-
runs: [
|
|
272
|
-
run(1, 1, 'running', { lockedBy: 'worker-dead:1', lockedUntil: new Date(Date.now() - 1_000) }),
|
|
273
|
-
run(2, 2, 'queued'),
|
|
274
|
-
],
|
|
275
|
-
globalMaxConcurrency: 1,
|
|
276
|
-
});
|
|
277
|
-
|
|
278
|
-
const claimed = await stateMachine.claimNext('worker-a', 30_000);
|
|
279
|
-
|
|
280
|
-
// Without reclaim the crashed run would hold the single global slot forever and nothing
|
|
281
|
-
// could be claimed. Once requeued it is the oldest candidate, so it is picked up first.
|
|
282
|
-
expect(claimed?.run.id).toBe(1);
|
|
283
|
-
expect(String(claimed?.leaseToken)).toMatch(/^worker-a:/);
|
|
284
|
-
expect(repositories.get('agentLoopRuns')?.rows[0]).toMatchObject({ status: 'preparing' });
|
|
285
|
-
expect(repositories.get('agentLoopRuns')?.rows[1].status).toBe('queued');
|
|
286
|
-
expect(repositories.get('agentLoopEvents')?.rows).toMatchObject([
|
|
287
|
-
{ runId: 1, type: 'run_lease_expired', status: 'queued' },
|
|
288
|
-
{ runId: 1, type: 'run_claimed', status: 'preparing' },
|
|
289
|
-
]);
|
|
290
|
-
});
|
|
291
|
-
|
|
292
|
-
it('does not reclaim an active run that has no lease owner yet', async () => {
|
|
293
|
-
const { repositories, stateMachine } = createStateMachine({
|
|
294
|
-
runs: [run(1, 1, 'running', { lockedBy: null, lockedUntil: null }), run(2, 2, 'queued')],
|
|
295
|
-
globalMaxConcurrency: 1,
|
|
296
|
-
});
|
|
297
|
-
|
|
298
|
-
// An owner-less active run still occupies its slot; requeueing it would double-dispatch work.
|
|
299
|
-
await expect(stateMachine.claimNext('worker-a', 30_000)).resolves.toBeNull();
|
|
300
|
-
expect(repositories.get('agentLoopRuns')?.rows[0].status).toBe('running');
|
|
301
|
-
expect(repositories.get('agentLoopRuns')?.rows[1].status).toBe('queued');
|
|
302
|
-
expect(repositories.get('agentLoopEvents')?.rows).toHaveLength(0);
|
|
303
|
-
});
|
|
304
|
-
|
|
305
|
-
it('stops a worker from carrying a paused or canceled run forward', async () => {
|
|
306
|
-
const { repositories, stateMachine } = createStateMachine({ runs: [run(1, 1, 'queued')] });
|
|
307
|
-
const claimed = await stateMachine.claimNext('worker-a', 30_000);
|
|
308
|
-
if (!claimed) throw new Error('Expected the queued run to be claimed.');
|
|
309
|
-
await stateMachine.transition({
|
|
310
|
-
runId: 1,
|
|
311
|
-
to: 'running',
|
|
312
|
-
actorType: 'worker',
|
|
313
|
-
actorIdentity: 'worker-a',
|
|
314
|
-
leaseToken: claimed.leaseToken,
|
|
315
|
-
});
|
|
316
|
-
|
|
317
|
-
// A human pause leaves the worker lease in place, so the lease check alone would let the
|
|
318
|
-
// in-flight worker continue. `paused -> verifying` is a legal edge for a human resume, and
|
|
319
|
-
// without the actor guard the worker would walk straight through the pause.
|
|
320
|
-
const runRow = repositories.get('agentLoopRuns')?.rows[0];
|
|
321
|
-
if (!runRow) throw new Error('Expected the claimed run.');
|
|
322
|
-
await stateMachine.transition({ runId: 1, to: 'paused', actorType: 'human', actorIdentity: '1' });
|
|
323
|
-
expect(runRow.status).toBe('paused');
|
|
324
|
-
|
|
325
|
-
await expect(
|
|
326
|
-
stateMachine.transition({
|
|
327
|
-
runId: 1,
|
|
328
|
-
to: 'verifying',
|
|
329
|
-
actorType: 'worker',
|
|
330
|
-
actorIdentity: 'worker-a',
|
|
331
|
-
leaseToken: claimed.leaseToken,
|
|
332
|
-
}),
|
|
333
|
-
).rejects.toThrow('A worker cannot transition a run that is paused.');
|
|
334
|
-
expect(runRow.status).toBe('paused');
|
|
335
|
-
|
|
336
|
-
// The human resume path is unaffected.
|
|
337
|
-
await expect(
|
|
338
|
-
stateMachine.transition({ runId: 1, to: 'queued', actorType: 'human', actorIdentity: '1' }),
|
|
339
|
-
).resolves.toMatchObject({ from: 'paused', to: 'queued' });
|
|
340
|
-
});
|
|
341
|
-
});
|
|
1
|
+
import type { Database } from '@nocobase/database';
|
|
2
|
+
import { describe, expect, it } from 'vitest';
|
|
3
|
+
import { loopPatternPolicySchema } from '../services/LoopPatternSchema';
|
|
4
|
+
import { LoopRunStateMachine } from '../services/LoopRunStateMachine';
|
|
5
|
+
|
|
6
|
+
type Row = Record<string, unknown>;
|
|
7
|
+
|
|
8
|
+
function matchesValue(actual: unknown, expected: unknown) {
|
|
9
|
+
if (expected && typeof expected === 'object' && !Array.isArray(expected)) {
|
|
10
|
+
const operators = expected as Record<string, unknown>;
|
|
11
|
+
if (Array.isArray(operators.$in)) return operators.$in.includes(actual);
|
|
12
|
+
}
|
|
13
|
+
return actual === expected;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
function matches(row: Row, filter: Row) {
|
|
17
|
+
return Object.entries(filter).every(([key, value]) => matchesValue(row[key], value));
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
class MemoryRepository {
|
|
21
|
+
private nextId = 1;
|
|
22
|
+
|
|
23
|
+
constructor(readonly rows: Row[] = []) {
|
|
24
|
+
this.resetSequence();
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
async find(options: Row = {}) {
|
|
28
|
+
const filter = (options.filter as Row | undefined) || {};
|
|
29
|
+
const rows = this.rows.filter((row) => matches(row, filter));
|
|
30
|
+
const sort = (options.sort as string[] | undefined) || [];
|
|
31
|
+
rows.sort((left, right) => {
|
|
32
|
+
for (const field of sort) {
|
|
33
|
+
const descending = field.startsWith('-');
|
|
34
|
+
const key = descending ? field.slice(1) : field;
|
|
35
|
+
const leftValue = left[key];
|
|
36
|
+
const rightValue = right[key];
|
|
37
|
+
if (leftValue === rightValue) continue;
|
|
38
|
+
const result = String(leftValue) < String(rightValue) ? -1 : 1;
|
|
39
|
+
return descending ? -result : result;
|
|
40
|
+
}
|
|
41
|
+
return 0;
|
|
42
|
+
});
|
|
43
|
+
return rows.slice(0, Number(options.limit) || rows.length);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
async findOne(options: Row) {
|
|
47
|
+
if (options.filterByTk !== undefined) return this.rows.find((row) => row.id === options.filterByTk) || null;
|
|
48
|
+
return (await this.find(options))[0] || null;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
async count(options: Row = {}) {
|
|
52
|
+
return (await this.find(options)).length;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
async create(options: Row) {
|
|
56
|
+
const row = { id: this.nextId++, ...((options.values as Row | undefined) || {}) };
|
|
57
|
+
this.rows.push(row);
|
|
58
|
+
return row;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
async update(options: Row) {
|
|
62
|
+
const values = (options.values as Row | undefined) || {};
|
|
63
|
+
for (const row of this.rows) {
|
|
64
|
+
if (options.filterByTk !== undefined && row.id === options.filterByTk) Object.assign(row, values);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
restore(rows: Row[]) {
|
|
69
|
+
this.rows.splice(0, this.rows.length, ...structuredClone(rows));
|
|
70
|
+
this.resetSequence();
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
private resetSequence() {
|
|
74
|
+
this.nextId = this.rows.reduce((maximum, row) => Math.max(maximum, Number(row.id) || 0), 0) + 1;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
class TransactionQueue {
|
|
79
|
+
private tail = Promise.resolve();
|
|
80
|
+
|
|
81
|
+
async run<T>(operation: () => Promise<T>) {
|
|
82
|
+
let release = () => {};
|
|
83
|
+
const previous = this.tail;
|
|
84
|
+
this.tail = new Promise<void>((resolve) => {
|
|
85
|
+
release = resolve;
|
|
86
|
+
});
|
|
87
|
+
await previous;
|
|
88
|
+
try {
|
|
89
|
+
return await operation();
|
|
90
|
+
} finally {
|
|
91
|
+
release();
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function run(id: number, patternId: number, status: string, values: Row = {}) {
|
|
97
|
+
return {
|
|
98
|
+
id,
|
|
99
|
+
rootRunId: `loop-${id}`,
|
|
100
|
+
runtimeVersion: 'control-plane-v2',
|
|
101
|
+
patternId,
|
|
102
|
+
status,
|
|
103
|
+
policySnapshot: loopPatternPolicySchema.parse({ maxConcurrency: 1 }),
|
|
104
|
+
createdAt: new Date(`2026-08-03T00:00:${String(id).padStart(2, '0')}.000Z`),
|
|
105
|
+
lockedBy: null,
|
|
106
|
+
lockedUntil: null,
|
|
107
|
+
...values,
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
function createStateMachine(input: { runs: Row[]; patterns?: Row[]; globalMaxConcurrency?: number }) {
|
|
112
|
+
const repositories = new Map<string, MemoryRepository>([
|
|
113
|
+
['agentLoopRuns', new MemoryRepository(input.runs)],
|
|
114
|
+
[
|
|
115
|
+
'agentLoopPatterns',
|
|
116
|
+
new MemoryRepository(
|
|
117
|
+
input.patterns || [
|
|
118
|
+
{ id: 1, enabled: true },
|
|
119
|
+
{ id: 2, enabled: true },
|
|
120
|
+
{ id: 3, enabled: true },
|
|
121
|
+
],
|
|
122
|
+
),
|
|
123
|
+
],
|
|
124
|
+
['agentLoopEvents', new MemoryRepository()],
|
|
125
|
+
[
|
|
126
|
+
'agentLoopControlSettings',
|
|
127
|
+
new MemoryRepository([
|
|
128
|
+
{
|
|
129
|
+
id: 1,
|
|
130
|
+
key: 'global',
|
|
131
|
+
state: 'running',
|
|
132
|
+
acceptNewRuns: true,
|
|
133
|
+
globalMaxConcurrency: input.globalMaxConcurrency || 5,
|
|
134
|
+
dailyMaxTokens: null,
|
|
135
|
+
dailyMaxCost: null,
|
|
136
|
+
},
|
|
137
|
+
]),
|
|
138
|
+
],
|
|
139
|
+
]);
|
|
140
|
+
const transaction = { LOCK: { UPDATE: 'UPDATE' } };
|
|
141
|
+
const queue = new TransactionQueue();
|
|
142
|
+
const database = {
|
|
143
|
+
getRepository(name: string) {
|
|
144
|
+
const repository = repositories.get(name);
|
|
145
|
+
if (!repository) throw new Error(`Unexpected repository ${name}`);
|
|
146
|
+
return repository;
|
|
147
|
+
},
|
|
148
|
+
sequelize: {
|
|
149
|
+
transaction<T>(callback: (value: typeof transaction) => Promise<T>) {
|
|
150
|
+
return queue.run(async () => {
|
|
151
|
+
const before = new Map(
|
|
152
|
+
Array.from(repositories, ([name, repository]) => [name, structuredClone(repository.rows)]),
|
|
153
|
+
);
|
|
154
|
+
try {
|
|
155
|
+
return await callback(transaction);
|
|
156
|
+
} catch (error) {
|
|
157
|
+
for (const [name, rows] of before) repositories.get(name)?.restore(rows);
|
|
158
|
+
throw error;
|
|
159
|
+
}
|
|
160
|
+
});
|
|
161
|
+
},
|
|
162
|
+
},
|
|
163
|
+
};
|
|
164
|
+
return {
|
|
165
|
+
repositories,
|
|
166
|
+
stateMachine: new LoopRunStateMachine(database as unknown as Database),
|
|
167
|
+
};
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
describe('LoopRunStateMachine worker leases', () => {
|
|
171
|
+
it('allows only one of two workers to claim a queued run', async () => {
|
|
172
|
+
const { repositories, stateMachine } = createStateMachine({ runs: [run(1, 1, 'queued')] });
|
|
173
|
+
const claims = await Promise.all([
|
|
174
|
+
stateMachine.claimNext('worker-a', 30_000),
|
|
175
|
+
stateMachine.claimNext('worker-b', 30_000),
|
|
176
|
+
]);
|
|
177
|
+
|
|
178
|
+
expect(claims.filter(Boolean)).toHaveLength(1);
|
|
179
|
+
expect(repositories.get('agentLoopRuns')?.rows[0]).toMatchObject({ status: 'preparing' });
|
|
180
|
+
expect(String(repositories.get('agentLoopRuns')?.rows[0].lockedBy)).toMatch(/^worker-[ab]:/);
|
|
181
|
+
expect(repositories.get('agentLoopEvents')?.rows).toHaveLength(1);
|
|
182
|
+
expect(repositories.get('agentLoopEvents')?.rows[0]).toMatchObject({ type: 'run_claimed', status: 'preparing' });
|
|
183
|
+
});
|
|
184
|
+
|
|
185
|
+
it('blocks malformed snapshots and skips a pattern at capacity without hiding another pattern', async () => {
|
|
186
|
+
const { repositories, stateMachine } = createStateMachine({
|
|
187
|
+
runs: [
|
|
188
|
+
run(1, 3, 'queued', { policySnapshot: { maxConcurrency: 0 } }),
|
|
189
|
+
run(2, 1, 'running'),
|
|
190
|
+
run(3, 1, 'queued'),
|
|
191
|
+
run(4, 2, 'queued'),
|
|
192
|
+
],
|
|
193
|
+
});
|
|
194
|
+
|
|
195
|
+
const claimed = await stateMachine.claimNext('worker-a', 30_000);
|
|
196
|
+
|
|
197
|
+
expect(claimed?.run.id).toBe(4);
|
|
198
|
+
expect(repositories.get('agentLoopRuns')?.rows[0]).toMatchObject({
|
|
199
|
+
status: 'blocked',
|
|
200
|
+
blockedReason: 'Run policy snapshot is invalid.',
|
|
201
|
+
});
|
|
202
|
+
expect(repositories.get('agentLoopRuns')?.rows[2].status).toBe('queued');
|
|
203
|
+
expect(repositories.get('agentLoopRuns')?.rows[3].status).toBe('preparing');
|
|
204
|
+
expect(repositories.get('agentLoopEvents')?.rows).toMatchObject([
|
|
205
|
+
{ runId: 1, type: 'run_blocked' },
|
|
206
|
+
{ runId: 4, type: 'run_claimed' },
|
|
207
|
+
]);
|
|
208
|
+
});
|
|
209
|
+
|
|
210
|
+
it('enforces global concurrency before claiming another run', async () => {
|
|
211
|
+
const { repositories, stateMachine } = createStateMachine({
|
|
212
|
+
runs: [run(1, 1, 'running'), run(2, 2, 'queued')],
|
|
213
|
+
globalMaxConcurrency: 1,
|
|
214
|
+
});
|
|
215
|
+
|
|
216
|
+
await expect(stateMachine.claimNext('worker-a', 30_000)).resolves.toBeNull();
|
|
217
|
+
expect(repositories.get('agentLoopRuns')?.rows[1].status).toBe('queued');
|
|
218
|
+
expect(repositories.get('agentLoopEvents')?.rows).toHaveLength(0);
|
|
219
|
+
});
|
|
220
|
+
|
|
221
|
+
it('rejects stale worker transitions and accepts the current unexpired lease', async () => {
|
|
222
|
+
const { repositories, stateMachine } = createStateMachine({ runs: [run(1, 1, 'queued')] });
|
|
223
|
+
const claimed = await stateMachine.claimNext('worker-a', 30_000);
|
|
224
|
+
if (!claimed) throw new Error('Expected the queued run to be claimed.');
|
|
225
|
+
|
|
226
|
+
const transition = {
|
|
227
|
+
runId: 1,
|
|
228
|
+
to: 'running' as const,
|
|
229
|
+
actorType: 'worker',
|
|
230
|
+
actorIdentity: 'worker-a',
|
|
231
|
+
};
|
|
232
|
+
await expect(stateMachine.transition(transition)).rejects.toThrow('no longer owned');
|
|
233
|
+
await expect(stateMachine.transition({ ...transition, leaseToken: 'worker-b:wrong' })).rejects.toThrow(
|
|
234
|
+
'no longer owned',
|
|
235
|
+
);
|
|
236
|
+
|
|
237
|
+
const expiredRunRow = repositories.get('agentLoopRuns')?.rows[0];
|
|
238
|
+
if (!expiredRunRow) throw new Error('Expected claimed run.');
|
|
239
|
+
expiredRunRow.lockedUntil = new Date(Date.now() - 1_000);
|
|
240
|
+
await expect(stateMachine.transition({ ...transition, leaseToken: claimed.leaseToken })).rejects.toThrow('expired');
|
|
241
|
+
|
|
242
|
+
const currentRunRow = repositories.get('agentLoopRuns')?.rows[0];
|
|
243
|
+
if (!currentRunRow) throw new Error('Expected claimed run after rollback.');
|
|
244
|
+
currentRunRow.lockedUntil = new Date(Date.now() + 30_000);
|
|
245
|
+
await expect(stateMachine.transition({ ...transition, leaseToken: claimed.leaseToken })).resolves.toMatchObject({
|
|
246
|
+
from: 'preparing',
|
|
247
|
+
to: 'running',
|
|
248
|
+
});
|
|
249
|
+
expect(currentRunRow.status).toBe('running');
|
|
250
|
+
expect(repositories.get('agentLoopEvents')?.rows).toMatchObject([
|
|
251
|
+
{ type: 'run_claimed', status: 'preparing' },
|
|
252
|
+
{ type: 'run_status_changed', status: 'running' },
|
|
253
|
+
]);
|
|
254
|
+
});
|
|
255
|
+
|
|
256
|
+
it('renews and releases only the matching worker lease', async () => {
|
|
257
|
+
const { repositories, stateMachine } = createStateMachine({ runs: [run(1, 1, 'queued')] });
|
|
258
|
+
const claimed = await stateMachine.claimNext('worker-a', 30_000);
|
|
259
|
+
if (!claimed) throw new Error('Expected the queued run to be claimed.');
|
|
260
|
+
|
|
261
|
+
await expect(stateMachine.renewLease(1, 'worker-b:wrong', 30_000)).rejects.toThrow('no longer owned');
|
|
262
|
+
const renewedUntil = await stateMachine.renewLease(1, claimed.leaseToken, 30_000);
|
|
263
|
+
expect(renewedUntil.getTime()).toBeGreaterThan(Date.now());
|
|
264
|
+
await expect(stateMachine.releaseLease(1, 'worker-b:wrong')).rejects.toThrow('no longer owned');
|
|
265
|
+
await expect(stateMachine.releaseLease(1, claimed.leaseToken)).resolves.toBe(true);
|
|
266
|
+
expect(repositories.get('agentLoopRuns')?.rows[0]).toMatchObject({ lockedBy: null, lockedUntil: null });
|
|
267
|
+
});
|
|
268
|
+
|
|
269
|
+
it('reclaims a crashed run with an expired lease and frees its concurrency slot', async () => {
|
|
270
|
+
const { repositories, stateMachine } = createStateMachine({
|
|
271
|
+
runs: [
|
|
272
|
+
run(1, 1, 'running', { lockedBy: 'worker-dead:1', lockedUntil: new Date(Date.now() - 1_000) }),
|
|
273
|
+
run(2, 2, 'queued'),
|
|
274
|
+
],
|
|
275
|
+
globalMaxConcurrency: 1,
|
|
276
|
+
});
|
|
277
|
+
|
|
278
|
+
const claimed = await stateMachine.claimNext('worker-a', 30_000);
|
|
279
|
+
|
|
280
|
+
// Without reclaim the crashed run would hold the single global slot forever and nothing
|
|
281
|
+
// could be claimed. Once requeued it is the oldest candidate, so it is picked up first.
|
|
282
|
+
expect(claimed?.run.id).toBe(1);
|
|
283
|
+
expect(String(claimed?.leaseToken)).toMatch(/^worker-a:/);
|
|
284
|
+
expect(repositories.get('agentLoopRuns')?.rows[0]).toMatchObject({ status: 'preparing' });
|
|
285
|
+
expect(repositories.get('agentLoopRuns')?.rows[1].status).toBe('queued');
|
|
286
|
+
expect(repositories.get('agentLoopEvents')?.rows).toMatchObject([
|
|
287
|
+
{ runId: 1, type: 'run_lease_expired', status: 'queued' },
|
|
288
|
+
{ runId: 1, type: 'run_claimed', status: 'preparing' },
|
|
289
|
+
]);
|
|
290
|
+
});
|
|
291
|
+
|
|
292
|
+
it('does not reclaim an active run that has no lease owner yet', async () => {
|
|
293
|
+
const { repositories, stateMachine } = createStateMachine({
|
|
294
|
+
runs: [run(1, 1, 'running', { lockedBy: null, lockedUntil: null }), run(2, 2, 'queued')],
|
|
295
|
+
globalMaxConcurrency: 1,
|
|
296
|
+
});
|
|
297
|
+
|
|
298
|
+
// An owner-less active run still occupies its slot; requeueing it would double-dispatch work.
|
|
299
|
+
await expect(stateMachine.claimNext('worker-a', 30_000)).resolves.toBeNull();
|
|
300
|
+
expect(repositories.get('agentLoopRuns')?.rows[0].status).toBe('running');
|
|
301
|
+
expect(repositories.get('agentLoopRuns')?.rows[1].status).toBe('queued');
|
|
302
|
+
expect(repositories.get('agentLoopEvents')?.rows).toHaveLength(0);
|
|
303
|
+
});
|
|
304
|
+
|
|
305
|
+
it('stops a worker from carrying a paused or canceled run forward', async () => {
|
|
306
|
+
const { repositories, stateMachine } = createStateMachine({ runs: [run(1, 1, 'queued')] });
|
|
307
|
+
const claimed = await stateMachine.claimNext('worker-a', 30_000);
|
|
308
|
+
if (!claimed) throw new Error('Expected the queued run to be claimed.');
|
|
309
|
+
await stateMachine.transition({
|
|
310
|
+
runId: 1,
|
|
311
|
+
to: 'running',
|
|
312
|
+
actorType: 'worker',
|
|
313
|
+
actorIdentity: 'worker-a',
|
|
314
|
+
leaseToken: claimed.leaseToken,
|
|
315
|
+
});
|
|
316
|
+
|
|
317
|
+
// A human pause leaves the worker lease in place, so the lease check alone would let the
|
|
318
|
+
// in-flight worker continue. `paused -> verifying` is a legal edge for a human resume, and
|
|
319
|
+
// without the actor guard the worker would walk straight through the pause.
|
|
320
|
+
const runRow = repositories.get('agentLoopRuns')?.rows[0];
|
|
321
|
+
if (!runRow) throw new Error('Expected the claimed run.');
|
|
322
|
+
await stateMachine.transition({ runId: 1, to: 'paused', actorType: 'human', actorIdentity: '1' });
|
|
323
|
+
expect(runRow.status).toBe('paused');
|
|
324
|
+
|
|
325
|
+
await expect(
|
|
326
|
+
stateMachine.transition({
|
|
327
|
+
runId: 1,
|
|
328
|
+
to: 'verifying',
|
|
329
|
+
actorType: 'worker',
|
|
330
|
+
actorIdentity: 'worker-a',
|
|
331
|
+
leaseToken: claimed.leaseToken,
|
|
332
|
+
}),
|
|
333
|
+
).rejects.toThrow('A worker cannot transition a run that is paused.');
|
|
334
|
+
expect(runRow.status).toBe('paused');
|
|
335
|
+
|
|
336
|
+
// The human resume path is unaffected.
|
|
337
|
+
await expect(
|
|
338
|
+
stateMachine.transition({ runId: 1, to: 'queued', actorType: 'human', actorIdentity: '1' }),
|
|
339
|
+
).resolves.toMatchObject({ from: 'paused', to: 'queued' });
|
|
340
|
+
});
|
|
341
|
+
});
|