plugin-agent-orchestrator 1.1.6 → 1.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client/index.js +1 -1
- package/dist/client-v2/index.js +1 -1
- package/dist/externalVersion.js +7 -7
- package/dist/server/collections/agent-loop-runs.js +6 -0
- package/dist/server/collections/skill-metrics.js +66 -0
- package/dist/server/plugin.js +91 -2
- package/dist/server/resources/agent-loop-events-stream.js +1 -0
- package/dist/server/resources/agent-loop.js +1 -1
- package/dist/server/services/DurableCircuitBreakerService.js +9 -12
- package/dist/server/services/ExecutionSpanService.js +8 -3
- package/dist/server/services/HarnessProfileService.js +23 -26
- package/dist/server/services/LoopBudgetService.js +36 -39
- package/dist/server/services/LoopControlService.js +8 -11
- package/dist/server/services/LoopPatternSchema.js +51 -17
- package/dist/server/services/LoopPatternService.js +31 -11
- package/dist/server/services/LoopRunRepository.js +2 -5
- package/dist/server/services/LoopRunStateMachine.js +31 -34
- package/dist/server/services/LoopSchedulerService.js +4 -7
- package/dist/server/services/LoopTriggerService.js +11 -6
- package/dist/server/services/LoopWorkerService.js +88 -22
- package/dist/server/services/PathLockService.js +19 -22
- package/dist/server/services/PluginAiRuntimeAdapter.js +51 -8
- package/dist/server/services/RunEventBus.js +14 -2
- package/dist/server/services/SkillAccessService.js +50 -1
- package/dist/server/services/TokenTracker.js +17 -1
- package/dist/server/services/ToolLoopDetectionService.js +14 -17
- package/dist/server/services/VerificationService.js +47 -16
- package/dist/server/services/harness-runtime-policy.js +7 -12
- package/dist/server/skill-hub/plugin.js +34 -27
- package/dist/server/utils/ha-lock.js +73 -0
- package/dist/server/utils/loop-pattern-context.js +42 -5
- package/dist/server/utils/record-utils.js +44 -0
- package/package.json +1 -1
- package/src/client/plugin.tsx +1 -1
- package/src/client-v2/plugin.tsx +14 -13
- package/src/server/__tests__/tool-loop-detection.test.ts +1 -0
- package/src/server/collections/agent-loop-runs.ts +7 -0
- package/src/server/collections/skill-metrics.ts +36 -0
- package/src/server/plugin.ts +117 -1
- package/src/server/resources/agent-loop-events-stream.ts +108 -105
- package/src/server/resources/agent-loop.ts +1 -1
- package/src/server/resources/tracing.ts +267 -266
- package/src/server/services/DurableCircuitBreakerService.ts +235 -239
- package/src/server/services/ExecutionSpanService.ts +37 -12
- package/src/server/services/HarnessProfileService.ts +1 -5
- package/src/server/services/LoopBudgetService.ts +271 -275
- package/src/server/services/LoopControlService.ts +120 -124
- package/src/server/services/LoopPatternSchema.ts +60 -14
- package/src/server/services/LoopPatternService.ts +171 -143
- package/src/server/services/LoopRunRepository.ts +1 -5
- package/src/server/services/LoopRunStateMachine.ts +1 -5
- package/src/server/services/LoopSchedulerService.ts +195 -199
- package/src/server/services/LoopTriggerService.ts +7 -5
- package/src/server/services/LoopWorkerService.ts +102 -12
- package/src/server/services/PathLockService.ts +317 -321
- package/src/server/services/PluginAiRuntimeAdapter.ts +59 -4
- package/src/server/services/RunEventBus.ts +13 -0
- package/src/server/services/SkillAccessService.ts +80 -5
- package/src/server/services/TokenTracker.ts +59 -7
- package/src/server/services/ToolLoopDetectionService.ts +8 -11
- package/src/server/services/VerificationSchema.ts +31 -31
- package/src/server/services/VerificationService.ts +60 -11
- package/src/server/services/harness-runtime-policy.ts +2 -9
- package/src/server/skill-hub/plugin.ts +36 -28
- package/src/server/types.ts +5 -39
- package/src/server/utils/ensure-indexes.ts +96 -96
- package/src/server/utils/ha-lock.ts +80 -0
- package/src/server/utils/loop-pattern-context.ts +84 -20
- package/src/server/utils/record-utils.ts +19 -0
|
@@ -1,199 +1,195 @@
|
|
|
1
|
-
import type { Database, Model, Transaction } from '@nocobase/database';
|
|
2
|
-
import type { Application, Plugin } from '@nocobase/server';
|
|
3
|
-
import type { LoopPattern } from './LoopPatternSchema';
|
|
4
|
-
import { LoopPatternService } from './LoopPatternService';
|
|
5
|
-
import { LoopTriggerService } from './LoopTriggerService';
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
return
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
function
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
this.
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
this.
|
|
66
|
-
this.
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
this.
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
if (!
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
patternIds.
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
const
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
await
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
this.
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
error,
|
|
197
|
-
});
|
|
198
|
-
}
|
|
199
|
-
}
|
|
1
|
+
import type { Database, Model, Transaction } from '@nocobase/database';
|
|
2
|
+
import type { Application, Plugin } from '@nocobase/server';
|
|
3
|
+
import type { LoopPattern } from './LoopPatternSchema';
|
|
4
|
+
import { LoopPatternService } from './LoopPatternService';
|
|
5
|
+
import { LoopTriggerService } from './LoopTriggerService';
|
|
6
|
+
import { read } from '../utils/record-utils';
|
|
7
|
+
|
|
8
|
+
type ManagedCronJob = ReturnType<Application['cronJobManager']['addJob']>;
|
|
9
|
+
|
|
10
|
+
type ScheduledPattern = {
|
|
11
|
+
job: ManagedCronJob;
|
|
12
|
+
signature: string;
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
type TransactionOptions = {
|
|
16
|
+
transaction?: Transaction;
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
export type LoopSchedulerSyncMessage = {
|
|
20
|
+
type: 'loop-pattern-schedule-changed';
|
|
21
|
+
patternId: number;
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
function positiveInteger(value: unknown) {
|
|
25
|
+
const parsed = Number(value);
|
|
26
|
+
return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : null;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function isLockAcquireError(error: unknown) {
|
|
30
|
+
return error instanceof Error && error.constructor.name === 'LockAcquireError';
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function tickDate(pattern: LoopPattern, now: Date) {
|
|
34
|
+
const fields = pattern.cronExpression?.split(/\s+/).filter(Boolean).length || 5;
|
|
35
|
+
const interval = fields === 6 ? 1_000 : 60_000;
|
|
36
|
+
return new Date(Math.floor(now.getTime() / interval) * interval);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export class LoopSchedulerService {
|
|
40
|
+
private readonly scheduled = new Map<number, ScheduledPattern>();
|
|
41
|
+
private started = false;
|
|
42
|
+
private installed = false;
|
|
43
|
+
|
|
44
|
+
constructor(
|
|
45
|
+
private readonly plugin: Plugin,
|
|
46
|
+
private readonly patterns: LoopPatternService,
|
|
47
|
+
private readonly triggers: LoopTriggerService,
|
|
48
|
+
) {}
|
|
49
|
+
|
|
50
|
+
install() {
|
|
51
|
+
if (this.installed) return;
|
|
52
|
+
this.installed = true;
|
|
53
|
+
this.plugin.db.on('agentLoopPatterns.afterCreate', this.onPatternChanged);
|
|
54
|
+
this.plugin.db.on('agentLoopPatterns.afterUpdate', this.onPatternChanged);
|
|
55
|
+
this.plugin.db.on('agentLoopPatterns.afterDestroy', this.onPatternChanged);
|
|
56
|
+
this.plugin.app.on('afterStart', this.start);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
async dispose() {
|
|
60
|
+
if (!this.installed) return;
|
|
61
|
+
this.installed = false;
|
|
62
|
+
this.plugin.db.off('agentLoopPatterns.afterCreate', this.onPatternChanged);
|
|
63
|
+
this.plugin.db.off('agentLoopPatterns.afterUpdate', this.onPatternChanged);
|
|
64
|
+
this.plugin.db.off('agentLoopPatterns.afterDestroy', this.onPatternChanged);
|
|
65
|
+
this.plugin.app.off('afterStart', this.start);
|
|
66
|
+
this.stop();
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
readonly start = async () => {
|
|
70
|
+
if (this.started) return;
|
|
71
|
+
this.started = true;
|
|
72
|
+
await this.resyncAll();
|
|
73
|
+
};
|
|
74
|
+
|
|
75
|
+
stop() {
|
|
76
|
+
this.started = false;
|
|
77
|
+
for (const { job } of this.scheduled.values()) this.plugin.app.cronJobManager.removeJob(job);
|
|
78
|
+
this.scheduled.clear();
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
async handleSyncMessage(message: unknown) {
|
|
82
|
+
if (!message || typeof message !== 'object') return;
|
|
83
|
+
const value = message as Partial<LoopSchedulerSyncMessage>;
|
|
84
|
+
if (value.type !== 'loop-pattern-schedule-changed') return;
|
|
85
|
+
const patternId = positiveInteger(value.patternId);
|
|
86
|
+
if (!patternId) return;
|
|
87
|
+
await this.resyncPattern(patternId);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
async resyncAll() {
|
|
91
|
+
if (!this.started) return;
|
|
92
|
+
const records = await this.plugin.db.getRepository('agentLoopPatterns').find({ fields: ['id'] });
|
|
93
|
+
const patternIds = new Set<number>();
|
|
94
|
+
for (const record of records) {
|
|
95
|
+
const patternId = positiveInteger(read(record, 'id'));
|
|
96
|
+
if (!patternId) continue;
|
|
97
|
+
patternIds.add(patternId);
|
|
98
|
+
await this.resyncPattern(patternId);
|
|
99
|
+
}
|
|
100
|
+
for (const patternId of this.scheduled.keys()) {
|
|
101
|
+
if (!patternIds.has(patternId)) this.remove(patternId);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
async resyncPattern(patternId: number) {
|
|
106
|
+
if (!this.started) return;
|
|
107
|
+
const record = await this.plugin.db.getRepository('agentLoopPatterns').findOne({ filterByTk: patternId });
|
|
108
|
+
if (!record || read(record, 'enabled') !== true || read(record, 'triggerType') !== 'cron') {
|
|
109
|
+
this.remove(patternId);
|
|
110
|
+
return;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
try {
|
|
114
|
+
const { pattern } = await this.patterns.get(patternId);
|
|
115
|
+
const signature = JSON.stringify([pattern.cronExpression, pattern.timezone]);
|
|
116
|
+
if (this.scheduled.get(patternId)?.signature === signature) return;
|
|
117
|
+
this.remove(patternId);
|
|
118
|
+
const job = this.plugin.app.cronJobManager.addJob({
|
|
119
|
+
cronTime: pattern.cronExpression as string,
|
|
120
|
+
timeZone: pattern.timezone as string,
|
|
121
|
+
onTick: () => this.onTick(patternId, pattern).catch((error) => this.logTickError(patternId, error)),
|
|
122
|
+
});
|
|
123
|
+
this.scheduled.set(patternId, { job, signature });
|
|
124
|
+
if (this.plugin.app.cronJobManager.started) job.start();
|
|
125
|
+
this.plugin.app.logger.info(`[AgentOrchestrator] Scheduled loop pattern ${pattern.key}.`, {
|
|
126
|
+
patternId,
|
|
127
|
+
cronExpression: pattern.cronExpression,
|
|
128
|
+
timezone: pattern.timezone,
|
|
129
|
+
});
|
|
130
|
+
} catch (error) {
|
|
131
|
+
this.remove(patternId);
|
|
132
|
+
this.plugin.app.logger.error(`[AgentOrchestrator] Loop pattern ${patternId} could not be scheduled.`, { error });
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
private readonly onPatternChanged = async (record: Model, options: TransactionOptions = {}) => {
|
|
137
|
+
const patternId = positiveInteger(read(record, 'id'));
|
|
138
|
+
if (!patternId) return;
|
|
139
|
+
const apply = async () => {
|
|
140
|
+
await this.resyncPattern(patternId);
|
|
141
|
+
await this.plugin.sendSyncMessage({ type: 'loop-pattern-schedule-changed', patternId });
|
|
142
|
+
};
|
|
143
|
+
if (options.transaction) {
|
|
144
|
+
// afterCommit does not await its callback, so a rejection here would be
|
|
145
|
+
// an unhandled promise. Catch and log so a failed resync/broadcast does
|
|
146
|
+
// not silently drop the schedule change.
|
|
147
|
+
options.transaction.afterCommit(() =>
|
|
148
|
+
apply().catch((error) =>
|
|
149
|
+
this.plugin.app.logger.error(
|
|
150
|
+
`[AgentOrchestrator] Failed to apply schedule change for loop pattern ${patternId} after commit.`,
|
|
151
|
+
{ error },
|
|
152
|
+
),
|
|
153
|
+
),
|
|
154
|
+
);
|
|
155
|
+
return;
|
|
156
|
+
}
|
|
157
|
+
await apply();
|
|
158
|
+
};
|
|
159
|
+
|
|
160
|
+
private async onTick(patternId: number, pattern: LoopPattern) {
|
|
161
|
+
const scheduledAt = tickDate(pattern, new Date());
|
|
162
|
+
const triggerKey = `cron:${pattern.key}:${scheduledAt.toISOString()}`;
|
|
163
|
+
let lock;
|
|
164
|
+
try {
|
|
165
|
+
lock = await this.plugin.app.lockManager.tryAcquire(
|
|
166
|
+
`agent-loop:schedule:${this.plugin.app.name}:${patternId}:${triggerKey}`,
|
|
167
|
+
);
|
|
168
|
+
} catch (error) {
|
|
169
|
+
if (isLockAcquireError(error)) return;
|
|
170
|
+
throw error;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
await lock.runExclusive(async () => {
|
|
174
|
+
await this.triggers.enqueue({
|
|
175
|
+
patternId,
|
|
176
|
+
triggerType: 'cron',
|
|
177
|
+
triggerKey,
|
|
178
|
+
triggerPayload: { scheduledAt: scheduledAt.toISOString(), timezone: pattern.timezone },
|
|
179
|
+
});
|
|
180
|
+
}, 60_000);
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
private remove(patternId: number) {
|
|
184
|
+
const existing = this.scheduled.get(patternId);
|
|
185
|
+
if (!existing) return;
|
|
186
|
+
this.plugin.app.cronJobManager.removeJob(existing.job);
|
|
187
|
+
this.scheduled.delete(patternId);
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
private logTickError(patternId: number, error: unknown) {
|
|
191
|
+
this.plugin.app.logger.error(`[AgentOrchestrator] Scheduled loop pattern ${patternId} failed to enqueue.`, {
|
|
192
|
+
error,
|
|
193
|
+
});
|
|
194
|
+
}
|
|
195
|
+
}
|
|
@@ -3,14 +3,10 @@ import type { Database, Model } from '@nocobase/database';
|
|
|
3
3
|
import { LoopControlService } from './LoopControlService';
|
|
4
4
|
import { LoopPatternService } from './LoopPatternService';
|
|
5
5
|
import type { CompiledPatternSnapshot, HarnessSnapshot } from './LoopPatternService';
|
|
6
|
+
import { read } from '../utils/record-utils';
|
|
6
7
|
|
|
7
8
|
type TriggerType = 'manual' | 'cron' | 'event';
|
|
8
9
|
|
|
9
|
-
function read(record: Model | Record<string, unknown>, key: string) {
|
|
10
|
-
const model = record as Model & { get?: (name: string) => unknown };
|
|
11
|
-
return typeof model.get === 'function' ? model.get(key) : (record as Record<string, unknown>)[key];
|
|
12
|
-
}
|
|
13
|
-
|
|
14
10
|
function plain(record: Model | Record<string, unknown>) {
|
|
15
11
|
return typeof (record as Model).toJSON === 'function'
|
|
16
12
|
? ((record as Model).toJSON() as Record<string, unknown>)
|
|
@@ -43,6 +39,7 @@ export class LoopTriggerService {
|
|
|
43
39
|
constructor(
|
|
44
40
|
private readonly database: Database,
|
|
45
41
|
private readonly patterns: LoopPatternService,
|
|
42
|
+
private readonly logger?: { info?: (...args: unknown[]) => void; debug?: (...args: unknown[]) => void },
|
|
46
43
|
) {
|
|
47
44
|
this.control = new LoopControlService(database);
|
|
48
45
|
}
|
|
@@ -151,6 +148,11 @@ export class LoopTriggerService {
|
|
|
151
148
|
filter: { patternId: input.patternId, triggerKey },
|
|
152
149
|
});
|
|
153
150
|
if (!existing) throw error;
|
|
151
|
+
this.logger?.debug?.(
|
|
152
|
+
`[LoopTrigger] Deduplicated trigger key "${triggerKey}" for pattern ${input.patternId}: run ${Number(
|
|
153
|
+
existing.get?.('id') || (existing as Record<string, unknown>).id,
|
|
154
|
+
)} already exists.`,
|
|
155
|
+
);
|
|
154
156
|
return { created: false, run: plain(existing) };
|
|
155
157
|
}
|
|
156
158
|
}
|
|
@@ -13,7 +13,10 @@ import { PathLockService } from './PathLockService';
|
|
|
13
13
|
import { PluginAiRuntimeAdapter } from './PluginAiRuntimeAdapter';
|
|
14
14
|
import type { InvocationOutcome, ToolCallDecision } from './PluginAiRuntimeAdapter';
|
|
15
15
|
import { ToolLoopDetectionService, toolLoopNotice, toolLoopReason } from './ToolLoopDetectionService';
|
|
16
|
+
import { TokenTracker } from './TokenTracker';
|
|
16
17
|
import { VerificationService } from './VerificationService';
|
|
18
|
+
import { read } from '../utils/record-utils';
|
|
19
|
+
import { asObject } from '../utils/ctx-utils';
|
|
17
20
|
|
|
18
21
|
const POLL_INTERVAL_MS = 3_000;
|
|
19
22
|
const LEASE_MS = 120_000;
|
|
@@ -43,6 +46,37 @@ type ResumeContext = {
|
|
|
43
46
|
makerQueue: string[];
|
|
44
47
|
};
|
|
45
48
|
|
|
49
|
+
export type RunProgress = {
|
|
50
|
+
currentPhase: 'leader' | 'maker' | 'verifying' | 'complete';
|
|
51
|
+
totalPhases: number;
|
|
52
|
+
completedPhases: number;
|
|
53
|
+
currentStepTitle: string;
|
|
54
|
+
currentRole: string;
|
|
55
|
+
estimatedMsRemaining: number | null;
|
|
56
|
+
startedAt: string;
|
|
57
|
+
updatedAt: string;
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
function buildProgress(
|
|
61
|
+
phase: RunProgress['currentPhase'],
|
|
62
|
+
totalPhases: number,
|
|
63
|
+
completedPhases: number,
|
|
64
|
+
stepTitle: string,
|
|
65
|
+
role: string,
|
|
66
|
+
startedAt: Date,
|
|
67
|
+
): RunProgress {
|
|
68
|
+
return {
|
|
69
|
+
currentPhase: phase,
|
|
70
|
+
totalPhases,
|
|
71
|
+
completedPhases,
|
|
72
|
+
currentStepTitle: stepTitle,
|
|
73
|
+
currentRole: role,
|
|
74
|
+
estimatedMsRemaining: null,
|
|
75
|
+
startedAt: startedAt.toISOString(),
|
|
76
|
+
updatedAt: new Date().toISOString(),
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
|
|
46
80
|
type RunSnapshot = {
|
|
47
81
|
id: number;
|
|
48
82
|
patternId: number;
|
|
@@ -62,15 +96,6 @@ type RunSnapshot = {
|
|
|
62
96
|
resumeContext: ResumeContext | null;
|
|
63
97
|
};
|
|
64
98
|
|
|
65
|
-
function read(record: Model | Record<string, unknown>, key: string) {
|
|
66
|
-
const model = record as Model & { get?: (name: string) => unknown };
|
|
67
|
-
return typeof model.get === 'function' ? model.get(key) : (record as Record<string, unknown>)[key];
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
function asObject(value: unknown): Record<string, unknown> {
|
|
71
|
-
return value && typeof value === 'object' && !Array.isArray(value) ? (value as Record<string, unknown>) : {};
|
|
72
|
-
}
|
|
73
|
-
|
|
74
99
|
function asStringArray(value: unknown): string[] {
|
|
75
100
|
return Array.isArray(value) ? value.map((item) => String(item)).filter(Boolean) : [];
|
|
76
101
|
}
|
|
@@ -119,6 +144,7 @@ export class LoopWorkerService {
|
|
|
119
144
|
private readonly runtime: PluginAiRuntimeAdapter;
|
|
120
145
|
private readonly verification: VerificationService;
|
|
121
146
|
private readonly loopGuard: ToolLoopDetectionService;
|
|
147
|
+
private readonly tokenTracker: TokenTracker;
|
|
122
148
|
private readonly workerId: string;
|
|
123
149
|
private readonly active = new Map<number, AbortController>();
|
|
124
150
|
private readonly heldPathLocks = new Map<number, string>();
|
|
@@ -140,7 +166,8 @@ export class LoopWorkerService {
|
|
|
140
166
|
this.circuits = new DurableCircuitBreakerService(database, distributedLock);
|
|
141
167
|
this.pathLocks = new PathLockService(database, distributedLock);
|
|
142
168
|
this.runtime = new PluginAiRuntimeAdapter(app);
|
|
143
|
-
this.
|
|
169
|
+
this.tokenTracker = new TokenTracker({ app, db: database });
|
|
170
|
+
this.verification = new VerificationService(database, this.runtime, this.stateMachine, this.tokenTracker);
|
|
144
171
|
this.loopGuard = new ToolLoopDetectionService(database);
|
|
145
172
|
this.workerId = `${app.name}:${process.pid}:${randomUUID()}`;
|
|
146
173
|
}
|
|
@@ -197,6 +224,18 @@ export class LoopWorkerService {
|
|
|
197
224
|
await this.execute(claimed);
|
|
198
225
|
}
|
|
199
226
|
|
|
227
|
+
private async updateProgress(runId: number, progress: RunProgress) {
|
|
228
|
+
await this.database
|
|
229
|
+
.getRepository('agentLoopRuns')
|
|
230
|
+
.update({
|
|
231
|
+
filterByTk: runId,
|
|
232
|
+
values: { progress },
|
|
233
|
+
})
|
|
234
|
+
.catch((error: unknown) => {
|
|
235
|
+
this.app.logger?.warn?.(`[AgentOrchestrator] Failed to update progress for run ${runId}`, { error });
|
|
236
|
+
});
|
|
237
|
+
}
|
|
238
|
+
|
|
200
239
|
private async execute(claimed: ClaimedLoopRun) {
|
|
201
240
|
const runId = Number(claimed.run.id);
|
|
202
241
|
const controller = new AbortController();
|
|
@@ -287,6 +326,8 @@ export class LoopWorkerService {
|
|
|
287
326
|
// Loops recorded before this claim (for example before a park for approval) keep warning the
|
|
288
327
|
// prompts composed later in the run.
|
|
289
328
|
const loopNotices = await this.loopGuard.existingNotices(snapshot.id);
|
|
329
|
+
const totalPhases = 1 + snapshot.makerUsernames.length + 1;
|
|
330
|
+
const runStartedAt = new Date();
|
|
290
331
|
|
|
291
332
|
let makerReports: string[];
|
|
292
333
|
let makerQueue: string[];
|
|
@@ -340,6 +381,10 @@ export class LoopWorkerService {
|
|
|
340
381
|
loopNotices,
|
|
341
382
|
);
|
|
342
383
|
if (leaderGuard === 'halt') return;
|
|
384
|
+
await this.updateProgress(
|
|
385
|
+
snapshot.id,
|
|
386
|
+
buildProgress('leader', totalPhases, 1, 'Leader plan complete', 'leader', runStartedAt),
|
|
387
|
+
);
|
|
343
388
|
|
|
344
389
|
// plugin-ai stops the graph on an approval-gated tool. Approval is a human decision, so the
|
|
345
390
|
// run parks here instead of the worker deciding on the operator's behalf.
|
|
@@ -397,6 +442,11 @@ export class LoopWorkerService {
|
|
|
397
442
|
return;
|
|
398
443
|
}
|
|
399
444
|
makerReports.push(`# ${username}\n${maker.content}`);
|
|
445
|
+
const completedMakers = snapshot.makerUsernames.length - makerQueue.length + 1;
|
|
446
|
+
await this.updateProgress(
|
|
447
|
+
snapshot.id,
|
|
448
|
+
buildProgress('maker', totalPhases, 1 + completedMakers, `Maker "${username}" complete`, 'maker', runStartedAt),
|
|
449
|
+
);
|
|
400
450
|
makerQueue = makerQueue.slice(1);
|
|
401
451
|
}
|
|
402
452
|
const makerSummary = makerReports.join('\n\n');
|
|
@@ -413,6 +463,17 @@ export class LoopWorkerService {
|
|
|
413
463
|
correlationKey: `verifying:${snapshot.id}`,
|
|
414
464
|
values: { currentRole: 'verifier', finalAnswer: makerSummary },
|
|
415
465
|
});
|
|
466
|
+
await this.updateProgress(
|
|
467
|
+
snapshot.id,
|
|
468
|
+
buildProgress(
|
|
469
|
+
'verifying',
|
|
470
|
+
totalPhases,
|
|
471
|
+
totalPhases - 1,
|
|
472
|
+
`Verifier "${snapshot.verifierUsername}" reviewing`,
|
|
473
|
+
'verifier',
|
|
474
|
+
runStartedAt,
|
|
475
|
+
),
|
|
476
|
+
);
|
|
416
477
|
|
|
417
478
|
await this.budgets.reserve({
|
|
418
479
|
runId: snapshot.id,
|
|
@@ -430,6 +491,7 @@ export class LoopWorkerService {
|
|
|
430
491
|
verifierHarness: snapshot.verifierHarness,
|
|
431
492
|
policy: snapshot.policy,
|
|
432
493
|
makerSummary,
|
|
494
|
+
actingOn: snapshot.actingOn,
|
|
433
495
|
userId: snapshot.userId,
|
|
434
496
|
workerId: this.workerId,
|
|
435
497
|
leaseToken,
|
|
@@ -452,6 +514,17 @@ export class LoopWorkerService {
|
|
|
452
514
|
return;
|
|
453
515
|
}
|
|
454
516
|
await this.circuits.recordSuccess(snapshot.patternId, `pattern:${snapshot.patternId}`, snapshot.id);
|
|
517
|
+
await this.updateProgress(
|
|
518
|
+
snapshot.id,
|
|
519
|
+
buildProgress(
|
|
520
|
+
'complete',
|
|
521
|
+
totalPhases,
|
|
522
|
+
totalPhases,
|
|
523
|
+
'Run completed successfully',
|
|
524
|
+
snapshot.currentRole || 'verifier',
|
|
525
|
+
runStartedAt,
|
|
526
|
+
),
|
|
527
|
+
);
|
|
455
528
|
}
|
|
456
529
|
|
|
457
530
|
private async acquirePaths(snapshot: RunSnapshot, leaseToken: string) {
|
|
@@ -538,6 +611,15 @@ export class LoopWorkerService {
|
|
|
538
611
|
signal: input.signal,
|
|
539
612
|
});
|
|
540
613
|
|
|
614
|
+
// Track actual token usage from the LLM response; fall back to text-length estimation
|
|
615
|
+
await this.tokenTracker.trackActualUsage(
|
|
616
|
+
read(step, 'id'),
|
|
617
|
+
outcome.tokenUsage ?? null,
|
|
618
|
+
input.prompt,
|
|
619
|
+
outcome.content,
|
|
620
|
+
snapshot.id,
|
|
621
|
+
);
|
|
622
|
+
|
|
541
623
|
await this.database.getRepository('agentLoopSteps').update({
|
|
542
624
|
filterByTk: read(step, 'id'),
|
|
543
625
|
values: {
|
|
@@ -599,6 +681,15 @@ export class LoopWorkerService {
|
|
|
599
681
|
signal,
|
|
600
682
|
});
|
|
601
683
|
|
|
684
|
+
// Track actual token usage from the LLM response; fall back to text-length estimation
|
|
685
|
+
await this.tokenTracker.trackActualUsage(
|
|
686
|
+
read(step, 'id'),
|
|
687
|
+
outcome.tokenUsage ?? null,
|
|
688
|
+
resume.role === 'leader' ? LEADER_SYSTEM_MESSAGE : MAKER_SYSTEM_MESSAGE,
|
|
689
|
+
outcome.content,
|
|
690
|
+
snapshot.id,
|
|
691
|
+
);
|
|
692
|
+
|
|
602
693
|
await this.database.getRepository('agentLoopSteps').update({
|
|
603
694
|
filterByTk: read(step, 'id'),
|
|
604
695
|
values: {
|
|
@@ -785,8 +876,7 @@ export class LoopWorkerService {
|
|
|
785
876
|
}
|
|
786
877
|
|
|
787
878
|
private async nextSequence(runId: number) {
|
|
788
|
-
|
|
789
|
-
return steps.length;
|
|
879
|
+
return this.database.getRepository('agentLoopSteps').count({ filter: { runId } });
|
|
790
880
|
}
|
|
791
881
|
|
|
792
882
|
private snapshotFrom(run: Record<string, unknown>): RunSnapshot {
|