plugin-agent-orchestrator 1.1.5 → 1.1.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/dist/client/index.js +1 -1
  2. package/dist/client-v2/index.js +1 -1
  3. package/dist/externalVersion.js +7 -7
  4. package/dist/server/collections/agent-loop-runs.js +6 -0
  5. package/dist/server/collections/skill-metrics.js +66 -0
  6. package/dist/server/plugin.js +91 -2
  7. package/dist/server/resources/agent-loop-events-stream.js +1 -0
  8. package/dist/server/resources/agent-loop.js +1 -1
  9. package/dist/server/services/DurableCircuitBreakerService.js +9 -12
  10. package/dist/server/services/ExecutionSpanService.js +8 -3
  11. package/dist/server/services/HarnessProfileService.js +23 -26
  12. package/dist/server/services/LoopBudgetService.js +36 -39
  13. package/dist/server/services/LoopControlService.js +8 -11
  14. package/dist/server/services/LoopPatternSchema.js +51 -17
  15. package/dist/server/services/LoopPatternService.js +31 -11
  16. package/dist/server/services/LoopRunRepository.js +2 -5
  17. package/dist/server/services/LoopRunStateMachine.js +31 -34
  18. package/dist/server/services/LoopSchedulerService.js +4 -7
  19. package/dist/server/services/LoopTriggerService.js +11 -6
  20. package/dist/server/services/LoopWorkerService.js +88 -22
  21. package/dist/server/services/PathLockService.js +19 -22
  22. package/dist/server/services/PluginAiRuntimeAdapter.js +51 -8
  23. package/dist/server/services/RunEventBus.js +14 -2
  24. package/dist/server/services/SkillAccessService.js +50 -1
  25. package/dist/server/services/TokenTracker.js +17 -1
  26. package/dist/server/services/ToolLoopDetectionService.js +14 -17
  27. package/dist/server/services/VerificationService.js +47 -16
  28. package/dist/server/services/harness-runtime-policy.js +7 -12
  29. package/dist/server/skill-hub/plugin.js +34 -27
  30. package/dist/server/utils/ha-lock.js +73 -0
  31. package/dist/server/utils/loop-pattern-context.js +42 -5
  32. package/dist/server/utils/record-utils.js +44 -0
  33. package/package.json +1 -1
  34. package/src/client/plugin.tsx +1 -1
  35. package/src/client-v2/plugin.tsx +14 -13
  36. package/src/server/__tests__/tool-loop-detection.test.ts +1 -0
  37. package/src/server/collections/agent-loop-runs.ts +7 -0
  38. package/src/server/collections/skill-metrics.ts +36 -0
  39. package/src/server/plugin.ts +117 -1
  40. package/src/server/resources/agent-loop-events-stream.ts +108 -105
  41. package/src/server/resources/agent-loop.ts +1 -1
  42. package/src/server/resources/tracing.ts +267 -266
  43. package/src/server/services/DurableCircuitBreakerService.ts +235 -239
  44. package/src/server/services/ExecutionSpanService.ts +37 -12
  45. package/src/server/services/HarnessProfileService.ts +1 -5
  46. package/src/server/services/LoopBudgetService.ts +271 -275
  47. package/src/server/services/LoopControlService.ts +120 -124
  48. package/src/server/services/LoopPatternSchema.ts +60 -14
  49. package/src/server/services/LoopPatternService.ts +171 -143
  50. package/src/server/services/LoopRunRepository.ts +1 -5
  51. package/src/server/services/LoopRunStateMachine.ts +1 -5
  52. package/src/server/services/LoopSchedulerService.ts +195 -199
  53. package/src/server/services/LoopTriggerService.ts +7 -5
  54. package/src/server/services/LoopWorkerService.ts +102 -12
  55. package/src/server/services/PathLockService.ts +317 -321
  56. package/src/server/services/PluginAiRuntimeAdapter.ts +59 -4
  57. package/src/server/services/RunEventBus.ts +13 -0
  58. package/src/server/services/SkillAccessService.ts +80 -5
  59. package/src/server/services/TokenTracker.ts +59 -7
  60. package/src/server/services/ToolLoopDetectionService.ts +8 -11
  61. package/src/server/services/VerificationSchema.ts +31 -31
  62. package/src/server/services/VerificationService.ts +60 -11
  63. package/src/server/services/harness-runtime-policy.ts +2 -9
  64. package/src/server/skill-hub/plugin.ts +36 -28
  65. package/src/server/types.ts +5 -39
  66. package/src/server/utils/ensure-indexes.ts +96 -96
  67. package/src/server/utils/ha-lock.ts +80 -0
  68. package/src/server/utils/loop-pattern-context.ts +84 -20
  69. package/src/server/utils/record-utils.ts +19 -0
@@ -1,199 +1,195 @@
1
- import type { Database, Model, Transaction } from '@nocobase/database';
2
- import type { Application, Plugin } from '@nocobase/server';
3
- import type { LoopPattern } from './LoopPatternSchema';
4
- import { LoopPatternService } from './LoopPatternService';
5
- import { LoopTriggerService } from './LoopTriggerService';
6
-
7
- type ManagedCronJob = ReturnType<Application['cronJobManager']['addJob']>;
8
-
9
- type ScheduledPattern = {
10
- job: ManagedCronJob;
11
- signature: string;
12
- };
13
-
14
- type TransactionOptions = {
15
- transaction?: Transaction;
16
- };
17
-
18
- export type LoopSchedulerSyncMessage = {
19
- type: 'loop-pattern-schedule-changed';
20
- patternId: number;
21
- };
22
-
23
- function read(record: Model | Record<string, unknown>, key: string) {
24
- const model = record as Model & { get?: (name: string) => unknown };
25
- return typeof model.get === 'function' ? model.get(key) : (record as Record<string, unknown>)[key];
26
- }
27
-
28
- function positiveInteger(value: unknown) {
29
- const parsed = Number(value);
30
- return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : null;
31
- }
32
-
33
- function isLockAcquireError(error: unknown) {
34
- return error instanceof Error && error.constructor.name === 'LockAcquireError';
35
- }
36
-
37
- function tickDate(pattern: LoopPattern, now: Date) {
38
- const fields = pattern.cronExpression?.split(/\s+/).filter(Boolean).length || 5;
39
- const interval = fields === 6 ? 1_000 : 60_000;
40
- return new Date(Math.floor(now.getTime() / interval) * interval);
41
- }
42
-
43
- export class LoopSchedulerService {
44
- private readonly scheduled = new Map<number, ScheduledPattern>();
45
- private started = false;
46
- private installed = false;
47
-
48
- constructor(
49
- private readonly plugin: Plugin,
50
- private readonly patterns: LoopPatternService,
51
- private readonly triggers: LoopTriggerService,
52
- ) {}
53
-
54
- install() {
55
- if (this.installed) return;
56
- this.installed = true;
57
- this.plugin.db.on('agentLoopPatterns.afterCreate', this.onPatternChanged);
58
- this.plugin.db.on('agentLoopPatterns.afterUpdate', this.onPatternChanged);
59
- this.plugin.db.on('agentLoopPatterns.afterDestroy', this.onPatternChanged);
60
- this.plugin.app.on('afterStart', this.start);
61
- }
62
-
63
- async dispose() {
64
- if (!this.installed) return;
65
- this.installed = false;
66
- this.plugin.db.off('agentLoopPatterns.afterCreate', this.onPatternChanged);
67
- this.plugin.db.off('agentLoopPatterns.afterUpdate', this.onPatternChanged);
68
- this.plugin.db.off('agentLoopPatterns.afterDestroy', this.onPatternChanged);
69
- this.plugin.app.off('afterStart', this.start);
70
- this.stop();
71
- }
72
-
73
- readonly start = async () => {
74
- if (this.started) return;
75
- this.started = true;
76
- await this.resyncAll();
77
- };
78
-
79
- stop() {
80
- this.started = false;
81
- for (const { job } of this.scheduled.values()) this.plugin.app.cronJobManager.removeJob(job);
82
- this.scheduled.clear();
83
- }
84
-
85
- async handleSyncMessage(message: unknown) {
86
- if (!message || typeof message !== 'object') return;
87
- const value = message as Partial<LoopSchedulerSyncMessage>;
88
- if (value.type !== 'loop-pattern-schedule-changed') return;
89
- const patternId = positiveInteger(value.patternId);
90
- if (!patternId) return;
91
- await this.resyncPattern(patternId);
92
- }
93
-
94
- async resyncAll() {
95
- if (!this.started) return;
96
- const records = await this.plugin.db.getRepository('agentLoopPatterns').find({ fields: ['id'] });
97
- const patternIds = new Set<number>();
98
- for (const record of records) {
99
- const patternId = positiveInteger(read(record, 'id'));
100
- if (!patternId) continue;
101
- patternIds.add(patternId);
102
- await this.resyncPattern(patternId);
103
- }
104
- for (const patternId of this.scheduled.keys()) {
105
- if (!patternIds.has(patternId)) this.remove(patternId);
106
- }
107
- }
108
-
109
- async resyncPattern(patternId: number) {
110
- if (!this.started) return;
111
- const record = await this.plugin.db.getRepository('agentLoopPatterns').findOne({ filterByTk: patternId });
112
- if (!record || read(record, 'enabled') !== true || read(record, 'triggerType') !== 'cron') {
113
- this.remove(patternId);
114
- return;
115
- }
116
-
117
- try {
118
- const { pattern } = await this.patterns.get(patternId);
119
- const signature = JSON.stringify([pattern.cronExpression, pattern.timezone]);
120
- if (this.scheduled.get(patternId)?.signature === signature) return;
121
- this.remove(patternId);
122
- const job = this.plugin.app.cronJobManager.addJob({
123
- cronTime: pattern.cronExpression as string,
124
- timeZone: pattern.timezone as string,
125
- onTick: () => this.onTick(patternId, pattern).catch((error) => this.logTickError(patternId, error)),
126
- });
127
- this.scheduled.set(patternId, { job, signature });
128
- if (this.plugin.app.cronJobManager.started) job.start();
129
- this.plugin.app.logger.info(`[AgentOrchestrator] Scheduled loop pattern ${pattern.key}.`, {
130
- patternId,
131
- cronExpression: pattern.cronExpression,
132
- timezone: pattern.timezone,
133
- });
134
- } catch (error) {
135
- this.remove(patternId);
136
- this.plugin.app.logger.error(`[AgentOrchestrator] Loop pattern ${patternId} could not be scheduled.`, { error });
137
- }
138
- }
139
-
140
- private readonly onPatternChanged = async (record: Model, options: TransactionOptions = {}) => {
141
- const patternId = positiveInteger(read(record, 'id'));
142
- if (!patternId) return;
143
- const apply = async () => {
144
- await this.resyncPattern(patternId);
145
- await this.plugin.sendSyncMessage({ type: 'loop-pattern-schedule-changed', patternId });
146
- };
147
- if (options.transaction) {
148
- // afterCommit does not await its callback, so a rejection here would be
149
- // an unhandled promise. Catch and log so a failed resync/broadcast does
150
- // not silently drop the schedule change.
151
- options.transaction.afterCommit(() =>
152
- apply().catch((error) =>
153
- this.plugin.app.logger.error(
154
- `[AgentOrchestrator] Failed to apply schedule change for loop pattern ${patternId} after commit.`,
155
- { error },
156
- ),
157
- ),
158
- );
159
- return;
160
- }
161
- await apply();
162
- };
163
-
164
- private async onTick(patternId: number, pattern: LoopPattern) {
165
- const scheduledAt = tickDate(pattern, new Date());
166
- const triggerKey = `cron:${pattern.key}:${scheduledAt.toISOString()}`;
167
- let lock;
168
- try {
169
- lock = await this.plugin.app.lockManager.tryAcquire(
170
- `agent-loop:schedule:${this.plugin.app.name}:${patternId}:${triggerKey}`,
171
- );
172
- } catch (error) {
173
- if (isLockAcquireError(error)) return;
174
- throw error;
175
- }
176
-
177
- await lock.runExclusive(async () => {
178
- await this.triggers.enqueue({
179
- patternId,
180
- triggerType: 'cron',
181
- triggerKey,
182
- triggerPayload: { scheduledAt: scheduledAt.toISOString(), timezone: pattern.timezone },
183
- });
184
- }, 60_000);
185
- }
186
-
187
- private remove(patternId: number) {
188
- const existing = this.scheduled.get(patternId);
189
- if (!existing) return;
190
- this.plugin.app.cronJobManager.removeJob(existing.job);
191
- this.scheduled.delete(patternId);
192
- }
193
-
194
- private logTickError(patternId: number, error: unknown) {
195
- this.plugin.app.logger.error(`[AgentOrchestrator] Scheduled loop pattern ${patternId} failed to enqueue.`, {
196
- error,
197
- });
198
- }
199
- }
1
+ import type { Database, Model, Transaction } from '@nocobase/database';
2
+ import type { Application, Plugin } from '@nocobase/server';
3
+ import type { LoopPattern } from './LoopPatternSchema';
4
+ import { LoopPatternService } from './LoopPatternService';
5
+ import { LoopTriggerService } from './LoopTriggerService';
6
+ import { read } from '../utils/record-utils';
7
+
8
+ type ManagedCronJob = ReturnType<Application['cronJobManager']['addJob']>;
9
+
10
+ type ScheduledPattern = {
11
+ job: ManagedCronJob;
12
+ signature: string;
13
+ };
14
+
15
+ type TransactionOptions = {
16
+ transaction?: Transaction;
17
+ };
18
+
19
+ export type LoopSchedulerSyncMessage = {
20
+ type: 'loop-pattern-schedule-changed';
21
+ patternId: number;
22
+ };
23
+
24
+ function positiveInteger(value: unknown) {
25
+ const parsed = Number(value);
26
+ return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : null;
27
+ }
28
+
29
+ function isLockAcquireError(error: unknown) {
30
+ return error instanceof Error && error.constructor.name === 'LockAcquireError';
31
+ }
32
+
33
+ function tickDate(pattern: LoopPattern, now: Date) {
34
+ const fields = pattern.cronExpression?.split(/\s+/).filter(Boolean).length || 5;
35
+ const interval = fields === 6 ? 1_000 : 60_000;
36
+ return new Date(Math.floor(now.getTime() / interval) * interval);
37
+ }
38
+
39
+ export class LoopSchedulerService {
40
+ private readonly scheduled = new Map<number, ScheduledPattern>();
41
+ private started = false;
42
+ private installed = false;
43
+
44
+ constructor(
45
+ private readonly plugin: Plugin,
46
+ private readonly patterns: LoopPatternService,
47
+ private readonly triggers: LoopTriggerService,
48
+ ) {}
49
+
50
+ install() {
51
+ if (this.installed) return;
52
+ this.installed = true;
53
+ this.plugin.db.on('agentLoopPatterns.afterCreate', this.onPatternChanged);
54
+ this.plugin.db.on('agentLoopPatterns.afterUpdate', this.onPatternChanged);
55
+ this.plugin.db.on('agentLoopPatterns.afterDestroy', this.onPatternChanged);
56
+ this.plugin.app.on('afterStart', this.start);
57
+ }
58
+
59
+ async dispose() {
60
+ if (!this.installed) return;
61
+ this.installed = false;
62
+ this.plugin.db.off('agentLoopPatterns.afterCreate', this.onPatternChanged);
63
+ this.plugin.db.off('agentLoopPatterns.afterUpdate', this.onPatternChanged);
64
+ this.plugin.db.off('agentLoopPatterns.afterDestroy', this.onPatternChanged);
65
+ this.plugin.app.off('afterStart', this.start);
66
+ this.stop();
67
+ }
68
+
69
+ readonly start = async () => {
70
+ if (this.started) return;
71
+ this.started = true;
72
+ await this.resyncAll();
73
+ };
74
+
75
+ stop() {
76
+ this.started = false;
77
+ for (const { job } of this.scheduled.values()) this.plugin.app.cronJobManager.removeJob(job);
78
+ this.scheduled.clear();
79
+ }
80
+
81
+ async handleSyncMessage(message: unknown) {
82
+ if (!message || typeof message !== 'object') return;
83
+ const value = message as Partial<LoopSchedulerSyncMessage>;
84
+ if (value.type !== 'loop-pattern-schedule-changed') return;
85
+ const patternId = positiveInteger(value.patternId);
86
+ if (!patternId) return;
87
+ await this.resyncPattern(patternId);
88
+ }
89
+
90
+ async resyncAll() {
91
+ if (!this.started) return;
92
+ const records = await this.plugin.db.getRepository('agentLoopPatterns').find({ fields: ['id'] });
93
+ const patternIds = new Set<number>();
94
+ for (const record of records) {
95
+ const patternId = positiveInteger(read(record, 'id'));
96
+ if (!patternId) continue;
97
+ patternIds.add(patternId);
98
+ await this.resyncPattern(patternId);
99
+ }
100
+ for (const patternId of this.scheduled.keys()) {
101
+ if (!patternIds.has(patternId)) this.remove(patternId);
102
+ }
103
+ }
104
+
105
+ async resyncPattern(patternId: number) {
106
+ if (!this.started) return;
107
+ const record = await this.plugin.db.getRepository('agentLoopPatterns').findOne({ filterByTk: patternId });
108
+ if (!record || read(record, 'enabled') !== true || read(record, 'triggerType') !== 'cron') {
109
+ this.remove(patternId);
110
+ return;
111
+ }
112
+
113
+ try {
114
+ const { pattern } = await this.patterns.get(patternId);
115
+ const signature = JSON.stringify([pattern.cronExpression, pattern.timezone]);
116
+ if (this.scheduled.get(patternId)?.signature === signature) return;
117
+ this.remove(patternId);
118
+ const job = this.plugin.app.cronJobManager.addJob({
119
+ cronTime: pattern.cronExpression as string,
120
+ timeZone: pattern.timezone as string,
121
+ onTick: () => this.onTick(patternId, pattern).catch((error) => this.logTickError(patternId, error)),
122
+ });
123
+ this.scheduled.set(patternId, { job, signature });
124
+ if (this.plugin.app.cronJobManager.started) job.start();
125
+ this.plugin.app.logger.info(`[AgentOrchestrator] Scheduled loop pattern ${pattern.key}.`, {
126
+ patternId,
127
+ cronExpression: pattern.cronExpression,
128
+ timezone: pattern.timezone,
129
+ });
130
+ } catch (error) {
131
+ this.remove(patternId);
132
+ this.plugin.app.logger.error(`[AgentOrchestrator] Loop pattern ${patternId} could not be scheduled.`, { error });
133
+ }
134
+ }
135
+
136
+ private readonly onPatternChanged = async (record: Model, options: TransactionOptions = {}) => {
137
+ const patternId = positiveInteger(read(record, 'id'));
138
+ if (!patternId) return;
139
+ const apply = async () => {
140
+ await this.resyncPattern(patternId);
141
+ await this.plugin.sendSyncMessage({ type: 'loop-pattern-schedule-changed', patternId });
142
+ };
143
+ if (options.transaction) {
144
+ // afterCommit does not await its callback, so a rejection here would be
145
+ // an unhandled promise. Catch and log so a failed resync/broadcast does
146
+ // not silently drop the schedule change.
147
+ options.transaction.afterCommit(() =>
148
+ apply().catch((error) =>
149
+ this.plugin.app.logger.error(
150
+ `[AgentOrchestrator] Failed to apply schedule change for loop pattern ${patternId} after commit.`,
151
+ { error },
152
+ ),
153
+ ),
154
+ );
155
+ return;
156
+ }
157
+ await apply();
158
+ };
159
+
160
+ private async onTick(patternId: number, pattern: LoopPattern) {
161
+ const scheduledAt = tickDate(pattern, new Date());
162
+ const triggerKey = `cron:${pattern.key}:${scheduledAt.toISOString()}`;
163
+ let lock;
164
+ try {
165
+ lock = await this.plugin.app.lockManager.tryAcquire(
166
+ `agent-loop:schedule:${this.plugin.app.name}:${patternId}:${triggerKey}`,
167
+ );
168
+ } catch (error) {
169
+ if (isLockAcquireError(error)) return;
170
+ throw error;
171
+ }
172
+
173
+ await lock.runExclusive(async () => {
174
+ await this.triggers.enqueue({
175
+ patternId,
176
+ triggerType: 'cron',
177
+ triggerKey,
178
+ triggerPayload: { scheduledAt: scheduledAt.toISOString(), timezone: pattern.timezone },
179
+ });
180
+ }, 60_000);
181
+ }
182
+
183
+ private remove(patternId: number) {
184
+ const existing = this.scheduled.get(patternId);
185
+ if (!existing) return;
186
+ this.plugin.app.cronJobManager.removeJob(existing.job);
187
+ this.scheduled.delete(patternId);
188
+ }
189
+
190
+ private logTickError(patternId: number, error: unknown) {
191
+ this.plugin.app.logger.error(`[AgentOrchestrator] Scheduled loop pattern ${patternId} failed to enqueue.`, {
192
+ error,
193
+ });
194
+ }
195
+ }
@@ -3,14 +3,10 @@ import type { Database, Model } from '@nocobase/database';
3
3
  import { LoopControlService } from './LoopControlService';
4
4
  import { LoopPatternService } from './LoopPatternService';
5
5
  import type { CompiledPatternSnapshot, HarnessSnapshot } from './LoopPatternService';
6
+ import { read } from '../utils/record-utils';
6
7
 
7
8
  type TriggerType = 'manual' | 'cron' | 'event';
8
9
 
9
- function read(record: Model | Record<string, unknown>, key: string) {
10
- const model = record as Model & { get?: (name: string) => unknown };
11
- return typeof model.get === 'function' ? model.get(key) : (record as Record<string, unknown>)[key];
12
- }
13
-
14
10
  function plain(record: Model | Record<string, unknown>) {
15
11
  return typeof (record as Model).toJSON === 'function'
16
12
  ? ((record as Model).toJSON() as Record<string, unknown>)
@@ -43,6 +39,7 @@ export class LoopTriggerService {
43
39
  constructor(
44
40
  private readonly database: Database,
45
41
  private readonly patterns: LoopPatternService,
42
+ private readonly logger?: { info?: (...args: unknown[]) => void; debug?: (...args: unknown[]) => void },
46
43
  ) {
47
44
  this.control = new LoopControlService(database);
48
45
  }
@@ -151,6 +148,11 @@ export class LoopTriggerService {
151
148
  filter: { patternId: input.patternId, triggerKey },
152
149
  });
153
150
  if (!existing) throw error;
151
+ this.logger?.debug?.(
152
+ `[LoopTrigger] Deduplicated trigger key "${triggerKey}" for pattern ${input.patternId}: run ${Number(
153
+ existing.get?.('id') || (existing as Record<string, unknown>).id,
154
+ )} already exists.`,
155
+ );
154
156
  return { created: false, run: plain(existing) };
155
157
  }
156
158
  }
@@ -13,7 +13,10 @@ import { PathLockService } from './PathLockService';
13
13
  import { PluginAiRuntimeAdapter } from './PluginAiRuntimeAdapter';
14
14
  import type { InvocationOutcome, ToolCallDecision } from './PluginAiRuntimeAdapter';
15
15
  import { ToolLoopDetectionService, toolLoopNotice, toolLoopReason } from './ToolLoopDetectionService';
16
+ import { TokenTracker } from './TokenTracker';
16
17
  import { VerificationService } from './VerificationService';
18
+ import { read } from '../utils/record-utils';
19
+ import { asObject } from '../utils/ctx-utils';
17
20
 
18
21
  const POLL_INTERVAL_MS = 3_000;
19
22
  const LEASE_MS = 120_000;
@@ -43,6 +46,37 @@ type ResumeContext = {
43
46
  makerQueue: string[];
44
47
  };
45
48
 
49
+ export type RunProgress = {
50
+ currentPhase: 'leader' | 'maker' | 'verifying' | 'complete';
51
+ totalPhases: number;
52
+ completedPhases: number;
53
+ currentStepTitle: string;
54
+ currentRole: string;
55
+ estimatedMsRemaining: number | null;
56
+ startedAt: string;
57
+ updatedAt: string;
58
+ };
59
+
60
+ function buildProgress(
61
+ phase: RunProgress['currentPhase'],
62
+ totalPhases: number,
63
+ completedPhases: number,
64
+ stepTitle: string,
65
+ role: string,
66
+ startedAt: Date,
67
+ ): RunProgress {
68
+ return {
69
+ currentPhase: phase,
70
+ totalPhases,
71
+ completedPhases,
72
+ currentStepTitle: stepTitle,
73
+ currentRole: role,
74
+ estimatedMsRemaining: null,
75
+ startedAt: startedAt.toISOString(),
76
+ updatedAt: new Date().toISOString(),
77
+ };
78
+ }
79
+
46
80
  type RunSnapshot = {
47
81
  id: number;
48
82
  patternId: number;
@@ -62,15 +96,6 @@ type RunSnapshot = {
62
96
  resumeContext: ResumeContext | null;
63
97
  };
64
98
 
65
- function read(record: Model | Record<string, unknown>, key: string) {
66
- const model = record as Model & { get?: (name: string) => unknown };
67
- return typeof model.get === 'function' ? model.get(key) : (record as Record<string, unknown>)[key];
68
- }
69
-
70
- function asObject(value: unknown): Record<string, unknown> {
71
- return value && typeof value === 'object' && !Array.isArray(value) ? (value as Record<string, unknown>) : {};
72
- }
73
-
74
99
  function asStringArray(value: unknown): string[] {
75
100
  return Array.isArray(value) ? value.map((item) => String(item)).filter(Boolean) : [];
76
101
  }
@@ -119,6 +144,7 @@ export class LoopWorkerService {
119
144
  private readonly runtime: PluginAiRuntimeAdapter;
120
145
  private readonly verification: VerificationService;
121
146
  private readonly loopGuard: ToolLoopDetectionService;
147
+ private readonly tokenTracker: TokenTracker;
122
148
  private readonly workerId: string;
123
149
  private readonly active = new Map<number, AbortController>();
124
150
  private readonly heldPathLocks = new Map<number, string>();
@@ -140,7 +166,8 @@ export class LoopWorkerService {
140
166
  this.circuits = new DurableCircuitBreakerService(database, distributedLock);
141
167
  this.pathLocks = new PathLockService(database, distributedLock);
142
168
  this.runtime = new PluginAiRuntimeAdapter(app);
143
- this.verification = new VerificationService(database, this.runtime, this.stateMachine);
169
+ this.tokenTracker = new TokenTracker({ app, db: database });
170
+ this.verification = new VerificationService(database, this.runtime, this.stateMachine, this.tokenTracker);
144
171
  this.loopGuard = new ToolLoopDetectionService(database);
145
172
  this.workerId = `${app.name}:${process.pid}:${randomUUID()}`;
146
173
  }
@@ -197,6 +224,18 @@ export class LoopWorkerService {
197
224
  await this.execute(claimed);
198
225
  }
199
226
 
227
+ private async updateProgress(runId: number, progress: RunProgress) {
228
+ await this.database
229
+ .getRepository('agentLoopRuns')
230
+ .update({
231
+ filterByTk: runId,
232
+ values: { progress },
233
+ })
234
+ .catch((error: unknown) => {
235
+ this.app.logger?.warn?.(`[AgentOrchestrator] Failed to update progress for run ${runId}`, { error });
236
+ });
237
+ }
238
+
200
239
  private async execute(claimed: ClaimedLoopRun) {
201
240
  const runId = Number(claimed.run.id);
202
241
  const controller = new AbortController();
@@ -287,6 +326,8 @@ export class LoopWorkerService {
287
326
  // Loops recorded before this claim (for example before a park for approval) keep warning the
288
327
  // prompts composed later in the run.
289
328
  const loopNotices = await this.loopGuard.existingNotices(snapshot.id);
329
+ const totalPhases = 1 + snapshot.makerUsernames.length + 1;
330
+ const runStartedAt = new Date();
290
331
 
291
332
  let makerReports: string[];
292
333
  let makerQueue: string[];
@@ -340,6 +381,10 @@ export class LoopWorkerService {
340
381
  loopNotices,
341
382
  );
342
383
  if (leaderGuard === 'halt') return;
384
+ await this.updateProgress(
385
+ snapshot.id,
386
+ buildProgress('leader', totalPhases, 1, 'Leader plan complete', 'leader', runStartedAt),
387
+ );
343
388
 
344
389
  // plugin-ai stops the graph on an approval-gated tool. Approval is a human decision, so the
345
390
  // run parks here instead of the worker deciding on the operator's behalf.
@@ -397,6 +442,11 @@ export class LoopWorkerService {
397
442
  return;
398
443
  }
399
444
  makerReports.push(`# ${username}\n${maker.content}`);
445
+ const completedMakers = snapshot.makerUsernames.length - makerQueue.length + 1;
446
+ await this.updateProgress(
447
+ snapshot.id,
448
+ buildProgress('maker', totalPhases, 1 + completedMakers, `Maker "${username}" complete`, 'maker', runStartedAt),
449
+ );
400
450
  makerQueue = makerQueue.slice(1);
401
451
  }
402
452
  const makerSummary = makerReports.join('\n\n');
@@ -413,6 +463,17 @@ export class LoopWorkerService {
413
463
  correlationKey: `verifying:${snapshot.id}`,
414
464
  values: { currentRole: 'verifier', finalAnswer: makerSummary },
415
465
  });
466
+ await this.updateProgress(
467
+ snapshot.id,
468
+ buildProgress(
469
+ 'verifying',
470
+ totalPhases,
471
+ totalPhases - 1,
472
+ `Verifier "${snapshot.verifierUsername}" reviewing`,
473
+ 'verifier',
474
+ runStartedAt,
475
+ ),
476
+ );
416
477
 
417
478
  await this.budgets.reserve({
418
479
  runId: snapshot.id,
@@ -430,6 +491,7 @@ export class LoopWorkerService {
430
491
  verifierHarness: snapshot.verifierHarness,
431
492
  policy: snapshot.policy,
432
493
  makerSummary,
494
+ actingOn: snapshot.actingOn,
433
495
  userId: snapshot.userId,
434
496
  workerId: this.workerId,
435
497
  leaseToken,
@@ -452,6 +514,17 @@ export class LoopWorkerService {
452
514
  return;
453
515
  }
454
516
  await this.circuits.recordSuccess(snapshot.patternId, `pattern:${snapshot.patternId}`, snapshot.id);
517
+ await this.updateProgress(
518
+ snapshot.id,
519
+ buildProgress(
520
+ 'complete',
521
+ totalPhases,
522
+ totalPhases,
523
+ 'Run completed successfully',
524
+ snapshot.currentRole || 'verifier',
525
+ runStartedAt,
526
+ ),
527
+ );
455
528
  }
456
529
 
457
530
  private async acquirePaths(snapshot: RunSnapshot, leaseToken: string) {
@@ -538,6 +611,15 @@ export class LoopWorkerService {
538
611
  signal: input.signal,
539
612
  });
540
613
 
614
+ // Track actual token usage from the LLM response; fall back to text-length estimation
615
+ await this.tokenTracker.trackActualUsage(
616
+ read(step, 'id'),
617
+ outcome.tokenUsage ?? null,
618
+ input.prompt,
619
+ outcome.content,
620
+ snapshot.id,
621
+ );
622
+
541
623
  await this.database.getRepository('agentLoopSteps').update({
542
624
  filterByTk: read(step, 'id'),
543
625
  values: {
@@ -599,6 +681,15 @@ export class LoopWorkerService {
599
681
  signal,
600
682
  });
601
683
 
684
+ // Track actual token usage from the LLM response; fall back to text-length estimation
685
+ await this.tokenTracker.trackActualUsage(
686
+ read(step, 'id'),
687
+ outcome.tokenUsage ?? null,
688
+ resume.role === 'leader' ? LEADER_SYSTEM_MESSAGE : MAKER_SYSTEM_MESSAGE,
689
+ outcome.content,
690
+ snapshot.id,
691
+ );
692
+
602
693
  await this.database.getRepository('agentLoopSteps').update({
603
694
  filterByTk: read(step, 'id'),
604
695
  values: {
@@ -785,8 +876,7 @@ export class LoopWorkerService {
785
876
  }
786
877
 
787
878
  private async nextSequence(runId: number) {
788
- const steps = await this.database.getRepository('agentLoopSteps').find({ filter: { runId } });
789
- return steps.length;
879
+ return this.database.getRepository('agentLoopSteps').count({ filter: { runId } });
790
880
  }
791
881
 
792
882
  private snapshotFrom(run: Record<string, unknown>): RunSnapshot {