plugin-agent-orchestrator 1.1.5 → 1.1.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/dist/client/index.js +1 -1
  2. package/dist/client-v2/index.js +1 -1
  3. package/dist/externalVersion.js +7 -7
  4. package/dist/server/collections/agent-loop-runs.js +6 -0
  5. package/dist/server/collections/skill-metrics.js +66 -0
  6. package/dist/server/plugin.js +91 -2
  7. package/dist/server/resources/agent-loop-events-stream.js +1 -0
  8. package/dist/server/resources/agent-loop.js +1 -1
  9. package/dist/server/services/DurableCircuitBreakerService.js +9 -12
  10. package/dist/server/services/ExecutionSpanService.js +8 -3
  11. package/dist/server/services/HarnessProfileService.js +23 -26
  12. package/dist/server/services/LoopBudgetService.js +36 -39
  13. package/dist/server/services/LoopControlService.js +8 -11
  14. package/dist/server/services/LoopPatternSchema.js +51 -17
  15. package/dist/server/services/LoopPatternService.js +31 -11
  16. package/dist/server/services/LoopRunRepository.js +2 -5
  17. package/dist/server/services/LoopRunStateMachine.js +31 -34
  18. package/dist/server/services/LoopSchedulerService.js +4 -7
  19. package/dist/server/services/LoopTriggerService.js +11 -6
  20. package/dist/server/services/LoopWorkerService.js +88 -22
  21. package/dist/server/services/PathLockService.js +19 -22
  22. package/dist/server/services/PluginAiRuntimeAdapter.js +51 -8
  23. package/dist/server/services/RunEventBus.js +14 -2
  24. package/dist/server/services/SkillAccessService.js +50 -1
  25. package/dist/server/services/TokenTracker.js +17 -1
  26. package/dist/server/services/ToolLoopDetectionService.js +14 -17
  27. package/dist/server/services/VerificationService.js +47 -16
  28. package/dist/server/services/harness-runtime-policy.js +7 -12
  29. package/dist/server/skill-hub/plugin.js +34 -27
  30. package/dist/server/utils/ha-lock.js +73 -0
  31. package/dist/server/utils/loop-pattern-context.js +42 -5
  32. package/dist/server/utils/record-utils.js +44 -0
  33. package/package.json +1 -1
  34. package/src/client/plugin.tsx +1 -1
  35. package/src/client-v2/plugin.tsx +14 -13
  36. package/src/server/__tests__/tool-loop-detection.test.ts +1 -0
  37. package/src/server/collections/agent-loop-runs.ts +7 -0
  38. package/src/server/collections/skill-metrics.ts +36 -0
  39. package/src/server/plugin.ts +117 -1
  40. package/src/server/resources/agent-loop-events-stream.ts +108 -105
  41. package/src/server/resources/agent-loop.ts +1 -1
  42. package/src/server/resources/tracing.ts +267 -266
  43. package/src/server/services/DurableCircuitBreakerService.ts +235 -239
  44. package/src/server/services/ExecutionSpanService.ts +37 -12
  45. package/src/server/services/HarnessProfileService.ts +1 -5
  46. package/src/server/services/LoopBudgetService.ts +271 -275
  47. package/src/server/services/LoopControlService.ts +120 -124
  48. package/src/server/services/LoopPatternSchema.ts +60 -14
  49. package/src/server/services/LoopPatternService.ts +171 -143
  50. package/src/server/services/LoopRunRepository.ts +1 -5
  51. package/src/server/services/LoopRunStateMachine.ts +1 -5
  52. package/src/server/services/LoopSchedulerService.ts +195 -199
  53. package/src/server/services/LoopTriggerService.ts +7 -5
  54. package/src/server/services/LoopWorkerService.ts +102 -12
  55. package/src/server/services/PathLockService.ts +317 -321
  56. package/src/server/services/PluginAiRuntimeAdapter.ts +59 -4
  57. package/src/server/services/RunEventBus.ts +13 -0
  58. package/src/server/services/SkillAccessService.ts +80 -5
  59. package/src/server/services/TokenTracker.ts +59 -7
  60. package/src/server/services/ToolLoopDetectionService.ts +8 -11
  61. package/src/server/services/VerificationSchema.ts +31 -31
  62. package/src/server/services/VerificationService.ts +60 -11
  63. package/src/server/services/harness-runtime-policy.ts +2 -9
  64. package/src/server/skill-hub/plugin.ts +36 -28
  65. package/src/server/types.ts +5 -39
  66. package/src/server/utils/ensure-indexes.ts +96 -96
  67. package/src/server/utils/ha-lock.ts +80 -0
  68. package/src/server/utils/loop-pattern-context.ts +84 -20
  69. package/src/server/utils/record-utils.ts +19 -0
@@ -2,9 +2,11 @@ import type { Model } from '@nocobase/database';
2
2
  import type { Application } from '@nocobase/server';
3
3
  import { AIEmployee } from '@nocobase/plugin-ai/server';
4
4
  import { listSystemTools } from '@nocobase/ai';
5
+ import { estimateCost } from './TokenTracker';
5
6
  import { getAIToolsManager } from '../utils/ai-manager';
6
7
  import { decideTool } from './HarnessCompiler';
7
8
  import type { CompiledHarness } from './HarnessCompiler';
9
+ import { read } from '../utils/record-utils';
8
10
 
9
11
  // plugin-ai keeps every tool whose name is in `listSystemTools()` regardless of the constructor
10
12
  // tool filter (ai-employee.ts getAIEmployeeTools). The harness `filter` cannot remove them, so a
@@ -30,11 +32,19 @@ export type InterruptedToolCall = {
30
32
  interruptId: string;
31
33
  };
32
34
 
35
+ export type TokenUsageReport = {
36
+ inputTokens: number;
37
+ outputTokens: number;
38
+ totalTokens: number;
39
+ cost: number;
40
+ };
41
+
33
42
  export type InvocationOutcome = {
34
43
  sessionId: string;
35
44
  messageId: string;
36
45
  interrupted: InterruptedToolCall[];
37
46
  content: string;
47
+ tokenUsage: TokenUsageReport | null;
38
48
  };
39
49
 
40
50
  type ConversationsManager = {
@@ -58,9 +68,48 @@ type PluginAi = {
58
68
  aiEmployeesManager: EmployeesManager;
59
69
  };
60
70
 
61
- function read(record: Model | Record<string, unknown>, key: string) {
62
- const model = record as Model & { get?: (name: string) => unknown };
63
- return typeof model.get === 'function' ? model.get(key) : (record as Record<string, unknown>)[key];
71
+ function extractTokenUsageFromResult(result: Record<string, unknown>): TokenUsageReport | null {
72
+ // Try direct usage_metadata from the last AI message (LangChain standard)
73
+ const messages = result.messages;
74
+ if (Array.isArray(messages)) {
75
+ let totalInput = 0;
76
+ let totalOutput = 0;
77
+ let totalAll = 0;
78
+ for (const msg of messages) {
79
+ const um = (msg as Record<string, unknown>)?.usage_metadata as Record<string, unknown> | undefined;
80
+ if (um) {
81
+ totalInput += Number(um.input_tokens || 0);
82
+ totalOutput += Number(um.output_tokens || 0);
83
+ totalAll += Number(um.total_tokens || 0);
84
+ }
85
+ }
86
+ if (totalAll > 0) {
87
+ return {
88
+ inputTokens: totalInput,
89
+ outputTokens: totalOutput,
90
+ totalTokens: totalAll,
91
+ cost: estimateCost(totalInput, totalOutput),
92
+ };
93
+ }
94
+ }
95
+
96
+ // Try usage field (some providers return this)
97
+ const usage = result.usage as Record<string, unknown> | undefined;
98
+ if (usage) {
99
+ const inputTokens = Number(usage.prompt_tokens || usage.input_tokens || 0);
100
+ const outputTokens = Number(usage.completion_tokens || usage.output_tokens || 0);
101
+ const totalTokens = Number(usage.total_tokens || inputTokens + outputTokens);
102
+ if (totalTokens > 0) {
103
+ return {
104
+ inputTokens,
105
+ outputTokens,
106
+ totalTokens,
107
+ cost: estimateCost(inputTokens, outputTokens),
108
+ };
109
+ }
110
+ }
111
+
112
+ return null;
64
113
  }
65
114
 
66
115
  function textContent(messages: unknown): string {
@@ -211,6 +260,7 @@ export class PluginAiRuntimeAdapter {
211
260
  messageId: String(result.messageId || ''),
212
261
  interrupted: interruptedFrom(result),
213
262
  content: textContent(result.messages),
263
+ tokenUsage: extractTokenUsageFromResult(result),
214
264
  };
215
265
  }
216
266
 
@@ -259,6 +309,7 @@ export class PluginAiRuntimeAdapter {
259
309
  messageId: String(result.messageId || input.messageId),
260
310
  interrupted: interruptedFrom(result),
261
311
  content: textContent(result.messages),
312
+ tokenUsage: extractTokenUsageFromResult(result),
262
313
  };
263
314
  }
264
315
 
@@ -283,7 +334,11 @@ export class PluginAiRuntimeAdapter {
283
334
  db: this.app.db,
284
335
  log: this.app.log,
285
336
  logger: this.app.log,
286
- state: { currentRoles },
337
+ state: {
338
+ currentRoles,
339
+ // Populate currentUser so currentUserId(ctx) resolves correctly.
340
+ currentUser: input.userId ? { id: input.userId } : undefined,
341
+ },
287
342
  auth: { user: { id: input.userId } },
288
343
  action: { params: { values: { sessionId: input.sessionId, model } } },
289
344
  };
@@ -33,6 +33,11 @@ class RunEventBusImpl {
33
33
  listenerCount(runId: string | number): number {
34
34
  return this.listeners.get(runId)?.size || 0;
35
35
  }
36
+
37
+ /** Remove all listeners for all runs. Call during plugin shutdown. */
38
+ destroy() {
39
+ this.listeners.clear();
40
+ }
36
41
  }
37
42
 
38
43
  let instance: RunEventBusImpl | null = null;
@@ -43,3 +48,11 @@ export function getRunEventBus(): RunEventBusImpl {
43
48
  }
44
49
  return instance;
45
50
  }
51
+
52
+ /** Reset the singleton (for tests or shutdown). */
53
+ export function resetRunEventBus() {
54
+ if (instance) {
55
+ instance.destroy();
56
+ instance = null;
57
+ }
58
+ }
@@ -10,9 +10,22 @@ function bindingName(binding: ToolBinding): string {
10
10
  }
11
11
 
12
12
  export class SkillAccessService {
13
- constructor(private readonly plugin: { db: { getRepository: (name: string) => any } }) {}
13
+ constructor(
14
+ private readonly plugin: {
15
+ db: {
16
+ getRepository: (name: string) =>
17
+ | {
18
+ find: (opts?: Record<string, unknown>) => Promise<unknown[]>;
19
+ findOne: (opts: Record<string, unknown>) => Promise<unknown>;
20
+ update: (opts: Record<string, unknown>) => Promise<unknown>;
21
+ }
22
+ | undefined;
23
+ };
24
+ app?: { logger?: { info?: (...args: unknown[]) => void } };
25
+ },
26
+ ) {}
14
27
 
15
- async resolveAgentUsername(ctx: any): Promise<string | undefined> {
28
+ async resolveAgentUsername(ctx: Record<string, unknown>): Promise<string | undefined> {
16
29
  const executionIdentity = getAgentExecutionContext();
17
30
  if (executionIdentity?.employeeUsername) return executionIdentity.employeeUsername;
18
31
 
@@ -59,7 +72,7 @@ export class SkillAccessService {
59
72
  }
60
73
 
61
74
  async assertCanExecute(
62
- ctx: any,
75
+ ctx: Record<string, unknown>,
63
76
  skill: unknown,
64
77
  options: { privileged?: boolean } = {},
65
78
  ): Promise<string | undefined> {
@@ -76,7 +89,7 @@ export class SkillAccessService {
76
89
  return agentUsername;
77
90
  }
78
91
 
79
- async filterAccessibleSkills(ctx: any, skills: unknown[]): Promise<unknown[]> {
92
+ async filterAccessibleSkills(ctx: Record<string, unknown>, skills: unknown[]): Promise<unknown[]> {
80
93
  if (isAdminUser(ctx)) return skills;
81
94
  const agentUsername = await this.resolveAgentUsername(ctx);
82
95
  if (!agentUsername) return [];
@@ -86,7 +99,8 @@ export class SkillAccessService {
86
99
  }
87
100
 
88
101
  async findAgentUsernamesUsingTool(toolName: string): Promise<string[]> {
89
- const employees = await this.plugin.db.getRepository('aiEmployees').find({});
102
+ // Cap at 500 employees to prevent unbounded scans. Most deployments have <100.
103
+ const employees = await this.plugin.db.getRepository('aiEmployees').find({ limit: 500 });
90
104
  const usernames: string[] = [];
91
105
  for (const employee of employees || []) {
92
106
  const settings = asObject(readRecordValue(employee, 'skillSettings'));
@@ -97,4 +111,65 @@ export class SkillAccessService {
97
111
  }
98
112
  return usernames;
99
113
  }
114
+
115
+ /**
116
+ * Cascade-disable a skill across all AI employee bindings.
117
+ *
118
+ * When a skill is disabled or deleted, this method removes its tool from every employee's
119
+ * `skillSettings.tools` array. This prevents stale bindings from allowing execution of
120
+ * skills that are no longer available.
121
+ *
122
+ * Returns the list of affected employee usernames for audit logging.
123
+ */
124
+ async cascadeDisableSkill(skillToolName: string): Promise<string[]> {
125
+ const affectedUsernames: string[] = [];
126
+ // Cap at 500 employees to prevent unbounded scans. Most deployments have <100.
127
+ const employees = await this.plugin.db.getRepository('aiEmployees').find({ limit: 500 });
128
+
129
+ // Collect all updates first, then apply in parallel to reduce round-trips.
130
+ const updates: Array<{ username: string; skillSettings: Record<string, unknown> }> = [];
131
+
132
+ for (const employee of employees || []) {
133
+ const skillSettings = asObject(readRecordValue(employee, 'skillSettings'));
134
+ const tools = Array.isArray(skillSettings.tools) ? (skillSettings.tools as ToolBinding[]) : [];
135
+ const filteredTools = tools.filter((binding) => bindingName(binding) !== skillToolName);
136
+
137
+ if (filteredTools.length === tools.length) continue;
138
+
139
+ const username = normalizeEmployeeUsername(readRecordValue(employee, 'username'));
140
+ if (!username) continue;
141
+
142
+ updates.push({
143
+ username,
144
+ skillSettings: {
145
+ ...skillSettings,
146
+ tools: filteredTools,
147
+ },
148
+ });
149
+ }
150
+
151
+ // Apply updates in parallel (max 10 concurrent to avoid DB overload).
152
+ const BATCH_SIZE = 10;
153
+ for (let i = 0; i < updates.length; i += BATCH_SIZE) {
154
+ const batch = updates.slice(i, i + BATCH_SIZE);
155
+ await Promise.all(
156
+ batch.map((update) =>
157
+ this.plugin.db
158
+ .getRepository('aiEmployees')
159
+ .update({
160
+ filter: { username: update.username },
161
+ values: { skillSettings: update.skillSettings },
162
+ })
163
+ .then(() => {
164
+ affectedUsernames.push(update.username);
165
+ this.plugin.app?.logger?.info?.(
166
+ `[SkillAccess] Cascade-removed skill tool "${skillToolName}" from employee "${update.username}".`,
167
+ );
168
+ }),
169
+ ),
170
+ );
171
+ }
172
+
173
+ return affectedUsernames;
174
+ }
100
175
  }
@@ -27,7 +27,7 @@ const PRICE_PER_1K_OUTPUT = Number(process.env.ORCHESTRATOR_PRICE_PER_1K_OUTPUT
27
27
  /**
28
28
  * Estimate cost based on token counts.
29
29
  */
30
- function estimateCost(inputTokens: number, outputTokens: number): number {
30
+ export function estimateCost(inputTokens: number, outputTokens: number): number {
31
31
  return (inputTokens / 1000) * PRICE_PER_1K_INPUT + (outputTokens / 1000) * PRICE_PER_1K_OUTPUT;
32
32
  }
33
33
 
@@ -46,7 +46,25 @@ function estimateCost(inputTokens: number, outputTokens: number): number {
46
46
  * total_tokens: number,
47
47
  * }
48
48
  */
49
- export function extractTokenUsage(finalState: any): TokenUsage | null {
49
+ type LegacyUsageMetadata = {
50
+ input_tokens?: number;
51
+ output_tokens?: number;
52
+ total_tokens?: number;
53
+ };
54
+
55
+ type LegacyAIMessage = {
56
+ usage_metadata?: LegacyUsageMetadata;
57
+ [key: string]: unknown;
58
+ };
59
+
60
+ type LegacyFinalState =
61
+ | {
62
+ messages?: LegacyAIMessage[];
63
+ }
64
+ | null
65
+ | undefined;
66
+
67
+ export function extractTokenUsage(finalState: LegacyFinalState): TokenUsage | null {
50
68
  if (!finalState?.messages || !Array.isArray(finalState.messages)) return null;
51
69
 
52
70
  // Accumulate usage across all AI messages in the run (each LLM call adds usage)
@@ -83,7 +101,18 @@ export function extractTokenUsage(finalState: any): TokenUsage | null {
83
101
  * 4. Enforce budget limits (max tokens / max cost per run)
84
102
  */
85
103
  export class TokenTracker {
86
- constructor(private readonly plugin: any) {}
104
+ constructor(
105
+ private readonly plugin: {
106
+ app: { log?: { warn?: (...args: unknown[]) => void } };
107
+ db: {
108
+ getRepository: (name: string) => {
109
+ find?: (opts?: Record<string, unknown>) => Promise<unknown[]>;
110
+ findOne?: (opts?: Record<string, unknown>) => Promise<unknown>;
111
+ update?: (opts: Record<string, unknown>) => Promise<unknown>;
112
+ };
113
+ };
114
+ },
115
+ ) {}
87
116
 
88
117
  get db() {
89
118
  return this.plugin.db;
@@ -121,7 +150,7 @@ export class TokenTracker {
121
150
  if (runId != null) {
122
151
  await this.accumulateToRun(runId);
123
152
  }
124
- } catch (e: any) {
153
+ } catch (e: unknown) {
125
154
  this.app.log?.warn?.('[TokenTracker] Failed to track span tokens', e);
126
155
  }
127
156
  }
@@ -134,8 +163,11 @@ export class TokenTracker {
134
163
  const spansRepo = this.db.getRepository('agentExecutionSpans');
135
164
  if (!spansRepo) return;
136
165
 
166
+ // Project only token fields to minimize memory; cap at 10K spans per run.
137
167
  const spans = await spansRepo.find({
138
- filter: { 'metadata.agentLoopRunId': String(runId) },
168
+ filter: { agentLoopRunId: String(runId) },
169
+ fields: ['inputTokens', 'outputTokens', 'cost'],
170
+ limit: 10_000,
139
171
  });
140
172
 
141
173
  let totalInput = 0;
@@ -160,7 +192,7 @@ export class TokenTracker {
160
192
  totalCost,
161
193
  },
162
194
  });
163
- } catch (e: any) {
195
+ } catch (e: unknown) {
164
196
  this.app.log?.warn?.('[TokenTracker] Failed to accumulate run totals', e);
165
197
  }
166
198
  }
@@ -201,12 +233,32 @@ export class TokenTracker {
201
233
  }
202
234
 
203
235
  return { allowed: true };
204
- } catch (e: any) {
236
+ } catch (e: unknown) {
205
237
  this.app.log?.warn?.('[TokenTracker] Budget check failed, allowing', e);
206
238
  return { allowed: true };
207
239
  }
208
240
  }
209
241
 
242
+ /**
243
+ * Track actual token usage from the LLM response on a span.
244
+ * This is preferred over estimateAndTrack when the LLM provides real usage metadata.
245
+ * Falls back to estimation if usage is null.
246
+ */
247
+ async trackActualUsage(
248
+ spanId: string | number | undefined,
249
+ tokenUsage: { inputTokens: number; outputTokens: number; totalTokens: number; cost: number } | null,
250
+ input: string,
251
+ output: string,
252
+ runId?: string | number,
253
+ ): Promise<void> {
254
+ if (tokenUsage && tokenUsage.totalTokens > 0) {
255
+ await this.trackSpan(spanId, tokenUsage, runId);
256
+ return;
257
+ }
258
+ // Fall back to estimation when actual usage is not available
259
+ await this.estimateAndTrack(spanId, input, output, runId);
260
+ }
261
+
210
262
  /**
211
263
  * Estimate tokens from text length and track on a span.
212
264
  * Rough heuristic: ~4 chars per token for English text.
@@ -1,7 +1,9 @@
1
+ import { asObject } from '../utils/ctx-utils';
1
2
  import { createHash } from 'node:crypto';
2
3
  import type { Database, Model } from '@nocobase/database';
3
4
  import { getRunEventBus } from './RunEventBus';
4
5
  import type { LoopPatternPolicy } from './LoopPatternSchema';
6
+ import { read } from '../utils/record-utils';
5
7
 
6
8
  export type ToolLoopLevel = 'none' | 'warn' | 'block' | 'escalate';
7
9
 
@@ -27,15 +29,6 @@ export type ToolLoopRecordContext = {
27
29
  actorIdentity: string;
28
30
  };
29
31
 
30
- function read(record: Model | Record<string, unknown>, key: string) {
31
- const model = record as Model & { get?: (name: string) => unknown };
32
- return typeof model.get === 'function' ? model.get(key) : (record as Record<string, unknown>)[key];
33
- }
34
-
35
- function asObject(value: unknown): Record<string, unknown> {
36
- return value && typeof value === 'object' && !Array.isArray(value) ? (value as Record<string, unknown>) : {};
37
- }
38
-
39
32
  // Key order alone must not change the signature, so object keys are sorted before hashing.
40
33
  function canonicalize(value: unknown): unknown {
41
34
  if (Array.isArray(value)) return value.map((entry) => canonicalize(entry));
@@ -116,8 +109,12 @@ export class ToolLoopDetectionService {
116
109
  }
117
110
  if (!sessionIds.size) return null;
118
111
 
112
+ // Limit message scan to prevent memory pressure on long-running loops.
113
+ // Only load the toolCalls field since that's all we inspect.
119
114
  const messages = await this.database.getRepository('aiMessages').find({
120
115
  filter: { sessionId: { $in: [...sessionIds] } },
116
+ fields: ['toolCalls'],
117
+ limit: 5_000,
121
118
  });
122
119
  const calls: ToolCallEntry[] = [];
123
120
  for (const message of messages) {
@@ -150,12 +147,12 @@ export class ToolLoopDetectionService {
150
147
  if (existing) return false;
151
148
 
152
149
  const now = new Date();
153
- const runSteps = await steps.find({ filter: { runId: context.runId } });
150
+ const stepCount = await steps.count({ filter: { runId: context.runId } });
154
151
  const step = await steps.create({
155
152
  values: {
156
153
  runId: context.runId,
157
154
  runtimeVersion: 'control-plane-v2',
158
- sequence: runSteps.length,
155
+ sequence: stepCount,
159
156
  role: context.role,
160
157
  kind: 'guard',
161
158
  type: 'tool_loop',
@@ -1,31 +1,31 @@
1
- import { z } from 'zod';
2
-
3
- export const verificationVerdictSchema = z.object({
4
- verdict: z.enum(['pass', 'reject', 'escalate']),
5
- summary: z.string().min(1),
6
- checks: z
7
- .array(
8
- z.object({
9
- name: z.string().min(1),
10
- status: z.enum(['pass', 'fail', 'skipped']),
11
- evidenceArtifactIds: z.array(z.number().int().positive()).default([]),
12
- }),
13
- )
14
- .min(1),
15
- residualRisks: z.array(z.string()).default([]),
16
- });
17
-
18
- export type VerificationVerdict = z.infer<typeof verificationVerdictSchema>;
19
-
20
- export function parseVerificationVerdict(value: unknown) {
21
- return verificationVerdictSchema.parse(value);
22
- }
23
-
24
- export function isPassingVerification(value: unknown) {
25
- const parsed = verificationVerdictSchema.safeParse(value);
26
- return (
27
- parsed.success &&
28
- parsed.data.verdict === 'pass' &&
29
- parsed.data.checks.every((check) => check.status === 'pass' && check.evidenceArtifactIds.length > 0)
30
- );
31
- }
1
+ import { z } from 'zod';
2
+
3
+ export const verificationVerdictSchema = z.object({
4
+ verdict: z.enum(['pass', 'reject', 'escalate']),
5
+ summary: z.string().min(1),
6
+ checks: z
7
+ .array(
8
+ z.object({
9
+ name: z.string().min(1),
10
+ status: z.enum(['pass', 'fail', 'skipped']),
11
+ evidenceArtifactIds: z.array(z.number().int().positive()).default([]),
12
+ }),
13
+ )
14
+ .min(1),
15
+ residualRisks: z.array(z.string()).default([]),
16
+ });
17
+
18
+ export type VerificationVerdict = z.infer<typeof verificationVerdictSchema>;
19
+
20
+ export function parseVerificationVerdict(value: unknown) {
21
+ return verificationVerdictSchema.parse(value);
22
+ }
23
+
24
+ export function isPassingVerification(value: unknown) {
25
+ const parsed = verificationVerdictSchema.safeParse(value);
26
+ return (
27
+ parsed.success &&
28
+ parsed.data.verdict === 'pass' &&
29
+ parsed.data.checks.every((check) => check.status === 'pass' && check.evidenceArtifactIds.length > 0)
30
+ );
31
+ }
@@ -6,6 +6,8 @@ import type { PluginAiRuntimeAdapter } from './PluginAiRuntimeAdapter';
6
6
  import { ToolLoopDetectionService, toolLoopReason } from './ToolLoopDetectionService';
7
7
  import { parseVerificationVerdict } from './VerificationSchema';
8
8
  import type { VerificationVerdict } from './VerificationSchema';
9
+ import { read } from '../utils/record-utils';
10
+ import { TokenTracker } from './TokenTracker';
9
11
 
10
12
  export type VerificationRequest = {
11
13
  runId: number;
@@ -17,6 +19,7 @@ export type VerificationRequest = {
17
19
  verifierHarness: CompiledHarness;
18
20
  policy: LoopPatternPolicy;
19
21
  makerSummary: string;
22
+ actingOn?: string[];
20
23
  userId?: number;
21
24
  workerId: string;
22
25
  leaseToken: string;
@@ -29,11 +32,6 @@ export type VerificationOutcome = {
29
32
  finalStatus: 'succeeded' | 'waiting_human' | 'failed' | 'halted';
30
33
  };
31
34
 
32
- function read(record: Model | Record<string, unknown>, key: string) {
33
- const model = record as Model & { get?: (name: string) => unknown };
34
- return typeof model.get === 'function' ? model.get(key) : (record as Record<string, unknown>)[key];
35
- }
36
-
37
35
  // The verifier answers in prose around its JSON verdict, so the object is located rather than
38
36
  // assumed to be the whole response.
39
37
  function extractVerdictObject(content: string): unknown {
@@ -47,7 +45,47 @@ function extractVerdictObject(content: string): unknown {
47
45
  return JSON.parse(candidate.slice(start, end + 1));
48
46
  }
49
47
 
50
- function verifierPrompt(input: VerificationRequest, artifacts: Array<Record<string, unknown>>) {
48
+ /**
49
+ * Compute additional verification checks from the run's runtime context.
50
+ *
51
+ * When `policy.verification.dynamicChecks` is configured, this inspects the number of affected
52
+ * paths, whether any match high-risk patterns, and adds corresponding check names to the list
53
+ * that the verifier must address.
54
+ */
55
+ function dynamicChecksFor(input: VerificationRequest): string[] {
56
+ const config = input.policy.verification.dynamicChecks;
57
+ if (!config) return [];
58
+
59
+ const extra: string[] = [];
60
+ const affectedCount = input.actingOn?.length ?? 0;
61
+
62
+ if (affectedCount >= config.codeChangeThreshold) {
63
+ extra.push('broad_change_review');
64
+ }
65
+ if (config.riskPathPatterns.length > 0 && input.actingOn?.length) {
66
+ const hasRiskPath = input.actingOn.some((path) =>
67
+ config.riskPathPatterns.some((pattern) => path.startsWith(pattern) || path.includes(pattern)),
68
+ );
69
+ if (hasRiskPath) {
70
+ extra.push('risk_path_review');
71
+ }
72
+ }
73
+ if (config.requireSecurityReview) {
74
+ extra.push('security_review');
75
+ }
76
+ if (config.requirePerformanceReview) {
77
+ extra.push('performance_review');
78
+ }
79
+
80
+ return [...new Set(extra)];
81
+ }
82
+
83
+ function verifierPrompt(
84
+ input: VerificationRequest,
85
+ artifacts: Array<Record<string, unknown>>,
86
+ dynamicChecks: string[],
87
+ ) {
88
+ const allChecks = [...new Set([...input.policy.verification.requiredChecks, ...dynamicChecks])];
51
89
  const catalogue = artifacts.length
52
90
  ? artifacts.map((artifact) => `- id=${artifact.id} kind=${artifact.kind} title=${artifact.title || ''}`).join('\n')
53
91
  : '- (none)';
@@ -61,7 +99,7 @@ function verifierPrompt(input: VerificationRequest, artifacts: Array<Record<stri
61
99
  'Evidence artifacts recorded for this run:',
62
100
  catalogue,
63
101
  '',
64
- `Required checks: ${input.policy.verification.requiredChecks.join(', ')}`,
102
+ `Required checks: ${allChecks.join(', ')}`,
65
103
  '',
66
104
  ...(input.loopNotices ?? []),
67
105
  'Reply with a single JSON object and nothing else:',
@@ -79,6 +117,7 @@ export class VerificationService {
79
117
  private readonly database: Database,
80
118
  private readonly runtime: PluginAiRuntimeAdapter,
81
119
  private readonly stateMachine: LoopRunStateMachine,
120
+ private readonly tokenTracker: TokenTracker,
82
121
  ) {
83
122
  this.loopGuard = new ToolLoopDetectionService(database);
84
123
  }
@@ -159,8 +198,7 @@ export class VerificationService {
159
198
  }
160
199
 
161
200
  private async nextSequence(runId: number) {
162
- const steps = await this.database.getRepository('agentLoopSteps').find({ filter: { runId } });
163
- return steps.length;
201
+ return this.database.getRepository('agentLoopSteps').count({ filter: { runId } });
164
202
  }
165
203
 
166
204
  private async collectVerdict(input: VerificationRequest, artifacts: Array<Record<string, unknown>>) {
@@ -195,9 +233,19 @@ export class VerificationService {
195
233
  systemMessage:
196
234
  'You verify completed work against its stated goal using only recorded evidence. You never modify anything.',
197
235
  harness: input.verifierHarness,
198
- prompt: verifierPrompt(input, artifacts),
236
+ prompt: verifierPrompt(input, artifacts, dynamicChecksFor(input)),
199
237
  signal: input.signal,
200
238
  });
239
+
240
+ // Track actual token usage from the verifier's LLM response
241
+ await this.tokenTracker.trackActualUsage(
242
+ read(step, 'id'),
243
+ outcome.tokenUsage ?? null,
244
+ verifierPrompt(input, artifacts, dynamicChecksFor(input)),
245
+ outcome.content,
246
+ input.runId,
247
+ );
248
+
201
249
  await this.database.getRepository('agentLoopSteps').update({
202
250
  filterByTk: read(step, 'id'),
203
251
  values: {
@@ -212,7 +260,8 @@ export class VerificationService {
212
260
  throw new Error('The verifier requested an approval-gated tool; verification cannot complete.');
213
261
  }
214
262
  const verdict = parseVerificationVerdict(extractVerdictObject(outcome.content));
215
- const required = new Set(input.policy.verification.requiredChecks);
263
+ const dynamicChecks = dynamicChecksFor(input);
264
+ const required = new Set([...input.policy.verification.requiredChecks, ...dynamicChecks]);
216
265
  for (const check of verdict.checks) required.delete(check.name);
217
266
  if (required.size) {
218
267
  throw new Error(`The verifier omitted required checks: ${Array.from(required).join(', ')}.`);
@@ -1,23 +1,16 @@
1
+ import { asObject } from '../utils/ctx-utils';
1
2
  import type { Database } from '@nocobase/database';
2
3
  import { compileHarness } from './HarnessCompiler';
3
4
  import type { CompiledHarness } from './HarnessCompiler';
4
5
  import { HarnessProfileService } from './HarnessProfileService';
5
6
  import { parseHarnessSettings } from './HarnessSchema';
7
+ import { read } from '../utils/record-utils';
6
8
 
7
9
  export type TraceLike = {
8
10
  agentLoopRunId?: string | number;
9
11
  employeeUsername?: string;
10
12
  };
11
13
 
12
- function read(record: unknown, key: string) {
13
- const model = record as { get?: (name: string) => unknown } | null;
14
- return typeof model?.get === 'function' ? model.get(key) : (model as Record<string, unknown> | null)?.[key];
15
- }
16
-
17
- function asObject(value: unknown): Record<string, unknown> {
18
- return value && typeof value === 'object' && !Array.isArray(value) ? (value as Record<string, unknown>) : {};
19
- }
20
-
21
14
  // Run snapshots were compiled at enqueue time, possibly by an older schema revision. Re-parsing
22
15
  // through the current schema backfills new policy fields with their defaults. `sources` is a
23
16
  // compiler artifact the strict schema rejects, so it is stripped first.