plugin-agent-orchestrator 1.1.6 → 1.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client/index.js +1 -1
- package/dist/client-v2/index.js +1 -1
- package/dist/externalVersion.js +7 -7
- package/dist/server/collections/agent-loop-runs.js +6 -0
- package/dist/server/collections/skill-metrics.js +66 -0
- package/dist/server/plugin.js +91 -2
- package/dist/server/resources/agent-loop-events-stream.js +1 -0
- package/dist/server/resources/agent-loop.js +1 -1
- package/dist/server/services/DurableCircuitBreakerService.js +9 -12
- package/dist/server/services/ExecutionSpanService.js +8 -3
- package/dist/server/services/HarnessProfileService.js +23 -26
- package/dist/server/services/LoopBudgetService.js +36 -39
- package/dist/server/services/LoopControlService.js +8 -11
- package/dist/server/services/LoopPatternSchema.js +51 -17
- package/dist/server/services/LoopPatternService.js +31 -11
- package/dist/server/services/LoopRunRepository.js +2 -5
- package/dist/server/services/LoopRunStateMachine.js +31 -34
- package/dist/server/services/LoopSchedulerService.js +4 -7
- package/dist/server/services/LoopTriggerService.js +11 -6
- package/dist/server/services/LoopWorkerService.js +88 -22
- package/dist/server/services/PathLockService.js +19 -22
- package/dist/server/services/PluginAiRuntimeAdapter.js +51 -8
- package/dist/server/services/RunEventBus.js +14 -2
- package/dist/server/services/SkillAccessService.js +50 -1
- package/dist/server/services/TokenTracker.js +17 -1
- package/dist/server/services/ToolLoopDetectionService.js +14 -17
- package/dist/server/services/VerificationService.js +47 -16
- package/dist/server/services/harness-runtime-policy.js +7 -12
- package/dist/server/skill-hub/plugin.js +34 -27
- package/dist/server/utils/ha-lock.js +73 -0
- package/dist/server/utils/loop-pattern-context.js +42 -5
- package/dist/server/utils/record-utils.js +44 -0
- package/package.json +1 -1
- package/src/client/OrchestratorSettings.tsx +2 -2
- package/src/client/plugin.tsx +2 -1
- package/src/client-v2/plugin.tsx +26 -13
- package/src/server/__tests__/tool-loop-detection.test.ts +1 -0
- package/src/server/collections/agent-loop-runs.ts +7 -0
- package/src/server/collections/skill-metrics.ts +36 -0
- package/src/server/plugin.ts +117 -1
- package/src/server/resources/agent-loop-events-stream.ts +108 -105
- package/src/server/resources/agent-loop.ts +1 -1
- package/src/server/resources/tracing.ts +267 -266
- package/src/server/services/DurableCircuitBreakerService.ts +235 -239
- package/src/server/services/ExecutionSpanService.ts +37 -12
- package/src/server/services/HarnessProfileService.ts +1 -5
- package/src/server/services/LoopBudgetService.ts +271 -275
- package/src/server/services/LoopControlService.ts +120 -124
- package/src/server/services/LoopPatternSchema.ts +60 -14
- package/src/server/services/LoopPatternService.ts +171 -143
- package/src/server/services/LoopRunRepository.ts +1 -5
- package/src/server/services/LoopRunStateMachine.ts +1 -5
- package/src/server/services/LoopSchedulerService.ts +195 -199
- package/src/server/services/LoopTriggerService.ts +7 -5
- package/src/server/services/LoopWorkerService.ts +102 -12
- package/src/server/services/PathLockService.ts +317 -321
- package/src/server/services/PluginAiRuntimeAdapter.ts +59 -4
- package/src/server/services/RunEventBus.ts +13 -0
- package/src/server/services/SkillAccessService.ts +80 -5
- package/src/server/services/TokenTracker.ts +59 -7
- package/src/server/services/ToolLoopDetectionService.ts +8 -11
- package/src/server/services/VerificationSchema.ts +31 -31
- package/src/server/services/VerificationService.ts +60 -11
- package/src/server/services/harness-runtime-policy.ts +2 -9
- package/src/server/skill-hub/plugin.ts +36 -28
- package/src/server/types.ts +5 -39
- package/src/server/utils/ensure-indexes.ts +96 -96
- package/src/server/utils/ha-lock.ts +80 -0
- package/src/server/utils/loop-pattern-context.ts +84 -20
- package/src/server/utils/record-utils.ts +19 -0
|
@@ -2,9 +2,11 @@ import type { Model } from '@nocobase/database';
|
|
|
2
2
|
import type { Application } from '@nocobase/server';
|
|
3
3
|
import { AIEmployee } from '@nocobase/plugin-ai/server';
|
|
4
4
|
import { listSystemTools } from '@nocobase/ai';
|
|
5
|
+
import { estimateCost } from './TokenTracker';
|
|
5
6
|
import { getAIToolsManager } from '../utils/ai-manager';
|
|
6
7
|
import { decideTool } from './HarnessCompiler';
|
|
7
8
|
import type { CompiledHarness } from './HarnessCompiler';
|
|
9
|
+
import { read } from '../utils/record-utils';
|
|
8
10
|
|
|
9
11
|
// plugin-ai keeps every tool whose name is in `listSystemTools()` regardless of the constructor
|
|
10
12
|
// tool filter (ai-employee.ts getAIEmployeeTools). The harness `filter` cannot remove them, so a
|
|
@@ -30,11 +32,19 @@ export type InterruptedToolCall = {
|
|
|
30
32
|
interruptId: string;
|
|
31
33
|
};
|
|
32
34
|
|
|
35
|
+
export type TokenUsageReport = {
|
|
36
|
+
inputTokens: number;
|
|
37
|
+
outputTokens: number;
|
|
38
|
+
totalTokens: number;
|
|
39
|
+
cost: number;
|
|
40
|
+
};
|
|
41
|
+
|
|
33
42
|
export type InvocationOutcome = {
|
|
34
43
|
sessionId: string;
|
|
35
44
|
messageId: string;
|
|
36
45
|
interrupted: InterruptedToolCall[];
|
|
37
46
|
content: string;
|
|
47
|
+
tokenUsage: TokenUsageReport | null;
|
|
38
48
|
};
|
|
39
49
|
|
|
40
50
|
type ConversationsManager = {
|
|
@@ -58,9 +68,48 @@ type PluginAi = {
|
|
|
58
68
|
aiEmployeesManager: EmployeesManager;
|
|
59
69
|
};
|
|
60
70
|
|
|
61
|
-
function
|
|
62
|
-
|
|
63
|
-
|
|
71
|
+
function extractTokenUsageFromResult(result: Record<string, unknown>): TokenUsageReport | null {
|
|
72
|
+
// Try direct usage_metadata from the last AI message (LangChain standard)
|
|
73
|
+
const messages = result.messages;
|
|
74
|
+
if (Array.isArray(messages)) {
|
|
75
|
+
let totalInput = 0;
|
|
76
|
+
let totalOutput = 0;
|
|
77
|
+
let totalAll = 0;
|
|
78
|
+
for (const msg of messages) {
|
|
79
|
+
const um = (msg as Record<string, unknown>)?.usage_metadata as Record<string, unknown> | undefined;
|
|
80
|
+
if (um) {
|
|
81
|
+
totalInput += Number(um.input_tokens || 0);
|
|
82
|
+
totalOutput += Number(um.output_tokens || 0);
|
|
83
|
+
totalAll += Number(um.total_tokens || 0);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
if (totalAll > 0) {
|
|
87
|
+
return {
|
|
88
|
+
inputTokens: totalInput,
|
|
89
|
+
outputTokens: totalOutput,
|
|
90
|
+
totalTokens: totalAll,
|
|
91
|
+
cost: estimateCost(totalInput, totalOutput),
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
// Try usage field (some providers return this)
|
|
97
|
+
const usage = result.usage as Record<string, unknown> | undefined;
|
|
98
|
+
if (usage) {
|
|
99
|
+
const inputTokens = Number(usage.prompt_tokens || usage.input_tokens || 0);
|
|
100
|
+
const outputTokens = Number(usage.completion_tokens || usage.output_tokens || 0);
|
|
101
|
+
const totalTokens = Number(usage.total_tokens || inputTokens + outputTokens);
|
|
102
|
+
if (totalTokens > 0) {
|
|
103
|
+
return {
|
|
104
|
+
inputTokens,
|
|
105
|
+
outputTokens,
|
|
106
|
+
totalTokens,
|
|
107
|
+
cost: estimateCost(inputTokens, outputTokens),
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
return null;
|
|
64
113
|
}
|
|
65
114
|
|
|
66
115
|
function textContent(messages: unknown): string {
|
|
@@ -211,6 +260,7 @@ export class PluginAiRuntimeAdapter {
|
|
|
211
260
|
messageId: String(result.messageId || ''),
|
|
212
261
|
interrupted: interruptedFrom(result),
|
|
213
262
|
content: textContent(result.messages),
|
|
263
|
+
tokenUsage: extractTokenUsageFromResult(result),
|
|
214
264
|
};
|
|
215
265
|
}
|
|
216
266
|
|
|
@@ -259,6 +309,7 @@ export class PluginAiRuntimeAdapter {
|
|
|
259
309
|
messageId: String(result.messageId || input.messageId),
|
|
260
310
|
interrupted: interruptedFrom(result),
|
|
261
311
|
content: textContent(result.messages),
|
|
312
|
+
tokenUsage: extractTokenUsageFromResult(result),
|
|
262
313
|
};
|
|
263
314
|
}
|
|
264
315
|
|
|
@@ -283,7 +334,11 @@ export class PluginAiRuntimeAdapter {
|
|
|
283
334
|
db: this.app.db,
|
|
284
335
|
log: this.app.log,
|
|
285
336
|
logger: this.app.log,
|
|
286
|
-
state: {
|
|
337
|
+
state: {
|
|
338
|
+
currentRoles,
|
|
339
|
+
// Populate currentUser so currentUserId(ctx) resolves correctly.
|
|
340
|
+
currentUser: input.userId ? { id: input.userId } : undefined,
|
|
341
|
+
},
|
|
287
342
|
auth: { user: { id: input.userId } },
|
|
288
343
|
action: { params: { values: { sessionId: input.sessionId, model } } },
|
|
289
344
|
};
|
|
@@ -33,6 +33,11 @@ class RunEventBusImpl {
|
|
|
33
33
|
listenerCount(runId: string | number): number {
|
|
34
34
|
return this.listeners.get(runId)?.size || 0;
|
|
35
35
|
}
|
|
36
|
+
|
|
37
|
+
/** Remove all listeners for all runs. Call during plugin shutdown. */
|
|
38
|
+
destroy() {
|
|
39
|
+
this.listeners.clear();
|
|
40
|
+
}
|
|
36
41
|
}
|
|
37
42
|
|
|
38
43
|
let instance: RunEventBusImpl | null = null;
|
|
@@ -43,3 +48,11 @@ export function getRunEventBus(): RunEventBusImpl {
|
|
|
43
48
|
}
|
|
44
49
|
return instance;
|
|
45
50
|
}
|
|
51
|
+
|
|
52
|
+
/** Reset the singleton (for tests or shutdown). */
|
|
53
|
+
export function resetRunEventBus() {
|
|
54
|
+
if (instance) {
|
|
55
|
+
instance.destroy();
|
|
56
|
+
instance = null;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
@@ -10,9 +10,22 @@ function bindingName(binding: ToolBinding): string {
|
|
|
10
10
|
}
|
|
11
11
|
|
|
12
12
|
export class SkillAccessService {
|
|
13
|
-
constructor(
|
|
13
|
+
constructor(
|
|
14
|
+
private readonly plugin: {
|
|
15
|
+
db: {
|
|
16
|
+
getRepository: (name: string) =>
|
|
17
|
+
| {
|
|
18
|
+
find: (opts?: Record<string, unknown>) => Promise<unknown[]>;
|
|
19
|
+
findOne: (opts: Record<string, unknown>) => Promise<unknown>;
|
|
20
|
+
update: (opts: Record<string, unknown>) => Promise<unknown>;
|
|
21
|
+
}
|
|
22
|
+
| undefined;
|
|
23
|
+
};
|
|
24
|
+
app?: { logger?: { info?: (...args: unknown[]) => void } };
|
|
25
|
+
},
|
|
26
|
+
) {}
|
|
14
27
|
|
|
15
|
-
async resolveAgentUsername(ctx:
|
|
28
|
+
async resolveAgentUsername(ctx: Record<string, unknown>): Promise<string | undefined> {
|
|
16
29
|
const executionIdentity = getAgentExecutionContext();
|
|
17
30
|
if (executionIdentity?.employeeUsername) return executionIdentity.employeeUsername;
|
|
18
31
|
|
|
@@ -59,7 +72,7 @@ export class SkillAccessService {
|
|
|
59
72
|
}
|
|
60
73
|
|
|
61
74
|
async assertCanExecute(
|
|
62
|
-
ctx:
|
|
75
|
+
ctx: Record<string, unknown>,
|
|
63
76
|
skill: unknown,
|
|
64
77
|
options: { privileged?: boolean } = {},
|
|
65
78
|
): Promise<string | undefined> {
|
|
@@ -76,7 +89,7 @@ export class SkillAccessService {
|
|
|
76
89
|
return agentUsername;
|
|
77
90
|
}
|
|
78
91
|
|
|
79
|
-
async filterAccessibleSkills(ctx:
|
|
92
|
+
async filterAccessibleSkills(ctx: Record<string, unknown>, skills: unknown[]): Promise<unknown[]> {
|
|
80
93
|
if (isAdminUser(ctx)) return skills;
|
|
81
94
|
const agentUsername = await this.resolveAgentUsername(ctx);
|
|
82
95
|
if (!agentUsername) return [];
|
|
@@ -86,7 +99,8 @@ export class SkillAccessService {
|
|
|
86
99
|
}
|
|
87
100
|
|
|
88
101
|
async findAgentUsernamesUsingTool(toolName: string): Promise<string[]> {
|
|
89
|
-
|
|
102
|
+
// Cap at 500 employees to prevent unbounded scans. Most deployments have <100.
|
|
103
|
+
const employees = await this.plugin.db.getRepository('aiEmployees').find({ limit: 500 });
|
|
90
104
|
const usernames: string[] = [];
|
|
91
105
|
for (const employee of employees || []) {
|
|
92
106
|
const settings = asObject(readRecordValue(employee, 'skillSettings'));
|
|
@@ -97,4 +111,65 @@ export class SkillAccessService {
|
|
|
97
111
|
}
|
|
98
112
|
return usernames;
|
|
99
113
|
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* Cascade-disable a skill across all AI employee bindings.
|
|
117
|
+
*
|
|
118
|
+
* When a skill is disabled or deleted, this method removes its tool from every employee's
|
|
119
|
+
* `skillSettings.tools` array. This prevents stale bindings from allowing execution of
|
|
120
|
+
* skills that are no longer available.
|
|
121
|
+
*
|
|
122
|
+
* Returns the list of affected employee usernames for audit logging.
|
|
123
|
+
*/
|
|
124
|
+
async cascadeDisableSkill(skillToolName: string): Promise<string[]> {
|
|
125
|
+
const affectedUsernames: string[] = [];
|
|
126
|
+
// Cap at 500 employees to prevent unbounded scans. Most deployments have <100.
|
|
127
|
+
const employees = await this.plugin.db.getRepository('aiEmployees').find({ limit: 500 });
|
|
128
|
+
|
|
129
|
+
// Collect all updates first, then apply in parallel to reduce round-trips.
|
|
130
|
+
const updates: Array<{ username: string; skillSettings: Record<string, unknown> }> = [];
|
|
131
|
+
|
|
132
|
+
for (const employee of employees || []) {
|
|
133
|
+
const skillSettings = asObject(readRecordValue(employee, 'skillSettings'));
|
|
134
|
+
const tools = Array.isArray(skillSettings.tools) ? (skillSettings.tools as ToolBinding[]) : [];
|
|
135
|
+
const filteredTools = tools.filter((binding) => bindingName(binding) !== skillToolName);
|
|
136
|
+
|
|
137
|
+
if (filteredTools.length === tools.length) continue;
|
|
138
|
+
|
|
139
|
+
const username = normalizeEmployeeUsername(readRecordValue(employee, 'username'));
|
|
140
|
+
if (!username) continue;
|
|
141
|
+
|
|
142
|
+
updates.push({
|
|
143
|
+
username,
|
|
144
|
+
skillSettings: {
|
|
145
|
+
...skillSettings,
|
|
146
|
+
tools: filteredTools,
|
|
147
|
+
},
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
// Apply updates in parallel (max 10 concurrent to avoid DB overload).
|
|
152
|
+
const BATCH_SIZE = 10;
|
|
153
|
+
for (let i = 0; i < updates.length; i += BATCH_SIZE) {
|
|
154
|
+
const batch = updates.slice(i, i + BATCH_SIZE);
|
|
155
|
+
await Promise.all(
|
|
156
|
+
batch.map((update) =>
|
|
157
|
+
this.plugin.db
|
|
158
|
+
.getRepository('aiEmployees')
|
|
159
|
+
.update({
|
|
160
|
+
filter: { username: update.username },
|
|
161
|
+
values: { skillSettings: update.skillSettings },
|
|
162
|
+
})
|
|
163
|
+
.then(() => {
|
|
164
|
+
affectedUsernames.push(update.username);
|
|
165
|
+
this.plugin.app?.logger?.info?.(
|
|
166
|
+
`[SkillAccess] Cascade-removed skill tool "${skillToolName}" from employee "${update.username}".`,
|
|
167
|
+
);
|
|
168
|
+
}),
|
|
169
|
+
),
|
|
170
|
+
);
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
return affectedUsernames;
|
|
174
|
+
}
|
|
100
175
|
}
|
|
@@ -27,7 +27,7 @@ const PRICE_PER_1K_OUTPUT = Number(process.env.ORCHESTRATOR_PRICE_PER_1K_OUTPUT
|
|
|
27
27
|
/**
|
|
28
28
|
* Estimate cost based on token counts.
|
|
29
29
|
*/
|
|
30
|
-
function estimateCost(inputTokens: number, outputTokens: number): number {
|
|
30
|
+
export function estimateCost(inputTokens: number, outputTokens: number): number {
|
|
31
31
|
return (inputTokens / 1000) * PRICE_PER_1K_INPUT + (outputTokens / 1000) * PRICE_PER_1K_OUTPUT;
|
|
32
32
|
}
|
|
33
33
|
|
|
@@ -46,7 +46,25 @@ function estimateCost(inputTokens: number, outputTokens: number): number {
|
|
|
46
46
|
* total_tokens: number,
|
|
47
47
|
* }
|
|
48
48
|
*/
|
|
49
|
-
|
|
49
|
+
type LegacyUsageMetadata = {
|
|
50
|
+
input_tokens?: number;
|
|
51
|
+
output_tokens?: number;
|
|
52
|
+
total_tokens?: number;
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
type LegacyAIMessage = {
|
|
56
|
+
usage_metadata?: LegacyUsageMetadata;
|
|
57
|
+
[key: string]: unknown;
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
type LegacyFinalState =
|
|
61
|
+
| {
|
|
62
|
+
messages?: LegacyAIMessage[];
|
|
63
|
+
}
|
|
64
|
+
| null
|
|
65
|
+
| undefined;
|
|
66
|
+
|
|
67
|
+
export function extractTokenUsage(finalState: LegacyFinalState): TokenUsage | null {
|
|
50
68
|
if (!finalState?.messages || !Array.isArray(finalState.messages)) return null;
|
|
51
69
|
|
|
52
70
|
// Accumulate usage across all AI messages in the run (each LLM call adds usage)
|
|
@@ -83,7 +101,18 @@ export function extractTokenUsage(finalState: any): TokenUsage | null {
|
|
|
83
101
|
* 4. Enforce budget limits (max tokens / max cost per run)
|
|
84
102
|
*/
|
|
85
103
|
export class TokenTracker {
|
|
86
|
-
constructor(
|
|
104
|
+
constructor(
|
|
105
|
+
private readonly plugin: {
|
|
106
|
+
app: { log?: { warn?: (...args: unknown[]) => void } };
|
|
107
|
+
db: {
|
|
108
|
+
getRepository: (name: string) => {
|
|
109
|
+
find?: (opts?: Record<string, unknown>) => Promise<unknown[]>;
|
|
110
|
+
findOne?: (opts?: Record<string, unknown>) => Promise<unknown>;
|
|
111
|
+
update?: (opts: Record<string, unknown>) => Promise<unknown>;
|
|
112
|
+
};
|
|
113
|
+
};
|
|
114
|
+
},
|
|
115
|
+
) {}
|
|
87
116
|
|
|
88
117
|
get db() {
|
|
89
118
|
return this.plugin.db;
|
|
@@ -121,7 +150,7 @@ export class TokenTracker {
|
|
|
121
150
|
if (runId != null) {
|
|
122
151
|
await this.accumulateToRun(runId);
|
|
123
152
|
}
|
|
124
|
-
} catch (e:
|
|
153
|
+
} catch (e: unknown) {
|
|
125
154
|
this.app.log?.warn?.('[TokenTracker] Failed to track span tokens', e);
|
|
126
155
|
}
|
|
127
156
|
}
|
|
@@ -134,8 +163,11 @@ export class TokenTracker {
|
|
|
134
163
|
const spansRepo = this.db.getRepository('agentExecutionSpans');
|
|
135
164
|
if (!spansRepo) return;
|
|
136
165
|
|
|
166
|
+
// Project only token fields to minimize memory; cap at 10K spans per run.
|
|
137
167
|
const spans = await spansRepo.find({
|
|
138
|
-
filter: {
|
|
168
|
+
filter: { agentLoopRunId: String(runId) },
|
|
169
|
+
fields: ['inputTokens', 'outputTokens', 'cost'],
|
|
170
|
+
limit: 10_000,
|
|
139
171
|
});
|
|
140
172
|
|
|
141
173
|
let totalInput = 0;
|
|
@@ -160,7 +192,7 @@ export class TokenTracker {
|
|
|
160
192
|
totalCost,
|
|
161
193
|
},
|
|
162
194
|
});
|
|
163
|
-
} catch (e:
|
|
195
|
+
} catch (e: unknown) {
|
|
164
196
|
this.app.log?.warn?.('[TokenTracker] Failed to accumulate run totals', e);
|
|
165
197
|
}
|
|
166
198
|
}
|
|
@@ -201,12 +233,32 @@ export class TokenTracker {
|
|
|
201
233
|
}
|
|
202
234
|
|
|
203
235
|
return { allowed: true };
|
|
204
|
-
} catch (e:
|
|
236
|
+
} catch (e: unknown) {
|
|
205
237
|
this.app.log?.warn?.('[TokenTracker] Budget check failed, allowing', e);
|
|
206
238
|
return { allowed: true };
|
|
207
239
|
}
|
|
208
240
|
}
|
|
209
241
|
|
|
242
|
+
/**
|
|
243
|
+
* Track actual token usage from the LLM response on a span.
|
|
244
|
+
* This is preferred over estimateAndTrack when the LLM provides real usage metadata.
|
|
245
|
+
* Falls back to estimation if usage is null.
|
|
246
|
+
*/
|
|
247
|
+
async trackActualUsage(
|
|
248
|
+
spanId: string | number | undefined,
|
|
249
|
+
tokenUsage: { inputTokens: number; outputTokens: number; totalTokens: number; cost: number } | null,
|
|
250
|
+
input: string,
|
|
251
|
+
output: string,
|
|
252
|
+
runId?: string | number,
|
|
253
|
+
): Promise<void> {
|
|
254
|
+
if (tokenUsage && tokenUsage.totalTokens > 0) {
|
|
255
|
+
await this.trackSpan(spanId, tokenUsage, runId);
|
|
256
|
+
return;
|
|
257
|
+
}
|
|
258
|
+
// Fall back to estimation when actual usage is not available
|
|
259
|
+
await this.estimateAndTrack(spanId, input, output, runId);
|
|
260
|
+
}
|
|
261
|
+
|
|
210
262
|
/**
|
|
211
263
|
* Estimate tokens from text length and track on a span.
|
|
212
264
|
* Rough heuristic: ~4 chars per token for English text.
|
|
@@ -1,7 +1,9 @@
|
|
|
1
|
+
import { asObject } from '../utils/ctx-utils';
|
|
1
2
|
import { createHash } from 'node:crypto';
|
|
2
3
|
import type { Database, Model } from '@nocobase/database';
|
|
3
4
|
import { getRunEventBus } from './RunEventBus';
|
|
4
5
|
import type { LoopPatternPolicy } from './LoopPatternSchema';
|
|
6
|
+
import { read } from '../utils/record-utils';
|
|
5
7
|
|
|
6
8
|
export type ToolLoopLevel = 'none' | 'warn' | 'block' | 'escalate';
|
|
7
9
|
|
|
@@ -27,15 +29,6 @@ export type ToolLoopRecordContext = {
|
|
|
27
29
|
actorIdentity: string;
|
|
28
30
|
};
|
|
29
31
|
|
|
30
|
-
function read(record: Model | Record<string, unknown>, key: string) {
|
|
31
|
-
const model = record as Model & { get?: (name: string) => unknown };
|
|
32
|
-
return typeof model.get === 'function' ? model.get(key) : (record as Record<string, unknown>)[key];
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
function asObject(value: unknown): Record<string, unknown> {
|
|
36
|
-
return value && typeof value === 'object' && !Array.isArray(value) ? (value as Record<string, unknown>) : {};
|
|
37
|
-
}
|
|
38
|
-
|
|
39
32
|
// Key order alone must not change the signature, so object keys are sorted before hashing.
|
|
40
33
|
function canonicalize(value: unknown): unknown {
|
|
41
34
|
if (Array.isArray(value)) return value.map((entry) => canonicalize(entry));
|
|
@@ -116,8 +109,12 @@ export class ToolLoopDetectionService {
|
|
|
116
109
|
}
|
|
117
110
|
if (!sessionIds.size) return null;
|
|
118
111
|
|
|
112
|
+
// Limit message scan to prevent memory pressure on long-running loops.
|
|
113
|
+
// Only load the toolCalls field since that's all we inspect.
|
|
119
114
|
const messages = await this.database.getRepository('aiMessages').find({
|
|
120
115
|
filter: { sessionId: { $in: [...sessionIds] } },
|
|
116
|
+
fields: ['toolCalls'],
|
|
117
|
+
limit: 5_000,
|
|
121
118
|
});
|
|
122
119
|
const calls: ToolCallEntry[] = [];
|
|
123
120
|
for (const message of messages) {
|
|
@@ -150,12 +147,12 @@ export class ToolLoopDetectionService {
|
|
|
150
147
|
if (existing) return false;
|
|
151
148
|
|
|
152
149
|
const now = new Date();
|
|
153
|
-
const
|
|
150
|
+
const stepCount = await steps.count({ filter: { runId: context.runId } });
|
|
154
151
|
const step = await steps.create({
|
|
155
152
|
values: {
|
|
156
153
|
runId: context.runId,
|
|
157
154
|
runtimeVersion: 'control-plane-v2',
|
|
158
|
-
sequence:
|
|
155
|
+
sequence: stepCount,
|
|
159
156
|
role: context.role,
|
|
160
157
|
kind: 'guard',
|
|
161
158
|
type: 'tool_loop',
|
|
@@ -1,31 +1,31 @@
|
|
|
1
|
-
import { z } from 'zod';
|
|
2
|
-
|
|
3
|
-
export const verificationVerdictSchema = z.object({
|
|
4
|
-
verdict: z.enum(['pass', 'reject', 'escalate']),
|
|
5
|
-
summary: z.string().min(1),
|
|
6
|
-
checks: z
|
|
7
|
-
.array(
|
|
8
|
-
z.object({
|
|
9
|
-
name: z.string().min(1),
|
|
10
|
-
status: z.enum(['pass', 'fail', 'skipped']),
|
|
11
|
-
evidenceArtifactIds: z.array(z.number().int().positive()).default([]),
|
|
12
|
-
}),
|
|
13
|
-
)
|
|
14
|
-
.min(1),
|
|
15
|
-
residualRisks: z.array(z.string()).default([]),
|
|
16
|
-
});
|
|
17
|
-
|
|
18
|
-
export type VerificationVerdict = z.infer<typeof verificationVerdictSchema>;
|
|
19
|
-
|
|
20
|
-
export function parseVerificationVerdict(value: unknown) {
|
|
21
|
-
return verificationVerdictSchema.parse(value);
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
export function isPassingVerification(value: unknown) {
|
|
25
|
-
const parsed = verificationVerdictSchema.safeParse(value);
|
|
26
|
-
return (
|
|
27
|
-
parsed.success &&
|
|
28
|
-
parsed.data.verdict === 'pass' &&
|
|
29
|
-
parsed.data.checks.every((check) => check.status === 'pass' && check.evidenceArtifactIds.length > 0)
|
|
30
|
-
);
|
|
31
|
-
}
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
|
|
3
|
+
export const verificationVerdictSchema = z.object({
|
|
4
|
+
verdict: z.enum(['pass', 'reject', 'escalate']),
|
|
5
|
+
summary: z.string().min(1),
|
|
6
|
+
checks: z
|
|
7
|
+
.array(
|
|
8
|
+
z.object({
|
|
9
|
+
name: z.string().min(1),
|
|
10
|
+
status: z.enum(['pass', 'fail', 'skipped']),
|
|
11
|
+
evidenceArtifactIds: z.array(z.number().int().positive()).default([]),
|
|
12
|
+
}),
|
|
13
|
+
)
|
|
14
|
+
.min(1),
|
|
15
|
+
residualRisks: z.array(z.string()).default([]),
|
|
16
|
+
});
|
|
17
|
+
|
|
18
|
+
export type VerificationVerdict = z.infer<typeof verificationVerdictSchema>;
|
|
19
|
+
|
|
20
|
+
export function parseVerificationVerdict(value: unknown) {
|
|
21
|
+
return verificationVerdictSchema.parse(value);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export function isPassingVerification(value: unknown) {
|
|
25
|
+
const parsed = verificationVerdictSchema.safeParse(value);
|
|
26
|
+
return (
|
|
27
|
+
parsed.success &&
|
|
28
|
+
parsed.data.verdict === 'pass' &&
|
|
29
|
+
parsed.data.checks.every((check) => check.status === 'pass' && check.evidenceArtifactIds.length > 0)
|
|
30
|
+
);
|
|
31
|
+
}
|
|
@@ -6,6 +6,8 @@ import type { PluginAiRuntimeAdapter } from './PluginAiRuntimeAdapter';
|
|
|
6
6
|
import { ToolLoopDetectionService, toolLoopReason } from './ToolLoopDetectionService';
|
|
7
7
|
import { parseVerificationVerdict } from './VerificationSchema';
|
|
8
8
|
import type { VerificationVerdict } from './VerificationSchema';
|
|
9
|
+
import { read } from '../utils/record-utils';
|
|
10
|
+
import { TokenTracker } from './TokenTracker';
|
|
9
11
|
|
|
10
12
|
export type VerificationRequest = {
|
|
11
13
|
runId: number;
|
|
@@ -17,6 +19,7 @@ export type VerificationRequest = {
|
|
|
17
19
|
verifierHarness: CompiledHarness;
|
|
18
20
|
policy: LoopPatternPolicy;
|
|
19
21
|
makerSummary: string;
|
|
22
|
+
actingOn?: string[];
|
|
20
23
|
userId?: number;
|
|
21
24
|
workerId: string;
|
|
22
25
|
leaseToken: string;
|
|
@@ -29,11 +32,6 @@ export type VerificationOutcome = {
|
|
|
29
32
|
finalStatus: 'succeeded' | 'waiting_human' | 'failed' | 'halted';
|
|
30
33
|
};
|
|
31
34
|
|
|
32
|
-
function read(record: Model | Record<string, unknown>, key: string) {
|
|
33
|
-
const model = record as Model & { get?: (name: string) => unknown };
|
|
34
|
-
return typeof model.get === 'function' ? model.get(key) : (record as Record<string, unknown>)[key];
|
|
35
|
-
}
|
|
36
|
-
|
|
37
35
|
// The verifier answers in prose around its JSON verdict, so the object is located rather than
|
|
38
36
|
// assumed to be the whole response.
|
|
39
37
|
function extractVerdictObject(content: string): unknown {
|
|
@@ -47,7 +45,47 @@ function extractVerdictObject(content: string): unknown {
|
|
|
47
45
|
return JSON.parse(candidate.slice(start, end + 1));
|
|
48
46
|
}
|
|
49
47
|
|
|
50
|
-
|
|
48
|
+
/**
|
|
49
|
+
* Compute additional verification checks from the run's runtime context.
|
|
50
|
+
*
|
|
51
|
+
* When `policy.verification.dynamicChecks` is configured, this inspects the number of affected
|
|
52
|
+
* paths, whether any match high-risk patterns, and adds corresponding check names to the list
|
|
53
|
+
* that the verifier must address.
|
|
54
|
+
*/
|
|
55
|
+
function dynamicChecksFor(input: VerificationRequest): string[] {
|
|
56
|
+
const config = input.policy.verification.dynamicChecks;
|
|
57
|
+
if (!config) return [];
|
|
58
|
+
|
|
59
|
+
const extra: string[] = [];
|
|
60
|
+
const affectedCount = input.actingOn?.length ?? 0;
|
|
61
|
+
|
|
62
|
+
if (affectedCount >= config.codeChangeThreshold) {
|
|
63
|
+
extra.push('broad_change_review');
|
|
64
|
+
}
|
|
65
|
+
if (config.riskPathPatterns.length > 0 && input.actingOn?.length) {
|
|
66
|
+
const hasRiskPath = input.actingOn.some((path) =>
|
|
67
|
+
config.riskPathPatterns.some((pattern) => path.startsWith(pattern) || path.includes(pattern)),
|
|
68
|
+
);
|
|
69
|
+
if (hasRiskPath) {
|
|
70
|
+
extra.push('risk_path_review');
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
if (config.requireSecurityReview) {
|
|
74
|
+
extra.push('security_review');
|
|
75
|
+
}
|
|
76
|
+
if (config.requirePerformanceReview) {
|
|
77
|
+
extra.push('performance_review');
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
return [...new Set(extra)];
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function verifierPrompt(
|
|
84
|
+
input: VerificationRequest,
|
|
85
|
+
artifacts: Array<Record<string, unknown>>,
|
|
86
|
+
dynamicChecks: string[],
|
|
87
|
+
) {
|
|
88
|
+
const allChecks = [...new Set([...input.policy.verification.requiredChecks, ...dynamicChecks])];
|
|
51
89
|
const catalogue = artifacts.length
|
|
52
90
|
? artifacts.map((artifact) => `- id=${artifact.id} kind=${artifact.kind} title=${artifact.title || ''}`).join('\n')
|
|
53
91
|
: '- (none)';
|
|
@@ -61,7 +99,7 @@ function verifierPrompt(input: VerificationRequest, artifacts: Array<Record<stri
|
|
|
61
99
|
'Evidence artifacts recorded for this run:',
|
|
62
100
|
catalogue,
|
|
63
101
|
'',
|
|
64
|
-
`Required checks: ${
|
|
102
|
+
`Required checks: ${allChecks.join(', ')}`,
|
|
65
103
|
'',
|
|
66
104
|
...(input.loopNotices ?? []),
|
|
67
105
|
'Reply with a single JSON object and nothing else:',
|
|
@@ -79,6 +117,7 @@ export class VerificationService {
|
|
|
79
117
|
private readonly database: Database,
|
|
80
118
|
private readonly runtime: PluginAiRuntimeAdapter,
|
|
81
119
|
private readonly stateMachine: LoopRunStateMachine,
|
|
120
|
+
private readonly tokenTracker: TokenTracker,
|
|
82
121
|
) {
|
|
83
122
|
this.loopGuard = new ToolLoopDetectionService(database);
|
|
84
123
|
}
|
|
@@ -159,8 +198,7 @@ export class VerificationService {
|
|
|
159
198
|
}
|
|
160
199
|
|
|
161
200
|
private async nextSequence(runId: number) {
|
|
162
|
-
|
|
163
|
-
return steps.length;
|
|
201
|
+
return this.database.getRepository('agentLoopSteps').count({ filter: { runId } });
|
|
164
202
|
}
|
|
165
203
|
|
|
166
204
|
private async collectVerdict(input: VerificationRequest, artifacts: Array<Record<string, unknown>>) {
|
|
@@ -195,9 +233,19 @@ export class VerificationService {
|
|
|
195
233
|
systemMessage:
|
|
196
234
|
'You verify completed work against its stated goal using only recorded evidence. You never modify anything.',
|
|
197
235
|
harness: input.verifierHarness,
|
|
198
|
-
prompt: verifierPrompt(input, artifacts),
|
|
236
|
+
prompt: verifierPrompt(input, artifacts, dynamicChecksFor(input)),
|
|
199
237
|
signal: input.signal,
|
|
200
238
|
});
|
|
239
|
+
|
|
240
|
+
// Track actual token usage from the verifier's LLM response
|
|
241
|
+
await this.tokenTracker.trackActualUsage(
|
|
242
|
+
read(step, 'id'),
|
|
243
|
+
outcome.tokenUsage ?? null,
|
|
244
|
+
verifierPrompt(input, artifacts, dynamicChecksFor(input)),
|
|
245
|
+
outcome.content,
|
|
246
|
+
input.runId,
|
|
247
|
+
);
|
|
248
|
+
|
|
201
249
|
await this.database.getRepository('agentLoopSteps').update({
|
|
202
250
|
filterByTk: read(step, 'id'),
|
|
203
251
|
values: {
|
|
@@ -212,7 +260,8 @@ export class VerificationService {
|
|
|
212
260
|
throw new Error('The verifier requested an approval-gated tool; verification cannot complete.');
|
|
213
261
|
}
|
|
214
262
|
const verdict = parseVerificationVerdict(extractVerdictObject(outcome.content));
|
|
215
|
-
const
|
|
263
|
+
const dynamicChecks = dynamicChecksFor(input);
|
|
264
|
+
const required = new Set([...input.policy.verification.requiredChecks, ...dynamicChecks]);
|
|
216
265
|
for (const check of verdict.checks) required.delete(check.name);
|
|
217
266
|
if (required.size) {
|
|
218
267
|
throw new Error(`The verifier omitted required checks: ${Array.from(required).join(', ')}.`);
|
|
@@ -1,23 +1,16 @@
|
|
|
1
|
+
import { asObject } from '../utils/ctx-utils';
|
|
1
2
|
import type { Database } from '@nocobase/database';
|
|
2
3
|
import { compileHarness } from './HarnessCompiler';
|
|
3
4
|
import type { CompiledHarness } from './HarnessCompiler';
|
|
4
5
|
import { HarnessProfileService } from './HarnessProfileService';
|
|
5
6
|
import { parseHarnessSettings } from './HarnessSchema';
|
|
7
|
+
import { read } from '../utils/record-utils';
|
|
6
8
|
|
|
7
9
|
export type TraceLike = {
|
|
8
10
|
agentLoopRunId?: string | number;
|
|
9
11
|
employeeUsername?: string;
|
|
10
12
|
};
|
|
11
13
|
|
|
12
|
-
function read(record: unknown, key: string) {
|
|
13
|
-
const model = record as { get?: (name: string) => unknown } | null;
|
|
14
|
-
return typeof model?.get === 'function' ? model.get(key) : (model as Record<string, unknown> | null)?.[key];
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
function asObject(value: unknown): Record<string, unknown> {
|
|
18
|
-
return value && typeof value === 'object' && !Array.isArray(value) ? (value as Record<string, unknown>) : {};
|
|
19
|
-
}
|
|
20
|
-
|
|
21
14
|
// Run snapshots were compiled at enqueue time, possibly by an older schema revision. Re-parsing
|
|
22
15
|
// through the current schema backfills new policy fields with their defaults. `sources` is a
|
|
23
16
|
// compiler artifact the strict schema rejects, so it is stripped first.
|