plugin-agent-orchestrator 1.1.2 → 1.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client/index.js +1 -1
- package/dist/client-v2/214.b063f33ba044a808.js +10 -0
- package/dist/client-v2/253.59577b40d9875be0.js +10 -0
- package/dist/client-v2/264.7ed8bee71fa5935c.js +10 -0
- package/dist/client-v2/{41.ba6e080cc0488143.js → 41.3deb9e57a0f64fc4.js} +1 -1
- package/dist/client-v2/418.58fa350f5496efd3.js +10 -0
- package/dist/client-v2/{53.7c29c28043e9112d.js → 53.9a10455323390f14.js} +1 -1
- package/dist/client-v2/619.0a2144aaffefe018.js +10 -0
- package/dist/client-v2/{70.bda9518881c05360.js → 70.ffff2997c10fd4b0.js} +1 -1
- package/dist/client-v2/{794.9bbb8d3a61b2bdf2.js → 794.a67a2614405ba627.js} +1 -1
- package/dist/client-v2/925.e5925bf21ed68e7a.js +10 -0
- package/dist/client-v2/{993.82778b6fd6db3cb1.js → 993.b0688fdc5ccb000a.js} +1 -1
- package/dist/client-v2/index.js +1 -1
- package/dist/externalVersion.js +8 -6
- package/dist/locale/en-US.json +48 -28
- package/dist/locale/vi-VN.json +48 -28
- package/dist/locale/zh-CN.json +48 -28
- package/dist/server/collections/agent-execution-spans.js +47 -0
- package/dist/server/collections/agent-harness-profile-versions.js +56 -0
- package/dist/server/collections/agent-harness-profiles.js +12 -0
- package/dist/server/collections/agent-loop-action-approvals.js +68 -0
- package/dist/server/collections/agent-loop-artifacts.js +55 -0
- package/dist/server/collections/agent-loop-circuit-states.js +53 -0
- package/dist/server/collections/agent-loop-control-settings.js +51 -0
- package/dist/server/collections/agent-loop-events.js +27 -0
- package/dist/server/collections/agent-loop-path-locks.js +57 -0
- package/dist/server/collections/agent-loop-patterns.js +72 -0
- package/dist/server/collections/agent-loop-runs.js +163 -2
- package/dist/server/collections/agent-loop-steps.js +69 -0
- package/dist/server/collections/agent-loop-usage-buckets.js +52 -0
- package/dist/server/collections/agent-loop-worktrees.js +55 -0
- package/dist/server/migrations/{20260427000000-change-packages-to-text.js → 20260427000002-change-packages-to-text.js} +2 -0
- package/dist/server/migrations/20260429000000-add-llm-fields.js +3 -2
- package/dist/server/migrations/{20260429000000-fix-inputargs-json-to-text.js → 20260429000001-fix-inputargs-json-to-text.js} +14 -6
- package/dist/server/migrations/20260503000000-add-orchestrator-trace-fields.js +2 -1
- package/dist/server/migrations/20260524000000-add-agent-loop-fields-to-skill-executions.js +2 -1
- package/dist/server/migrations/20260803010000-backfill-loop-control-plane-data.js +262 -0
- package/dist/server/plugin.js +52 -150
- package/dist/server/resources/agent-loop-approval.js +83 -0
- package/dist/server/resources/agent-loop-artifact.js +86 -0
- package/dist/server/resources/agent-loop-events-stream.js +130 -0
- package/dist/server/resources/agent-loop.js +181 -139
- package/dist/server/resources/resource-helpers.js +63 -0
- package/dist/server/resources/tracing.js +121 -107
- package/dist/server/services/DurableCircuitBreakerService.js +229 -0
- package/dist/server/services/HarnessCompiler.js +142 -0
- package/dist/server/services/HarnessProfileService.js +168 -0
- package/dist/server/services/HarnessSchema.js +167 -0
- package/dist/server/services/LoopBudgetService.js +269 -0
- package/dist/server/services/LoopControlService.js +139 -0
- package/dist/server/services/LoopPatternSchema.js +180 -0
- package/dist/server/services/LoopPatternService.js +132 -0
- package/dist/server/services/LoopRunRepository.js +205 -0
- package/dist/server/services/LoopRunStateMachine.js +389 -0
- package/dist/server/services/LoopSchedulerService.js +193 -0
- package/dist/server/services/LoopTriggerService.js +128 -0
- package/dist/server/services/LoopWorkerService.js +461 -0
- package/dist/server/services/PathLockService.js +288 -0
- package/dist/server/services/PluginAiRuntimeAdapter.js +215 -0
- package/dist/server/services/VerificationSchema.js +59 -0
- package/dist/server/services/VerificationService.js +169 -0
- package/dist/server/skill-hub/actions/git-import.js +13 -7
- package/dist/server/skill-hub/services/RegistrySkillSnapshotService.js +1 -5
- package/dist/server/utils/ensure-indexes.js +111 -0
- package/dist/server/utils/loop-pattern-context.js +56 -0
- package/package.json +2 -2
- package/src/client/OrchestratorSettings.tsx +44 -0
- package/src/client-v2/components/AgentRunsTab.tsx +576 -494
- package/src/client-v2/components/HarnessProfilesTab.tsx +13 -8
- package/src/client-v2/components/MemoryInspectorTab.tsx +19 -3
- package/src/client-v2/components/TracingTab.tsx +515 -432
- package/src/client-v2/hooks/useApiRequest.ts +141 -121
- package/src/client-v2/skill-hub/components/SkillMetrics.tsx +8 -6
- package/src/locale/en-US.json +52 -28
- package/src/locale/vi-VN.json +52 -28
- package/src/locale/zh-CN.json +52 -28
- package/src/server/__tests__/circuit-breaker.test.ts +225 -146
- package/src/server/__tests__/harness-compiler.test.ts +131 -0
- package/src/server/__tests__/harness-profile-service.test.ts +116 -0
- package/src/server/__tests__/harness-tool-plan.test.ts +148 -0
- package/src/server/__tests__/loop-budget-service.test.ts +255 -0
- package/src/server/__tests__/loop-control-plane-backfill.test.ts +266 -0
- package/src/server/__tests__/loop-pattern-schema.test.ts +56 -0
- package/src/server/__tests__/loop-pattern-service.test.ts +148 -0
- package/src/server/__tests__/loop-run-state-machine.test.ts +90 -0
- package/src/server/__tests__/loop-worker-claim.test.ts +341 -0
- package/src/server/__tests__/loop-worker-execution.test.ts +236 -0
- package/src/server/__tests__/path-lock-service.test.ts +254 -0
- package/src/server/__tests__/smoke.test.ts +62 -21
- package/src/server/collections/agent-execution-spans.ts +47 -0
- package/src/server/collections/agent-harness-profile-versions.ts +26 -0
- package/src/server/collections/agent-harness-profiles.ts +12 -0
- package/src/server/collections/agent-loop-action-approvals.ts +38 -0
- package/src/server/collections/agent-loop-artifacts.ts +25 -0
- package/src/server/collections/agent-loop-circuit-states.ts +23 -0
- package/src/server/collections/agent-loop-control-settings.ts +21 -0
- package/src/server/collections/agent-loop-events.ts +27 -0
- package/src/server/collections/agent-loop-path-locks.ts +27 -0
- package/src/server/collections/agent-loop-patterns.ts +42 -0
- package/src/server/collections/agent-loop-runs.ts +163 -2
- package/src/server/collections/agent-loop-steps.ts +69 -0
- package/src/server/collections/agent-loop-usage-buckets.ts +22 -0
- package/src/server/collections/agent-loop-worktrees.ts +25 -0
- package/src/server/migrations/{20260427000000-change-packages-to-text.ts → 20260427000002-change-packages-to-text.ts} +3 -0
- package/src/server/migrations/20260429000000-add-llm-fields.ts +3 -2
- package/src/server/migrations/{20260429000000-fix-inputargs-json-to-text.ts → 20260429000001-fix-inputargs-json-to-text.ts} +13 -6
- package/src/server/migrations/20260503000000-add-orchestrator-trace-fields.ts +2 -1
- package/src/server/migrations/20260524000000-add-agent-loop-fields-to-skill-executions.ts +2 -1
- package/src/server/migrations/20260803010000-backfill-loop-control-plane-data.ts +293 -0
- package/src/server/plugin.ts +53 -167
- package/src/server/resources/agent-loop-approval.ts +52 -0
- package/src/server/resources/agent-loop-artifact.ts +56 -0
- package/src/server/resources/agent-loop-events-stream.ts +105 -0
- package/src/server/resources/agent-loop.ts +202 -143
- package/src/server/resources/agent-monitor.ts +482 -482
- package/src/server/resources/resource-helpers.ts +33 -0
- package/src/server/resources/tracing.ts +145 -137
- package/src/server/services/DurableCircuitBreakerService.ts +239 -0
- package/src/server/services/HarnessCompiler.ts +133 -0
- package/src/server/services/HarnessProfileService.ts +157 -0
- package/src/server/services/HarnessSchema.ts +163 -0
- package/src/server/services/LoopBudgetService.ts +275 -0
- package/src/server/services/LoopControlService.ts +124 -0
- package/src/server/services/LoopPatternSchema.ts +167 -0
- package/src/server/services/LoopPatternService.ts +143 -0
- package/src/server/services/LoopRunRepository.ts +237 -0
- package/src/server/services/LoopRunStateMachine.ts +432 -0
- package/src/server/services/LoopSchedulerService.ts +199 -0
- package/src/server/services/LoopTriggerService.ts +115 -0
- package/src/server/services/LoopWorkerService.ts +522 -0
- package/src/server/services/PathLockService.ts +321 -0
- package/src/server/services/PluginAiRuntimeAdapter.ts +315 -0
- package/src/server/services/VerificationSchema.ts +31 -0
- package/src/server/services/VerificationService.ts +185 -0
- package/src/server/skill-hub/actions/__tests__/git-import.test.ts +62 -0
- package/src/server/skill-hub/actions/git-import.ts +9 -3
- package/src/server/skill-hub/services/RegistrySkillSnapshotService.ts +1 -5
- package/src/server/skill-hub/services/__tests__/RegistrySkillSnapshotService.test.ts +1 -3
- package/src/server/utils/ensure-indexes.ts +96 -0
- package/src/server/utils/logging.ts +0 -1
- package/src/server/utils/loop-pattern-context.ts +20 -0
- package/dist/client-v2/214.79650a549273f163.js +0 -10
- package/dist/client-v2/253.9f0537713cee9e6c.js +0 -10
- package/dist/client-v2/264.718a107e43fc163c.js +0 -10
- package/dist/client-v2/418.ce8f5b2b8a229d1d.js +0 -10
- package/dist/client-v2/619.bd3c5698b40705c3.js +0 -10
- package/dist/client-v2/925.f5370de8f6632d65.js +0 -10
- package/dist/server/services/AgentHarness.js +0 -81
- package/dist/server/services/AgentLoopController.js +0 -1023
- package/dist/server/services/AgentLoopRepository.js +0 -192
- package/dist/server/services/AgentLoopService.js +0 -142
- package/dist/server/services/AgentPlanValidator.js +0 -99
- package/dist/server/services/AgentPlannerService.js +0 -99
- package/dist/server/services/CircuitBreaker.js +0 -125
- package/dist/server/services/ContextAggregator.js +0 -201
- package/dist/server/services/PlanningPromptService.js +0 -95
- package/dist/server/tools/agent-loop.js +0 -373
- package/dist/server/tools/orchestrator-plan.js +0 -254
- package/src/server/__tests__/agent-loop-controller.test.ts +0 -375
- package/src/server/__tests__/context-aggregator.test.ts +0 -222
- package/src/server/__tests__/parallel-execution.test.ts +0 -318
- package/src/server/__tests__/planning-prompt-service.test.ts +0 -23
- package/src/server/services/AgentHarness.ts +0 -67
- package/src/server/services/AgentLoopController.ts +0 -1263
- package/src/server/services/AgentLoopRepository.ts +0 -178
- package/src/server/services/AgentLoopService.ts +0 -177
- package/src/server/services/AgentPlanValidator.ts +0 -73
- package/src/server/services/AgentPlannerService.ts +0 -80
- package/src/server/services/CircuitBreaker.ts +0 -126
- package/src/server/services/ContextAggregator.ts +0 -239
- package/src/server/services/PlanningPromptService.ts +0 -72
- package/src/server/tools/agent-loop.ts +0 -360
- package/src/server/tools/orchestrator-plan.ts +0 -255
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import type { Context } from '@nocobase/actions';
|
|
2
|
+
import { ZodError } from 'zod';
|
|
3
|
+
import { currentUserId, isAdminUser } from '../utils/ctx-utils';
|
|
4
|
+
import { LoopRunAccessError } from '../services/LoopRunRepository';
|
|
5
|
+
|
|
6
|
+
export type RequestActor = {
|
|
7
|
+
userId: number;
|
|
8
|
+
isAdmin: boolean;
|
|
9
|
+
};
|
|
10
|
+
|
|
11
|
+
export function requestActor(ctx: Context): RequestActor {
|
|
12
|
+
const userId = Number(currentUserId(ctx));
|
|
13
|
+
if (!Number.isSafeInteger(userId) || userId <= 0) {
|
|
14
|
+
throw new LoopRunAccessError(401, 'Authentication is required.');
|
|
15
|
+
}
|
|
16
|
+
return { userId, isAdmin: isAdminUser(ctx) };
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export function requireAdminActor(ctx: Context) {
|
|
20
|
+
const actor = requestActor(ctx);
|
|
21
|
+
if (!actor.isAdmin) throw new LoopRunAccessError(403, 'Administrator access is required.');
|
|
22
|
+
return actor;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export function throwResourceError(ctx: Context, error: unknown): never {
|
|
26
|
+
if (error instanceof LoopRunAccessError) {
|
|
27
|
+
ctx.throw(error.status, error.message);
|
|
28
|
+
}
|
|
29
|
+
if (error instanceof ZodError) {
|
|
30
|
+
ctx.throw(400, error.issues.map((issue) => issue.message).join(' '));
|
|
31
|
+
}
|
|
32
|
+
ctx.throw(409, error instanceof Error ? error.message : 'The requested operation could not be completed.');
|
|
33
|
+
}
|
|
@@ -1,8 +1,16 @@
|
|
|
1
|
-
import { Plugin } from '@nocobase/server';
|
|
1
|
+
import type { Plugin } from '@nocobase/server';
|
|
2
|
+
import { LoopRunAccessError, LoopRunRepository } from '../services/LoopRunRepository';
|
|
3
|
+
import { requestActor, throwResourceError } from './resource-helpers';
|
|
2
4
|
|
|
3
|
-
|
|
5
|
+
type TraceRecord = Record<string, unknown>;
|
|
4
6
|
|
|
5
|
-
function
|
|
7
|
+
function plainRecord(raw: unknown): TraceRecord {
|
|
8
|
+
if (!raw || typeof raw !== 'object') return {};
|
|
9
|
+
const record = raw as { toJSON?: () => TraceRecord };
|
|
10
|
+
return typeof record.toJSON === 'function' ? record.toJSON() : (raw as TraceRecord);
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
function normalizeSpanFilter(filter: Record<string, unknown> = {}) {
|
|
6
14
|
const next = { ...filter };
|
|
7
15
|
if (next.subAgentUsername) {
|
|
8
16
|
next.employeeUsername = next.subAgentUsername;
|
|
@@ -11,16 +19,33 @@ function normalizeSpanFilter(filter: any = {}) {
|
|
|
11
19
|
return next;
|
|
12
20
|
}
|
|
13
21
|
|
|
14
|
-
function
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
22
|
+
function mergeFilter(filter: Record<string, unknown>, required: Record<string, unknown>) {
|
|
23
|
+
return Object.keys(filter).length > 0 ? { $and: [filter, required] } : required;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
function asObject(value: unknown): Record<string, unknown> {
|
|
27
|
+
return value && typeof value === 'object' && !Array.isArray(value) ? (value as Record<string, unknown>) : {};
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
function asArray(value: unknown) {
|
|
31
|
+
return Array.isArray(value) ? value : [];
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function positiveId(value: unknown) {
|
|
35
|
+
const parsed = Number(value);
|
|
36
|
+
return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : null;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function spanTitle(row: TraceRecord) {
|
|
40
|
+
if (typeof row.title === 'string' && row.title) return row.title;
|
|
41
|
+
if (row.type === 'sub_agent') return `${String(row.leaderUsername || '-')} -> ${String(row.employeeUsername || '-')}`;
|
|
42
|
+
return String(row.toolName || row.type || 'span');
|
|
18
43
|
}
|
|
19
44
|
|
|
20
|
-
function formatSpanListRow(raw:
|
|
21
|
-
const row =
|
|
22
|
-
const metadata = row.metadata
|
|
23
|
-
const input = row.input
|
|
45
|
+
function formatSpanListRow(raw: unknown) {
|
|
46
|
+
const row = plainRecord(raw);
|
|
47
|
+
const metadata = asObject(row.metadata);
|
|
48
|
+
const input = asObject(row.input);
|
|
24
49
|
return {
|
|
25
50
|
id: row.id,
|
|
26
51
|
rootRunId: row.rootRunId,
|
|
@@ -45,45 +70,44 @@ function formatSpanListRow(raw: any) {
|
|
|
45
70
|
};
|
|
46
71
|
}
|
|
47
72
|
|
|
48
|
-
function buildSpanTree(rows:
|
|
49
|
-
const
|
|
50
|
-
const
|
|
51
|
-
const roots: any[] = [];
|
|
73
|
+
function buildSpanTree(rows: unknown[]) {
|
|
74
|
+
const byId = new Map<string, TraceRecord & { children: TraceRecord[] }>();
|
|
75
|
+
const roots: Array<TraceRecord & { children: TraceRecord[] }> = [];
|
|
52
76
|
|
|
53
|
-
for (const
|
|
54
|
-
const
|
|
55
|
-
byId.set(String(row.id),
|
|
77
|
+
for (const raw of rows) {
|
|
78
|
+
const row = plainRecord(raw);
|
|
79
|
+
byId.set(String(row.id), { ...row, children: [] });
|
|
56
80
|
}
|
|
57
81
|
|
|
58
82
|
for (const row of byId.values()) {
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
roots.push(row);
|
|
63
|
-
}
|
|
83
|
+
const parent = row.parentSpanId ? byId.get(String(row.parentSpanId)) : undefined;
|
|
84
|
+
if (parent) parent.children.push(row);
|
|
85
|
+
else roots.push(row);
|
|
64
86
|
}
|
|
65
87
|
|
|
66
|
-
const sortTree = (items:
|
|
88
|
+
const sortTree = (items: Array<TraceRecord & { children: TraceRecord[] }>) => {
|
|
67
89
|
items.sort(
|
|
68
|
-
(
|
|
69
|
-
new Date(
|
|
90
|
+
(left, right) =>
|
|
91
|
+
new Date(String(left.startedAt || left.createdAt || 0)).getTime() -
|
|
92
|
+
new Date(String(right.startedAt || right.createdAt || 0)).getTime(),
|
|
70
93
|
);
|
|
71
|
-
for (const item of items) sortTree(item.children
|
|
94
|
+
for (const item of items) sortTree(item.children as Array<TraceRecord & { children: TraceRecord[] }>);
|
|
72
95
|
};
|
|
73
96
|
sortTree(roots);
|
|
74
97
|
return roots;
|
|
75
98
|
}
|
|
76
99
|
|
|
77
|
-
function flattenSpanTimeline(rows:
|
|
100
|
+
function flattenSpanTimeline(rows: unknown[]) {
|
|
78
101
|
return rows
|
|
79
|
-
.map(
|
|
102
|
+
.map(plainRecord)
|
|
80
103
|
.sort(
|
|
81
|
-
(
|
|
82
|
-
new Date(
|
|
104
|
+
(left, right) =>
|
|
105
|
+
new Date(String(left.startedAt || left.createdAt || 0)).getTime() -
|
|
106
|
+
new Date(String(right.startedAt || right.createdAt || 0)).getTime(),
|
|
83
107
|
)
|
|
84
108
|
.map((row) => {
|
|
85
|
-
const input = row.input
|
|
86
|
-
const metadata = row.metadata
|
|
109
|
+
const input = asObject(row.input);
|
|
110
|
+
const metadata = asObject(row.metadata);
|
|
87
111
|
return {
|
|
88
112
|
type: row.type,
|
|
89
113
|
at: row.startedAt || row.createdAt,
|
|
@@ -99,80 +123,81 @@ function flattenSpanTimeline(rows: any[]) {
|
|
|
99
123
|
});
|
|
100
124
|
}
|
|
101
125
|
|
|
102
|
-
/**
|
|
103
|
-
* Custom resource for the Swarm Tracing admin UI (Phase 5).
|
|
104
|
-
* Queries the dedicated orchestratorLogs collection instead of
|
|
105
|
-
* filtering aiConversations by JSONB (P2 fix: DB-engine agnostic).
|
|
106
|
-
*/
|
|
107
126
|
export function registerTracingResource(plugin: Plugin) {
|
|
108
|
-
const
|
|
127
|
+
const runs = new LoopRunRepository(plugin.db);
|
|
109
128
|
|
|
110
|
-
app.resource({
|
|
129
|
+
plugin.app.resource({
|
|
111
130
|
name: 'orchestratorTracing',
|
|
112
131
|
actions: {
|
|
113
|
-
/**
|
|
114
|
-
* List all delegation execution logs.
|
|
115
|
-
*/
|
|
116
132
|
async list(ctx, next) {
|
|
117
|
-
const { page = 1, pageSize = 50, sort = ['-createdAt'], filter = {} } = ctx.action.params;
|
|
118
|
-
|
|
119
133
|
try {
|
|
134
|
+
const actor = requestActor(ctx);
|
|
135
|
+
const { page = 1, pageSize = 50, sort = ['-createdAt'], filter = {} } = ctx.action.params;
|
|
136
|
+
const ownedRunIds = actor.isAdmin ? [] : await runs.listOwnedRunIds(actor.userId, false);
|
|
137
|
+
const spanAccess = actor.isAdmin
|
|
138
|
+
? {}
|
|
139
|
+
: {
|
|
140
|
+
$or: [
|
|
141
|
+
{ agentLoopRunId: { $in: ownedRunIds.length > 0 ? ownedRunIds : [-1] } },
|
|
142
|
+
{ agentLoopRunId: null, userId: actor.userId },
|
|
143
|
+
],
|
|
144
|
+
};
|
|
145
|
+
const spanFilter = mergeFilter(
|
|
146
|
+
{ ...normalizeSpanFilter(filter), parentSpanId: null, type: 'sub_agent' },
|
|
147
|
+
spanAccess,
|
|
148
|
+
);
|
|
120
149
|
const spanRepo = ctx.db.getRepository('agentExecutionSpans');
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
const [spanRows, spanCount] = await spanRepo.findAndCount({
|
|
128
|
-
filter: spanFilter,
|
|
129
|
-
sort,
|
|
130
|
-
offset: (Number(page) - 1) * Number(pageSize),
|
|
131
|
-
limit: Number(pageSize),
|
|
132
|
-
});
|
|
150
|
+
const [spanRows, spanCount] = await spanRepo.findAndCount({
|
|
151
|
+
filter: spanFilter,
|
|
152
|
+
sort,
|
|
153
|
+
offset: (Number(page) - 1) * Number(pageSize),
|
|
154
|
+
limit: Number(pageSize),
|
|
155
|
+
});
|
|
133
156
|
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
}
|
|
157
|
+
if (spanCount > 0) {
|
|
158
|
+
ctx.body = {
|
|
159
|
+
data: spanRows.map(formatSpanListRow),
|
|
160
|
+
meta: {
|
|
161
|
+
count: spanCount,
|
|
162
|
+
page: Number(page),
|
|
163
|
+
pageSize: Number(pageSize),
|
|
164
|
+
totalPage: Math.ceil(spanCount / Number(pageSize)),
|
|
165
|
+
},
|
|
166
|
+
};
|
|
167
|
+
await next();
|
|
168
|
+
return;
|
|
147
169
|
}
|
|
148
170
|
|
|
149
|
-
const
|
|
150
|
-
const [rows, count] = await
|
|
151
|
-
filter,
|
|
171
|
+
const logFilter = actor.isAdmin ? filter : mergeFilter(filter, { userId: actor.userId });
|
|
172
|
+
const [rows, count] = await ctx.db.getRepository('orchestratorLogs').findAndCount({
|
|
173
|
+
filter: logFilter,
|
|
152
174
|
sort,
|
|
153
175
|
offset: (Number(page) - 1) * Number(pageSize),
|
|
154
176
|
limit: Number(pageSize),
|
|
155
177
|
});
|
|
156
178
|
|
|
157
179
|
ctx.body = {
|
|
158
|
-
data: rows.map((
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
180
|
+
data: rows.map((raw) => {
|
|
181
|
+
const row = plainRecord(raw);
|
|
182
|
+
return {
|
|
183
|
+
id: row.id,
|
|
184
|
+
leaderUsername: row.leaderUsername,
|
|
185
|
+
subAgentUsername: row.subAgentUsername,
|
|
186
|
+
toolName: row.toolName,
|
|
187
|
+
task: row.task,
|
|
188
|
+
context: row.context,
|
|
189
|
+
result: row.result,
|
|
190
|
+
status: row.status,
|
|
191
|
+
depth: row.depth,
|
|
192
|
+
durationMs: row.durationMs,
|
|
193
|
+
error: row.error,
|
|
194
|
+
userId: row.userId,
|
|
195
|
+
createdAt: row.createdAt,
|
|
196
|
+
traceCount: asArray(row.trace).length,
|
|
197
|
+
messageCount: asArray(row.messages).length,
|
|
198
|
+
hasUnifiedTrace: false,
|
|
199
|
+
};
|
|
200
|
+
}),
|
|
176
201
|
meta: {
|
|
177
202
|
count,
|
|
178
203
|
page: Number(page),
|
|
@@ -180,53 +205,43 @@ export function registerTracingResource(plugin: Plugin) {
|
|
|
180
205
|
totalPage: Math.ceil(count / Number(pageSize)),
|
|
181
206
|
},
|
|
182
207
|
};
|
|
183
|
-
} catch (
|
|
184
|
-
|
|
185
|
-
ctx.log.error('[AgentOrchestrator] Tracing list error', e);
|
|
186
|
-
ctx.throw(500, e?.message || 'Failed to load tracing logs');
|
|
208
|
+
} catch (error) {
|
|
209
|
+
throwResourceError(ctx, error);
|
|
187
210
|
}
|
|
188
|
-
|
|
189
211
|
await next();
|
|
190
212
|
},
|
|
191
213
|
|
|
192
|
-
/**
|
|
193
|
-
* Get a single delegation log by ID.
|
|
194
|
-
*/
|
|
195
214
|
async get(ctx, next) {
|
|
196
|
-
const { filterByTk, source } = ctx.action.params;
|
|
197
|
-
|
|
198
|
-
if (!filterByTk) {
|
|
199
|
-
ctx.throw(400, 'id is required');
|
|
200
|
-
return;
|
|
201
|
-
}
|
|
202
|
-
|
|
203
215
|
try {
|
|
204
|
-
const
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
filter: { id: filterByTk },
|
|
208
|
-
});
|
|
216
|
+
const actor = requestActor(ctx);
|
|
217
|
+
const { filterByTk, source } = ctx.action.params;
|
|
218
|
+
if (!filterByTk) ctx.throw(400, 'id is required');
|
|
209
219
|
|
|
220
|
+
const spanRepo = ctx.db.getRepository('agentExecutionSpans');
|
|
221
|
+
if (source !== 'log') {
|
|
222
|
+
const span = await spanRepo.findOne({ filterByTk });
|
|
210
223
|
if (span) {
|
|
211
|
-
const plainSpan =
|
|
224
|
+
const plainSpan = plainRecord(span);
|
|
225
|
+
const runId = positiveId(plainSpan.agentLoopRunId);
|
|
226
|
+
if (runId) await runs.requireOwnedRun(runId, actor.userId, actor.isAdmin);
|
|
227
|
+
else if (!actor.isAdmin && Number(plainSpan.userId) !== actor.userId) {
|
|
228
|
+
throw new LoopRunAccessError(403, 'You cannot access this execution trace.');
|
|
229
|
+
}
|
|
230
|
+
|
|
212
231
|
const rows = await spanRepo.find({
|
|
213
232
|
filter: { rootRunId: plainSpan.rootRunId },
|
|
214
233
|
sort: ['createdAt'],
|
|
215
234
|
});
|
|
216
|
-
const
|
|
217
|
-
const timeline = flattenSpanTimeline(rows);
|
|
218
|
-
const rootRow = formatSpanListRow(plainSpan);
|
|
219
|
-
const metadata = plainSpan.metadata || {};
|
|
220
|
-
|
|
235
|
+
const metadata = asObject(plainSpan.metadata);
|
|
221
236
|
ctx.body = {
|
|
222
237
|
data: {
|
|
223
|
-
...
|
|
238
|
+
...formatSpanListRow(plainSpan),
|
|
224
239
|
input: plainSpan.input,
|
|
225
240
|
output: plainSpan.output,
|
|
226
241
|
metadata,
|
|
227
|
-
trace:
|
|
228
|
-
messages: metadata.messages
|
|
229
|
-
children:
|
|
242
|
+
trace: flattenSpanTimeline(rows),
|
|
243
|
+
messages: asArray(metadata.messages),
|
|
244
|
+
children: buildSpanTree(rows),
|
|
230
245
|
},
|
|
231
246
|
};
|
|
232
247
|
await next();
|
|
@@ -234,25 +249,18 @@ export function registerTracingResource(plugin: Plugin) {
|
|
|
234
249
|
}
|
|
235
250
|
}
|
|
236
251
|
|
|
237
|
-
const
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
ctx.body = { data: plainLog ? { ...plainLog, hasUnifiedTrace: false } : null };
|
|
247
|
-
} catch (e: any) {
|
|
248
|
-
ctx.log.error('[AgentOrchestrator] Tracing get error', e);
|
|
249
|
-
ctx.throw(500, e?.message || 'Failed to load tracing log');
|
|
252
|
+
const log = source === 'span' ? null : await ctx.db.getRepository('orchestratorLogs').findOne({ filterByTk });
|
|
253
|
+
if (!log) throw new LoopRunAccessError(404, `Execution trace ${String(filterByTk)} was not found.`);
|
|
254
|
+
const plainLog = plainRecord(log);
|
|
255
|
+
if (!actor.isAdmin && Number(plainLog.userId) !== actor.userId) {
|
|
256
|
+
throw new LoopRunAccessError(403, 'You cannot access this execution trace.');
|
|
257
|
+
}
|
|
258
|
+
ctx.body = { data: { ...plainLog, hasUnifiedTrace: false } };
|
|
259
|
+
} catch (error) {
|
|
260
|
+
throwResourceError(ctx, error);
|
|
250
261
|
}
|
|
251
|
-
|
|
252
262
|
await next();
|
|
253
263
|
},
|
|
254
264
|
},
|
|
255
265
|
});
|
|
256
|
-
|
|
257
|
-
// ACL: access is granted by the parent plugin's snippet (pm.plugin-agent-orchestrator → orchestratorTracing:*)
|
|
258
266
|
}
|
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import type { Database, Model } from '@nocobase/database';
|
|
3
|
+
import type { LoopPatternPolicy } from './LoopPatternSchema';
|
|
4
|
+
|
|
5
|
+
type DistributedLock = {
|
|
6
|
+
runExclusive<T>(key: string, operation: () => Promise<T>, ttl?: number): Promise<T>;
|
|
7
|
+
};
|
|
8
|
+
|
|
9
|
+
type CircuitSnapshot = {
|
|
10
|
+
id: number;
|
|
11
|
+
state: 'closed' | 'open' | 'half_open';
|
|
12
|
+
attempts: number;
|
|
13
|
+
consecutiveFailures: number;
|
|
14
|
+
errorSignature: string;
|
|
15
|
+
repeatedErrorCount: number;
|
|
16
|
+
retryAt: Date | null;
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
function read(record: Model | Record<string, unknown>, key: string) {
|
|
20
|
+
const model = record as Model & { get?: (name: string) => unknown };
|
|
21
|
+
return typeof model.get === 'function' ? model.get(key) : (record as Record<string, unknown>)[key];
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function count(value: unknown, label: string) {
|
|
25
|
+
if (value === null || value === undefined) return 0;
|
|
26
|
+
const parsed = Number(value);
|
|
27
|
+
if (!Number.isSafeInteger(parsed) || parsed < 0) throw new Error(`${label} counter is invalid.`);
|
|
28
|
+
return parsed;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function dateOrNull(value: unknown) {
|
|
32
|
+
if (!value) return null;
|
|
33
|
+
const date = value instanceof Date ? value : new Date(String(value));
|
|
34
|
+
return Number.isNaN(date.getTime()) ? null : date;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function snapshot(record: Model | Record<string, unknown>): CircuitSnapshot {
|
|
38
|
+
const id = Number(read(record, 'id'));
|
|
39
|
+
const state = read(record, 'state');
|
|
40
|
+
if (!Number.isSafeInteger(id) || id <= 0) throw new Error('Circuit breaker state has an invalid identifier.');
|
|
41
|
+
if (state !== 'closed' && state !== 'open' && state !== 'half_open') {
|
|
42
|
+
throw new Error('Circuit breaker state is invalid.');
|
|
43
|
+
}
|
|
44
|
+
return {
|
|
45
|
+
id,
|
|
46
|
+
state,
|
|
47
|
+
attempts: count(read(record, 'attempts'), 'Circuit attempt'),
|
|
48
|
+
consecutiveFailures: count(read(record, 'consecutiveFailures'), 'Circuit consecutive failure'),
|
|
49
|
+
errorSignature: typeof read(record, 'errorSignature') === 'string' ? String(read(record, 'errorSignature')) : '',
|
|
50
|
+
repeatedErrorCount: count(read(record, 'repeatedErrorCount'), 'Circuit repeated error'),
|
|
51
|
+
retryAt: dateOrNull(read(record, 'retryAt')),
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function normalizeScopeKey(scopeKey: string) {
|
|
56
|
+
const normalized = scopeKey.trim();
|
|
57
|
+
if (!normalized) throw new Error('Circuit breaker scope key is required.');
|
|
58
|
+
if (normalized.length > 300) throw new Error('Circuit breaker scope key cannot exceed 300 characters.');
|
|
59
|
+
return normalized;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export function normalizeErrorSignature(error: unknown) {
|
|
63
|
+
const raw = error instanceof Error ? `${error.name}:${error.message}` : String(error);
|
|
64
|
+
const normalized = raw
|
|
65
|
+
.toLowerCase()
|
|
66
|
+
.replace(/[a-z]:[\\/][^\s"']+/gi, '<path>')
|
|
67
|
+
.replace(/\/(?:[^\s/]+\/)+[^\s"']*/g, '<path>')
|
|
68
|
+
.replace(/\b[0-9a-f]{8}-[0-9a-f-]{27,}\b/gi, '<uuid>')
|
|
69
|
+
.replace(/\b\d+\b/g, '<n>')
|
|
70
|
+
.replace(/\s+/g, ' ')
|
|
71
|
+
.trim();
|
|
72
|
+
return createHash('sha256')
|
|
73
|
+
.update(normalized || 'unknown-error')
|
|
74
|
+
.digest('hex');
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
export class DurableCircuitBreakerService {
|
|
78
|
+
constructor(
|
|
79
|
+
private readonly database: Database,
|
|
80
|
+
private readonly distributedLock: DistributedLock,
|
|
81
|
+
) {}
|
|
82
|
+
|
|
83
|
+
async authorizeAttempt(input: {
|
|
84
|
+
patternId: number;
|
|
85
|
+
scopeKey: string;
|
|
86
|
+
policy: LoopPatternPolicy['circuit'];
|
|
87
|
+
now?: Date;
|
|
88
|
+
}) {
|
|
89
|
+
const scopeKey = normalizeScopeKey(input.scopeKey);
|
|
90
|
+
const now = input.now || new Date();
|
|
91
|
+
return this.withScopeLock(input.patternId, scopeKey, async () => {
|
|
92
|
+
const decision = await this.database.sequelize.transaction(async (transaction) => {
|
|
93
|
+
const repository = this.database.getRepository('agentLoopCircuitStates');
|
|
94
|
+
let record = await repository.findOne({
|
|
95
|
+
filter: { patternId: input.patternId, scopeKey },
|
|
96
|
+
transaction,
|
|
97
|
+
lock: transaction.LOCK.UPDATE,
|
|
98
|
+
});
|
|
99
|
+
if (!record) {
|
|
100
|
+
record = await repository.create({
|
|
101
|
+
values: {
|
|
102
|
+
patternId: input.patternId,
|
|
103
|
+
scopeKey,
|
|
104
|
+
state: 'closed',
|
|
105
|
+
attempts: 0,
|
|
106
|
+
consecutiveFailures: 0,
|
|
107
|
+
repeatedErrorCount: 0,
|
|
108
|
+
},
|
|
109
|
+
transaction,
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
const current = snapshot(record);
|
|
113
|
+
if (current.state === 'half_open') {
|
|
114
|
+
return { allowed: false as const, reason: 'probe' as const, retryAt: current.retryAt };
|
|
115
|
+
}
|
|
116
|
+
if (current.state === 'open' && (!current.retryAt || current.retryAt.getTime() > now.getTime())) {
|
|
117
|
+
return { allowed: false as const, reason: 'open' as const, retryAt: current.retryAt };
|
|
118
|
+
}
|
|
119
|
+
if (current.state === 'closed' && current.attempts >= input.policy.maxAttempts) {
|
|
120
|
+
const retryAt = new Date(now.getTime() + input.policy.cooldownMs);
|
|
121
|
+
await repository.update({
|
|
122
|
+
filterByTk: current.id,
|
|
123
|
+
values: { state: 'open', openedAt: now, retryAt },
|
|
124
|
+
transaction,
|
|
125
|
+
});
|
|
126
|
+
return { allowed: false as const, reason: 'attempts' as const, retryAt };
|
|
127
|
+
}
|
|
128
|
+
const nextState = current.state === 'open' ? 'half_open' : 'closed';
|
|
129
|
+
await repository.update({
|
|
130
|
+
filterByTk: current.id,
|
|
131
|
+
values: { state: nextState, attempts: current.attempts + 1 },
|
|
132
|
+
transaction,
|
|
133
|
+
});
|
|
134
|
+
return { allowed: true as const, state: nextState, attempt: current.attempts + 1 };
|
|
135
|
+
});
|
|
136
|
+
if (decision.allowed) return decision;
|
|
137
|
+
if (decision.reason === 'probe') throw new Error('Circuit breaker probe is already in progress.');
|
|
138
|
+
if (decision.reason === 'attempts') throw new Error('Circuit breaker maximum attempts reached.');
|
|
139
|
+
throw new Error(`Circuit breaker is open until ${decision.retryAt?.toISOString() || 'manual reset'}.`);
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
async recordSuccess(patternId: number, scopeKey: string, lastRunId?: number) {
|
|
144
|
+
const normalizedScope = normalizeScopeKey(scopeKey);
|
|
145
|
+
return this.withScopeLock(patternId, normalizedScope, async () =>
|
|
146
|
+
this.database.sequelize.transaction(async (transaction) => {
|
|
147
|
+
const repository = this.database.getRepository('agentLoopCircuitStates');
|
|
148
|
+
const record = await repository.findOne({
|
|
149
|
+
filter: { patternId, scopeKey: normalizedScope },
|
|
150
|
+
transaction,
|
|
151
|
+
lock: transaction.LOCK.UPDATE,
|
|
152
|
+
});
|
|
153
|
+
if (!record) return;
|
|
154
|
+
await repository.update({
|
|
155
|
+
filterByTk: read(record, 'id'),
|
|
156
|
+
values: {
|
|
157
|
+
state: 'closed',
|
|
158
|
+
attempts: 0,
|
|
159
|
+
consecutiveFailures: 0,
|
|
160
|
+
errorSignature: null,
|
|
161
|
+
repeatedErrorCount: 0,
|
|
162
|
+
lastRunId: lastRunId || null,
|
|
163
|
+
openedAt: null,
|
|
164
|
+
retryAt: null,
|
|
165
|
+
},
|
|
166
|
+
transaction,
|
|
167
|
+
});
|
|
168
|
+
}),
|
|
169
|
+
);
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
async recordFailure(input: {
|
|
173
|
+
patternId: number;
|
|
174
|
+
scopeKey: string;
|
|
175
|
+
policy: LoopPatternPolicy['circuit'];
|
|
176
|
+
error: unknown;
|
|
177
|
+
lastRunId?: number;
|
|
178
|
+
now?: Date;
|
|
179
|
+
}) {
|
|
180
|
+
const scopeKey = normalizeScopeKey(input.scopeKey);
|
|
181
|
+
const signature = normalizeErrorSignature(input.error);
|
|
182
|
+
const now = input.now || new Date();
|
|
183
|
+
return this.withScopeLock(input.patternId, scopeKey, async () =>
|
|
184
|
+
this.database.sequelize.transaction(async (transaction) => {
|
|
185
|
+
const repository = this.database.getRepository('agentLoopCircuitStates');
|
|
186
|
+
let record = await repository.findOne({
|
|
187
|
+
filter: { patternId: input.patternId, scopeKey },
|
|
188
|
+
transaction,
|
|
189
|
+
lock: transaction.LOCK.UPDATE,
|
|
190
|
+
});
|
|
191
|
+
if (!record) {
|
|
192
|
+
record = await repository.create({
|
|
193
|
+
values: {
|
|
194
|
+
patternId: input.patternId,
|
|
195
|
+
scopeKey,
|
|
196
|
+
state: 'closed',
|
|
197
|
+
attempts: 1,
|
|
198
|
+
consecutiveFailures: 0,
|
|
199
|
+
repeatedErrorCount: 0,
|
|
200
|
+
},
|
|
201
|
+
transaction,
|
|
202
|
+
});
|
|
203
|
+
}
|
|
204
|
+
const current = snapshot(record);
|
|
205
|
+
const consecutiveFailures = current.consecutiveFailures + 1;
|
|
206
|
+
const repeatedErrorCount = current.errorSignature === signature ? current.repeatedErrorCount + 1 : 1;
|
|
207
|
+
const shouldOpen =
|
|
208
|
+
current.state === 'half_open' ||
|
|
209
|
+
current.attempts >= input.policy.maxAttempts ||
|
|
210
|
+
consecutiveFailures >= input.policy.maxConsecutiveFailures ||
|
|
211
|
+
repeatedErrorCount >= input.policy.maxRepeatedError;
|
|
212
|
+
await repository.update({
|
|
213
|
+
filterByTk: current.id,
|
|
214
|
+
values: {
|
|
215
|
+
state: shouldOpen ? 'open' : 'closed',
|
|
216
|
+
consecutiveFailures,
|
|
217
|
+
errorSignature: signature,
|
|
218
|
+
repeatedErrorCount,
|
|
219
|
+
lastRunId: input.lastRunId || null,
|
|
220
|
+
openedAt: shouldOpen ? now : null,
|
|
221
|
+
retryAt: shouldOpen ? new Date(now.getTime() + input.policy.cooldownMs) : null,
|
|
222
|
+
},
|
|
223
|
+
transaction,
|
|
224
|
+
});
|
|
225
|
+
return { open: shouldOpen, signature, consecutiveFailures, repeatedErrorCount };
|
|
226
|
+
}),
|
|
227
|
+
);
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
private withScopeLock<T>(patternId: number, scopeKey: string, operation: () => Promise<T>) {
|
|
231
|
+
const normalizedScope = scopeKey.trim();
|
|
232
|
+
if (!normalizedScope) throw new Error('Circuit breaker scope key is required.');
|
|
233
|
+
return this.distributedLock.runExclusive(
|
|
234
|
+
`agent-loop:circuit:${patternId}:${createHash('sha256').update(normalizedScope).digest('hex')}`,
|
|
235
|
+
operation,
|
|
236
|
+
30_000,
|
|
237
|
+
);
|
|
238
|
+
}
|
|
239
|
+
}
|