plugin-agent-orchestrator 1.0.33 → 1.0.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/externalVersion.js +6 -6
- package/dist/server/collections/skill-executions.js +21 -0
- package/dist/server/plugin.js +153 -3
- package/dist/server/resources/agent-loop.js +1 -2
- package/dist/server/services/AgentLoopController.js +39 -1
- package/dist/server/services/AgentLoopService.js +3 -0
- package/dist/server/services/AgentRuntimeLifecycle.js +70 -0
- package/dist/server/services/FileManager.js +12 -0
- package/dist/server/services/NativeSubAgentObserver.js +20 -0
- package/dist/server/services/PlanningPromptService.js +95 -0
- package/dist/server/services/SandboxRunner.js +251 -56
- package/dist/server/services/TokenTracker.js +20 -0
- package/dist/server/services/WorkerEnvManager.js +76 -10
- package/dist/server/services/pythonGuardTemplate.js +105 -0
- package/dist/server/services/sandboxCapabilities.js +286 -0
- package/dist/server/skill-hub/plugin.js +123 -28
- package/dist/server/skill-hub/tasks/SkillExecutionTask.js +53 -11
- package/dist/server/utils/skill-settings.js +1 -1
- package/package.json +47 -47
- package/src/server/__tests__/agent-runtime-lifecycle.test.ts +46 -0
- package/src/server/__tests__/planning-prompt-service.test.ts +23 -0
- package/src/server/__tests__/skill-sandbox-worker-mode.test.ts +15 -0
- package/src/server/collections/skill-executions.ts +21 -0
- package/src/server/plugin.ts +178 -0
- package/src/server/resources/agent-loop.ts +1 -2
- package/src/server/services/AgentLoopController.ts +50 -1
- package/src/server/services/AgentLoopService.ts +4 -0
- package/src/server/services/AgentRuntimeLifecycle.ts +64 -0
- package/src/server/services/FileManager.ts +16 -1
- package/src/server/services/NativeSubAgentObserver.ts +21 -0
- package/src/server/services/PlanningPromptService.ts +72 -0
- package/src/server/services/SandboxRunner.ts +341 -74
- package/src/server/services/TokenTracker.ts +26 -0
- package/src/server/services/WorkerEnvManager.ts +95 -9
- package/src/server/services/pythonGuardTemplate.ts +87 -0
- package/src/server/services/sandboxCapabilities.ts +348 -0
- package/src/server/skill-hub/plugin.ts +1120 -1016
- package/src/server/skill-hub/tasks/SkillExecutionTask.ts +515 -470
- package/src/server/utils/skill-settings.ts +1 -0
package/package.json
CHANGED
|
@@ -1,47 +1,47 @@
|
|
|
1
|
-
{
|
|
2
|
-
"name": "plugin-agent-orchestrator",
|
|
3
|
-
"displayName": "Agent Orchestrator",
|
|
4
|
-
"displayName.zh-CN": "代理协调器",
|
|
5
|
-
"displayName.vi-VN": "Điều phối Agent",
|
|
6
|
-
"description": "Native AI Employee sub-agent monitoring, policy, and memory context layer for NocoBase plugin-ai.",
|
|
7
|
-
"version": "1.0.
|
|
8
|
-
"license": "Apache-2.0",
|
|
9
|
-
"main": "dist/server/index.js",
|
|
10
|
-
"keywords": [
|
|
11
|
-
"AI",
|
|
12
|
-
"Agent",
|
|
13
|
-
"Orchestrator",
|
|
14
|
-
"Multi-Agent",
|
|
15
|
-
"Swarm"
|
|
16
|
-
],
|
|
17
|
-
"files": [
|
|
18
|
-
"dist",
|
|
19
|
-
"src",
|
|
20
|
-
"client.js",
|
|
21
|
-
"server.js",
|
|
22
|
-
"client.d.ts",
|
|
23
|
-
"server.d.ts",
|
|
24
|
-
"client-v2.js",
|
|
25
|
-
"client-v2.d.ts"
|
|
26
|
-
],
|
|
27
|
-
"nocobase": {
|
|
28
|
-
"supportedVersions": [
|
|
29
|
-
"2.x"
|
|
30
|
-
],
|
|
31
|
-
"editionLevel": 0
|
|
32
|
-
},
|
|
33
|
-
"dependencies": {
|
|
34
|
-
"adm-zip": "^0.5.10",
|
|
35
|
-
"simple-git": "^3.22.0",
|
|
36
|
-
"zod": "^3.23.0"
|
|
37
|
-
},
|
|
38
|
-
"peerDependencies": {
|
|
39
|
-
"@nocobase/client": "2.x",
|
|
40
|
-
"@nocobase/server": "2.x",
|
|
41
|
-
"@nocobase/database": "2.x",
|
|
42
|
-
"@nocobase/ai": "2.x",
|
|
43
|
-
"@nocobase/plugin-ai": "2.x",
|
|
44
|
-
"@nocobase/client-v2": "2.x",
|
|
45
|
-
"@nocobase/flow-engine": "2.x"
|
|
46
|
-
}
|
|
47
|
-
}
|
|
1
|
+
{
|
|
2
|
+
"name": "plugin-agent-orchestrator",
|
|
3
|
+
"displayName": "Agent Orchestrator",
|
|
4
|
+
"displayName.zh-CN": "代理协调器",
|
|
5
|
+
"displayName.vi-VN": "Điều phối Agent",
|
|
6
|
+
"description": "Native AI Employee sub-agent monitoring, policy, and memory context layer for NocoBase plugin-ai.",
|
|
7
|
+
"version": "1.0.36",
|
|
8
|
+
"license": "Apache-2.0",
|
|
9
|
+
"main": "dist/server/index.js",
|
|
10
|
+
"keywords": [
|
|
11
|
+
"AI",
|
|
12
|
+
"Agent",
|
|
13
|
+
"Orchestrator",
|
|
14
|
+
"Multi-Agent",
|
|
15
|
+
"Swarm"
|
|
16
|
+
],
|
|
17
|
+
"files": [
|
|
18
|
+
"dist",
|
|
19
|
+
"src",
|
|
20
|
+
"client.js",
|
|
21
|
+
"server.js",
|
|
22
|
+
"client.d.ts",
|
|
23
|
+
"server.d.ts",
|
|
24
|
+
"client-v2.js",
|
|
25
|
+
"client-v2.d.ts"
|
|
26
|
+
],
|
|
27
|
+
"nocobase": {
|
|
28
|
+
"supportedVersions": [
|
|
29
|
+
"2.x"
|
|
30
|
+
],
|
|
31
|
+
"editionLevel": 0
|
|
32
|
+
},
|
|
33
|
+
"dependencies": {
|
|
34
|
+
"adm-zip": "^0.5.10",
|
|
35
|
+
"simple-git": "^3.22.0",
|
|
36
|
+
"zod": "^3.23.0"
|
|
37
|
+
},
|
|
38
|
+
"peerDependencies": {
|
|
39
|
+
"@nocobase/client": "2.x",
|
|
40
|
+
"@nocobase/server": "2.x",
|
|
41
|
+
"@nocobase/database": "2.x",
|
|
42
|
+
"@nocobase/ai": "2.x",
|
|
43
|
+
"@nocobase/plugin-ai": "2.x",
|
|
44
|
+
"@nocobase/client-v2": "2.x",
|
|
45
|
+
"@nocobase/flow-engine": "2.x"
|
|
46
|
+
}
|
|
47
|
+
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { AgentRuntimeLifecycle } from '../services/AgentRuntimeLifecycle';
|
|
2
|
+
|
|
3
|
+
describe('AgentRuntimeLifecycle', () => {
|
|
4
|
+
it('runs registered hooks in registration order and supports cleanup', async () => {
|
|
5
|
+
const lifecycle = new AgentRuntimeLifecycle();
|
|
6
|
+
const calls: string[] = [];
|
|
7
|
+
const cleanup = lifecycle.registerBeforeRunHook('first', () => calls.push('first'));
|
|
8
|
+
lifecycle.registerBeforeRunHook('second', () => calls.push('second'));
|
|
9
|
+
const context = {
|
|
10
|
+
ctx: {},
|
|
11
|
+
source: 'api' as const,
|
|
12
|
+
employee: {},
|
|
13
|
+
sessionId: 'session-1',
|
|
14
|
+
messages: [],
|
|
15
|
+
metadata: {},
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
await lifecycle.runBeforeHooks(context);
|
|
19
|
+
cleanup();
|
|
20
|
+
await lifecycle.runBeforeHooks(context);
|
|
21
|
+
|
|
22
|
+
expect(calls).toEqual(['first', 'second', 'second']);
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
it('isolates a failed hook and continues with later hooks', async () => {
|
|
26
|
+
const failures: string[] = [];
|
|
27
|
+
const calls: string[] = [];
|
|
28
|
+
const lifecycle = new AgentRuntimeLifecycle((hookType, name) => failures.push(`${hookType}:${name}`));
|
|
29
|
+
lifecycle.registerBeforeRunHook('broken', () => {
|
|
30
|
+
throw new Error('boom');
|
|
31
|
+
});
|
|
32
|
+
lifecycle.registerBeforeRunHook('healthy', () => calls.push('healthy'));
|
|
33
|
+
|
|
34
|
+
await lifecycle.runBeforeHooks({
|
|
35
|
+
ctx: {},
|
|
36
|
+
source: 'api',
|
|
37
|
+
employee: {},
|
|
38
|
+
sessionId: 'session-1',
|
|
39
|
+
messages: [],
|
|
40
|
+
metadata: {},
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
expect(failures).toEqual(['beforeRun:broken']);
|
|
44
|
+
expect(calls).toEqual(['healthy']);
|
|
45
|
+
});
|
|
46
|
+
});
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { PlanningPromptService } from '../services/PlanningPromptService';
|
|
2
|
+
|
|
3
|
+
describe('PlanningPromptService', () => {
|
|
4
|
+
it('adds planning instructions as a system message without modifying the user request', () => {
|
|
5
|
+
const service = new PlanningPromptService();
|
|
6
|
+
const messages = [{ role: 'user', content: { type: 'text', content: 'Create a plan for the rollout' } }];
|
|
7
|
+
|
|
8
|
+
expect(service.applyPlanningContext(messages)).toBe(true);
|
|
9
|
+
expect(messages[0]).toMatchObject({ role: 'system' });
|
|
10
|
+
expect(messages[1]).toEqual({
|
|
11
|
+
role: 'user',
|
|
12
|
+
content: { type: 'text', content: 'Create a plan for the rollout' },
|
|
13
|
+
});
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
it('does not add planning context for a simple request', () => {
|
|
17
|
+
const service = new PlanningPromptService();
|
|
18
|
+
const messages = [{ role: 'user', content: 'Show the current status' }];
|
|
19
|
+
|
|
20
|
+
expect(service.applyPlanningContext(messages)).toBe(false);
|
|
21
|
+
expect(messages).toHaveLength(1);
|
|
22
|
+
});
|
|
23
|
+
});
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { shouldRunSkillSandbox } from '../skill-hub/plugin';
|
|
2
|
+
|
|
3
|
+
describe('Skill Hub sandbox worker mode', () => {
|
|
4
|
+
it.each([
|
|
5
|
+
[{}, true],
|
|
6
|
+
[{ SKILL_HUB_SANDBOX: 'false' }, false],
|
|
7
|
+
[{ WORKER_MODE: 'main' }, false],
|
|
8
|
+
[{ WORKER_MODE: 'workflow:process' }, false],
|
|
9
|
+
[{ WORKER_MODE: 'skill-hub:sandbox' }, true],
|
|
10
|
+
[{ WORKER_MODE: 'skill-hub.task' }, true],
|
|
11
|
+
[{ WORKER_MODE: '*' }, true],
|
|
12
|
+
])('evaluates %o as %s', (env, expected) => {
|
|
13
|
+
expect(shouldRunSkillSandbox(env)).toBe(expected);
|
|
14
|
+
});
|
|
15
|
+
});
|
|
@@ -50,6 +50,27 @@ export default {
|
|
|
50
50
|
name: 'durationMs',
|
|
51
51
|
type: 'integer',
|
|
52
52
|
},
|
|
53
|
+
{
|
|
54
|
+
name: 'startedAt',
|
|
55
|
+
type: 'date',
|
|
56
|
+
allowNull: true,
|
|
57
|
+
},
|
|
58
|
+
{
|
|
59
|
+
name: 'heartbeatAt',
|
|
60
|
+
type: 'date',
|
|
61
|
+
allowNull: true,
|
|
62
|
+
},
|
|
63
|
+
{
|
|
64
|
+
name: 'workerId',
|
|
65
|
+
type: 'string',
|
|
66
|
+
length: 200,
|
|
67
|
+
allowNull: true,
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
name: 'retryCount',
|
|
71
|
+
type: 'integer',
|
|
72
|
+
defaultValue: 0,
|
|
73
|
+
},
|
|
53
74
|
{
|
|
54
75
|
name: 'sessionId',
|
|
55
76
|
type: 'string',
|
package/src/server/plugin.ts
CHANGED
|
@@ -3,10 +3,15 @@ import path from 'path';
|
|
|
3
3
|
import { createExternalRagSearchTool } from './tools/external-rag-search';
|
|
4
4
|
import { registerTracingResource } from './resources/tracing';
|
|
5
5
|
import { registerAgentMonitorResource } from './resources/agent-monitor';
|
|
6
|
+
import { registerAgentLoopResource } from './resources/agent-loop';
|
|
6
7
|
import SkillHubSubFeature from './skill-hub/plugin';
|
|
7
8
|
import { NativeSubAgentObserver } from './services/NativeSubAgentObserver';
|
|
9
|
+
import { AgentLoopService } from './services/AgentLoopService';
|
|
10
|
+
import { createAgentLoopTools } from './tools/agent-loop';
|
|
11
|
+
import { PlanningPromptService } from './services/PlanningPromptService';
|
|
8
12
|
import { asObject, isAdminUser, currentUserId } from './utils/ctx-utils';
|
|
9
13
|
import { getAIToolsManager } from './utils/ai-manager';
|
|
14
|
+
import { AgentRuntimeLifecycle } from './services/AgentRuntimeLifecycle';
|
|
10
15
|
|
|
11
16
|
function normalizeOptionalString(value: unknown) {
|
|
12
17
|
return typeof value === 'string' ? value.trim() : '';
|
|
@@ -123,6 +128,7 @@ async function resolveTracingRetentionDays(plugin: { db: any; app: any }) {
|
|
|
123
128
|
export class PluginAgentOrchestratorServer extends Plugin {
|
|
124
129
|
skillHub: SkillHubSubFeature;
|
|
125
130
|
nativeObserver: NativeSubAgentObserver;
|
|
131
|
+
agentRuntimeLifecycle: AgentRuntimeLifecycle;
|
|
126
132
|
private readonly installNativeObserver = () => {
|
|
127
133
|
this.nativeObserver?.install();
|
|
128
134
|
};
|
|
@@ -130,6 +136,13 @@ export class PluginAgentOrchestratorServer extends Plugin {
|
|
|
130
136
|
async afterAdd() {
|
|
131
137
|
this.skillHub = new SkillHubSubFeature(this);
|
|
132
138
|
this.nativeObserver = new NativeSubAgentObserver(this);
|
|
139
|
+
this.agentRuntimeLifecycle = new AgentRuntimeLifecycle((hookType, name, error) => {
|
|
140
|
+
this.app.logger.warn(`[AgentOrchestrator] ${hookType} hook "${name}" failed; continuing agent execution.`, {
|
|
141
|
+
error,
|
|
142
|
+
});
|
|
143
|
+
});
|
|
144
|
+
(this.app as typeof this.app & { agentRuntimeLifecycle: AgentRuntimeLifecycle }).agentRuntimeLifecycle =
|
|
145
|
+
this.agentRuntimeLifecycle;
|
|
133
146
|
}
|
|
134
147
|
|
|
135
148
|
async beforeLoad() {
|
|
@@ -182,6 +195,7 @@ export class PluginAgentOrchestratorServer extends Plugin {
|
|
|
182
195
|
this.app.acl.allow('skillHub', 'download', 'loggedIn');
|
|
183
196
|
this.app.acl.allow('skillHub', 'listTemplates', 'loggedIn');
|
|
184
197
|
this.app.acl.allow('agentMonitor', ['list', 'get'], 'loggedIn');
|
|
198
|
+
this.app.acl.allow('agentLoops', ['list', 'get'], 'loggedIn');
|
|
185
199
|
|
|
186
200
|
// Data scoping for skillExecutions: a logged-in non-admin user may only
|
|
187
201
|
// read their own executions. Rows hold inputArgs / stdout / output files,
|
|
@@ -214,6 +228,58 @@ export class PluginAgentOrchestratorServer extends Plugin {
|
|
|
214
228
|
{ tag: 'orchestrator-agent-memory-context-policy', after: 'acl' },
|
|
215
229
|
);
|
|
216
230
|
|
|
231
|
+
// --- Planning Prompt Injection ---
|
|
232
|
+
// When a user message contains plan-related keywords, prepend planning
|
|
233
|
+
// instructions so the AI agent creates a plan via agent_loop_start before
|
|
234
|
+
// executing.
|
|
235
|
+
const planningPrompt = new PlanningPromptService();
|
|
236
|
+
this.agentRuntimeLifecycle.registerBeforeRunHook('orchestrator-planning', (context) => {
|
|
237
|
+
if (planningPrompt.applyPlanningContext(context.messages)) {
|
|
238
|
+
context.metadata.planningPromptInjected = true;
|
|
239
|
+
}
|
|
240
|
+
});
|
|
241
|
+
this.app.resourceManager.use(
|
|
242
|
+
async (ctx, next) => {
|
|
243
|
+
const { resourceName, actionName } = ctx.action || {};
|
|
244
|
+
if (resourceName === 'aiConversations' && actionName === 'sendMessages') {
|
|
245
|
+
const values = ctx.action.params.values || {};
|
|
246
|
+
const messages = values.messages;
|
|
247
|
+
if (Array.isArray(messages)) {
|
|
248
|
+
const runtimeContext = {
|
|
249
|
+
ctx,
|
|
250
|
+
source: 'chat-ui' as const,
|
|
251
|
+
employee: undefined,
|
|
252
|
+
sessionId: String(ctx.state?.sessionId || ''),
|
|
253
|
+
userId: currentUserId(ctx),
|
|
254
|
+
messages,
|
|
255
|
+
metadata: {},
|
|
256
|
+
};
|
|
257
|
+
await this.agentRuntimeLifecycle.runBeforeHooks(runtimeContext);
|
|
258
|
+
ctx.state.agentRuntimeContext = runtimeContext;
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
const runtimeContext = ctx.state?.agentRuntimeContext;
|
|
262
|
+
try {
|
|
263
|
+
await next();
|
|
264
|
+
if (runtimeContext) {
|
|
265
|
+
await this.agentRuntimeLifecycle.runAfterHooks(runtimeContext, {
|
|
266
|
+
succeeded: ctx.status < 400,
|
|
267
|
+
value: ctx.body,
|
|
268
|
+
});
|
|
269
|
+
}
|
|
270
|
+
} catch (error) {
|
|
271
|
+
if (runtimeContext) {
|
|
272
|
+
await this.agentRuntimeLifecycle.runAfterHooks(runtimeContext, {
|
|
273
|
+
succeeded: false,
|
|
274
|
+
error: error instanceof Error ? error : new Error(String(error)),
|
|
275
|
+
});
|
|
276
|
+
}
|
|
277
|
+
throw error;
|
|
278
|
+
}
|
|
279
|
+
},
|
|
280
|
+
{ tag: 'orchestrator-planning-prompt', after: 'acl' },
|
|
281
|
+
);
|
|
282
|
+
|
|
217
283
|
// --- Register External Tool Only ---
|
|
218
284
|
// Native @nocobase/plugin-ai owns sub-agent dispatch. The orchestrator no
|
|
219
285
|
// longer registers delegate_* / dispatch_subagents_* tools or controller
|
|
@@ -221,11 +287,21 @@ export class PluginAgentOrchestratorServer extends Plugin {
|
|
|
221
287
|
const toolsManager = getAIToolsManager(this.app);
|
|
222
288
|
toolsManager.registerTools(createExternalRagSearchTool(this));
|
|
223
289
|
|
|
290
|
+
// --- Register Agent Loop Tools & Resource ---
|
|
291
|
+
const agentLoopService = new AgentLoopService(this);
|
|
292
|
+
toolsManager.registerTools(createAgentLoopTools(this, agentLoopService));
|
|
293
|
+
registerAgentLoopResource(this, agentLoopService);
|
|
294
|
+
|
|
224
295
|
// --- Native plugin-ai Monitor Resource ---
|
|
225
296
|
registerAgentMonitorResource(this);
|
|
226
297
|
this.installNativeObserver();
|
|
227
298
|
this.app.on?.('afterStart', this.installNativeObserver);
|
|
228
299
|
|
|
300
|
+
// --- Crash Recovery ---
|
|
301
|
+
// On startup, scan for steps/runs left in "running" state from a previous
|
|
302
|
+
// crash and reset them so they can be retried.
|
|
303
|
+
this.app.on?.('afterStart', this.recoverAfterCrash);
|
|
304
|
+
|
|
229
305
|
// --- Register Tracing Resource (Phase 5) ---
|
|
230
306
|
// Custom read-only resource for the Swarm Tracing admin page.
|
|
231
307
|
registerTracingResource(this);
|
|
@@ -277,9 +353,111 @@ export class PluginAgentOrchestratorServer extends Plugin {
|
|
|
277
353
|
async beforeStop() {
|
|
278
354
|
this.app.off?.('afterStart', this.installNativeObserver);
|
|
279
355
|
this.app.removeListener?.('afterStart', this.installNativeObserver);
|
|
356
|
+
this.app.off?.('afterStart', this.recoverAfterCrash);
|
|
357
|
+
this.app.removeListener?.('afterStart', this.recoverAfterCrash);
|
|
280
358
|
this.nativeObserver?.uninstall();
|
|
281
359
|
await this.skillHub.beforeStop();
|
|
282
360
|
}
|
|
361
|
+
|
|
362
|
+
// --- Crash Recovery ---
|
|
363
|
+
// Scans for agentLoopSteps stuck in "running" status from a previous crash
|
|
364
|
+
// and resets them to "pending". Also cleans up orphaned agentLoopRuns where
|
|
365
|
+
// all steps have reached terminal states but the run itself wasn't finalized.
|
|
366
|
+
private readonly recoverAfterCrash = async () => {
|
|
367
|
+
const timeoutMs = Number(process.env.ORCHESTRATOR_CRASH_RECOVERY_TIMEOUT_MS || 600000); // 10 min default
|
|
368
|
+
const cutoff = new Date(Date.now() - timeoutMs);
|
|
369
|
+
|
|
370
|
+
try {
|
|
371
|
+
const stepsRepo = this.db.getRepository('agentLoopSteps');
|
|
372
|
+
const runsRepo = this.db.getRepository('agentLoopRuns');
|
|
373
|
+
|
|
374
|
+
// 1. Reset stale running steps back to pending
|
|
375
|
+
if (stepsRepo) {
|
|
376
|
+
const staleSteps = await stepsRepo.find({
|
|
377
|
+
filter: {
|
|
378
|
+
status: 'running',
|
|
379
|
+
startedAt: { $lt: cutoff.toISOString() },
|
|
380
|
+
},
|
|
381
|
+
});
|
|
382
|
+
|
|
383
|
+
for (const step of staleSteps) {
|
|
384
|
+
const stepId = readModelValue(step, 'id');
|
|
385
|
+
const runId = readModelValue(step, 'runId');
|
|
386
|
+
await stepsRepo.update({
|
|
387
|
+
filterByTk: stepId,
|
|
388
|
+
values: {
|
|
389
|
+
status: 'pending',
|
|
390
|
+
startedAt: null,
|
|
391
|
+
error: `Recovered after crash at ${new Date().toISOString()}`,
|
|
392
|
+
},
|
|
393
|
+
});
|
|
394
|
+
this.app.logger?.warn?.(
|
|
395
|
+
`[AgentOrchestrator] Crash recovery: reset step ${stepId} (run ${runId}) from running to pending`,
|
|
396
|
+
);
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
if (staleSteps.length > 0) {
|
|
400
|
+
this.app.logger?.info?.(
|
|
401
|
+
`[AgentOrchestrator] Crash recovery: reset ${staleSteps.length} stale running step(s)`,
|
|
402
|
+
);
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
// 2. Finalize orphaned runs — runs marked "running" where all steps are terminal
|
|
407
|
+
if (runsRepo) {
|
|
408
|
+
const runningRuns = await runsRepo.find({
|
|
409
|
+
filter: { status: 'running' },
|
|
410
|
+
});
|
|
411
|
+
|
|
412
|
+
for (const run of runningRuns) {
|
|
413
|
+
const runId = readModelValue(run, 'id');
|
|
414
|
+
const steps = await stepsRepo?.find({
|
|
415
|
+
filter: { runId },
|
|
416
|
+
});
|
|
417
|
+
|
|
418
|
+
if (!steps || steps.length === 0) {
|
|
419
|
+
// No steps at all — orphaned, mark as failed
|
|
420
|
+
await runsRepo.update({
|
|
421
|
+
filterByTk: runId,
|
|
422
|
+
values: {
|
|
423
|
+
status: 'failed',
|
|
424
|
+
endedAt: new Date(),
|
|
425
|
+
summary: 'Run was orphaned after server crash (no steps found).',
|
|
426
|
+
},
|
|
427
|
+
});
|
|
428
|
+
this.app.logger?.warn?.(
|
|
429
|
+
`[AgentOrchestrator] Crash recovery: marked orphaned run ${runId} as failed (no steps)`,
|
|
430
|
+
);
|
|
431
|
+
continue;
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
const allTerminal = steps.every((s: any) => {
|
|
435
|
+
const st = readModelValue(s, 'status');
|
|
436
|
+
return st === 'succeeded' || st === 'failed' || st === 'skipped';
|
|
437
|
+
});
|
|
438
|
+
|
|
439
|
+
if (!allTerminal) continue;
|
|
440
|
+
|
|
441
|
+
const anyFailed = steps.some((s: any) => readModelValue(s, 'status') === 'failed');
|
|
442
|
+
const finalStatus = anyFailed ? 'failed' : 'succeeded';
|
|
443
|
+
|
|
444
|
+
await runsRepo.update({
|
|
445
|
+
filterByTk: runId,
|
|
446
|
+
values: {
|
|
447
|
+
status: finalStatus,
|
|
448
|
+
endedAt: new Date(),
|
|
449
|
+
summary: `Run finalized as ${finalStatus} after server crash recovery.`,
|
|
450
|
+
},
|
|
451
|
+
});
|
|
452
|
+
this.app.logger?.warn?.(
|
|
453
|
+
`[AgentOrchestrator] Crash recovery: finalized orphaned run ${runId} as ${finalStatus}`,
|
|
454
|
+
);
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
} catch (error) {
|
|
458
|
+
this.app.logger?.error?.('[AgentOrchestrator] Crash recovery scan failed', error);
|
|
459
|
+
}
|
|
460
|
+
};
|
|
283
461
|
}
|
|
284
462
|
|
|
285
463
|
export default PluginAgentOrchestratorServer;
|
|
@@ -97,9 +97,8 @@ export function registerAgentLoopResource(plugin: Plugin, service: AgentLoopServ
|
|
|
97
97
|
return;
|
|
98
98
|
}
|
|
99
99
|
ctx.body = {
|
|
100
|
-
data: await service.
|
|
100
|
+
data: await service.approvePlan(runId, {
|
|
101
101
|
userId: currentUserId(ctx),
|
|
102
|
-
ctx,
|
|
103
102
|
reason: body.reason,
|
|
104
103
|
}),
|
|
105
104
|
};
|
|
@@ -272,6 +272,45 @@ export class AgentLoopController {
|
|
|
272
272
|
return this.getRunDetail(run.id);
|
|
273
273
|
}
|
|
274
274
|
|
|
275
|
+
async approvePlan(
|
|
276
|
+
runId: string | number,
|
|
277
|
+
options: { userId?: string | number; reason?: string } = {},
|
|
278
|
+
) {
|
|
279
|
+
const run = await this.repository.requireRun(runId);
|
|
280
|
+
if (TERMINAL_RUN_STATUSES.has(run.status)) {
|
|
281
|
+
throw new Error(`Agent loop run ${run.id} is already ${run.status}.`);
|
|
282
|
+
}
|
|
283
|
+
if (!['waiting_plan_approval', 'approved', 'needs_replan'].includes(run.status)) {
|
|
284
|
+
throw new Error(`Run ${run.id} is not waiting for plan approval.`);
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
await this.repository.updateRun(run.id, {
|
|
288
|
+
status: 'approved',
|
|
289
|
+
approvalStatus: 'approved',
|
|
290
|
+
approvedById: options.userId,
|
|
291
|
+
approvedAt: now(),
|
|
292
|
+
rejectionReason: '',
|
|
293
|
+
changeRequest: '',
|
|
294
|
+
updatedAt: now(),
|
|
295
|
+
});
|
|
296
|
+
|
|
297
|
+
await this.repository.createEvent({
|
|
298
|
+
runId: run.id,
|
|
299
|
+
type: 'plan_approved',
|
|
300
|
+
title: 'Plan approved',
|
|
301
|
+
content: options.reason || '',
|
|
302
|
+
status: 'approved',
|
|
303
|
+
userId: options.userId,
|
|
304
|
+
payload: { planVersion: run.planVersion || 1 },
|
|
305
|
+
});
|
|
306
|
+
|
|
307
|
+
return this.getRunDetail(run.id);
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
/**
|
|
311
|
+
* @deprecated Use approvePlan() instead. Auto-execution via AgentHarness is retired;
|
|
312
|
+
* step execution is now driven by the AI via agent_loop_update_step tools.
|
|
313
|
+
*/
|
|
275
314
|
async approvePlanAndExecute(
|
|
276
315
|
runId: string | number,
|
|
277
316
|
options: { userId?: string | number; ctx?: any; reason?: string } = {},
|
|
@@ -602,6 +641,11 @@ export class AgentLoopController {
|
|
|
602
641
|
userId: options.userId,
|
|
603
642
|
});
|
|
604
643
|
|
|
644
|
+
// Circuit breaker: record success for sub_agent steps
|
|
645
|
+
if (step.type === 'sub_agent' && step.target && run.leaderUsername) {
|
|
646
|
+
getCircuitBreaker().recordSuccess(subAgentCircuitKey(run.leaderUsername, step.target));
|
|
647
|
+
}
|
|
648
|
+
|
|
605
649
|
return this.getRunSnapshot(run.id);
|
|
606
650
|
}
|
|
607
651
|
|
|
@@ -644,6 +688,11 @@ export class AgentLoopController {
|
|
|
644
688
|
},
|
|
645
689
|
});
|
|
646
690
|
|
|
691
|
+
// Circuit breaker: record failure for sub_agent steps
|
|
692
|
+
if (step.type === 'sub_agent' && step.target && run.leaderUsername) {
|
|
693
|
+
getCircuitBreaker().recordFailure(subAgentCircuitKey(run.leaderUsername, step.target));
|
|
694
|
+
}
|
|
695
|
+
|
|
647
696
|
return this.getRunSnapshot(run.id);
|
|
648
697
|
}
|
|
649
698
|
|
|
@@ -786,7 +835,7 @@ export class AgentLoopController {
|
|
|
786
835
|
});
|
|
787
836
|
|
|
788
837
|
if (options.approved) {
|
|
789
|
-
return this.
|
|
838
|
+
return this.getRunSnapshot(run.id);
|
|
790
839
|
}
|
|
791
840
|
return this.getRunSnapshot(run.id);
|
|
792
841
|
}
|
|
@@ -95,6 +95,10 @@ export class AgentLoopService {
|
|
|
95
95
|
return this.controller.revisePlanGoal(runId, plan, options);
|
|
96
96
|
}
|
|
97
97
|
|
|
98
|
+
async approvePlan(runId: any, options: any = {}) {
|
|
99
|
+
return this.controller.approvePlan(runId, options);
|
|
100
|
+
}
|
|
101
|
+
|
|
98
102
|
async approvePlanAndExecute(runId: any, options: any = {}) {
|
|
99
103
|
return this.controller.approvePlanAndExecute(runId, options);
|
|
100
104
|
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
export type AgentRuntimeSource = 'chat-ui' | 'api' | 'workflow' | 'sub-agent';
|
|
2
|
+
|
|
3
|
+
export interface AgentRuntimeMessage {
|
|
4
|
+
role: string;
|
|
5
|
+
content: unknown;
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
export interface AgentRuntimeContext {
|
|
9
|
+
ctx: unknown;
|
|
10
|
+
source: AgentRuntimeSource;
|
|
11
|
+
employee: unknown;
|
|
12
|
+
sessionId: string;
|
|
13
|
+
userId?: number;
|
|
14
|
+
messages: AgentRuntimeMessage[];
|
|
15
|
+
metadata: Record<string, unknown>;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export interface AgentRuntimeResult {
|
|
19
|
+
succeeded: boolean;
|
|
20
|
+
value?: unknown;
|
|
21
|
+
error?: Error;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
type BeforeRunHook = (context: AgentRuntimeContext) => Promise<void> | void;
|
|
25
|
+
type AfterRunHook = (context: AgentRuntimeContext, result: AgentRuntimeResult) => Promise<void> | void;
|
|
26
|
+
|
|
27
|
+
export class AgentRuntimeLifecycle {
|
|
28
|
+
private readonly beforeRunHooks = new Map<string, BeforeRunHook>();
|
|
29
|
+
private readonly afterRunHooks = new Map<string, AfterRunHook>();
|
|
30
|
+
|
|
31
|
+
constructor(
|
|
32
|
+
private readonly onHookError?: (hookType: 'beforeRun' | 'afterRun', name: string, error: Error) => void,
|
|
33
|
+
) {}
|
|
34
|
+
|
|
35
|
+
registerBeforeRunHook(name: string, hook: BeforeRunHook): () => void {
|
|
36
|
+
this.beforeRunHooks.set(name, hook);
|
|
37
|
+
return () => this.beforeRunHooks.delete(name);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
registerAfterRunHook(name: string, hook: AfterRunHook): () => void {
|
|
41
|
+
this.afterRunHooks.set(name, hook);
|
|
42
|
+
return () => this.afterRunHooks.delete(name);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
async runBeforeHooks(context: AgentRuntimeContext): Promise<void> {
|
|
46
|
+
for (const [name, hook] of this.beforeRunHooks) {
|
|
47
|
+
try {
|
|
48
|
+
await hook(context);
|
|
49
|
+
} catch (error) {
|
|
50
|
+
this.onHookError?.('beforeRun', name, error instanceof Error ? error : new Error(String(error)));
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
async runAfterHooks(context: AgentRuntimeContext, result: AgentRuntimeResult): Promise<void> {
|
|
56
|
+
for (const [name, hook] of this.afterRunHooks) {
|
|
57
|
+
try {
|
|
58
|
+
await hook(context, result);
|
|
59
|
+
} catch (error) {
|
|
60
|
+
this.onHookError?.('afterRun', name, error instanceof Error ? error : new Error(String(error)));
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { mkdirSync, readdirSync, statSync, existsSync, rmSync } from 'fs';
|
|
1
|
+
import { mkdirSync, readdirSync, statSync, existsSync, rmSync, chownSync } from 'fs';
|
|
2
2
|
import { resolve, join } from 'path';
|
|
3
3
|
|
|
4
4
|
export interface OutputFileInfo {
|
|
@@ -52,6 +52,21 @@ export class FileManager {
|
|
|
52
52
|
return resolve(this.executionsDir, execId, 'output');
|
|
53
53
|
}
|
|
54
54
|
|
|
55
|
+
/**
|
|
56
|
+
* Hand the execution directory to the dropped-privilege sandbox user so the
|
|
57
|
+
* child can read its script and write output. Requires root (chown), which
|
|
58
|
+
* matches the only case privilege drop is active.
|
|
59
|
+
*/
|
|
60
|
+
chownExecDir(execId: string, uid: number, gid: number): void {
|
|
61
|
+
const dir = resolve(this.executionsDir, execId);
|
|
62
|
+
chownSync(dir, uid, gid);
|
|
63
|
+
// Direct children too (script + output dir): under a restrictive umask
|
|
64
|
+
// (e.g. 077) the root-written script would stay unreadable for the sandbox uid.
|
|
65
|
+
for (const entry of readdirSync(dir)) {
|
|
66
|
+
chownSync(resolve(dir, entry), uid, gid);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
55
70
|
/**
|
|
56
71
|
* Returns absolute path to an output file, or null if invalid/not found.
|
|
57
72
|
* Includes path traversal protection.
|