plugin-agent-orchestrator 1.1.3 → 1.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +204 -114
- package/dist/client/index.js +1 -1
- package/dist/client-v2/117.7565cff57c39318a.js +10 -0
- package/dist/client-v2/214.106635011859dede.js +10 -0
- package/dist/client-v2/253.59577b40d9875be0.js +10 -0
- package/dist/client-v2/264.7ed8bee71fa5935c.js +10 -0
- package/dist/client-v2/{41.ba6e080cc0488143.js → 41.3deb9e57a0f64fc4.js} +1 -1
- package/dist/client-v2/418.41ed200e7d6da332.js +10 -0
- package/dist/client-v2/53.349fff8db80b19d4.js +10 -0
- package/dist/client-v2/619.0a2144aaffefe018.js +10 -0
- package/dist/client-v2/{70.bda9518881c05360.js → 70.ffff2997c10fd4b0.js} +1 -1
- package/dist/client-v2/{794.9bbb8d3a61b2bdf2.js → 794.a67a2614405ba627.js} +1 -1
- package/dist/client-v2/981.e4a1f41cfc94a856.js +10 -0
- package/dist/client-v2/{993.82778b6fd6db3cb1.js → 993.b0688fdc5ccb000a.js} +1 -1
- package/dist/client-v2/index.js +1 -1
- package/dist/externalVersion.js +8 -7
- package/dist/locale/en-US.json +199 -123
- package/dist/locale/vi-VN.json +199 -123
- package/dist/locale/zh-CN.json +199 -123
- package/dist/server/collections/agent-execution-spans.js +55 -0
- package/dist/server/collections/agent-harness-profile-versions.js +56 -0
- package/dist/server/collections/agent-harness-profiles.js +12 -0
- package/dist/server/collections/agent-loop-action-approvals.js +69 -0
- package/dist/server/collections/agent-loop-artifacts.js +55 -0
- package/dist/server/collections/agent-loop-circuit-states.js +53 -0
- package/dist/server/collections/agent-loop-control-settings.js +51 -0
- package/dist/server/collections/agent-loop-events.js +27 -0
- package/dist/server/collections/agent-loop-path-locks.js +57 -0
- package/dist/server/collections/agent-loop-patterns.js +72 -0
- package/dist/server/collections/agent-loop-runs.js +170 -3
- package/dist/server/collections/agent-loop-steps.js +76 -0
- package/dist/server/collections/agent-loop-usage-buckets.js +52 -0
- package/dist/server/collections/agent-loop-worktrees.js +55 -0
- package/dist/server/collections/skill-executions.js +8 -0
- package/dist/server/migrations/{20260427000000-change-packages-to-text.js → 20260427000002-change-packages-to-text.js} +2 -0
- package/dist/server/migrations/20260429000000-add-llm-fields.js +3 -2
- package/dist/server/migrations/{20260429000000-fix-inputargs-json-to-text.js → 20260429000001-fix-inputargs-json-to-text.js} +14 -6
- package/dist/server/migrations/20260503000000-add-orchestrator-trace-fields.js +2 -1
- package/dist/server/migrations/20260524000000-add-agent-loop-fields-to-skill-executions.js +2 -1
- package/dist/server/migrations/20260803010000-backfill-loop-control-plane-data.js +262 -0
- package/dist/server/plugin.js +54 -150
- package/dist/server/resources/agent-harness-profile.js +117 -0
- package/dist/server/resources/agent-knowledge-insights.js +1 -3
- package/dist/server/resources/agent-loop-approval.js +108 -0
- package/dist/server/resources/agent-loop-artifact.js +86 -0
- package/dist/server/resources/agent-loop-events-stream.js +130 -0
- package/dist/server/resources/agent-loop.js +183 -139
- package/dist/server/resources/resource-helpers.js +63 -0
- package/dist/server/resources/tracing.js +121 -107
- package/dist/server/services/AgentMemoryContextService.js +34 -0
- package/dist/server/services/DurableCircuitBreakerService.js +229 -0
- package/dist/server/services/HarnessCompiler.js +175 -0
- package/dist/server/services/HarnessProfileService.js +233 -0
- package/dist/server/services/HarnessSchema.js +194 -0
- package/dist/server/services/LoopBudgetService.js +269 -0
- package/dist/server/services/LoopControlService.js +139 -0
- package/dist/server/services/LoopPatternSchema.js +195 -0
- package/dist/server/services/LoopPatternService.js +132 -0
- package/dist/server/services/LoopRunRepository.js +285 -0
- package/dist/server/services/LoopRunStateMachine.js +478 -0
- package/dist/server/services/LoopSchedulerService.js +193 -0
- package/dist/server/services/LoopTriggerService.js +165 -0
- package/dist/server/services/LoopWorkerService.js +699 -0
- package/dist/server/services/NativeSubAgentObserver.js +165 -23
- package/dist/server/services/PathLockService.js +288 -0
- package/dist/server/services/PluginAiRuntimeAdapter.js +226 -0
- package/dist/server/services/ToolLoopDetectionService.js +210 -0
- package/dist/server/services/VerificationSchema.js +59 -0
- package/dist/server/services/VerificationService.js +235 -0
- package/dist/server/services/harness-runtime-policy.js +139 -0
- package/dist/server/skill-hub/actions/git-import.js +13 -7
- package/dist/server/skill-hub/plugin.js +36 -3
- package/dist/server/skill-hub/services/RegistrySkillSnapshotService.js +1 -5
- package/dist/server/skill-hub/tasks/SkillExecutionTask.js +29 -0
- package/dist/server/utils/ensure-indexes.js +111 -0
- package/dist/server/utils/loop-pattern-context.js +56 -0
- package/package.json +53 -53
- package/src/client/ApprovalsTab.tsx +2 -0
- package/src/client/OrchestratorSettings.tsx +55 -0
- package/src/client/skill-hub/components/GitSkillImport.tsx +3 -3
- package/src/client/skill-hub/components/SkillMetrics.tsx +15 -10
- package/src/client/skill-hub/index.tsx +0 -52
- package/src/client-v2/components/AgentRunsTab.tsx +100 -18
- package/src/client-v2/components/ApprovalsTab.tsx +337 -0
- package/src/client-v2/components/HarnessProfilesTab.tsx +385 -100
- package/src/client-v2/components/MemoryInspectorTab.tsx +141 -125
- package/src/client-v2/components/TracingTab.tsx +160 -77
- package/src/client-v2/hooks/useApiRequest.ts +28 -8
- package/src/client-v2/pages/ApprovalsPage.tsx +10 -0
- package/src/client-v2/plugin.tsx +6 -6
- package/src/client-v2/skill-hub/components/SkillMetrics.tsx +11 -6
- package/src/locale/en-US.json +199 -123
- package/src/locale/vi-VN.json +199 -123
- package/src/locale/zh-CN.json +199 -123
- package/src/server/__tests__/circuit-breaker.test.ts +248 -169
- package/src/server/__tests__/harness-compiler.test.ts +214 -0
- package/src/server/__tests__/harness-profile-service.test.ts +167 -0
- package/src/server/__tests__/harness-runtime-policy.test.ts +293 -0
- package/src/server/__tests__/harness-tool-plan.test.ts +172 -0
- package/src/server/__tests__/loop-approval-decision.test.ts +512 -0
- package/src/server/__tests__/loop-approval-expiry.test.ts +258 -0
- package/src/server/__tests__/loop-budget-service.test.ts +255 -0
- package/src/server/__tests__/loop-control-plane-backfill.test.ts +266 -0
- package/src/server/__tests__/loop-pattern-schema.test.ts +73 -0
- package/src/server/__tests__/loop-pattern-service.test.ts +148 -0
- package/src/server/__tests__/loop-run-state-machine.test.ts +101 -0
- package/src/server/__tests__/loop-trigger-service.test.ts +148 -0
- package/src/server/__tests__/loop-worker-claim.test.ts +341 -0
- package/src/server/__tests__/loop-worker-execution.test.ts +317 -0
- package/src/server/__tests__/native-sub-agent-observer.test.ts +288 -4
- package/src/server/__tests__/path-lock-service.test.ts +254 -0
- package/src/server/__tests__/skill-execution-task-policy.test.ts +140 -0
- package/src/server/__tests__/smoke.test.ts +162 -121
- package/src/server/__tests__/tool-loop-detection.test.ts +275 -0
- package/src/server/collections/agent-execution-spans.ts +55 -0
- package/src/server/collections/agent-harness-profile-versions.ts +26 -0
- package/src/server/collections/agent-harness-profiles.ts +71 -59
- package/src/server/collections/agent-loop-action-approvals.ts +39 -0
- package/src/server/collections/agent-loop-artifacts.ts +25 -0
- package/src/server/collections/agent-loop-circuit-states.ts +23 -0
- package/src/server/collections/agent-loop-control-settings.ts +21 -0
- package/src/server/collections/agent-loop-events.ts +98 -71
- package/src/server/collections/agent-loop-path-locks.ts +27 -0
- package/src/server/collections/agent-loop-patterns.ts +42 -0
- package/src/server/collections/agent-loop-runs.ts +170 -3
- package/src/server/collections/agent-loop-steps.ts +76 -0
- package/src/server/collections/agent-loop-usage-buckets.ts +22 -0
- package/src/server/collections/agent-loop-worktrees.ts +25 -0
- package/src/server/collections/skill-executions.ts +8 -0
- package/src/server/migrations/{20260427000000-change-packages-to-text.ts → 20260427000002-change-packages-to-text.ts} +52 -49
- package/src/server/migrations/20260429000000-add-llm-fields.ts +47 -46
- package/src/server/migrations/{20260429000000-fix-inputargs-json-to-text.ts → 20260429000001-fix-inputargs-json-to-text.ts} +45 -38
- package/src/server/migrations/20260503000000-add-orchestrator-trace-fields.ts +33 -32
- package/src/server/migrations/20260524000000-add-agent-loop-fields-to-skill-executions.ts +31 -30
- package/src/server/migrations/20260803010000-backfill-loop-control-plane-data.ts +293 -0
- package/src/server/plugin.ts +55 -167
- package/src/server/resources/agent-harness-profile.ts +94 -0
- package/src/server/resources/agent-knowledge-insights.ts +5 -3
- package/src/server/resources/agent-loop-approval.ts +84 -0
- package/src/server/resources/agent-loop-artifact.ts +56 -0
- package/src/server/resources/agent-loop-events-stream.ts +105 -0
- package/src/server/resources/agent-loop.ts +204 -143
- package/src/server/resources/agent-monitor.ts +482 -482
- package/src/server/resources/resource-helpers.ts +33 -0
- package/src/server/resources/tracing.ts +266 -258
- package/src/server/services/AgentMemoryContextService.ts +44 -0
- package/src/server/services/DurableCircuitBreakerService.ts +239 -0
- package/src/server/services/ExecutionSpanService.ts +2 -0
- package/src/server/services/HarnessCompiler.ts +170 -0
- package/src/server/services/HarnessProfileService.ts +239 -0
- package/src/server/services/HarnessSchema.ts +194 -0
- package/src/server/services/LoopBudgetService.ts +275 -0
- package/src/server/services/LoopControlService.ts +124 -0
- package/src/server/services/LoopPatternSchema.ts +185 -0
- package/src/server/services/LoopPatternService.ts +143 -0
- package/src/server/services/LoopRunRepository.ts +336 -0
- package/src/server/services/LoopRunStateMachine.ts +529 -0
- package/src/server/services/LoopSchedulerService.ts +199 -0
- package/src/server/services/LoopTriggerService.ts +157 -0
- package/src/server/services/LoopWorkerService.ts +822 -0
- package/src/server/services/NativeSubAgentObserver.ts +186 -21
- package/src/server/services/PathLockService.ts +321 -0
- package/src/server/services/PluginAiRuntimeAdapter.ts +332 -0
- package/src/server/services/ToolLoopDetectionService.ts +213 -0
- package/src/server/services/VerificationSchema.ts +31 -0
- package/src/server/services/VerificationService.ts +264 -0
- package/src/server/services/harness-runtime-policy.ts +141 -0
- package/src/server/skill-hub/actions/__tests__/git-import.test.ts +15 -1
- package/src/server/skill-hub/actions/git-import.ts +9 -3
- package/src/server/skill-hub/plugin.ts +41 -3
- package/src/server/skill-hub/services/RegistrySkillSnapshotService.ts +235 -239
- package/src/server/skill-hub/services/__tests__/RegistrySkillSnapshotService.test.ts +61 -63
- package/src/server/skill-hub/tasks/SkillExecutionTask.ts +43 -2
- package/src/server/utils/ensure-indexes.ts +96 -0
- package/src/server/utils/logging.ts +62 -63
- package/src/server/utils/loop-pattern-context.ts +20 -0
- package/dist/client-v2/214.79650a549273f163.js +0 -10
- package/dist/client-v2/253.9f0537713cee9e6c.js +0 -10
- package/dist/client-v2/264.718a107e43fc163c.js +0 -10
- package/dist/client-v2/373.f5d5292e53c4e832.js +0 -10
- package/dist/client-v2/418.ce8f5b2b8a229d1d.js +0 -10
- package/dist/client-v2/53.7c29c28043e9112d.js +0 -10
- package/dist/client-v2/619.bd3c5698b40705c3.js +0 -10
- package/dist/client-v2/677.a991ce0250ff5c77.js +0 -10
- package/dist/client-v2/925.f5370de8f6632d65.js +0 -10
- package/dist/server/services/AgentHarness.js +0 -81
- package/dist/server/services/AgentLoopController.js +0 -1023
- package/dist/server/services/AgentLoopRepository.js +0 -192
- package/dist/server/services/AgentLoopService.js +0 -142
- package/dist/server/services/AgentPlanValidator.js +0 -99
- package/dist/server/services/AgentPlannerService.js +0 -99
- package/dist/server/services/CircuitBreaker.js +0 -125
- package/dist/server/services/ContextAggregator.js +0 -201
- package/dist/server/services/PlanningPromptService.js +0 -95
- package/dist/server/tools/agent-loop.js +0 -373
- package/dist/server/tools/orchestrator-plan.js +0 -254
- package/src/client/AIEmployeeSelect.tsx +0 -47
- package/src/client/AIEmployeesContext.tsx +0 -102
- package/src/client/RulesTab.tsx +0 -2
- package/src/client/TracingTab.tsx +0 -436
- package/src/client/skill-hub/components/ExecutionProgress.tsx +0 -51
- package/src/client-v2/components/RulesTab.tsx +0 -2
- package/src/client-v2/pages/AgentRunsPage.tsx +0 -13
- package/src/server/__tests__/agent-loop-controller.test.ts +0 -375
- package/src/server/__tests__/context-aggregator.test.ts +0 -222
- package/src/server/__tests__/parallel-execution.test.ts +0 -318
- package/src/server/__tests__/planning-prompt-service.test.ts +0 -23
- package/src/server/services/AgentHarness.ts +0 -67
- package/src/server/services/AgentLoopController.ts +0 -1263
- package/src/server/services/AgentLoopRepository.ts +0 -178
- package/src/server/services/AgentLoopService.ts +0 -177
- package/src/server/services/AgentPlanValidator.ts +0 -73
- package/src/server/services/AgentPlannerService.ts +0 -80
- package/src/server/services/CircuitBreaker.ts +0 -126
- package/src/server/services/ContextAggregator.ts +0 -239
- package/src/server/services/PlanningPromptService.ts +0 -72
- package/src/server/tools/agent-loop.ts +0 -360
- package/src/server/tools/orchestrator-plan.ts +0 -255
|
@@ -1,8 +1,13 @@
|
|
|
1
1
|
import { createHash } from 'crypto';
|
|
2
|
+
import { mkdirSync, writeFileSync } from 'fs';
|
|
3
|
+
import { resolve } from 'path';
|
|
2
4
|
import { ExecutionSpanService } from './ExecutionSpanService';
|
|
3
5
|
import { AgentMemoryContextService } from './AgentMemoryContextService';
|
|
4
6
|
import { TokenTracker } from './TokenTracker';
|
|
5
|
-
import { runWithAgentExecutionContext } from './AgentExecutionContext';
|
|
7
|
+
import { getAgentExecutionContext, runWithAgentExecutionContext } from './AgentExecutionContext';
|
|
8
|
+
import { compileHarness } from './HarnessCompiler';
|
|
9
|
+
import type { CompiledHarness, HarnessLayer } from './HarnessCompiler';
|
|
10
|
+
import { redactSecretsIn, resolveRunRoleHarness, spilledText } from './harness-runtime-policy';
|
|
6
11
|
import { asObject, currentUserId, toPlain, trimText } from '../utils/ctx-utils';
|
|
7
12
|
|
|
8
13
|
const WRAP_STATE_KEY = Symbol.for('plugin-agent-orchestrator.nativeSubAgentObserver');
|
|
@@ -57,6 +62,7 @@ type RunState = {
|
|
|
57
62
|
leaderUsername?: string;
|
|
58
63
|
employeeUsername?: string;
|
|
59
64
|
userId?: string | number;
|
|
65
|
+
policy: CompiledHarness;
|
|
60
66
|
toolSpans: Map<string, ToolSpanState>;
|
|
61
67
|
pending: Set<Promise<void>>;
|
|
62
68
|
};
|
|
@@ -184,13 +190,28 @@ export class NativeSubAgentObserver {
|
|
|
184
190
|
}
|
|
185
191
|
|
|
186
192
|
const parentSessionId = this.resolveParentSessionId(task) || undefined;
|
|
193
|
+
const parentConversation = await this.safeFindParentConversation(task.ctx, parentSessionId);
|
|
194
|
+
const parentRunId = this.controlPlaneRunIdOf(parentConversation);
|
|
195
|
+
// A sub-agent dispatched from a loop session inherits the role's frozen harness as an extra
|
|
196
|
+
// compile layer, so delegation caps, sharing and memory restrictions do not reset at the hop.
|
|
197
|
+
const parentRunHarness = parentRunId
|
|
198
|
+
? await this.safeResolveParentRunHarness(
|
|
199
|
+
parentRunId,
|
|
200
|
+
normalizeString(readValue(parentConversation, 'aiEmployeeUsername')),
|
|
201
|
+
)
|
|
202
|
+
: null;
|
|
203
|
+
const policy = this.compilePolicy(settings, parentRunHarness);
|
|
204
|
+
const spansEnabled = policy.observability.sharing !== 'disabled';
|
|
205
|
+
|
|
187
206
|
const subSessionId = normalizeString(task.sessionId) || undefined;
|
|
188
207
|
const employeeUsername = modelUsername(task.employee);
|
|
189
208
|
const userId = currentUserId(task.ctx) || task.ctx?.auth?.user?.id;
|
|
190
209
|
const dispatchToolMessage = await this.resolveDispatchToolMessage(task.ctx, parentSessionId, subSessionId);
|
|
191
210
|
const toolCallId = dispatchToolMessage?.toolCallId;
|
|
192
|
-
const leaderUsername =
|
|
211
|
+
const leaderUsername = this.leaderUsernameFor(task.ctx, parentConversation);
|
|
193
212
|
const rootRunId = makeNativeRunId([parentSessionId, subSessionId, toolCallId, employeeUsername]);
|
|
213
|
+
const delegation = await this.resolveDelegationContext(parentSessionId);
|
|
214
|
+
const childDepth = delegation.depth + 1;
|
|
194
215
|
const memory = await this.safeBuildMemoryContext({
|
|
195
216
|
userId,
|
|
196
217
|
aiEmployeeUsername: employeeUsername,
|
|
@@ -219,6 +240,7 @@ export class NativeSubAgentObserver {
|
|
|
219
240
|
memoryScopes: memory.appliedScopes,
|
|
220
241
|
memoryContextChars: memory.chars,
|
|
221
242
|
harnessTag: settings.harnessTag || 'default',
|
|
243
|
+
...(parentRunId ? { agentLoopRunId: parentRunId } : {}),
|
|
222
244
|
},
|
|
223
245
|
parentSessionId,
|
|
224
246
|
subSessionId,
|
|
@@ -226,11 +248,34 @@ export class NativeSubAgentObserver {
|
|
|
226
248
|
leaderUsername,
|
|
227
249
|
employeeUsername,
|
|
228
250
|
userId,
|
|
251
|
+
policy,
|
|
229
252
|
toolSpans: new Map(),
|
|
230
253
|
pending: new Set(),
|
|
231
254
|
};
|
|
232
255
|
|
|
233
|
-
|
|
256
|
+
if (spansEnabled) {
|
|
257
|
+
await this.safeCreateRootSpan(state, task.question, {
|
|
258
|
+
depth: childDepth,
|
|
259
|
+
parentSpanId: delegation.parentSpanId,
|
|
260
|
+
agentLoopRunId: parentRunId,
|
|
261
|
+
});
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
// Depth is persisted on the span, not carried in-process, so a resumed child arrives with its
|
|
265
|
+
// true depth. The cap comes from the compiled policy, which includes the inherited parent
|
|
266
|
+
// layer; exceeding it fails the dispatch instead of silently deepening the recursion.
|
|
267
|
+
const maxDepth = policy.delegation.maxDepth;
|
|
268
|
+
if (maxDepth !== null && childDepth > maxDepth) {
|
|
269
|
+
const message = `Sub-agent delegation depth ${childDepth} exceeds the harness limit of ${maxDepth}.`;
|
|
270
|
+
await this.spanService.finish(state.rootSpanId, 'error', state.rootStartedAt, {
|
|
271
|
+
error: message,
|
|
272
|
+
metadata: {
|
|
273
|
+
...state.rootMetadata,
|
|
274
|
+
failedAt: new Date().toISOString(),
|
|
275
|
+
},
|
|
276
|
+
});
|
|
277
|
+
throw new Error(message);
|
|
278
|
+
}
|
|
234
279
|
|
|
235
280
|
const originalWriter = task.writer;
|
|
236
281
|
observedTask.writer = (chunk: Record<string, unknown>) => {
|
|
@@ -242,7 +287,7 @@ export class NativeSubAgentObserver {
|
|
|
242
287
|
};
|
|
243
288
|
|
|
244
289
|
try {
|
|
245
|
-
const
|
|
290
|
+
const rawResult = await runWithAgentExecutionContext(
|
|
246
291
|
{
|
|
247
292
|
rootRunId,
|
|
248
293
|
spanId: state.rootSpanId ? String(state.rootSpanId) : undefined,
|
|
@@ -253,9 +298,11 @@ export class NativeSubAgentObserver {
|
|
|
253
298
|
},
|
|
254
299
|
() => this.runWithKnowledgeBaseAgentContext(employeeUsername, () => originalRun(observedTask)),
|
|
255
300
|
);
|
|
301
|
+
const result = this.spillLargeResult(state, rawResult);
|
|
256
302
|
await this.flushPending(state);
|
|
303
|
+
const capturedOutput = this.spanOutput(state, trimText(result, 20000));
|
|
257
304
|
await this.spanService.finish(state.rootSpanId, 'success', state.rootStartedAt, {
|
|
258
|
-
output:
|
|
305
|
+
...(capturedOutput !== undefined ? { output: capturedOutput } : {}),
|
|
259
306
|
metadata: {
|
|
260
307
|
...state.rootMetadata,
|
|
261
308
|
completedAt: new Date().toISOString(),
|
|
@@ -270,8 +317,9 @@ export class NativeSubAgentObserver {
|
|
|
270
317
|
return result;
|
|
271
318
|
} catch (error) {
|
|
272
319
|
await this.flushPending(state);
|
|
320
|
+
const capturedError = this.spanOutput(state, trimText(errorMessage(error), 10000));
|
|
273
321
|
await this.spanService.finish(state.rootSpanId, 'error', state.rootStartedAt, {
|
|
274
|
-
error:
|
|
322
|
+
...(capturedError !== undefined ? { error: capturedError } : {}),
|
|
275
323
|
metadata: {
|
|
276
324
|
...state.rootMetadata,
|
|
277
325
|
failedAt: new Date().toISOString(),
|
|
@@ -296,6 +344,65 @@ export class NativeSubAgentObserver {
|
|
|
296
344
|
}
|
|
297
345
|
}
|
|
298
346
|
|
|
347
|
+
// Span payload gates: inputs are captured only under 'full' sharing, outputs under 'full' or
|
|
348
|
+
// 'feedback-only'. Redaction scrubs secret-looking keys on whatever is captured; free text is
|
|
349
|
+
// deliberately left alone.
|
|
350
|
+
private spanInput(state: RunState, input: Record<string, unknown>): Record<string, unknown> | undefined {
|
|
351
|
+
const observability = state.policy.observability;
|
|
352
|
+
if (observability.sharing !== 'full' || !observability.captureInputs) return undefined;
|
|
353
|
+
return observability.redactSecrets ? redactSecretsIn(input) : input;
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
private spanOutput(state: RunState, output: unknown): unknown | undefined {
|
|
357
|
+
const observability = state.policy.observability;
|
|
358
|
+
if (observability.sharing === 'disabled' || !observability.captureOutputs) return undefined;
|
|
359
|
+
return observability.redactSecrets ? redactSecretsIn(output) : output;
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
// Resolved settings are a raw profile record plus the harnessTag marker; the marker is not a
|
|
363
|
+
// schema field. On any compile trouble fall back to an empty (all-default) harness so the
|
|
364
|
+
// safety defaults — redaction on, sharing full — still govern the spans.
|
|
365
|
+
private compilePolicy(settings: Record<string, unknown>, parentRunHarness: CompiledHarness | null): CompiledHarness {
|
|
366
|
+
const tag =
|
|
367
|
+
typeof settings.harnessTag === 'string' && settings.harnessTag.trim() ? settings.harnessTag.trim() : 'default';
|
|
368
|
+
const { harnessTag: _harnessTag, ...rest } = settings;
|
|
369
|
+
try {
|
|
370
|
+
const layers: HarnessLayer[] = [];
|
|
371
|
+
if (parentRunHarness) {
|
|
372
|
+
// `sources` is a compiler artifact the strict schema rejects, so it is stripped before the
|
|
373
|
+
// compiled parent harness is re-used as a layer. Most-restrictive-wins then carries the
|
|
374
|
+
// parent's delegation caps, sharing mode and memory limits into the child.
|
|
375
|
+
const { sources, ...inherited } = parentRunHarness;
|
|
376
|
+
layers.push({ source: `run-role:${sources.join('+') || 'snapshot'}`, settings: inherited });
|
|
377
|
+
}
|
|
378
|
+
layers.push({ source: `profile:${tag}`, settings: rest });
|
|
379
|
+
return compileHarness(layers);
|
|
380
|
+
} catch (error) {
|
|
381
|
+
this.plugin.app.logger?.warn?.('[AgentOrchestrator] Failed to compile observer harness policy', error);
|
|
382
|
+
return compileHarness([{ source: 'default', settings: {} }]);
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
// Oversized sub-agent results are stored on disk and replaced by a head/tail preview plus the
|
|
387
|
+
// file path, so the parent conversation can read the full text back with its file tools.
|
|
388
|
+
// Best effort: a spill failure keeps the original result inline.
|
|
389
|
+
private spillLargeResult(state: RunState, result: string): string {
|
|
390
|
+
const maxInlineBytes = state.policy.context.spill.maxInlineBytes;
|
|
391
|
+
if (!maxInlineBytes || typeof result !== 'string') return result;
|
|
392
|
+
try {
|
|
393
|
+
const spillDir = resolve(process.cwd(), 'storage', 'plugin-agent-orchestrator', 'spills');
|
|
394
|
+
mkdirSync(spillDir, { recursive: true });
|
|
395
|
+
const spillPath = resolve(spillDir, `${state.rootRunId}-${Date.now()}.txt`);
|
|
396
|
+
const replaced = spilledText(result, maxInlineBytes, spillPath);
|
|
397
|
+
if (!replaced) return result;
|
|
398
|
+
writeFileSync(spillPath, result, 'utf8');
|
|
399
|
+
return replaced;
|
|
400
|
+
} catch (error) {
|
|
401
|
+
this.plugin.app.logger?.warn?.('[AgentOrchestrator] Sub-agent result spill failed', error);
|
|
402
|
+
return result;
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
|
|
299
406
|
private async safeBuildMemoryContext(options: {
|
|
300
407
|
userId?: string | number;
|
|
301
408
|
aiEmployeeUsername?: string;
|
|
@@ -309,10 +416,16 @@ export class NativeSubAgentObserver {
|
|
|
309
416
|
}
|
|
310
417
|
}
|
|
311
418
|
|
|
312
|
-
private async safeCreateRootSpan(
|
|
419
|
+
private async safeCreateRootSpan(
|
|
420
|
+
state: RunState,
|
|
421
|
+
question: string,
|
|
422
|
+
delegationInfo: { depth: number; parentSpanId?: string; agentLoopRunId?: number },
|
|
423
|
+
) {
|
|
313
424
|
const span = (await this.spanService.create({
|
|
314
425
|
rootRunId: state.rootRunId,
|
|
315
|
-
parentSpanId:
|
|
426
|
+
parentSpanId: delegationInfo.parentSpanId,
|
|
427
|
+
depth: delegationInfo.depth,
|
|
428
|
+
agentLoopRunId: delegationInfo.agentLoopRunId,
|
|
316
429
|
source: NATIVE_SOURCE,
|
|
317
430
|
parentSessionId: state.parentSessionId,
|
|
318
431
|
subSessionId: state.subSessionId,
|
|
@@ -323,11 +436,11 @@ export class NativeSubAgentObserver {
|
|
|
323
436
|
employeeUsername: state.employeeUsername,
|
|
324
437
|
toolName: 'dispatch-sub-agent-task',
|
|
325
438
|
title: `${state.leaderUsername || 'main'} -> ${state.employeeUsername || 'sub-agent'}`,
|
|
326
|
-
input: {
|
|
439
|
+
input: this.spanInput(state, {
|
|
327
440
|
question,
|
|
328
441
|
parentSessionId: state.parentSessionId,
|
|
329
442
|
subSessionId: state.subSessionId,
|
|
330
|
-
},
|
|
443
|
+
}),
|
|
331
444
|
metadata: state.rootMetadata,
|
|
332
445
|
userId: state.userId,
|
|
333
446
|
})) as SpanRecord | null;
|
|
@@ -369,6 +482,7 @@ export class NativeSubAgentObserver {
|
|
|
369
482
|
}
|
|
370
483
|
|
|
371
484
|
private async handleBeforeToolCall(state: RunState, chunk: Record<string, unknown>) {
|
|
485
|
+
if (state.policy.observability.sharing === 'disabled') return;
|
|
372
486
|
const body = chunkBody(chunk);
|
|
373
487
|
const toolCall = asObject(body.toolCall);
|
|
374
488
|
const toolCallId = normalizeString(toolCall.id);
|
|
@@ -402,7 +516,7 @@ export class NativeSubAgentObserver {
|
|
|
402
516
|
employeeUsername: normalizeString(conversation.username) || state.employeeUsername,
|
|
403
517
|
toolName,
|
|
404
518
|
title: toolName || 'tool call',
|
|
405
|
-
input: asObject(toolCall.args),
|
|
519
|
+
input: this.spanInput(state, asObject(toolCall.args)),
|
|
406
520
|
metadata: {
|
|
407
521
|
source: NATIVE_SOURCE,
|
|
408
522
|
currentConversation: conversation,
|
|
@@ -443,8 +557,8 @@ export class NativeSubAgentObserver {
|
|
|
443
557
|
status,
|
|
444
558
|
endedAt: new Date(),
|
|
445
559
|
durationMs,
|
|
446
|
-
output: forceError ? undefined : trimText(output, 20000),
|
|
447
|
-
error: forceError ? trimText(errorMessage(error), 10000) : undefined,
|
|
560
|
+
output: forceError ? undefined : this.spanOutput(state, trimText(output, 20000)),
|
|
561
|
+
error: forceError ? this.spanOutput(state, trimText(errorMessage(error), 10000)) : undefined,
|
|
448
562
|
skillExecutionId,
|
|
449
563
|
metadata: {
|
|
450
564
|
source: NATIVE_SOURCE,
|
|
@@ -483,24 +597,75 @@ export class NativeSubAgentObserver {
|
|
|
483
597
|
return normalizeString(values.sessionId) || undefined;
|
|
484
598
|
}
|
|
485
599
|
|
|
486
|
-
private
|
|
600
|
+
private leaderUsernameFor(ctx: NativeSubAgentTask['ctx'], parentConversation: unknown) {
|
|
487
601
|
const values = ctx?.action?.params?.values || {};
|
|
488
602
|
const direct =
|
|
489
603
|
normalizeString(values.aiEmployeeUsername) ||
|
|
490
604
|
normalizeString((asObject(values.aiEmployee) as Record<string, unknown>).username);
|
|
491
|
-
|
|
492
|
-
|
|
605
|
+
return direct || normalizeString(readValue(parentConversation, 'aiEmployeeUsername')) || undefined;
|
|
606
|
+
}
|
|
493
607
|
|
|
608
|
+
private async safeFindParentConversation(ctx: NativeSubAgentTask['ctx'], parentSessionId?: string) {
|
|
609
|
+
if (!parentSessionId) return null;
|
|
494
610
|
try {
|
|
495
611
|
const repo = ctx.db?.getRepository?.('aiConversations') as
|
|
496
612
|
| { findOne?: (options: Record<string, unknown>) => Promise<unknown> }
|
|
497
613
|
| undefined;
|
|
498
|
-
const conversation = await repo?.findOne?.({
|
|
499
|
-
|
|
500
|
-
});
|
|
501
|
-
return normalizeString(readValue(conversation, 'aiEmployeeUsername')) || undefined;
|
|
614
|
+
const conversation = await repo?.findOne?.({ filter: { sessionId: parentSessionId } });
|
|
615
|
+
return conversation || null;
|
|
502
616
|
} catch {
|
|
503
|
-
return
|
|
617
|
+
return null;
|
|
618
|
+
}
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
private controlPlaneRunIdOf(conversation: unknown): number | undefined {
|
|
622
|
+
const runId = Number(asObject(readValue(conversation, 'options')).controlPlaneRunId);
|
|
623
|
+
return Number.isSafeInteger(runId) && runId > 0 ? runId : undefined;
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
private async safeResolveParentRunHarness(runId: number, parentUsername: string): Promise<CompiledHarness | null> {
|
|
627
|
+
if (!parentUsername) return null;
|
|
628
|
+
try {
|
|
629
|
+
return await resolveRunRoleHarness(this.plugin.db, runId, parentUsername);
|
|
630
|
+
} catch (error) {
|
|
631
|
+
this.plugin.app.logger?.warn?.('[AgentOrchestrator] Failed to resolve parent run harness for delegation', error);
|
|
632
|
+
return null;
|
|
633
|
+
}
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
// The parent's persisted span is the authoritative depth source: it was recorded when the
|
|
637
|
+
// parent itself was dispatched, so a resumed child arrives with its true depth even though its
|
|
638
|
+
// in-process context is fresh. The runtime context can only confirm or deepen the count, never
|
|
639
|
+
// lower it.
|
|
640
|
+
private async resolveDelegationContext(parentSessionId?: string): Promise<{ parentSpanId?: string; depth: number }> {
|
|
641
|
+
const candidates: Array<{ id?: string | number; depth?: unknown }> = [];
|
|
642
|
+
const context = getAgentExecutionContext();
|
|
643
|
+
if (context?.spanId) {
|
|
644
|
+
const span = await this.safeFindSpan({ filterByTk: context.spanId });
|
|
645
|
+
if (span) candidates.push(span);
|
|
646
|
+
}
|
|
647
|
+
if (parentSessionId) {
|
|
648
|
+
const span = await this.safeFindSpan({
|
|
649
|
+
filter: { subSessionId: parentSessionId, type: 'sub_agent' },
|
|
650
|
+
sort: ['-id'],
|
|
651
|
+
});
|
|
652
|
+
if (span) candidates.push(span);
|
|
653
|
+
}
|
|
654
|
+
const depth = candidates.reduce((deepest, span) => Math.max(deepest, Number(span.depth) || 0), 0);
|
|
655
|
+
const parentSpanId = candidates.find((span) => span.id !== undefined)?.id;
|
|
656
|
+
return { parentSpanId: parentSpanId === undefined ? undefined : String(parentSpanId), depth };
|
|
657
|
+
}
|
|
658
|
+
|
|
659
|
+
private async safeFindSpan(
|
|
660
|
+
options: Record<string, unknown>,
|
|
661
|
+
): Promise<{ id?: string | number; depth?: unknown } | null> {
|
|
662
|
+
try {
|
|
663
|
+
const repo = this.plugin.db.getRepository('agentExecutionSpans');
|
|
664
|
+
if (!repo || typeof repo.findOne !== 'function') return null;
|
|
665
|
+
return ((await repo.findOne(options)) as { id?: string | number; depth?: unknown } | null) || null;
|
|
666
|
+
} catch (error) {
|
|
667
|
+
this.plugin.app.logger?.warn?.('[AgentOrchestrator] Failed to resolve delegation parent span', error);
|
|
668
|
+
return null;
|
|
504
669
|
}
|
|
505
670
|
}
|
|
506
671
|
|
|
@@ -0,0 +1,321 @@
|
|
|
1
|
+
import type { Database, Model, Transaction } from '@nocobase/database';
|
|
2
|
+
|
|
3
|
+
type DistributedLock = {
|
|
4
|
+
runExclusive<T>(key: string, operation: () => Promise<T>, ttl?: number): Promise<T>;
|
|
5
|
+
};
|
|
6
|
+
|
|
7
|
+
type PathLockRow = {
|
|
8
|
+
id: number;
|
|
9
|
+
runId: number;
|
|
10
|
+
repositoryKey: string;
|
|
11
|
+
paths: string[];
|
|
12
|
+
status: 'waiting' | 'held' | 'released' | 'expired';
|
|
13
|
+
blockerRunIds: number[];
|
|
14
|
+
};
|
|
15
|
+
|
|
16
|
+
export type PathLockAcquisition =
|
|
17
|
+
| { acquired: true; lock: PathLockRow; blockers: number[] }
|
|
18
|
+
| { acquired: false; deadlocked: boolean; blockers: number[] };
|
|
19
|
+
|
|
20
|
+
function read(record: Model | Record<string, unknown>, key: string) {
|
|
21
|
+
const model = record as Model & { get?: (name: string) => unknown };
|
|
22
|
+
return typeof model.get === 'function' ? model.get(key) : (record as Record<string, unknown>)[key];
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function positiveInteger(value: unknown) {
|
|
26
|
+
const parsed = Number(value);
|
|
27
|
+
return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : null;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
function asDate(value: unknown): Date | null {
|
|
31
|
+
if (value instanceof Date) return Number.isNaN(value.getTime()) ? null : value;
|
|
32
|
+
if (typeof value === 'string' || typeof value === 'number') {
|
|
33
|
+
const parsed = new Date(value);
|
|
34
|
+
return Number.isNaN(parsed.getTime()) ? null : parsed;
|
|
35
|
+
}
|
|
36
|
+
return null;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
// A held lock without a readable expiry is treated as expired: a crashed worker must never
|
|
40
|
+
// leave a row that blocks every later run forever.
|
|
41
|
+
function lockIsExpired(record: Model | Record<string, unknown>, now: Date) {
|
|
42
|
+
const expiresAt = asDate(read(record, 'expiresAt'));
|
|
43
|
+
return !expiresAt || expiresAt.getTime() <= now.getTime();
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function positiveTtl(ttlMs: number) {
|
|
47
|
+
if (!Number.isSafeInteger(ttlMs) || ttlMs <= 0) {
|
|
48
|
+
throw new Error('Path lock TTL must be a positive number of milliseconds.');
|
|
49
|
+
}
|
|
50
|
+
return ttlMs;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export function normalizeLockPath(value: string) {
|
|
54
|
+
const normalized = value
|
|
55
|
+
.trim()
|
|
56
|
+
.replace(/\\/g, '/')
|
|
57
|
+
.replace(/^\.\//, '')
|
|
58
|
+
.replace(/\/{2,}/g, '/');
|
|
59
|
+
if (!normalized || normalized.startsWith('/') || /^[a-zA-Z]:\//.test(normalized)) {
|
|
60
|
+
throw new Error(`Lock path must be repository-relative: ${value}`);
|
|
61
|
+
}
|
|
62
|
+
const segments = normalized.split('/');
|
|
63
|
+
if (segments.some((segment) => segment === '..' || segment === '.')) {
|
|
64
|
+
throw new Error(`Lock path cannot contain dot segments: ${value}`);
|
|
65
|
+
}
|
|
66
|
+
return normalized.replace(/\/$/, '');
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function literalPrefix(pattern: string) {
|
|
70
|
+
const result: string[] = [];
|
|
71
|
+
for (const segment of normalizeLockPath(pattern).split('/')) {
|
|
72
|
+
if (/[*?{[\]]/.test(segment)) break;
|
|
73
|
+
result.push(segment);
|
|
74
|
+
}
|
|
75
|
+
return result;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export function lockPathsOverlap(left: string, right: string) {
|
|
79
|
+
const leftNormalized = normalizeLockPath(left);
|
|
80
|
+
const rightNormalized = normalizeLockPath(right);
|
|
81
|
+
if (leftNormalized === rightNormalized) return true;
|
|
82
|
+
|
|
83
|
+
const leftPrefix = literalPrefix(leftNormalized);
|
|
84
|
+
const rightPrefix = literalPrefix(rightNormalized);
|
|
85
|
+
if (leftPrefix.length === 0 || rightPrefix.length === 0) return true;
|
|
86
|
+
const shared = Math.min(leftPrefix.length, rightPrefix.length);
|
|
87
|
+
for (let index = 0; index < shared; index += 1) {
|
|
88
|
+
if (leftPrefix[index] !== rightPrefix[index]) return false;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
const leftHasPattern = leftPrefix.length < leftNormalized.split('/').length;
|
|
92
|
+
const rightHasPattern = rightPrefix.length < rightNormalized.split('/').length;
|
|
93
|
+
if (leftHasPattern || rightHasPattern) return true;
|
|
94
|
+
return leftPrefix.length !== rightPrefix.length;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
export function pathSetsOverlap(left: string[], right: string[]) {
|
|
98
|
+
return left.some((leftPath) => right.some((rightPath) => lockPathsOverlap(leftPath, rightPath)));
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
function row(record: Model | Record<string, unknown>): PathLockRow {
|
|
102
|
+
const status = read(record, 'status');
|
|
103
|
+
return {
|
|
104
|
+
id: positiveInteger(read(record, 'id')) || 0,
|
|
105
|
+
runId: positiveInteger(read(record, 'runId')) || 0,
|
|
106
|
+
repositoryKey: String(read(record, 'repositoryKey') || ''),
|
|
107
|
+
paths: Array.isArray(read(record, 'paths')) ? (read(record, 'paths') as string[]) : [],
|
|
108
|
+
status: status === 'held' || status === 'released' || status === 'expired' ? status : 'waiting',
|
|
109
|
+
blockerRunIds: Array.isArray(read(record, 'blockerRunIds'))
|
|
110
|
+
? (read(record, 'blockerRunIds') as unknown[])
|
|
111
|
+
.map(positiveInteger)
|
|
112
|
+
.filter((value): value is number => value !== null)
|
|
113
|
+
: [],
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
export function waitGraphHasCycle(rows: PathLockRow[], runId: number, blockers: number[]) {
|
|
118
|
+
const graph = new Map<number, number[]>();
|
|
119
|
+
for (const lock of rows) {
|
|
120
|
+
if (lock.status === 'waiting') graph.set(lock.runId, lock.blockerRunIds);
|
|
121
|
+
}
|
|
122
|
+
graph.set(runId, blockers);
|
|
123
|
+
const visiting = new Set<number>();
|
|
124
|
+
const visited = new Set<number>();
|
|
125
|
+
|
|
126
|
+
const visit = (node: number): boolean => {
|
|
127
|
+
if (visiting.has(node)) return true;
|
|
128
|
+
if (visited.has(node)) return false;
|
|
129
|
+
visiting.add(node);
|
|
130
|
+
for (const blocker of graph.get(node) || []) {
|
|
131
|
+
if (visit(blocker)) return true;
|
|
132
|
+
}
|
|
133
|
+
visiting.delete(node);
|
|
134
|
+
visited.add(node);
|
|
135
|
+
return false;
|
|
136
|
+
};
|
|
137
|
+
|
|
138
|
+
return visit(runId);
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
export class PathLockService {
|
|
142
|
+
constructor(
|
|
143
|
+
private readonly database: Database,
|
|
144
|
+
private readonly distributedLock: DistributedLock,
|
|
145
|
+
) {}
|
|
146
|
+
|
|
147
|
+
async acquire(input: {
|
|
148
|
+
runId: number;
|
|
149
|
+
repositoryKey: string;
|
|
150
|
+
owner: string;
|
|
151
|
+
paths: string[];
|
|
152
|
+
ttlMs: number;
|
|
153
|
+
waitUntil?: Date;
|
|
154
|
+
}): Promise<PathLockAcquisition> {
|
|
155
|
+
const paths = Array.from(new Set(input.paths.map(normalizeLockPath))).sort();
|
|
156
|
+
if (!paths.length) throw new Error('At least one repository path is required for a path lock.');
|
|
157
|
+
const repositoryKey = input.repositoryKey.trim();
|
|
158
|
+
if (!repositoryKey) throw new Error('Repository key is required for a path lock.');
|
|
159
|
+
const ttlMs = positiveTtl(input.ttlMs);
|
|
160
|
+
|
|
161
|
+
return this.distributedLock.runExclusive(
|
|
162
|
+
`agent-loop:path:${repositoryKey}`,
|
|
163
|
+
() =>
|
|
164
|
+
this.database.sequelize.transaction(async (transaction) => {
|
|
165
|
+
const repository = this.database.getRepository('agentLoopPathLocks');
|
|
166
|
+
const now = new Date();
|
|
167
|
+
const existing = await repository.findOne({
|
|
168
|
+
filter: { runId: input.runId, repositoryKey },
|
|
169
|
+
transaction,
|
|
170
|
+
lock: transaction.LOCK.UPDATE,
|
|
171
|
+
});
|
|
172
|
+
if (existing && read(existing, 'status') === 'held') {
|
|
173
|
+
const sameOwner = String(read(existing, 'owner') || '') === input.owner;
|
|
174
|
+
const samePaths = JSON.stringify(row(existing).paths.slice().sort()) === JSON.stringify(paths);
|
|
175
|
+
if (sameOwner && samePaths && !lockIsExpired(existing, now)) {
|
|
176
|
+
return { acquired: true, lock: row(existing), blockers: [] as number[] };
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
const held = await repository.find({
|
|
181
|
+
filter: { repositoryKey, status: 'held' },
|
|
182
|
+
transaction,
|
|
183
|
+
lock: transaction.LOCK.UPDATE,
|
|
184
|
+
});
|
|
185
|
+
const active: typeof held = [];
|
|
186
|
+
for (const lock of held) {
|
|
187
|
+
if (!lockIsExpired(lock, now)) {
|
|
188
|
+
active.push(lock);
|
|
189
|
+
continue;
|
|
190
|
+
}
|
|
191
|
+
await repository.update({
|
|
192
|
+
filterByTk: read(lock, 'id'),
|
|
193
|
+
values: { status: 'expired', releasedAt: now, blockerRunIds: [] },
|
|
194
|
+
transaction,
|
|
195
|
+
});
|
|
196
|
+
}
|
|
197
|
+
const blockers = active
|
|
198
|
+
.map(row)
|
|
199
|
+
.filter((lock) => lock.runId !== input.runId && pathSetsOverlap(paths, lock.paths))
|
|
200
|
+
.map((lock) => lock.runId);
|
|
201
|
+
|
|
202
|
+
if (blockers.length) {
|
|
203
|
+
const waitingRows = (
|
|
204
|
+
await repository.find({
|
|
205
|
+
filter: { repositoryKey, status: 'waiting' },
|
|
206
|
+
transaction,
|
|
207
|
+
lock: transaction.LOCK.UPDATE,
|
|
208
|
+
})
|
|
209
|
+
).map(row);
|
|
210
|
+
if (waitGraphHasCycle(waitingRows, input.runId, blockers)) {
|
|
211
|
+
return { acquired: false, deadlocked: true, blockers };
|
|
212
|
+
}
|
|
213
|
+
const values = {
|
|
214
|
+
runId: input.runId,
|
|
215
|
+
repositoryKey,
|
|
216
|
+
owner: input.owner,
|
|
217
|
+
paths,
|
|
218
|
+
status: 'waiting',
|
|
219
|
+
blockerRunIds: blockers,
|
|
220
|
+
waitUntil: input.waitUntil || null,
|
|
221
|
+
acquiredAt: null,
|
|
222
|
+
expiresAt: null,
|
|
223
|
+
releasedAt: null,
|
|
224
|
+
};
|
|
225
|
+
if (existing) {
|
|
226
|
+
await repository.update({ filterByTk: read(existing, 'id'), values, transaction });
|
|
227
|
+
} else {
|
|
228
|
+
await repository.create({ values, transaction });
|
|
229
|
+
}
|
|
230
|
+
return { acquired: false, deadlocked: false, blockers };
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
const values = {
|
|
234
|
+
runId: input.runId,
|
|
235
|
+
repositoryKey,
|
|
236
|
+
owner: input.owner,
|
|
237
|
+
paths,
|
|
238
|
+
status: 'held',
|
|
239
|
+
blockerRunIds: [],
|
|
240
|
+
acquiredAt: now,
|
|
241
|
+
expiresAt: new Date(now.getTime() + ttlMs),
|
|
242
|
+
waitUntil: null,
|
|
243
|
+
releasedAt: null,
|
|
244
|
+
};
|
|
245
|
+
const locked = existing
|
|
246
|
+
? await this.updateAndRead(repository, Number(read(existing, 'id')), values, transaction)
|
|
247
|
+
: await repository.create({ values, transaction });
|
|
248
|
+
return { acquired: true, lock: row(locked), blockers: [] as number[] };
|
|
249
|
+
}),
|
|
250
|
+
Math.max(ttlMs, 5000),
|
|
251
|
+
);
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
async renew(runId: number, repositoryKey: string, ttlMs: number, owner?: string) {
|
|
255
|
+
const ttl = positiveTtl(ttlMs);
|
|
256
|
+
return this.distributedLock.runExclusive(`agent-loop:path:${repositoryKey}`, async () => {
|
|
257
|
+
const repository = this.database.getRepository('agentLoopPathLocks');
|
|
258
|
+
// An expired lock is marked `expired` in its own committed transaction. Doing the audit
|
|
259
|
+
// write inside the renew transaction and then throwing would roll the mark back, leaving a
|
|
260
|
+
// stale `held` row that still reads as expired to the next acquirer but never audits it.
|
|
261
|
+
const expired = await this.database.sequelize.transaction(async (transaction) => {
|
|
262
|
+
const lock = await repository.findOne({
|
|
263
|
+
filter: { runId, repositoryKey, status: 'held' },
|
|
264
|
+
transaction,
|
|
265
|
+
lock: transaction.LOCK.UPDATE,
|
|
266
|
+
});
|
|
267
|
+
if (!lock) throw new Error('Held path lock was not found.');
|
|
268
|
+
if (owner !== undefined && String(read(lock, 'owner') || '') !== owner) {
|
|
269
|
+
throw new Error('Path lock is no longer owned by this worker.');
|
|
270
|
+
}
|
|
271
|
+
const now = new Date();
|
|
272
|
+
if (lockIsExpired(lock, now)) return { id: read(lock, 'id'), at: now };
|
|
273
|
+
await repository.update({
|
|
274
|
+
filterByTk: read(lock, 'id'),
|
|
275
|
+
values: { expiresAt: new Date(now.getTime() + ttl) },
|
|
276
|
+
transaction,
|
|
277
|
+
});
|
|
278
|
+
return null;
|
|
279
|
+
});
|
|
280
|
+
|
|
281
|
+
if (expired) {
|
|
282
|
+
await this.database.sequelize.transaction(async (transaction) => {
|
|
283
|
+
await repository.update({
|
|
284
|
+
filterByTk: expired.id,
|
|
285
|
+
values: { status: 'expired', releasedAt: expired.at, blockerRunIds: [] },
|
|
286
|
+
transaction,
|
|
287
|
+
});
|
|
288
|
+
});
|
|
289
|
+
throw new Error('Path lock has expired and must be reacquired.');
|
|
290
|
+
}
|
|
291
|
+
});
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
// The owner token guards against a reclaimed run: once a stale worker's lease expires and another
|
|
295
|
+
// worker takes over the same run, the original worker must not free a lock the new owner now holds.
|
|
296
|
+
async release(runId: number, repositoryKey: string, owner?: string) {
|
|
297
|
+
return this.distributedLock.runExclusive(`agent-loop:path:${repositoryKey}`, async () => {
|
|
298
|
+
const repository = this.database.getRepository('agentLoopPathLocks');
|
|
299
|
+
const lock = await repository.findOne({ filter: { runId, repositoryKey } });
|
|
300
|
+
if (!lock || ['released', 'expired'].includes(String(read(lock, 'status')))) return false;
|
|
301
|
+
if (owner !== undefined && String(read(lock, 'owner') || '') !== owner) return false;
|
|
302
|
+
await repository.update({
|
|
303
|
+
filterByTk: read(lock, 'id'),
|
|
304
|
+
values: { status: 'released', releasedAt: new Date(), blockerRunIds: [] },
|
|
305
|
+
});
|
|
306
|
+
return true;
|
|
307
|
+
});
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
private async updateAndRead(
|
|
311
|
+
repository: ReturnType<Database['getRepository']>,
|
|
312
|
+
id: number,
|
|
313
|
+
values: Record<string, unknown>,
|
|
314
|
+
transaction: Transaction,
|
|
315
|
+
) {
|
|
316
|
+
await repository.update({ filterByTk: id, values, transaction });
|
|
317
|
+
const updated = await repository.findOne({ filterByTk: id, transaction });
|
|
318
|
+
if (!updated) throw new Error('Path lock disappeared during update.');
|
|
319
|
+
return updated;
|
|
320
|
+
}
|
|
321
|
+
}
|