pilotswarm-sdk 0.4.1 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/api/src/protocol.js +7 -0
- package/dist/blob-store.d.ts +10 -1
- package/dist/blob-store.d.ts.map +1 -1
- package/dist/blob-store.js +275 -13
- package/dist/blob-store.js.map +1 -1
- package/dist/client.d.ts.map +1 -1
- package/dist/client.js +15 -1
- package/dist/client.js.map +1 -1
- package/dist/cms-migrations.d.ts.map +1 -1
- package/dist/cms-migrations.js +164 -0
- package/dist/cms-migrations.js.map +1 -1
- package/dist/cms.d.ts +6 -0
- package/dist/cms.d.ts.map +1 -1
- package/dist/cms.js +3 -0
- package/dist/cms.js.map +1 -1
- package/dist/fault-injection.d.ts +2 -0
- package/dist/fault-injection.d.ts.map +1 -0
- package/dist/fault-injection.js +45 -0
- package/dist/fault-injection.js.map +1 -0
- package/dist/model-providers.js +1 -1
- package/dist/model-providers.js.map +1 -1
- package/dist/orchestration/agents.d.ts.map +1 -1
- package/dist/orchestration/agents.js +9 -1
- package/dist/orchestration/agents.js.map +1 -1
- package/dist/orchestration/index.d.ts +3 -3
- package/dist/orchestration/index.js +1 -1
- package/dist/orchestration/lifecycle.d.ts +9 -2
- package/dist/orchestration/lifecycle.d.ts.map +1 -1
- package/dist/orchestration/lifecycle.js +21 -55
- package/dist/orchestration/lifecycle.js.map +1 -1
- package/dist/orchestration/runtime.d.ts +1 -1
- package/dist/orchestration/state.d.ts +7 -3
- package/dist/orchestration/state.d.ts.map +1 -1
- package/dist/orchestration/state.js +13 -4
- package/dist/orchestration/state.js.map +1 -1
- package/dist/orchestration/turn.d.ts.map +1 -1
- package/dist/orchestration/turn.js +126 -66
- package/dist/orchestration/turn.js.map +1 -1
- package/dist/orchestration-registry.d.ts.map +1 -1
- package/dist/orchestration-registry.js +8 -2
- package/dist/orchestration-registry.js.map +1 -1
- package/dist/orchestration-version.d.ts +1 -1
- package/dist/orchestration-version.js +1 -1
- package/dist/orchestration.d.ts +2 -2
- package/dist/orchestration.js +4 -4
- package/dist/orchestration_1_0_46.d.ts +1 -1
- package/dist/orchestration_1_0_47.d.ts +1 -1
- package/dist/orchestration_1_0_48.d.ts +1 -1
- package/dist/orchestration_1_0_49.d.ts +1 -1
- package/dist/orchestration_1_0_50.d.ts +1 -1
- package/dist/orchestration_1_0_56/agents.d.ts +41 -0
- package/dist/orchestration_1_0_56/agents.d.ts.map +1 -0
- package/dist/orchestration_1_0_56/agents.js +758 -0
- package/dist/orchestration_1_0_56/agents.js.map +1 -0
- package/dist/orchestration_1_0_56/index.d.ts +24 -0
- package/dist/orchestration_1_0_56/index.d.ts.map +1 -0
- package/dist/orchestration_1_0_56/index.js +13 -0
- package/dist/orchestration_1_0_56/index.js.map +1 -0
- package/dist/orchestration_1_0_56/lifecycle.d.ts +41 -0
- package/dist/orchestration_1_0_56/lifecycle.d.ts.map +1 -0
- package/dist/orchestration_1_0_56/lifecycle.js +580 -0
- package/dist/orchestration_1_0_56/lifecycle.js.map +1 -0
- package/dist/orchestration_1_0_56/queue.d.ts +7 -0
- package/dist/orchestration_1_0_56/queue.d.ts.map +1 -0
- package/dist/orchestration_1_0_56/queue.js +644 -0
- package/dist/orchestration_1_0_56/queue.js.map +1 -0
- package/dist/orchestration_1_0_56/runtime.d.ts +29 -0
- package/dist/orchestration_1_0_56/runtime.d.ts.map +1 -0
- package/dist/orchestration_1_0_56/runtime.js +194 -0
- package/dist/orchestration_1_0_56/runtime.js.map +1 -0
- package/dist/orchestration_1_0_56/state.d.ts +126 -0
- package/dist/orchestration_1_0_56/state.d.ts.map +1 -0
- package/dist/orchestration_1_0_56/state.js +106 -0
- package/dist/orchestration_1_0_56/state.js.map +1 -0
- package/dist/orchestration_1_0_56/turn.d.ts +30 -0
- package/dist/orchestration_1_0_56/turn.d.ts.map +1 -0
- package/dist/orchestration_1_0_56/turn.js +1075 -0
- package/dist/orchestration_1_0_56/turn.js.map +1 -0
- package/dist/orchestration_1_0_56/utils.d.ts +22 -0
- package/dist/orchestration_1_0_56/utils.d.ts.map +1 -0
- package/dist/orchestration_1_0_56/utils.js +226 -0
- package/dist/orchestration_1_0_56/utils.js.map +1 -0
- package/dist/orchestration_1_0_57/agents.d.ts +41 -0
- package/dist/orchestration_1_0_57/agents.d.ts.map +1 -0
- package/dist/orchestration_1_0_57/agents.js +758 -0
- package/dist/orchestration_1_0_57/agents.js.map +1 -0
- package/dist/orchestration_1_0_57/index.d.ts +24 -0
- package/dist/orchestration_1_0_57/index.d.ts.map +1 -0
- package/dist/orchestration_1_0_57/index.js +13 -0
- package/dist/orchestration_1_0_57/index.js.map +1 -0
- package/dist/orchestration_1_0_57/lifecycle.d.ts +49 -0
- package/dist/orchestration_1_0_57/lifecycle.d.ts.map +1 -0
- package/dist/orchestration_1_0_57/lifecycle.js +583 -0
- package/dist/orchestration_1_0_57/lifecycle.js.map +1 -0
- package/dist/orchestration_1_0_57/queue.d.ts +7 -0
- package/dist/orchestration_1_0_57/queue.d.ts.map +1 -0
- package/dist/orchestration_1_0_57/queue.js +644 -0
- package/dist/orchestration_1_0_57/queue.js.map +1 -0
- package/dist/orchestration_1_0_57/runtime.d.ts +29 -0
- package/dist/orchestration_1_0_57/runtime.d.ts.map +1 -0
- package/dist/orchestration_1_0_57/runtime.js +198 -0
- package/dist/orchestration_1_0_57/runtime.js.map +1 -0
- package/dist/orchestration_1_0_57/state.d.ts +133 -0
- package/dist/orchestration_1_0_57/state.d.ts.map +1 -0
- package/dist/orchestration_1_0_57/state.js +115 -0
- package/dist/orchestration_1_0_57/state.js.map +1 -0
- package/dist/orchestration_1_0_57/turn.d.ts +30 -0
- package/dist/orchestration_1_0_57/turn.d.ts.map +1 -0
- package/dist/orchestration_1_0_57/turn.js +1110 -0
- package/dist/orchestration_1_0_57/turn.js.map +1 -0
- package/dist/orchestration_1_0_57/utils.d.ts +22 -0
- package/dist/orchestration_1_0_57/utils.d.ts.map +1 -0
- package/dist/orchestration_1_0_57/utils.js +226 -0
- package/dist/orchestration_1_0_57/utils.js.map +1 -0
- package/dist/orchestration_1_0_58/agents.d.ts +41 -0
- package/dist/orchestration_1_0_58/agents.d.ts.map +1 -0
- package/dist/orchestration_1_0_58/agents.js +758 -0
- package/dist/orchestration_1_0_58/agents.js.map +1 -0
- package/dist/orchestration_1_0_58/index.d.ts +24 -0
- package/dist/orchestration_1_0_58/index.d.ts.map +1 -0
- package/dist/orchestration_1_0_58/index.js +13 -0
- package/dist/orchestration_1_0_58/index.js.map +1 -0
- package/dist/orchestration_1_0_58/lifecycle.d.ts +48 -0
- package/dist/orchestration_1_0_58/lifecycle.d.ts.map +1 -0
- package/dist/orchestration_1_0_58/lifecycle.js +546 -0
- package/dist/orchestration_1_0_58/lifecycle.js.map +1 -0
- package/dist/orchestration_1_0_58/queue.d.ts +7 -0
- package/dist/orchestration_1_0_58/queue.d.ts.map +1 -0
- package/dist/orchestration_1_0_58/queue.js +644 -0
- package/dist/orchestration_1_0_58/queue.js.map +1 -0
- package/dist/orchestration_1_0_58/runtime.d.ts +29 -0
- package/dist/orchestration_1_0_58/runtime.d.ts.map +1 -0
- package/dist/orchestration_1_0_58/runtime.js +199 -0
- package/dist/orchestration_1_0_58/runtime.js.map +1 -0
- package/dist/orchestration_1_0_58/state.d.ts +130 -0
- package/dist/orchestration_1_0_58/state.d.ts.map +1 -0
- package/dist/orchestration_1_0_58/state.js +115 -0
- package/dist/orchestration_1_0_58/state.js.map +1 -0
- package/dist/orchestration_1_0_58/turn.d.ts +30 -0
- package/dist/orchestration_1_0_58/turn.d.ts.map +1 -0
- package/dist/orchestration_1_0_58/turn.js +1107 -0
- package/dist/orchestration_1_0_58/turn.js.map +1 -0
- package/dist/orchestration_1_0_58/utils.d.ts +22 -0
- package/dist/orchestration_1_0_58/utils.d.ts.map +1 -0
- package/dist/orchestration_1_0_58/utils.js +226 -0
- package/dist/orchestration_1_0_58/utils.js.map +1 -0
- package/dist/session-lifecycle.d.ts +114 -0
- package/dist/session-lifecycle.d.ts.map +1 -0
- package/dist/session-lifecycle.js +324 -0
- package/dist/session-lifecycle.js.map +1 -0
- package/dist/session-manager.d.ts +21 -0
- package/dist/session-manager.d.ts.map +1 -1
- package/dist/session-manager.js +137 -1
- package/dist/session-manager.js.map +1 -1
- package/dist/session-proxy.d.ts +5 -1
- package/dist/session-proxy.d.ts.map +1 -1
- package/dist/session-proxy.js +1158 -954
- package/dist/session-proxy.js.map +1 -1
- package/dist/session-store.d.ts +48 -3
- package/dist/session-store.d.ts.map +1 -1
- package/dist/session-store.js +394 -28
- package/dist/session-store.js.map +1 -1
- package/dist/snapshot-protocol.d.ts +101 -0
- package/dist/snapshot-protocol.d.ts.map +1 -0
- package/dist/snapshot-protocol.js +120 -0
- package/dist/snapshot-protocol.js.map +1 -0
- package/dist/types.d.ts +24 -1
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/dist/wait-affinity.d.ts +11 -0
- package/dist/wait-affinity.d.ts.map +1 -1
- package/dist/wait-affinity.js +6 -0
- package/dist/wait-affinity.js.map +1 -1
- package/dist/web/web-management-client.d.ts +28 -9
- package/dist/web/web-management-client.d.ts.map +1 -1
- package/dist/web/web-management-client.js +30 -15
- package/dist/web/web-management-client.js.map +1 -1
- package/dist/worker.d.ts +16 -1
- package/dist/worker.d.ts.map +1 -1
- package/dist/worker.js +63 -6
- package/dist/worker.js.map +1 -1
- package/package.json +7 -4
package/dist/session-proxy.js
CHANGED
|
@@ -13,7 +13,10 @@ import { computeCronAtNextFire } from "./cron-at.js";
|
|
|
13
13
|
import { SpanStatusCode, trace as otelTrace } from "@opentelemetry/api";
|
|
14
14
|
import os from "node:os";
|
|
15
15
|
import fs from "node:fs";
|
|
16
|
+
import path from "node:path";
|
|
16
17
|
import { randomUUID } from "node:crypto";
|
|
18
|
+
import { attemptStoreRecovery, runTurnCommit, runTurnPreamble } from "./session-lifecycle.js";
|
|
19
|
+
import { supportsVersionedSnapshots, writeTurnSentinel } from "./snapshot-protocol.js";
|
|
17
20
|
const SYSTEM_AGENT_IDS = new Set(["pilotswarm", "sweeper", "resourcemgr", "facts-manager", "agent-tuner"]);
|
|
18
21
|
const SESSION_RECOVERY_NOTICE = "[SYSTEM: The runtime recovered this session after the live Copilot session was lost on a worker. " +
|
|
19
22
|
"Some very recent in-memory state may have been lost. Re-read the visible conversation and continue carefully from the latest durable state.]";
|
|
@@ -23,6 +26,8 @@ const LOSSY_SESSION_REPLAY_NOTICE = "[SYSTEM: The runtime is replaying this turn
|
|
|
23
26
|
const CORRUPTED_TRANSCRIPT_REPLAY_NOTICE = "[SYSTEM: The runtime recreated this session after the live Copilot transcript became inconsistent. " +
|
|
24
27
|
"Some recent in-memory work may be missing, and the previous turn may have partially executed. " +
|
|
25
28
|
"Re-read the visible conversation and continue carefully from the latest durable state.]";
|
|
29
|
+
const REHYDRATED_SESSION_NOTICE = "[SYSTEM: This session was rehydrated on a different worker from its durable snapshot. " +
|
|
30
|
+
"The conversation history is fully preserved. Continue seamlessly.]";
|
|
26
31
|
function normalizeJsonObject(value) {
|
|
27
32
|
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
28
33
|
return null;
|
|
@@ -429,6 +434,7 @@ export function createSessionProxy(ctx, sessionId, affinityKey, config) {
|
|
|
429
434
|
...(turnMeta?.clientMessageIds && turnMeta.clientMessageIds.length > 0
|
|
430
435
|
? { clientMessageIds: turnMeta.clientMessageIds }
|
|
431
436
|
: {}),
|
|
437
|
+
...(turnMeta?.snapshot ? { snapshot: turnMeta.snapshot } : {}),
|
|
432
438
|
}, affinityKey);
|
|
433
439
|
},
|
|
434
440
|
dehydrate(reason, eventData) {
|
|
@@ -618,7 +624,7 @@ export function createSessionManagerProxy(ctx) {
|
|
|
618
624
|
// ─── Activity Registration ───────────────────────────────────────
|
|
619
625
|
// Thin dispatchers — each is a one-liner that calls the corresponding
|
|
620
626
|
// SessionManager or ManagedSession method.
|
|
621
|
-
export function registerActivities(runtime, sessionManager,
|
|
627
|
+
export function registerActivities(runtime, sessionManager, sessionStore, githubToken, catalog, provider, storeUrl, cmsSchema, clientConfig,
|
|
622
628
|
/** Loaded system agents — used by resolveAgentConfig activity. */
|
|
623
629
|
systemAgents,
|
|
624
630
|
/** Worker-level session policy — used by getWorkerSessionPolicy activity. */
|
|
@@ -715,1024 +721,1207 @@ workerNodeId) {
|
|
|
715
721
|
let finalTurnResult = null;
|
|
716
722
|
try {
|
|
717
723
|
finalTurnResult = await sessionManager.withRunTurnLock(input.sessionId, "runTurn", async () => {
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
724
|
+
// ── Session lifecycle protocol preamble (proposal §3.3) ─────────
|
|
725
|
+
// Only active when the orchestration (1.0.57+) supplied snapshot
|
|
726
|
+
// coordinates AND the store implements the versioned CAS contract.
|
|
727
|
+
// Absent either, this activity behaves exactly as before.
|
|
728
|
+
const lifecycle = input.snapshot && sessionStore && supportsVersionedSnapshots(sessionStore)
|
|
729
|
+
? {
|
|
730
|
+
store: sessionStore,
|
|
731
|
+
sessionStateDir: sessionManager.getSessionStateDir(),
|
|
732
|
+
sessionId: input.sessionId,
|
|
733
|
+
// Store-wins: expectedVersion is non-load-bearing and 1.0.59
|
|
734
|
+
// stops sending it (dead for the reconcile — the worker
|
|
735
|
+
// ignores it for every control decision). Default to 0 for
|
|
736
|
+
// frozen versions that still supply it, and for the two
|
|
737
|
+
// observability-only reads (lossy flag, snapshot_store_empty).
|
|
738
|
+
expectedVersion: input.snapshot.expectedVersion ?? 0,
|
|
739
|
+
turnKey: input.snapshot.turnKey,
|
|
740
|
+
dropWarmSession: () => sessionManager.dropWarmSession(input.sessionId),
|
|
723
741
|
trace,
|
|
724
|
-
lockHeld: true,
|
|
725
|
-
});
|
|
726
|
-
}
|
|
727
|
-
catch (err) {
|
|
728
|
-
const message = err?.message || String(err);
|
|
729
|
-
if (isMissingSessionStateErrorMessage(message) || isLiveSessionLostErrorMessage(message)) {
|
|
730
|
-
const detail = isMissingSessionStateErrorMessage(message)
|
|
731
|
-
? stripMissingSessionStatePrefix(message)
|
|
732
|
-
: message;
|
|
733
|
-
trace(`session=${input.sessionId} missing resumable state before turn ${input.turnIndex ?? "unknown"}; ` +
|
|
734
|
-
"starting lossy fresh-session replay");
|
|
735
|
-
await recordLossyHandoffEvent(catalog, input.sessionId, workerNodeId, {
|
|
736
|
-
cause: "missing_resumable_state_before_run_turn",
|
|
737
|
-
message: buildLossyReplayMessage(input.sessionId, detail),
|
|
738
|
-
detail,
|
|
739
|
-
error: message,
|
|
740
|
-
recoveryMode: "fresh_session_replay",
|
|
741
|
-
nextStep: "replay_pending_turn_with_recreated_copilot_session",
|
|
742
|
-
...(input.turnIndex != null ? { iteration: input.turnIndex } : {}),
|
|
743
|
-
}, (failureMessage) => activityCtx.traceInfo(`[runTurn] ${failureMessage}`));
|
|
744
|
-
if (catalog) {
|
|
745
|
-
await cmsRetryBestEffort(`runTurn.recordEvent system.message-lossy-replay session=${input.sessionId}`, () => catalog.recordEvents(input.sessionId, [{
|
|
746
|
-
eventType: "system.message",
|
|
747
|
-
data: {
|
|
748
|
-
content: "The runtime is replaying this turn after a worker restart lost the live Copilot session state before durable dehydrate completed. " +
|
|
749
|
-
"Some recent work may be missing or partially executed.",
|
|
750
|
-
},
|
|
751
|
-
}], workerNodeId), (msg) => activityCtx.traceInfo(msg));
|
|
752
|
-
}
|
|
753
|
-
try {
|
|
754
|
-
session = await sessionManager.getOrCreate(input.sessionId, runConfig, {
|
|
755
|
-
turnIndex: 0,
|
|
756
|
-
trace,
|
|
757
|
-
lockHeld: true,
|
|
758
|
-
});
|
|
759
|
-
}
|
|
760
|
-
catch (recoveryErr) {
|
|
761
|
-
const recoveryMessage = recoveryErr?.message || String(recoveryErr);
|
|
762
|
-
const fatalMessage = isMissingSessionStateErrorMessage(recoveryMessage)
|
|
763
|
-
? buildUnrecoverableSessionLossMessage(input.sessionId, stripMissingSessionStatePrefix(recoveryMessage))
|
|
764
|
-
: buildUnrecoverableSessionLossMessage(input.sessionId, recoveryMessage);
|
|
765
|
-
trace(`session=${input.sessionId} lossy replay session recreation failed error=${fatalMessage}`);
|
|
766
|
-
return await failForMissingState(fatalMessage);
|
|
767
|
-
}
|
|
768
|
-
effectivePrompt = mergePromptSections([LOSSY_SESSION_REPLAY_NOTICE, input.prompt]) || input.prompt;
|
|
769
742
|
}
|
|
770
|
-
|
|
771
|
-
|
|
743
|
+
: null;
|
|
744
|
+
let lifecycleBaseVersion = 0;
|
|
745
|
+
let lifecycleRehydrated = false;
|
|
746
|
+
// Protocol-native persistence stats: the legacy dehydrate/hydrate
|
|
747
|
+
// activities no longer run, so the preamble/commit paths feed the
|
|
748
|
+
// same CMS summary + events the Stats pane reads. Best-effort only.
|
|
749
|
+
const recordLifecycleHydration = async (version) => {
|
|
750
|
+
if (!catalog)
|
|
751
|
+
return;
|
|
752
|
+
await cmsRetryBestEffort(`runTurn.lifecycleHydrated session=${input.sessionId}`, async () => {
|
|
753
|
+
await catalog.upsertSessionMetricSummary(input.sessionId, {
|
|
754
|
+
hydrationCountIncrement: 1,
|
|
755
|
+
lastHydratedAt: true,
|
|
756
|
+
});
|
|
757
|
+
await catalog.recordEvents(input.sessionId, [{
|
|
758
|
+
eventType: "session.hydrated",
|
|
759
|
+
data: { version, protocol: "lifecycle" },
|
|
760
|
+
}], workerNodeId);
|
|
761
|
+
}, (msg) => activityCtx.traceInfo(msg));
|
|
762
|
+
};
|
|
763
|
+
if (lifecycle) {
|
|
764
|
+
const pre = await runTurnPreamble(lifecycle);
|
|
765
|
+
if (pre.kind === "already-committed") {
|
|
766
|
+
activityCtx.traceInfo(`[runTurn] session=${input.sessionId} already-committed recovery at v${pre.version}; ` +
|
|
767
|
+
`returning stored result without re-running the turn`);
|
|
768
|
+
await recordLifecycleHydration(pre.version);
|
|
769
|
+
return { ...pre.result, snapshotVersion: pre.version };
|
|
772
770
|
}
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
if (
|
|
776
|
-
|
|
777
|
-
if (inlineSdkClientPromise)
|
|
778
|
-
return await inlineSdkClientPromise;
|
|
779
|
-
if (!storeUrl)
|
|
780
|
-
throw new Error("No storeUrl — cannot create PilotSwarmClient");
|
|
781
|
-
inlineSdkClientPromise = (async () => {
|
|
782
|
-
const startedClient = new PilotSwarmClient(internalClientConfig());
|
|
783
|
-
await startedClient.start();
|
|
784
|
-
inlineSdkClient = startedClient;
|
|
785
|
-
return startedClient;
|
|
786
|
-
})();
|
|
787
|
-
try {
|
|
788
|
-
return await inlineSdkClientPromise;
|
|
771
|
+
lifecycleBaseVersion = pre.baseVersion;
|
|
772
|
+
lifecycleRehydrated = pre.kind === "hydrated";
|
|
773
|
+
if (lifecycleRehydrated) {
|
|
774
|
+
await recordLifecycleHydration(pre.baseVersion);
|
|
789
775
|
}
|
|
790
|
-
|
|
791
|
-
|
|
776
|
+
if (pre.kind === "hydrated" && pre.regressed && catalog) {
|
|
777
|
+
// Store-wins anomaly: the store was BELOW this worker's marker
|
|
778
|
+
// (backup restore / data loss). The store still won — we
|
|
779
|
+
// hydrated it — but surface the regression. Best-effort (P7).
|
|
780
|
+
const regressed = pre.regressed;
|
|
781
|
+
await cmsRetryBestEffort(`runTurn.recordEvent snapshot-regressed session=${input.sessionId}`, () => catalog.recordEvents(input.sessionId, [{
|
|
782
|
+
eventType: "session.snapshot_regressed",
|
|
783
|
+
data: {
|
|
784
|
+
markerVersion: regressed.markerVersion,
|
|
785
|
+
storeVersion: regressed.storeVersion,
|
|
786
|
+
hydratedVersion: pre.baseVersion,
|
|
787
|
+
message: "Snapshot store version was below the worker's local marker (restore from an older backup, or store data loss); store wins — hydrated the stored version.",
|
|
788
|
+
},
|
|
789
|
+
}], workerNodeId), (msg) => activityCtx.traceInfo(msg));
|
|
792
790
|
}
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
const sanitizeAutonomousSystemSessionFilters = (args) => {
|
|
802
|
-
if (!args || !isAutonomousSystemTurn())
|
|
803
|
-
return args;
|
|
804
|
-
if (!args.owner_query && !args.owner_kind)
|
|
805
|
-
return args;
|
|
806
|
-
const sanitized = { ...args };
|
|
807
|
-
delete sanitized.owner_query;
|
|
808
|
-
delete sanitized.owner_kind;
|
|
809
|
-
return sanitized;
|
|
810
|
-
};
|
|
811
|
-
const isReadOnlyTuner = () => runConfig.agentIdentity === "agent-tuner";
|
|
812
|
-
const normalizeAgentLookup = (value) => (value || "").toLowerCase().replace(/[^a-z0-9]+/g, "");
|
|
813
|
-
const resolveAgentConfigInline = (agentName) => {
|
|
814
|
-
const agents = [
|
|
815
|
-
...(userAgents ?? []).map(a => ({ ...a, system: false, creatable: true })),
|
|
816
|
-
...(systemAgents ?? []).map(a => ({ ...a, creatable: false })),
|
|
817
|
-
];
|
|
818
|
-
let lookupNamespace;
|
|
819
|
-
let rawName = agentName;
|
|
820
|
-
if (agentName.includes(":")) {
|
|
821
|
-
const parts = agentName.split(":");
|
|
822
|
-
lookupNamespace = parts[0];
|
|
823
|
-
rawName = parts.slice(1).join(":");
|
|
791
|
+
if (pre.kind === "fresh" && pre.lossy && catalog) {
|
|
792
|
+
await cmsRetryBestEffort(`runTurn.recordEvent snapshot-store-empty session=${input.sessionId}`, () => catalog.recordEvents(input.sessionId, [{
|
|
793
|
+
eventType: "session.snapshot_store_empty",
|
|
794
|
+
data: {
|
|
795
|
+
expectedVersion: input.snapshot.expectedVersion ?? 0,
|
|
796
|
+
message: "Snapshot store held no data for a session with committed turns; falling back to fresh-session recovery.",
|
|
797
|
+
},
|
|
798
|
+
}], workerNodeId), (msg) => activityCtx.traceInfo(msg));
|
|
824
799
|
}
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
const sdkClient = await getInlineClient();
|
|
836
|
-
const sessions = await sdkClient.listSessions();
|
|
837
|
-
const directChildren = sessions.filter(s => s.parentSessionId === input.sessionId);
|
|
838
|
-
return await Promise.all(directChildren.map(async (child) => {
|
|
839
|
-
const info = await sdkClient._getSessionInfo(child.sessionId);
|
|
840
|
-
const outcome = catalog ? await catalog.getChildOutcome(child.sessionId).catch(() => null) : null;
|
|
841
|
-
const outcomeResult = normalizeJsonObject(outcome?.resultJson?.current);
|
|
842
|
-
const contractCurrent = normalizeJsonObject(outcome?.contractJson?.current);
|
|
843
|
-
return {
|
|
844
|
-
orchId: `session-${child.sessionId}`,
|
|
845
|
-
sessionId: child.sessionId,
|
|
846
|
-
title: info.title ?? child.title,
|
|
847
|
-
status: info.status,
|
|
848
|
-
iterations: info.iterations ?? child.iterations ?? 0,
|
|
849
|
-
parentSessionId: child.parentSessionId,
|
|
850
|
-
isSystem: child.isSystem ?? info.isSystem ?? false,
|
|
851
|
-
agentId: child.agentId ?? info.agentId,
|
|
852
|
-
result: outcome?.summary ?? (typeof outcomeResult?.summary === "string" ? outcomeResult.summary : info.result),
|
|
853
|
-
contract: contractCurrent ?? undefined,
|
|
854
|
-
contractStatus: outcome?.contractJson ? "contracted" : undefined,
|
|
855
|
-
verdict: outcome?.verdict ?? undefined,
|
|
856
|
-
contractViolations: Array.isArray(outcomeResult?.contractViolations) ? outcomeResult.contractViolations : undefined,
|
|
857
|
-
error: info.error,
|
|
858
|
-
};
|
|
859
|
-
}));
|
|
860
|
-
};
|
|
861
|
-
const resolveManagedChild = async (agentId) => {
|
|
862
|
-
const targetOrchId = agentId.startsWith("session-") ? agentId : `session-${agentId}`;
|
|
863
|
-
const children = await loadDirectChildSessions();
|
|
864
|
-
const child = children.find(entry => entry.orchId === targetOrchId);
|
|
865
|
-
if (!child) {
|
|
866
|
-
throw new Error(`agent "${targetOrchId}" not found. Known agents: ${children.filter(entry => !entry.isSystem).map(entry => entry.orchId).join(", ") || "none"}`);
|
|
800
|
+
}
|
|
801
|
+
const executeTurnBody = async () => {
|
|
802
|
+
let session = null;
|
|
803
|
+
let effectivePrompt = input.prompt;
|
|
804
|
+
try {
|
|
805
|
+
session = await sessionManager.getOrCreate(input.sessionId, runConfig, {
|
|
806
|
+
turnIndex: input.turnIndex,
|
|
807
|
+
trace,
|
|
808
|
+
lockHeld: true,
|
|
809
|
+
});
|
|
867
810
|
}
|
|
868
|
-
|
|
869
|
-
|
|
811
|
+
catch (err) {
|
|
812
|
+
const message = err?.message || String(err);
|
|
813
|
+
if (isMissingSessionStateErrorMessage(message) || isLiveSessionLostErrorMessage(message)) {
|
|
814
|
+
const detail = isMissingSessionStateErrorMessage(message)
|
|
815
|
+
? stripMissingSessionStatePrefix(message)
|
|
816
|
+
: message;
|
|
817
|
+
// ── Lifecycle store recovery BEFORE lossy replay ─────────
|
|
818
|
+
// The store may hold a perfect committed snapshot (e.g. the
|
|
819
|
+
// SDK refused to resume from locally damaged files). Restore
|
|
820
|
+
// it and retry — lossy replay (whose turn-0 reset DELETES
|
|
821
|
+
// the store snapshot) is only reachable when the store is
|
|
822
|
+
// empty, making that delete a no-op.
|
|
823
|
+
if (lifecycle) {
|
|
824
|
+
try {
|
|
825
|
+
const recoveredVersion = await attemptStoreRecovery(lifecycle);
|
|
826
|
+
if (recoveredVersion != null) {
|
|
827
|
+
trace(`session=${input.sessionId} restored committed snapshot v${recoveredVersion} ` +
|
|
828
|
+
`after getOrCreate failure (${detail.slice(0, 120)}); retrying resume`);
|
|
829
|
+
session = await sessionManager.getOrCreate(input.sessionId, runConfig, {
|
|
830
|
+
turnIndex: input.turnIndex,
|
|
831
|
+
trace,
|
|
832
|
+
lockHeld: true,
|
|
833
|
+
});
|
|
834
|
+
lifecycleBaseVersion = recoveredVersion;
|
|
835
|
+
lifecycleRehydrated = true;
|
|
836
|
+
}
|
|
837
|
+
}
|
|
838
|
+
catch (storeRecoveryErr) {
|
|
839
|
+
trace(`session=${input.sessionId} snapshot-store recovery failed: ` +
|
|
840
|
+
`${storeRecoveryErr?.message ?? storeRecoveryErr}; falling back to lossy replay`);
|
|
841
|
+
session = null;
|
|
842
|
+
}
|
|
843
|
+
}
|
|
844
|
+
if (session) {
|
|
845
|
+
// Recovered losslessly — skip the lossy replay machinery.
|
|
846
|
+
}
|
|
847
|
+
else {
|
|
848
|
+
trace(`session=${input.sessionId} missing resumable state before turn ${input.turnIndex ?? "unknown"}; ` +
|
|
849
|
+
"starting lossy fresh-session replay");
|
|
850
|
+
await recordLossyHandoffEvent(catalog, input.sessionId, workerNodeId, {
|
|
851
|
+
cause: "missing_resumable_state_before_run_turn",
|
|
852
|
+
message: buildLossyReplayMessage(input.sessionId, detail),
|
|
853
|
+
detail,
|
|
854
|
+
error: message,
|
|
855
|
+
recoveryMode: "fresh_session_replay",
|
|
856
|
+
nextStep: "replay_pending_turn_with_recreated_copilot_session",
|
|
857
|
+
...(input.turnIndex != null ? { iteration: input.turnIndex } : {}),
|
|
858
|
+
}, (failureMessage) => activityCtx.traceInfo(`[runTurn] ${failureMessage}`));
|
|
859
|
+
if (catalog) {
|
|
860
|
+
await cmsRetryBestEffort(`runTurn.recordEvent system.message-lossy-replay session=${input.sessionId}`, () => catalog.recordEvents(input.sessionId, [{
|
|
861
|
+
eventType: "system.message",
|
|
862
|
+
data: {
|
|
863
|
+
content: "The runtime is replaying this turn after a worker restart lost the live Copilot session state before durable dehydrate completed. " +
|
|
864
|
+
"Some recent work may be missing or partially executed.",
|
|
865
|
+
},
|
|
866
|
+
}], workerNodeId), (msg) => activityCtx.traceInfo(msg));
|
|
867
|
+
}
|
|
868
|
+
try {
|
|
869
|
+
session = await sessionManager.getOrCreate(input.sessionId, runConfig, {
|
|
870
|
+
turnIndex: 0,
|
|
871
|
+
trace,
|
|
872
|
+
lockHeld: true,
|
|
873
|
+
});
|
|
874
|
+
}
|
|
875
|
+
catch (recoveryErr) {
|
|
876
|
+
const recoveryMessage = recoveryErr?.message || String(recoveryErr);
|
|
877
|
+
const fatalMessage = isMissingSessionStateErrorMessage(recoveryMessage)
|
|
878
|
+
? buildUnrecoverableSessionLossMessage(input.sessionId, stripMissingSessionStatePrefix(recoveryMessage))
|
|
879
|
+
: buildUnrecoverableSessionLossMessage(input.sessionId, recoveryMessage);
|
|
880
|
+
trace(`session=${input.sessionId} lossy replay session recreation failed error=${fatalMessage}`);
|
|
881
|
+
return await failForMissingState(fatalMessage);
|
|
882
|
+
}
|
|
883
|
+
effectivePrompt = mergePromptSections([LOSSY_SESSION_REPLAY_NOTICE, input.prompt]) || input.prompt;
|
|
884
|
+
}
|
|
885
|
+
}
|
|
886
|
+
else {
|
|
887
|
+
throw err;
|
|
888
|
+
}
|
|
870
889
|
}
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
890
|
+
// ── Lifecycle p3: sentinel marks the dir as mid-mutation from here
|
|
891
|
+
// until the post-turn commit clears it. Written after getOrCreate so
|
|
892
|
+
// the turn-0 reset path cannot wipe it, before any body mutation.
|
|
893
|
+
if (lifecycle) {
|
|
894
|
+
writeTurnSentinel(path.join(sessionManager.getSessionStateDir(), input.sessionId), input.snapshot.turnKey);
|
|
895
|
+
if (lifecycleRehydrated) {
|
|
896
|
+
effectivePrompt = mergePromptSections([REHYDRATED_SESSION_NOTICE, effectivePrompt]) || effectivePrompt;
|
|
897
|
+
}
|
|
898
|
+
}
|
|
899
|
+
const getInlineClient = async () => {
|
|
900
|
+
if (inlineSdkClient)
|
|
901
|
+
return inlineSdkClient;
|
|
902
|
+
if (inlineSdkClientPromise)
|
|
903
|
+
return await inlineSdkClientPromise;
|
|
904
|
+
if (!storeUrl)
|
|
905
|
+
throw new Error("No storeUrl — cannot create PilotSwarmClient");
|
|
906
|
+
inlineSdkClientPromise = (async () => {
|
|
907
|
+
const startedClient = new PilotSwarmClient(internalClientConfig());
|
|
908
|
+
await startedClient.start();
|
|
909
|
+
inlineSdkClient = startedClient;
|
|
910
|
+
return startedClient;
|
|
911
|
+
})();
|
|
875
912
|
try {
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
913
|
+
return await inlineSdkClientPromise;
|
|
914
|
+
}
|
|
915
|
+
finally {
|
|
916
|
+
inlineSdkClientPromise = null;
|
|
917
|
+
}
|
|
918
|
+
};
|
|
919
|
+
const isAutonomousSystemTurn = () => {
|
|
920
|
+
if (!SYSTEM_AGENT_IDS.has(runConfig.agentIdentity || ""))
|
|
921
|
+
return false;
|
|
922
|
+
if (input.bootstrap === true)
|
|
923
|
+
return true;
|
|
924
|
+
return /^\s*\[SYSTEM:/i.test(String(input.prompt ?? ""));
|
|
925
|
+
};
|
|
926
|
+
const sanitizeAutonomousSystemSessionFilters = (args) => {
|
|
927
|
+
if (!args || !isAutonomousSystemTurn())
|
|
928
|
+
return args;
|
|
929
|
+
if (!args.owner_query && !args.owner_kind)
|
|
930
|
+
return args;
|
|
931
|
+
const sanitized = { ...args };
|
|
932
|
+
delete sanitized.owner_query;
|
|
933
|
+
delete sanitized.owner_kind;
|
|
934
|
+
return sanitized;
|
|
935
|
+
};
|
|
936
|
+
const isReadOnlyTuner = () => runConfig.agentIdentity === "agent-tuner";
|
|
937
|
+
const normalizeAgentLookup = (value) => (value || "").toLowerCase().replace(/[^a-z0-9]+/g, "");
|
|
938
|
+
const resolveAgentConfigInline = (agentName) => {
|
|
939
|
+
const agents = [
|
|
940
|
+
...(userAgents ?? []).map(a => ({ ...a, system: false, creatable: true })),
|
|
941
|
+
...(systemAgents ?? []).map(a => ({ ...a, creatable: false })),
|
|
942
|
+
];
|
|
943
|
+
let lookupNamespace;
|
|
944
|
+
let rawName = agentName;
|
|
945
|
+
if (agentName.includes(":")) {
|
|
946
|
+
const parts = agentName.split(":");
|
|
947
|
+
lookupNamespace = parts[0];
|
|
948
|
+
rawName = parts.slice(1).join(":");
|
|
949
|
+
}
|
|
950
|
+
const lookup = normalizeAgentLookup(rawName);
|
|
951
|
+
const lookupBase = lookup.replace(/agent$/, "");
|
|
952
|
+
return agents.find(a => {
|
|
953
|
+
if (lookupNamespace && normalizeAgentLookup(a.namespace) !== normalizeAgentLookup(lookupNamespace))
|
|
954
|
+
return false;
|
|
955
|
+
const candidates = [a.name, a.id, a.title].map(normalizeAgentLookup).filter(Boolean);
|
|
956
|
+
return candidates.includes(lookup) || (lookupBase && candidates.includes(lookupBase));
|
|
957
|
+
}) ?? null;
|
|
958
|
+
};
|
|
959
|
+
const loadDirectChildSessions = async () => {
|
|
960
|
+
const sdkClient = await getInlineClient();
|
|
961
|
+
const sessions = await sdkClient.listSessions();
|
|
962
|
+
const directChildren = sessions.filter(s => s.parentSessionId === input.sessionId);
|
|
963
|
+
return await Promise.all(directChildren.map(async (child) => {
|
|
964
|
+
const info = await sdkClient._getSessionInfo(child.sessionId);
|
|
965
|
+
const outcome = catalog ? await catalog.getChildOutcome(child.sessionId).catch(() => null) : null;
|
|
966
|
+
const outcomeResult = normalizeJsonObject(outcome?.resultJson?.current);
|
|
967
|
+
const contractCurrent = normalizeJsonObject(outcome?.contractJson?.current);
|
|
968
|
+
return {
|
|
969
|
+
orchId: `session-${child.sessionId}`,
|
|
970
|
+
sessionId: child.sessionId,
|
|
971
|
+
title: info.title ?? child.title,
|
|
972
|
+
status: info.status,
|
|
973
|
+
iterations: info.iterations ?? child.iterations ?? 0,
|
|
974
|
+
parentSessionId: child.parentSessionId,
|
|
975
|
+
isSystem: child.isSystem ?? info.isSystem ?? false,
|
|
976
|
+
agentId: child.agentId ?? info.agentId,
|
|
977
|
+
result: outcome?.summary ?? (typeof outcomeResult?.summary === "string" ? outcomeResult.summary : info.result),
|
|
978
|
+
contract: contractCurrent ?? undefined,
|
|
979
|
+
contractStatus: outcome?.contractJson ? "contracted" : undefined,
|
|
980
|
+
verdict: outcome?.verdict ?? undefined,
|
|
981
|
+
contractViolations: Array.isArray(outcomeResult?.contractViolations) ? outcomeResult.contractViolations : undefined,
|
|
982
|
+
error: info.error,
|
|
983
|
+
};
|
|
984
|
+
}));
|
|
985
|
+
};
|
|
986
|
+
const resolveManagedChild = async (agentId) => {
|
|
987
|
+
const targetOrchId = agentId.startsWith("session-") ? agentId : `session-${agentId}`;
|
|
988
|
+
const children = await loadDirectChildSessions();
|
|
989
|
+
const child = children.find(entry => entry.orchId === targetOrchId);
|
|
990
|
+
if (!child) {
|
|
991
|
+
throw new Error(`agent "${targetOrchId}" not found. Known agents: ${children.filter(entry => !entry.isSystem).map(entry => entry.orchId).join(", ") || "none"}`);
|
|
992
|
+
}
|
|
993
|
+
if (child.isSystem) {
|
|
994
|
+
throw new Error(`agent "${targetOrchId}" is a worker-managed system agent and is not a controllable spawned sub-agent`);
|
|
995
|
+
}
|
|
996
|
+
return child;
|
|
997
|
+
};
|
|
998
|
+
const controlToolBridge = {
|
|
999
|
+
spawnAgent: async (args) => {
|
|
1000
|
+
try {
|
|
1001
|
+
const childNestingLevel = (input.nestingLevel ?? 0) + 1;
|
|
1002
|
+
if (childNestingLevel > MAX_NESTING_LEVEL) {
|
|
1003
|
+
return `[SYSTEM: spawn_agent failed — you are already at nesting level ${input.nestingLevel ?? 0} (max ${MAX_NESTING_LEVEL}). ` +
|
|
1004
|
+
`Sub-agents at this depth cannot spawn further sub-agents. Handle the task directly instead.]`;
|
|
1005
|
+
}
|
|
1006
|
+
const existingChildren = (await loadDirectChildSessions()).filter(child => !child.isSystem);
|
|
1007
|
+
const activeCount = existingChildren.filter(child => child.status === "running").length;
|
|
1008
|
+
if (activeCount >= MAX_SUB_AGENTS) {
|
|
1009
|
+
return `[SYSTEM: spawn_agent failed — you already have ${activeCount} running sub-agents (max ${MAX_SUB_AGENTS}). ` +
|
|
1010
|
+
`Wait for some to complete before spawning more.]`;
|
|
1011
|
+
}
|
|
1012
|
+
let agentTask = args.task || "";
|
|
1013
|
+
let agentSystemMessage = args.system_message;
|
|
1014
|
+
let agentToolNames = args.tool_names;
|
|
1015
|
+
let agentModel = args.model;
|
|
1016
|
+
let agentReasoningEffort = args.reasoning_effort;
|
|
1017
|
+
let agentIsSystem = false;
|
|
1018
|
+
const explicitAgentTitle = typeof args.title === "string" && args.title.trim() ? args.title.trim() : undefined;
|
|
1019
|
+
let agentTitle = explicitAgentTitle;
|
|
1020
|
+
let agentTitleIsExplicit = Boolean(explicitAgentTitle);
|
|
1021
|
+
let agentId;
|
|
1022
|
+
let agentSplash;
|
|
1023
|
+
let agentSplashMobile;
|
|
1024
|
+
let boundAgentName;
|
|
1025
|
+
let promptLayeringKind;
|
|
1026
|
+
let resolvedAgentName = args.agent_name;
|
|
1027
|
+
const applyAgentDef = (agentDef, useDefinitionDefaults = false) => {
|
|
1028
|
+
agentTask = useDefinitionDefaults
|
|
1029
|
+
? (agentDef.initialPrompt || `You are the ${agentDef.name} agent. Begin your work.`)
|
|
1030
|
+
: (args.task || agentDef.initialPrompt || `You are the ${agentDef.name} agent. Begin your work.`);
|
|
1031
|
+
agentSystemMessage = useDefinitionDefaults ? undefined : args.system_message;
|
|
1032
|
+
agentToolNames = useDefinitionDefaults
|
|
1033
|
+
? (agentDef.tools ?? undefined)
|
|
1034
|
+
: (args.tool_names ?? agentDef.tools ?? undefined);
|
|
1035
|
+
agentIsSystem = agentDef.system ?? false;
|
|
1036
|
+
if (!agentTitleIsExplicit)
|
|
1037
|
+
agentTitle = agentDef.title;
|
|
1038
|
+
agentId = agentDef.id ?? resolvedAgentName;
|
|
1039
|
+
agentSplash = agentDef.splash;
|
|
1040
|
+
agentSplashMobile = agentDef.splashMobile;
|
|
1041
|
+
boundAgentName = agentDef.name;
|
|
1042
|
+
promptLayeringKind = agentDef.promptLayerKind
|
|
1043
|
+
?? (agentDef.system
|
|
1044
|
+
? ((agentDef.namespace || "pilotswarm") === "pilotswarm"
|
|
1045
|
+
? "pilotswarm-system-agent"
|
|
1046
|
+
: "app-system-agent")
|
|
1047
|
+
: "app-agent");
|
|
1048
|
+
};
|
|
1049
|
+
if (resolvedAgentName) {
|
|
1050
|
+
const agentDef = resolveAgentConfigInline(resolvedAgentName);
|
|
1051
|
+
if (!agentDef) {
|
|
1052
|
+
return `[SYSTEM: spawn_agent failed — agent "${resolvedAgentName}" not found. Use ps_list_agents to see available agents.]`;
|
|
1053
|
+
}
|
|
1054
|
+
if (agentDef.system && agentDef.creatable === false) {
|
|
1055
|
+
return `[SYSTEM: spawn_agent failed — agent "${resolvedAgentName}" is a worker-managed system agent and cannot be spawned from a session. ` +
|
|
1056
|
+
`If it is missing, the workers likely need to be restarted.]`;
|
|
1057
|
+
}
|
|
1058
|
+
applyAgentDef(agentDef, resolvedAgentName !== args.agent_name);
|
|
1059
|
+
}
|
|
1060
|
+
if (agentModel && !agentModel.includes(":")) {
|
|
1061
|
+
return `[SYSTEM: spawn_agent failed — model "${agentModel}" is not allowed. ` +
|
|
1062
|
+
`When overriding a sub-agent model, first call list_available_models and then use the exact provider:model value from that list. ` +
|
|
1063
|
+
`If you are unsure, omit model so the sub-agent inherits your current model.]`;
|
|
1064
|
+
}
|
|
1065
|
+
// v1.0.49: same-name duplicate spawns are allowed. The
|
|
1066
|
+
// global MAX_SUB_AGENTS cap and per-spawn nesting limit
|
|
1067
|
+
// still apply. The parent is responsible for closing
|
|
1068
|
+
// each instance with complete_agent / cancel_agent /
|
|
1069
|
+
// delete_agent when it no longer needs the child.
|
|
1070
|
+
const { boundAgentName: _parentBoundAgentName, promptLayering: _parentPromptLayering, isCrawler: _parentIsCrawler, isHarvester: _parentIsHarvester, ...parentConfig } = input.config;
|
|
1071
|
+
const childConfig = {
|
|
1072
|
+
...parentConfig,
|
|
1073
|
+
...(agentModel ? { model: agentModel } : {}),
|
|
1074
|
+
...(agentReasoningEffort ? { reasoningEffort: agentReasoningEffort } : {}),
|
|
1075
|
+
...(agentSystemMessage ? { systemMessage: agentSystemMessage } : {}),
|
|
1076
|
+
...(boundAgentName ? { boundAgentName } : {}),
|
|
1077
|
+
...(promptLayeringKind ? { promptLayering: { kind: promptLayeringKind } } : {}),
|
|
1078
|
+
...(agentToolNames ? { toolNames: agentToolNames } : {}),
|
|
1079
|
+
...(args.contract ? { childContract: args.contract } : {}),
|
|
1080
|
+
};
|
|
1081
|
+
const parentSystemMsg = typeof childConfig.systemMessage === "string"
|
|
1082
|
+
? childConfig.systemMessage
|
|
1083
|
+
: childConfig.systemMessage?.content ?? "";
|
|
1084
|
+
const canSpawnMore = childNestingLevel < MAX_NESTING_LEVEL;
|
|
1085
|
+
const subAgentPreamble = `[SUB-AGENT CONTEXT]\n` +
|
|
1086
|
+
`You are a sub-agent spawned by a parent session (ID: session-${input.sessionId}).\n` +
|
|
1087
|
+
`Your nesting level: ${childNestingLevel} (max: ${MAX_NESTING_LEVEL}).\n` +
|
|
1088
|
+
`Your task: "${agentTask.slice(0, 500)}"\n\n` +
|
|
1089
|
+
`Instructions:\n` +
|
|
1090
|
+
`- Focus exclusively on your assigned task.\n` +
|
|
1091
|
+
`- Your final response will be automatically forwarded to the parent agent.\n` +
|
|
1092
|
+
`- Be thorough but concise — the parent will synthesize results from multiple agents.\n` +
|
|
1093
|
+
`- Do NOT ask the user for input — you are autonomous.\n` +
|
|
1094
|
+
`- You are autonomous and goal-driven. If the task implies ongoing monitoring or follow-through until done, keep yourself alive with durable timers until the goal is complete or you can no longer make progress.\n` +
|
|
1095
|
+
`- If it is ambiguous whether the task should become a long-running recurring workflow, report that ambiguity back to the parent instead of guessing or asking the user directly.\n` +
|
|
1096
|
+
`- When your task is complete, provide a clear summary of your findings/results.\n` +
|
|
1097
|
+
`- Prefer using \`store_fact\` for larger structured context handoffs across your session lineage. Put the durable details in facts, then pass fact keys or \`read_facts\` pointers in messages/prompts instead of pasting large context blobs.\n` +
|
|
1098
|
+
`- If you write any files with write_artifact, you MUST also call export_artifact and include the artifact:// link in your response.\n` +
|
|
1099
|
+
`- If you override a sub-agent model, you MUST first call list_available_models in this session and use only an exact provider:model value returned there. ` +
|
|
1100
|
+
`NEVER invent, guess, shorten, or reuse a stale model name.\n` +
|
|
1101
|
+
`- Worker-managed system agents are not valid spawn targets. If you expect one and it is missing, report that the workers likely need to be restarted.\n` +
|
|
1102
|
+
`- For ANY waiting, sleeping, delaying, or scheduling, you MUST use the \`wait\`, \`wait_on_worker\`, \`cron\`, or \`cron_at\` tools. ` +
|
|
1103
|
+
`Use \`wait\` or \`wait_on_worker\` for one-shot delays. Use \`cron\` for fixed recurring intervals and \`cron_at\` for wall-clock schedules. ` +
|
|
1104
|
+
`Do NOT implement wall-clock schedules by waking every N minutes to check the clock; use \`cron_at\` with an explicit IANA timezone instead. ` +
|
|
1105
|
+
`Do NOT burn tokens polling inside one LLM turn; after a brief immediate re-check at most, yield with a durable timer. ` +
|
|
1106
|
+
`NEVER use setTimeout, sleep, setInterval, or any other timing mechanism.\n` +
|
|
1107
|
+
(canSpawnMore
|
|
1108
|
+
? `- If your parent task explicitly asks you to spawn sub-agents, delegate, fan out, or parallelize work, you SHOULD do so within runtime limits instead of collapsing the task into a direct answer. ` +
|
|
1109
|
+
`If delegation was not explicitly requested, use your judgment and avoid unnecessary fan-out. ` +
|
|
1110
|
+
`You have ${MAX_NESTING_LEVEL - childNestingLevel} level(s) of nesting remaining. After spawning, call wait_for_agents to block until they finish.\n`
|
|
1111
|
+
: `- You CANNOT spawn sub-agents — you are at the maximum nesting depth. Handle everything directly.\n`);
|
|
1112
|
+
childConfig.systemMessage = subAgentPreamble + (parentSystemMsg ? "\n\n" + parentSystemMsg : "");
|
|
1113
|
+
const sdkClient = await getInlineClient();
|
|
1114
|
+
const normalizedModel = sessionManager.normalizeModelRef(childConfig.model);
|
|
1115
|
+
if (normalizedModel)
|
|
1116
|
+
childConfig.model = normalizedModel;
|
|
1117
|
+
const childSession = await sdkClient.createSession({
|
|
1118
|
+
parentSessionId: input.sessionId,
|
|
1119
|
+
nestingLevel: childNestingLevel,
|
|
1120
|
+
model: childConfig.model,
|
|
1121
|
+
reasoningEffort: childConfig.reasoningEffort,
|
|
1122
|
+
systemMessage: childConfig.systemMessage,
|
|
1123
|
+
boundAgentName: childConfig.boundAgentName,
|
|
1124
|
+
promptLayering: childConfig.promptLayering,
|
|
1125
|
+
toolNames: childConfig.toolNames,
|
|
1126
|
+
waitThreshold: childConfig.waitThreshold,
|
|
1127
|
+
agentId,
|
|
1128
|
+
});
|
|
1129
|
+
if (catalog) {
|
|
1130
|
+
const meta = {};
|
|
1131
|
+
if (agentTitle) {
|
|
1132
|
+
meta.title = (agentTitleIsExplicit || agentIsSystem)
|
|
1133
|
+
? agentTitle
|
|
1134
|
+
: `${agentTitle}: ${childSession.sessionId.slice(0, 8)}`;
|
|
1135
|
+
}
|
|
1136
|
+
if (agentId)
|
|
1137
|
+
meta.agentId = agentId;
|
|
1138
|
+
if (agentSplash)
|
|
1139
|
+
meta.splash = agentSplash;
|
|
1140
|
+
if (agentSplashMobile)
|
|
1141
|
+
meta.splashMobile = agentSplashMobile;
|
|
1142
|
+
if (Object.keys(meta).length > 0) {
|
|
1143
|
+
// Best-effort: child has been created and is about to be
|
|
1144
|
+
// sent its bootstrap. A failed meta update means the row
|
|
1145
|
+
// is missing title/agentId/splash, not that the spawn
|
|
1146
|
+
// failed. Don't escalate to a thrown spawn_agent.
|
|
1147
|
+
const capturedMeta = meta;
|
|
1148
|
+
await cmsRetryBestEffort(`runTurn.spawn.updateSession meta session=${childSession.sessionId}`, () => catalog.updateSession(childSession.sessionId, capturedMeta), (msg) => activityCtx.traceInfo(msg));
|
|
1149
|
+
}
|
|
1150
|
+
const contractJson = buildContractJson(args.contract, input.sessionId, childSession.sessionId);
|
|
1151
|
+
if (contractJson) {
|
|
1152
|
+
await cmsRetryCritical(`runTurn.spawn.upsertChildOutcome contract child=${childSession.sessionId}`, () => catalog.upsertChildOutcome({
|
|
1153
|
+
childSessionId: childSession.sessionId,
|
|
1154
|
+
parentSessionId: input.sessionId,
|
|
1155
|
+
contractJson,
|
|
1156
|
+
}), (msg) => activityCtx.traceInfo(msg));
|
|
1157
|
+
}
|
|
1158
|
+
}
|
|
1159
|
+
await childSession.send(agentTask, { bootstrap: true });
|
|
1160
|
+
if (catalog) {
|
|
1161
|
+
await cmsRetryBestEffort(`runTurn.spawn.recordEvent agent_spawned session=${input.sessionId}`, () => catalog.recordEvents(input.sessionId, [{
|
|
1162
|
+
eventType: "session.agent_spawned",
|
|
1163
|
+
data: { childSessionId: childSession.sessionId, agentId: agentId || undefined, task: agentTask.slice(0, 500) },
|
|
1164
|
+
}], workerNodeId), (msg) => activityCtx.traceInfo(msg));
|
|
1165
|
+
}
|
|
1166
|
+
const childOrchId = `session-${childSession.sessionId}`;
|
|
1167
|
+
return `[SYSTEM: Sub-agent spawned successfully.\n` +
|
|
1168
|
+
` Agent ID: ${childOrchId}\n` +
|
|
1169
|
+
` ${resolvedAgentName ? `Agent: ${resolvedAgentName}\n ` : ``}Task: "${agentTask.slice(0, 200)}"\n` +
|
|
1170
|
+
` The agent is now running autonomously. Continue your work in this SAME turn and keep following the user's remaining steps. ` +
|
|
1171
|
+
`Do NOT stop just because the child started. If your plan says to pause, call wait or wait_for_agents explicitly. ` +
|
|
1172
|
+
`You can also use check_agents to poll status, ` +
|
|
1173
|
+
`or message_agent to send instructions.]`;
|
|
880
1174
|
}
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
if (activeCount >= MAX_SUB_AGENTS) {
|
|
884
|
-
return `[SYSTEM: spawn_agent failed — you already have ${activeCount} running sub-agents (max ${MAX_SUB_AGENTS}). ` +
|
|
885
|
-
`Wait for some to complete before spawning more.]`;
|
|
1175
|
+
catch (err) {
|
|
1176
|
+
return `[SYSTEM: spawn_agent failed: ${err?.message || String(err)}]`;
|
|
886
1177
|
}
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
: (args.tool_names ?? agentDef.tools ?? undefined);
|
|
910
|
-
agentIsSystem = agentDef.system ?? false;
|
|
911
|
-
if (!agentTitleIsExplicit)
|
|
912
|
-
agentTitle = agentDef.title;
|
|
913
|
-
agentId = agentDef.id ?? resolvedAgentName;
|
|
914
|
-
agentSplash = agentDef.splash;
|
|
915
|
-
agentSplashMobile = agentDef.splashMobile;
|
|
916
|
-
boundAgentName = agentDef.name;
|
|
917
|
-
promptLayeringKind = agentDef.promptLayerKind
|
|
918
|
-
?? (agentDef.system
|
|
919
|
-
? ((agentDef.namespace || "pilotswarm") === "pilotswarm"
|
|
920
|
-
? "pilotswarm-system-agent"
|
|
921
|
-
: "app-system-agent")
|
|
922
|
-
: "app-agent");
|
|
923
|
-
};
|
|
924
|
-
if (resolvedAgentName) {
|
|
925
|
-
const agentDef = resolveAgentConfigInline(resolvedAgentName);
|
|
926
|
-
if (!agentDef) {
|
|
927
|
-
return `[SYSTEM: spawn_agent failed — agent "${resolvedAgentName}" not found. Use ps_list_agents to see available agents.]`;
|
|
1178
|
+
},
|
|
1179
|
+
setSessionModel: async (args) => {
|
|
1180
|
+
try {
|
|
1181
|
+
const requestedModel = String(args.model || "").trim();
|
|
1182
|
+
if (!requestedModel)
|
|
1183
|
+
return "[SYSTEM: set_session_model failed: model is required.]";
|
|
1184
|
+
if (!storeUrl)
|
|
1185
|
+
return "[SYSTEM: set_session_model failed: no storeUrl is configured.]";
|
|
1186
|
+
const resolved = sessionManager.resolveModelSwitchConfig(requestedModel, "reasoning_effort" in args ? args.reasoning_effort ?? null : undefined);
|
|
1187
|
+
const sdkClient = new PilotSwarmClient(internalClientConfig());
|
|
1188
|
+
try {
|
|
1189
|
+
await sdkClient.start();
|
|
1190
|
+
await sdkClient.duroxideClient.enqueueEvent(`session-${input.sessionId}`, "messages", JSON.stringify({
|
|
1191
|
+
type: "cmd",
|
|
1192
|
+
cmd: "set_model",
|
|
1193
|
+
id: `set-model-tool-${randomUUID()}`,
|
|
1194
|
+
args: {
|
|
1195
|
+
model: resolved.model,
|
|
1196
|
+
reasoningEffort: resolved.reasoningEffort,
|
|
1197
|
+
source: "tool",
|
|
1198
|
+
},
|
|
1199
|
+
}));
|
|
928
1200
|
}
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
`If it is missing, the workers likely need to be restarted.]`;
|
|
1201
|
+
finally {
|
|
1202
|
+
await sdkClient.stop().catch(() => { });
|
|
932
1203
|
}
|
|
933
|
-
|
|
1204
|
+
const effortText = resolved.reasoningEffort ? `:${resolved.reasoningEffort}` : "";
|
|
1205
|
+
return `[SYSTEM: Model switch accepted. Stop this turn now; the runtime will automatically continue on ${resolved.model}${effortText}.]`;
|
|
934
1206
|
}
|
|
935
|
-
|
|
936
|
-
return `[SYSTEM:
|
|
937
|
-
`When overriding a sub-agent model, first call list_available_models and then use the exact provider:model value from that list. ` +
|
|
938
|
-
`If you are unsure, omit model so the sub-agent inherits your current model.]`;
|
|
1207
|
+
catch (err) {
|
|
1208
|
+
return `[SYSTEM: set_session_model failed: ${err?.message || String(err)}]`;
|
|
939
1209
|
}
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
...(args.contract ? { childContract: args.contract } : {}),
|
|
955
|
-
};
|
|
956
|
-
const parentSystemMsg = typeof childConfig.systemMessage === "string"
|
|
957
|
-
? childConfig.systemMessage
|
|
958
|
-
: childConfig.systemMessage?.content ?? "";
|
|
959
|
-
const canSpawnMore = childNestingLevel < MAX_NESTING_LEVEL;
|
|
960
|
-
const subAgentPreamble = `[SUB-AGENT CONTEXT]\n` +
|
|
961
|
-
`You are a sub-agent spawned by a parent session (ID: session-${input.sessionId}).\n` +
|
|
962
|
-
`Your nesting level: ${childNestingLevel} (max: ${MAX_NESTING_LEVEL}).\n` +
|
|
963
|
-
`Your task: "${agentTask.slice(0, 500)}"\n\n` +
|
|
964
|
-
`Instructions:\n` +
|
|
965
|
-
`- Focus exclusively on your assigned task.\n` +
|
|
966
|
-
`- Your final response will be automatically forwarded to the parent agent.\n` +
|
|
967
|
-
`- Be thorough but concise — the parent will synthesize results from multiple agents.\n` +
|
|
968
|
-
`- Do NOT ask the user for input — you are autonomous.\n` +
|
|
969
|
-
`- You are autonomous and goal-driven. If the task implies ongoing monitoring or follow-through until done, keep yourself alive with durable timers until the goal is complete or you can no longer make progress.\n` +
|
|
970
|
-
`- If it is ambiguous whether the task should become a long-running recurring workflow, report that ambiguity back to the parent instead of guessing or asking the user directly.\n` +
|
|
971
|
-
`- When your task is complete, provide a clear summary of your findings/results.\n` +
|
|
972
|
-
`- Prefer using \`store_fact\` for larger structured context handoffs across your session lineage. Put the durable details in facts, then pass fact keys or \`read_facts\` pointers in messages/prompts instead of pasting large context blobs.\n` +
|
|
973
|
-
`- If you write any files with write_artifact, you MUST also call export_artifact and include the artifact:// link in your response.\n` +
|
|
974
|
-
`- If you override a sub-agent model, you MUST first call list_available_models in this session and use only an exact provider:model value returned there. ` +
|
|
975
|
-
`NEVER invent, guess, shorten, or reuse a stale model name.\n` +
|
|
976
|
-
`- Worker-managed system agents are not valid spawn targets. If you expect one and it is missing, report that the workers likely need to be restarted.\n` +
|
|
977
|
-
`- For ANY waiting, sleeping, delaying, or scheduling, you MUST use the \`wait\`, \`wait_on_worker\`, \`cron\`, or \`cron_at\` tools. ` +
|
|
978
|
-
`Use \`wait\` or \`wait_on_worker\` for one-shot delays. Use \`cron\` for fixed recurring intervals and \`cron_at\` for wall-clock schedules. ` +
|
|
979
|
-
`Do NOT implement wall-clock schedules by waking every N minutes to check the clock; use \`cron_at\` with an explicit IANA timezone instead. ` +
|
|
980
|
-
`Do NOT burn tokens polling inside one LLM turn; after a brief immediate re-check at most, yield with a durable timer. ` +
|
|
981
|
-
`NEVER use setTimeout, sleep, setInterval, or any other timing mechanism.\n` +
|
|
982
|
-
(canSpawnMore
|
|
983
|
-
? `- If your parent task explicitly asks you to spawn sub-agents, delegate, fan out, or parallelize work, you SHOULD do so within runtime limits instead of collapsing the task into a direct answer. ` +
|
|
984
|
-
`If delegation was not explicitly requested, use your judgment and avoid unnecessary fan-out. ` +
|
|
985
|
-
`You have ${MAX_NESTING_LEVEL - childNestingLevel} level(s) of nesting remaining. After spawning, call wait_for_agents to block until they finish.\n`
|
|
986
|
-
: `- You CANNOT spawn sub-agents — you are at the maximum nesting depth. Handle everything directly.\n`);
|
|
987
|
-
childConfig.systemMessage = subAgentPreamble + (parentSystemMsg ? "\n\n" + parentSystemMsg : "");
|
|
988
|
-
const sdkClient = await getInlineClient();
|
|
989
|
-
const normalizedModel = sessionManager.normalizeModelRef(childConfig.model);
|
|
990
|
-
if (normalizedModel)
|
|
991
|
-
childConfig.model = normalizedModel;
|
|
992
|
-
const childSession = await sdkClient.createSession({
|
|
993
|
-
parentSessionId: input.sessionId,
|
|
994
|
-
nestingLevel: childNestingLevel,
|
|
995
|
-
model: childConfig.model,
|
|
996
|
-
reasoningEffort: childConfig.reasoningEffort,
|
|
997
|
-
systemMessage: childConfig.systemMessage,
|
|
998
|
-
boundAgentName: childConfig.boundAgentName,
|
|
999
|
-
promptLayering: childConfig.promptLayering,
|
|
1000
|
-
toolNames: childConfig.toolNames,
|
|
1001
|
-
waitThreshold: childConfig.waitThreshold,
|
|
1002
|
-
agentId,
|
|
1003
|
-
});
|
|
1004
|
-
if (catalog) {
|
|
1005
|
-
const meta = {};
|
|
1006
|
-
if (agentTitle) {
|
|
1007
|
-
meta.title = (agentTitleIsExplicit || agentIsSystem)
|
|
1008
|
-
? agentTitle
|
|
1009
|
-
: `${agentTitle}: ${childSession.sessionId.slice(0, 8)}`;
|
|
1210
|
+
},
|
|
1211
|
+
messageAgent: async (args) => {
|
|
1212
|
+
try {
|
|
1213
|
+
const child = await resolveManagedChild(args.agent_id);
|
|
1214
|
+
if (catalog && args.contract_patch) {
|
|
1215
|
+
const existing = await catalog.getChildOutcome(child.sessionId);
|
|
1216
|
+
const contractJson = appendContractPatchJson(existing?.contractJson ?? null, args.contract_patch, input.sessionId, child.sessionId);
|
|
1217
|
+
if (contractJson) {
|
|
1218
|
+
await cmsRetryCritical(`runTurn.message.upsertChildOutcome contract child=${child.sessionId}`, () => catalog.upsertChildOutcome({
|
|
1219
|
+
childSessionId: child.sessionId,
|
|
1220
|
+
parentSessionId: input.sessionId,
|
|
1221
|
+
contractJson,
|
|
1222
|
+
}), (msg) => activityCtx.traceInfo(msg));
|
|
1223
|
+
}
|
|
1010
1224
|
}
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1225
|
+
const sdkClient = await getInlineClient();
|
|
1226
|
+
await sdkClient._getDuroxideClient().enqueueEvent(child.orchId, "messages", JSON.stringify({ prompt: args.message }));
|
|
1227
|
+
return `[SYSTEM: Message sent to sub-agent ${child.orchId}: "${args.message.slice(0, 200)}". ` +
|
|
1228
|
+
`Continue your work in this SAME turn. If you are waiting on the child, call wait_for_agents explicitly rather than stopping here.]`;
|
|
1229
|
+
}
|
|
1230
|
+
catch (err) {
|
|
1231
|
+
return `[SYSTEM: message_agent failed: ${err?.message || String(err)}]`;
|
|
1232
|
+
}
|
|
1233
|
+
},
|
|
1234
|
+
checkAgents: async () => {
|
|
1235
|
+
try {
|
|
1236
|
+
const children = (await loadDirectChildSessions()).filter(child => !child.isSystem);
|
|
1237
|
+
if (children.length === 0) {
|
|
1238
|
+
return `[SYSTEM: No sub-agents have been spawned yet.]`;
|
|
1024
1239
|
}
|
|
1025
|
-
const
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1240
|
+
const statusLines = children.map((agent) => ` - Agent ${agent.orchId}\n` +
|
|
1241
|
+
` Title: ${agent.title ?? "(untitled)"}\n` +
|
|
1242
|
+
` Status: ${agent.status}\n` +
|
|
1243
|
+
` Contract: ${agent.contractStatus ?? "none"}\n` +
|
|
1244
|
+
` Verdict: ${agent.verdict ?? "none"}\n` +
|
|
1245
|
+
` Iterations: ${agent.iterations ?? 0}\n` +
|
|
1246
|
+
(Array.isArray(agent.contractViolations) && agent.contractViolations.length > 0
|
|
1247
|
+
? ` Violations: ${agent.contractViolations.map((v) => v?.code || v?.message || "violation").join(", ")}\n`
|
|
1248
|
+
: "") +
|
|
1249
|
+
` Output: ${agent.result ?? agent.error ?? "(no output yet)"}`);
|
|
1250
|
+
return `[SYSTEM: Sub-agent status report (${children.length} agents):\n${statusLines.join("\n")}]`;
|
|
1251
|
+
}
|
|
1252
|
+
catch (err) {
|
|
1253
|
+
return `[SYSTEM: check_agents failed: ${err?.message || String(err)}]`;
|
|
1254
|
+
}
|
|
1255
|
+
},
|
|
1256
|
+
resolveWaitForAgents: async (agentIds) => {
|
|
1257
|
+
const children = (await loadDirectChildSessions()).filter(child => !child.isSystem);
|
|
1258
|
+
if (agentIds && agentIds.length > 0) {
|
|
1259
|
+
return await Promise.all(agentIds.map(async (agentId) => (await resolveManagedChild(agentId)).orchId));
|
|
1260
|
+
}
|
|
1261
|
+
const running = children.filter(child => child.status === "running").map(child => child.orchId);
|
|
1262
|
+
return running.length > 0 ? running : children.map(child => child.orchId);
|
|
1263
|
+
},
|
|
1264
|
+
listSessions: async (args) => {
|
|
1265
|
+
try {
|
|
1266
|
+
const sdkClient = await getInlineClient();
|
|
1267
|
+
const effectiveArgs = sanitizeAutonomousSystemSessionFilters(args);
|
|
1268
|
+
const limit = Math.max(1, Math.min(100, Number(effectiveArgs?.limit) || 50));
|
|
1269
|
+
const query = String(effectiveArgs?.query || "").trim().toLowerCase();
|
|
1270
|
+
const exactSessionId = String(effectiveArgs?.session_id || "").replace(/^session-/, "");
|
|
1271
|
+
const agentId = String(effectiveArgs?.agent_id || "").trim();
|
|
1272
|
+
const state = String(effectiveArgs?.state || "").trim().toLowerCase();
|
|
1273
|
+
const parentSessionId = String(effectiveArgs?.parent_session_id || "").replace(/^session-/, "");
|
|
1274
|
+
const groupFilterRaw = effectiveArgs?.group_id;
|
|
1275
|
+
const groupFilter = typeof groupFilterRaw === "string" ? groupFilterRaw.trim() : undefined;
|
|
1276
|
+
const includeChildren = effectiveArgs?.include_children === true;
|
|
1277
|
+
const updatedSince = Date.parse(String(effectiveArgs?.updated_since || ""));
|
|
1278
|
+
const summaryUpdatedSince = Date.parse(String(effectiveArgs?.summary_updated_since || ""));
|
|
1279
|
+
const sessions = (await sdkClient.listSessions()).filter((session) => matchesSessionOwnerFilters(session, {
|
|
1280
|
+
includeSystem: effectiveArgs?.include_system === true,
|
|
1281
|
+
ownerQuery: effectiveArgs?.owner_query,
|
|
1282
|
+
ownerKind: effectiveArgs?.owner_kind,
|
|
1283
|
+
})).filter((session) => {
|
|
1284
|
+
if (!includeChildren && session.parentSessionId)
|
|
1285
|
+
return false;
|
|
1286
|
+
if (exactSessionId && session.sessionId !== exactSessionId)
|
|
1287
|
+
return false;
|
|
1288
|
+
if (agentId && session.agentId !== agentId)
|
|
1289
|
+
return false;
|
|
1290
|
+
if (state && String(session.status || "").toLowerCase() !== state)
|
|
1291
|
+
return false;
|
|
1292
|
+
if (parentSessionId && session.parentSessionId !== parentSessionId)
|
|
1293
|
+
return false;
|
|
1294
|
+
if (groupFilter === "null" && session.groupId)
|
|
1295
|
+
return false;
|
|
1296
|
+
if (groupFilter && groupFilter !== "null" && session.groupId !== groupFilter)
|
|
1297
|
+
return false;
|
|
1298
|
+
if (Number.isFinite(updatedSince)) {
|
|
1299
|
+
const updatedAt = Date.parse(session.updatedAt || session.lastActiveAt || session.createdAt || "");
|
|
1300
|
+
if (!Number.isFinite(updatedAt) || updatedAt < updatedSince)
|
|
1301
|
+
return false;
|
|
1302
|
+
}
|
|
1303
|
+
if (Number.isFinite(summaryUpdatedSince)) {
|
|
1304
|
+
const summaryUpdatedAt = Date.parse(session.summaryUpdatedAt || "");
|
|
1305
|
+
if (!Number.isFinite(summaryUpdatedAt) || summaryUpdatedAt < summaryUpdatedSince)
|
|
1306
|
+
return false;
|
|
1307
|
+
}
|
|
1308
|
+
if (query) {
|
|
1309
|
+
const haystack = [
|
|
1310
|
+
session.sessionId,
|
|
1311
|
+
session.title,
|
|
1312
|
+
session.agentId,
|
|
1313
|
+
session.shortSummary,
|
|
1314
|
+
session.summaryState?.intent,
|
|
1315
|
+
session.summaryState?.summary,
|
|
1316
|
+
formatSessionOwnerLabel(session),
|
|
1317
|
+
].map((part) => String(part || "").toLowerCase()).join(" ");
|
|
1318
|
+
if (!haystack.includes(query))
|
|
1319
|
+
return false;
|
|
1320
|
+
}
|
|
1321
|
+
return true;
|
|
1322
|
+
}).slice(0, limit);
|
|
1323
|
+
if (sessions.length === 0) {
|
|
1324
|
+
return "[SYSTEM: Active sessions (0). No sessions matched the requested filters.]";
|
|
1032
1325
|
}
|
|
1326
|
+
const lines = sessions.map((s) => ` - ${s.sessionId}${s.sessionId === input.sessionId ? " (this session)" : ""}\n` +
|
|
1327
|
+
` Title: ${s.title ?? "(untitled)"}\n` +
|
|
1328
|
+
` Owner: ${formatSessionOwnerLabel(s)}\n` +
|
|
1329
|
+
` Agent: ${s.agentId ?? "generic"}\n` +
|
|
1330
|
+
` Group: ${s.groupId ?? "none"}\n` +
|
|
1331
|
+
` Summary: ${s.shortSummary ?? s.summaryState?.summary ?? "(no summary)"}\n` +
|
|
1332
|
+
` Summary Updated: ${s.summaryUpdatedAt ? new Date(s.summaryUpdatedAt).toISOString() : "never"}\n` +
|
|
1333
|
+
` Status: ${s.status}, Iterations: ${s.iterations ?? 0}\n` +
|
|
1334
|
+
` Parent: ${s.parentSessionId ?? "none"}`);
|
|
1335
|
+
return `[SYSTEM: Active sessions (${sessions.length}):\n${lines.join("\n")}]`;
|
|
1033
1336
|
}
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
await cmsRetryBestEffort(`runTurn.spawn.recordEvent agent_spawned session=${input.sessionId}`, () => catalog.recordEvents(input.sessionId, [{
|
|
1037
|
-
eventType: "session.agent_spawned",
|
|
1038
|
-
data: { childSessionId: childSession.sessionId, agentId: agentId || undefined, task: agentTask.slice(0, 500) },
|
|
1039
|
-
}], workerNodeId), (msg) => activityCtx.traceInfo(msg));
|
|
1337
|
+
catch (err) {
|
|
1338
|
+
return `[SYSTEM: list_sessions failed: ${err?.message || String(err)}]`;
|
|
1040
1339
|
}
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
` Agent ID: ${childOrchId}\n` +
|
|
1044
|
-
` ${resolvedAgentName ? `Agent: ${resolvedAgentName}\n ` : ``}Task: "${agentTask.slice(0, 200)}"\n` +
|
|
1045
|
-
` The agent is now running autonomously. Continue your work in this SAME turn and keep following the user's remaining steps. ` +
|
|
1046
|
-
`Do NOT stop just because the child started. If your plan says to pause, call wait or wait_for_agents explicitly. ` +
|
|
1047
|
-
`You can also use check_agents to poll status, ` +
|
|
1048
|
-
`or message_agent to send instructions.]`;
|
|
1049
|
-
}
|
|
1050
|
-
catch (err) {
|
|
1051
|
-
return `[SYSTEM: spawn_agent failed: ${err?.message || String(err)}]`;
|
|
1052
|
-
}
|
|
1053
|
-
},
|
|
1054
|
-
setSessionModel: async (args) => {
|
|
1055
|
-
try {
|
|
1056
|
-
const requestedModel = String(args.model || "").trim();
|
|
1057
|
-
if (!requestedModel)
|
|
1058
|
-
return "[SYSTEM: set_session_model failed: model is required.]";
|
|
1059
|
-
if (!storeUrl)
|
|
1060
|
-
return "[SYSTEM: set_session_model failed: no storeUrl is configured.]";
|
|
1061
|
-
const resolved = sessionManager.resolveModelSwitchConfig(requestedModel, "reasoning_effort" in args ? args.reasoning_effort ?? null : undefined);
|
|
1062
|
-
const sdkClient = new PilotSwarmClient(internalClientConfig());
|
|
1340
|
+
},
|
|
1341
|
+
updateSessionSummary: async (args) => {
|
|
1063
1342
|
try {
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1343
|
+
if (isReadOnlyTuner())
|
|
1344
|
+
return `[SYSTEM: update_session_summary is disabled for read-only agent-tuner sessions.]`;
|
|
1345
|
+
if (!catalog)
|
|
1346
|
+
return `[SYSTEM: update_session_summary failed: CMS catalog is unavailable.]`;
|
|
1347
|
+
const hasSummaryState = Object.prototype.hasOwnProperty.call(args ?? {}, "summary_state") && args.summary_state !== undefined && args.summary_state !== null;
|
|
1348
|
+
const hasTitle = Object.prototype.hasOwnProperty.call(args ?? {}, "title");
|
|
1349
|
+
if (!hasSummaryState && !hasTitle) {
|
|
1350
|
+
return `[SYSTEM: update_session_summary failed: summary_state or title is required.]`;
|
|
1351
|
+
}
|
|
1352
|
+
let normalizedSummary;
|
|
1353
|
+
if (hasSummaryState) {
|
|
1354
|
+
const summaryResult = normalizeSummaryState(args.summary_state);
|
|
1355
|
+
if (!summaryResult.ok)
|
|
1356
|
+
return `[SYSTEM: update_session_summary failed: ${summaryResult.message}]`;
|
|
1357
|
+
normalizedSummary = summaryResult;
|
|
1358
|
+
}
|
|
1359
|
+
const titleUpdate = hasTitle
|
|
1360
|
+
? await prepareStickySessionTitleUpdate(catalog, input.sessionId, args.title)
|
|
1361
|
+
: undefined;
|
|
1362
|
+
if (titleUpdate && !titleUpdate.ok)
|
|
1363
|
+
return `[SYSTEM: update_session_summary failed: ${titleUpdate.message}]`;
|
|
1364
|
+
const updated = [];
|
|
1365
|
+
if (normalizedSummary) {
|
|
1366
|
+
await catalog.updateSessionSummary(input.sessionId, normalizedSummary.summaryState, args.short_summary);
|
|
1367
|
+
updated.push("summary");
|
|
1368
|
+
}
|
|
1369
|
+
if (titleUpdate?.ok) {
|
|
1370
|
+
await catalog.updateSession(input.sessionId, titleUpdate.updates);
|
|
1371
|
+
updated.push("title");
|
|
1372
|
+
}
|
|
1373
|
+
return `[SYSTEM: Session ${updated.join(" and ")} updated.]`;
|
|
1075
1374
|
}
|
|
1076
|
-
|
|
1077
|
-
|
|
1375
|
+
catch (err) {
|
|
1376
|
+
return `[SYSTEM: update_session_summary failed: ${err?.message || String(err)}]`;
|
|
1078
1377
|
}
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1378
|
+
},
|
|
1379
|
+
sendSessionMessage: async (args) => {
|
|
1380
|
+
try {
|
|
1381
|
+
if (isReadOnlyTuner())
|
|
1382
|
+
return `[SYSTEM: send_session_message is disabled for read-only agent-tuner sessions.]`;
|
|
1383
|
+
const targetSessionId = String(args.session_id || "").replace(/^session-/, "");
|
|
1384
|
+
if (!targetSessionId)
|
|
1385
|
+
return `[SYSTEM: send_session_message failed: session_id is required.]`;
|
|
1386
|
+
if (targetSessionId === input.sessionId)
|
|
1387
|
+
return `[SYSTEM: send_session_message failed: target session is this session.]`;
|
|
1388
|
+
if (!args.subject?.trim() || !args.body?.trim())
|
|
1389
|
+
return `[SYSTEM: send_session_message failed: subject and body are required.]`;
|
|
1390
|
+
if (args.body.length > 8192)
|
|
1391
|
+
return `[SYSTEM: send_session_message failed: body exceeds 8 KB.]`;
|
|
1392
|
+
if (!catalog)
|
|
1393
|
+
return `[SYSTEM: send_session_message failed: CMS catalog is unavailable.]`;
|
|
1394
|
+
const sdkClient = await getInlineClient();
|
|
1395
|
+
const allowedReasons = new Set(["help", "guidance", "fact-request", "status-request", "handoff"]);
|
|
1396
|
+
const reason = allowedReasons.has(String(args.reason || "")) ? args.reason : undefined;
|
|
1397
|
+
const { requestId } = await sendInternalSessionMessage({
|
|
1398
|
+
catalog,
|
|
1399
|
+
duroxideClient: sdkClient._getDuroxideClient(),
|
|
1400
|
+
}, {
|
|
1401
|
+
fromSessionId: input.sessionId,
|
|
1402
|
+
toSessionId: targetSessionId,
|
|
1403
|
+
subject: args.subject,
|
|
1404
|
+
body: args.body,
|
|
1405
|
+
reason,
|
|
1406
|
+
expectsResponse: args.expects_response === true,
|
|
1407
|
+
expiresAt: args.expires_at,
|
|
1408
|
+
});
|
|
1409
|
+
return `[SYSTEM: Cross-session message queued. Request ID: ${requestId}. Target: session-${targetSessionId}.]`;
|
|
1410
|
+
}
|
|
1411
|
+
catch (err) {
|
|
1412
|
+
return `[SYSTEM: send_session_message failed: ${err?.message || String(err)}]`;
|
|
1413
|
+
}
|
|
1414
|
+
},
|
|
1415
|
+
replySessionMessage: async (args) => {
|
|
1416
|
+
try {
|
|
1417
|
+
if (isReadOnlyTuner())
|
|
1418
|
+
return `[SYSTEM: reply_session_message is disabled for read-only agent-tuner sessions.]`;
|
|
1419
|
+
const targetSessionId = String(args.session_id || "").replace(/^session-/, "");
|
|
1420
|
+
if (!targetSessionId || !args.request_id || !args.body?.trim())
|
|
1421
|
+
return `[SYSTEM: reply_session_message failed: request_id, session_id, and body are required.]`;
|
|
1422
|
+
if (!catalog)
|
|
1423
|
+
return `[SYSTEM: reply_session_message failed: CMS catalog is unavailable.]`;
|
|
1424
|
+
const sdkClient = await getInlineClient();
|
|
1425
|
+
const allowedVerdicts = new Set(["answered", "declined", "blocked", "stale"]);
|
|
1426
|
+
const verdict = allowedVerdicts.has(String(args.verdict || "")) ? args.verdict : "answered";
|
|
1427
|
+
await replyInternalSessionMessage({
|
|
1428
|
+
catalog,
|
|
1429
|
+
duroxideClient: sdkClient._getDuroxideClient(),
|
|
1430
|
+
}, {
|
|
1431
|
+
requestId: args.request_id,
|
|
1432
|
+
fromSessionId: input.sessionId,
|
|
1433
|
+
toSessionId: targetSessionId,
|
|
1434
|
+
verdict,
|
|
1435
|
+
body: args.body,
|
|
1436
|
+
});
|
|
1437
|
+
return `[SYSTEM: Cross-session reply queued for request ${args.request_id}. Target: session-${targetSessionId}.]`;
|
|
1438
|
+
}
|
|
1439
|
+
catch (err) {
|
|
1440
|
+
return `[SYSTEM: reply_session_message failed: ${err?.message || String(err)}]`;
|
|
1441
|
+
}
|
|
1442
|
+
},
|
|
1443
|
+
completeAgent: async (args) => {
|
|
1444
|
+
try {
|
|
1445
|
+
const child = await resolveManagedChild(args.agent_id);
|
|
1446
|
+
let builtOutcome = null;
|
|
1447
|
+
if (catalog) {
|
|
1448
|
+
const existing = await catalog.getChildOutcome(child.sessionId).catch(() => null);
|
|
1449
|
+
const hasExistingResult = Boolean(normalizeJsonObject(existing?.resultJson?.current));
|
|
1450
|
+
if (args.result || (existing?.contractJson && !hasExistingResult)) {
|
|
1451
|
+
builtOutcome = buildResultJson(args.result, child, existing?.contractJson ?? null, existing?.resultJson ?? null, "success", "completed_without_result");
|
|
1452
|
+
}
|
|
1453
|
+
if (builtOutcome?.strictBlocked) {
|
|
1454
|
+
return `[SYSTEM: complete_agent blocked by strict contract validation for ${child.orchId}: ${builtOutcome.violations.map(v => v.message || v.code).join("; ")}]`;
|
|
1455
|
+
}
|
|
1456
|
+
}
|
|
1457
|
+
if (catalog && builtOutcome) {
|
|
1458
|
+
await cmsRetryCritical(`runTurn.complete.upsertChildOutcome result child=${child.sessionId}`, () => catalog.upsertChildOutcome({
|
|
1094
1459
|
childSessionId: child.sessionId,
|
|
1095
1460
|
parentSessionId: input.sessionId,
|
|
1096
|
-
|
|
1461
|
+
resultJson: builtOutcome.resultJson,
|
|
1462
|
+
verdict: builtOutcome.verdict,
|
|
1463
|
+
summary: builtOutcome.summary,
|
|
1464
|
+
completedAt: new Date(),
|
|
1097
1465
|
}), (msg) => activityCtx.traceInfo(msg));
|
|
1098
1466
|
}
|
|
1467
|
+
const sdkClient = await getInlineClient();
|
|
1468
|
+
const cmdId = `done-inline-${Date.now()}`;
|
|
1469
|
+
await sdkClient._getDuroxideClient().enqueueEvent(child.orchId, "messages", JSON.stringify({ type: "cmd", cmd: "done", id: cmdId, args: { reason: "Completed by parent" } }));
|
|
1470
|
+
return `[SYSTEM: Graceful completion requested for sub-agent ${child.orchId}. ` +
|
|
1471
|
+
`Use check_agents or wait_for_agents to observe final completion.]`;
|
|
1099
1472
|
}
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
return `[SYSTEM: Message sent to sub-agent ${child.orchId}: "${args.message.slice(0, 200)}". ` +
|
|
1103
|
-
`Continue your work in this SAME turn. If you are waiting on the child, call wait_for_agents explicitly rather than stopping here.]`;
|
|
1104
|
-
}
|
|
1105
|
-
catch (err) {
|
|
1106
|
-
return `[SYSTEM: message_agent failed: ${err?.message || String(err)}]`;
|
|
1107
|
-
}
|
|
1108
|
-
},
|
|
1109
|
-
checkAgents: async () => {
|
|
1110
|
-
try {
|
|
1111
|
-
const children = (await loadDirectChildSessions()).filter(child => !child.isSystem);
|
|
1112
|
-
if (children.length === 0) {
|
|
1113
|
-
return `[SYSTEM: No sub-agents have been spawned yet.]`;
|
|
1473
|
+
catch (err) {
|
|
1474
|
+
return `[SYSTEM: complete_agent failed: ${err?.message || String(err)}]`;
|
|
1114
1475
|
}
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
}
|
|
1127
|
-
catch (err) {
|
|
1128
|
-
return `[SYSTEM: check_agents failed: ${err?.message || String(err)}]`;
|
|
1129
|
-
}
|
|
1130
|
-
},
|
|
1131
|
-
resolveWaitForAgents: async (agentIds) => {
|
|
1132
|
-
const children = (await loadDirectChildSessions()).filter(child => !child.isSystem);
|
|
1133
|
-
if (agentIds && agentIds.length > 0) {
|
|
1134
|
-
return await Promise.all(agentIds.map(async (agentId) => (await resolveManagedChild(agentId)).orchId));
|
|
1135
|
-
}
|
|
1136
|
-
const running = children.filter(child => child.status === "running").map(child => child.orchId);
|
|
1137
|
-
return running.length > 0 ? running : children.map(child => child.orchId);
|
|
1138
|
-
},
|
|
1139
|
-
listSessions: async (args) => {
|
|
1140
|
-
try {
|
|
1141
|
-
const sdkClient = await getInlineClient();
|
|
1142
|
-
const effectiveArgs = sanitizeAutonomousSystemSessionFilters(args);
|
|
1143
|
-
const limit = Math.max(1, Math.min(100, Number(effectiveArgs?.limit) || 50));
|
|
1144
|
-
const query = String(effectiveArgs?.query || "").trim().toLowerCase();
|
|
1145
|
-
const exactSessionId = String(effectiveArgs?.session_id || "").replace(/^session-/, "");
|
|
1146
|
-
const agentId = String(effectiveArgs?.agent_id || "").trim();
|
|
1147
|
-
const state = String(effectiveArgs?.state || "").trim().toLowerCase();
|
|
1148
|
-
const parentSessionId = String(effectiveArgs?.parent_session_id || "").replace(/^session-/, "");
|
|
1149
|
-
const groupFilterRaw = effectiveArgs?.group_id;
|
|
1150
|
-
const groupFilter = typeof groupFilterRaw === "string" ? groupFilterRaw.trim() : undefined;
|
|
1151
|
-
const includeChildren = effectiveArgs?.include_children === true;
|
|
1152
|
-
const updatedSince = Date.parse(String(effectiveArgs?.updated_since || ""));
|
|
1153
|
-
const summaryUpdatedSince = Date.parse(String(effectiveArgs?.summary_updated_since || ""));
|
|
1154
|
-
const sessions = (await sdkClient.listSessions()).filter((session) => matchesSessionOwnerFilters(session, {
|
|
1155
|
-
includeSystem: effectiveArgs?.include_system === true,
|
|
1156
|
-
ownerQuery: effectiveArgs?.owner_query,
|
|
1157
|
-
ownerKind: effectiveArgs?.owner_kind,
|
|
1158
|
-
})).filter((session) => {
|
|
1159
|
-
if (!includeChildren && session.parentSessionId)
|
|
1160
|
-
return false;
|
|
1161
|
-
if (exactSessionId && session.sessionId !== exactSessionId)
|
|
1162
|
-
return false;
|
|
1163
|
-
if (agentId && session.agentId !== agentId)
|
|
1164
|
-
return false;
|
|
1165
|
-
if (state && String(session.status || "").toLowerCase() !== state)
|
|
1166
|
-
return false;
|
|
1167
|
-
if (parentSessionId && session.parentSessionId !== parentSessionId)
|
|
1168
|
-
return false;
|
|
1169
|
-
if (groupFilter === "null" && session.groupId)
|
|
1170
|
-
return false;
|
|
1171
|
-
if (groupFilter && groupFilter !== "null" && session.groupId !== groupFilter)
|
|
1172
|
-
return false;
|
|
1173
|
-
if (Number.isFinite(updatedSince)) {
|
|
1174
|
-
const updatedAt = Date.parse(session.updatedAt || session.lastActiveAt || session.createdAt || "");
|
|
1175
|
-
if (!Number.isFinite(updatedAt) || updatedAt < updatedSince)
|
|
1176
|
-
return false;
|
|
1476
|
+
},
|
|
1477
|
+
cancelAgent: async (args) => {
|
|
1478
|
+
try {
|
|
1479
|
+
const child = await resolveManagedChild(args.agent_id);
|
|
1480
|
+
let builtOutcome = null;
|
|
1481
|
+
if (catalog) {
|
|
1482
|
+
const existing = await catalog.getChildOutcome(child.sessionId).catch(() => null);
|
|
1483
|
+
const hasExistingResult = Boolean(normalizeJsonObject(existing?.resultJson?.current));
|
|
1484
|
+
if (args.partial_result || (existing?.contractJson && !hasExistingResult)) {
|
|
1485
|
+
builtOutcome = buildResultJson(args.partial_result, child, existing?.contractJson ?? null, existing?.resultJson ?? null, "cancelled", "completed_without_result");
|
|
1486
|
+
}
|
|
1177
1487
|
}
|
|
1178
|
-
if (
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1488
|
+
if (catalog && builtOutcome) {
|
|
1489
|
+
await cmsRetryCritical(`runTurn.cancel.upsertChildOutcome partial child=${child.sessionId}`, () => catalog.upsertChildOutcome({
|
|
1490
|
+
childSessionId: child.sessionId,
|
|
1491
|
+
parentSessionId: input.sessionId,
|
|
1492
|
+
resultJson: builtOutcome.resultJson,
|
|
1493
|
+
verdict: builtOutcome.verdict,
|
|
1494
|
+
summary: builtOutcome.summary,
|
|
1495
|
+
completedAt: new Date(),
|
|
1496
|
+
}), (msg) => activityCtx.traceInfo(msg));
|
|
1182
1497
|
}
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1498
|
+
const sdkClient = await getInlineClient();
|
|
1499
|
+
const cmdId = `cancel-inline-${Date.now()}`;
|
|
1500
|
+
await sdkClient._getDuroxideClient().enqueueEvent(child.orchId, "messages", JSON.stringify({ type: "cmd", cmd: "cancel", id: cmdId, args: { reason: args.reason ?? "Cancelled by parent" } }));
|
|
1501
|
+
return `[SYSTEM: Graceful cancellation requested for sub-agent ${child.orchId}. ` +
|
|
1502
|
+
`Use check_agents or wait_for_agents to observe final termination.${args.reason ? ` Reason: ${args.reason}` : ""}]`;
|
|
1503
|
+
}
|
|
1504
|
+
catch (err) {
|
|
1505
|
+
return `[SYSTEM: cancel_agent failed: ${err?.message || String(err)}]`;
|
|
1506
|
+
}
|
|
1507
|
+
},
|
|
1508
|
+
deleteAgent: async (args) => {
|
|
1509
|
+
try {
|
|
1510
|
+
const child = await resolveManagedChild(args.agent_id);
|
|
1511
|
+
const sdkClient = await getInlineClient();
|
|
1512
|
+
if (child.status === "completed" || child.status === "failed" || child.status === "cancelled") {
|
|
1513
|
+
await sdkClient.deleteSession(child.sessionId);
|
|
1514
|
+
return `[SYSTEM: Sub-agent ${child.orchId} has been deleted.${args.reason ? ` Reason: ${args.reason}` : ""}]`;
|
|
1195
1515
|
}
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1516
|
+
const cmdId = `delete-inline-${Date.now()}`;
|
|
1517
|
+
await sdkClient._getDuroxideClient().enqueueEvent(child.orchId, "messages", JSON.stringify({ type: "cmd", cmd: "delete", id: cmdId, args: { reason: args.reason ?? "Deleted by parent" } }));
|
|
1518
|
+
return `[SYSTEM: Graceful deletion requested for sub-agent ${child.orchId}. ` +
|
|
1519
|
+
`It will cancel its descendants first and then delete itself.${args.reason ? ` Reason: ${args.reason}` : ""}]`;
|
|
1200
1520
|
}
|
|
1201
|
-
|
|
1202
|
-
`
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1521
|
+
catch (err) {
|
|
1522
|
+
return `[SYSTEM: delete_agent failed: ${err?.message || String(err)}]`;
|
|
1523
|
+
}
|
|
1524
|
+
},
|
|
1525
|
+
};
|
|
1526
|
+
// Cooperative cancellation: poll for lock steal
|
|
1527
|
+
let cancelled = false;
|
|
1528
|
+
cancelPoll = setInterval(() => {
|
|
1529
|
+
if (activityCtx.isCancelled()) {
|
|
1530
|
+
cancelled = true;
|
|
1531
|
+
session?.abort?.();
|
|
1532
|
+
if (cancelPoll)
|
|
1533
|
+
clearInterval(cancelPoll);
|
|
1214
1534
|
}
|
|
1215
|
-
},
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1535
|
+
}, 2_000);
|
|
1536
|
+
// Build onEvent callback: write each non-ephemeral event to CMS as it fires.
|
|
1537
|
+
// We track every in-flight CMS recordEvents promise so we can flush them
|
|
1538
|
+
// before posting `session.turn_completed`. Without this barrier the
|
|
1539
|
+
// turn_completed insert can race ahead of the SDK's `assistant.message`
|
|
1540
|
+
// insert and CMS will assign the smaller `seq` to turn_completed,
|
|
1541
|
+
// breaking event-ordering invariants downstream (see cms-seq-nodemap).
|
|
1542
|
+
const pendingEventWrites = [];
|
|
1543
|
+
const trackEventWrite = (promise) => {
|
|
1544
|
+
if (!promise || typeof promise.then !== "function")
|
|
1545
|
+
return;
|
|
1546
|
+
pendingEventWrites.push(promise.catch(() => { }));
|
|
1547
|
+
};
|
|
1548
|
+
const EPHEMERAL_TYPES = new Set([
|
|
1549
|
+
"assistant.message_delta",
|
|
1550
|
+
"assistant.streaming_delta",
|
|
1551
|
+
"assistant.reasoning_delta",
|
|
1552
|
+
"user.message", // Already recorded explicitly above — skip the SDK's duplicate
|
|
1553
|
+
]);
|
|
1554
|
+
const onEvent = catalog
|
|
1555
|
+
? (event) => {
|
|
1556
|
+
if (EPHEMERAL_TYPES.has(event.eventType))
|
|
1557
|
+
return;
|
|
1558
|
+
const persistedEvent = summarizeSdkSystemPromptEchoEvent(event);
|
|
1559
|
+
if (!persistedEvent)
|
|
1560
|
+
return;
|
|
1561
|
+
if (event.eventType === "session.wait_started") {
|
|
1562
|
+
const data = (event.data ?? {});
|
|
1563
|
+
void cmsRetryBestEffort(`runTurn.onEvent updateSession state=waiting session=${input.sessionId}`, () => catalog.updateSession(input.sessionId, {
|
|
1564
|
+
state: "waiting",
|
|
1565
|
+
waitReason: data.reason ?? null,
|
|
1566
|
+
lastActiveAt: new Date(),
|
|
1567
|
+
}), (msg) => activityCtx.traceInfo(msg));
|
|
1568
|
+
}
|
|
1569
|
+
else if (event.eventType === "session.input_required_started") {
|
|
1570
|
+
const data = (event.data ?? {});
|
|
1571
|
+
void cmsRetryBestEffort(`runTurn.onEvent updateSession state=input_required session=${input.sessionId}`, () => catalog.updateSession(input.sessionId, {
|
|
1572
|
+
state: "input_required",
|
|
1573
|
+
waitReason: data.question ?? null,
|
|
1574
|
+
lastActiveAt: new Date(),
|
|
1575
|
+
}), (msg) => activityCtx.traceInfo(msg));
|
|
1226
1576
|
}
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1577
|
+
if (event.eventType === "assistant.usage") {
|
|
1578
|
+
const usageUpsert = buildUsageSummaryUpsert(event.data);
|
|
1579
|
+
if (usageUpsert) {
|
|
1580
|
+
turnTelemetry.tokensInput += usageUpsert.tokensInputIncrement ?? 0;
|
|
1581
|
+
turnTelemetry.tokensOutput += usageUpsert.tokensOutputIncrement ?? 0;
|
|
1582
|
+
turnTelemetry.tokensCacheRead += usageUpsert.tokensCacheReadIncrement ?? 0;
|
|
1583
|
+
turnTelemetry.tokensCacheWrite += usageUpsert.tokensCacheWriteIncrement ?? 0;
|
|
1584
|
+
}
|
|
1233
1585
|
}
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1586
|
+
else if (event.eventType === "tool.execution_start") {
|
|
1587
|
+
turnTelemetry.toolCalls += 1;
|
|
1588
|
+
const eventData = normalizeEventData(event.data);
|
|
1589
|
+
const toolName = typeof eventData?.toolName === "string"
|
|
1590
|
+
? eventData.toolName
|
|
1591
|
+
: typeof eventData?.name === "string"
|
|
1592
|
+
? eventData.name
|
|
1593
|
+
: undefined;
|
|
1594
|
+
if (toolName)
|
|
1595
|
+
turnTelemetry.toolNames.add(toolName);
|
|
1243
1596
|
}
|
|
1244
|
-
if (
|
|
1245
|
-
|
|
1246
|
-
updated.push("title");
|
|
1597
|
+
else if (event.eventType === "tool.execution_complete" && isFailureToolCompletion(event.data)) {
|
|
1598
|
+
turnTelemetry.toolErrors += 1;
|
|
1247
1599
|
}
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1600
|
+
// Best-effort with one transient retry. trackEventWrite tracks
|
|
1601
|
+
// the wrapped promise so the post-turn barrier waits for the
|
|
1602
|
+
// retry to settle before emitting turn_completed.
|
|
1603
|
+
const writePromise = cmsRetryBestEffort(`runTurn.onEvent recordEvent ${persistedEvent.eventType} session=${input.sessionId}`, () => catalog.recordEvents(input.sessionId, [persistedEvent], workerNodeId), (msg) => activityCtx.traceInfo(msg));
|
|
1604
|
+
trackEventWrite(writePromise);
|
|
1252
1605
|
}
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
subject: args.subject,
|
|
1279
|
-
body: args.body,
|
|
1280
|
-
reason,
|
|
1281
|
-
expectsResponse: args.expects_response === true,
|
|
1282
|
-
expiresAt: args.expires_at,
|
|
1283
|
-
});
|
|
1284
|
-
return `[SYSTEM: Cross-session message queued. Request ID: ${requestId}. Target: session-${targetSessionId}.]`;
|
|
1606
|
+
: undefined;
|
|
1607
|
+
// Record the user prompt as a CMS event before running the turn.
|
|
1608
|
+
// Skip internal timer continuation prompts — they're system-generated, not user input.
|
|
1609
|
+
const isTimerPrompt = /^The \d+ second wait is now complete\./i.test(input.prompt);
|
|
1610
|
+
const isRetryAttempt = (input.retryCount ?? 0) > 0;
|
|
1611
|
+
if (catalog && input.config.turnSystemPrompt && !isRetryAttempt) {
|
|
1612
|
+
const persistedSystemPrompt = decorateRehydrationSystemPrompt(input.config.turnSystemPrompt, workerNodeId);
|
|
1613
|
+
void cmsRetryBestEffort(`runTurn.recordEvent system-prompt session=${input.sessionId}`, () => catalog.recordEvents(input.sessionId, [{
|
|
1614
|
+
eventType: "system.message",
|
|
1615
|
+
data: { content: persistedSystemPrompt },
|
|
1616
|
+
}], workerNodeId), (msg) => activityCtx.traceInfo(msg));
|
|
1617
|
+
}
|
|
1618
|
+
if (catalog && !isTimerPrompt && !input.bootstrap && !isRetryAttempt) {
|
|
1619
|
+
const promptEventType = isInternalSystemPrompt(input.prompt) ? "system.message" : "user.message";
|
|
1620
|
+
// v1.0.47: when the orchestration tagged the turn with one or
|
|
1621
|
+
// more clientMessageIds (the UI-generated identities of the
|
|
1622
|
+
// contributing local outbox items), persist them on the
|
|
1623
|
+
// durable user.message event so the client can ack/cancel by
|
|
1624
|
+
// exact id rather than text match.
|
|
1625
|
+
const incomingClientMessageIds = Array.isArray(input.clientMessageIds)
|
|
1626
|
+
? input.clientMessageIds.filter((id) => typeof id === "string" && id)
|
|
1627
|
+
: [];
|
|
1628
|
+
const eventData = { content: input.prompt };
|
|
1629
|
+
if (incomingClientMessageIds.length > 0) {
|
|
1630
|
+
eventData.clientMessageIds = incomingClientMessageIds;
|
|
1285
1631
|
}
|
|
1286
|
-
|
|
1287
|
-
|
|
1632
|
+
const capturedPromptEventType = promptEventType;
|
|
1633
|
+
void cmsRetryBestEffort(`runTurn.recordEvent ${capturedPromptEventType} session=${input.sessionId}`, () => catalog.recordEvents(input.sessionId, [{
|
|
1634
|
+
eventType: capturedPromptEventType,
|
|
1635
|
+
data: eventData,
|
|
1636
|
+
}], workerNodeId), (msg) => activityCtx.traceInfo(msg));
|
|
1637
|
+
}
|
|
1638
|
+
// Mark session as "running" in CMS before the turn, and publish the
|
|
1639
|
+
// in-flight turn index so stopSessionTurn() can address the
|
|
1640
|
+
// turn-scoped stop queue (stopTurn.<turnIndex>).
|
|
1641
|
+
if (catalog) {
|
|
1642
|
+
await cmsRetryBestEffort(`runTurn.preTurn updateSession state=running session=${input.sessionId}`, () => catalog.updateSession(input.sessionId, {
|
|
1643
|
+
state: "running",
|
|
1644
|
+
lastActiveAt: new Date(),
|
|
1645
|
+
}), (msg) => activityCtx.traceInfo(msg));
|
|
1646
|
+
await cmsRetryBestEffort(`runTurn.preTurn setActiveTurnIndex session=${input.sessionId} turn=${input.turnIndex ?? 0}`, () => catalog.setActiveTurnIndex(input.sessionId, input.turnIndex ?? 0), (msg) => activityCtx.traceInfo(msg));
|
|
1647
|
+
}
|
|
1648
|
+
activityCtx.traceInfo(`[runTurn] invoking ManagedSession.runTurn for ${input.sessionId}`);
|
|
1649
|
+
// Record turn_started CMS event
|
|
1650
|
+
if (catalog) {
|
|
1651
|
+
void cmsRetryBestEffort(`runTurn.recordEvent turn_started session=${input.sessionId}`, () => catalog.recordEvents(input.sessionId, [{
|
|
1652
|
+
eventType: "session.turn_started",
|
|
1653
|
+
data: { iteration: input.turnIndex ?? 0 },
|
|
1654
|
+
}], workerNodeId), (msg) => activityCtx.traceInfo(msg));
|
|
1655
|
+
}
|
|
1656
|
+
const runTurnWithPrompt = async (targetSession, prompt) => {
|
|
1657
|
+
return await targetSession.runTurn(prompt, {
|
|
1658
|
+
onEvent,
|
|
1659
|
+
modelSummary: sessionManager.getModelSummary(),
|
|
1660
|
+
bootstrap: input.bootstrap,
|
|
1661
|
+
requiredTool: input.requiredTool,
|
|
1662
|
+
cycleOrigin: input.cycleOrigin,
|
|
1663
|
+
turnIndex: input.turnIndex,
|
|
1664
|
+
controlToolBridge,
|
|
1665
|
+
});
|
|
1666
|
+
};
|
|
1667
|
+
let result = await runTurnWithPrompt(session, effectivePrompt);
|
|
1668
|
+
if (result.type === "error" && isLiveSessionLostErrorMessage(result.message)) {
|
|
1669
|
+
activityCtx.traceInfo(`[runTurn] live Copilot session lost for ${input.sessionId}; invalidating warm session and attempting recovery`);
|
|
1670
|
+
await sessionManager.invalidateWarmSession(input.sessionId, { lockHeld: true }).catch((err) => {
|
|
1671
|
+
activityCtx.traceInfo(`[runTurn] warm-session invalidation failed (non-fatal): ${err?.message ?? err}`);
|
|
1672
|
+
});
|
|
1673
|
+
if (catalog) {
|
|
1674
|
+
await cmsRetryBestEffort(`runTurn.recordEvent system.message-recovery session=${input.sessionId}`, () => catalog.recordEvents(input.sessionId, [{
|
|
1675
|
+
eventType: "system.message",
|
|
1676
|
+
data: {
|
|
1677
|
+
content: "The runtime recovered this session after the worker lost the live Copilot session. " +
|
|
1678
|
+
"Some very recent in-memory state may have been lost.",
|
|
1679
|
+
},
|
|
1680
|
+
}], workerNodeId), (msg) => activityCtx.traceInfo(msg));
|
|
1288
1681
|
}
|
|
1289
|
-
},
|
|
1290
|
-
replySessionMessage: async (args) => {
|
|
1291
1682
|
try {
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
return `[SYSTEM: reply_session_message failed: request_id, session_id, and body are required.]`;
|
|
1297
|
-
if (!catalog)
|
|
1298
|
-
return `[SYSTEM: reply_session_message failed: CMS catalog is unavailable.]`;
|
|
1299
|
-
const sdkClient = await getInlineClient();
|
|
1300
|
-
const allowedVerdicts = new Set(["answered", "declined", "blocked", "stale"]);
|
|
1301
|
-
const verdict = allowedVerdicts.has(String(args.verdict || "")) ? args.verdict : "answered";
|
|
1302
|
-
await replyInternalSessionMessage({
|
|
1303
|
-
catalog,
|
|
1304
|
-
duroxideClient: sdkClient._getDuroxideClient(),
|
|
1305
|
-
}, {
|
|
1306
|
-
requestId: args.request_id,
|
|
1307
|
-
fromSessionId: input.sessionId,
|
|
1308
|
-
toSessionId: targetSessionId,
|
|
1309
|
-
verdict,
|
|
1310
|
-
body: args.body,
|
|
1683
|
+
session = await sessionManager.getOrCreate(input.sessionId, runConfig, {
|
|
1684
|
+
turnIndex: input.turnIndex,
|
|
1685
|
+
trace,
|
|
1686
|
+
lockHeld: true,
|
|
1311
1687
|
});
|
|
1312
|
-
return `[SYSTEM: Cross-session reply queued for request ${args.request_id}. Target: session-${targetSessionId}.]`;
|
|
1313
|
-
}
|
|
1314
|
-
catch (err) {
|
|
1315
|
-
return `[SYSTEM: reply_session_message failed: ${err?.message || String(err)}]`;
|
|
1316
|
-
}
|
|
1317
|
-
},
|
|
1318
|
-
completeAgent: async (args) => {
|
|
1319
|
-
try {
|
|
1320
|
-
const child = await resolveManagedChild(args.agent_id);
|
|
1321
|
-
let builtOutcome = null;
|
|
1322
|
-
if (catalog) {
|
|
1323
|
-
const existing = await catalog.getChildOutcome(child.sessionId).catch(() => null);
|
|
1324
|
-
const hasExistingResult = Boolean(normalizeJsonObject(existing?.resultJson?.current));
|
|
1325
|
-
if (args.result || (existing?.contractJson && !hasExistingResult)) {
|
|
1326
|
-
builtOutcome = buildResultJson(args.result, child, existing?.contractJson ?? null, existing?.resultJson ?? null, "success", "completed_without_result");
|
|
1327
|
-
}
|
|
1328
|
-
if (builtOutcome?.strictBlocked) {
|
|
1329
|
-
return `[SYSTEM: complete_agent blocked by strict contract validation for ${child.orchId}: ${builtOutcome.violations.map(v => v.message || v.code).join("; ")}]`;
|
|
1330
|
-
}
|
|
1331
|
-
}
|
|
1332
|
-
if (catalog && builtOutcome) {
|
|
1333
|
-
await cmsRetryCritical(`runTurn.complete.upsertChildOutcome result child=${child.sessionId}`, () => catalog.upsertChildOutcome({
|
|
1334
|
-
childSessionId: child.sessionId,
|
|
1335
|
-
parentSessionId: input.sessionId,
|
|
1336
|
-
resultJson: builtOutcome.resultJson,
|
|
1337
|
-
verdict: builtOutcome.verdict,
|
|
1338
|
-
summary: builtOutcome.summary,
|
|
1339
|
-
completedAt: new Date(),
|
|
1340
|
-
}), (msg) => activityCtx.traceInfo(msg));
|
|
1341
|
-
}
|
|
1342
|
-
const sdkClient = await getInlineClient();
|
|
1343
|
-
const cmdId = `done-inline-${Date.now()}`;
|
|
1344
|
-
await sdkClient._getDuroxideClient().enqueueEvent(child.orchId, "messages", JSON.stringify({ type: "cmd", cmd: "done", id: cmdId, args: { reason: "Completed by parent" } }));
|
|
1345
|
-
return `[SYSTEM: Graceful completion requested for sub-agent ${child.orchId}. ` +
|
|
1346
|
-
`Use check_agents or wait_for_agents to observe final completion.]`;
|
|
1347
1688
|
}
|
|
1348
1689
|
catch (err) {
|
|
1349
|
-
|
|
1690
|
+
const recoveryMessage = err?.message || String(err);
|
|
1691
|
+
const fatalMessage = isMissingSessionStateErrorMessage(recoveryMessage)
|
|
1692
|
+
? buildUnrecoverableSessionLossMessage(input.sessionId, stripMissingSessionStatePrefix(recoveryMessage))
|
|
1693
|
+
: buildUnrecoverableSessionLossMessage(input.sessionId, recoveryMessage);
|
|
1694
|
+
activityCtx.traceInfo(`[runTurn] unrecoverable session loss for ${input.sessionId}: ${fatalMessage}`);
|
|
1695
|
+
return await failForMissingState(fatalMessage);
|
|
1350
1696
|
}
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
const
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
const existing = await catalog.getChildOutcome(child.sessionId).catch(() => null);
|
|
1358
|
-
const hasExistingResult = Boolean(normalizeJsonObject(existing?.resultJson?.current));
|
|
1359
|
-
if (args.partial_result || (existing?.contractJson && !hasExistingResult)) {
|
|
1360
|
-
builtOutcome = buildResultJson(args.partial_result, child, existing?.contractJson ?? null, existing?.resultJson ?? null, "cancelled", "completed_without_result");
|
|
1361
|
-
}
|
|
1362
|
-
}
|
|
1363
|
-
if (catalog && builtOutcome) {
|
|
1364
|
-
await cmsRetryCritical(`runTurn.cancel.upsertChildOutcome partial child=${child.sessionId}`, () => catalog.upsertChildOutcome({
|
|
1365
|
-
childSessionId: child.sessionId,
|
|
1366
|
-
parentSessionId: input.sessionId,
|
|
1367
|
-
resultJson: builtOutcome.resultJson,
|
|
1368
|
-
verdict: builtOutcome.verdict,
|
|
1369
|
-
summary: builtOutcome.summary,
|
|
1370
|
-
completedAt: new Date(),
|
|
1371
|
-
}), (msg) => activityCtx.traceInfo(msg));
|
|
1372
|
-
}
|
|
1373
|
-
const sdkClient = await getInlineClient();
|
|
1374
|
-
const cmdId = `cancel-inline-${Date.now()}`;
|
|
1375
|
-
await sdkClient._getDuroxideClient().enqueueEvent(child.orchId, "messages", JSON.stringify({ type: "cmd", cmd: "cancel", id: cmdId, args: { reason: args.reason ?? "Cancelled by parent" } }));
|
|
1376
|
-
return `[SYSTEM: Graceful cancellation requested for sub-agent ${child.orchId}. ` +
|
|
1377
|
-
`Use check_agents or wait_for_agents to observe final termination.${args.reason ? ` Reason: ${args.reason}` : ""}]`;
|
|
1697
|
+
const recoveredPrompt = mergePromptSections([SESSION_RECOVERY_NOTICE, input.prompt]) || input.prompt;
|
|
1698
|
+
result = await runTurnWithPrompt(session, recoveredPrompt);
|
|
1699
|
+
if (result.type === "error" && isLiveSessionLostErrorMessage(result.message)) {
|
|
1700
|
+
const fatalMessage = buildUnrecoverableSessionLossMessage(input.sessionId, result.message);
|
|
1701
|
+
activityCtx.traceInfo(`[runTurn] recovery re-run still reported lost session for ${input.sessionId}: ${fatalMessage}`);
|
|
1702
|
+
return await failForMissingState(fatalMessage);
|
|
1378
1703
|
}
|
|
1379
|
-
|
|
1380
|
-
|
|
1704
|
+
}
|
|
1705
|
+
if (result.type === "error" && isToolCallTranscriptCorruptionErrorMessage(result.message)) {
|
|
1706
|
+
const transcriptError = result.message;
|
|
1707
|
+
activityCtx.traceInfo(`[runTurn] corrupted live Copilot transcript for ${input.sessionId}; resetting stored session state and attempting fresh-session replay`);
|
|
1708
|
+
await sessionManager.resetSessionState(input.sessionId, { lockHeld: true }).catch((err) => {
|
|
1709
|
+
activityCtx.traceInfo(`[runTurn] stored-session reset failed (non-fatal): ${err?.message ?? err}`);
|
|
1710
|
+
});
|
|
1711
|
+
await recordLossyHandoffEvent(catalog, input.sessionId, workerNodeId, {
|
|
1712
|
+
cause: "corrupted_tool_call_transcript_during_run_turn",
|
|
1713
|
+
message: "The runtime detected an inconsistent live Copilot transcript and recreated a fresh session to replay the pending turn.",
|
|
1714
|
+
detail: transcriptError,
|
|
1715
|
+
error: transcriptError,
|
|
1716
|
+
recoveryMode: "fresh_session_replay",
|
|
1717
|
+
nextStep: "replay_pending_turn_with_recreated_copilot_session",
|
|
1718
|
+
...(input.turnIndex != null ? { iteration: input.turnIndex } : {}),
|
|
1719
|
+
}, (failureMessage) => activityCtx.traceInfo(`[runTurn] ${failureMessage}`));
|
|
1720
|
+
if (catalog) {
|
|
1721
|
+
await cmsRetryBestEffort(`runTurn.recordEvent system.message-corrupted-transcript session=${input.sessionId}`, () => catalog.recordEvents(input.sessionId, [{
|
|
1722
|
+
eventType: "system.message",
|
|
1723
|
+
data: {
|
|
1724
|
+
content: "The runtime recreated this session after the live Copilot transcript became inconsistent. " +
|
|
1725
|
+
"Some recent in-memory work may be missing or partially executed.",
|
|
1726
|
+
},
|
|
1727
|
+
}], workerNodeId), (msg) => activityCtx.traceInfo(msg));
|
|
1381
1728
|
}
|
|
1382
|
-
},
|
|
1383
|
-
deleteAgent: async (args) => {
|
|
1384
1729
|
try {
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
}
|
|
1391
|
-
const cmdId = `delete-inline-${Date.now()}`;
|
|
1392
|
-
await sdkClient._getDuroxideClient().enqueueEvent(child.orchId, "messages", JSON.stringify({ type: "cmd", cmd: "delete", id: cmdId, args: { reason: args.reason ?? "Deleted by parent" } }));
|
|
1393
|
-
return `[SYSTEM: Graceful deletion requested for sub-agent ${child.orchId}. ` +
|
|
1394
|
-
`It will cancel its descendants first and then delete itself.${args.reason ? ` Reason: ${args.reason}` : ""}]`;
|
|
1730
|
+
session = await sessionManager.getOrCreate(input.sessionId, runConfig, {
|
|
1731
|
+
turnIndex: 0,
|
|
1732
|
+
trace,
|
|
1733
|
+
lockHeld: true,
|
|
1734
|
+
});
|
|
1395
1735
|
}
|
|
1396
1736
|
catch (err) {
|
|
1397
|
-
|
|
1737
|
+
const recoveryMessage = err?.message || String(err);
|
|
1738
|
+
const fatalMessage = `Live Copilot transcript became inconsistent for ${input.sessionId}, and fresh-session recovery failed. ${recoveryMessage}`;
|
|
1739
|
+
activityCtx.traceInfo(`[runTurn] unrecoverable corrupted transcript for ${input.sessionId}: ${fatalMessage}`);
|
|
1740
|
+
return { type: "error", message: fatalMessage };
|
|
1398
1741
|
}
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
// Cooperative cancellation: poll for lock steal
|
|
1402
|
-
let cancelled = false;
|
|
1403
|
-
cancelPoll = setInterval(() => {
|
|
1404
|
-
if (activityCtx.isCancelled()) {
|
|
1405
|
-
cancelled = true;
|
|
1406
|
-
session?.abort?.();
|
|
1407
|
-
if (cancelPoll)
|
|
1408
|
-
clearInterval(cancelPoll);
|
|
1742
|
+
const recoveredPrompt = mergePromptSections([CORRUPTED_TRANSCRIPT_REPLAY_NOTICE, input.prompt]) || input.prompt;
|
|
1743
|
+
result = await runTurnWithPrompt(session, recoveredPrompt);
|
|
1409
1744
|
}
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1745
|
+
if (input.parentSessionId
|
|
1746
|
+
&& result.type === "completed"
|
|
1747
|
+
&& typeof result.content === "string"
|
|
1748
|
+
&& /^QUESTION FOR PARENT:/i.test(result.content.trim())) {
|
|
1749
|
+
result = {
|
|
1750
|
+
...result,
|
|
1751
|
+
type: "wait",
|
|
1752
|
+
seconds: 60,
|
|
1753
|
+
reason: "waiting for parent answer",
|
|
1754
|
+
content: result.content.trim(),
|
|
1755
|
+
};
|
|
1756
|
+
}
|
|
1757
|
+
activityCtx.traceInfo(`[runTurn] ManagedSession.runTurn completed for ${input.sessionId} type=${result.type}`);
|
|
1758
|
+
// Drain event writes before the atomic post-turn writeback records
|
|
1759
|
+
// session.turn_completed.
|
|
1760
|
+
//
|
|
1761
|
+
// Ordering matters here: every per-event CMS write fired by the
|
|
1762
|
+
// SDK's onEvent callback (assistant.message, tool calls, usage,
|
|
1763
|
+
// etc.) must land in `session_events` with a smaller `seq` than
|
|
1764
|
+
// `session.turn_completed`, otherwise downstream consumers that
|
|
1765
|
+
// walk events in seq order (e.g. cms-seq-nodemap) see the turn
|
|
1766
|
+
// close before its own assistant.message.
|
|
1767
|
+
//
|
|
1768
|
+
// `runTurn` resolves once the SDK has emitted its final events,
|
|
1769
|
+
// but the corresponding `recordEvents` calls were dispatched
|
|
1770
|
+
// fire-and-forget. We therefore:
|
|
1771
|
+
// 1. Sleep 100ms to let any straggler onEvent callbacks fire
|
|
1772
|
+
// and enqueue their CMS write (quiesce window).
|
|
1773
|
+
// 2. Drain `pendingEventWrites` and await all of them with
|
|
1774
|
+
// allSettled — failed writes were already logged inline,
|
|
1775
|
+
// we only need the ordering guarantee.
|
|
1776
|
+
// 3. Await the turn_completed insert so subsequent activity
|
|
1777
|
+
// logic and any immediate CMS readers see a consistent tail.
|
|
1778
|
+
//
|
|
1779
|
+
// The 100ms is a deliberate, bounded pause. If it ever needs to
|
|
1780
|
+
// grow, prefer a real "SDK turn fully flushed" signal over a
|
|
1781
|
+
// larger sleep.
|
|
1782
|
+
if (catalog) {
|
|
1783
|
+
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
1784
|
+
const pendingWritesAtBarrier = pendingEventWrites.splice(0);
|
|
1785
|
+
if (pendingWritesAtBarrier.length > 0) {
|
|
1786
|
+
await Promise.allSettled(pendingWritesAtBarrier);
|
|
1443
1787
|
}
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1788
|
+
}
|
|
1789
|
+
if (cancelled)
|
|
1790
|
+
return { type: "cancelled" };
|
|
1791
|
+
// ── Activity-level writeback: sync turn result → CMS ──
|
|
1792
|
+
// This lets listSessions() read entirely from CMS without
|
|
1793
|
+
// hitting duroxide for every session's customStatus.
|
|
1794
|
+
if (catalog) {
|
|
1795
|
+
const statusMap = {
|
|
1796
|
+
completed: "idle", // orchestration decides idle vs completed; default to idle
|
|
1797
|
+
wait: "waiting",
|
|
1798
|
+
cron: "running",
|
|
1799
|
+
input_required: "input_required",
|
|
1800
|
+
error: "error",
|
|
1801
|
+
cancelled: "idle",
|
|
1802
|
+
stopped: "idle",
|
|
1803
|
+
spawn_agent: "running",
|
|
1804
|
+
message_agent: "running",
|
|
1805
|
+
check_agents: "running",
|
|
1806
|
+
wait_for_agents: "waiting",
|
|
1807
|
+
list_sessions: "running",
|
|
1808
|
+
complete_agent: "running",
|
|
1809
|
+
cancel_agent: "running",
|
|
1810
|
+
delete_agent: "running",
|
|
1811
|
+
};
|
|
1812
|
+
const liveStatus = statusMap[result.type] ?? "idle";
|
|
1813
|
+
const updates = {
|
|
1814
|
+
state: liveStatus,
|
|
1815
|
+
lastActiveAt: new Date(),
|
|
1816
|
+
};
|
|
1817
|
+
if (result.type === "error") {
|
|
1818
|
+
updates.lastError = result.message ?? null;
|
|
1819
|
+
updates.waitReason = null;
|
|
1451
1820
|
}
|
|
1452
|
-
if (
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
turnTelemetry.tokensInput += usageUpsert.tokensInputIncrement ?? 0;
|
|
1456
|
-
turnTelemetry.tokensOutput += usageUpsert.tokensOutputIncrement ?? 0;
|
|
1457
|
-
turnTelemetry.tokensCacheRead += usageUpsert.tokensCacheReadIncrement ?? 0;
|
|
1458
|
-
turnTelemetry.tokensCacheWrite += usageUpsert.tokensCacheWriteIncrement ?? 0;
|
|
1459
|
-
}
|
|
1821
|
+
else if (result.type === "wait") {
|
|
1822
|
+
updates.waitReason = result.reason ?? null;
|
|
1823
|
+
updates.lastError = null;
|
|
1460
1824
|
}
|
|
1461
|
-
else if (
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
const toolName = typeof eventData?.toolName === "string"
|
|
1465
|
-
? eventData.toolName
|
|
1466
|
-
: typeof eventData?.name === "string"
|
|
1467
|
-
? eventData.name
|
|
1468
|
-
: undefined;
|
|
1469
|
-
if (toolName)
|
|
1470
|
-
turnTelemetry.toolNames.add(toolName);
|
|
1825
|
+
else if (result.type === "input_required") {
|
|
1826
|
+
updates.waitReason = result.question ?? null;
|
|
1827
|
+
updates.lastError = null;
|
|
1471
1828
|
}
|
|
1472
|
-
else
|
|
1473
|
-
|
|
1829
|
+
else {
|
|
1830
|
+
updates.waitReason = null;
|
|
1831
|
+
updates.lastError = null;
|
|
1474
1832
|
}
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1833
|
+
const turnEndedAt = new Date();
|
|
1834
|
+
await cmsRetryBestEffort(`runTurn.postTurn completeTurnWriteback state=${updates.state} session=${input.sessionId}`, () => catalog.completeTurnWriteback({
|
|
1835
|
+
sessionId: input.sessionId,
|
|
1836
|
+
agentId: null,
|
|
1837
|
+
model: input.config.model ?? null,
|
|
1838
|
+
reasoningEffort: input.config.reasoningEffort ?? null,
|
|
1839
|
+
turnIndex: input.turnIndex ?? 0,
|
|
1840
|
+
startedAt: turnStartedAt,
|
|
1841
|
+
endedAt: turnEndedAt,
|
|
1842
|
+
durationMs: Math.max(0, turnEndedAt.getTime() - turnStartedAt.getTime()),
|
|
1843
|
+
tokensInput: turnTelemetry.tokensInput,
|
|
1844
|
+
tokensOutput: turnTelemetry.tokensOutput,
|
|
1845
|
+
tokensCacheRead: turnTelemetry.tokensCacheRead,
|
|
1846
|
+
tokensCacheWrite: turnTelemetry.tokensCacheWrite,
|
|
1847
|
+
toolCalls: turnTelemetry.toolCalls,
|
|
1848
|
+
toolErrors: turnTelemetry.toolErrors,
|
|
1849
|
+
toolNames: Array.from(turnTelemetry.toolNames).sort(),
|
|
1850
|
+
resultType: result.type,
|
|
1851
|
+
errorMessage: result.type === "error" ? (result.message ?? null) : null,
|
|
1852
|
+
workerNodeId: workerNodeId ?? null,
|
|
1853
|
+
state: updates.state ?? "idle",
|
|
1854
|
+
lastActiveAt: updates.lastActiveAt ?? turnEndedAt,
|
|
1855
|
+
lastError: updates.lastError ?? null,
|
|
1856
|
+
waitReason: updates.waitReason ?? null,
|
|
1857
|
+
currentIteration: input.turnIndex ?? 0,
|
|
1858
|
+
}), (msg) => activityCtx.traceInfo(msg));
|
|
1480
1859
|
}
|
|
1481
|
-
|
|
1482
|
-
// Record the user prompt as a CMS event before running the turn.
|
|
1483
|
-
// Skip internal timer continuation prompts — they're system-generated, not user input.
|
|
1484
|
-
const isTimerPrompt = /^The \d+ second wait is now complete\./i.test(input.prompt);
|
|
1485
|
-
const isRetryAttempt = (input.retryCount ?? 0) > 0;
|
|
1486
|
-
if (catalog && input.config.turnSystemPrompt && !isRetryAttempt) {
|
|
1487
|
-
const persistedSystemPrompt = decorateRehydrationSystemPrompt(input.config.turnSystemPrompt, workerNodeId);
|
|
1488
|
-
void cmsRetryBestEffort(`runTurn.recordEvent system-prompt session=${input.sessionId}`, () => catalog.recordEvents(input.sessionId, [{
|
|
1489
|
-
eventType: "system.message",
|
|
1490
|
-
data: { content: persistedSystemPrompt },
|
|
1491
|
-
}], workerNodeId), (msg) => activityCtx.traceInfo(msg));
|
|
1492
|
-
}
|
|
1493
|
-
if (catalog && !isTimerPrompt && !input.bootstrap && !isRetryAttempt) {
|
|
1494
|
-
const promptEventType = isInternalSystemPrompt(input.prompt) ? "system.message" : "user.message";
|
|
1495
|
-
// v1.0.47: when the orchestration tagged the turn with one or
|
|
1496
|
-
// more clientMessageIds (the UI-generated identities of the
|
|
1497
|
-
// contributing local outbox items), persist them on the
|
|
1498
|
-
// durable user.message event so the client can ack/cancel by
|
|
1499
|
-
// exact id rather than text match.
|
|
1500
|
-
const incomingClientMessageIds = Array.isArray(input.clientMessageIds)
|
|
1501
|
-
? input.clientMessageIds.filter((id) => typeof id === "string" && id)
|
|
1502
|
-
: [];
|
|
1503
|
-
const eventData = { content: input.prompt };
|
|
1504
|
-
if (incomingClientMessageIds.length > 0) {
|
|
1505
|
-
eventData.clientMessageIds = incomingClientMessageIds;
|
|
1506
|
-
}
|
|
1507
|
-
const capturedPromptEventType = promptEventType;
|
|
1508
|
-
void cmsRetryBestEffort(`runTurn.recordEvent ${capturedPromptEventType} session=${input.sessionId}`, () => catalog.recordEvents(input.sessionId, [{
|
|
1509
|
-
eventType: capturedPromptEventType,
|
|
1510
|
-
data: eventData,
|
|
1511
|
-
}], workerNodeId), (msg) => activityCtx.traceInfo(msg));
|
|
1512
|
-
}
|
|
1513
|
-
// Mark session as "running" in CMS before the turn, and publish the
|
|
1514
|
-
// in-flight turn index so stopSessionTurn() can address the
|
|
1515
|
-
// turn-scoped stop queue (stopTurn.<turnIndex>).
|
|
1516
|
-
if (catalog) {
|
|
1517
|
-
await cmsRetryBestEffort(`runTurn.preTurn updateSession state=running session=${input.sessionId}`, () => catalog.updateSession(input.sessionId, {
|
|
1518
|
-
state: "running",
|
|
1519
|
-
lastActiveAt: new Date(),
|
|
1520
|
-
}), (msg) => activityCtx.traceInfo(msg));
|
|
1521
|
-
await cmsRetryBestEffort(`runTurn.preTurn setActiveTurnIndex session=${input.sessionId} turn=${input.turnIndex ?? 0}`, () => catalog.setActiveTurnIndex(input.sessionId, input.turnIndex ?? 0), (msg) => activityCtx.traceInfo(msg));
|
|
1522
|
-
}
|
|
1523
|
-
activityCtx.traceInfo(`[runTurn] invoking ManagedSession.runTurn for ${input.sessionId}`);
|
|
1524
|
-
// Record turn_started CMS event
|
|
1525
|
-
if (catalog) {
|
|
1526
|
-
void cmsRetryBestEffort(`runTurn.recordEvent turn_started session=${input.sessionId}`, () => catalog.recordEvents(input.sessionId, [{
|
|
1527
|
-
eventType: "session.turn_started",
|
|
1528
|
-
data: { iteration: input.turnIndex ?? 0 },
|
|
1529
|
-
}], workerNodeId), (msg) => activityCtx.traceInfo(msg));
|
|
1530
|
-
}
|
|
1531
|
-
const runTurnWithPrompt = async (targetSession, prompt) => {
|
|
1532
|
-
return await targetSession.runTurn(prompt, {
|
|
1533
|
-
onEvent,
|
|
1534
|
-
modelSummary: sessionManager.getModelSummary(),
|
|
1535
|
-
bootstrap: input.bootstrap,
|
|
1536
|
-
requiredTool: input.requiredTool,
|
|
1537
|
-
cycleOrigin: input.cycleOrigin,
|
|
1538
|
-
turnIndex: input.turnIndex,
|
|
1539
|
-
controlToolBridge,
|
|
1540
|
-
});
|
|
1860
|
+
return result;
|
|
1541
1861
|
};
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
? buildUnrecoverableSessionLossMessage(input.sessionId, stripMissingSessionStatePrefix(recoveryMessage))
|
|
1568
|
-
: buildUnrecoverableSessionLossMessage(input.sessionId, recoveryMessage);
|
|
1569
|
-
activityCtx.traceInfo(`[runTurn] unrecoverable session loss for ${input.sessionId}: ${fatalMessage}`);
|
|
1570
|
-
return await failForMissingState(fatalMessage);
|
|
1571
|
-
}
|
|
1572
|
-
const recoveredPrompt = mergePromptSections([SESSION_RECOVERY_NOTICE, input.prompt]) || input.prompt;
|
|
1573
|
-
result = await runTurnWithPrompt(session, recoveredPrompt);
|
|
1574
|
-
if (result.type === "error" && isLiveSessionLostErrorMessage(result.message)) {
|
|
1575
|
-
const fatalMessage = buildUnrecoverableSessionLossMessage(input.sessionId, result.message);
|
|
1576
|
-
activityCtx.traceInfo(`[runTurn] recovery re-run still reported lost session for ${input.sessionId}: ${fatalMessage}`);
|
|
1577
|
-
return await failForMissingState(fatalMessage);
|
|
1578
|
-
}
|
|
1579
|
-
}
|
|
1580
|
-
if (result.type === "error" && isToolCallTranscriptCorruptionErrorMessage(result.message)) {
|
|
1581
|
-
const transcriptError = result.message;
|
|
1582
|
-
activityCtx.traceInfo(`[runTurn] corrupted live Copilot transcript for ${input.sessionId}; resetting stored session state and attempting fresh-session replay`);
|
|
1583
|
-
await sessionManager.resetSessionState(input.sessionId, { lockHeld: true }).catch((err) => {
|
|
1584
|
-
activityCtx.traceInfo(`[runTurn] stored-session reset failed (non-fatal): ${err?.message ?? err}`);
|
|
1585
|
-
});
|
|
1586
|
-
await recordLossyHandoffEvent(catalog, input.sessionId, workerNodeId, {
|
|
1587
|
-
cause: "corrupted_tool_call_transcript_during_run_turn",
|
|
1588
|
-
message: "The runtime detected an inconsistent live Copilot transcript and recreated a fresh session to replay the pending turn.",
|
|
1589
|
-
detail: transcriptError,
|
|
1590
|
-
error: transcriptError,
|
|
1591
|
-
recoveryMode: "fresh_session_replay",
|
|
1592
|
-
nextStep: "replay_pending_turn_with_recreated_copilot_session",
|
|
1593
|
-
...(input.turnIndex != null ? { iteration: input.turnIndex } : {}),
|
|
1594
|
-
}, (failureMessage) => activityCtx.traceInfo(`[runTurn] ${failureMessage}`));
|
|
1595
|
-
if (catalog) {
|
|
1596
|
-
await cmsRetryBestEffort(`runTurn.recordEvent system.message-corrupted-transcript session=${input.sessionId}`, () => catalog.recordEvents(input.sessionId, [{
|
|
1597
|
-
eventType: "system.message",
|
|
1862
|
+
const bodyResult = await executeTurnBody();
|
|
1863
|
+
// ── Session lifecycle protocol commit (proposal §3.2) ───────────
|
|
1864
|
+
// The turn and its snapshot durability are one activity completion:
|
|
1865
|
+
// no observable state exists where the turn "happened" but its
|
|
1866
|
+
// snapshot doesn't. Commit on every result the body produced —
|
|
1867
|
+
// matching today's semantics where the warm session keeps whatever
|
|
1868
|
+
// state the turn (even a cancelled/errored one) left behind. The
|
|
1869
|
+
// layout guard skips paths where no snapshottable session exists:
|
|
1870
|
+
// the SDK writes workspace.yaml on session create, so a dir without
|
|
1871
|
+
// it (fake test clients, failed creates — possibly containing only
|
|
1872
|
+
// our own sentinel) has nothing the store could meaningfully hold,
|
|
1873
|
+
// and committing it would fail the whole turn.
|
|
1874
|
+
const lifecycleSessionDir = lifecycle
|
|
1875
|
+
? path.join(sessionManager.getSessionStateDir(), input.sessionId)
|
|
1876
|
+
: null;
|
|
1877
|
+
if (lifecycle && lifecycleSessionDir && fs.existsSync(path.join(lifecycleSessionDir, "workspace.yaml"))) {
|
|
1878
|
+
const committed = await runTurnCommit(lifecycle, lifecycleBaseVersion, bodyResult);
|
|
1879
|
+
if (!committed.published && catalog) {
|
|
1880
|
+
// Store-wins: the snapshot was NOT advanced — a user stop, or a
|
|
1881
|
+
// discarded/foreign turn superseded this one at the store. The
|
|
1882
|
+
// turn's result still returns; record the memory loss so it is
|
|
1883
|
+
// operator-visible (never a bare heartbeat). Best-effort (P7).
|
|
1884
|
+
const reason = committed.unpublishedReason ?? "superseded";
|
|
1885
|
+
await cmsRetryBestEffort(`runTurn.recordEvent snapshot-unpublished session=${input.sessionId}`, () => catalog.recordEvents(input.sessionId, [{
|
|
1886
|
+
eventType: "session.snapshot_unpublished",
|
|
1598
1887
|
data: {
|
|
1599
|
-
|
|
1600
|
-
|
|
1888
|
+
reason,
|
|
1889
|
+
// The accepted-loss contract must be diagnosable: this
|
|
1890
|
+
// turn's model memory did NOT reach the store, and (for
|
|
1891
|
+
// a supersede) here is what won the base — so operators
|
|
1892
|
+
// can tell a foreign writer / restore race / discarded
|
|
1893
|
+
// turn apart, not just "loss happened".
|
|
1894
|
+
modelMemoryCommitted: false,
|
|
1895
|
+
baseVersion: lifecycleBaseVersion,
|
|
1896
|
+
turnKey: input.snapshot?.turnKey,
|
|
1897
|
+
...(input.turnIndex != null ? { turnIndex: input.turnIndex } : {}),
|
|
1898
|
+
...(committed.observedStoreVersion != null ? { observedStoreVersion: committed.observedStoreVersion } : {}),
|
|
1899
|
+
...(committed.observedStoreTurnKey ? { observedStoreTurnKey: committed.observedStoreTurnKey } : {}),
|
|
1900
|
+
...(bodyResult?.type ? { resultType: bodyResult.type } : {}),
|
|
1901
|
+
message: reason === "stopped"
|
|
1902
|
+
? "User-stopped turn: snapshot not committed (intentional discard)."
|
|
1903
|
+
: "Turn snapshot superseded: the store advanced off this turn's base before it committed; the next turn rehydrates the winner.",
|
|
1601
1904
|
},
|
|
1602
1905
|
}], workerNodeId), (msg) => activityCtx.traceInfo(msg));
|
|
1603
1906
|
}
|
|
1604
|
-
|
|
1605
|
-
session
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
});
|
|
1907
|
+
if (catalog && committed.sizeBytes != null) {
|
|
1908
|
+
await cmsRetryBestEffort(`runTurn.commitSummary session=${input.sessionId}`, () => catalog.upsertSessionMetricSummary(input.sessionId, {
|
|
1909
|
+
snapshotSizeBytes: committed.sizeBytes,
|
|
1910
|
+
...(committed.rawSizeBytes != null ? { rawSizeBytes: committed.rawSizeBytes } : {}),
|
|
1911
|
+
}), (msg) => activityCtx.traceInfo(msg));
|
|
1610
1912
|
}
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
}
|
|
1620
|
-
if (input.parentSessionId
|
|
1621
|
-
&& result.type === "completed"
|
|
1622
|
-
&& typeof result.content === "string"
|
|
1623
|
-
&& /^QUESTION FOR PARENT:/i.test(result.content.trim())) {
|
|
1624
|
-
result = {
|
|
1625
|
-
...result,
|
|
1626
|
-
type: "wait",
|
|
1627
|
-
seconds: 60,
|
|
1628
|
-
reason: "waiting for parent answer",
|
|
1629
|
-
content: result.content.trim(),
|
|
1630
|
-
};
|
|
1631
|
-
}
|
|
1632
|
-
activityCtx.traceInfo(`[runTurn] ManagedSession.runTurn completed for ${input.sessionId} type=${result.type}`);
|
|
1633
|
-
// Drain event writes before the atomic post-turn writeback records
|
|
1634
|
-
// session.turn_completed.
|
|
1635
|
-
//
|
|
1636
|
-
// Ordering matters here: every per-event CMS write fired by the
|
|
1637
|
-
// SDK's onEvent callback (assistant.message, tool calls, usage,
|
|
1638
|
-
// etc.) must land in `session_events` with a smaller `seq` than
|
|
1639
|
-
// `session.turn_completed`, otherwise downstream consumers that
|
|
1640
|
-
// walk events in seq order (e.g. cms-seq-nodemap) see the turn
|
|
1641
|
-
// close before its own assistant.message.
|
|
1642
|
-
//
|
|
1643
|
-
// `runTurn` resolves once the SDK has emitted its final events,
|
|
1644
|
-
// but the corresponding `recordEvents` calls were dispatched
|
|
1645
|
-
// fire-and-forget. We therefore:
|
|
1646
|
-
// 1. Sleep 100ms to let any straggler onEvent callbacks fire
|
|
1647
|
-
// and enqueue their CMS write (quiesce window).
|
|
1648
|
-
// 2. Drain `pendingEventWrites` and await all of them with
|
|
1649
|
-
// allSettled — failed writes were already logged inline,
|
|
1650
|
-
// we only need the ordering guarantee.
|
|
1651
|
-
// 3. Await the turn_completed insert so subsequent activity
|
|
1652
|
-
// logic and any immediate CMS readers see a consistent tail.
|
|
1653
|
-
//
|
|
1654
|
-
// The 100ms is a deliberate, bounded pause. If it ever needs to
|
|
1655
|
-
// grow, prefer a real "SDK turn fully flushed" signal over a
|
|
1656
|
-
// larger sleep.
|
|
1657
|
-
if (catalog) {
|
|
1658
|
-
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
1659
|
-
const pendingWritesAtBarrier = pendingEventWrites.splice(0);
|
|
1660
|
-
if (pendingWritesAtBarrier.length > 0) {
|
|
1661
|
-
await Promise.allSettled(pendingWritesAtBarrier);
|
|
1913
|
+
if (committed.alreadyCommitted && committed.storedResult !== undefined) {
|
|
1914
|
+
// A racing attempt of this same turn won the CAS. Its
|
|
1915
|
+
// snapshot was restored (restore-not-replay, §3.2 r1–r3) —
|
|
1916
|
+
// return ITS result so the recorded outcome matches the
|
|
1917
|
+
// durable lineage instead of this attempt's divergent body.
|
|
1918
|
+
activityCtx.traceInfo(`[runTurn] session=${input.sessionId} racing attempt committed v${committed.version} first; ` +
|
|
1919
|
+
`adopting its stored result`);
|
|
1920
|
+
return { ...committed.storedResult, snapshotVersion: committed.version };
|
|
1662
1921
|
}
|
|
1922
|
+
return { ...bodyResult, snapshotVersion: committed.version };
|
|
1663
1923
|
}
|
|
1664
|
-
|
|
1665
|
-
return { type: "cancelled" };
|
|
1666
|
-
// ── Activity-level writeback: sync turn result → CMS ──
|
|
1667
|
-
// This lets listSessions() read entirely from CMS without
|
|
1668
|
-
// hitting duroxide for every session's customStatus.
|
|
1669
|
-
if (catalog) {
|
|
1670
|
-
const statusMap = {
|
|
1671
|
-
completed: "idle", // orchestration decides idle vs completed; default to idle
|
|
1672
|
-
wait: "waiting",
|
|
1673
|
-
cron: "running",
|
|
1674
|
-
input_required: "input_required",
|
|
1675
|
-
error: "error",
|
|
1676
|
-
cancelled: "idle",
|
|
1677
|
-
stopped: "idle",
|
|
1678
|
-
spawn_agent: "running",
|
|
1679
|
-
message_agent: "running",
|
|
1680
|
-
check_agents: "running",
|
|
1681
|
-
wait_for_agents: "waiting",
|
|
1682
|
-
list_sessions: "running",
|
|
1683
|
-
complete_agent: "running",
|
|
1684
|
-
cancel_agent: "running",
|
|
1685
|
-
delete_agent: "running",
|
|
1686
|
-
};
|
|
1687
|
-
const liveStatus = statusMap[result.type] ?? "idle";
|
|
1688
|
-
const updates = {
|
|
1689
|
-
state: liveStatus,
|
|
1690
|
-
lastActiveAt: new Date(),
|
|
1691
|
-
};
|
|
1692
|
-
if (result.type === "error") {
|
|
1693
|
-
updates.lastError = result.message ?? null;
|
|
1694
|
-
updates.waitReason = null;
|
|
1695
|
-
}
|
|
1696
|
-
else if (result.type === "wait") {
|
|
1697
|
-
updates.waitReason = result.reason ?? null;
|
|
1698
|
-
updates.lastError = null;
|
|
1699
|
-
}
|
|
1700
|
-
else if (result.type === "input_required") {
|
|
1701
|
-
updates.waitReason = result.question ?? null;
|
|
1702
|
-
updates.lastError = null;
|
|
1703
|
-
}
|
|
1704
|
-
else {
|
|
1705
|
-
updates.waitReason = null;
|
|
1706
|
-
updates.lastError = null;
|
|
1707
|
-
}
|
|
1708
|
-
const turnEndedAt = new Date();
|
|
1709
|
-
await cmsRetryBestEffort(`runTurn.postTurn completeTurnWriteback state=${updates.state} session=${input.sessionId}`, () => catalog.completeTurnWriteback({
|
|
1710
|
-
sessionId: input.sessionId,
|
|
1711
|
-
agentId: null,
|
|
1712
|
-
model: input.config.model ?? null,
|
|
1713
|
-
reasoningEffort: input.config.reasoningEffort ?? null,
|
|
1714
|
-
turnIndex: input.turnIndex ?? 0,
|
|
1715
|
-
startedAt: turnStartedAt,
|
|
1716
|
-
endedAt: turnEndedAt,
|
|
1717
|
-
durationMs: Math.max(0, turnEndedAt.getTime() - turnStartedAt.getTime()),
|
|
1718
|
-
tokensInput: turnTelemetry.tokensInput,
|
|
1719
|
-
tokensOutput: turnTelemetry.tokensOutput,
|
|
1720
|
-
tokensCacheRead: turnTelemetry.tokensCacheRead,
|
|
1721
|
-
tokensCacheWrite: turnTelemetry.tokensCacheWrite,
|
|
1722
|
-
toolCalls: turnTelemetry.toolCalls,
|
|
1723
|
-
toolErrors: turnTelemetry.toolErrors,
|
|
1724
|
-
toolNames: Array.from(turnTelemetry.toolNames).sort(),
|
|
1725
|
-
resultType: result.type,
|
|
1726
|
-
errorMessage: result.type === "error" ? (result.message ?? null) : null,
|
|
1727
|
-
workerNodeId: workerNodeId ?? null,
|
|
1728
|
-
state: updates.state ?? "idle",
|
|
1729
|
-
lastActiveAt: updates.lastActiveAt ?? turnEndedAt,
|
|
1730
|
-
lastError: updates.lastError ?? null,
|
|
1731
|
-
waitReason: updates.waitReason ?? null,
|
|
1732
|
-
currentIteration: input.turnIndex ?? 0,
|
|
1733
|
-
}), (msg) => activityCtx.traceInfo(msg));
|
|
1734
|
-
}
|
|
1735
|
-
return result;
|
|
1924
|
+
return bodyResult;
|
|
1736
1925
|
}, { trace });
|
|
1737
1926
|
if (!finalTurnResult) {
|
|
1738
1927
|
throw new Error("runTurn completed without a turn result");
|
|
@@ -1812,6 +2001,21 @@ workerNodeId) {
|
|
|
1812
2001
|
activityCtx.traceInfo(`[abortTurn] session=${input.sessionId} outcome=${result.outcome}${result.detail ? ` (${result.detail})` : ""}`);
|
|
1813
2002
|
return result;
|
|
1814
2003
|
});
|
|
2004
|
+
// ── LEGACY LIFECYCLE ACTIVITIES (compat surface, not dead code) ──
|
|
2005
|
+
//
|
|
2006
|
+
// dehydrateSession / hydrateSession / needsHydrationSession /
|
|
2007
|
+
// checkpointSession are scheduled ONLY by frozen orchestration versions
|
|
2008
|
+
// (≤ 1.0.56: their processPrompt probes needsHydrationSession before
|
|
2009
|
+
// every turn, their wait policy dehydrates, their warm continue-as-new
|
|
2010
|
+
// checkpoints) and by 1.0.57's one-shot normalization of a legacy CAN
|
|
2011
|
+
// input (hydrateSession when input.needsHydration=true). The lifecycle
|
|
2012
|
+
// protocol itself never schedules any of them — hydrate/commit live
|
|
2013
|
+
// inside the runTurn activity (P5).
|
|
2014
|
+
//
|
|
2015
|
+
// Retirement condition: when the ≤1.0.56 handlers are dropped from
|
|
2016
|
+
// DURABLE_SESSION_ORCHESTRATION_REGISTRY, delete these registrations
|
|
2017
|
+
// (and the session-proxy methods that schedule them) in the same
|
|
2018
|
+
// change. Until then they must stay byte-compatible.
|
|
1815
2019
|
// ── dehydrateSession ────────────────────────────────────
|
|
1816
2020
|
runtime.registerActivity("dehydrateSession", async (activityCtx, input) => {
|
|
1817
2021
|
const reason = input.reason ?? "unknown";
|
|
@@ -1884,7 +2088,7 @@ workerNodeId) {
|
|
|
1884
2088
|
trace(`session=${input.sessionId} complete reason=${reason}`);
|
|
1885
2089
|
dehydrationSpan.setAttribute("pilotswarm.dehydration_result", "completed");
|
|
1886
2090
|
if (catalog) {
|
|
1887
|
-
const snapshotSizeBytes = await tryReadSnapshotSizeBytes(
|
|
2091
|
+
const snapshotSizeBytes = await tryReadSnapshotSizeBytes(sessionStore, input.sessionId);
|
|
1888
2092
|
if (snapshotSizeBytes != null) {
|
|
1889
2093
|
dehydrationSpan.setAttribute("pilotswarm.snapshot_size_bytes", snapshotSizeBytes);
|
|
1890
2094
|
}
|
|
@@ -1971,7 +2175,7 @@ workerNodeId) {
|
|
|
1971
2175
|
runtime.registerActivity("checkpointSession", async (activityCtx, input) => {
|
|
1972
2176
|
await sessionManager.checkpoint(input.sessionId);
|
|
1973
2177
|
if (catalog) {
|
|
1974
|
-
const snapshotSizeBytes = await tryReadSnapshotSizeBytes(
|
|
2178
|
+
const snapshotSizeBytes = await tryReadSnapshotSizeBytes(sessionStore, input.sessionId);
|
|
1975
2179
|
// Best-effort: metric summary is observability only. The blob has
|
|
1976
2180
|
// already been written by sessionManager.checkpoint above.
|
|
1977
2181
|
await cmsRetryBestEffort(`checkpointSession.upsertSummary session=${input.sessionId}`, () => catalog.upsertSessionMetricSummary(input.sessionId, {
|