typeclaw 0.48.13 → 0.49.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +5 -5
- package/scripts/dump-system-prompt.ts +52 -110
- package/src/agent/index.ts +144 -72
- package/src/agent/live-sessions.ts +2 -2
- package/src/agent/live-subagents.ts +1 -0
- package/src/agent/plugin-tools.ts +53 -11
- package/src/agent/prompt-cache-key.ts +42 -0
- package/src/agent/provider-error.ts +179 -10
- package/src/agent/restart-handoff/index.ts +7 -14
- package/src/agent/reviewer-activity.ts +68 -0
- package/src/agent/session-origin.ts +33 -0
- package/src/agent/subagents.ts +13 -1
- package/src/agent/system-prompt.ts +68 -202
- package/src/agent/todo/continuation-policy.ts +12 -4
- package/src/agent/tool-file-safety.ts +113 -14
- package/src/agent/tool-not-found-nudge.ts +1 -1
- package/src/agent/tools/channel-history-render.ts +2 -1
- package/src/agent/tools/channel-reply.ts +23 -5
- package/src/agent/tools/channel-send.ts +49 -7
- package/src/agent/tools/curl-impersonate.ts +11 -2
- package/src/agent/tools/post-github-review.ts +77 -15
- package/src/agent/tools/skip-response.ts +2 -2
- package/src/agent/tools/spawn-subagent.ts +163 -12
- package/src/agent/tools/subagent-output.ts +8 -0
- package/src/bundled-plugins/backup/README.md +10 -9
- package/src/bundled-plugins/backup/runner.ts +140 -58
- package/src/bundled-plugins/backup/subagents.ts +58 -26
- package/src/bundled-plugins/github-cli-auth/gh-command.ts +34 -2
- package/src/bundled-plugins/github-cli-auth/granted-push.ts +135 -0
- package/src/bundled-plugins/github-cli-auth/index.ts +83 -11
- package/src/bundled-plugins/github-cli-auth/review-account.ts +23 -0
- package/src/bundled-plugins/github-cli-auth/review-checkout.ts +9 -2
- package/src/bundled-plugins/github-cli-auth/review-recorder.ts +45 -7
- package/src/bundled-plugins/operator/operator.ts +15 -4
- package/src/bundled-plugins/reviewer/skills/code-review.ts +1 -1
- package/src/bundled-plugins/security/index.ts +50 -2
- package/src/bundled-plugins/security/policies/git-exfil.ts +24 -12
- package/src/channels/adapters/discord-bot-reactions.ts +18 -0
- package/src/channels/adapters/discord-bot.ts +122 -67
- package/src/channels/adapters/discord-reactions.ts +18 -0
- package/src/channels/adapters/discord.ts +114 -8
- package/src/channels/adapters/github/inbound.ts +103 -8
- package/src/channels/adapters/github/index.ts +251 -19
- package/src/channels/adapters/github/last-alive.ts +27 -0
- package/src/channels/adapters/github/outbound.ts +54 -18
- package/src/channels/adapters/github/pending-review-store.ts +141 -0
- package/src/channels/adapters/github/permission-guidance.ts +1 -0
- package/src/channels/adapters/github/reactions.ts +100 -0
- package/src/channels/adapters/github/reconcile-cooldown-store.ts +197 -67
- package/src/channels/adapters/github/reconcile-open-prs.ts +418 -33
- package/src/channels/adapters/github/recovery.ts +106 -0
- package/src/channels/adapters/github/review-check.ts +510 -0
- package/src/channels/adapters/github/review-submitter.ts +34 -2
- package/src/channels/adapters/instagram.ts +32 -14
- package/src/channels/adapters/kakaotalk.ts +22 -7
- package/src/channels/adapters/line.ts +18 -3
- package/src/channels/adapters/outbound-account.ts +20 -0
- package/src/channels/adapters/own-reactions.ts +66 -0
- package/src/channels/adapters/recovery-correlation.ts +214 -0
- package/src/channels/adapters/slack-bot-classify.ts +18 -11
- package/src/channels/adapters/slack-bot-dedupe.ts +111 -28
- package/src/channels/adapters/slack-bot-reactions.ts +18 -0
- package/src/channels/adapters/slack-bot.ts +125 -57
- package/src/channels/adapters/slack-classify.ts +25 -4
- package/src/channels/adapters/slack-inbound-revision.ts +96 -0
- package/src/channels/adapters/slack-reactions.ts +18 -0
- package/src/channels/adapters/slack-thread-ts.ts +6 -0
- package/src/channels/adapters/slack.ts +48 -3
- package/src/channels/adapters/teams.ts +14 -4
- package/src/channels/adapters/telegram-bot.ts +56 -10
- package/src/channels/adapters/webex-bot.ts +12 -2
- package/src/channels/adapters/webex.ts +12 -2
- package/src/channels/background-handoff.ts +434 -0
- package/src/channels/background-obligations.ts +893 -0
- package/src/channels/continuity-types.ts +271 -0
- package/src/channels/github-review-check.ts +598 -0
- package/src/channels/github-review-turn-ledger.ts +102 -28
- package/src/channels/github-review-verdict-coordinator.ts +72 -12
- package/src/channels/github-token-bridge.ts +11 -5
- package/src/channels/inbound-journal.ts +1738 -0
- package/src/channels/index.ts +6 -1
- package/src/channels/manager.ts +17 -7
- package/src/channels/recovery-dispatcher.ts +264 -0
- package/src/channels/recovery-notice.ts +94 -0
- package/src/channels/recovery-outbox.ts +459 -0
- package/src/channels/router.ts +2539 -615
- package/src/channels/schema.ts +8 -1
- package/src/channels/subagent-completion-bridge.ts +17 -14
- package/src/channels/types.ts +121 -16
- package/src/channels/waiting-reactions.ts +664 -0
- package/src/cli/compose.ts +7 -4
- package/src/cli/hostd.ts +49 -3
- package/src/cli/inspect.ts +11 -14
- package/src/coding-agent/broker-body.ts +38 -0
- package/src/coding-agent/broker.ts +290 -0
- package/src/coding-agent/claude.ts +95 -0
- package/src/coding-agent/codex.ts +104 -0
- package/src/coding-agent/index.ts +506 -0
- package/src/coding-agent/omo-state.ts +69 -0
- package/src/coding-agent/omo-stream.ts +98 -0
- package/src/coding-agent/omo.ts +192 -0
- package/src/coding-agent/runner.ts +263 -0
- package/src/coding-agent/tool.ts +62 -0
- package/src/coding-agent/worktree.ts +355 -0
- package/src/compose/restart.ts +10 -3
- package/src/compose/start.ts +20 -3
- package/src/config/config.ts +10 -3
- package/src/config/providers.ts +12 -9
- package/src/container/exit-reason.ts +44 -0
- package/src/container/index.ts +5 -0
- package/src/container/log-archive.ts +51 -13
- package/src/container/memory-limit.ts +1 -1
- package/src/container/memory-oversubscription.ts +36 -6
- package/src/container/oom-events.ts +86 -0
- package/src/container/restart.ts +2 -0
- package/src/container/shared.ts +8 -1
- package/src/container/start.ts +25 -13
- package/src/container/stop.ts +6 -2
- package/src/container/tool-oom-priority.ts +32 -0
- package/src/cron/consumer.ts +10 -4
- package/src/cron/reloadable.ts +23 -2
- package/src/doctor/checks.ts +14 -2
- package/src/hostd/daemon.ts +20 -10
- package/src/hostd/spawn.ts +106 -5
- package/src/hostd/version.ts +20 -0
- package/src/hostd/watchdog.ts +241 -0
- package/src/init/dockerfile.ts +57 -592
- package/src/init/index.ts +2 -3
- package/src/mcp/client.ts +3 -1
- package/src/operations/classify-tool-outcome.ts +45 -1
- package/src/operations/incidents.ts +10 -5
- package/src/plugin/core-tool-names.ts +1 -1
- package/src/plugin/define.ts +1 -0
- package/src/plugin/index.ts +1 -0
- package/src/plugin/types.ts +6 -1
- package/src/run/background-handoff-boot.ts +86 -0
- package/src/run/index.ts +83 -83
- package/src/run/resource-sample.ts +19 -0
- package/src/sandbox/build.ts +4 -0
- package/src/sandbox/coding-agent.ts +214 -0
- package/src/sandbox/policy.ts +1 -0
- package/src/sandbox/runtime-private.ts +4 -1
- package/src/secrets/coding-agent-auth.ts +97 -0
- package/src/secrets/index.ts +0 -16
- package/src/server/command-runner.ts +6 -2
- package/src/server/index.ts +101 -40
- package/src/skills/typeclaw-channel-github/SKILL.md +7 -5
- package/src/skills/typeclaw-claude-code/SKILL.md +25 -260
- package/src/skills/typeclaw-claude-code/references/auth-flow.md +7 -23
- package/src/skills/typeclaw-claude-code/references/headless-sessions.md +11 -0
- package/src/skills/typeclaw-claude-code/references/results-and-worktrees.md +13 -0
- package/src/skills/typeclaw-codex-cli/SKILL.md +35 -270
- package/src/skills/typeclaw-codex-cli/references/auth-flow.md +13 -20
- package/src/skills/typeclaw-codex-cli/references/headless-sessions.md +26 -0
- package/src/skills/typeclaw-codex-cli/references/results-and-worktrees.md +28 -0
- package/src/skills/typeclaw-config/SKILL.md +21 -21
- package/src/skills/typeclaw-config/references/dockerfile.md +14 -13
- package/src/skills/typeclaw-cron/SKILL.md +1 -1
- package/src/skills/typeclaw-github-contributing/SKILL.md +2 -2
- package/src/skills/typeclaw-memory/SKILL.md +3 -3
- package/src/skills/typeclaw-omo-cli/SKILL.md +52 -0
- package/src/skills/typeclaw-omo-cli/references/auth-flow.md +11 -0
- package/src/skills/typeclaw-omo-cli/references/headless-sessions.md +11 -0
- package/src/skills/typeclaw-omo-cli/references/results-and-worktrees.md +23 -0
- package/src/skills/typeclaw-plugins/SKILL.md +15 -15
- package/src/skills/typeclaw-skills/SKILL.md +19 -2
- package/src/skills/typeclaw-troubleshooting/SKILL.md +15 -2
- package/typeclaw.schema.json +12 -1
- package/src/secrets/claude-credentials-json.ts +0 -129
- package/src/secrets/codex-auth-json.ts +0 -102
- package/src/secrets/export-claude-credentials-file.ts +0 -270
- package/src/secrets/export-codex-auth-file.ts +0 -250
- package/src/skills/typeclaw-claude-code/references/stop-hook.md +0 -101
- package/src/skills/typeclaw-claude-code/references/tmux-driving.md +0 -243
- package/src/skills/typeclaw-codex-cli/references/stop-hook.md +0 -92
- package/src/skills/typeclaw-codex-cli/references/tmux-driving.md +0 -239
package/src/channels/router.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto'
|
|
1
2
|
import { statSync } from 'node:fs'
|
|
2
3
|
import { basename } from 'node:path'
|
|
3
4
|
|
|
@@ -9,14 +10,14 @@ import { applyTurnThinkingLevel, getQuestionSignal, type QuestionSignal } from '
|
|
|
9
10
|
import { resolveFallbackChain } from '@/agent/model-fallback'
|
|
10
11
|
import { applyModelRuntimeOverrides } from '@/agent/model-overrides'
|
|
11
12
|
import { forgetSharedLoopGuardTool } from '@/agent/plugin-tools'
|
|
12
|
-
import { detectHardProviderError, isFailoverWorthy, subscribeProviderErrors } from '@/agent/provider-error'
|
|
13
13
|
import {
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
} from '@/agent/
|
|
14
|
+
detectHardProviderError,
|
|
15
|
+
isFailoverWorthy,
|
|
16
|
+
isTransientProviderFailure,
|
|
17
|
+
isUnclassifiedProviderNotice,
|
|
18
|
+
subscribeProviderErrors,
|
|
19
|
+
} from '@/agent/provider-error'
|
|
20
|
+
import type { RestartHandoff } from '@/agent/restart-handoff'
|
|
20
21
|
import type { ChannelParticipant, SessionOrigin } from '@/agent/session-origin'
|
|
21
22
|
import { renderSubagentCompletionReminder } from '@/agent/subagent-completion-reminder'
|
|
22
23
|
import {
|
|
@@ -39,9 +40,15 @@ import { extractClaimCode } from '@/role-claim'
|
|
|
39
40
|
import type { Stream } from '@/stream'
|
|
40
41
|
|
|
41
42
|
import { extractMentionedUserIds } from './adapters/mention-hints'
|
|
43
|
+
import {
|
|
44
|
+
BackgroundObligationStore,
|
|
45
|
+
type BackgroundObligation,
|
|
46
|
+
type BackgroundObligationRef,
|
|
47
|
+
} from './background-obligations'
|
|
42
48
|
import { formatChannelCommandHelp } from './commands'
|
|
43
49
|
import { isQualifyingWorkResult } from './completion-claim'
|
|
44
50
|
import { detectContinuationWillingness } from './continuation-willingness'
|
|
51
|
+
import type { RecoveryFailure, RecoveryRecord } from './continuity-types'
|
|
45
52
|
import { describeError } from './describe-error'
|
|
46
53
|
import {
|
|
47
54
|
countEffectiveHumans,
|
|
@@ -65,6 +72,7 @@ import {
|
|
|
65
72
|
hasGithubReviewRoundDismissalAttempt,
|
|
66
73
|
isGithubReviewRoundComplete,
|
|
67
74
|
isGithubReviewRoundPending,
|
|
75
|
+
isGithubReviewRoundSuperseded,
|
|
68
76
|
promoteGithubReviewRound,
|
|
69
77
|
registerGithubReviewRound,
|
|
70
78
|
restoreGithubReviewRound,
|
|
@@ -73,6 +81,7 @@ import {
|
|
|
73
81
|
validateGithubReviewRound,
|
|
74
82
|
} from './github-review-verdict-coordinator'
|
|
75
83
|
import { renderPrVerdictStandDownReminder } from './github-verdict-activity'
|
|
84
|
+
import { InboundJournal, type InboundRecord, type InboundRef } from './inbound-journal'
|
|
76
85
|
import {
|
|
77
86
|
MEMBERSHIP_COLD_FETCH_TIMEOUT_MS,
|
|
78
87
|
type MembershipCount,
|
|
@@ -89,6 +98,8 @@ import {
|
|
|
89
98
|
saveChannelSessions,
|
|
90
99
|
type ChannelSessionRecord,
|
|
91
100
|
} from './persistence'
|
|
101
|
+
import { recoveryNoticePartitionKey } from './recovery-notice'
|
|
102
|
+
import { RecoveryOutbox } from './recovery-outbox'
|
|
92
103
|
import {
|
|
93
104
|
ADAPTER_READ_CAPABILITIES,
|
|
94
105
|
ADAPTER_WRITE_CAPABILITIES,
|
|
@@ -119,6 +130,7 @@ import type {
|
|
|
119
130
|
GithubReviewFollowupRound,
|
|
120
131
|
GithubReviewThreadCloseout,
|
|
121
132
|
InboundMessage,
|
|
133
|
+
RouteReceipt,
|
|
122
134
|
InboundReferenceContext,
|
|
123
135
|
ListCallback,
|
|
124
136
|
ListChannelsArgs,
|
|
@@ -126,6 +138,11 @@ import type {
|
|
|
126
138
|
MessageGetCallback,
|
|
127
139
|
RemoveReactionCallback,
|
|
128
140
|
RemoveReactionRequest,
|
|
141
|
+
OwnReactionTarget,
|
|
142
|
+
PrepareOwnReactionCallback,
|
|
143
|
+
RemoveOwnReactionCallback,
|
|
144
|
+
RemoveOwnReactionRequest,
|
|
145
|
+
RemoveOwnReactionResult,
|
|
129
146
|
OutboundCallback,
|
|
130
147
|
OutboundMessage,
|
|
131
148
|
QuoteAnchorSource,
|
|
@@ -147,7 +164,14 @@ import type {
|
|
|
147
164
|
SubmitReviewResult,
|
|
148
165
|
TypingCallback,
|
|
149
166
|
} from './types'
|
|
150
|
-
import { channelKeyId } from './types'
|
|
167
|
+
import { channelKeyId, fallbackChannelAccountIdentity } from './types'
|
|
168
|
+
import type {
|
|
169
|
+
RecoveryAdapterCallbacks,
|
|
170
|
+
RecoveryReconcileResult,
|
|
171
|
+
RecoveryRedelivery,
|
|
172
|
+
SendOptions as TransportSendOptions,
|
|
173
|
+
} from './types'
|
|
174
|
+
import { createWaitingReactionCoordinator, type WaitingReactionHandle } from './waiting-reactions'
|
|
151
175
|
|
|
152
176
|
export const INITIAL_DEBOUNCE_MS = 600
|
|
153
177
|
export const HOT_DEBOUNCE_MS = 1500
|
|
@@ -219,6 +243,10 @@ export const SESSION_GC_INTERVAL_MS = 60 * 1000
|
|
|
219
243
|
export const MAX_CHANNEL_SENDS_PER_TURN = 10
|
|
220
244
|
export const ENGAGE_REACTION_EMOJI = 'eyes'
|
|
221
245
|
export const CONTINUATION_REACTION_EMOJI = 'hourglass_flowing_sand'
|
|
246
|
+
// Marks the triggering message of a turn that failed for a reason the user can
|
|
247
|
+
// only answer by retrying, in place of a chat notice. Adapters that cannot
|
|
248
|
+
// render it (GitHub's fixed set, reaction-less platforms) get the text notice.
|
|
249
|
+
export const TURN_FAILED_REACTION_EMOJI = 'warning'
|
|
222
250
|
// Best-effort "zipping it / going quiet" ack dropped on the triggering message
|
|
223
251
|
// when the model disengages (channel_disengage); fire-and-forget like engage :eyes:.
|
|
224
252
|
export const DISENGAGE_REACTION_EMOJI = 'zipper_mouth_face'
|
|
@@ -234,6 +262,8 @@ export function disengageReactionEmojiFor(adapter: AdapterId): string {
|
|
|
234
262
|
return DISENGAGE_REACTION_EMOJI_OVERRIDES[adapter] ?? DISENGAGE_REACTION_EMOJI
|
|
235
263
|
}
|
|
236
264
|
|
|
265
|
+
type PendingProviderError = { turnSeq: number; safeMessage: string; retryAndSee: boolean }
|
|
266
|
+
|
|
237
267
|
type SilentAckReason = 'github_review_output' | 'awaiting_background_child'
|
|
238
268
|
|
|
239
269
|
// Wake nudge pushed into a resumed channel session at boot so drain() has a
|
|
@@ -253,37 +283,6 @@ export const RESTART_RESUME_WAKE_REMINDER = [
|
|
|
253
283
|
'---',
|
|
254
284
|
].join('\n')
|
|
255
285
|
|
|
256
|
-
// The lost-work directive: names the interrupted subagents and tells the model
|
|
257
|
-
// to inform the thread, in the audience's language, that the result was lost —
|
|
258
|
-
// never to re-run it automatically (the human decides whether to re-ask). Used
|
|
259
|
-
// standalone when a racing inbound already provides the wake turn, and embedded
|
|
260
|
-
// in the fuller resume reminder when the synthetic wake fires. Rendered as plain
|
|
261
|
-
// text so a non-Latin subagent name survives intact.
|
|
262
|
-
export function buildInterruptedSubagentNotice(interruptedSubagents: readonly string[]): string {
|
|
263
|
-
const names = interruptedSubagents.join(', ')
|
|
264
|
-
return [
|
|
265
|
-
'---',
|
|
266
|
-
'**[SYSTEM MESSAGE — not from a human]**',
|
|
267
|
-
'',
|
|
268
|
-
`A background task you had promised a result for was lost when the container`,
|
|
269
|
-
`restarted (interrupted subagent(s): ${names}). Briefly tell the people in`,
|
|
270
|
-
`this conversation, in their own language, that the result was lost to a`,
|
|
271
|
-
`restart and they can ask again if they still want it. Do not silently`,
|
|
272
|
-
`re-run it. Do not acknowledge or reply to this notice itself.`,
|
|
273
|
-
'',
|
|
274
|
-
'---',
|
|
275
|
-
].join('\n')
|
|
276
|
-
}
|
|
277
|
-
|
|
278
|
-
// The synthetic "I'm back" wake, optionally carrying the lost-work directive
|
|
279
|
-
// when the restart interrupted background subagents this session had promised
|
|
280
|
-
// results for. With none, it is the plain wake reminder unchanged.
|
|
281
|
-
export function buildRestartResumeWakeReminder(interruptedSubagents?: readonly string[]): string {
|
|
282
|
-
if (interruptedSubagents === undefined || interruptedSubagents.length === 0) {
|
|
283
|
-
return RESTART_RESUME_WAKE_REMINDER
|
|
284
|
-
}
|
|
285
|
-
return `${RESTART_RESUME_WAKE_REMINDER}\n\n${buildInterruptedSubagentNotice(interruptedSubagents)}`
|
|
286
|
-
}
|
|
287
286
|
// Ceiling on tool-source channel sends that a same-turn router policy DENIED
|
|
288
287
|
// without delivering — `skip-locked`, `turn-cap`, or `duplicate`. Such denials
|
|
289
288
|
// return a soft error and do NOT increment `consecutiveSends`, so a model that
|
|
@@ -309,13 +308,13 @@ export const MAX_POLICY_DENIED_CHANNEL_SENDS_PER_TURN = 3
|
|
|
309
308
|
// post-tool follow-up turn — can run the model's full output capability
|
|
310
309
|
// (~116s of garbage) before `stopReason: 'length'`. The terminal-reply hook
|
|
311
310
|
// below removes the turn that triggers this; the cap bounds any other path that
|
|
312
|
-
// still reaches a channel LLM call.
|
|
313
|
-
// reply (
|
|
314
|
-
//
|
|
311
|
+
// still reaches a channel LLM call. 16384 leaves reasoning models room to think
|
|
312
|
+
// and reply: 4096 (sized from kimi's ~317-token healthy turns) truncated ordinary
|
|
313
|
+
// glm-5.3 channel turns that need 4-9K tokens. Deliberately NOT lowered in `providers.ts`, where `maxTokens` is
|
|
315
314
|
// the model's true capability that compaction math reads.
|
|
316
|
-
export const CHANNEL_MAX_OUTPUT_TOKENS =
|
|
315
|
+
export const CHANNEL_MAX_OUTPUT_TOKENS = 16384
|
|
317
316
|
// Raised output-token budget threaded into the ONE re-prompt that follows a
|
|
318
|
-
// `stopReason:'length'` empty turn. The default
|
|
317
|
+
// `stopReason:'length'` empty turn. The default backstop bounds kimi's
|
|
319
318
|
// degenerate repetition loop, but it is the same ceiling a *legitimate*
|
|
320
319
|
// reasoning-heavy turn hits when it spends the whole pool thinking and emits no
|
|
321
320
|
// prose — re-prompting under the identical cap reproduces the truncation. A
|
|
@@ -324,7 +323,7 @@ export const CHANNEL_MAX_OUTPUT_TOKENS = 4096
|
|
|
324
323
|
// turn that IS looping still can't burn the model's full output capability. Consumed
|
|
325
324
|
// one-shot via `LiveSession.nextPromptMaxTokens`, then reset at the next real
|
|
326
325
|
// user turn so the raised budget never leaks past the turn that needed it.
|
|
327
|
-
export const CHANNEL_EMPTY_TURN_RETRY_MAX_OUTPUT_TOKENS =
|
|
326
|
+
export const CHANNEL_EMPTY_TURN_RETRY_MAX_OUTPUT_TOKENS = 32768
|
|
328
327
|
// Ceiling on automatic re-prompts for a turn that ended with NO user-facing
|
|
329
328
|
// reply AND no attempted send — the pure "the model burned its budget thinking
|
|
330
329
|
// and produced nothing" failure. The canonical trigger is Fireworks'
|
|
@@ -422,7 +421,7 @@ export const STRANDED_TOOLUSE_CONTINUATION_NUDGE = [
|
|
|
422
421
|
// because the turn thrashed the send path). Replaces the historical silent
|
|
423
422
|
// drop so the human is never left staring at dead air after a degenerate turn.
|
|
424
423
|
export const EMPTY_TURN_FALLBACK_TEXT =
|
|
425
|
-
"
|
|
424
|
+
"I got stuck putting together a reply and couldn't finish. Could you rephrase or try again?"
|
|
426
425
|
export const GITHUB_REVIEW_THREAD_CLOSEOUT_FALLBACK_TEXT =
|
|
427
426
|
"I wasn't able to complete this follow-up. Leaving this thread open for manual review."
|
|
428
427
|
// Distinct from EMPTY_TURN_RETRY_NUDGE: that one diagnoses budget exhaustion
|
|
@@ -754,6 +753,7 @@ export type CreateSessionForChannel = (params: {
|
|
|
754
753
|
export type ConfigForAdapter = (adapter: ChannelKey['adapter']) => ChannelAdapterConfig | undefined
|
|
755
754
|
|
|
756
755
|
type QueuedInbound = {
|
|
756
|
+
inputId: string
|
|
757
757
|
text: string
|
|
758
758
|
referenceContext?: InboundReferenceContext
|
|
759
759
|
attachments?: readonly InboundAttachment[]
|
|
@@ -762,7 +762,7 @@ type QueuedInbound = {
|
|
|
762
762
|
authorIsBot: boolean
|
|
763
763
|
externalMessageId: string
|
|
764
764
|
reactionRef?: ReactionRef
|
|
765
|
-
engageReaction?:
|
|
765
|
+
engageReaction?: WaitingReactionHandle
|
|
766
766
|
isBotMention: boolean
|
|
767
767
|
isBotMentionOnly?: boolean
|
|
768
768
|
replyToBotMessageId: string | null
|
|
@@ -832,17 +832,59 @@ type ChannelAgentSession = AgentSession & { getAbortReason?: () => string | unde
|
|
|
832
832
|
// TEXT was rejected: a wakeup body that happened to equal a nudge constant
|
|
833
833
|
// would silently misclassify, and every future enqueue site would inherit the
|
|
834
834
|
// default instead of being forced to choose.
|
|
835
|
-
type PendingSystemReminder = {
|
|
835
|
+
type PendingSystemReminder = {
|
|
836
|
+
text: string
|
|
837
|
+
kind: 'retry' | 'wakeup'
|
|
838
|
+
githubReviewRoundKey?: string
|
|
839
|
+
backgroundObligationId?: string
|
|
840
|
+
// Set on a retry handed to a reload successor: a reminder-only retry never runs
|
|
841
|
+
// beginWakeupTurn, so the successor restores the logical turn's provenance here.
|
|
842
|
+
backgroundWake?: boolean
|
|
843
|
+
inboundRetry?: { inputIds: string[]; turnId: string }
|
|
844
|
+
backgroundRetry?: {
|
|
845
|
+
turnId: string
|
|
846
|
+
obligationIds: string[]
|
|
847
|
+
emptyTurnRetries: number
|
|
848
|
+
toolLeakRetries: number
|
|
849
|
+
willingnessNudges: number
|
|
850
|
+
nextPromptMaxTokens: number | undefined
|
|
851
|
+
}
|
|
852
|
+
}
|
|
853
|
+
|
|
854
|
+
// Identity-only snapshot of the deferred durable coverage a session reloaded
|
|
855
|
+
// mid-obligation still owes (typically a turn waiting on a running background
|
|
856
|
+
// child). The journal claims stay authoritative under `turnId`/`ownerSessionId`
|
|
857
|
+
// until the successor's in-lane consume moves them; nothing here prompts.
|
|
858
|
+
type PendingReloadCoverage = {
|
|
859
|
+
inboundIds: string[]
|
|
860
|
+
backgroundIds: string[]
|
|
861
|
+
turnId: string
|
|
862
|
+
ownerSessionId: string
|
|
863
|
+
logicalTurnStartedAt: number
|
|
864
|
+
coverageDeferredSince: number | null
|
|
865
|
+
coverageDeferredParentSessionId: string | null
|
|
866
|
+
turnAccountIdentity?: string
|
|
867
|
+
// Author context a completion-only wake needs for author-scoped role
|
|
868
|
+
// resolution, exactly as the reloaded session would have restored it.
|
|
869
|
+
lastTurnAuthorId: string | null
|
|
870
|
+
lastTurnAuthorIds: string[]
|
|
871
|
+
}
|
|
836
872
|
|
|
837
873
|
type PendingReloadHandoff = {
|
|
838
874
|
key: ChannelKey
|
|
875
|
+
// Session the work was retained from. A /stop that finds no live session
|
|
876
|
+
// withdraws this session's transferred notices, as a live stop would.
|
|
877
|
+
sessionId: string
|
|
839
878
|
inbounds: QueuedInbound[]
|
|
840
879
|
observed: ObservedInbound[]
|
|
841
880
|
reminders: PendingSystemReminder[]
|
|
881
|
+
coverage: PendingReloadCoverage | null
|
|
842
882
|
retainedAt: number
|
|
843
883
|
estimatedBytes: number
|
|
844
884
|
}
|
|
845
885
|
|
|
886
|
+
const STOP_COMMAND_REPLY = 'Stopped the current turn.'
|
|
887
|
+
|
|
846
888
|
const retryReminder = (text: string): PendingSystemReminder => ({ text, kind: 'retry' })
|
|
847
889
|
|
|
848
890
|
const wakeupReminder = (text: string): PendingSystemReminder => ({ text, kind: 'wakeup' })
|
|
@@ -984,11 +1026,9 @@ type LiveSession = {
|
|
|
984
1026
|
// stop clear ALL of them, not just the latest. An anchor is added when its
|
|
985
1027
|
// 'tick' dispatches and removed when its 'stop' clear dispatches.
|
|
986
1028
|
dirtyTypingThreads: Set<string>
|
|
987
|
-
//
|
|
988
|
-
//
|
|
989
|
-
|
|
990
|
-
// removed after the reply. Empty on turns with no reactable inbound.
|
|
991
|
-
currentTurnEngageReactions: Array<Promise<ReactionRef | null>>
|
|
1029
|
+
// Captured durable owners for this coalesced turn. Releasing one owner
|
|
1030
|
+
// cannot remove another turn's or a protected acknowledgment's presence.
|
|
1031
|
+
currentTurnEngageReactions: WaitingReactionHandle[]
|
|
992
1032
|
// Model-requested `channel_react` reactions for THIS turn, held until the turn
|
|
993
1033
|
// ends. Flushed to the adapter only if the agent actually replied this turn;
|
|
994
1034
|
// discarded on silence (skip_response / empty / errored turns) so the bot never
|
|
@@ -998,32 +1038,12 @@ type LiveSession = {
|
|
|
998
1038
|
// deliberately deferred to a background child. Explicit NO_REPLY and
|
|
999
1039
|
// skip_response turns do not arm it: opting out must leave no reaction behind.
|
|
1000
1040
|
// Stamped with `turnSeq` so a stale flag from a crashed turn cannot leak into
|
|
1001
|
-
// the next one.
|
|
1002
|
-
//
|
|
1041
|
+
// the next one. Tuple arbitration preserves protected acknowledgment
|
|
1042
|
+
// presence when temporary engagement ownership is retired.
|
|
1003
1043
|
silentAckTurn: { turnSeq: number; reason: SilentAckReason } | null
|
|
1004
|
-
//
|
|
1005
|
-
//
|
|
1006
|
-
|
|
1007
|
-
// arm time, so a later replied-turn cleanup that snapshots this array always
|
|
1008
|
-
// sees every in-flight add and awaits it before removing — storing only the
|
|
1009
|
-
// resolved ref raced (cleanup could snapshot an empty array, then a slow add
|
|
1010
|
-
// append its ref afterward, stranding the :eyes:). Cross-turn state on purpose:
|
|
1011
|
-
// the mark means output is external or deferred, and once the same sticky
|
|
1012
|
-
// conversation gets a genuine reply it reads as stale — so it is retired on
|
|
1013
|
-
// the next replied turn (see dropSilentAckReactions).
|
|
1014
|
-
// Conversation-scoped, NOT per-trigger-message: coalescing means the silent
|
|
1015
|
-
// turn's trigger (message N) and the eventual reply's trigger (message N+1)
|
|
1016
|
-
// routinely differ, so exact-message scoping would strand the mark. NOT reset
|
|
1017
|
-
// in drain's outer per-turn finally. On teardown the entries are dropped
|
|
1018
|
-
// WITHOUT removing the reactions, unlike the transient engage :eyes: which
|
|
1019
|
-
// teardown must strip.
|
|
1020
|
-
activeSilentAckReactions: Array<Promise<ReactionRef | null>>
|
|
1021
|
-
// One add promise per willingness status posted by the agent, resolving to
|
|
1022
|
-
// the removable reaction-instance ref. Cross-turn state on purpose: the
|
|
1023
|
-
// hourglass stays visible until a substantive result, fallback, explicit
|
|
1024
|
-
// silence, stop, or teardown retires every outstanding promise. Storing the
|
|
1025
|
-
// promise synchronously prevents cleanup racing a slow reaction add.
|
|
1026
|
-
activeContinuationReactions: Array<Promise<ReactionRef | null>>
|
|
1044
|
+
// Temporary continuation indicators are owned across turns until result,
|
|
1045
|
+
// silence, stop, teardown, or recovery explicitly releases their owner.
|
|
1046
|
+
activeContinuationReactions: WaitingReactionHandle[]
|
|
1027
1047
|
lastTurnAuthorIds: Set<string>
|
|
1028
1048
|
// Mirror of currentTurnAuthorId at end-of-turn (the LAST speaker of the
|
|
1029
1049
|
// prior batch), preserved across the drain finally-block which resets
|
|
@@ -1049,6 +1069,16 @@ type LiveSession = {
|
|
|
1049
1069
|
// the drain loop's run condition checks BOTH queues so a system
|
|
1050
1070
|
// reminder alone is enough to trigger a turn.
|
|
1051
1071
|
pendingSystemReminders: PendingSystemReminder[]
|
|
1072
|
+
// Identity-only cache; every durable transition resolves ownership in its lane.
|
|
1073
|
+
backgroundCoverage: string[]
|
|
1074
|
+
inboundCoverage: string[]
|
|
1075
|
+
turnAccountIdentity?: string
|
|
1076
|
+
backgroundTurnId: string
|
|
1077
|
+
// Session holding the claims under `backgroundTurnId`: the creating session, a
|
|
1078
|
+
// reload source restored at adoption, or this session once a consume acquires a
|
|
1079
|
+
// turn. A successor may run under another ID than the one its adopted rows name.
|
|
1080
|
+
backgroundTurnOwnerSessionId: string
|
|
1081
|
+
backgroundStopVersion: number
|
|
1052
1082
|
// True only for the reminder-only iteration that consumed a willingness
|
|
1053
1083
|
// nudge. `willingnessNudges` persists across the logical turn, so it remains
|
|
1054
1084
|
// nonzero after a substantive result and would misclassify NO_REPLY from a
|
|
@@ -1101,6 +1131,17 @@ type LiveSession = {
|
|
|
1101
1131
|
// `startedAt` against, to tell "this turn spawned it" from "it was already
|
|
1102
1132
|
// running when unrelated work arrived".
|
|
1103
1133
|
logicalTurnStartedAt: number
|
|
1134
|
+
// `logicalTurnStartedAt` of the turn that deferred open coverage (inbound requests,
|
|
1135
|
+
// consumed child results, or both) to a background child, kept while fresh batches
|
|
1136
|
+
// carry that coverage forward. Each carrying batch advances `logicalTurnStartedAt`,
|
|
1137
|
+
// so without this a later batch would stop seeing the original child and declare
|
|
1138
|
+
// the coverage lost.
|
|
1139
|
+
coverageDeferredSince: number | null
|
|
1140
|
+
// Session that spawned the child that coverage was deferred to, kept with
|
|
1141
|
+
// `coverageDeferredSince` while it is carried; null means the claim owner. A carry
|
|
1142
|
+
// moves the claims to the carrying session, which after a reload can have another
|
|
1143
|
+
// session ID than that child's parent.
|
|
1144
|
+
coverageDeferredParentSessionId: string | null
|
|
1104
1145
|
// Snapshot of `successfulChannelSends` taken at turn start (same
|
|
1105
1146
|
// moment `turnSeq` increments). Lets `markTurnSkipped` detect "a
|
|
1106
1147
|
// channel send already landed in this turn" and reject the skip,
|
|
@@ -1274,6 +1315,17 @@ type LiveSession = {
|
|
|
1274
1315
|
// retry iterations that end the turn. Reset ONLY on a real user batch,
|
|
1275
1316
|
// beside `githubReviewOutputTurn`.
|
|
1276
1317
|
toolExecutionThisLogicalTurn: boolean
|
|
1318
|
+
// True while the logical turn was opened by background work (a completed
|
|
1319
|
+
// subagent's wakeup) rather than a user message. Only a turn answering a user
|
|
1320
|
+
// may error out in the channel: a failure here is logged for the operator and
|
|
1321
|
+
// its coverage settles silently, never as a provider-error, empty-turn or
|
|
1322
|
+
// live-turn-ended notice. Set by beginWakeupTurn, cleared by a real user batch.
|
|
1323
|
+
backgroundWakeLogicalTurn: boolean
|
|
1324
|
+
// Set when this logical turn's provider-error notice was suppressed because
|
|
1325
|
+
// the turn already ran tools. The same failure must not resurface as a
|
|
1326
|
+
// live-turn-ended notice, so turn-end settles held coverage silently.
|
|
1327
|
+
// Cleared with the other logical-turn state.
|
|
1328
|
+
providerErrorSilencedThisLogicalTurn: boolean
|
|
1277
1329
|
// Successful, non-communication, non-control tool output observed during
|
|
1278
1330
|
// this logical turn. Unlike toolExecutionThisLogicalTurn, channel_reply and
|
|
1279
1331
|
// todo/control calls cannot satisfy a later durable completion claim.
|
|
@@ -1288,6 +1340,19 @@ type LiveSession = {
|
|
|
1288
1340
|
// also fulfills it, while a continuation-willingness/status send preserves
|
|
1289
1341
|
// it. A real user batch resets it beside `toolExecutionThisLogicalTurn`.
|
|
1290
1342
|
promisedWorkOutstandingThisLogicalTurn: boolean
|
|
1343
|
+
// True while the latest landed output accounted to this logical turn was a
|
|
1344
|
+
// continuation-willingness/status message ("I'll check and get back to you"):
|
|
1345
|
+
// the turn told the user more was coming without the machine-readable
|
|
1346
|
+
// `more_work_this_turn` flag. A substantive send clears it; meta notices
|
|
1347
|
+
// (provider error, fallbacks) leave it. Together with
|
|
1348
|
+
// `promisedWorkOutstandingThisLogicalTurn` it marks promised unfinished work,
|
|
1349
|
+
// which explicit silence can never settle. Reset with the logical turn.
|
|
1350
|
+
statusOutputOutstandingThisLogicalTurn: boolean
|
|
1351
|
+
// Backing turn id whose owed coverage a documented guard deferred to the next
|
|
1352
|
+
// inbound (the GitHub recovery review guard: "the next inbound re-prompts the
|
|
1353
|
+
// model"). That fresh batch carries the coverage instead of transferring it as
|
|
1354
|
+
// abandoned; any other turn id is stale. Cleared when a fresh batch consumes it.
|
|
1355
|
+
nextInboundContinuationTurnId: string | null
|
|
1291
1356
|
// Captured by drain() at batch dequeue; read+cleared by send() on the
|
|
1292
1357
|
// first tool-source send of the turn. The anchor decision (delay
|
|
1293
1358
|
// threshold + intervening-observed check) is evaluated at SEND time
|
|
@@ -1334,7 +1399,7 @@ type LiveSession = {
|
|
|
1334
1399
|
// turn-end finally posts `safeMessage` ONLY if no reply landed, else discards.
|
|
1335
1400
|
// First error per turn wins. Keyed by `turnSeq` so a stale record from a
|
|
1336
1401
|
// crashed prior turn can't leak into the next.
|
|
1337
|
-
pendingProviderError:
|
|
1402
|
+
pendingProviderError: PendingProviderError | null
|
|
1338
1403
|
destroyed: boolean
|
|
1339
1404
|
// Set by tearDownAllLive when a reload/roles/auth swap wants to recreate this
|
|
1340
1405
|
// session but a turn is mid-flight. Aborting now would strand the in-flight
|
|
@@ -1389,7 +1454,7 @@ export type ExecuteCommandOptions = {
|
|
|
1389
1454
|
|
|
1390
1455
|
export type SendSource = 'tool' | 'system'
|
|
1391
1456
|
|
|
1392
|
-
export type SendOptions = {
|
|
1457
|
+
export type SendOptions = TransportSendOptions & {
|
|
1393
1458
|
source?: SendSource
|
|
1394
1459
|
// Apply tool-send accounting to the originating conversation while delivering
|
|
1395
1460
|
// elsewhere. The outbound target remains unchanged; only per-turn cap/dedup,
|
|
@@ -1417,8 +1482,26 @@ export const SKIP_RESPONSE_LOCK_ERROR =
|
|
|
1417
1482
|
'something to say, send it on the next turn.'
|
|
1418
1483
|
|
|
1419
1484
|
export type ChannelRouter = {
|
|
1420
|
-
route: (event: InboundMessage) => Promise<void>
|
|
1485
|
+
route: (event: InboundMessage, onAdmitted?: () => Promise<void>) => Promise<RouteReceipt>
|
|
1421
1486
|
send: (msg: OutboundMessage, opts?: SendOptions) => Promise<SendResult>
|
|
1487
|
+
acceptBackgroundResponse: (args: {
|
|
1488
|
+
parentSessionId: string
|
|
1489
|
+
parentSessionFile?: string
|
|
1490
|
+
key: ChannelKey
|
|
1491
|
+
taskId: string
|
|
1492
|
+
subagentName: string
|
|
1493
|
+
startedAt: number
|
|
1494
|
+
accountIdentity: string
|
|
1495
|
+
triggeringAuthorId?: string
|
|
1496
|
+
parentChat?: string
|
|
1497
|
+
}) => Promise<BackgroundObligationRef>
|
|
1498
|
+
suppressUnstartedBackgroundResponse: (ref: BackgroundObligationRef, reason: string) => Promise<void>
|
|
1499
|
+
attachBackgroundResultCoverage: (args: { parentSessionId: string; taskId: string }) => Promise<void>
|
|
1500
|
+
captureBackgroundResultCoverage?: (parentSessionId: string) => Promise<BackgroundObligationRef[]>
|
|
1501
|
+
captureInboundResultCoverage?: (parentSessionId: string) => Promise<InboundRef[]>
|
|
1502
|
+
// Scoped to `adapter`: an identity is only comparable with accounts on the same
|
|
1503
|
+
// platform, so a turn that came from another adapter has no binding to enforce.
|
|
1504
|
+
captureTurnAccountIdentity?: (parentSessionId: string, adapter: ChannelKey['adapter']) => Promise<string | undefined>
|
|
1422
1505
|
getConsecutiveSendCount: (target: {
|
|
1423
1506
|
adapter: ChannelKey['adapter']
|
|
1424
1507
|
workspace: string
|
|
@@ -1434,6 +1517,13 @@ export type ChannelRouter = {
|
|
|
1434
1517
|
}) => { count: number; windowMs: number }
|
|
1435
1518
|
registerOutbound: (adapter: ChannelKey['adapter'], cb: OutboundCallback) => void
|
|
1436
1519
|
unregisterOutbound: (adapter: ChannelKey['adapter'], cb: OutboundCallback) => void
|
|
1520
|
+
registerRecoveryAdapter: (adapter: ChannelKey['adapter'], callbacks: RecoveryAdapterCallbacks) => void
|
|
1521
|
+
unregisterRecoveryAdapter: (adapter: ChannelKey['adapter'], callbacks: RecoveryAdapterCallbacks) => void
|
|
1522
|
+
getRecoveryAccountIdentity: (adapter: ChannelKey['adapter'], workspace?: string) => Promise<string | undefined>
|
|
1523
|
+
validateRecovery: (record: RecoveryRecord) => Promise<RecoveryFailure | undefined>
|
|
1524
|
+
reconcileRecovery: (record: RecoveryRecord) => Promise<RecoveryReconcileResult>
|
|
1525
|
+
recoveryRedelivery?: (record: RecoveryRecord, eventKinds: readonly string[]) => RecoveryRedelivery
|
|
1526
|
+
setRecoveryStopHandler: (handler: (target: ChannelKey, parentSessionId: string) => Promise<void>) => void
|
|
1437
1527
|
// Reaction support is opt-in per adapter: an adapter that never calls
|
|
1438
1528
|
// registerReaction makes `react` resolve to `code: 'unsupported'`, and
|
|
1439
1529
|
// auto-react-on-engage becomes a silent no-op for it. Kept separate from
|
|
@@ -1450,6 +1540,12 @@ export type ChannelRouter = {
|
|
|
1450
1540
|
registerRemoveReaction: (adapter: ChannelKey['adapter'], cb: RemoveReactionCallback) => void
|
|
1451
1541
|
unregisterRemoveReaction: (adapter: ChannelKey['adapter'], cb: RemoveReactionCallback) => void
|
|
1452
1542
|
removeReaction: (req: RemoveReactionRequest) => Promise<ReactionResult>
|
|
1543
|
+
registerPrepareOwnReaction: (adapter: ChannelKey['adapter'], cb: PrepareOwnReactionCallback) => void
|
|
1544
|
+
unregisterPrepareOwnReaction: (adapter: ChannelKey['adapter'], cb: PrepareOwnReactionCallback) => void
|
|
1545
|
+
registerRemoveOwnReaction: (adapter: ChannelKey['adapter'], cb: RemoveOwnReactionCallback) => void
|
|
1546
|
+
unregisterRemoveOwnReaction: (adapter: ChannelKey['adapter'], cb: RemoveOwnReactionCallback) => void
|
|
1547
|
+
removeOwnReaction: (req: RemoveOwnReactionRequest) => Promise<RemoveOwnReactionResult>
|
|
1548
|
+
recoverWaitingReactions: (adapter?: ChannelKey['adapter']) => Promise<void>
|
|
1453
1549
|
registerTyping: (adapter: ChannelKey['adapter'], cb: TypingCallback) => void
|
|
1454
1550
|
unregisterTyping: (adapter: ChannelKey['adapter'], cb: TypingCallback) => void
|
|
1455
1551
|
// Deliberately separate from registerTyping: github registers a no-op typing
|
|
@@ -1569,7 +1665,7 @@ export type ChannelRouter = {
|
|
|
1569
1665
|
durationMs: number
|
|
1570
1666
|
error?: string
|
|
1571
1667
|
channelKey?: { adapter: string; workspace: string; chat: string; thread: string | null }
|
|
1572
|
-
}) => { kind: 'delivered'; keyId: string } | { kind: 'no-live-session' }
|
|
1668
|
+
}) => Promise<{ kind: 'delivered'; keyId: string } | { kind: 'no-live-session' }>
|
|
1573
1669
|
// Fan a LANDED formal review verdict out to the OTHER live sessions reviewing
|
|
1574
1670
|
// the same PR so they stand down from posting a redundant verdict. Targets every
|
|
1575
1671
|
// live github session whose chat is `pr:<prNumber>` in `workspace`, EXCLUDING the
|
|
@@ -1596,6 +1692,7 @@ export type ChannelRouter = {
|
|
|
1596
1692
|
prNumber: number
|
|
1597
1693
|
verdict: ReviewRoundOutcome
|
|
1598
1694
|
sessionId: string
|
|
1695
|
+
commitSha?: string
|
|
1599
1696
|
}) => Promise<{ kind: 'completed' | 'no-round' }>
|
|
1600
1697
|
finishGithubReviewThreadCloseout?: (args: {
|
|
1601
1698
|
sessionId: string
|
|
@@ -1609,7 +1706,9 @@ export type ChannelRouter = {
|
|
|
1609
1706
|
workspace: string
|
|
1610
1707
|
prNumber: number
|
|
1611
1708
|
state: ReviewOutputState
|
|
1612
|
-
|
|
1709
|
+
inboundCoverage?: InboundRef[]
|
|
1710
|
+
backgroundCoverage?: BackgroundObligationRef[]
|
|
1711
|
+
}) => Promise<{ kind: 'stamped' | 'no-live-session' }>
|
|
1613
1712
|
hasOutstandingGithubReviewThreadCloseout?: (sessionId: string) => boolean
|
|
1614
1713
|
// Record that the agent invoked `skip_response` during the current turn
|
|
1615
1714
|
// for the channel session identified by `parentSessionId`. The reason is
|
|
@@ -1634,10 +1733,9 @@ export type ChannelRouter = {
|
|
|
1634
1733
|
markTurnSkipped: (args: {
|
|
1635
1734
|
parentSessionId: string
|
|
1636
1735
|
reason: string
|
|
1637
|
-
}) =>
|
|
1638
|
-
| { kind: 'recorded'; keyId: string }
|
|
1639
|
-
|
|
1640
|
-
| { kind: 'no-live-session' }
|
|
1736
|
+
}) => Promise<
|
|
1737
|
+
{ kind: 'recorded'; keyId: string } | { kind: 'recorded-after-send'; keyId: string } | { kind: 'no-live-session' }
|
|
1738
|
+
>
|
|
1641
1739
|
// Force-clear every sticky credit for one channel key. Stickiness normally
|
|
1642
1740
|
// expires on TTL or is consumed on the next inbound, but in a busy group each
|
|
1643
1741
|
// reply re-grants a fresh credit, so the bot can stay force-engaged turn after
|
|
@@ -1662,12 +1760,9 @@ export type ChannelRouter = {
|
|
|
1662
1760
|
// Graceful-restart shutdown: mark + abort every live channel session so each
|
|
1663
1761
|
// scope's incomplete todos auto-continue on the next boot. See the impl.
|
|
1664
1762
|
markRestartAbortForAllLive: () => Promise<void>
|
|
1665
|
-
// Graceful-restart shutdown: persist a channel handoff naming the background
|
|
1666
|
-
// subagents a live session was still awaiting, so the resume greeting can tell
|
|
1667
|
-
// that thread the promised result was lost. Resolves true iff one was written.
|
|
1668
|
-
writeInterruptedSubagentHandoff: () => Promise<boolean>
|
|
1669
1763
|
liveCount: () => number
|
|
1670
1764
|
__testing?: {
|
|
1765
|
+
getRecoveryAccountingSnapshot: (key: ChannelKey) => Record<string, unknown> | undefined
|
|
1671
1766
|
flushDebounce: (key: ChannelKey) => Promise<void>
|
|
1672
1767
|
fireTypingHeartbeat: (key: ChannelKey, phase?: 'tick' | 'stop') => Promise<void>
|
|
1673
1768
|
fireTypingInterval: (key: ChannelKey) => Promise<void>
|
|
@@ -1701,11 +1796,6 @@ export type ChannelRouter = {
|
|
|
1701
1796
|
// (a continuation re-prompt queued after a willingness-ack strand) at the
|
|
1702
1797
|
// channels layer, without coupling to todo-file persistence formats.
|
|
1703
1798
|
injectContinuationReminder: (key: ChannelKey, text: string) => void
|
|
1704
|
-
// Enqueues a real user batch onto the live session's promptQueue via the same
|
|
1705
|
-
// `enqueue` path route() uses, letting a test reproduce a fresh inbound that
|
|
1706
|
-
// supersedes a still-pending staged fallback mid-drain (the A → staged B →
|
|
1707
|
-
// queued C question-signal supersession case) without the debounce timing dance.
|
|
1708
|
-
enqueueUserInbound: (key: ChannelKey, event: InboundMessage) => void
|
|
1709
1799
|
// Returns a shallow copy of `live.originRef.current` for the live
|
|
1710
1800
|
// session matching `key`, or undefined when no live session exists.
|
|
1711
1801
|
// Exists so tests can assert on the per-turn origin that tool.before
|
|
@@ -1732,6 +1822,10 @@ export type AliasesProvider = () => readonly string[]
|
|
|
1732
1822
|
|
|
1733
1823
|
export type CreateChannelRouterOptions = {
|
|
1734
1824
|
agentDir: string
|
|
1825
|
+
bootEpoch?: string
|
|
1826
|
+
backgroundObligations?: BackgroundObligationStore
|
|
1827
|
+
inboundJournal?: InboundJournal
|
|
1828
|
+
recoveryOutbox?: RecoveryOutbox
|
|
1735
1829
|
configForAdapter: ConfigForAdapter
|
|
1736
1830
|
configuredAliases?: AliasesProvider
|
|
1737
1831
|
createSessionForChannel?: CreateSessionForChannel
|
|
@@ -1832,11 +1926,6 @@ export type CreateChannelRouterOptions = {
|
|
|
1832
1926
|
// spawned child is still inside its window. Production wiring forwards the
|
|
1833
1927
|
// LiveSubagentRegistry; omitted (tests, no-subagent setups) means no pin.
|
|
1834
1928
|
newestRunningChildSubagentStartedAt?: (sessionId: string) => number | null
|
|
1835
|
-
// Names of the still-running BACKGROUND subagents for a parent session, used
|
|
1836
|
-
// by writeInterruptedSubagentHandoff on graceful restart to name the lost
|
|
1837
|
-
// work in the resume greeting. Background-only: a foreground child returns its
|
|
1838
|
-
// result inline, so it is not orphaned by the bounce. Omitted means none.
|
|
1839
|
-
listRunningBackgroundSubagentNames?: (sessionId: string) => string[]
|
|
1840
1929
|
cancelRunningSubagentsByWorkKey?: (
|
|
1841
1930
|
workKey: string,
|
|
1842
1931
|
reason: string,
|
|
@@ -1872,6 +1961,7 @@ export type RestartReservation = {
|
|
|
1872
1961
|
keyId: string
|
|
1873
1962
|
sawInbound: boolean
|
|
1874
1963
|
resume: () => Promise<void>
|
|
1964
|
+
release: () => void
|
|
1875
1965
|
}
|
|
1876
1966
|
|
|
1877
1967
|
export type ClaimHandler = (input: ClaimHandlerInput) => Promise<ClaimHandlerOutcome>
|
|
@@ -1888,6 +1978,12 @@ const GRANT_ALL_PERMISSIONS: PermissionService = {
|
|
|
1888
1978
|
export function createChannelRouter(options: CreateChannelRouterOptions): ChannelRouter {
|
|
1889
1979
|
const logger = options.logger ?? consoleLogger
|
|
1890
1980
|
const now = options.now ?? Date.now
|
|
1981
|
+
const backgroundObligations = options.backgroundObligations ?? new BackgroundObligationStore(options.agentDir)
|
|
1982
|
+
const inboundJournal = options.inboundJournal ?? new InboundJournal(options.agentDir, { backgroundObligations })
|
|
1983
|
+
const ready = inboundJournal.initialize()
|
|
1984
|
+
const recoveryOutbox =
|
|
1985
|
+
options.recoveryOutbox ?? new RecoveryOutbox(options.agentDir, { epoch: backgroundObligations.epoch })
|
|
1986
|
+
void ready.catch((error) => logger.error(`[channels] inbound journal initialization failed: ${describeError(error)}`))
|
|
1891
1987
|
const measureTranscriptBytes = options.measureTranscriptBytes ?? defaultMeasureTranscriptBytes
|
|
1892
1988
|
const ensureLiveTimeoutMs = options.ensureLiveTimeoutMs ?? ENSURE_LIVE_TIMEOUT_MS
|
|
1893
1989
|
const handoffRetryRetentionMs = options.handoffRetryRetentionMs ?? RELOAD_HANDOFF_RETENTION_MS
|
|
@@ -1903,6 +1999,20 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
1903
1999
|
const onRestart = options.onRestart
|
|
1904
2000
|
const newestRunningChildSubagentStartedAt = options.newestRunningChildSubagentStartedAt ?? (() => null)
|
|
1905
2001
|
const liveSessions = new Map<string, LiveSession>()
|
|
2002
|
+
const unsubscribeJournalFailure = inboundJournal.subscribeFailure((error) => {
|
|
2003
|
+
// Abort the SDK's active loop synchronously, not session.abort() (which waits
|
|
2004
|
+
// for that loop). A failed admission can occur inside another turn's fetch.
|
|
2005
|
+
for (const live of liveSessions.values()) {
|
|
2006
|
+
if (live.destroyed || !live.promptInFlight) continue
|
|
2007
|
+
live.abortReasonThisTurn = { turnSeq: live.turnSeq, reason: 'continuity-unavailable' }
|
|
2008
|
+
try {
|
|
2009
|
+
live.session.agent.abort()
|
|
2010
|
+
} catch (abortError) {
|
|
2011
|
+
logger.error(`[channels] continuity cancellation failed: ${describeError(abortError)}`)
|
|
2012
|
+
}
|
|
2013
|
+
}
|
|
2014
|
+
logger.error(`[channels] cancelling in-flight turns after continuity failure: ${describeError(error)}`)
|
|
2015
|
+
})
|
|
1906
2016
|
const creating = new Map<string, Promise<LiveSession>>()
|
|
1907
2017
|
let nextCreationToken = 0
|
|
1908
2018
|
const activeCreationAttempts = new Map<string, number>()
|
|
@@ -1935,8 +2045,12 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
1935
2045
|
// teardown was meant to clear.
|
|
1936
2046
|
let liveGeneration = 0
|
|
1937
2047
|
const outboundCallbacks = new Map<ChannelKey['adapter'], Set<OutboundCallback>>()
|
|
2048
|
+
const recoveryAdapters = new Map<ChannelKey['adapter'], RecoveryAdapterCallbacks>()
|
|
2049
|
+
let recoveryStopHandler: ((target: ChannelKey, parentSessionId: string) => Promise<void>) | undefined
|
|
1938
2050
|
const reactionCallbacks = new Map<ChannelKey['adapter'], Set<ReactionCallback>>()
|
|
1939
2051
|
const removeReactionCallbacks = new Map<ChannelKey['adapter'], Set<RemoveReactionCallback>>()
|
|
2052
|
+
const prepareOwnReactionCallbacks = new Map<ChannelKey['adapter'], Set<PrepareOwnReactionCallback>>()
|
|
2053
|
+
const removeOwnReactionCallbacks = new Map<ChannelKey['adapter'], Set<RemoveOwnReactionCallback>>()
|
|
1940
2054
|
const typingCallbacks = new Map<ChannelKey['adapter'], Set<TypingCallback>>()
|
|
1941
2055
|
const typingCapableAdapters = new Set<ChannelKey['adapter']>()
|
|
1942
2056
|
const typingHeartbeatIntervals = new Map<ChannelKey['adapter'], number>()
|
|
@@ -1974,7 +2088,7 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
1974
2088
|
// requiresLiveSession:true guarantees the dispatch layer resolved a
|
|
1975
2089
|
// session before running this handler, so `live` is non-null here.
|
|
1976
2090
|
await stopCurrentChannelTurn(live!, 'user_stop')
|
|
1977
|
-
return { reply:
|
|
2091
|
+
return { reply: STOP_COMMAND_REPLY }
|
|
1978
2092
|
},
|
|
1979
2093
|
},
|
|
1980
2094
|
]
|
|
@@ -2032,6 +2146,10 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
2032
2146
|
// when it evaluates; a write appended afterward would still race a caller that
|
|
2033
2147
|
// deletes the agent dir right after stop() resolves.
|
|
2034
2148
|
let closing = false
|
|
2149
|
+
// Set once a graceful restart aborts every live turn. Those turns end because
|
|
2150
|
+
// the process is restarting, so their owed coverage stays for boot recovery's
|
|
2151
|
+
// restart notice instead of being transferred as a live-turn-ended one.
|
|
2152
|
+
let restartShutdownPending = false
|
|
2035
2153
|
|
|
2036
2154
|
const ensureLoaded = async (): Promise<void> => {
|
|
2037
2155
|
if (mappings !== null) return
|
|
@@ -2278,10 +2396,13 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
2278
2396
|
live: LiveSession,
|
|
2279
2397
|
label: string,
|
|
2280
2398
|
startedAt = live.logicalTurnStartedAt,
|
|
2399
|
+
// Deferred coverage can wait on another session's child; a reload successor
|
|
2400
|
+
// under a new session ID owns none of them in the registry.
|
|
2401
|
+
parentSessionId = live.sessionId,
|
|
2281
2402
|
): boolean => {
|
|
2282
|
-
const childStartedAt = newestRunningChildSubagentStartedAt(
|
|
2403
|
+
const childStartedAt = newestRunningChildSubagentStartedAt(parentSessionId)
|
|
2283
2404
|
if (childStartedAt === null || childStartedAt < startedAt) return false
|
|
2284
|
-
return isPinnedByRunningChild(
|
|
2405
|
+
return isPinnedByRunningChild(parentSessionId, live.keyId, label)
|
|
2285
2406
|
}
|
|
2286
2407
|
|
|
2287
2408
|
const shouldRolloverLive = (live: LiveSession, idleMs: number): boolean => {
|
|
@@ -2311,86 +2432,416 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
2311
2432
|
}
|
|
2312
2433
|
|
|
2313
2434
|
const pendingReloadHandoffItemCount = (pending: PendingReloadHandoff): number =>
|
|
2314
|
-
pending.inbounds.length +
|
|
2435
|
+
pending.inbounds.length +
|
|
2436
|
+
pending.observed.length +
|
|
2437
|
+
pending.reminders.length +
|
|
2438
|
+
(pending.coverage === null ? 0 : pending.coverage.inboundIds.length + pending.coverage.backgroundIds.length)
|
|
2315
2439
|
|
|
2316
2440
|
const estimateReloadHandoffBytes = (
|
|
2317
2441
|
inbounds: QueuedInbound[],
|
|
2318
2442
|
observed: ObservedInbound[],
|
|
2319
2443
|
reminders: PendingSystemReminder[],
|
|
2444
|
+
coverage: PendingReloadCoverage | null,
|
|
2320
2445
|
): number => {
|
|
2321
2446
|
try {
|
|
2322
2447
|
const serializableInbounds = inbounds.map(({ engageReaction: _engageReaction, ...inbound }) => inbound)
|
|
2323
|
-
return Buffer.byteLength(
|
|
2448
|
+
return Buffer.byteLength(
|
|
2449
|
+
JSON.stringify({ inbounds: serializableInbounds, observed, reminders, coverage }),
|
|
2450
|
+
'utf8',
|
|
2451
|
+
)
|
|
2324
2452
|
} catch {
|
|
2325
2453
|
return Number.POSITIVE_INFINITY
|
|
2326
2454
|
}
|
|
2327
2455
|
}
|
|
2328
2456
|
|
|
2457
|
+
// Every durable identity a handoff owns, for the transfer that ends it
|
|
2458
|
+
// without a successor (capacity or expiry) and for /stop withdrawal.
|
|
2459
|
+
const pendingReloadHandoffInputIds = (pending: PendingReloadHandoff): string[] => [
|
|
2460
|
+
...(pending.coverage?.inboundIds ?? []),
|
|
2461
|
+
...pending.inbounds.flatMap((input) => (input.inputId ? [input.inputId] : [])),
|
|
2462
|
+
...pending.reminders.flatMap((r) => r.inboundRetry?.inputIds ?? []),
|
|
2463
|
+
]
|
|
2464
|
+
|
|
2465
|
+
const pendingReloadHandoffBackgroundIds = (pending: PendingReloadHandoff): string[] => [
|
|
2466
|
+
...(pending.coverage?.backgroundIds ?? []),
|
|
2467
|
+
...pending.reminders.flatMap((r) => [
|
|
2468
|
+
...(r.backgroundObligationId ? [r.backgroundObligationId] : []),
|
|
2469
|
+
...(r.backgroundRetry?.obligationIds ?? []),
|
|
2470
|
+
]),
|
|
2471
|
+
]
|
|
2472
|
+
|
|
2329
2473
|
const pendingReloadHandoffTotalBytes = (): number => {
|
|
2330
2474
|
let total = 0
|
|
2331
2475
|
for (const pending of pendingReloadHandoffs.values()) total += pending.estimatedBytes
|
|
2332
2476
|
return total
|
|
2333
2477
|
}
|
|
2334
2478
|
|
|
2335
|
-
|
|
2336
|
-
|
|
2337
|
-
|
|
2338
|
-
|
|
2339
|
-
|
|
2340
|
-
|
|
2341
|
-
|
|
2479
|
+
// Preparation advanced the source generation before publication/ownership could
|
|
2480
|
+
// fail. Resume only these cached identities, with their immutable frozen covers;
|
|
2481
|
+
// their old turn claims are no longer authority for this transition.
|
|
2482
|
+
const resumePreparedTransfersInLane = async (
|
|
2483
|
+
target: ChannelKey,
|
|
2484
|
+
inputIds: readonly string[],
|
|
2485
|
+
backgroundIds: readonly string[],
|
|
2486
|
+
): Promise<void> => {
|
|
2487
|
+
await ready
|
|
2488
|
+
inboundJournal.assertAvailable()
|
|
2489
|
+
const handled = new Set<string>()
|
|
2490
|
+
const resume = async (
|
|
2491
|
+
row: InboundRecord | BackgroundObligation | undefined,
|
|
2492
|
+
store: 'inbound' | 'background',
|
|
2493
|
+
id: string,
|
|
2494
|
+
) => {
|
|
2495
|
+
if (row?.phase !== 'notice-prepared') return
|
|
2496
|
+
const frozen = row.transfer
|
|
2497
|
+
if (
|
|
2498
|
+
!frozen ||
|
|
2499
|
+
channelKeyId(row.target) !== channelKeyId(target) ||
|
|
2500
|
+
channelKeyId(frozen.target) !== channelKeyId(target) ||
|
|
2501
|
+
!frozen.covers.some((cover) => cover.store === store && cover.id === id && cover.generation === row.generation)
|
|
2502
|
+
)
|
|
2503
|
+
throw new Error('Prepared notice coverage mismatch')
|
|
2504
|
+
if (handled.has(frozen.deliveryId)) return
|
|
2505
|
+
if (frozen.covers.some((cover) => cover.store === 'inbound')) {
|
|
2506
|
+
await inboundJournal.importPrepared(recoveryOutbox, frozen)
|
|
2507
|
+
} else {
|
|
2508
|
+
const imported = await recoveryOutbox.import(frozen)
|
|
2509
|
+
for (const cover of frozen.covers) {
|
|
2510
|
+
if (cover.store !== 'background') throw new Error('Invalid prepared background coverage')
|
|
2511
|
+
const owned = await backgroundObligations.ownNotice(cover.id, cover.generation, imported.deliveryId)
|
|
2512
|
+
if (!owned && imported.state !== 'delivered' && imported.state !== 'suppressed')
|
|
2513
|
+
throw new Error('Background notice ownership conflict')
|
|
2514
|
+
}
|
|
2515
|
+
await backgroundObligations.acknowledgeNotice(imported)
|
|
2516
|
+
}
|
|
2517
|
+
handled.add(frozen.deliveryId)
|
|
2518
|
+
}
|
|
2519
|
+
for (const id of new Set(inputIds)) await resume(inboundJournal.get(id), 'inbound', id)
|
|
2520
|
+
for (const id of new Set(backgroundIds)) await resume(await backgroundObligations.get(id), 'background', id)
|
|
2521
|
+
}
|
|
2522
|
+
|
|
2523
|
+
// Every transfer the router prepares ends live work in THIS process: a turn
|
|
2524
|
+
// that ended without answering, an account replaced under it, a reload or
|
|
2525
|
+
// lifecycle bound that refused its handoff. None of them is a restart, so the
|
|
2526
|
+
// frozen notice says the live turn ended. Restart notices are prepared only by
|
|
2527
|
+
// boot recovery of an older epoch; router stop() leaves its debt for that.
|
|
2528
|
+
const transferLostCoverageInLane = async (
|
|
2529
|
+
target: ChannelKey,
|
|
2530
|
+
inputIds: readonly string[],
|
|
2531
|
+
backgroundIds: readonly string[],
|
|
2532
|
+
ownerSessionId?: string,
|
|
2533
|
+
): Promise<void> => {
|
|
2534
|
+
await ready
|
|
2535
|
+
inboundJournal.assertAvailable()
|
|
2536
|
+
await resumePreparedTransfersInLane(target, inputIds, backgroundIds)
|
|
2537
|
+
// Frozen notices retain the admission's principal, including its original author,
|
|
2538
|
+
// grouped by the same partition boot recovery uses for old-epoch rows.
|
|
2539
|
+
const groups = new Map<string, { inboundRefs: InboundRef[]; backgroundRefs: BackgroundObligationRef[] }>()
|
|
2540
|
+
for (const ref of inboundJournal.resolve([...new Set(inputIds)])) {
|
|
2541
|
+
const row = inboundJournal.get(ref.inputId)
|
|
2542
|
+
if (!row || row.phase === 'closed' || row.transfer) continue
|
|
2543
|
+
const identity = recoveryNoticePartitionKey({
|
|
2544
|
+
target,
|
|
2545
|
+
accountIdentity: row.accountIdentity,
|
|
2546
|
+
principal: row.principal,
|
|
2547
|
+
parentSessionId: row.claim?.ownerSessionId ?? ownerSessionId ?? row.sourceParentSessionId,
|
|
2548
|
+
})
|
|
2549
|
+
let group = groups.get(identity)
|
|
2550
|
+
if (!group) {
|
|
2551
|
+
group = { inboundRefs: [], backgroundRefs: [] }
|
|
2552
|
+
groups.set(identity, group)
|
|
2553
|
+
}
|
|
2554
|
+
group.inboundRefs.push(ref)
|
|
2555
|
+
}
|
|
2556
|
+
const backgroundOnly: BackgroundObligationRef[] = []
|
|
2557
|
+
for (const id of new Set(backgroundIds)) {
|
|
2558
|
+
const row = await backgroundObligations.get(id)
|
|
2559
|
+
if (!row || row.phase === 'closed' || row.transfer || channelKeyId(row.target) !== channelKeyId(target)) continue
|
|
2560
|
+
const ref = { obligationId: id, generation: row.generation }
|
|
2561
|
+
const group = groups.get(
|
|
2562
|
+
recoveryNoticePartitionKey({
|
|
2563
|
+
target,
|
|
2564
|
+
accountIdentity: row.accountIdentity,
|
|
2565
|
+
principal: row.principal,
|
|
2566
|
+
parentSessionId: row.claim?.ownerSessionId ?? ownerSessionId ?? row.parentSessionId,
|
|
2567
|
+
}),
|
|
2568
|
+
)
|
|
2569
|
+
if (group) group.backgroundRefs.push(ref)
|
|
2570
|
+
else backgroundOnly.push(ref)
|
|
2571
|
+
}
|
|
2572
|
+
for (const group of groups.values()) {
|
|
2573
|
+
const record = await inboundJournal.prepareNotice(
|
|
2574
|
+
group.inboundRefs,
|
|
2575
|
+
target,
|
|
2576
|
+
group.backgroundRefs,
|
|
2577
|
+
ownerSessionId,
|
|
2578
|
+
'live-turn-ended',
|
|
2579
|
+
)
|
|
2580
|
+
await inboundJournal.importPrepared(recoveryOutbox, record)
|
|
2581
|
+
}
|
|
2582
|
+
for (const ref of backgroundOnly) {
|
|
2583
|
+
const prepared = await backgroundObligations.prepareNotice(ref.obligationId, ref.generation, 'live-turn-ended')
|
|
2584
|
+
if (!prepared?.transfer) continue
|
|
2585
|
+
await resumePreparedTransfersInLane(target, [], [prepared.obligationId])
|
|
2586
|
+
}
|
|
2587
|
+
}
|
|
2588
|
+
|
|
2589
|
+
const transferMismatchedAccountCoverageInLane = async (live: LiveSession): Promise<string | undefined> => {
|
|
2590
|
+
const currentAccount = currentAccountIdentityFor(live.key)
|
|
2591
|
+
if (currentAccount === undefined) return undefined
|
|
2592
|
+
const inputIds = new Set([
|
|
2593
|
+
...live.inboundCoverage,
|
|
2594
|
+
...live.promptQueue.map((input) => input.inputId),
|
|
2595
|
+
...live.pendingSystemReminders.flatMap((reminder) => reminder.inboundRetry?.inputIds ?? []),
|
|
2596
|
+
])
|
|
2597
|
+
const mismatchedInputs = new Set<string>()
|
|
2598
|
+
for (const id of inputIds) {
|
|
2599
|
+
const row = inboundJournal.get(id)
|
|
2600
|
+
if (row && row.phase !== 'closed' && row.accountIdentity !== currentAccount) mismatchedInputs.add(id)
|
|
2601
|
+
}
|
|
2602
|
+
const backgroundIds = new Set([
|
|
2603
|
+
...live.backgroundCoverage,
|
|
2604
|
+
...live.pendingSystemReminders.flatMap((reminder) => [
|
|
2605
|
+
...(reminder.backgroundObligationId ? [reminder.backgroundObligationId] : []),
|
|
2606
|
+
...(reminder.backgroundRetry?.obligationIds ?? []),
|
|
2607
|
+
]),
|
|
2608
|
+
])
|
|
2609
|
+
const mismatchedBackground = new Set<string>()
|
|
2610
|
+
for (const id of backgroundIds) {
|
|
2611
|
+
const row = await backgroundObligations.get(id)
|
|
2612
|
+
if (row && row.accountIdentity !== currentAccount) mismatchedBackground.add(id)
|
|
2613
|
+
}
|
|
2614
|
+
if (mismatchedInputs.size === 0 && mismatchedBackground.size === 0) return currentAccount
|
|
2615
|
+
await transferLostCoverageInLane(live.key, [...mismatchedInputs], [...mismatchedBackground], live.sessionId)
|
|
2616
|
+
// Publish only compatible cache entries after the original-account transfer commits.
|
|
2617
|
+
const affectedRetries = new Set(
|
|
2618
|
+
live.pendingSystemReminders.filter(
|
|
2619
|
+
(reminder) =>
|
|
2620
|
+
reminder.inboundRetry?.inputIds.some((id) => mismatchedInputs.has(id)) ||
|
|
2621
|
+
reminder.backgroundRetry?.obligationIds.some((id) => mismatchedBackground.has(id)),
|
|
2622
|
+
),
|
|
2342
2623
|
)
|
|
2343
|
-
|
|
2344
|
-
|
|
2345
|
-
|
|
2346
|
-
|
|
2347
|
-
|
|
2348
|
-
|
|
2349
|
-
|
|
2350
|
-
|
|
2351
|
-
|
|
2352
|
-
|
|
2353
|
-
|
|
2354
|
-
logger.error(`[channels] ${channelKeyId(pending.key)}: reload handoff loss notice threw: ${describeError(err)}`)
|
|
2355
|
-
return null
|
|
2624
|
+
live.promptQueue = live.promptQueue.filter((input) => !mismatchedInputs.has(input.inputId))
|
|
2625
|
+
live.inboundCoverage = live.inboundCoverage.filter((id) => !mismatchedInputs.has(id))
|
|
2626
|
+
retireBackgroundCache(live, mismatchedBackground)
|
|
2627
|
+
live.pendingSystemReminders = live.pendingSystemReminders.filter((reminder) => {
|
|
2628
|
+
if (reminder.inboundRetry)
|
|
2629
|
+
reminder.inboundRetry.inputIds = reminder.inboundRetry.inputIds.filter((id) => !mismatchedInputs.has(id))
|
|
2630
|
+
return (
|
|
2631
|
+
!affectedRetries.has(reminder) ||
|
|
2632
|
+
(reminder.inboundRetry?.inputIds.length ?? 0) > 0 ||
|
|
2633
|
+
(reminder.backgroundRetry?.obligationIds.length ?? 0) > 0
|
|
2634
|
+
)
|
|
2356
2635
|
})
|
|
2357
|
-
|
|
2358
|
-
|
|
2636
|
+
return currentAccount
|
|
2637
|
+
}
|
|
2638
|
+
|
|
2639
|
+
// Children of the session the coverage was deferred from and of its current
|
|
2640
|
+
// claim owner; the two differ once a reload successor under another session ID
|
|
2641
|
+
// carried it. True while one of them still runs inside the stuck backstop.
|
|
2642
|
+
const awaitsDeferredChild = (live: LiveSession, label: string): boolean => {
|
|
2643
|
+
const deferredSince = live.coverageDeferredSince ?? live.logicalTurnStartedAt
|
|
2644
|
+
return [
|
|
2645
|
+
...new Set([
|
|
2646
|
+
live.coverageDeferredParentSessionId ?? live.backgroundTurnOwnerSessionId,
|
|
2647
|
+
live.backgroundTurnOwnerSessionId,
|
|
2648
|
+
]),
|
|
2649
|
+
].some((parent) => isAwaitingBackgroundChild(live, label, deferredSince, parent))
|
|
2650
|
+
}
|
|
2651
|
+
|
|
2652
|
+
// Cached coverage IDs are hints. A row belongs to this cache's turn only while
|
|
2653
|
+
// the recorded backing turn and owner hold its current-epoch claim at the
|
|
2654
|
+
// current generation; one closed, transferred or re-owned elsewhere does not.
|
|
2655
|
+
const isHeldByLiveTurn = (live: LiveSession, row: InboundRecord | BackgroundObligation | undefined): boolean =>
|
|
2656
|
+
row?.phase === 'turn-owned' &&
|
|
2657
|
+
!row.transfer &&
|
|
2658
|
+
row.claim?.turnId === live.backgroundTurnId &&
|
|
2659
|
+
row.claim.ownerSessionId === live.backgroundTurnOwnerSessionId &&
|
|
2660
|
+
row.claim.epoch === inboundJournal.epoch &&
|
|
2661
|
+
row.claim.generation === row.generation &&
|
|
2662
|
+
channelKeyId(row.target) === live.keyId
|
|
2663
|
+
|
|
2664
|
+
// Ends a live cache's ownership of its turn's coverage: whatever the turn still
|
|
2665
|
+
// holds becomes a durable live-turn-ended notice. Call only in the target lane.
|
|
2666
|
+
// A failed transfer throws before the caches clear, so the owner keeps them and
|
|
2667
|
+
// can retry at its next decision instead of silently dropping the debt.
|
|
2668
|
+
const transferHeldCoverageInLane = async (live: LiveSession): Promise<void> => {
|
|
2669
|
+
if (live.inboundCoverage.length > 0 || live.backgroundCoverage.length > 0) {
|
|
2670
|
+
await resumePreparedTransfersInLane(live.key, live.inboundCoverage, live.backgroundCoverage)
|
|
2671
|
+
const inputIds = live.inboundCoverage.filter((id) => isHeldByLiveTurn(live, inboundJournal.get(id)))
|
|
2672
|
+
const backgroundIds: string[] = []
|
|
2673
|
+
for (const id of new Set(live.backgroundCoverage)) {
|
|
2674
|
+
if (isHeldByLiveTurn(live, await backgroundObligations.get(id))) backgroundIds.push(id)
|
|
2675
|
+
}
|
|
2676
|
+
if (inputIds.length > 0 || backgroundIds.length > 0) {
|
|
2677
|
+
await transferLostCoverageInLane(live.key, inputIds, backgroundIds, live.backgroundTurnOwnerSessionId)
|
|
2678
|
+
}
|
|
2679
|
+
}
|
|
2680
|
+
live.inboundCoverage = []
|
|
2681
|
+
live.backgroundCoverage = []
|
|
2682
|
+
live.coverageDeferredSince = null
|
|
2683
|
+
live.coverageDeferredParentSessionId = null
|
|
2684
|
+
}
|
|
2685
|
+
|
|
2686
|
+
// The identity caches and turn lineage a successor adopts with deferred coverage.
|
|
2687
|
+
const reloadCoverageSnapshot = (live: LiveSession): PendingReloadCoverage => ({
|
|
2688
|
+
inboundIds: [...live.inboundCoverage],
|
|
2689
|
+
backgroundIds: [...live.backgroundCoverage],
|
|
2690
|
+
turnId: live.backgroundTurnId,
|
|
2691
|
+
ownerSessionId: live.backgroundTurnOwnerSessionId,
|
|
2692
|
+
logicalTurnStartedAt: live.logicalTurnStartedAt,
|
|
2693
|
+
coverageDeferredSince: live.coverageDeferredSince,
|
|
2694
|
+
coverageDeferredParentSessionId: live.coverageDeferredParentSessionId,
|
|
2695
|
+
...(live.turnAccountIdentity !== undefined ? { turnAccountIdentity: live.turnAccountIdentity } : {}),
|
|
2696
|
+
lastTurnAuthorId: live.lastTurnAuthorId,
|
|
2697
|
+
lastTurnAuthorIds: [...live.lastTurnAuthorIds],
|
|
2698
|
+
})
|
|
2699
|
+
|
|
2700
|
+
// Shared by the idle (immediate) and mid-drain (deferred tail) reload paths.
|
|
2701
|
+
// Queued work and the identities of coverage still deferred to background work
|
|
2702
|
+
// (a running child of the deferring turn, or its completion wake already queued)
|
|
2703
|
+
// move together into one bounded handoff, so a reload never turns that wait
|
|
2704
|
+
// into an interruption notice. Debt the session was not waiting on ended with
|
|
2705
|
+
// its live turn and is transferred as a live-turn-ended notice. Coverage bound
|
|
2706
|
+
// to a replaced account is transferred under that original account first, so a
|
|
2707
|
+
// fresh-account successor never inherits it. Whatever the bounds refuse is
|
|
2708
|
+
// transferred the same way under its original principal.
|
|
2709
|
+
const retainLiveWorkInLane = async (live: LiveSession): Promise<boolean> => {
|
|
2710
|
+
await ready
|
|
2711
|
+
inboundJournal.assertAvailable()
|
|
2712
|
+
await transferMismatchedAccountCoverageInLane(live)
|
|
2713
|
+
// A previous refused handoff may already have frozen otherwise deferred debt.
|
|
2714
|
+
// Finish that decision before a reload can retain it as live-turn coverage.
|
|
2715
|
+
await resumePreparedTransfersInLane(
|
|
2716
|
+
live.key,
|
|
2717
|
+
[
|
|
2718
|
+
...live.inboundCoverage,
|
|
2719
|
+
...live.promptQueue.map((input) => input.inputId),
|
|
2720
|
+
...live.pendingSystemReminders.flatMap((reminder) => reminder.inboundRetry?.inputIds ?? []),
|
|
2721
|
+
],
|
|
2722
|
+
[
|
|
2723
|
+
...live.backgroundCoverage,
|
|
2724
|
+
...live.pendingSystemReminders.flatMap((reminder) => [
|
|
2725
|
+
...(reminder.backgroundObligationId ? [reminder.backgroundObligationId] : []),
|
|
2726
|
+
...(reminder.backgroundRetry?.obligationIds ?? []),
|
|
2727
|
+
]),
|
|
2728
|
+
],
|
|
2729
|
+
)
|
|
2730
|
+
const hasCoverage = live.inboundCoverage.length > 0 || live.backgroundCoverage.length > 0
|
|
2731
|
+
const deferredToBackground =
|
|
2732
|
+
hasCoverage &&
|
|
2733
|
+
(live.pendingSystemReminders.some((reminder) => reminder.backgroundObligationId !== undefined) ||
|
|
2734
|
+
awaitsDeferredChild(live, 'reload-handoff'))
|
|
2735
|
+
if (hasCoverage && !deferredToBackground) await transferHeldCoverageInLane(live)
|
|
2736
|
+
const inbounds = live.promptQueue.slice()
|
|
2737
|
+
const observed = live.contextBuffer.slice()
|
|
2738
|
+
const reminders = live.pendingSystemReminders.slice()
|
|
2739
|
+
const coverage = deferredToBackground ? reloadCoverageSnapshot(live) : null
|
|
2740
|
+
const incoming: PendingReloadHandoff = {
|
|
2741
|
+
key: live.key,
|
|
2742
|
+
sessionId: live.sessionId,
|
|
2743
|
+
inbounds,
|
|
2744
|
+
observed,
|
|
2745
|
+
reminders,
|
|
2746
|
+
coverage,
|
|
2747
|
+
retainedAt: now(),
|
|
2748
|
+
estimatedBytes: estimateReloadHandoffBytes(inbounds, observed, reminders, coverage),
|
|
2749
|
+
}
|
|
2750
|
+
const hasWork = pendingReloadHandoffItemCount(incoming) > 0
|
|
2751
|
+
const retained = hasWork && retainReloadHandoff(incoming)
|
|
2752
|
+
if (hasWork && !retained) {
|
|
2753
|
+
await transferLostCoverageInLane(
|
|
2754
|
+
live.key,
|
|
2755
|
+
pendingReloadHandoffInputIds(incoming),
|
|
2756
|
+
pendingReloadHandoffBackgroundIds(incoming),
|
|
2757
|
+
live.sessionId,
|
|
2758
|
+
)
|
|
2759
|
+
}
|
|
2760
|
+
live.promptQueue.length = 0
|
|
2761
|
+
live.contextBuffer.length = 0
|
|
2762
|
+
live.pendingSystemReminders.length = 0
|
|
2763
|
+
live.inboundCoverage = []
|
|
2764
|
+
live.backgroundCoverage = []
|
|
2765
|
+
return retained
|
|
2766
|
+
}
|
|
2767
|
+
|
|
2768
|
+
// Stale rollover runs inside ensureLive, which can already hold this target's
|
|
2769
|
+
// lane, so it performs no durable transition itself. The stale session's owed
|
|
2770
|
+
// coverage moves into the bounded reload handoff its fresh successor adopts;
|
|
2771
|
+
// that successor's first in-lane consume carries whatever is still deferred to
|
|
2772
|
+
// a running child and transfers the rest as a live-turn-ended notice. A refused
|
|
2773
|
+
// handoff is transferred the same way by its discard report.
|
|
2774
|
+
const handOffRolloverCoverage = (live: LiveSession): void => {
|
|
2775
|
+
if (live.inboundCoverage.length === 0 && live.backgroundCoverage.length === 0) return
|
|
2776
|
+
const coverage = reloadCoverageSnapshot(live)
|
|
2777
|
+
retainReloadHandoff({
|
|
2778
|
+
key: live.key,
|
|
2779
|
+
sessionId: live.sessionId,
|
|
2780
|
+
inbounds: [],
|
|
2781
|
+
observed: [],
|
|
2782
|
+
reminders: [],
|
|
2783
|
+
coverage,
|
|
2784
|
+
retainedAt: now(),
|
|
2785
|
+
estimatedBytes: estimateReloadHandoffBytes([], [], [], coverage),
|
|
2786
|
+
})
|
|
2787
|
+
live.inboundCoverage = []
|
|
2788
|
+
live.backgroundCoverage = []
|
|
2789
|
+
}
|
|
2790
|
+
|
|
2791
|
+
// Resolves false when the transfer failed: the rows stay owed and the caller
|
|
2792
|
+
// keeps the handoff so the transfer can be retried.
|
|
2793
|
+
const reportDiscardedReloadHandoff = async (pending: PendingReloadHandoff, reason: string): Promise<boolean> => {
|
|
2794
|
+
logger.error(
|
|
2795
|
+
`[channels] ${channelKeyId(pending.key)}: reload handoff unavailable (${pendingReloadHandoffItemCount(pending)} items): ${reason}`,
|
|
2796
|
+
)
|
|
2797
|
+
let transferred = true
|
|
2798
|
+
try {
|
|
2799
|
+
await backgroundObligations.withTargetLane(pending.key, () =>
|
|
2800
|
+
transferLostCoverageInLane(
|
|
2801
|
+
pending.key,
|
|
2802
|
+
pendingReloadHandoffInputIds(pending),
|
|
2803
|
+
pendingReloadHandoffBackgroundIds(pending),
|
|
2804
|
+
),
|
|
2805
|
+
)
|
|
2806
|
+
} catch (error) {
|
|
2807
|
+
// The durable rows remain owed; failed transfer must never become an ad-hoc send.
|
|
2808
|
+
transferred = false
|
|
2809
|
+
logger.error(`[channels] reload interruption transfer failed: ${describeError(error)}`)
|
|
2359
2810
|
}
|
|
2811
|
+
void dropPendingReloadHandoffEngageReactions(pending)
|
|
2812
|
+
return transferred
|
|
2813
|
+
}
|
|
2814
|
+
|
|
2815
|
+
const discardExpiredReloadHandoff = async (
|
|
2816
|
+
keyId: string,
|
|
2817
|
+
pending: PendingReloadHandoff,
|
|
2818
|
+
ageMs: number,
|
|
2819
|
+
): Promise<void> => {
|
|
2820
|
+
if (pendingReloadHandoffs.get(keyId) === pending) pendingReloadHandoffs.delete(keyId)
|
|
2821
|
+
const transferred = await reportDiscardedReloadHandoff(pending, `retention limit exceeded after ${ageMs}ms`)
|
|
2822
|
+
if (!transferred && !pendingReloadHandoffs.has(keyId)) pendingReloadHandoffs.set(keyId, pending)
|
|
2360
2823
|
}
|
|
2361
2824
|
|
|
2362
2825
|
const expirePendingReloadHandoffs = async (): Promise<void> => {
|
|
2363
|
-
const expired: Array<{ pending: PendingReloadHandoff; ageMs: number }> = []
|
|
2826
|
+
const expired: Array<{ keyId: string; pending: PendingReloadHandoff; ageMs: number }> = []
|
|
2364
2827
|
for (const [keyId, pending] of pendingReloadHandoffs) {
|
|
2365
2828
|
const ageMs = now() - pending.retainedAt
|
|
2366
2829
|
if (ageMs < handoffRetryRetentionMs) continue
|
|
2367
|
-
|
|
2368
|
-
expired.push({ pending, ageMs })
|
|
2830
|
+
expired.push({ keyId, pending, ageMs })
|
|
2369
2831
|
}
|
|
2370
|
-
await Promise.all(
|
|
2371
|
-
expired.map(({ pending, ageMs }) =>
|
|
2372
|
-
reportDiscardedReloadHandoff(pending, `retention limit exceeded after ${ageMs}ms`),
|
|
2373
|
-
),
|
|
2374
|
-
)
|
|
2832
|
+
await Promise.all(expired.map(({ keyId, pending, ageMs }) => discardExpiredReloadHandoff(keyId, pending, ageMs)))
|
|
2375
2833
|
}
|
|
2376
2834
|
|
|
2377
|
-
const retainReloadHandoff = (
|
|
2378
|
-
key: ChannelKey,
|
|
2379
|
-
inbounds: QueuedInbound[],
|
|
2380
|
-
observed: ObservedInbound[],
|
|
2381
|
-
reminders: PendingSystemReminder[],
|
|
2382
|
-
): boolean => {
|
|
2835
|
+
const retainReloadHandoff = (incoming: PendingReloadHandoff): boolean => {
|
|
2383
2836
|
void expirePendingReloadHandoffs()
|
|
2384
|
-
const keyId = channelKeyId(key)
|
|
2837
|
+
const keyId = channelKeyId(incoming.key)
|
|
2385
2838
|
const existing = pendingReloadHandoffs.get(keyId)
|
|
2386
2839
|
if (existing !== undefined) {
|
|
2387
|
-
|
|
2388
|
-
|
|
2389
|
-
|
|
2390
|
-
|
|
2391
|
-
|
|
2392
|
-
retainedAt: now(),
|
|
2393
|
-
estimatedBytes: estimateReloadHandoffBytes(inbounds, observed, reminders),
|
|
2840
|
+
// A successor resumes exactly one turn lineage; a second coverage snapshot
|
|
2841
|
+
// for the same conversation cannot share its cache.
|
|
2842
|
+
if (existing.coverage !== null && incoming.coverage !== null) {
|
|
2843
|
+
void reportDiscardedReloadHandoff(incoming, 'reload coverage already pending for this conversation')
|
|
2844
|
+
return false
|
|
2394
2845
|
}
|
|
2395
2846
|
if (pendingReloadHandoffItemCount(existing) + pendingReloadHandoffItemCount(incoming) > handoffRetryItemLimit) {
|
|
2396
2847
|
void reportDiscardedReloadHandoff(incoming, `per-key item limit ${handoffRetryItemLimit} would be exceeded`)
|
|
@@ -2407,37 +2858,33 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
2407
2858
|
)
|
|
2408
2859
|
return false
|
|
2409
2860
|
}
|
|
2410
|
-
existing.inbounds.push(...inbounds)
|
|
2411
|
-
existing.observed.push(...observed)
|
|
2412
|
-
existing.reminders.push(...reminders)
|
|
2861
|
+
existing.inbounds.push(...incoming.inbounds)
|
|
2862
|
+
existing.observed.push(...incoming.observed)
|
|
2863
|
+
existing.reminders.push(...incoming.reminders)
|
|
2864
|
+
if (incoming.coverage !== null) existing.coverage = incoming.coverage
|
|
2413
2865
|
existing.estimatedBytes += incoming.estimatedBytes
|
|
2414
2866
|
return true
|
|
2415
2867
|
}
|
|
2416
|
-
|
|
2417
|
-
key
|
|
2418
|
-
inbounds,
|
|
2419
|
-
observed,
|
|
2420
|
-
reminders,
|
|
2421
|
-
retainedAt: now(),
|
|
2422
|
-
estimatedBytes: estimateReloadHandoffBytes(inbounds, observed, reminders),
|
|
2423
|
-
}
|
|
2424
|
-
if (pendingReloadHandoffItemCount(pending) > handoffRetryItemLimit) {
|
|
2425
|
-
void reportDiscardedReloadHandoff(pending, `per-key item limit ${handoffRetryItemLimit} exceeded`)
|
|
2868
|
+
if (pendingReloadHandoffItemCount(incoming) > handoffRetryItemLimit) {
|
|
2869
|
+
void reportDiscardedReloadHandoff(incoming, `per-key item limit ${handoffRetryItemLimit} exceeded`)
|
|
2426
2870
|
return false
|
|
2427
2871
|
}
|
|
2428
|
-
if (
|
|
2429
|
-
void reportDiscardedReloadHandoff(
|
|
2872
|
+
if (incoming.estimatedBytes > handoffRetryByteLimit) {
|
|
2873
|
+
void reportDiscardedReloadHandoff(incoming, `per-key byte limit ${handoffRetryByteLimit} exceeded`)
|
|
2430
2874
|
return false
|
|
2431
2875
|
}
|
|
2432
|
-
if (pendingReloadHandoffTotalBytes() +
|
|
2433
|
-
void reportDiscardedReloadHandoff(
|
|
2876
|
+
if (pendingReloadHandoffTotalBytes() + incoming.estimatedBytes > MAX_PENDING_RELOAD_HANDOFF_BYTES_TOTAL) {
|
|
2877
|
+
void reportDiscardedReloadHandoff(
|
|
2878
|
+
incoming,
|
|
2879
|
+
`global byte limit ${MAX_PENDING_RELOAD_HANDOFF_BYTES_TOTAL} exceeded`,
|
|
2880
|
+
)
|
|
2434
2881
|
return false
|
|
2435
2882
|
}
|
|
2436
2883
|
if (pendingReloadHandoffs.size >= MAX_PENDING_RELOAD_HANDOFF_KEYS) {
|
|
2437
|
-
void reportDiscardedReloadHandoff(
|
|
2884
|
+
void reportDiscardedReloadHandoff(incoming, `key limit ${MAX_PENDING_RELOAD_HANDOFF_KEYS} reached`)
|
|
2438
2885
|
return false
|
|
2439
2886
|
}
|
|
2440
|
-
pendingReloadHandoffs.set(keyId,
|
|
2887
|
+
pendingReloadHandoffs.set(keyId, incoming)
|
|
2441
2888
|
return true
|
|
2442
2889
|
}
|
|
2443
2890
|
|
|
@@ -2446,16 +2893,54 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
2446
2893
|
if (pending === undefined) return
|
|
2447
2894
|
const ageMs = now() - pending.retainedAt
|
|
2448
2895
|
if (ageMs >= handoffRetryRetentionMs) {
|
|
2449
|
-
|
|
2450
|
-
void reportDiscardedReloadHandoff(pending, `retention limit exceeded after ${ageMs}ms`)
|
|
2896
|
+
void discardExpiredReloadHandoff(live.keyId, pending, ageMs)
|
|
2451
2897
|
return
|
|
2452
2898
|
}
|
|
2899
|
+
// Coverage identities join only a session with no turn lineage of its own; a
|
|
2900
|
+
// busy session leaves them pending (still bounded by expiry) rather than
|
|
2901
|
+
// mixing two lineages in one cache.
|
|
2902
|
+
const coverage = pending.coverage
|
|
2903
|
+
const adoptCoverage =
|
|
2904
|
+
coverage !== null &&
|
|
2905
|
+
!live.draining &&
|
|
2906
|
+
!live.pendingTeardown &&
|
|
2907
|
+
live.inboundCoverage.length === 0 &&
|
|
2908
|
+
live.backgroundCoverage.length === 0
|
|
2453
2909
|
// Delete before enqueueing: once ownership moves to the live queues, a
|
|
2454
2910
|
// second ensureLive must not replay the same batch and double-deliver it.
|
|
2455
|
-
|
|
2911
|
+
if (coverage !== null && !adoptCoverage) {
|
|
2912
|
+
pendingReloadHandoffs.set(live.keyId, {
|
|
2913
|
+
...pending,
|
|
2914
|
+
inbounds: [],
|
|
2915
|
+
observed: [],
|
|
2916
|
+
reminders: [],
|
|
2917
|
+
estimatedBytes: estimateReloadHandoffBytes([], [], [], coverage),
|
|
2918
|
+
})
|
|
2919
|
+
} else {
|
|
2920
|
+
pendingReloadHandoffs.delete(live.keyId)
|
|
2921
|
+
}
|
|
2456
2922
|
live.promptQueue.push(...pending.inbounds)
|
|
2457
2923
|
live.contextBuffer.push(...pending.observed)
|
|
2458
2924
|
live.pendingSystemReminders.push(...pending.reminders)
|
|
2925
|
+
if (adoptCoverage) {
|
|
2926
|
+
// Identity caches only. The journal claims stay with the source turn until
|
|
2927
|
+
// this session's in-lane consume revalidates and moves them, which happens
|
|
2928
|
+
// before any prompt; restoring the source turn and owner keeps that move and
|
|
2929
|
+
// any retry lineage valid. A coverage-only handoff never prompts by itself: the
|
|
2930
|
+
// child's completion (or the next real input) is what wakes this session.
|
|
2931
|
+
live.inboundCoverage = [...coverage.inboundIds]
|
|
2932
|
+
live.backgroundCoverage = [...coverage.backgroundIds]
|
|
2933
|
+
live.backgroundTurnId = coverage.turnId
|
|
2934
|
+
live.backgroundTurnOwnerSessionId = coverage.ownerSessionId
|
|
2935
|
+
live.logicalTurnStartedAt = coverage.logicalTurnStartedAt
|
|
2936
|
+
live.coverageDeferredSince = coverage.coverageDeferredSince
|
|
2937
|
+
live.coverageDeferredParentSessionId = coverage.coverageDeferredParentSessionId
|
|
2938
|
+
if (coverage.turnAccountIdentity !== undefined) live.turnAccountIdentity = coverage.turnAccountIdentity
|
|
2939
|
+
if (live.lastTurnAuthorId === null && coverage.lastTurnAuthorId !== null) {
|
|
2940
|
+
live.lastTurnAuthorId = coverage.lastTurnAuthorId
|
|
2941
|
+
live.lastTurnAuthorIds = new Set(coverage.lastTurnAuthorIds)
|
|
2942
|
+
}
|
|
2943
|
+
}
|
|
2459
2944
|
if ((pending.inbounds.length > 0 || pending.reminders.length > 0) && !live.draining) void drain(live)
|
|
2460
2945
|
}
|
|
2461
2946
|
|
|
@@ -2474,8 +2959,12 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
2474
2959
|
// parent-scoped from the first fetch (Discord threads resolve `chat` to the
|
|
2475
2960
|
// thread id, so an unscoped warm would prime the cache against the thread).
|
|
2476
2961
|
room?: InboundMessage['room'],
|
|
2962
|
+
// route captures this before journal/account precreation awaits, so a reload
|
|
2963
|
+
// during initialization cannot make a pre-reload inbound look newly arrived.
|
|
2964
|
+
generation = liveGeneration,
|
|
2477
2965
|
): Promise<LiveSession> => {
|
|
2478
2966
|
const keyId = channelKeyId(key)
|
|
2967
|
+
if (generation !== liveGeneration) throw new StaleLiveSessionError(keyId)
|
|
2479
2968
|
const existing = liveSessions.get(keyId)
|
|
2480
2969
|
if (existing && !existing.destroyed) {
|
|
2481
2970
|
// A resume that finds the key already live is a no-op for reopening: the
|
|
@@ -2489,6 +2978,7 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
2489
2978
|
// turn is never aborted by a follow-up's idle check (PR #359 incident).
|
|
2490
2979
|
const idleMs = now() - existing.lastInboundAt
|
|
2491
2980
|
if (shouldRolloverLive(existing, idleMs)) {
|
|
2981
|
+
handOffRolloverCoverage(existing)
|
|
2492
2982
|
await tearDownLive(existing)
|
|
2493
2983
|
liveSessions.delete(keyId)
|
|
2494
2984
|
if (mappings) {
|
|
@@ -2527,7 +3017,6 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
2527
3017
|
const inFlight = creating.get(keyId)
|
|
2528
3018
|
if (inFlight) return inFlight
|
|
2529
3019
|
|
|
2530
|
-
const generation = liveGeneration
|
|
2531
3020
|
const creationToken = ++nextCreationToken
|
|
2532
3021
|
latestCreationTokens.set(keyId, creationToken)
|
|
2533
3022
|
activeCreationAttempts.set(keyId, (activeCreationAttempts.get(keyId) ?? 0) + 1)
|
|
@@ -2754,6 +3243,11 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
2754
3243
|
pendingGithubReviewRoundCloseout: null,
|
|
2755
3244
|
promptQueue: [],
|
|
2756
3245
|
pendingSystemReminders: [],
|
|
3246
|
+
backgroundCoverage: [],
|
|
3247
|
+
inboundCoverage: [],
|
|
3248
|
+
backgroundStopVersion: 0,
|
|
3249
|
+
backgroundTurnId: randomUUID(),
|
|
3250
|
+
backgroundTurnOwnerSessionId: created.sessionId,
|
|
2757
3251
|
willingnessReminderIteration: false,
|
|
2758
3252
|
contextBuffer: [],
|
|
2759
3253
|
currentTurnAttachments: [],
|
|
@@ -2780,7 +3274,6 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
2780
3274
|
currentTurnEngageReactions: [],
|
|
2781
3275
|
pendingTurnReactions: [],
|
|
2782
3276
|
silentAckTurn: null,
|
|
2783
|
-
activeSilentAckReactions: [],
|
|
2784
3277
|
activeContinuationReactions: [],
|
|
2785
3278
|
// `lastTurnAuthorId` (string, used for `lastInboundAuthorId` in
|
|
2786
3279
|
// origin) and `lastTurnAuthorIds` (Set, used by
|
|
@@ -2802,6 +3295,8 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
2802
3295
|
successfulChannelSends: 0,
|
|
2803
3296
|
turnSeq: 0,
|
|
2804
3297
|
logicalTurnStartedAt: now(),
|
|
3298
|
+
coverageDeferredSince: null,
|
|
3299
|
+
coverageDeferredParentSessionId: null,
|
|
2805
3300
|
successfulSendsAtTurnStart: 0,
|
|
2806
3301
|
inFlightToolSends: new Map(),
|
|
2807
3302
|
policyDeniedToolSendsThisTurn: new Map(),
|
|
@@ -2822,8 +3317,12 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
2822
3317
|
stagedFallbackCause: null,
|
|
2823
3318
|
githubReviewOutputTurn: null,
|
|
2824
3319
|
toolExecutionThisLogicalTurn: false,
|
|
3320
|
+
backgroundWakeLogicalTurn: false,
|
|
3321
|
+
providerErrorSilencedThisLogicalTurn: false,
|
|
2825
3322
|
qualifyingWorkThisLogicalTurn: false,
|
|
2826
3323
|
promisedWorkOutstandingThisLogicalTurn: false,
|
|
3324
|
+
statusOutputOutstandingThisLogicalTurn: false,
|
|
3325
|
+
nextInboundContinuationTurnId: null,
|
|
2827
3326
|
pendingQuoteCandidate: null,
|
|
2828
3327
|
recentEngagedPeerBotTurns: [],
|
|
2829
3328
|
consecutiveEngagedPeerBotTurns: 0,
|
|
@@ -2851,7 +3350,11 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
2851
3350
|
live.unsubProviderErrors = subscribeProviderErrors(created.session, (err) => {
|
|
2852
3351
|
logger.error(`[channels] ${live.keyId}: LLM call failed: ${err.message}`)
|
|
2853
3352
|
if (live.pendingProviderError?.turnSeq === live.turnSeq) return
|
|
2854
|
-
live.pendingProviderError = {
|
|
3353
|
+
live.pendingProviderError = {
|
|
3354
|
+
turnSeq: live.turnSeq,
|
|
3355
|
+
safeMessage: err.safeMessage,
|
|
3356
|
+
retryAndSee: isTransientProviderFailure(err.message) || isUnclassifiedProviderNotice(err.safeMessage),
|
|
3357
|
+
}
|
|
2855
3358
|
})
|
|
2856
3359
|
live.unsubTodoOutcome = created.session.subscribe((event: unknown) => {
|
|
2857
3360
|
const usage = extractTurnUsage(event)
|
|
@@ -3243,12 +3746,145 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
3243
3746
|
// proceeds. The kimi 32k loop only recurs when the model genuinely has nothing
|
|
3244
3747
|
// left to say after a reply, which `more_work_this_turn` asserts is not the
|
|
3245
3748
|
// case; Layer 2's maxTokens cap still bounds any misuse.
|
|
3749
|
+
// IDs are cache hints, never generation authority. Call only inside the lane.
|
|
3750
|
+
const resolveBackgroundCoverage = async (
|
|
3751
|
+
live: LiveSession,
|
|
3752
|
+
ids: readonly string[],
|
|
3753
|
+
ownedOnly = false,
|
|
3754
|
+
): Promise<BackgroundObligationRef[]> => {
|
|
3755
|
+
const refs: BackgroundObligationRef[] = []
|
|
3756
|
+
for (const id of new Set(ids)) {
|
|
3757
|
+
const row = await backgroundObligations.get(id)
|
|
3758
|
+
if (
|
|
3759
|
+
!row ||
|
|
3760
|
+
row.transfer ||
|
|
3761
|
+
channelKeyId(row.target) !== live.keyId ||
|
|
3762
|
+
(row.phase !== 'result-ready' && row.phase !== 'turn-owned')
|
|
3763
|
+
)
|
|
3764
|
+
continue
|
|
3765
|
+
if (ownedOnly && (row.claim?.turnId !== live.backgroundTurnId || row.claim.ownerSessionId !== live.sessionId))
|
|
3766
|
+
continue
|
|
3767
|
+
refs.push({ obligationId: id, generation: row.generation })
|
|
3768
|
+
}
|
|
3769
|
+
if (refs.length > 0) backgroundObligations.assertAvailable()
|
|
3770
|
+
return refs
|
|
3771
|
+
}
|
|
3772
|
+
|
|
3773
|
+
const retireBackgroundCache = (live: LiveSession, covered: Set<string>): void => {
|
|
3774
|
+
live.backgroundCoverage = live.backgroundCoverage.filter((id) => !covered.has(id))
|
|
3775
|
+
live.pendingSystemReminders = live.pendingSystemReminders.filter(
|
|
3776
|
+
(reminder) => !reminder.backgroundObligationId || !covered.has(reminder.backgroundObligationId),
|
|
3777
|
+
)
|
|
3778
|
+
for (const reminder of live.pendingSystemReminders) {
|
|
3779
|
+
if (reminder.backgroundRetry) {
|
|
3780
|
+
reminder.backgroundRetry.obligationIds = reminder.backgroundRetry.obligationIds.filter((id) => !covered.has(id))
|
|
3781
|
+
}
|
|
3782
|
+
}
|
|
3783
|
+
}
|
|
3784
|
+
|
|
3785
|
+
// An inbound row the session's CURRENT turn owns: the claim this session took
|
|
3786
|
+
// for its backing turn in this epoch. Settlement that captured nothing uses this
|
|
3787
|
+
// rather than the cached ID list, which a later turn can have refilled.
|
|
3788
|
+
const isClaimedByCurrentTurn = (live: LiveSession, row: InboundRecord | undefined): boolean =>
|
|
3789
|
+
row?.phase === 'turn-owned' &&
|
|
3790
|
+
row.claim?.turnId === live.backgroundTurnId &&
|
|
3791
|
+
row.claim.ownerSessionId === live.sessionId &&
|
|
3792
|
+
row.claim.epoch === inboundJournal.epoch
|
|
3793
|
+
|
|
3794
|
+
const settleBackgroundCoverageInLane = async (
|
|
3795
|
+
live: LiveSession,
|
|
3796
|
+
kind: 'delivered' | 'intentionally-suppressed',
|
|
3797
|
+
reason: string,
|
|
3798
|
+
ids: readonly string[],
|
|
3799
|
+
capturedInbound?: readonly InboundRef[],
|
|
3800
|
+
capturedBackground?: readonly BackgroundObligationRef[],
|
|
3801
|
+
): Promise<void> => {
|
|
3802
|
+
const currentBackground = await resolveBackgroundCoverage(live, ids, capturedBackground === undefined)
|
|
3803
|
+
const refs = capturedBackground
|
|
3804
|
+
? currentBackground.filter((ref) =>
|
|
3805
|
+
capturedBackground.some(
|
|
3806
|
+
(captured) => captured.obligationId === ref.obligationId && captured.generation === ref.generation,
|
|
3807
|
+
),
|
|
3808
|
+
)
|
|
3809
|
+
: currentBackground
|
|
3810
|
+
const currentInbound = inboundJournal.resolve(
|
|
3811
|
+
capturedInbound ? capturedInbound.map((ref) => ref.inputId) : live.inboundCoverage,
|
|
3812
|
+
)
|
|
3813
|
+
const inboundRefs = capturedInbound
|
|
3814
|
+
? currentInbound.filter((ref) =>
|
|
3815
|
+
capturedInbound.some(
|
|
3816
|
+
(captured) => captured.inputId === ref.inputId && captured.generation === ref.generation,
|
|
3817
|
+
),
|
|
3818
|
+
)
|
|
3819
|
+
: currentInbound.filter((ref) => isClaimedByCurrentTurn(live, inboundJournal.get(ref.inputId)))
|
|
3820
|
+
if (inboundRefs.length > 0) {
|
|
3821
|
+
await inboundJournal.settle(inboundRefs, { kind, decisionId: randomUUID(), reason }, refs, live.key)
|
|
3822
|
+
} else if (refs.length > 0) {
|
|
3823
|
+
inboundJournal.assertAvailable()
|
|
3824
|
+
await backgroundObligations.settle(refs, { kind, decisionId: randomUUID(), reason })
|
|
3825
|
+
}
|
|
3826
|
+
const retiredInbound = new Set(inboundRefs.map((ref) => ref.inputId))
|
|
3827
|
+
live.inboundCoverage = live.inboundCoverage.filter((id) => !retiredInbound.has(id))
|
|
3828
|
+
retireBackgroundCache(live, new Set(refs.map((ref) => ref.obligationId)))
|
|
3829
|
+
}
|
|
3830
|
+
|
|
3831
|
+
const settleBackgroundCoverage = async (
|
|
3832
|
+
live: LiveSession,
|
|
3833
|
+
kind: 'delivered' | 'intentionally-suppressed',
|
|
3834
|
+
reason: string,
|
|
3835
|
+
refs?: BackgroundObligationRef[],
|
|
3836
|
+
inboundRefs?: readonly InboundRef[],
|
|
3837
|
+
): Promise<void> => {
|
|
3838
|
+
await backgroundObligations.withTargetLane(live.key, async () => {
|
|
3839
|
+
await settleBackgroundCoverageInLane(
|
|
3840
|
+
live,
|
|
3841
|
+
kind,
|
|
3842
|
+
reason,
|
|
3843
|
+
refs ? refs.map((ref) => ref.obligationId) : live.backgroundCoverage,
|
|
3844
|
+
inboundRefs,
|
|
3845
|
+
refs,
|
|
3846
|
+
)
|
|
3847
|
+
})
|
|
3848
|
+
}
|
|
3849
|
+
|
|
3850
|
+
// A landed response must never be replayed because its local outcome failed.
|
|
3851
|
+
const recordLandedBackgroundResponse = async (
|
|
3852
|
+
live: LiveSession,
|
|
3853
|
+
reason: string,
|
|
3854
|
+
refs: BackgroundObligationRef[],
|
|
3855
|
+
inboundRefs?: readonly InboundRef[],
|
|
3856
|
+
): Promise<void> => {
|
|
3857
|
+
try {
|
|
3858
|
+
await settleBackgroundCoverage(live, 'delivered', reason, refs, inboundRefs)
|
|
3859
|
+
} catch (error) {
|
|
3860
|
+
logger.error(`[channels] background response outcome failed: ${describeError(error)}`)
|
|
3861
|
+
}
|
|
3862
|
+
}
|
|
3863
|
+
|
|
3864
|
+
// A GitHub thread close-out decided (or still owed) this turn owns the turn's
|
|
3865
|
+
// end: the willingness nudge never runs for it, so future-intent wording there
|
|
3866
|
+
// ("해결할게요", "I'll go ahead and resolve") names the resolution itself, not
|
|
3867
|
+
// a follow-up. Settlement and the nudge must agree, or an answered request is
|
|
3868
|
+
// left owed until a live-turn-ended notice.
|
|
3869
|
+
const githubCloseoutOwnsTurn = (live: LiveSession): boolean =>
|
|
3870
|
+
live.githubReviewThreadCloseout !== null || live.githubReviewThreadCloseoutDecisionTurn === live.turnSeq
|
|
3871
|
+
|
|
3872
|
+
const promisesUnfinishedWork = (live: LiveSession, text: unknown): boolean =>
|
|
3873
|
+
typeof text === 'string' && detectContinuationWillingness(text) && !githubCloseoutOwnsTurn(live)
|
|
3874
|
+
|
|
3246
3875
|
const installChannelReplyTerminalHook = (live: LiveSession): void => {
|
|
3247
3876
|
const { agent } = live.session
|
|
3248
3877
|
const prior = agent.afterToolCall
|
|
3249
3878
|
agent.afterToolCall = async (context, signal) => {
|
|
3250
3879
|
const result = prior ? await prior(context, signal) : undefined
|
|
3251
|
-
const details = context.result.details as
|
|
3880
|
+
const details = context.result.details as
|
|
3881
|
+
| {
|
|
3882
|
+
ok?: unknown
|
|
3883
|
+
more_work_this_turn?: unknown
|
|
3884
|
+
backgroundCoverage?: BackgroundObligationRef[]
|
|
3885
|
+
inboundCoverage?: InboundRef[]
|
|
3886
|
+
}
|
|
3887
|
+
| undefined
|
|
3252
3888
|
if (
|
|
3253
3889
|
isQualifyingWorkResult({
|
|
3254
3890
|
toolName: context.toolCall.name,
|
|
@@ -3265,11 +3901,34 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
3265
3901
|
// cannot distinguish channel_reply from channel_send, but here the
|
|
3266
3902
|
// explicit more_work_this_turn flag tells us whether the delivered reply
|
|
3267
3903
|
// renewed the promise or terminally fulfilled it.
|
|
3904
|
+
const text = (context.toolCall.arguments as { text?: unknown } | undefined)?.text
|
|
3905
|
+
if (!keepTurnAlive && !promisesUnfinishedWork(live, text)) {
|
|
3906
|
+
await recordLandedBackgroundResponse(
|
|
3907
|
+
live,
|
|
3908
|
+
'terminal-channel-reply',
|
|
3909
|
+
details?.backgroundCoverage ?? [],
|
|
3910
|
+
details?.inboundCoverage ?? [],
|
|
3911
|
+
)
|
|
3912
|
+
}
|
|
3268
3913
|
live.promisedWorkOutstandingThisLogicalTurn = keepTurnAlive
|
|
3269
3914
|
if (keepTurnAlive) {
|
|
3270
3915
|
live.continueReplyTurn = { turnSeq: live.turnSeq, sendCount: live.successfulChannelSends }
|
|
3271
3916
|
}
|
|
3272
3917
|
}
|
|
3918
|
+
// A landed channel_send settles only the debt its tool captured from this
|
|
3919
|
+
// session before the send, and the tool returns a capture only for an exact
|
|
3920
|
+
// origin destination. A send elsewhere, a no-op success, or a progress/status
|
|
3921
|
+
// message never borrows the origin's debt. Unlike channel_reply this stays
|
|
3922
|
+
// non-terminal: the follow-up provider call still runs, so the model can keep
|
|
3923
|
+
// working after delivering its answer.
|
|
3924
|
+
if (context.toolCall.name === 'channel_send' && !context.isError && details?.ok === true) {
|
|
3925
|
+
const backgroundCoverage = Array.isArray(details.backgroundCoverage) ? details.backgroundCoverage : []
|
|
3926
|
+
const inboundCoverage = Array.isArray(details.inboundCoverage) ? details.inboundCoverage : []
|
|
3927
|
+
const text = (context.toolCall.arguments as { text?: unknown } | undefined)?.text
|
|
3928
|
+
if ((backgroundCoverage.length > 0 || inboundCoverage.length > 0) && !promisesUnfinishedWork(live, text)) {
|
|
3929
|
+
await recordLandedBackgroundResponse(live, 'origin-channel-send', backgroundCoverage, inboundCoverage)
|
|
3930
|
+
}
|
|
3931
|
+
}
|
|
3273
3932
|
if (succeeded && !keepTurnAlive && live.userStoppedTurnSeq !== live.turnSeq) {
|
|
3274
3933
|
logger.info(
|
|
3275
3934
|
`[channels] ${live.keyId} terminal_after_channel_reply site=terminal_after_channel_reply session=${live.sessionId} reason=terminal_after_channel_reply`,
|
|
@@ -3344,11 +4003,18 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
3344
4003
|
return stream
|
|
3345
4004
|
}
|
|
3346
4005
|
let maxTokens = streamOptions?.maxTokens
|
|
3347
|
-
if (maxTokens === undefined
|
|
3348
|
-
|
|
3349
|
-
live.nextPromptMaxTokens
|
|
4006
|
+
if (maxTokens === undefined) {
|
|
4007
|
+
let budget = CHANNEL_MAX_OUTPUT_TOKENS
|
|
4008
|
+
if (live.nextPromptMaxTokens !== undefined) {
|
|
4009
|
+
budget = live.nextPromptMaxTokens
|
|
4010
|
+
live.nextPromptMaxTokens = undefined
|
|
4011
|
+
}
|
|
4012
|
+
// Router-chosen budgets never exceed the model's own output maximum: pi-ai
|
|
4013
|
+
// clamps an explicit maxTokens only to remaining context, and endpoints that
|
|
4014
|
+
// enforce a smaller output limit reject the request outright.
|
|
4015
|
+
maxTokens = model.maxTokens > 0 ? Math.min(budget, model.maxTokens) : budget
|
|
3350
4016
|
}
|
|
3351
|
-
return await inner(model, context, { ...streamOptions, maxTokens
|
|
4017
|
+
return await inner(model, context, { ...streamOptions, maxTokens })
|
|
3352
4018
|
}
|
|
3353
4019
|
}
|
|
3354
4020
|
|
|
@@ -3470,9 +4136,19 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
3470
4136
|
}
|
|
3471
4137
|
|
|
3472
4138
|
const postEmptyTurnFallback = async (live: LiveSession, cause: string): Promise<void> => {
|
|
3473
|
-
logger.warn(`[channels] ${live.keyId} empty_turn_fallback cause=${cause}`)
|
|
3474
4139
|
live.emptyTurnFallbackTurn = live.turnSeq
|
|
4140
|
+
if (live.backgroundWakeLogicalTurn) {
|
|
4141
|
+
logger.warn(`[channels] ${live.keyId} empty_turn_fallback_suppressed reason=background_wake_turn cause=${cause}`)
|
|
4142
|
+
void dropContinuationReactions(live)
|
|
4143
|
+
return
|
|
4144
|
+
}
|
|
4145
|
+
logger.warn(`[channels] ${live.keyId} empty_turn_fallback cause=${cause}`)
|
|
3475
4146
|
try {
|
|
4147
|
+
const inboundCoverage = await captureInboundResultCoverage(live.sessionId)
|
|
4148
|
+
if (await reactTurnFailed(live)) {
|
|
4149
|
+
await recordLandedBackgroundResponse(live, 'empty-turn-fallback', [], inboundCoverage)
|
|
4150
|
+
return
|
|
4151
|
+
}
|
|
3476
4152
|
const result = await send(
|
|
3477
4153
|
{
|
|
3478
4154
|
adapter: live.key.adapter,
|
|
@@ -3485,6 +4161,8 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
3485
4161
|
)
|
|
3486
4162
|
if (!result.ok) {
|
|
3487
4163
|
logger.warn(`[channels] ${live.keyId}: empty-turn fallback send failed: ${result.error}`)
|
|
4164
|
+
} else {
|
|
4165
|
+
await recordLandedBackgroundResponse(live, 'empty-turn-fallback', [], inboundCoverage)
|
|
3488
4166
|
}
|
|
3489
4167
|
} finally {
|
|
3490
4168
|
void dropContinuationReactions(live)
|
|
@@ -3500,18 +4178,30 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
3500
4178
|
logger.warn(
|
|
3501
4179
|
`[channels] ${live.keyId} github_thread_closeout_fallback pr=${obligation.prNumber} root=${obligation.rootCommentId}`,
|
|
3502
4180
|
)
|
|
4181
|
+
const destination: ChannelKey = {
|
|
4182
|
+
adapter: 'github',
|
|
4183
|
+
workspace: obligation.workspace,
|
|
4184
|
+
chat: `pr:${obligation.prNumber}`,
|
|
4185
|
+
thread: obligation.rootCommentId,
|
|
4186
|
+
}
|
|
4187
|
+
// Like the empty-turn fallback, a close-out landed on this session's own
|
|
4188
|
+
// thread is the terminal answer to the requests the turn owns there.
|
|
4189
|
+
let inboundCoverage: InboundRef[] = []
|
|
4190
|
+
if (channelKeyId(destination) === live.keyId) {
|
|
4191
|
+
try {
|
|
4192
|
+
inboundCoverage = await captureInboundResultCoverage(live.sessionId)
|
|
4193
|
+
} catch (error) {
|
|
4194
|
+
logger.error(`[channels] ${live.keyId}: close-out fallback coverage capture failed: ${describeError(error)}`)
|
|
4195
|
+
}
|
|
4196
|
+
}
|
|
3503
4197
|
const result = await send(
|
|
3504
|
-
{
|
|
3505
|
-
adapter: 'github',
|
|
3506
|
-
workspace: obligation.workspace,
|
|
3507
|
-
chat: `pr:${obligation.prNumber}`,
|
|
3508
|
-
thread: obligation.rootCommentId,
|
|
3509
|
-
text: GITHUB_REVIEW_THREAD_CLOSEOUT_FALLBACK_TEXT,
|
|
3510
|
-
},
|
|
4198
|
+
{ ...destination, text: GITHUB_REVIEW_THREAD_CLOSEOUT_FALLBACK_TEXT },
|
|
3511
4199
|
{ source: 'system', outputKind: 'meta' },
|
|
3512
4200
|
)
|
|
3513
4201
|
if (!result.ok) {
|
|
3514
4202
|
logger.warn(`[channels] ${live.keyId}: github review-thread closeout fallback send failed: ${result.error}`)
|
|
4203
|
+
} else if (inboundCoverage.length > 0) {
|
|
4204
|
+
await recordLandedBackgroundResponse(live, 'github-thread-closeout-fallback', [], inboundCoverage)
|
|
3515
4205
|
}
|
|
3516
4206
|
}
|
|
3517
4207
|
|
|
@@ -3801,17 +4491,169 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
3801
4491
|
}
|
|
3802
4492
|
}
|
|
3803
4493
|
|
|
4494
|
+
// User-stop withdrawal for one conversation, independent of any live session:
|
|
4495
|
+
// its open, untransferred inbound rows and result-ready/turn-owned background
|
|
4496
|
+
// rows settle as intentionally suppressed. Rotation can leave the old actor's
|
|
4497
|
+
// active turn beside a new actor's queued input, and a journal decision must
|
|
4498
|
+
// keep one account scope, so each account settles separately. `withdrawn`
|
|
4499
|
+
// records every group as its decision commits, so a caller can retire exactly
|
|
4500
|
+
// those cache entries even when a later group fails. Call only in the lane.
|
|
4501
|
+
const withdrawStoppedWorkInLane = async (
|
|
4502
|
+
target: ChannelKey,
|
|
4503
|
+
inputIds: readonly string[],
|
|
4504
|
+
backgroundIds: readonly string[],
|
|
4505
|
+
withdrawn: { inbound: Set<string>; background: Set<string> } = { inbound: new Set(), background: new Set() },
|
|
4506
|
+
): Promise<void> => {
|
|
4507
|
+
const partitions = new Map<string, { inbound: InboundRef[]; background: BackgroundObligationRef[] }>()
|
|
4508
|
+
const partition = (accountIdentity: string) => {
|
|
4509
|
+
let group = partitions.get(accountIdentity)
|
|
4510
|
+
if (!group) {
|
|
4511
|
+
group = { inbound: [], background: [] }
|
|
4512
|
+
partitions.set(accountIdentity, group)
|
|
4513
|
+
}
|
|
4514
|
+
return group
|
|
4515
|
+
}
|
|
4516
|
+
const prepared = new Map<string, RecoveryRecord>()
|
|
4517
|
+
for (const ref of inboundJournal.resolve([...new Set(inputIds)])) {
|
|
4518
|
+
const row = inboundJournal.get(ref.inputId)!
|
|
4519
|
+
if (row.phase === 'notice-prepared') {
|
|
4520
|
+
if (!row.transfer || channelKeyId(row.target) !== channelKeyId(target))
|
|
4521
|
+
throw new Error('Prepared stop coverage mismatch')
|
|
4522
|
+
prepared.set(row.transfer.deliveryId, row.transfer)
|
|
4523
|
+
}
|
|
4524
|
+
if (row.phase !== 'closed' && !row.transfer) partition(row.accountIdentity).inbound.push(ref)
|
|
4525
|
+
}
|
|
4526
|
+
for (const id of new Set(backgroundIds)) {
|
|
4527
|
+
const row = await backgroundObligations.get(id)
|
|
4528
|
+
if (row?.phase === 'notice-prepared') {
|
|
4529
|
+
if (!row.transfer || channelKeyId(row.target) !== channelKeyId(target))
|
|
4530
|
+
throw new Error('Prepared stop coverage mismatch')
|
|
4531
|
+
prepared.set(row.transfer.deliveryId, row.transfer)
|
|
4532
|
+
}
|
|
4533
|
+
if (
|
|
4534
|
+
!row ||
|
|
4535
|
+
row.transfer ||
|
|
4536
|
+
channelKeyId(row.target) !== channelKeyId(target) ||
|
|
4537
|
+
(row.phase !== 'result-ready' && row.phase !== 'turn-owned')
|
|
4538
|
+
)
|
|
4539
|
+
continue
|
|
4540
|
+
backgroundObligations.assertAvailable()
|
|
4541
|
+
partition(row.accountIdentity).background.push({ obligationId: id, generation: row.generation })
|
|
4542
|
+
}
|
|
4543
|
+
for (const frozen of prepared.values()) {
|
|
4544
|
+
const decision = { decisionId: randomUUID(), reason: 'user-stop' }
|
|
4545
|
+
if (frozen.covers.some((cover) => cover.store === 'inbound')) {
|
|
4546
|
+
await inboundJournal.suppressNoticeCoverage(frozen, decision)
|
|
4547
|
+
} else {
|
|
4548
|
+
inboundJournal.assertAvailable()
|
|
4549
|
+
await backgroundObligations.suppressNoticeCoverage(frozen, decision)
|
|
4550
|
+
}
|
|
4551
|
+
for (const cover of frozen.covers) {
|
|
4552
|
+
if (cover.store === 'inbound') withdrawn.inbound.add(cover.id)
|
|
4553
|
+
if (cover.store === 'background') withdrawn.background.add(cover.id)
|
|
4554
|
+
}
|
|
4555
|
+
}
|
|
4556
|
+
for (const group of partitions.values()) {
|
|
4557
|
+
const outcome = { kind: 'intentionally-suppressed' as const, decisionId: randomUUID(), reason: 'user-stop' }
|
|
4558
|
+
if (group.inbound.length > 0) {
|
|
4559
|
+
await inboundJournal.settle(group.inbound, outcome, group.background, target)
|
|
4560
|
+
} else {
|
|
4561
|
+
inboundJournal.assertAvailable()
|
|
4562
|
+
await backgroundObligations.settle(group.background, outcome)
|
|
4563
|
+
}
|
|
4564
|
+
for (const ref of group.inbound) withdrawn.inbound.add(ref.inputId)
|
|
4565
|
+
for (const ref of group.background) withdrawn.background.add(ref.obligationId)
|
|
4566
|
+
}
|
|
4567
|
+
}
|
|
4568
|
+
|
|
4569
|
+
// A reload handoff holds stoppable work only when it carries owed or queued
|
|
4570
|
+
// work: queued inputs, reminders or deferred coverage. Observed context alone
|
|
4571
|
+
// is the idle bystander hasStoppableWork excludes, so /stop never answers for
|
|
4572
|
+
// it, never withdraws it and never posts "stopped" on its behalf.
|
|
4573
|
+
const isStoppableReloadHandoff = (pending: PendingReloadHandoff | undefined): pending is PendingReloadHandoff =>
|
|
4574
|
+
pending !== undefined && (pending.inbounds.length > 0 || pending.reminders.length > 0 || pending.coverage !== null)
|
|
4575
|
+
|
|
4576
|
+
// After a committed stop withdrawal the handoff keeps only its observed
|
|
4577
|
+
// context, which a live stop also leaves in place for the next turn.
|
|
4578
|
+
const retireStoppedReloadHandoff = (keyId: string, pending: PendingReloadHandoff): void => {
|
|
4579
|
+
if (pendingReloadHandoffs.get(keyId) !== pending) return
|
|
4580
|
+
void dropPendingReloadHandoffEngageReactions(pending)
|
|
4581
|
+
if (pending.observed.length === 0) {
|
|
4582
|
+
pendingReloadHandoffs.delete(keyId)
|
|
4583
|
+
return
|
|
4584
|
+
}
|
|
4585
|
+
pendingReloadHandoffs.set(keyId, {
|
|
4586
|
+
...pending,
|
|
4587
|
+
inbounds: [],
|
|
4588
|
+
reminders: [],
|
|
4589
|
+
coverage: null,
|
|
4590
|
+
estimatedBytes: estimateReloadHandoffBytes([], pending.observed, [], null),
|
|
4591
|
+
})
|
|
4592
|
+
}
|
|
4593
|
+
|
|
3804
4594
|
const stopCurrentChannelTurn = async (live: LiveSession, reason: string): Promise<boolean> => {
|
|
3805
|
-
live.
|
|
3806
|
-
|
|
3807
|
-
|
|
3808
|
-
|
|
3809
|
-
|
|
3810
|
-
|
|
3811
|
-
|
|
3812
|
-
|
|
3813
|
-
|
|
3814
|
-
|
|
4595
|
+
await backgroundObligations.withTargetLane(live.key, async () => {
|
|
4596
|
+
try {
|
|
4597
|
+
if (reason === 'user_stop') {
|
|
4598
|
+
// A reload handoff not yet adopted by this session is the same
|
|
4599
|
+
// conversation's owed work; the stop withdraws it too.
|
|
4600
|
+
const handoff = pendingReloadHandoffs.get(live.keyId)
|
|
4601
|
+
const pending = isStoppableReloadHandoff(handoff) ? handoff : undefined
|
|
4602
|
+
// Select after predecessors commit; queued entries contain IDs only.
|
|
4603
|
+
live.inboundCoverage = [
|
|
4604
|
+
...new Set([
|
|
4605
|
+
...live.inboundCoverage,
|
|
4606
|
+
...live.promptQueue.flatMap((input) => (input.inputId ? [input.inputId] : [])),
|
|
4607
|
+
...live.pendingSystemReminders.flatMap((r) => r.inboundRetry?.inputIds ?? []),
|
|
4608
|
+
]),
|
|
4609
|
+
]
|
|
4610
|
+
const withdrawn = { inbound: new Set<string>(), background: new Set<string>() }
|
|
4611
|
+
try {
|
|
4612
|
+
await withdrawStoppedWorkInLane(
|
|
4613
|
+
live.key,
|
|
4614
|
+
[...live.inboundCoverage, ...(pending ? pendingReloadHandoffInputIds(pending) : [])],
|
|
4615
|
+
[
|
|
4616
|
+
...live.backgroundCoverage,
|
|
4617
|
+
...live.pendingSystemReminders.flatMap((r) =>
|
|
4618
|
+
r.backgroundObligationId ? [r.backgroundObligationId] : [],
|
|
4619
|
+
),
|
|
4620
|
+
...live.pendingSystemReminders.flatMap((r) => r.backgroundRetry?.obligationIds ?? []),
|
|
4621
|
+
...(pending ? pendingReloadHandoffBackgroundIds(pending) : []),
|
|
4622
|
+
],
|
|
4623
|
+
withdrawn,
|
|
4624
|
+
)
|
|
4625
|
+
} finally {
|
|
4626
|
+
live.inboundCoverage = live.inboundCoverage.filter((id) => !withdrawn.inbound.has(id))
|
|
4627
|
+
retireBackgroundCache(live, withdrawn.background)
|
|
4628
|
+
}
|
|
4629
|
+
// Retired only once withdrawn: a failed withdrawal keeps it owed.
|
|
4630
|
+
if (pending) retireStoppedReloadHandoff(live.keyId, pending)
|
|
4631
|
+
}
|
|
4632
|
+
} catch (error) {
|
|
4633
|
+
// A failed withdrawal remains owed for conservative boot recovery.
|
|
4634
|
+
// Bookkeeping must never prevent the user's parent abort.
|
|
4635
|
+
logger.error(`[channels] background stop withdrawal failed: ${describeError(error)}`)
|
|
4636
|
+
} finally {
|
|
4637
|
+
live.backgroundStopVersion++
|
|
4638
|
+
live.userStoppedTurnSeq = live.turnSeq
|
|
4639
|
+
live.lastTerminalReplyCompletion = null
|
|
4640
|
+
live.pendingTerminalReplyStop = null
|
|
4641
|
+
live.abortReasonThisTurn = { turnSeq: live.turnSeq, reason }
|
|
4642
|
+
clearTimeout(live.debounceTimer ?? undefined)
|
|
4643
|
+
live.debounceTimer = null
|
|
4644
|
+
live.firstUnprocessedAt = 0
|
|
4645
|
+
live.promptQueue.length = 0
|
|
4646
|
+
live.pendingSystemReminders.length = 0
|
|
4647
|
+
live.continueReplyTurn = null
|
|
4648
|
+
}
|
|
4649
|
+
})
|
|
4650
|
+
if (reason === 'user_stop') {
|
|
4651
|
+
try {
|
|
4652
|
+
await recoveryStopHandler?.(live.key, live.sessionId)
|
|
4653
|
+
} catch (error) {
|
|
4654
|
+
logger.error(`[channels] recovery stop withdrawal failed: ${describeError(error)}`)
|
|
4655
|
+
}
|
|
4656
|
+
}
|
|
3815
4657
|
void dropContinuationReactions(live)
|
|
3816
4658
|
enqueueTodoOutcomeWrite(live, {
|
|
3817
4659
|
agentDir: options.agentDir,
|
|
@@ -3819,7 +4661,9 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
3819
4661
|
turnId: live.sessionId,
|
|
3820
4662
|
stopReason: 'aborted',
|
|
3821
4663
|
})
|
|
3822
|
-
|
|
4664
|
+
void stopTypingHeartbeat(live).catch((error) =>
|
|
4665
|
+
logger.warn(`[channels] stop typing failed: ${describeError(error)}`),
|
|
4666
|
+
)
|
|
3823
4667
|
let aborted = true
|
|
3824
4668
|
try {
|
|
3825
4669
|
await live.session.abort()
|
|
@@ -3846,6 +4690,69 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
3846
4690
|
return aborted
|
|
3847
4691
|
}
|
|
3848
4692
|
|
|
4693
|
+
// /stop for a conversation with no live session whose owed work is still a
|
|
4694
|
+
// reload handoff (its successor recreate failed or has not installed yet).
|
|
4695
|
+
// Withdraws that work by key without recreating a session, so a later
|
|
4696
|
+
// successor cannot adopt and answer a stopped request. The handoff is retired
|
|
4697
|
+
// only once the withdrawal commits; a failed withdrawal is logged and leaves
|
|
4698
|
+
// it owed, the live stop's bookkeeping contract. A successor that installs
|
|
4699
|
+
// meanwhile has adopted the handoff's queues, so it is stopped in turn.
|
|
4700
|
+
// Resolves false when there was nothing to stop, including a handoff that
|
|
4701
|
+
// holds only observed context.
|
|
4702
|
+
const stopPendingReloadHandoff = async (keyId: string): Promise<boolean> => {
|
|
4703
|
+
const pending = pendingReloadHandoffs.get(keyId)
|
|
4704
|
+
if (!isStoppableReloadHandoff(pending)) return false
|
|
4705
|
+
const sourceSessionId = await backgroundObligations.withTargetLane(pending.key, async () => {
|
|
4706
|
+
const current = pendingReloadHandoffs.get(keyId)
|
|
4707
|
+
if (!isStoppableReloadHandoff(current)) return undefined
|
|
4708
|
+
try {
|
|
4709
|
+
await ready
|
|
4710
|
+
inboundJournal.assertAvailable()
|
|
4711
|
+
await withdrawStoppedWorkInLane(
|
|
4712
|
+
current.key,
|
|
4713
|
+
pendingReloadHandoffInputIds(current),
|
|
4714
|
+
pendingReloadHandoffBackgroundIds(current),
|
|
4715
|
+
)
|
|
4716
|
+
retireStoppedReloadHandoff(keyId, current)
|
|
4717
|
+
} catch (error) {
|
|
4718
|
+
logger.error(`[channels] ${keyId}: reload handoff stop withdrawal failed: ${describeError(error)}`)
|
|
4719
|
+
}
|
|
4720
|
+
return current.sessionId
|
|
4721
|
+
})
|
|
4722
|
+
const successor = liveSessions.get(keyId)
|
|
4723
|
+
if (successor !== undefined && !successor.destroyed) {
|
|
4724
|
+
await stopCurrentChannelTurn(successor, 'user_stop')
|
|
4725
|
+
return true
|
|
4726
|
+
}
|
|
4727
|
+
if (sourceSessionId === undefined) return false
|
|
4728
|
+
try {
|
|
4729
|
+
await recoveryStopHandler?.(pending.key, sourceSessionId)
|
|
4730
|
+
} catch (error) {
|
|
4731
|
+
logger.error(`[channels] recovery stop withdrawal failed: ${describeError(error)}`)
|
|
4732
|
+
}
|
|
4733
|
+
logger.info(`[channels] ${keyId}: command /stop withdrew a pending reload handoff site=user_stop reason=user_stop`)
|
|
4734
|
+
return true
|
|
4735
|
+
}
|
|
4736
|
+
|
|
4737
|
+
// The exact-key / thread-null-fallback policy of resolveLiveSessionForCommand,
|
|
4738
|
+
// applied to reload handoffs no live session has adopted.
|
|
4739
|
+
const resolvePendingReloadHandoffForCommand = (
|
|
4740
|
+
key: ChannelKey,
|
|
4741
|
+
): { kind: 'found'; keyId: string } | { kind: 'ambiguous'; count: number } | { kind: 'none' } => {
|
|
4742
|
+
const exact = channelKeyId(key)
|
|
4743
|
+
if (isStoppableReloadHandoff(pendingReloadHandoffs.get(exact))) return { kind: 'found', keyId: exact }
|
|
4744
|
+
if (key.thread !== null) return { kind: 'none' }
|
|
4745
|
+
const matches = Array.from(pendingReloadHandoffs).filter(
|
|
4746
|
+
([, pending]) =>
|
|
4747
|
+
isStoppableReloadHandoff(pending) &&
|
|
4748
|
+
pending.key.adapter === key.adapter &&
|
|
4749
|
+
pending.key.workspace === key.workspace &&
|
|
4750
|
+
pending.key.chat === key.chat,
|
|
4751
|
+
)
|
|
4752
|
+
if (matches.length > 1) return { kind: 'ambiguous', count: matches.length }
|
|
4753
|
+
return matches[0] !== undefined ? { kind: 'found', keyId: matches[0][0] } : { kind: 'none' }
|
|
4754
|
+
}
|
|
4755
|
+
|
|
3849
4756
|
// ensureLive() installs a session BEFORE the engage/observe decision, so a
|
|
3850
4757
|
// bystander agent that only OBSERVED a thread still holds an exact-thread
|
|
3851
4758
|
// session with nothing running. Without this gate that agent aborts its idle
|
|
@@ -3926,6 +4833,12 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
3926
4833
|
)
|
|
3927
4834
|
}
|
|
3928
4835
|
|
|
4836
|
+
// The logical turn told the user more was coming and has not delivered it: a
|
|
4837
|
+
// `more_work_this_turn: true` reply, or a landed status/continuation message
|
|
4838
|
+
// that no substantive output followed. Silence cannot answer such a promise.
|
|
4839
|
+
const owesPromisedWork = (live: LiveSession): boolean =>
|
|
4840
|
+
live.promisedWorkOutstandingThisLogicalTurn || live.statusOutputOutstandingThisLogicalTurn
|
|
4841
|
+
|
|
3929
4842
|
const maybePostDeferredProviderError = async (
|
|
3930
4843
|
live: LiveSession,
|
|
3931
4844
|
completedReplyThisTurn: boolean,
|
|
@@ -3947,12 +4860,14 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
3947
4860
|
// fail over once `startedToolExecution` is set) also means a side effect
|
|
3948
4861
|
// may already be public. Surfacing the notice here strands a "connection
|
|
3949
4862
|
// dropped" warning above the agent's own successful work. A logical turn
|
|
3950
|
-
// holding an outstanding continue-reply
|
|
3951
|
-
// explicitly told the user more was coming, so dying
|
|
3952
|
-
// them waiting and the notice must still fire. Operators
|
|
3953
|
-
// failure from the `LLM call failed` line plus this suppression
|
|
3954
|
-
|
|
4863
|
+
// holding an outstanding promise (a continue-reply or a status message) is
|
|
4864
|
+
// the opposite case: it explicitly told the user more was coming, so dying
|
|
4865
|
+
// silently WOULD leave them waiting and the notice must still fire. Operators
|
|
4866
|
+
// still get the failure from the `LLM call failed` line plus this suppression
|
|
4867
|
+
// record.
|
|
4868
|
+
if (live.toolExecutionThisLogicalTurn && !owesPromisedWork(live)) {
|
|
3955
4869
|
live.pendingProviderError = null
|
|
4870
|
+
live.providerErrorSilencedThisLogicalTurn = true
|
|
3956
4871
|
logger.warn(
|
|
3957
4872
|
`[channels] ${live.keyId}: provider_error_notice_suppressed reason=tool_activity_this_turn notice="${pending.safeMessage}"`,
|
|
3958
4873
|
)
|
|
@@ -3966,21 +4881,50 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
3966
4881
|
// a false-failure notice above the retry's eventual reply — the same defect
|
|
3967
4882
|
// one logical turn later.
|
|
3968
4883
|
if (retryQueued) {
|
|
3969
|
-
live.pendingProviderError = { turnSeq: live.turnSeq + 1
|
|
4884
|
+
live.pendingProviderError = { ...pending, turnSeq: live.turnSeq + 1 }
|
|
3970
4885
|
return
|
|
3971
4886
|
}
|
|
3972
4887
|
live.pendingProviderError = null
|
|
4888
|
+
// Only a turn answering a user errors out in the channel. A turn opened by
|
|
4889
|
+
// background work stays silent; the `LLM call failed` line plus this record
|
|
4890
|
+
// carry the failure to the operator, and turn-end settles its coverage.
|
|
4891
|
+
if (live.backgroundWakeLogicalTurn) {
|
|
4892
|
+
logger.warn(
|
|
4893
|
+
`[channels] ${live.keyId}: provider_error_notice_suppressed reason=background_wake_turn notice="${pending.safeMessage}"`,
|
|
4894
|
+
)
|
|
4895
|
+
return
|
|
4896
|
+
}
|
|
3973
4897
|
// Genuinely empty turn with no retry left: surface the REDACTED
|
|
3974
4898
|
// `safeMessage` (never raw provider text, which can carry response bodies /
|
|
3975
4899
|
// URLs / tokens) via a 'system' send — the one-shot bypass path that lands
|
|
3976
4900
|
// regardless of per-turn send caps — so the human doesn't just see silence.
|
|
4901
|
+
// The notice is this turn's terminal answer to the requests it owns, captured
|
|
4902
|
+
// before the send so a newer claim can never be closed by it. Consumed child
|
|
4903
|
+
// results are not answered by a provider failure and stay owed.
|
|
4904
|
+
let inboundCoverage: InboundRef[] = []
|
|
4905
|
+
try {
|
|
4906
|
+
inboundCoverage = await captureInboundResultCoverage(live.sessionId)
|
|
4907
|
+
} catch (error) {
|
|
4908
|
+
logger.error(`[channels] ${live.keyId}: provider-error notice coverage capture failed: ${describeError(error)}`)
|
|
4909
|
+
}
|
|
4910
|
+
// Only a retry-and-see failure (transient, or unclassified with nothing the
|
|
4911
|
+
// user could act on) collapses to a reaction. Auth, billing, quota,
|
|
4912
|
+
// model-config and rate limits keep the text notice: every later request
|
|
4913
|
+
// fails too until someone acts, so telling the user is worth the noise.
|
|
4914
|
+
if (pending.retryAndSee && (await reactTurnFailed(live))) {
|
|
4915
|
+
logger.warn(`[channels] ${live.keyId}: provider-error marked with reaction notice="${pending.safeMessage}"`)
|
|
4916
|
+
if (inboundCoverage.length > 0) {
|
|
4917
|
+
await recordLandedBackgroundResponse(live, 'provider-error-notice', [], inboundCoverage)
|
|
4918
|
+
}
|
|
4919
|
+
return
|
|
4920
|
+
}
|
|
3977
4921
|
const result = await send(
|
|
3978
4922
|
{
|
|
3979
4923
|
adapter: live.key.adapter,
|
|
3980
4924
|
workspace: live.key.workspace,
|
|
3981
4925
|
chat: live.key.chat,
|
|
3982
4926
|
thread: live.key.thread,
|
|
3983
|
-
text:
|
|
4927
|
+
text: pending.safeMessage,
|
|
3984
4928
|
},
|
|
3985
4929
|
{ source: 'system', outputKind: 'meta' },
|
|
3986
4930
|
).catch((sendErr) => {
|
|
@@ -3989,6 +4933,90 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
3989
4933
|
})
|
|
3990
4934
|
if (result !== null && !result.ok) {
|
|
3991
4935
|
logger.warn(`[channels] ${live.keyId}: provider-error notice send failed: ${result.error}`)
|
|
4936
|
+
} else if (result !== null && inboundCoverage.length > 0) {
|
|
4937
|
+
await recordLandedBackgroundResponse(live, 'provider-error-notice', [], inboundCoverage)
|
|
4938
|
+
}
|
|
4939
|
+
}
|
|
4940
|
+
|
|
4941
|
+
// The owning terminal decision for a logical turn that ended still holding
|
|
4942
|
+
// owed coverage. Runs after todo continuation and staged-fallback resolution
|
|
4943
|
+
// had their chance to queue a continuation. Work that provably continues keeps
|
|
4944
|
+
// its coverage: queued input or reminders (the next iteration decides), a
|
|
4945
|
+
// staged fallback, a running deferred child, an outstanding GitHub thread
|
|
4946
|
+
// close-out, or a recovery the review guard deferred to the next inbound.
|
|
4947
|
+
// Anything else ended here, in this process: it is transferred now as a
|
|
4948
|
+
// durable live-turn-ended notice — never a restart notice, and never debt
|
|
4949
|
+
// grafted onto a later unrelated request. A graceful restart's aborted turns
|
|
4950
|
+
// are the exception: the process is restarting, so boot recovery owns them.
|
|
4951
|
+
// A failed transfer keeps the caches, so the next lifecycle decision retries;
|
|
4952
|
+
// the journal rows stay owed meanwhile.
|
|
4953
|
+
// A logical turn that may not error out in the channel: one opened by background
|
|
4954
|
+
// work rather than a user, or one whose provider-error notice was already
|
|
4955
|
+
// suppressed because it ran tools. Whatever such a turn leaves owed settles
|
|
4956
|
+
// silently instead of becoming a live-turn-ended notice, wherever it ends.
|
|
4957
|
+
const silentTurnEnd = (live: LiveSession): { log: string; reason: string } | null =>
|
|
4958
|
+
live.backgroundWakeLogicalTurn
|
|
4959
|
+
? { log: 'background_wake_turn_ended_silently', reason: 'background-wake-turn-ended' }
|
|
4960
|
+
: live.providerErrorSilencedThisLogicalTurn
|
|
4961
|
+
? {
|
|
4962
|
+
log: 'turn_end_notice_suppressed reason=provider_error_after_tool_activity',
|
|
4963
|
+
reason: 'provider-error-after-tool-activity',
|
|
4964
|
+
}
|
|
4965
|
+
: null
|
|
4966
|
+
|
|
4967
|
+
// Settles a source turn's held rows by their exact generations, for when a fresh
|
|
4968
|
+
// batch abandons a silent turn: the rows may be owned by a predecessor session,
|
|
4969
|
+
// so they cannot go through the current turn's settlement. Call only in the lane.
|
|
4970
|
+
const settleAbandonedSilentlyInLane = async (
|
|
4971
|
+
live: LiveSession,
|
|
4972
|
+
inbound: readonly InboundRecord[],
|
|
4973
|
+
background: readonly BackgroundObligation[],
|
|
4974
|
+
reason: string,
|
|
4975
|
+
): Promise<void> => {
|
|
4976
|
+
const inboundRefs = inbound.map((row) => ({ inputId: row.inputId, generation: row.generation }))
|
|
4977
|
+
const backgroundRefs = background.map((row) => ({ obligationId: row.obligationId, generation: row.generation }))
|
|
4978
|
+
const outcome = { kind: 'intentionally-suppressed' as const, decisionId: randomUUID(), reason }
|
|
4979
|
+
if (inboundRefs.length > 0) {
|
|
4980
|
+
await inboundJournal.settle(inboundRefs, outcome, backgroundRefs, live.key)
|
|
4981
|
+
} else if (backgroundRefs.length > 0) {
|
|
4982
|
+
inboundJournal.assertAvailable()
|
|
4983
|
+
await backgroundObligations.settle(backgroundRefs, outcome)
|
|
4984
|
+
}
|
|
4985
|
+
}
|
|
4986
|
+
|
|
4987
|
+
const transferAbandonedTurnCoverage = async (live: LiveSession): Promise<void> => {
|
|
4988
|
+
const keepsCoverage = (): boolean =>
|
|
4989
|
+
restartShutdownPending ||
|
|
4990
|
+
live.destroyed ||
|
|
4991
|
+
live.pendingTeardown ||
|
|
4992
|
+
live.promptQueue.length > 0 ||
|
|
4993
|
+
live.pendingSystemReminders.length > 0 ||
|
|
4994
|
+
live.stagedFallbackCause !== null ||
|
|
4995
|
+
live.githubReviewThreadCloseout !== null ||
|
|
4996
|
+
live.nextInboundContinuationTurnId === live.backgroundTurnId ||
|
|
4997
|
+
(live.inboundCoverage.length === 0 && live.backgroundCoverage.length === 0)
|
|
4998
|
+
if (keepsCoverage()) return
|
|
4999
|
+
try {
|
|
5000
|
+
await backgroundObligations.withTargetLane(live.key, async () => {
|
|
5001
|
+
if (keepsCoverage() || awaitsDeferredChild(live, 'turn-end-coverage')) return
|
|
5002
|
+
// Anything a silent turn cannot settle (owned elsewhere) still transfers below.
|
|
5003
|
+
const silentEnd = silentTurnEnd(live)
|
|
5004
|
+
if (silentEnd !== null) {
|
|
5005
|
+
logger.warn(
|
|
5006
|
+
`[channels] ${live.keyId}: ${silentEnd.log} ` +
|
|
5007
|
+
`inbound=${live.inboundCoverage.length} background=${live.backgroundCoverage.length}`,
|
|
5008
|
+
)
|
|
5009
|
+
await settleBackgroundCoverageInLane(
|
|
5010
|
+
live,
|
|
5011
|
+
'intentionally-suppressed',
|
|
5012
|
+
silentEnd.reason,
|
|
5013
|
+
live.backgroundCoverage,
|
|
5014
|
+
)
|
|
5015
|
+
}
|
|
5016
|
+
await transferHeldCoverageInLane(live)
|
|
5017
|
+
})
|
|
5018
|
+
} catch (error) {
|
|
5019
|
+
logger.error(`[channels] ${live.keyId}: live-turn-ended transfer failed: ${describeError(error)}`)
|
|
3992
5020
|
}
|
|
3993
5021
|
}
|
|
3994
5022
|
|
|
@@ -4032,6 +5060,8 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
4032
5060
|
live.nextPromptMaxTokens = undefined
|
|
4033
5061
|
live.githubReviewOutputTurn = null
|
|
4034
5062
|
live.toolExecutionThisLogicalTurn = false
|
|
5063
|
+
live.backgroundWakeLogicalTurn = true
|
|
5064
|
+
live.providerErrorSilencedThisLogicalTurn = false
|
|
4035
5065
|
live.qualifyingWorkThisLogicalTurn = false
|
|
4036
5066
|
}
|
|
4037
5067
|
|
|
@@ -4058,20 +5088,306 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
4058
5088
|
// without re-entering route(), the route() call site alone wouldn't
|
|
4059
5089
|
// keep the indicator alive across multiple drain iterations.
|
|
4060
5090
|
startTypingHeartbeat(live)
|
|
4061
|
-
|
|
4062
|
-
const
|
|
4063
|
-
|
|
4064
|
-
|
|
4065
|
-
|
|
4066
|
-
|
|
4067
|
-
|
|
4068
|
-
|
|
4069
|
-
|
|
4070
|
-
|
|
4071
|
-
|
|
4072
|
-
|
|
4073
|
-
|
|
4074
|
-
|
|
5091
|
+
// Result claims commit before content leaves the queues or enters a prompt.
|
|
5092
|
+
const consume = async () => {
|
|
5093
|
+
if (live.destroyed || live.pendingTeardown) return undefined
|
|
5094
|
+
if (live.promptQueue.length === 0 && live.pendingSystemReminders.length === 0) return undefined
|
|
5095
|
+
await ready
|
|
5096
|
+
await transferMismatchedAccountCoverageInLane(live)
|
|
5097
|
+
await resumePreparedTransfersInLane(
|
|
5098
|
+
live.key,
|
|
5099
|
+
[
|
|
5100
|
+
...live.inboundCoverage,
|
|
5101
|
+
...live.promptQueue.map((input) => input.inputId),
|
|
5102
|
+
...live.pendingSystemReminders.flatMap((reminder) => reminder.inboundRetry?.inputIds ?? []),
|
|
5103
|
+
],
|
|
5104
|
+
[
|
|
5105
|
+
...live.backgroundCoverage,
|
|
5106
|
+
...live.pendingSystemReminders.flatMap((reminder) => [
|
|
5107
|
+
...(reminder.backgroundObligationId ? [reminder.backgroundObligationId] : []),
|
|
5108
|
+
...(reminder.backgroundRetry?.obligationIds ?? []),
|
|
5109
|
+
]),
|
|
5110
|
+
],
|
|
5111
|
+
)
|
|
5112
|
+
// A refused handoff may retain identities only on its retry reminder.
|
|
5113
|
+
// Completed transfers must neither enter a new prompt nor reach move().
|
|
5114
|
+
const stillUntransferred = (id: string) => {
|
|
5115
|
+
const row = inboundJournal.get(id)
|
|
5116
|
+
return row !== undefined && row.phase !== 'closed' && !row.transfer
|
|
5117
|
+
}
|
|
5118
|
+
live.promptQueue = live.promptQueue.filter((input) => stillUntransferred(input.inputId))
|
|
5119
|
+
live.inboundCoverage = live.inboundCoverage.filter(stillUntransferred)
|
|
5120
|
+
const staleBackground = new Set<string>()
|
|
5121
|
+
for (const id of new Set([
|
|
5122
|
+
...live.backgroundCoverage,
|
|
5123
|
+
...live.pendingSystemReminders.flatMap((reminder) => [
|
|
5124
|
+
...(reminder.backgroundObligationId ? [reminder.backgroundObligationId] : []),
|
|
5125
|
+
...(reminder.backgroundRetry?.obligationIds ?? []),
|
|
5126
|
+
]),
|
|
5127
|
+
])) {
|
|
5128
|
+
const row = await backgroundObligations.get(id)
|
|
5129
|
+
if (!row || row.phase === 'closed' || row.transfer) staleBackground.add(id)
|
|
5130
|
+
}
|
|
5131
|
+
const identityRetries = new Set(
|
|
5132
|
+
live.pendingSystemReminders.filter((reminder) => reminder.inboundRetry || reminder.backgroundRetry),
|
|
5133
|
+
)
|
|
5134
|
+
retireBackgroundCache(live, staleBackground)
|
|
5135
|
+
live.pendingSystemReminders = live.pendingSystemReminders.filter((reminder) => {
|
|
5136
|
+
if (reminder.inboundRetry)
|
|
5137
|
+
reminder.inboundRetry.inputIds = reminder.inboundRetry.inputIds.filter(stillUntransferred)
|
|
5138
|
+
return (
|
|
5139
|
+
!identityRetries.has(reminder) ||
|
|
5140
|
+
(reminder.inboundRetry?.inputIds.length ?? 0) > 0 ||
|
|
5141
|
+
(reminder.backgroundRetry?.obligationIds.length ?? 0) > 0
|
|
5142
|
+
)
|
|
5143
|
+
})
|
|
5144
|
+
if (live.promptQueue.length === 0 && live.pendingSystemReminders.length === 0) return undefined
|
|
5145
|
+
const batchCount = live.promptQueue.length
|
|
5146
|
+
const observedCount = live.contextBuffer.length
|
|
5147
|
+
const reminderCount = live.pendingSystemReminders.length
|
|
5148
|
+
const queued = live.pendingSystemReminders.slice(0, reminderCount)
|
|
5149
|
+
const eligible = new Set<PendingSystemReminder>()
|
|
5150
|
+
const refs: BackgroundObligationRef[] = []
|
|
5151
|
+
const readyRows: BackgroundObligation[] = []
|
|
5152
|
+
for (const reminder of queued) {
|
|
5153
|
+
if (!reminder.backgroundObligationId) {
|
|
5154
|
+
eligible.add(reminder)
|
|
5155
|
+
continue
|
|
5156
|
+
}
|
|
5157
|
+
const row = await backgroundObligations.get(reminder.backgroundObligationId)
|
|
5158
|
+
// Fetch/settlement may have overtaken this wake. Durable ownership
|
|
5159
|
+
// wins: retire the obsolete entry instead of poisoning later input.
|
|
5160
|
+
if (
|
|
5161
|
+
!row ||
|
|
5162
|
+
row.phase !== 'result-ready' ||
|
|
5163
|
+
row.claim ||
|
|
5164
|
+
row.transfer ||
|
|
5165
|
+
channelKeyId(row.target) !== live.keyId
|
|
5166
|
+
)
|
|
5167
|
+
continue
|
|
5168
|
+
eligible.add(reminder)
|
|
5169
|
+
refs.push({ obligationId: row.obligationId, generation: row.generation })
|
|
5170
|
+
readyRows.push(row)
|
|
5171
|
+
}
|
|
5172
|
+
const fresh = batchCount > 0 || queued.some((r) => eligible.has(r) && r.kind === 'wakeup')
|
|
5173
|
+
const carried = fresh ? undefined : queued.find((r) => r.backgroundRetry)?.backgroundRetry
|
|
5174
|
+
const turnId = fresh ? randomUUID() : (carried?.turnId ?? live.backgroundTurnId)
|
|
5175
|
+
await ready
|
|
5176
|
+
inboundJournal.assertAvailable()
|
|
5177
|
+
const inboundRetry = fresh ? undefined : queued.find((r) => r.inboundRetry)?.inboundRetry
|
|
5178
|
+
const inputIds = live.promptQueue
|
|
5179
|
+
.slice(0, batchCount)
|
|
5180
|
+
.flatMap((input) => (input.inputId ? [input.inputId] : []))
|
|
5181
|
+
const inboundRefs = await inboundJournal.resolve(inputIds)
|
|
5182
|
+
let carriedBackground: BackgroundObligationRef[] = []
|
|
5183
|
+
if (live.inboundCoverage.length === 0 && live.backgroundCoverage.length === 0) {
|
|
5184
|
+
live.coverageDeferredSince = null
|
|
5185
|
+
live.coverageDeferredParentSessionId = null
|
|
5186
|
+
}
|
|
5187
|
+
if (fresh && (live.inboundCoverage.length > 0 || live.backgroundCoverage.length > 0)) {
|
|
5188
|
+
// Coverage left open by a turn deferred to background work is owed, not
|
|
5189
|
+
// abandoned: carry it forward like a wakeup instead of ending it. That
|
|
5190
|
+
// covers deferred inputs and child results the deferring turn already
|
|
5191
|
+
// consumed, together or alone. Cached IDs are hints (a reload successor
|
|
5192
|
+
// adopts them), so only rows the recorded source turn and owner still hold
|
|
5193
|
+
// at their current generation, target and account are carried.
|
|
5194
|
+
const sourceOwner = live.backgroundTurnOwnerSessionId
|
|
5195
|
+
const sourceAccount = live.turnAccountIdentity ?? currentAccountIdentityFor(live.key)
|
|
5196
|
+
const cached: Array<InboundRecord | BackgroundObligation | undefined> = live.inboundCoverage.map((id) =>
|
|
5197
|
+
inboundJournal.get(id),
|
|
5198
|
+
)
|
|
5199
|
+
for (const id of new Set(live.backgroundCoverage)) cached.push(await backgroundObligations.get(id))
|
|
5200
|
+
const held = cached.filter(
|
|
5201
|
+
(row): row is InboundRecord | BackgroundObligation =>
|
|
5202
|
+
row?.phase === 'turn-owned' &&
|
|
5203
|
+
!row.transfer &&
|
|
5204
|
+
row.claim?.turnId === live.backgroundTurnId &&
|
|
5205
|
+
row.claim.ownerSessionId === sourceOwner &&
|
|
5206
|
+
row.claim.epoch === inboundJournal.epoch &&
|
|
5207
|
+
row.claim.generation === row.generation &&
|
|
5208
|
+
channelKeyId(row.target) === live.keyId &&
|
|
5209
|
+
row.accountIdentity === sourceAccount,
|
|
5210
|
+
)
|
|
5211
|
+
const deferred = held.flatMap((row) => ('inputId' in row ? [row] : []))
|
|
5212
|
+
const deferredBackground = held.flatMap((row) => ('obligationId' in row ? [row] : []))
|
|
5213
|
+
live.inboundCoverage = deferred.map((row) => row.inputId)
|
|
5214
|
+
live.backgroundCoverage = deferredBackground.map((row) => row.obligationId)
|
|
5215
|
+
const deferredSince = live.coverageDeferredSince ?? live.logicalTurnStartedAt
|
|
5216
|
+
// The deferred work is a child of the session the coverage was deferred from
|
|
5217
|
+
// or of the session now holding its claims; the two differ once a successor
|
|
5218
|
+
// under another session ID has carried it.
|
|
5219
|
+
const deferredParent = live.coverageDeferredParentSessionId ?? sourceOwner
|
|
5220
|
+
const parents = [...new Set([deferredParent, sourceOwner])]
|
|
5221
|
+
// A child of the deferring turn that already finished no longer shows as
|
|
5222
|
+
// running, yet its durable result-ready wake in this same batch still
|
|
5223
|
+
// carries the debt into the coalesced turn. Only an exact match counts:
|
|
5224
|
+
// spawned by one of those sessions, during the deferred interval, under the
|
|
5225
|
+
// source account. Generic wake text or an unrelated child never does.
|
|
5226
|
+
const awaitingDeferredWork =
|
|
5227
|
+
readyRows.some(
|
|
5228
|
+
(row) =>
|
|
5229
|
+
parents.includes(row.parentSessionId) &&
|
|
5230
|
+
row.accountIdentity === sourceAccount &&
|
|
5231
|
+
row.acceptedAt >= deferredSince,
|
|
5232
|
+
) ||
|
|
5233
|
+
parents.some((parent) =>
|
|
5234
|
+
isAwaitingBackgroundChild(live, 'fresh-deferred-coverage', deferredSince, parent),
|
|
5235
|
+
)
|
|
5236
|
+
// A documented guard deferred the source turn's recovery to the next
|
|
5237
|
+
// inbound (see `nextInboundContinuationTurnId`): this batch continues it.
|
|
5238
|
+
const continuesSourceTurn =
|
|
5239
|
+
awaitingDeferredWork || live.nextInboundContinuationTurnId === live.backgroundTurnId
|
|
5240
|
+
if (held.length === 0) {
|
|
5241
|
+
live.coverageDeferredSince = null
|
|
5242
|
+
live.coverageDeferredParentSessionId = null
|
|
5243
|
+
} else if (batchCount > 0 && !continuesSourceTurn) {
|
|
5244
|
+
// Nothing continues the source turn, so its debt ended with it. A new
|
|
5245
|
+
// human batch is independent work that never answers it: the remainder
|
|
5246
|
+
// becomes a durable live-turn-ended notice, not debt on the new request,
|
|
5247
|
+
// unless the source turn was one that may not error out.
|
|
5248
|
+
const silentEnd = silentTurnEnd(live)
|
|
5249
|
+
if (silentEnd !== null) {
|
|
5250
|
+
logger.warn(
|
|
5251
|
+
`[channels] ${live.keyId}: ${silentEnd.log} abandoned_by=fresh_batch ` +
|
|
5252
|
+
`inbound=${deferred.length} background=${deferredBackground.length}`,
|
|
5253
|
+
)
|
|
5254
|
+
await settleAbandonedSilentlyInLane(live, deferred, deferredBackground, silentEnd.reason)
|
|
5255
|
+
} else {
|
|
5256
|
+
await transferLostCoverageInLane(live.key, live.inboundCoverage, live.backgroundCoverage, sourceOwner)
|
|
5257
|
+
}
|
|
5258
|
+
live.coverageDeferredSince = null
|
|
5259
|
+
live.coverageDeferredParentSessionId = null
|
|
5260
|
+
} else {
|
|
5261
|
+
// A wake-only batch still carries deferred inputs, but a consumed child
|
|
5262
|
+
// result moves only on evidence that the source turn continues here. An
|
|
5263
|
+
// abandoned one ended with its turn: it is transferred as a live-turn-ended
|
|
5264
|
+
// notice rather than dropped from every live cache while still owed.
|
|
5265
|
+
if (!continuesSourceTurn && deferredBackground.length > 0) {
|
|
5266
|
+
const silentEnd = silentTurnEnd(live)
|
|
5267
|
+
if (silentEnd !== null) {
|
|
5268
|
+
logger.warn(
|
|
5269
|
+
`[channels] ${live.keyId}: ${silentEnd.log} abandoned_by=wake ` +
|
|
5270
|
+
`background=${deferredBackground.length}`,
|
|
5271
|
+
)
|
|
5272
|
+
await settleAbandonedSilentlyInLane(live, [], deferredBackground, silentEnd.reason)
|
|
5273
|
+
} else {
|
|
5274
|
+
await transferLostCoverageInLane(
|
|
5275
|
+
live.key,
|
|
5276
|
+
[],
|
|
5277
|
+
deferredBackground.map((row) => row.obligationId),
|
|
5278
|
+
sourceOwner,
|
|
5279
|
+
)
|
|
5280
|
+
}
|
|
5281
|
+
}
|
|
5282
|
+
const background = continuesSourceTurn
|
|
5283
|
+
? deferredBackground.map((row) => ({ obligationId: row.obligationId, generation: row.generation }))
|
|
5284
|
+
: []
|
|
5285
|
+
if (batchCount > 0) live.coverageDeferredSince = deferredSince
|
|
5286
|
+
live.coverageDeferredParentSessionId = deferredParent
|
|
5287
|
+
if (deferred.length > 0 || background.length > 0) {
|
|
5288
|
+
// One journal decision moves both kinds out of the source turn.
|
|
5289
|
+
const moved = await inboundJournal.move(
|
|
5290
|
+
inboundJournal.resolve(live.inboundCoverage),
|
|
5291
|
+
{ fromTurnId: live.backgroundTurnId, turnId, ownerSessionId: live.sessionId, target: live.key },
|
|
5292
|
+
background,
|
|
5293
|
+
)
|
|
5294
|
+
inboundRefs.push(...moved.inboundRefs)
|
|
5295
|
+
carriedBackground = moved.backgroundRefs
|
|
5296
|
+
}
|
|
5297
|
+
}
|
|
5298
|
+
}
|
|
5299
|
+
if (fresh) live.nextInboundContinuationTurnId = null
|
|
5300
|
+
const owner = { turnId, ownerSessionId: live.sessionId, target: live.key }
|
|
5301
|
+
const claimedCoverage =
|
|
5302
|
+
inboundRefs.length > 0
|
|
5303
|
+
? await inboundJournal.claim(inboundRefs, owner, refs)
|
|
5304
|
+
: {
|
|
5305
|
+
inboundRefs: [],
|
|
5306
|
+
backgroundRefs: refs.length > 0 ? await backgroundObligations.claim(refs, owner) : [],
|
|
5307
|
+
}
|
|
5308
|
+
const claimed = claimedCoverage.backgroundRefs
|
|
5309
|
+
let carriedInboundIds = fresh ? [] : live.inboundCoverage
|
|
5310
|
+
const retryRefs = inboundRetry ? await inboundJournal.resolve(inboundRetry.inputIds) : []
|
|
5311
|
+
const moved: BackgroundObligationRef[] = []
|
|
5312
|
+
if (carried) {
|
|
5313
|
+
for (const id of new Set(carried.obligationIds)) {
|
|
5314
|
+
const row = await backgroundObligations.get(id)
|
|
5315
|
+
if (
|
|
5316
|
+
!row ||
|
|
5317
|
+
row.transfer ||
|
|
5318
|
+
row.phase !== 'turn-owned' ||
|
|
5319
|
+
row.claim?.turnId !== carried.turnId ||
|
|
5320
|
+
channelKeyId(row.target) !== live.keyId
|
|
5321
|
+
)
|
|
5322
|
+
continue
|
|
5323
|
+
moved.push({ obligationId: id, generation: row.generation })
|
|
5324
|
+
}
|
|
5325
|
+
}
|
|
5326
|
+
if (retryRefs.length > 0) {
|
|
5327
|
+
const movedCoverage = await inboundJournal.move(
|
|
5328
|
+
retryRefs,
|
|
5329
|
+
{ ...owner, fromTurnId: inboundRetry!.turnId },
|
|
5330
|
+
moved,
|
|
5331
|
+
)
|
|
5332
|
+
carriedInboundIds = movedCoverage.inboundRefs.map((ref) => ref.inputId)
|
|
5333
|
+
moved.splice(0, moved.length, ...movedCoverage.backgroundRefs)
|
|
5334
|
+
} else if (moved.length > 0) {
|
|
5335
|
+
const movedRefs = await backgroundObligations.move(moved, { ...owner, fromTurnId: carried!.turnId })
|
|
5336
|
+
moved.splice(0, moved.length, ...movedRefs)
|
|
5337
|
+
}
|
|
5338
|
+
live.inboundCoverage = [...carriedInboundIds, ...claimedCoverage.inboundRefs.map((ref) => ref.inputId)]
|
|
5339
|
+
live.backgroundTurnId = turnId
|
|
5340
|
+
if (fresh || carried) live.backgroundTurnOwnerSessionId = live.sessionId
|
|
5341
|
+
// Moved results and newly claimed completions are disjoint: one was
|
|
5342
|
+
// turn-owned, the other unclaimed and result-ready when the lane read it.
|
|
5343
|
+
if (fresh) live.backgroundCoverage = carriedBackground.map((ref) => ref.obligationId)
|
|
5344
|
+
else
|
|
5345
|
+
live.backgroundCoverage = (await resolveBackgroundCoverage(live, live.backgroundCoverage, true)).map(
|
|
5346
|
+
(ref) => ref.obligationId,
|
|
5347
|
+
)
|
|
5348
|
+
live.backgroundCoverage.push(...claimed.map((ref) => ref.obligationId))
|
|
5349
|
+
if (carried) {
|
|
5350
|
+
live.backgroundCoverage = [...moved, ...claimed].map((ref) => ref.obligationId)
|
|
5351
|
+
live.emptyTurnRetries = carried.emptyTurnRetries
|
|
5352
|
+
live.toolLeakRetries = carried.toolLeakRetries
|
|
5353
|
+
live.willingnessNudges = carried.willingnessNudges
|
|
5354
|
+
live.nextPromptMaxTokens = carried.nextPromptMaxTokens
|
|
5355
|
+
}
|
|
5356
|
+
const carriedWake = fresh ? undefined : queued.find((r) => r.backgroundWake !== undefined)?.backgroundWake
|
|
5357
|
+
if (carriedWake !== undefined) live.backgroundWakeLogicalTurn = carriedWake
|
|
5358
|
+
const ownedRow = live.inboundCoverage[0] ? inboundJournal.get(live.inboundCoverage[0]) : undefined
|
|
5359
|
+
const ownedInput = ownedRow?.phase === 'closed' ? undefined : ownedRow
|
|
5360
|
+
const ownedBackground = live.backgroundCoverage[0]
|
|
5361
|
+
? await backgroundObligations.get(live.backgroundCoverage[0])
|
|
5362
|
+
: undefined
|
|
5363
|
+
live.turnAccountIdentity =
|
|
5364
|
+
ownedInput?.accountIdentity ??
|
|
5365
|
+
ownedBackground?.accountIdentity ??
|
|
5366
|
+
(fresh ? undefined : live.turnAccountIdentity)
|
|
5367
|
+
return {
|
|
5368
|
+
batch: live.promptQueue.splice(0, batchCount),
|
|
5369
|
+
observed: live.contextBuffer.splice(0, observedCount),
|
|
5370
|
+
reminders: live.pendingSystemReminders
|
|
5371
|
+
.splice(0, reminderCount)
|
|
5372
|
+
.filter((reminder) => eligible.has(reminder)),
|
|
5373
|
+
stopVersion: live.backgroundStopVersion,
|
|
5374
|
+
}
|
|
5375
|
+
}
|
|
5376
|
+
const drained = await backgroundObligations.withTargetLane(live.key, consume)
|
|
5377
|
+
if (drained === undefined) break
|
|
5378
|
+
const { batch, observed, reminders, stopVersion } = drained
|
|
5379
|
+
if (live.backgroundStopVersion !== stopVersion || (batch.length === 0 && reminders.length === 0)) continue
|
|
5380
|
+
live.currentTurnAttachments = collectTurnAttachments(observed, batch)
|
|
5381
|
+
// A reminder-only iteration carrying externally-injected work opens a
|
|
5382
|
+
// NEW logical turn. Without this, the completed-subagent wakeup that
|
|
5383
|
+
// follows a turn which spawned it inherits that turn's
|
|
5384
|
+
// `toolExecutionThisLogicalTurn`, and a provider failure on the wakeup
|
|
5385
|
+
// is misread as "the turn already ran tools, so it wasn't silent" —
|
|
5386
|
+
// suppressing the notice for a prompt that ran no tool and produced
|
|
5387
|
+
// nothing. Observed in production as a review verdict lost with zero
|
|
5388
|
+
// user-visible signal. Retry nudges deliberately do NOT trip this: they
|
|
5389
|
+
// ARE the same logical turn retrying.
|
|
5390
|
+
const wakeupTurn = batch.length === 0 && reminders.some((r) => r.kind === 'wakeup')
|
|
4075
5391
|
// Gated on `!wakeupTurn` because a willingness nudge can coalesce into the
|
|
4076
5392
|
// same iteration as a wakeup. The wakeup wins the boundary and clears the
|
|
4077
5393
|
// nudge budget, so the willingness bookkeeping now describes a superseded
|
|
@@ -4168,8 +5484,11 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
4168
5484
|
// empty-turn fallback across the reminder-only retry iterations.
|
|
4169
5485
|
live.githubReviewOutputTurn = null
|
|
4170
5486
|
live.toolExecutionThisLogicalTurn = false
|
|
5487
|
+
live.backgroundWakeLogicalTurn = false
|
|
5488
|
+
live.providerErrorSilencedThisLogicalTurn = false
|
|
4171
5489
|
live.qualifyingWorkThisLogicalTurn = false
|
|
4172
5490
|
live.promisedWorkOutstandingThisLogicalTurn = false
|
|
5491
|
+
live.statusOutputOutstandingThisLogicalTurn = false
|
|
4173
5492
|
} else if (live.lastTurnAuthorId !== null) {
|
|
4174
5493
|
live.currentTurnEngageReactions = []
|
|
4175
5494
|
// Reminder-only turn (batch.length === 0, reminders.length > 0):
|
|
@@ -4249,6 +5568,16 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
4249
5568
|
live.pendingUserTurnSignal = { signal: getQuestionSignal(retrievalQuery) }
|
|
4250
5569
|
}
|
|
4251
5570
|
const retrievalContext = await fireSessionTurnStart(live, retrievalQuery)
|
|
5571
|
+
if (live.backgroundStopVersion !== stopVersion) continue
|
|
5572
|
+
const accountMayPrompt = await backgroundObligations.withTargetLane(live.key, async () => {
|
|
5573
|
+
const currentAccount = await transferMismatchedAccountCoverageInLane(live)
|
|
5574
|
+
return (
|
|
5575
|
+
currentAccount === undefined ||
|
|
5576
|
+
live.turnAccountIdentity === undefined ||
|
|
5577
|
+
currentAccount === live.turnAccountIdentity
|
|
5578
|
+
)
|
|
5579
|
+
})
|
|
5580
|
+
if (!accountMayPrompt || live.backgroundStopVersion !== stopVersion) continue
|
|
4252
5581
|
const promptText = retrievalContext.results.length > 0 ? `${text}\n\n${retrievalContext.results}` : text
|
|
4253
5582
|
applyTurnThinkingLevel(live.session, retrievalQuery, live.turnThinkingDefault, live.lastQuestionSignal)
|
|
4254
5583
|
live.promptInFlight = true
|
|
@@ -4259,7 +5588,13 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
4259
5588
|
session: live.session,
|
|
4260
5589
|
text: promptText,
|
|
4261
5590
|
shouldFailover: (err) => isFailoverWorthy(err.message),
|
|
4262
|
-
|
|
5591
|
+
// The resume never re-runs a tool (it continues from the completed
|
|
5592
|
+
// tool result), so the only duplicate it could cause is a second
|
|
5593
|
+
// reply. Allow it while the user has seen nothing beyond a pending
|
|
5594
|
+
// progress ack, and never after a /stop.
|
|
5595
|
+
authorizeRetryAfterCompletedToolResult: () =>
|
|
5596
|
+
live.backgroundStopVersion === stopVersion &&
|
|
5597
|
+
(live.successfulChannelSends === successfulSendsBeforePrompt || hasPendingContinueReply(live)),
|
|
4263
5598
|
...(options.retryRandom !== undefined ? { retryRandom: options.retryRandom } : {}),
|
|
4264
5599
|
...(options.onRetryBackoffStart !== undefined ? { onRetryBackoffStart: options.onRetryBackoffStart } : {}),
|
|
4265
5600
|
setModelForRef: async (ref) => {
|
|
@@ -4267,6 +5602,19 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
4267
5602
|
live.activeModelRef = ref
|
|
4268
5603
|
},
|
|
4269
5604
|
beforeAttempt: () => {
|
|
5605
|
+
inboundJournal.assertAvailable()
|
|
5606
|
+
if (live.backgroundStopVersion !== stopVersion) throw new Error('Channel turn stopped before dispatch')
|
|
5607
|
+
const currentAccount = currentAccountIdentityFor(live.key)
|
|
5608
|
+
if (
|
|
5609
|
+
currentAccount !== undefined &&
|
|
5610
|
+
live.turnAccountIdentity !== undefined &&
|
|
5611
|
+
currentAccount !== live.turnAccountIdentity
|
|
5612
|
+
) {
|
|
5613
|
+
void backgroundObligations
|
|
5614
|
+
.withTargetLane(live.key, () => transferMismatchedAccountCoverageInLane(live))
|
|
5615
|
+
.catch((error) => logger.error(`[channels] account-change transfer failed: ${describeError(error)}`))
|
|
5616
|
+
throw new Error('Channel turn account identity changed before dispatch')
|
|
5617
|
+
}
|
|
4270
5618
|
applyTurnThinkingLevel(live.session, retrievalQuery, live.turnThinkingDefault, live.lastQuestionSignal)
|
|
4271
5619
|
},
|
|
4272
5620
|
onAttemptFailed: (attempt) => {
|
|
@@ -4325,7 +5673,13 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
4325
5673
|
// this turnSeq by subscribeProviderErrors (first-detected wins).
|
|
4326
5674
|
const hardProviderError = detectHardProviderError(err)
|
|
4327
5675
|
if (hardProviderError !== null && live.pendingProviderError?.turnSeq !== live.turnSeq) {
|
|
4328
|
-
live.pendingProviderError = {
|
|
5676
|
+
live.pendingProviderError = {
|
|
5677
|
+
turnSeq: live.turnSeq,
|
|
5678
|
+
safeMessage: hardProviderError.safeMessage,
|
|
5679
|
+
retryAndSee:
|
|
5680
|
+
isTransientProviderFailure(hardProviderError.message) ||
|
|
5681
|
+
isUnclassifiedProviderNotice(hardProviderError.safeMessage),
|
|
5682
|
+
}
|
|
4329
5683
|
}
|
|
4330
5684
|
live.consecutiveSends.clear()
|
|
4331
5685
|
live.lastSentText.clear()
|
|
@@ -4342,13 +5696,9 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
4342
5696
|
// also silence, skip_response, an empty turn, or a provider error
|
|
4343
5697
|
// (observe-after-engage). Leaving it only on the reply path stranded the
|
|
4344
5698
|
// ack permanently on messages the agent looked at but never answered.
|
|
4345
|
-
|
|
4346
|
-
//
|
|
4347
|
-
|
|
4348
|
-
// it reuses the transient engage reaction's message/emoji/actor, so wait
|
|
4349
|
-
// for that removal to reach the adapter before adding the persistent one.
|
|
4350
|
-
if (live.silentAckTurn?.turnSeq === live.turnSeq) await dropDone
|
|
4351
|
-
else void dropDone
|
|
5699
|
+
// Tuple lanes order removal before protected acknowledgment addition,
|
|
5700
|
+
// without making transport cleanup a barrier for the next response.
|
|
5701
|
+
void dropEngageReactions(live, engageAddPromises)
|
|
4352
5702
|
reactOnSilentAck(live)
|
|
4353
5703
|
// Held channel_react reactions apply only when the agent posted a
|
|
4354
5704
|
// genuine reply this turn — NOT an empty-turn fallback or provider-
|
|
@@ -4371,11 +5721,7 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
4371
5721
|
live.emptyTurnRetries > emptyTurnRetriesBeforePrompt ||
|
|
4372
5722
|
live.toolLeakRetries > toolLeakRetriesBeforePrompt ||
|
|
4373
5723
|
live.willingnessNudges > willingnessNudgesBeforePrompt
|
|
4374
|
-
await maybePostDeferredProviderError(
|
|
4375
|
-
live,
|
|
4376
|
-
sentReplyThisTurn && !live.promisedWorkOutstandingThisLogicalTurn,
|
|
4377
|
-
retryQueuedThisTurn,
|
|
4378
|
-
)
|
|
5724
|
+
await maybePostDeferredProviderError(live, sentReplyThisTurn && !owesPromisedWork(live), retryQueuedThisTurn)
|
|
4379
5725
|
// Provider notices own the first terminal output opportunity. A notice
|
|
4380
5726
|
// that lands on the owed thread clears the obligation through send();
|
|
4381
5727
|
// a suppressed or failed notice leaves it intact for this bounded net.
|
|
@@ -4400,11 +5746,19 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
4400
5746
|
if (!isAwaitingBackgroundChild(live, 'staged-fallback')) {
|
|
4401
5747
|
await resolveStagedFallback(live)
|
|
4402
5748
|
}
|
|
5749
|
+
// A carrier waiting on its own reviewer child has not ended without a
|
|
5750
|
+
// verdict: the child's completion reminder is the turn that posts it.
|
|
5751
|
+
// Failing over now hands the round to a sibling, which then denies the
|
|
5752
|
+
// carrier's verdict and runs a duplicate review. Session-scoped, not
|
|
5753
|
+
// turn-scoped: a later silent inbound on the carrier thread must not
|
|
5754
|
+
// release the round while that reviewer is still running.
|
|
4403
5755
|
const logicalTurnStillOpen =
|
|
4404
5756
|
live.pendingSystemReminders.length > 0 ||
|
|
4405
5757
|
live.stagedFallbackCause !== null ||
|
|
4406
|
-
live.promisedWorkOutstandingThisLogicalTurn
|
|
5758
|
+
live.promisedWorkOutstandingThisLogicalTurn ||
|
|
5759
|
+
isPinnedByRunningChild(live.sessionId, live.keyId, 'github-review-failover')
|
|
4407
5760
|
if (!logicalTurnStillOpen) await failoverGithubReviewRound(live)
|
|
5761
|
+
await transferAbandonedTurnCoverage(live)
|
|
4408
5762
|
live.lastTurnAuthorIds = new Set(live.currentTurnAuthorIds)
|
|
4409
5763
|
if (live.currentTurnAuthorId !== null) {
|
|
4410
5764
|
live.lastTurnAuthorId = live.currentTurnAuthorId
|
|
@@ -4434,18 +5788,78 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
4434
5788
|
// stop()/idle-gc that already tore it down wins the race without a double
|
|
4435
5789
|
// teardown.
|
|
4436
5790
|
if (live.pendingTeardown && !live.destroyed) {
|
|
4437
|
-
live.pendingTeardown = false
|
|
4438
5791
|
// Snapshot the post-reload inbounds that arrived during the held prompt
|
|
4439
5792
|
// BEFORE tearDownLive (its clearQueuedEngageReactions must not drop the
|
|
4440
5793
|
// engage acks we are handing to the successor). Clear the source arrays so
|
|
4441
5794
|
// the dying session owns nothing we are moving forward.
|
|
4442
|
-
const
|
|
4443
|
-
|
|
4444
|
-
|
|
4445
|
-
|
|
4446
|
-
|
|
4447
|
-
|
|
4448
|
-
|
|
5795
|
+
const retained = await backgroundObligations.withTargetLane(live.key, async () => {
|
|
5796
|
+
await resumePreparedTransfersInLane(
|
|
5797
|
+
live.key,
|
|
5798
|
+
[
|
|
5799
|
+
...live.inboundCoverage,
|
|
5800
|
+
...live.pendingSystemReminders.flatMap((reminder) => reminder.inboundRetry?.inputIds ?? []),
|
|
5801
|
+
],
|
|
5802
|
+
[
|
|
5803
|
+
...live.backgroundCoverage,
|
|
5804
|
+
...live.pendingSystemReminders.flatMap((reminder) => reminder.backgroundRetry?.obligationIds ?? []),
|
|
5805
|
+
],
|
|
5806
|
+
)
|
|
5807
|
+
const reminders: PendingSystemReminder[] = []
|
|
5808
|
+
for (const reminder of live.pendingSystemReminders) {
|
|
5809
|
+
const hadRetryCoverage = reminder.inboundRetry !== undefined || reminder.backgroundRetry !== undefined
|
|
5810
|
+
if (reminder.inboundRetry) {
|
|
5811
|
+
reminder.inboundRetry.inputIds = reminder.inboundRetry.inputIds.filter((id) => {
|
|
5812
|
+
const row = inboundJournal.get(id)
|
|
5813
|
+
return row !== undefined && row.phase !== 'closed' && !row.transfer
|
|
5814
|
+
})
|
|
5815
|
+
}
|
|
5816
|
+
if (reminder.backgroundRetry) {
|
|
5817
|
+
const ids: string[] = []
|
|
5818
|
+
for (const id of reminder.backgroundRetry.obligationIds) {
|
|
5819
|
+
const row = await backgroundObligations.get(id)
|
|
5820
|
+
if (row && row.phase !== 'closed' && !row.transfer) ids.push(id)
|
|
5821
|
+
}
|
|
5822
|
+
reminder.backgroundRetry.obligationIds = ids
|
|
5823
|
+
}
|
|
5824
|
+
if (
|
|
5825
|
+
!hadRetryCoverage ||
|
|
5826
|
+
(reminder.inboundRetry?.inputIds.length ?? 0) > 0 ||
|
|
5827
|
+
(reminder.backgroundRetry?.obligationIds.length ?? 0) > 0
|
|
5828
|
+
)
|
|
5829
|
+
reminders.push(reminder)
|
|
5830
|
+
}
|
|
5831
|
+
live.pendingSystemReminders = reminders
|
|
5832
|
+
const retry =
|
|
5833
|
+
live.promptQueue.length === 0 ? live.pendingSystemReminders.find((r) => r.kind === 'retry') : undefined
|
|
5834
|
+
if (retry) {
|
|
5835
|
+
// A queued retry already carries this turn's lineage and retry budgets
|
|
5836
|
+
// forward; the coverage rides on it rather than as a separate snapshot.
|
|
5837
|
+
retry.backgroundWake = live.backgroundWakeLogicalTurn
|
|
5838
|
+
const refs = await resolveBackgroundCoverage(live, live.backgroundCoverage, true)
|
|
5839
|
+
const inputIds = live.inboundCoverage.filter((id) => isHeldByLiveTurn(live, inboundJournal.get(id)))
|
|
5840
|
+
if (refs.length > 0 || inputIds.length > 0) {
|
|
5841
|
+
retry.backgroundRetry = {
|
|
5842
|
+
turnId: live.backgroundTurnId,
|
|
5843
|
+
obligationIds: refs.map((ref) => ref.obligationId),
|
|
5844
|
+
emptyTurnRetries: live.emptyTurnRetries,
|
|
5845
|
+
toolLeakRetries: live.toolLeakRetries,
|
|
5846
|
+
willingnessNudges: live.willingnessNudges,
|
|
5847
|
+
nextPromptMaxTokens: live.nextPromptMaxTokens,
|
|
5848
|
+
}
|
|
5849
|
+
}
|
|
5850
|
+
if (inputIds.length > 0) {
|
|
5851
|
+
retry.inboundRetry = { inputIds, turnId: live.backgroundTurnId }
|
|
5852
|
+
}
|
|
5853
|
+
live.inboundCoverage = []
|
|
5854
|
+
live.backgroundCoverage = []
|
|
5855
|
+
}
|
|
5856
|
+
const kept = await retainLiveWorkInLane(live)
|
|
5857
|
+
// Unpublish inside the lane: an admission queued behind this lane must
|
|
5858
|
+
// reach the successor (which adopts the handoff), not the dying session.
|
|
5859
|
+
if (liveSessions.get(live.keyId) === live) liveSessions.delete(live.keyId)
|
|
5860
|
+
return kept
|
|
5861
|
+
})
|
|
5862
|
+
live.pendingTeardown = false
|
|
4449
5863
|
await tearDownLive(live)
|
|
4450
5864
|
if (retained) await handOffToSuccessor(live.key)
|
|
4451
5865
|
}
|
|
@@ -4496,11 +5910,21 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
4496
5910
|
// recreate attempt so a timeout leaves it available to the next successful
|
|
4497
5911
|
// ensureLive rather than dropping the user's messages at the handoff seam.
|
|
4498
5912
|
const handOffToSuccessor = async (key: ChannelKey): Promise<void> => {
|
|
5913
|
+
const keyId = channelKeyId(key)
|
|
4499
5914
|
try {
|
|
5915
|
+
// A creation begun before this reload cannot install anymore, but ensureLive
|
|
5916
|
+
// would hand back its pending rejection. Let it settle first (bounded) so
|
|
5917
|
+
// the recreate runs under the current generation and adopts the handoff.
|
|
5918
|
+
const inFlight = creating.get(keyId)
|
|
5919
|
+
if (inFlight !== undefined) {
|
|
5920
|
+
await raceWithTimeout(inFlight, ensureLiveTimeoutMs, `[channels] ${keyId} superseded creation`).catch(
|
|
5921
|
+
() => undefined,
|
|
5922
|
+
)
|
|
5923
|
+
}
|
|
4500
5924
|
await ensureLive(key)
|
|
4501
5925
|
} catch (err) {
|
|
4502
5926
|
logger.warn(
|
|
4503
|
-
`[channels] ${
|
|
5927
|
+
`[channels] ${keyId}: successor recreate after reload failed; queued work retained for the next successful session: ${describeError(err)}`,
|
|
4504
5928
|
)
|
|
4505
5929
|
}
|
|
4506
5930
|
}
|
|
@@ -4576,24 +6000,30 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
4576
6000
|
// at the call sites — this helper only runs the handler and delivers the reply.
|
|
4577
6001
|
const runChannelCommand = async (event: InboundMessage, live: LiveSession | null): Promise<CommandResult> => {
|
|
4578
6002
|
const result = await commands.execute(event.text, { live, event, invokerId: event.authorId })
|
|
4579
|
-
if (result.kind === 'handled' && result.reply !== undefined)
|
|
4580
|
-
await send(
|
|
4581
|
-
{
|
|
4582
|
-
adapter: event.adapter,
|
|
4583
|
-
workspace: event.workspace,
|
|
4584
|
-
chat: event.chat,
|
|
4585
|
-
thread: event.thread,
|
|
4586
|
-
text: result.reply,
|
|
4587
|
-
},
|
|
4588
|
-
{ source: 'system', outputKind: 'meta' },
|
|
4589
|
-
)
|
|
4590
|
-
}
|
|
6003
|
+
if (result.kind === 'handled' && result.reply !== undefined) await postCommandReply(event, result.reply)
|
|
4591
6004
|
return result
|
|
4592
6005
|
}
|
|
4593
6006
|
|
|
4594
|
-
const
|
|
6007
|
+
const postCommandReply = async (event: InboundMessage, reply: string): Promise<void> => {
|
|
6008
|
+
await send(
|
|
6009
|
+
{
|
|
6010
|
+
adapter: event.adapter,
|
|
6011
|
+
workspace: event.workspace,
|
|
6012
|
+
chat: event.chat,
|
|
6013
|
+
thread: event.thread,
|
|
6014
|
+
text: reply,
|
|
6015
|
+
},
|
|
6016
|
+
{
|
|
6017
|
+
source: 'system',
|
|
6018
|
+
outputKind: 'meta',
|
|
6019
|
+
expectedAccountIdentity: event.accountIdentity ?? currentAccountIdentityFor(event),
|
|
6020
|
+
},
|
|
6021
|
+
)
|
|
6022
|
+
}
|
|
6023
|
+
|
|
6024
|
+
const route = async (event: InboundMessage, onAdmitted?: () => Promise<void>): Promise<RouteReceipt> => {
|
|
4595
6025
|
const adapterConfig = options.configForAdapter(event.adapter)
|
|
4596
|
-
if (!adapterConfig) return
|
|
6026
|
+
if (!adapterConfig) return { kind: 'denied' }
|
|
4597
6027
|
|
|
4598
6028
|
const key: ChannelKey = {
|
|
4599
6029
|
adapter: event.adapter,
|
|
@@ -4632,9 +6062,13 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
4632
6062
|
thread: event.thread,
|
|
4633
6063
|
text: outcome.reply,
|
|
4634
6064
|
},
|
|
4635
|
-
{
|
|
6065
|
+
{
|
|
6066
|
+
source: 'system',
|
|
6067
|
+
outputKind: 'meta',
|
|
6068
|
+
expectedAccountIdentity: event.accountIdentity ?? currentAccountIdentityFor(event),
|
|
6069
|
+
},
|
|
4636
6070
|
)
|
|
4637
|
-
return
|
|
6071
|
+
return { kind: 'control' }
|
|
4638
6072
|
}
|
|
4639
6073
|
}
|
|
4640
6074
|
|
|
@@ -4654,7 +6088,7 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
4654
6088
|
// messages all fall through to the gate unchanged.
|
|
4655
6089
|
if (parsedCommand !== null && commandInfo?.permission === 'none' && !commandInfo.requiresLiveSession) {
|
|
4656
6090
|
await runChannelCommand(event, null)
|
|
4657
|
-
return
|
|
6091
|
+
return { kind: 'control' }
|
|
4658
6092
|
}
|
|
4659
6093
|
|
|
4660
6094
|
if (isChannelRespondDenied(event)) {
|
|
@@ -4662,7 +6096,7 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
4662
6096
|
logger.info(
|
|
4663
6097
|
`[channels] ${channelKeyId(key)}: denied by permissions (channel.respond) author=${event.authorId} id=${event.externalMessageId}`,
|
|
4664
6098
|
)
|
|
4665
|
-
return
|
|
6099
|
+
return { kind: 'denied' }
|
|
4666
6100
|
}
|
|
4667
6101
|
|
|
4668
6102
|
if (parsedCommand !== null) {
|
|
@@ -4671,14 +6105,14 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
4671
6105
|
const keyId = channelKeyId(key)
|
|
4672
6106
|
if (commandInfo === undefined) {
|
|
4673
6107
|
logger.info(`[channels] ${keyId}: ignoring unknown command /${parsedCommand.name}`)
|
|
4674
|
-
return
|
|
6108
|
+
return { kind: 'control' }
|
|
4675
6109
|
}
|
|
4676
6110
|
const requiredPermission = commandPermissionString(commandInfo.permission)
|
|
4677
6111
|
if (requiredPermission !== null && !permissions.has(inboundAuthorOrigin(event), requiredPermission)) {
|
|
4678
6112
|
logger.info(
|
|
4679
6113
|
`[channels] ${keyId}: denied command /${parsedCommand.name} by permissions (${requiredPermission}) author=${event.authorId}`,
|
|
4680
6114
|
)
|
|
4681
|
-
return
|
|
6115
|
+
return { kind: 'denied' }
|
|
4682
6116
|
}
|
|
4683
6117
|
// Session-less commands (e.g. /help) are informational and run without a
|
|
4684
6118
|
// live session; their handler reply is posted straight back to the channel.
|
|
@@ -4688,15 +6122,56 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
4688
6122
|
if (commandInfo.requiresLiveSession) {
|
|
4689
6123
|
existingLive = liveSessions.get(keyId) ?? null
|
|
4690
6124
|
if (existingLive === null || existingLive.destroyed) {
|
|
6125
|
+
// Work a reload retained for a successor that never installed still
|
|
6126
|
+
// belongs to this conversation; /stop withdraws it by key.
|
|
6127
|
+
if (commandInfo.name === 'stop' && (await stopPendingReloadHandoff(keyId))) {
|
|
6128
|
+
await postCommandReply(event, STOP_COMMAND_REPLY)
|
|
6129
|
+
return { kind: 'control' }
|
|
6130
|
+
}
|
|
4691
6131
|
logger.info(`[channels] ${keyId}: ignoring command /${parsedCommand.name} with no live session`)
|
|
4692
|
-
return
|
|
6132
|
+
return { kind: 'control' }
|
|
4693
6133
|
}
|
|
4694
6134
|
} else if (commandInfo.wantsLiveSession) {
|
|
4695
6135
|
const candidate = liveSessions.get(keyId) ?? null
|
|
4696
6136
|
existingLive = candidate !== null && !candidate.destroyed ? candidate : null
|
|
4697
6137
|
}
|
|
4698
6138
|
const commandResult = await runChannelCommand(event, existingLive)
|
|
4699
|
-
if (commandResult.kind !== 'not-command') return
|
|
6139
|
+
if (commandResult.kind !== 'not-command') return { kind: 'control' }
|
|
6140
|
+
}
|
|
6141
|
+
const generation = liveGeneration
|
|
6142
|
+
await ready.catch(() => undefined)
|
|
6143
|
+
const accountIdentity = event.accountIdentity ?? (await getRecoveryAccountIdentity(event.adapter, event.workspace))
|
|
6144
|
+
// The early duplicate lookup and the in-lane admit share this identity and
|
|
6145
|
+
// its principal, so a duplicate is checked against the same author
|
|
6146
|
+
// provenance admission would record: a different author is a conflict,
|
|
6147
|
+
// never a fresh slot or a silent duplicate.
|
|
6148
|
+
const identity = {
|
|
6149
|
+
accountIdentity: accountIdentity ?? '',
|
|
6150
|
+
target: key,
|
|
6151
|
+
principal: {
|
|
6152
|
+
kind: 'channel' as const,
|
|
6153
|
+
adapter: event.adapter,
|
|
6154
|
+
workspace: event.workspace,
|
|
6155
|
+
chat: event.chat,
|
|
6156
|
+
...(event.room?.parentChat !== undefined ? { parentChat: event.room.parentChat } : {}),
|
|
6157
|
+
lastInboundAuthorId: event.authorId,
|
|
6158
|
+
},
|
|
6159
|
+
messageId: event.externalMessageId || undefined,
|
|
6160
|
+
eventKind: event.eventKind ?? 'message',
|
|
6161
|
+
revision: event.revision ?? '',
|
|
6162
|
+
receiptId: event.receiptId,
|
|
6163
|
+
}
|
|
6164
|
+
const existingAdmission =
|
|
6165
|
+
accountIdentity && inboundJournal.health().available
|
|
6166
|
+
? await backgroundObligations.withTargetLane(key, async () => inboundJournal.lookupAdmission(identity))
|
|
6167
|
+
: undefined
|
|
6168
|
+
if (existingAdmission) {
|
|
6169
|
+
if (existingAdmission.phase !== 'closed') await onAdmitted?.()
|
|
6170
|
+
return {
|
|
6171
|
+
kind: 'duplicate',
|
|
6172
|
+
inputId: existingAdmission.inputId,
|
|
6173
|
+
outcome: existingAdmission.phase === 'closed' ? existingAdmission.outcome.kind : undefined,
|
|
6174
|
+
}
|
|
4700
6175
|
}
|
|
4701
6176
|
|
|
4702
6177
|
// If a boot restart-resume reservation is pending for this key, mark that a
|
|
@@ -4706,7 +6181,7 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
4706
6181
|
const reservation = restartReservations.get(channelKeyId(key))
|
|
4707
6182
|
if (reservation !== undefined) reservation.sawInbound = true
|
|
4708
6183
|
|
|
4709
|
-
|
|
6184
|
+
let live = await ensureLive(key, event.externalMessageId, event.authorId, undefined, event.room, generation)
|
|
4710
6185
|
live.room = event.room
|
|
4711
6186
|
|
|
4712
6187
|
const isNewAuthor = !live.participants.some((p) => p.authorId === event.authorId)
|
|
@@ -4760,46 +6235,68 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
4760
6235
|
// shape mirrors `prompting batch=` so log scraping can pair them.
|
|
4761
6236
|
logger.info(`[channels] ${live.keyId} observed id=${event.externalMessageId}`)
|
|
4762
6237
|
observe(live, event)
|
|
4763
|
-
return
|
|
6238
|
+
return { kind: 'observed' }
|
|
4764
6239
|
}
|
|
4765
6240
|
|
|
4766
|
-
|
|
4767
|
-
|
|
4768
|
-
|
|
4769
|
-
|
|
4770
|
-
|
|
4771
|
-
|
|
4772
|
-
// engage so it self-clears once the first turn advances `turnSeq`; explicit
|
|
4773
|
-
// address (mention/reply/DM) keeps the historical silent-on-empty path.
|
|
4774
|
-
live.coldStartSoloFallbackTurnActive =
|
|
4775
|
-
live.createdFromColdStart &&
|
|
4776
|
-
live.turnSeq === 0 &&
|
|
4777
|
-
effectiveHumans <= 1 &&
|
|
4778
|
-
!event.authorIsBot &&
|
|
4779
|
-
!event.isDm &&
|
|
4780
|
-
!event.isBotMention &&
|
|
4781
|
-
event.replyToBotMessageId === null &&
|
|
4782
|
-
!live.multiHumanGroup
|
|
4783
|
-
|
|
4784
|
-
const engageReaction = autoReactOnEngage(event)
|
|
4785
|
-
|
|
4786
|
-
updateLoopGuard(live, event)
|
|
4787
|
-
|
|
4788
|
-
enqueue(live, event, engageReaction)
|
|
4789
|
-
|
|
4790
|
-
// Start showing "typing..." the moment we know we're going to engage,
|
|
4791
|
-
// so users see the indicator during the debounce window — not just
|
|
4792
|
-
// during LLM generation. drain() will keep it alive across iterations
|
|
4793
|
-
// and the finally-block will stop it when the queue empties.
|
|
4794
|
-
startTypingHeartbeat(live)
|
|
4795
|
-
|
|
4796
|
-
if (live.draining) {
|
|
4797
|
-
// In-flight turn; let coalesce-on-drain pick it up. Same-author abort
|
|
4798
|
-
// is a v0.2 enhancement once we have safe abort semantics through
|
|
4799
|
-
// pi-coding-agent for in-flight tool calls.
|
|
4800
|
-
return
|
|
6241
|
+
await ready
|
|
6242
|
+
if (!accountIdentity) {
|
|
6243
|
+
logger.warn(
|
|
6244
|
+
`[channels] ${live.keyId}: authenticated account is not ready; admitting as unbound so the message is not dropped`,
|
|
6245
|
+
)
|
|
6246
|
+
identity.accountIdentity = `unbound:${event.adapter}:${event.workspace}`
|
|
4801
6247
|
}
|
|
4802
|
-
|
|
6248
|
+
return backgroundObligations.withTargetLane(key, async (): Promise<RouteReceipt> => {
|
|
6249
|
+
if (live.destroyed || liveSessions.get(live.keyId) !== live) {
|
|
6250
|
+
live = await ensureLive(key, event.externalMessageId, event.authorId, undefined, event.room)
|
|
6251
|
+
live.room = event.room
|
|
6252
|
+
}
|
|
6253
|
+
if (isChannelRespondDenied(event)) return { kind: 'denied' }
|
|
6254
|
+
const admission = await inboundJournal.admit({ ...identity, ownerSessionId: live.sessionId })
|
|
6255
|
+
if (admission.kind === 'duplicate') {
|
|
6256
|
+
if (admission.outcome === undefined) await onAdmitted?.()
|
|
6257
|
+
return { kind: 'duplicate', inputId: admission.inputId, outcome: admission.outcome?.kind }
|
|
6258
|
+
}
|
|
6259
|
+
// Adapter-owned durable obligations must exist before this input can run.
|
|
6260
|
+
await onAdmitted?.()
|
|
6261
|
+
publishInbound(event, 'engage', live.sessionId)
|
|
6262
|
+
|
|
6263
|
+
// Arm cold-start bare-empty recovery only for the exact incident shape: the
|
|
6264
|
+
// FIRST prompt (`turnSeq === 0`) of a freshly cold-started session that
|
|
6265
|
+
// engaged via the solo-human answer-everything fallback — a lone human, no
|
|
6266
|
+
// explicit mention/reply/DM, not a multi-human group. Recomputed on every
|
|
6267
|
+
// engage so it self-clears once the first turn advances `turnSeq`; explicit
|
|
6268
|
+
// address (mention/reply/DM) keeps the historical silent-on-empty path.
|
|
6269
|
+
live.coldStartSoloFallbackTurnActive =
|
|
6270
|
+
live.createdFromColdStart &&
|
|
6271
|
+
live.turnSeq === 0 &&
|
|
6272
|
+
effectiveHumans <= 1 &&
|
|
6273
|
+
!event.authorIsBot &&
|
|
6274
|
+
!event.isDm &&
|
|
6275
|
+
!event.isBotMention &&
|
|
6276
|
+
event.replyToBotMessageId === null &&
|
|
6277
|
+
!live.multiHumanGroup
|
|
6278
|
+
|
|
6279
|
+
const engageReaction = autoReactOnEngage(event)
|
|
6280
|
+
|
|
6281
|
+
updateLoopGuard(live, event)
|
|
6282
|
+
|
|
6283
|
+
enqueue(live, event, engageReaction, admission.inputId)
|
|
6284
|
+
|
|
6285
|
+
// Start showing "typing..." the moment we know we're going to engage,
|
|
6286
|
+
// so users see the indicator during the debounce window — not just
|
|
6287
|
+
// during LLM generation. drain() will keep it alive across iterations
|
|
6288
|
+
// and the finally-block will stop it when the queue empties.
|
|
6289
|
+
startTypingHeartbeat(live)
|
|
6290
|
+
|
|
6291
|
+
if (live.draining) {
|
|
6292
|
+
// In-flight turn; let coalesce-on-drain pick it up. Same-author abort
|
|
6293
|
+
// is a v0.2 enhancement once we have safe abort semantics through
|
|
6294
|
+
// pi-coding-agent for in-flight tool calls.
|
|
6295
|
+
return admission
|
|
6296
|
+
}
|
|
6297
|
+
scheduleDebouncedDrain(live)
|
|
6298
|
+
return admission
|
|
6299
|
+
})
|
|
4803
6300
|
}
|
|
4804
6301
|
|
|
4805
6302
|
const inboundAuthorOrigin = (event: InboundMessage): SessionOrigin => ({
|
|
@@ -4894,10 +6391,12 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
4894
6391
|
const enqueue = (
|
|
4895
6392
|
live: LiveSession,
|
|
4896
6393
|
event: InboundMessage,
|
|
4897
|
-
engageReaction:
|
|
6394
|
+
engageReaction: WaitingReactionHandle | null,
|
|
6395
|
+
inputId: string,
|
|
4898
6396
|
): void => {
|
|
4899
6397
|
clearQueuedEngageReactions(live)
|
|
4900
6398
|
live.promptQueue.push({
|
|
6399
|
+
inputId,
|
|
4901
6400
|
text: event.text,
|
|
4902
6401
|
...(event.referenceContext !== undefined ? { referenceContext: event.referenceContext } : {}),
|
|
4903
6402
|
...(event.attachments !== undefined && event.attachments.length > 0 ? { attachments: event.attachments } : {}),
|
|
@@ -5044,6 +6543,69 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
5044
6543
|
return lastError ?? { ok: false, error: 'no reaction removal callback handled request', code: 'unsupported' }
|
|
5045
6544
|
}
|
|
5046
6545
|
|
|
6546
|
+
// Canonical preparation captures the adapter instance that authenticated it.
|
|
6547
|
+
// Reload may replace registration while identity lookup is still in flight.
|
|
6548
|
+
const preparedReactionAdds = new WeakMap<OwnReactionTarget, ReactionCallback>()
|
|
6549
|
+
const prepareOwnReaction = async (req: ReactionRequest): Promise<OwnReactionTarget | null> => {
|
|
6550
|
+
const adds = Array.from(reactionCallbacks.get(req.adapter) ?? [])
|
|
6551
|
+
// The manager has one live adapter per type. Do not guess account ownership
|
|
6552
|
+
// if an external caller installs ambiguous waiting-add capabilities.
|
|
6553
|
+
if (adds.length !== 1) return null
|
|
6554
|
+
const add = adds[0]!
|
|
6555
|
+
for (const cb of Array.from(prepareOwnReactionCallbacks.get(req.adapter) ?? [])) {
|
|
6556
|
+
const target = await cb(req).catch(() => null)
|
|
6557
|
+
if (target) {
|
|
6558
|
+
preparedReactionAdds.set(target, add)
|
|
6559
|
+
return target
|
|
6560
|
+
}
|
|
6561
|
+
}
|
|
6562
|
+
return null
|
|
6563
|
+
}
|
|
6564
|
+
const addPreparedWaitingReaction = async (
|
|
6565
|
+
req: ReactionRequest,
|
|
6566
|
+
prepared: OwnReactionTarget,
|
|
6567
|
+
): Promise<ReactionResult> => {
|
|
6568
|
+
const add = preparedReactionAdds.get(prepared)
|
|
6569
|
+
if (!add) return { ok: false, code: 'unsupported', error: 'authenticated waiting-add capability unavailable' }
|
|
6570
|
+
return add(req).catch((err): ReactionResult => ({ ok: false, code: 'transient', error: describeError(err) }))
|
|
6571
|
+
}
|
|
6572
|
+
const removeOwnReaction = async (req: RemoveOwnReactionRequest): Promise<RemoveOwnReactionResult> => {
|
|
6573
|
+
for (const cb of removeOwnReactionCallbacks.get(req.adapter) ?? []) {
|
|
6574
|
+
const result = await cb(req).catch(
|
|
6575
|
+
(): RemoveOwnReactionResult => ({ ok: false, code: 'transient', error: 'own reaction cleanup failed' }),
|
|
6576
|
+
)
|
|
6577
|
+
if (result.ok || result.code !== 'unsupported') return result
|
|
6578
|
+
}
|
|
6579
|
+
return { ok: false, code: 'unsupported', error: 'own reaction cleanup unsupported' }
|
|
6580
|
+
}
|
|
6581
|
+
const waitingReactions = createWaitingReactionCoordinator({
|
|
6582
|
+
agentDir: options.agentDir,
|
|
6583
|
+
epoch: options.bootEpoch,
|
|
6584
|
+
add: addPreparedWaitingReaction,
|
|
6585
|
+
prepare: prepareOwnReaction,
|
|
6586
|
+
remove: removeOwnReaction,
|
|
6587
|
+
})
|
|
6588
|
+
const recoverWaitingReactions = (adapter?: ChannelKey['adapter']): Promise<void> => waitingReactions.recover(adapter)
|
|
6589
|
+
const registerPrepareOwnReaction = (adapter: ChannelKey['adapter'], cb: PrepareOwnReactionCallback): void => {
|
|
6590
|
+
const callbacks = prepareOwnReactionCallbacks.get(adapter) ?? new Set<PrepareOwnReactionCallback>()
|
|
6591
|
+
callbacks.add(cb)
|
|
6592
|
+
prepareOwnReactionCallbacks.set(adapter, callbacks)
|
|
6593
|
+
}
|
|
6594
|
+
const unregisterPrepareOwnReaction = (adapter: ChannelKey['adapter'], cb: PrepareOwnReactionCallback): void => {
|
|
6595
|
+
prepareOwnReactionCallbacks.get(adapter)?.delete(cb)
|
|
6596
|
+
}
|
|
6597
|
+
const registerRemoveOwnReaction = (adapter: ChannelKey['adapter'], cb: RemoveOwnReactionCallback): void => {
|
|
6598
|
+
const callbacks = removeOwnReactionCallbacks.get(adapter) ?? new Set<RemoveOwnReactionCallback>()
|
|
6599
|
+
callbacks.add(cb)
|
|
6600
|
+
removeOwnReactionCallbacks.set(adapter, callbacks)
|
|
6601
|
+
void recoverWaitingReactions(adapter).catch((err) =>
|
|
6602
|
+
logger.warn(`[channels] waiting reaction recovery failed: ${describeError(err)}`),
|
|
6603
|
+
)
|
|
6604
|
+
}
|
|
6605
|
+
const unregisterRemoveOwnReaction = (adapter: ChannelKey['adapter'], cb: RemoveOwnReactionCallback): void => {
|
|
6606
|
+
removeOwnReactionCallbacks.get(adapter)?.delete(cb)
|
|
6607
|
+
}
|
|
6608
|
+
|
|
5047
6609
|
// Best-effort acknowledgment: drop an :eyes: on the triggering inbound the
|
|
5048
6610
|
// moment we decide to engage — but ONLY when the channel has no visible
|
|
5049
6611
|
// "typing…" indicator. Where typing renders (slack/discord/telegram) the
|
|
@@ -5054,10 +6616,9 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
5054
6616
|
// supporting reactions, a transient API error) can NEVER block engagement,
|
|
5055
6617
|
// enqueueing, or the agent's actual reply. No reactionRef = nothing reactable
|
|
5056
6618
|
// (synthetic inbounds, reaction-less adapters) = silent skip.
|
|
5057
|
-
const autoReactOnEngage = (event: InboundMessage):
|
|
5058
|
-
if (event.reactionRef === undefined) return null
|
|
5059
|
-
|
|
5060
|
-
const addResult = react({
|
|
6619
|
+
const autoReactOnEngage = (event: InboundMessage): WaitingReactionHandle | null => {
|
|
6620
|
+
if (event.reactionRef === undefined || typingCapableAdapters.has(event.adapter)) return null
|
|
6621
|
+
return waitingReactions.acquire({
|
|
5061
6622
|
adapter: event.adapter,
|
|
5062
6623
|
workspace: event.workspace,
|
|
5063
6624
|
chat: event.chat,
|
|
@@ -5065,28 +6626,12 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
5065
6626
|
reactionRef: event.reactionRef,
|
|
5066
6627
|
emoji: ENGAGE_REACTION_EMOJI,
|
|
5067
6628
|
})
|
|
5068
|
-
const addReactionRef = addResult.then((r) => (r.ok ? (r.reactionRef ?? null) : null)).catch(() => null)
|
|
5069
|
-
void addResult
|
|
5070
|
-
.then((result) => {
|
|
5071
|
-
if (!result.ok && result.code !== 'unsupported') {
|
|
5072
|
-
logger.info(`[channels] engage-react failed adapter=${event.adapter} chat=${event.chat}: ${result.error}`)
|
|
5073
|
-
}
|
|
5074
|
-
})
|
|
5075
|
-
.catch((err) => {
|
|
5076
|
-
logger.info(`[channels] engage-react threw adapter=${event.adapter} chat=${event.chat}: ${describeError(err)}`)
|
|
5077
|
-
})
|
|
5078
|
-
return addReactionRef
|
|
5079
6629
|
}
|
|
5080
6630
|
|
|
5081
|
-
//
|
|
5082
|
-
//
|
|
5083
|
-
|
|
5084
|
-
|
|
5085
|
-
const dropEngageReactions = (live: LiveSession, addPromises: Array<Promise<ReactionRef | null>>): Promise<void> => {
|
|
5086
|
-
return Promise.all(addPromises.map((addPromise) => dropOneEngageReaction(live.key, addPromise))).then(
|
|
5087
|
-
() => undefined,
|
|
5088
|
-
)
|
|
5089
|
-
}
|
|
6631
|
+
// Awaitable for scoped effect callers, never a response/notice barrier.
|
|
6632
|
+
// A protected successor may retain presence without a remote toggle.
|
|
6633
|
+
const dropEngageReactions = (live: LiveSession, owners: WaitingReactionHandle[]): Promise<void> =>
|
|
6634
|
+
Promise.all(owners.map((owner) => dropOneEngageReaction(live.key, owner))).then(() => undefined)
|
|
5090
6635
|
|
|
5091
6636
|
// Only the LAST engaging inbound of a coalesced batch should carry the eager
|
|
5092
6637
|
// :eyes:. Called from enqueue() before the new inbound is pushed to roll the
|
|
@@ -5094,43 +6639,20 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
5094
6639
|
// overwrite) is load-bearing: a newer engaging inbound that has no reactionRef
|
|
5095
6640
|
// must still strip the previous :eyes: rather than leave it stranded.
|
|
5096
6641
|
const clearQueuedEngageReactions = (live: LiveSession): void => {
|
|
5097
|
-
const
|
|
5098
|
-
if (
|
|
6642
|
+
const owners = live.promptQueue.flatMap((m) => (m.engageReaction !== undefined ? [m.engageReaction] : []))
|
|
6643
|
+
if (owners.length === 0) return
|
|
5099
6644
|
for (const item of live.promptQueue) delete item.engageReaction
|
|
5100
|
-
void dropEngageReactions(live,
|
|
6645
|
+
void dropEngageReactions(live, owners)
|
|
5101
6646
|
}
|
|
5102
6647
|
|
|
5103
6648
|
const dropPendingReloadHandoffEngageReactions = (pending: PendingReloadHandoff): Promise<void> => {
|
|
5104
|
-
const
|
|
5105
|
-
item.engageReaction !== undefined ? [item.engageReaction] : [],
|
|
5106
|
-
)
|
|
6649
|
+
const owners = pending.inbounds.flatMap((item) => (item.engageReaction !== undefined ? [item.engageReaction] : []))
|
|
5107
6650
|
for (const item of pending.inbounds) delete item.engageReaction
|
|
5108
|
-
return Promise.all(
|
|
5109
|
-
() => undefined,
|
|
5110
|
-
)
|
|
6651
|
+
return Promise.all(owners.map((owner) => dropOneEngageReaction(pending.key, owner))).then(() => undefined)
|
|
5111
6652
|
}
|
|
5112
6653
|
|
|
5113
|
-
const dropOneEngageReaction = (
|
|
5114
|
-
|
|
5115
|
-
.then((reactionRef) => {
|
|
5116
|
-
if (reactionRef === null) return undefined
|
|
5117
|
-
return removeReaction({
|
|
5118
|
-
adapter: key.adapter,
|
|
5119
|
-
workspace: key.workspace,
|
|
5120
|
-
chat: key.chat,
|
|
5121
|
-
thread: key.thread,
|
|
5122
|
-
reactionRef,
|
|
5123
|
-
})
|
|
5124
|
-
})
|
|
5125
|
-
.then((result) => {
|
|
5126
|
-
if (result && !result.ok && result.code !== 'unsupported' && result.code !== 'not-found') {
|
|
5127
|
-
logger.info(`[channels] engage-unreact failed adapter=${key.adapter} chat=${key.chat}: ${result.error}`)
|
|
5128
|
-
}
|
|
5129
|
-
})
|
|
5130
|
-
.catch((err) => {
|
|
5131
|
-
logger.info(`[channels] engage-unreact threw adapter=${key.adapter} chat=${key.chat}: ${describeError(err)}`)
|
|
5132
|
-
})
|
|
5133
|
-
}
|
|
6654
|
+
const dropOneEngageReaction = (_key: ChannelKey, owner: WaitingReactionHandle): Promise<void> =>
|
|
6655
|
+
waitingReactions.release(owner)
|
|
5134
6656
|
|
|
5135
6657
|
const unregisterOutbound = (adapter: ChannelKey['adapter'], cb: OutboundCallback): void => {
|
|
5136
6658
|
outboundCallbacks.get(adapter)?.delete(cb)
|
|
@@ -5465,7 +6987,47 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
5465
6987
|
if (submitter === undefined) {
|
|
5466
6988
|
return { ok: false, error: `adapter "${req.adapter}" does not support review submission`, code: 'unsupported' }
|
|
5467
6989
|
}
|
|
5468
|
-
|
|
6990
|
+
const sourceLive = req.sourceSessionId
|
|
6991
|
+
? [...liveSessions.values()].find(
|
|
6992
|
+
(live) =>
|
|
6993
|
+
!live.destroyed &&
|
|
6994
|
+
live.sessionId === req.sourceSessionId &&
|
|
6995
|
+
live.key.adapter === req.adapter &&
|
|
6996
|
+
live.key.workspace === req.workspace &&
|
|
6997
|
+
live.key.chat === req.chat,
|
|
6998
|
+
)
|
|
6999
|
+
: liveSessions.get(channelKeyId({ ...req, thread: null }))
|
|
7000
|
+
const expectedAccountIdentity =
|
|
7001
|
+
req.expectedAccountIdentity ?? activeTurnAccountIdentity(sourceLive) ?? currentAccountIdentityFor(req)
|
|
7002
|
+
if (
|
|
7003
|
+
sourceLive &&
|
|
7004
|
+
(sourceLive.promptInFlight || sourceLive.inboundCoverage.length > 0 || sourceLive.backgroundCoverage.length > 0)
|
|
7005
|
+
) {
|
|
7006
|
+
try {
|
|
7007
|
+
await ready
|
|
7008
|
+
inboundJournal.assertAvailable()
|
|
7009
|
+
} catch (error) {
|
|
7010
|
+
return { ok: false, error: `continuity persistence unavailable: ${describeError(error)}`, code: 'transient' }
|
|
7011
|
+
}
|
|
7012
|
+
}
|
|
7013
|
+
const currentAccount = currentAccountIdentityFor(req)
|
|
7014
|
+
if (
|
|
7015
|
+
expectedAccountIdentity !== undefined &&
|
|
7016
|
+
currentAccount !== undefined &&
|
|
7017
|
+
expectedAccountIdentity !== currentAccount
|
|
7018
|
+
) {
|
|
7019
|
+
if (sourceLive) {
|
|
7020
|
+
try {
|
|
7021
|
+
await backgroundObligations.withTargetLane(sourceLive.key, () =>
|
|
7022
|
+
transferMismatchedAccountCoverageInLane(sourceLive),
|
|
7023
|
+
)
|
|
7024
|
+
} catch (error) {
|
|
7025
|
+
logger.error(`[channels] review account-change transfer failed: ${describeError(error)}`)
|
|
7026
|
+
}
|
|
7027
|
+
}
|
|
7028
|
+
return { ok: false, error: 'original channel account identity changed', code: 'permission-denied' }
|
|
7029
|
+
}
|
|
7030
|
+
return await submitter({ ...req, expectedAccountIdentity }).catch(
|
|
5469
7031
|
(err): SubmitReviewResult => ({ ok: false, error: describeError(err), code: 'transient' }),
|
|
5470
7032
|
)
|
|
5471
7033
|
}
|
|
@@ -5557,6 +7119,16 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
5557
7119
|
const flood = checkOutboundFlood(authoredText)
|
|
5558
7120
|
if (!flood.ok) return { ok: false, error: OUTBOUND_FLOOD_ERROR, code: 'outbound-flood' }
|
|
5559
7121
|
}
|
|
7122
|
+
// Recovery is a transport-only operation. Do not even look up the live
|
|
7123
|
+
// session: doing so would let a delayed notice alter a newer logical turn.
|
|
7124
|
+
if (opts?.accounting === 'recovery') {
|
|
7125
|
+
for (const callback of Array.from(callbacks)) {
|
|
7126
|
+
const result = await callback({ ...msg, sendOptions: opts })
|
|
7127
|
+
if (result.ok) return result
|
|
7128
|
+
return result
|
|
7129
|
+
}
|
|
7130
|
+
return { ok: false, error: 'no recovery transport available', code: 'no-adapter' }
|
|
7131
|
+
}
|
|
5560
7132
|
|
|
5561
7133
|
const accountingTarget = opts?.accountingTarget ?? {
|
|
5562
7134
|
adapter: msg.adapter,
|
|
@@ -5571,6 +7143,23 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
5571
7143
|
accountingTarget.thread === (msg.thread ?? null)
|
|
5572
7144
|
const keyId = channelKeyId(accountingTarget)
|
|
5573
7145
|
const live = liveSessions.get(keyId)
|
|
7146
|
+
const originalTurnAccount = activeTurnAccountIdentity(live)
|
|
7147
|
+
const sameAdapterAccountScope =
|
|
7148
|
+
accountingTarget.adapter === msg.adapter && accountingTarget.workspace === msg.workspace
|
|
7149
|
+
const expectedAccountIdentity =
|
|
7150
|
+
opts?.expectedAccountIdentity ?? (sameAdapterAccountScope ? originalTurnAccount : currentAccountIdentityFor(msg))
|
|
7151
|
+
const dependentLiveSend =
|
|
7152
|
+
live &&
|
|
7153
|
+
(source === 'tool' || opts?.outputKind === 'substantive' || opts?.accounting === 'live-turn') &&
|
|
7154
|
+
(live.promptInFlight || live.inboundCoverage.length > 0 || live.backgroundCoverage.length > 0)
|
|
7155
|
+
if (dependentLiveSend) {
|
|
7156
|
+
try {
|
|
7157
|
+
await ready
|
|
7158
|
+
inboundJournal.assertAvailable()
|
|
7159
|
+
} catch (error) {
|
|
7160
|
+
return { ok: false, error: `continuity persistence unavailable: ${describeError(error)}` }
|
|
7161
|
+
}
|
|
7162
|
+
}
|
|
5574
7163
|
// Credit sticky where the reply was PUBLISHED, not where it was accounted.
|
|
5575
7164
|
// The two diverge when a caller answers from one sub-surface but publishes
|
|
5576
7165
|
// to another in the same room — `postDuplicateRequestChangesComment` sends
|
|
@@ -5722,7 +7311,35 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
5722
7311
|
let reactionRef: ReactionRef | undefined
|
|
5723
7312
|
try {
|
|
5724
7313
|
for (const cb of snapshot) {
|
|
5725
|
-
|
|
7314
|
+
if (dependentLiveSend && !inboundJournal.health().available) {
|
|
7315
|
+
lastError = 'continuity persistence unavailable'
|
|
7316
|
+
break
|
|
7317
|
+
}
|
|
7318
|
+
if (dependentLiveSend && live && originalTurnAccount !== undefined) {
|
|
7319
|
+
const currentSourceAccount = currentAccountIdentityFor(live.key)
|
|
7320
|
+
if (currentSourceAccount !== undefined && currentSourceAccount !== originalTurnAccount) {
|
|
7321
|
+
await backgroundObligations.withTargetLane(live.key, () => transferMismatchedAccountCoverageInLane(live))
|
|
7322
|
+
lastError = 'original channel account identity changed'
|
|
7323
|
+
break
|
|
7324
|
+
}
|
|
7325
|
+
}
|
|
7326
|
+
if (expectedAccountIdentity !== undefined) {
|
|
7327
|
+
const currentAccount = currentAccountIdentityFor(msg)
|
|
7328
|
+
if (currentAccount !== undefined && currentAccount !== expectedAccountIdentity) {
|
|
7329
|
+
if (live && sameAdapterAccountScope)
|
|
7330
|
+
await backgroundObligations.withTargetLane(live.key, () => transferMismatchedAccountCoverageInLane(live))
|
|
7331
|
+
lastError = 'original channel account identity changed'
|
|
7332
|
+
break
|
|
7333
|
+
}
|
|
7334
|
+
}
|
|
7335
|
+
const result = await cb(
|
|
7336
|
+
expectedAccountIdentity === undefined
|
|
7337
|
+
? msg
|
|
7338
|
+
: {
|
|
7339
|
+
...msg,
|
|
7340
|
+
sendOptions: { ...opts, accounting: 'live-turn', expectedAccountIdentity },
|
|
7341
|
+
},
|
|
7342
|
+
)
|
|
5726
7343
|
if (result.ok) {
|
|
5727
7344
|
delivered = true
|
|
5728
7345
|
messageId = result.messageId
|
|
@@ -5768,15 +7385,21 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
5768
7385
|
live.successfulChannelSends++
|
|
5769
7386
|
// Promise fulfillment follows OUTPUT KIND, never source:
|
|
5770
7387
|
// (a) ordinary substantive tool output clears;
|
|
5771
|
-
// (b) multilingual continuation-willingness/status output preserves
|
|
7388
|
+
// (b) multilingual continuation-willingness/status output preserves, and
|
|
7389
|
+
// itself promises more (`statusOutputOutstandingThisLogicalTurn`);
|
|
5772
7390
|
// (c) recovery prose explicitly marked substantive clears even though
|
|
5773
7391
|
// it uses the system bypass;
|
|
5774
7392
|
// (d) provider-error, empty-turn-fallback, and control notices are
|
|
5775
7393
|
// explicitly meta and preserve, so a warning can never self-clear.
|
|
5776
7394
|
// The channel_reply afterToolCall hook above remains authoritative for its
|
|
5777
7395
|
// machine-readable more_work_this_turn flag.
|
|
5778
|
-
if (outputKind === 'substantive')
|
|
5779
|
-
|
|
7396
|
+
if (outputKind === 'substantive') {
|
|
7397
|
+
live.promisedWorkOutstandingThisLogicalTurn = false
|
|
7398
|
+
live.statusOutputOutstandingThisLogicalTurn = false
|
|
7399
|
+
} else if (outputKind === 'status') {
|
|
7400
|
+
live.statusOutputOutstandingThisLogicalTurn = true
|
|
7401
|
+
}
|
|
7402
|
+
live.lastSendLeafId = branchLeafEntry(live.session)?.id ?? null
|
|
5780
7403
|
live.policyDeniedToolSendsThisTurn.delete(sendKey)
|
|
5781
7404
|
// Don't stop the heartbeat here: the agent may still be mid-turn and
|
|
5782
7405
|
// about to send another reply. drain()'s finally block owns turn-end
|
|
@@ -5848,13 +7471,18 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
5848
7471
|
const record = live.lastTerminalReplyCompletion
|
|
5849
7472
|
live.lastTerminalReplyCompletion = null
|
|
5850
7473
|
if (record === null || record.turnSeq !== live.turnSeq) return
|
|
5851
|
-
if (live
|
|
5852
|
-
return
|
|
5853
|
-
}
|
|
7474
|
+
if (githubCloseoutOwnsTurn(live)) return
|
|
5854
7475
|
if (live.willingnessNudges >= MAX_WILLINGNESS_NUDGES) return
|
|
5855
7476
|
if (live.promptQueue.length > 0) return
|
|
5856
7477
|
if (record.text === undefined) return
|
|
5857
7478
|
if (!detectContinuationWillingness(record.text)) return
|
|
7479
|
+
// A status promise can refer to a child from an earlier logical turn. Gate
|
|
7480
|
+
// only willingness recovery session-wide; ordinary empty-turn recovery stays
|
|
7481
|
+
// turn-scoped so unrelated later requests still get their own answer.
|
|
7482
|
+
if (isPinnedByRunningChild(live.sessionId, live.keyId, 'willingness_nudge')) {
|
|
7483
|
+
logger.info(`[channels] ${live.keyId} willingness_nudge_suppressed cause=awaiting_background_child`)
|
|
7484
|
+
return
|
|
7485
|
+
}
|
|
5858
7486
|
live.willingnessNudges++
|
|
5859
7487
|
logger.info(
|
|
5860
7488
|
`[channels] ${live.keyId} willingness_nudge attempt=${live.willingnessNudges}/${MAX_WILLINGNESS_NUDGES}`,
|
|
@@ -6131,7 +7759,7 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
6131
7759
|
}
|
|
6132
7760
|
return
|
|
6133
7761
|
}
|
|
6134
|
-
if (live.session
|
|
7762
|
+
if (branchLeafEntry(live.session)?.id === live.lastSendLeafId) return
|
|
6135
7763
|
}
|
|
6136
7764
|
|
|
6137
7765
|
if (unfinishedToolUseWithoutReply) {
|
|
@@ -6331,10 +7959,29 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
6331
7959
|
live.successfulChannelSends === successfulSendsBeforePrompt &&
|
|
6332
7960
|
live.promptQueue.length === 0
|
|
6333
7961
|
) {
|
|
7962
|
+
// Recheck at fallback time too: a reminder can already be queued when
|
|
7963
|
+
// the running child becomes visible to the router.
|
|
7964
|
+
if (isPinnedByRunningChild(live.sessionId, live.keyId, 'no_reply_after_willingness_nudge')) {
|
|
7965
|
+
logger.info(`[channels] ${live.keyId} empty_turn_suppressed cause=awaiting_background_child`)
|
|
7966
|
+
armSilentTurnAck(live, 'awaiting_background_child')
|
|
7967
|
+
return
|
|
7968
|
+
}
|
|
6334
7969
|
await postEmptyTurnFallback(live, 'no_reply_after_willingness_nudge')
|
|
6335
7970
|
return
|
|
6336
7971
|
}
|
|
6337
7972
|
const leakedReasoning = !isNoReplySignal(assistantText)
|
|
7973
|
+
// Explicit silence answers only an inbound-only turn that promised nothing.
|
|
7974
|
+
// After a status/continue-reply promise the user is still waiting, so the
|
|
7975
|
+
// debt stays owed for the turn-end decision instead of closing silently.
|
|
7976
|
+
if (
|
|
7977
|
+
assistantText.trim() !== '' &&
|
|
7978
|
+
live.backgroundCoverage.length === 0 &&
|
|
7979
|
+
live.inboundCoverage.length > 0 &&
|
|
7980
|
+
!live.willingnessReminderIteration &&
|
|
7981
|
+
!owesPromisedWork(live)
|
|
7982
|
+
) {
|
|
7983
|
+
await settleBackgroundCoverage(live, 'intentionally-suppressed', 'explicit-no-reply')
|
|
7984
|
+
}
|
|
6338
7985
|
logger.info(`[channels] ${live.keyId} no_reply${leakedReasoning ? ' (with_leaked_reasoning)' : ''}`)
|
|
6339
7986
|
void dropContinuationReactions(live)
|
|
6340
7987
|
return
|
|
@@ -6441,6 +8088,8 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
6441
8088
|
logger.warn(
|
|
6442
8089
|
`[channels] ${live.keyId}: suppressed recovery (github review guard) reason=${JSON.stringify(recoveryBlock)} text_len=${assistantText.length}`,
|
|
6443
8090
|
)
|
|
8091
|
+
// The documented continuation: the next inbound re-prompts this turn's work.
|
|
8092
|
+
live.nextInboundContinuationTurnId = live.backgroundTurnId
|
|
6444
8093
|
return
|
|
6445
8094
|
}
|
|
6446
8095
|
|
|
@@ -6461,6 +8110,8 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
6461
8110
|
logger.warn(
|
|
6462
8111
|
`[channels] ${live.keyId}: recovering assistant_text_without_channel_tool source=${source} text_len=${assistantText.length}`,
|
|
6463
8112
|
)
|
|
8113
|
+
const capturedCoverage = await captureBackgroundResultCoverage(live.sessionId)
|
|
8114
|
+
const capturedInbound = await captureInboundResultCoverage(live.sessionId)
|
|
6464
8115
|
const result = await send(
|
|
6465
8116
|
{
|
|
6466
8117
|
adapter: live.key.adapter,
|
|
@@ -6473,6 +8124,8 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
6473
8124
|
)
|
|
6474
8125
|
if (!result.ok) {
|
|
6475
8126
|
logger.warn(`[channels] ${live.keyId}: recovery send failed: ${result.error}`)
|
|
8127
|
+
} else if (!detectContinuationWillingness(assistantText)) {
|
|
8128
|
+
await recordLandedBackgroundResponse(live, 'recovered-terminal-response', capturedCoverage, capturedInbound)
|
|
6476
8129
|
}
|
|
6477
8130
|
}
|
|
6478
8131
|
|
|
@@ -6555,11 +8208,10 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
6555
8208
|
|
|
6556
8209
|
const tearDownLive = async (live: LiveSession): Promise<void> => {
|
|
6557
8210
|
live.destroyed = true
|
|
6558
|
-
//
|
|
6559
|
-
//
|
|
6560
|
-
// successful send). Remove both the queued and current-turn acks here.
|
|
8211
|
+
// Release queued and in-flight temporary owners even if teardown occurs
|
|
8212
|
+
// before the turn-end cleanup. Protected acknowledgments remain intact.
|
|
6561
8213
|
clearQueuedEngageReactions(live)
|
|
6562
|
-
dropEngageReactions(live, live.currentTurnEngageReactions)
|
|
8214
|
+
void dropEngageReactions(live, live.currentTurnEngageReactions)
|
|
6563
8215
|
live.currentTurnEngageReactions = []
|
|
6564
8216
|
void dropContinuationReactions(live)
|
|
6565
8217
|
if (live.debounceTimer) clearTimeout(live.debounceTimer)
|
|
@@ -6637,10 +8289,42 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
6637
8289
|
if (live.pendingSystemReminders.length > 0) continue
|
|
6638
8290
|
if (t - live.lastInboundAt <= SESSION_IDLE_MS) continue
|
|
6639
8291
|
if (isPinnedByRunningChild(live.sessionId, live.keyId, 'idle_gc evicting')) continue
|
|
8292
|
+
// Coverage deferred to another session's running child (a reload successor
|
|
8293
|
+
// carried it) waits on that child exactly like the session's own.
|
|
8294
|
+
if (
|
|
8295
|
+
(live.inboundCoverage.length > 0 || live.backgroundCoverage.length > 0) &&
|
|
8296
|
+
awaitsDeferredChild(live, 'idle_gc evicting')
|
|
8297
|
+
)
|
|
8298
|
+
continue
|
|
6640
8299
|
victims.push(live)
|
|
6641
8300
|
}
|
|
6642
8301
|
for (const live of victims) {
|
|
6643
|
-
|
|
8302
|
+
// Owed coverage never outlives its cache unowned: what the session still
|
|
8303
|
+
// holds is transferred as a live-turn-ended notice, in the target lane,
|
|
8304
|
+
// before eviction. A failed transfer keeps the session and its ownership for
|
|
8305
|
+
// the next tick; the journal rows stay owed meanwhile.
|
|
8306
|
+
const evicted = await backgroundObligations
|
|
8307
|
+
.withTargetLane(live.key, async () => {
|
|
8308
|
+
if (
|
|
8309
|
+
live.destroyed ||
|
|
8310
|
+
liveSessions.get(live.keyId) !== live ||
|
|
8311
|
+
live.draining ||
|
|
8312
|
+
live.promptQueue.length > 0 ||
|
|
8313
|
+
live.pendingSystemReminders.length > 0 ||
|
|
8314
|
+
now() - live.lastInboundAt <= SESSION_IDLE_MS
|
|
8315
|
+
)
|
|
8316
|
+
return false
|
|
8317
|
+
await transferHeldCoverageInLane(live)
|
|
8318
|
+
liveSessions.delete(live.keyId)
|
|
8319
|
+
return true
|
|
8320
|
+
})
|
|
8321
|
+
.catch((error: unknown) => {
|
|
8322
|
+
logger.error(
|
|
8323
|
+
`[channels] ${live.keyId} idle_gc kept session: owed coverage transfer failed: ${describeError(error)}`,
|
|
8324
|
+
)
|
|
8325
|
+
return false
|
|
8326
|
+
})
|
|
8327
|
+
if (!evicted) continue
|
|
6644
8328
|
logger.info(`[channels] ${live.keyId} idle_gc evicting after ${t - live.lastInboundAt}ms idle`)
|
|
6645
8329
|
await tearDownLive(live)
|
|
6646
8330
|
}
|
|
@@ -6655,20 +8339,29 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
6655
8339
|
|
|
6656
8340
|
const stop = async (): Promise<void> => {
|
|
6657
8341
|
closing = true
|
|
8342
|
+
if (!options.inboundJournal) inboundJournal.cancelInitialization()
|
|
8343
|
+
unsubscribeJournalFailure()
|
|
6658
8344
|
if (gcTimer) clearInterval(gcTimer)
|
|
6659
8345
|
gcTimer = null
|
|
6660
8346
|
liveGeneration++
|
|
8347
|
+
// Process shutdown is a real restart: owed rows, including those a reload
|
|
8348
|
+
// handoff holds, stay in this epoch's journal for boot recovery to transfer.
|
|
8349
|
+
// Nothing is prepared here, so a restart notice never comes from a live epoch.
|
|
6661
8350
|
const pendingHandoffs = Array.from(pendingReloadHandoffs.values())
|
|
6662
8351
|
pendingReloadHandoffs.clear()
|
|
6663
|
-
for (const pending of pendingHandoffs)
|
|
6664
|
-
await reportDiscardedReloadHandoff(pending, 'router stopped before replay could complete')
|
|
6665
|
-
}
|
|
8352
|
+
for (const pending of pendingHandoffs) void dropPendingReloadHandoffEngageReactions(pending)
|
|
6666
8353
|
const all = Array.from(liveSessions.values())
|
|
6667
8354
|
liveSessions.clear()
|
|
6668
8355
|
for (const live of all) {
|
|
6669
8356
|
await tearDownLive(live)
|
|
6670
8357
|
}
|
|
6671
|
-
|
|
8358
|
+
try {
|
|
8359
|
+
await persistChain
|
|
8360
|
+
} finally {
|
|
8361
|
+
if (!options.inboundJournal) {
|
|
8362
|
+
await inboundJournal.close()
|
|
8363
|
+
}
|
|
8364
|
+
}
|
|
6672
8365
|
}
|
|
6673
8366
|
|
|
6674
8367
|
// Drops every in-memory session but KEEPS the on-disk records, so the next
|
|
@@ -6691,24 +8384,40 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
6691
8384
|
// stays in liveSessions (so a concurrent inbound coalesces into the running
|
|
6692
8385
|
// turn instead of spawning a duplicate) and is torn down by the drain
|
|
6693
8386
|
// finally once its turn drains. Idle sessions tear down immediately below.
|
|
6694
|
-
const deferred: LiveSession[] = []
|
|
6695
8387
|
const immediate: LiveSession[] = []
|
|
6696
8388
|
for (const live of all) {
|
|
6697
8389
|
if (live.draining && !live.destroyed) {
|
|
6698
8390
|
live.pendingTeardown = true
|
|
6699
|
-
|
|
6700
|
-
|
|
8391
|
+
} else {
|
|
8392
|
+
// A failed immediate reload still fences the old configuration's prompt.
|
|
8393
|
+
live.pendingTeardown = true
|
|
8394
|
+
immediate.push(live)
|
|
8395
|
+
}
|
|
6701
8396
|
}
|
|
6702
|
-
liveSessions.clear()
|
|
6703
|
-
for (const live of deferred) liveSessions.set(live.keyId, live)
|
|
6704
8397
|
// Seal only around the flush — unlike stop() the router keeps serving after a
|
|
6705
8398
|
// roles reload, so re-enable persist() once pending writes have drained.
|
|
6706
8399
|
closing = true
|
|
6707
|
-
|
|
6708
|
-
|
|
8400
|
+
const successors = new Map<string, ChannelKey>()
|
|
8401
|
+
try {
|
|
8402
|
+
for (const live of immediate) {
|
|
8403
|
+
const retained = await backgroundObligations.withTargetLane(live.key, async () => {
|
|
8404
|
+
const kept = await retainLiveWorkInLane(live)
|
|
8405
|
+
// A failed transfer must keep the original live cache reachable.
|
|
8406
|
+
if (liveSessions.get(live.keyId) === live) liveSessions.delete(live.keyId)
|
|
8407
|
+
return kept
|
|
8408
|
+
})
|
|
8409
|
+
if (retained) successors.set(live.keyId, live.key)
|
|
8410
|
+
await tearDownLive(live)
|
|
8411
|
+
}
|
|
8412
|
+
// A handoff from an earlier reload whose successor never installed is recreated too.
|
|
8413
|
+
for (const [keyId, pending] of pendingReloadHandoffs) {
|
|
8414
|
+
if (!liveSessions.has(keyId)) successors.set(keyId, pending.key)
|
|
8415
|
+
}
|
|
8416
|
+
await persistChain
|
|
8417
|
+
} finally {
|
|
8418
|
+
closing = false
|
|
6709
8419
|
}
|
|
6710
|
-
await
|
|
6711
|
-
closing = false
|
|
8420
|
+
await Promise.all(Array.from(successors.values(), handOffToSuccessor))
|
|
6712
8421
|
}
|
|
6713
8422
|
|
|
6714
8423
|
// Graceful-restart shutdown: mark every live channel session's todo scope so
|
|
@@ -6717,6 +8426,7 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
6717
8426
|
// continues its incomplete todos instead of waiting for a human. Best-effort
|
|
6718
8427
|
// and bounded by the caller's shutdown deadline.
|
|
6719
8428
|
const markRestartAbortForAllLive = async (): Promise<void> => {
|
|
8429
|
+
restartShutdownPending = true
|
|
6720
8430
|
for (const live of Array.from(liveSessions.values())) {
|
|
6721
8431
|
const origin = buildLiveOrigin(live)
|
|
6722
8432
|
await markRestartAbortPendingForOrigin(options.agentDir, origin).catch((err) =>
|
|
@@ -6733,73 +8443,6 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
6733
8443
|
}
|
|
6734
8444
|
}
|
|
6735
8445
|
|
|
6736
|
-
// Graceful-restart hint: if a live channel session has background subagents
|
|
6737
|
-
// still running, record their names in the restart handoff so the boot resume
|
|
6738
|
-
// can tell that thread its promised result was lost. Only one handoff exists
|
|
6739
|
-
// on disk, so the FIRST session with running background children wins; the
|
|
6740
|
-
// rare "two threads mid-research at once" case notifies one. The fresh/stale
|
|
6741
|
-
// and augment-vs-fresh-write decision is documented inline below.
|
|
6742
|
-
//
|
|
6743
|
-
// Returns whether the handoff now carries interrupted names, propagated from
|
|
6744
|
-
// the writer's real result so a swallowed filesystem failure reports false.
|
|
6745
|
-
const writeInterruptedSubagentHandoff = async (): Promise<boolean> => {
|
|
6746
|
-
const listNames = options.listRunningBackgroundSubagentNames
|
|
6747
|
-
if (listNames === undefined) return false
|
|
6748
|
-
|
|
6749
|
-
// Take the same lock `/restart` holds so our peek→select→write is atomic
|
|
6750
|
-
// relative to its post-ACK write: either it commits first and we augment the
|
|
6751
|
-
// exact handoff, or we commit first and it overwrites with its own origin —
|
|
6752
|
-
// never an interleaved read-modify-write that drops one producer's data.
|
|
6753
|
-
const release = await acquireRestartHandoffLock(options.agentDir)
|
|
6754
|
-
try {
|
|
6755
|
-
// A FRESH existing handoff is authoritative: it is the accepted in-session
|
|
6756
|
-
// restart's, so we augment it (never clobber its origin/author with a
|
|
6757
|
-
// different live session's) — or, if its own session has no running
|
|
6758
|
-
// children, leave it untouched and record nothing. A STALE handoff is not
|
|
6759
|
-
// authoritative: peekRestartHandoff applies no TTL, so an unclaimed TUI
|
|
6760
|
-
// handoff left on disk by kind-aware consume can surface here; honoring it
|
|
6761
|
-
// would suppress THIS restart's note or preserve its old restartedAt so the
|
|
6762
|
-
// next boot discards the note as stale — so we ignore it and write fresh
|
|
6763
|
-
// from the current live sessions. When we augment, stamp restartedAt=now()
|
|
6764
|
-
// so the note survives the boot TTL.
|
|
6765
|
-
const existing = await peekRestartHandoff(options.agentDir)
|
|
6766
|
-
const existingIsFresh = existing !== null && now() - Date.parse(existing.restartedAt) <= RESTART_HANDOFF_TTL_MS
|
|
6767
|
-
if (existing !== null && existingIsFresh) {
|
|
6768
|
-
const names = listNames(existing.originatingSessionId)
|
|
6769
|
-
if (names.length === 0) return false
|
|
6770
|
-
return await writeRestartHandoff(options.agentDir, {
|
|
6771
|
-
...existing,
|
|
6772
|
-
restartedAt: new Date(now()).toISOString(),
|
|
6773
|
-
interruptedSubagents: names,
|
|
6774
|
-
})
|
|
6775
|
-
}
|
|
6776
|
-
|
|
6777
|
-
for (const live of Array.from(liveSessions.values())) {
|
|
6778
|
-
const names = listNames(live.sessionId)
|
|
6779
|
-
if (names.length === 0) continue
|
|
6780
|
-
const sessionFile = live.getTranscriptPath?.()
|
|
6781
|
-
if (sessionFile === undefined) continue
|
|
6782
|
-
// Carry the session's author (same precedence as buildRestartCommandContext)
|
|
6783
|
-
// so boot re-seeds lastTurnAuthorId. Without it an author-scoped role demotes
|
|
6784
|
-
// on resume and the reminder-only turn can lose channel.send — i.e. fail to
|
|
6785
|
-
// deliver the very lost-work notice this handoff exists for.
|
|
6786
|
-
const triggeringAuthorId = live.currentTurnAuthorId ?? live.lastTurnAuthorId ?? undefined
|
|
6787
|
-
return await writeRestartHandoff(options.agentDir, {
|
|
6788
|
-
schemaVersion: 2,
|
|
6789
|
-
restartedAt: new Date(now()).toISOString(),
|
|
6790
|
-
originatingSessionId: live.sessionId,
|
|
6791
|
-
origin: { kind: 'channel', key: live.key },
|
|
6792
|
-
originatingSessionFile: basename(sessionFile),
|
|
6793
|
-
interruptedSubagents: names,
|
|
6794
|
-
...(triggeringAuthorId !== undefined ? { triggeringAuthorId } : {}),
|
|
6795
|
-
})
|
|
6796
|
-
}
|
|
6797
|
-
return false
|
|
6798
|
-
} finally {
|
|
6799
|
-
release()
|
|
6800
|
-
}
|
|
6801
|
-
}
|
|
6802
|
-
|
|
6803
8446
|
// Boot-time resume for a restart that originated from a channel session, in
|
|
6804
8447
|
// two phases to close the race with adapters that begin receiving inbounds.
|
|
6805
8448
|
//
|
|
@@ -6836,6 +8479,7 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
6836
8479
|
logger.warn(`[channels] ${keyId}: restart-resume skipped — adapter not configured`)
|
|
6837
8480
|
return null
|
|
6838
8481
|
}
|
|
8482
|
+
if (restartReservations.has(keyId)) throw new Error(`restart reservation already exists for ${keyId}`)
|
|
6839
8483
|
|
|
6840
8484
|
let resolveGate!: (live: LiveSession) => void
|
|
6841
8485
|
let rejectGate!: (err: unknown) => void
|
|
@@ -6852,13 +8496,23 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
6852
8496
|
const reservation: RestartReservation = {
|
|
6853
8497
|
keyId,
|
|
6854
8498
|
sawInbound: false,
|
|
8499
|
+
release: () => {
|
|
8500
|
+
if (creating.get(keyId) === gate) creating.delete(keyId)
|
|
8501
|
+
if (restartReservations.get(keyId) === reservation) restartReservations.delete(keyId)
|
|
8502
|
+
rejectGate(new StaleLiveSessionError(keyId))
|
|
8503
|
+
},
|
|
6855
8504
|
resume: async () => {
|
|
6856
8505
|
// Drop our own seed BEFORE calling ensureLive, or ensureLive would
|
|
6857
8506
|
// await the gate we are about to resolve and deadlock.
|
|
6858
8507
|
if (creating.get(keyId) === gate) creating.delete(keyId)
|
|
6859
8508
|
restartReservations.delete(keyId)
|
|
6860
8509
|
|
|
6861
|
-
|
|
8510
|
+
try {
|
|
8511
|
+
await ensureLoaded()
|
|
8512
|
+
} catch (error) {
|
|
8513
|
+
reservation.release()
|
|
8514
|
+
throw error
|
|
8515
|
+
}
|
|
6862
8516
|
const record = mappings ? findRecord(mappings, key) : undefined
|
|
6863
8517
|
if (record?.sessionId !== handoff.originatingSessionId) {
|
|
6864
8518
|
logger.warn(
|
|
@@ -6893,27 +8547,8 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
6893
8547
|
// A real inbound coalesced onto the reservation during boot: it is the
|
|
6894
8548
|
// wake. Adding the synthetic "I'm back" turn on top would duplicate
|
|
6895
8549
|
// work / stack a spurious turn, so skip it and let the inbound drain.
|
|
6896
|
-
// The interrupted-subagent directive is NOT part of that generic wake,
|
|
6897
|
-
// though: it is the only signal that a promised result was lost, and the
|
|
6898
|
-
// boot already consumed the handoff, so if we drop it here no later turn
|
|
6899
|
-
// re-delivers it. Queue it AND drain ourselves: `sawInbound` is set
|
|
6900
|
-
// before engagement is decided, so an observe-only inbound returns
|
|
6901
|
-
// without draining and would strand the reminder. drain() is guarded
|
|
6902
|
-
// (`draining || destroyed` no-ops) so if the inbound WILL engage this is
|
|
6903
|
-
// a harmless second call, and its loop consumes pendingSystemReminders
|
|
6904
|
-
// either way. Still skip the generic synthetic wake.
|
|
6905
8550
|
if (reservation.sawInbound) {
|
|
6906
|
-
|
|
6907
|
-
live.pendingSystemReminders.push(
|
|
6908
|
-
wakeupReminder(buildInterruptedSubagentNotice(handoff.interruptedSubagents)),
|
|
6909
|
-
)
|
|
6910
|
-
logger.info(
|
|
6911
|
-
`[channels] ${keyId}: restart-resume coalesced with a real inbound; delivering interrupted-subagent notice`,
|
|
6912
|
-
)
|
|
6913
|
-
void drain(live)
|
|
6914
|
-
} else {
|
|
6915
|
-
logger.info(`[channels] ${keyId}: restart-resume coalesced with a real inbound; skipping synthetic wake`)
|
|
6916
|
-
}
|
|
8551
|
+
logger.info(`[channels] ${keyId}: restart-resume coalesced with a real inbound; skipping synthetic wake`)
|
|
6917
8552
|
return
|
|
6918
8553
|
}
|
|
6919
8554
|
|
|
@@ -6927,7 +8562,7 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
6927
8562
|
logger.error(`[channels] ${keyId}: restart-resume clear abort suppression failed: ${describeError(err)}`),
|
|
6928
8563
|
)
|
|
6929
8564
|
|
|
6930
|
-
live.pendingSystemReminders.push(wakeupReminder(
|
|
8565
|
+
live.pendingSystemReminders.push(wakeupReminder(RESTART_RESUME_WAKE_REMINDER))
|
|
6931
8566
|
logger.info(`[channels] ${keyId}: restart-resume waking session ${live.sessionId}`)
|
|
6932
8567
|
void drain(live)
|
|
6933
8568
|
},
|
|
@@ -6984,6 +8619,14 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
6984
8619
|
if (commandInfo.requiresLiveSession) {
|
|
6985
8620
|
const resolved = resolveLiveSessionForCommand(liveSessions, key)
|
|
6986
8621
|
if (resolved.kind === 'none') {
|
|
8622
|
+
if (lowered !== 'stop') return { kind: 'no-live-session' }
|
|
8623
|
+
// A reload handoff whose successor never installed is this
|
|
8624
|
+
// conversation's owed work; /stop withdraws it without a session.
|
|
8625
|
+
const pending = resolvePendingReloadHandoffForCommand(key)
|
|
8626
|
+
if (pending.kind === 'ambiguous') return { kind: 'ambiguous', matchCount: pending.count }
|
|
8627
|
+
if (pending.kind === 'found' && (await stopPendingReloadHandoff(pending.keyId))) {
|
|
8628
|
+
return { kind: 'handled', name: commandInfo.name, reply: STOP_COMMAND_REPLY }
|
|
8629
|
+
}
|
|
6987
8630
|
return { kind: 'no-live-session' }
|
|
6988
8631
|
}
|
|
6989
8632
|
if (resolved.kind === 'ambiguous') {
|
|
@@ -6999,7 +8642,12 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
6999
8642
|
// empty and incorrectly reported no-live-session. Always pass that exact
|
|
7000
8643
|
// session to stopCurrentChannelTurn so the user's escape hatch remains.
|
|
7001
8644
|
const exactDiscordSession = key.adapter === 'discord-bot' && resolved.session.keyId === channelKeyId(key)
|
|
7002
|
-
if (
|
|
8645
|
+
if (
|
|
8646
|
+
lowered === 'stop' &&
|
|
8647
|
+
!exactDiscordSession &&
|
|
8648
|
+
!hasStoppableWork(resolved.session) &&
|
|
8649
|
+
!isStoppableReloadHandoff(pendingReloadHandoffs.get(resolved.session.keyId))
|
|
8650
|
+
) {
|
|
7003
8651
|
return { kind: 'no-live-session' }
|
|
7004
8652
|
}
|
|
7005
8653
|
live = resolved.session
|
|
@@ -7021,6 +8669,124 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
7021
8669
|
return { kind: 'unknown-command', name: lowered }
|
|
7022
8670
|
}
|
|
7023
8671
|
|
|
8672
|
+
const acceptBackgroundResponse: ChannelRouter['acceptBackgroundResponse'] = async (args) =>
|
|
8673
|
+
backgroundObligations.withTargetLane(args.key, async () => {
|
|
8674
|
+
await ready
|
|
8675
|
+
inboundJournal.assertAvailable()
|
|
8676
|
+
const row = await backgroundObligations.accept(args)
|
|
8677
|
+
return { obligationId: row.obligationId, generation: row.generation }
|
|
8678
|
+
})
|
|
8679
|
+
|
|
8680
|
+
const suppressUnstartedBackgroundResponse: ChannelRouter['suppressUnstartedBackgroundResponse'] = async (
|
|
8681
|
+
ref,
|
|
8682
|
+
reason,
|
|
8683
|
+
) => {
|
|
8684
|
+
const row = await backgroundObligations.get(ref.obligationId)
|
|
8685
|
+
if (!row) throw new Error('Missing background launch obligation')
|
|
8686
|
+
await backgroundObligations.withTargetLane(row.target, async () => {
|
|
8687
|
+
await ready
|
|
8688
|
+
inboundJournal.assertAvailable()
|
|
8689
|
+
const current = await backgroundObligations.get(ref.obligationId)
|
|
8690
|
+
if (!current || current.phase === 'closed') return
|
|
8691
|
+
if (current.phase !== 'accepted') throw new Error('Background launch already progressed')
|
|
8692
|
+
await backgroundObligations.settle([{ obligationId: current.obligationId, generation: current.generation }], {
|
|
8693
|
+
kind: 'intentionally-suppressed',
|
|
8694
|
+
decisionId: randomUUID(),
|
|
8695
|
+
reason,
|
|
8696
|
+
})
|
|
8697
|
+
})
|
|
8698
|
+
}
|
|
8699
|
+
|
|
8700
|
+
const attachBackgroundResultCoverage: ChannelRouter['attachBackgroundResultCoverage'] = async (args) => {
|
|
8701
|
+
const live = [...liveSessions.values()].find(
|
|
8702
|
+
(candidate) => !candidate.destroyed && candidate.sessionId === args.parentSessionId,
|
|
8703
|
+
)
|
|
8704
|
+
if (!live) return
|
|
8705
|
+
await backgroundObligations.withTargetLane(live.key, async () => {
|
|
8706
|
+
await ready
|
|
8707
|
+
inboundJournal.assertAvailable()
|
|
8708
|
+
if (live.destroyed) return
|
|
8709
|
+
const source =
|
|
8710
|
+
(await backgroundObligations.lookup(args.parentSessionId, args.taskId)) ??
|
|
8711
|
+
(await backgroundObligations.lookupTask(args.taskId, live.key))
|
|
8712
|
+
if (!source || channelKeyId(source.target) !== live.keyId) return
|
|
8713
|
+
const row = await backgroundObligations.resultReady(source.obligationId)
|
|
8714
|
+
if (!row || row.phase === 'closed' || row.transfer) {
|
|
8715
|
+
retireBackgroundCache(live, new Set([source.obligationId]))
|
|
8716
|
+
return
|
|
8717
|
+
}
|
|
8718
|
+
const refs = [{ obligationId: row.obligationId, generation: row.generation }]
|
|
8719
|
+
const owner = { turnId: live.backgroundTurnId, ownerSessionId: live.sessionId, target: live.key }
|
|
8720
|
+
if (
|
|
8721
|
+
row.claim &&
|
|
8722
|
+
(row.claim.turnId !== live.backgroundTurnId ||
|
|
8723
|
+
row.claim.ownerSessionId !== live.sessionId ||
|
|
8724
|
+
row.claim.epoch !== backgroundObligations.epoch)
|
|
8725
|
+
) {
|
|
8726
|
+
if (live.inboundCoverage.length > 0) {
|
|
8727
|
+
await inboundJournal.move([], { ...owner, fromTurnId: row.claim.turnId }, refs)
|
|
8728
|
+
} else {
|
|
8729
|
+
await backgroundObligations.move(refs, { ...owner, fromTurnId: row.claim.turnId })
|
|
8730
|
+
}
|
|
8731
|
+
} else if (!row.claim) {
|
|
8732
|
+
if (live.inboundCoverage.length > 0) {
|
|
8733
|
+
await inboundJournal.claim([], owner, refs)
|
|
8734
|
+
} else {
|
|
8735
|
+
await backgroundObligations.claim(refs, owner)
|
|
8736
|
+
}
|
|
8737
|
+
}
|
|
8738
|
+
// Fetch consumed this result into the active prompt. Its queued wake must
|
|
8739
|
+
// not advertise an old owner or start a second prompt for the same result.
|
|
8740
|
+
retireBackgroundCache(live, new Set([row.obligationId]))
|
|
8741
|
+
live.backgroundCoverage.push(row.obligationId)
|
|
8742
|
+
})
|
|
8743
|
+
}
|
|
8744
|
+
|
|
8745
|
+
const captureTurnAccountIdentity: NonNullable<ChannelRouter['captureTurnAccountIdentity']> = async (
|
|
8746
|
+
parentSessionId,
|
|
8747
|
+
adapter,
|
|
8748
|
+
) => {
|
|
8749
|
+
const live = [...liveSessions.values()].find(
|
|
8750
|
+
(candidate) => !candidate.destroyed && candidate.sessionId === parentSessionId,
|
|
8751
|
+
)
|
|
8752
|
+
if (!live || live.key.adapter !== adapter) return undefined
|
|
8753
|
+
return backgroundObligations.withTargetLane(live.key, async () => {
|
|
8754
|
+
await ready
|
|
8755
|
+
inboundJournal.assertAvailable()
|
|
8756
|
+
return activeTurnAccountIdentity(live) ?? currentAccountIdentityFor(live.key)
|
|
8757
|
+
})
|
|
8758
|
+
}
|
|
8759
|
+
|
|
8760
|
+
const captureInboundResultCoverage: NonNullable<ChannelRouter['captureInboundResultCoverage']> = async (
|
|
8761
|
+
parentSessionId,
|
|
8762
|
+
) => {
|
|
8763
|
+
const live = [...liveSessions.values()].find(
|
|
8764
|
+
(candidate) => !candidate.destroyed && candidate.sessionId === parentSessionId,
|
|
8765
|
+
)
|
|
8766
|
+
if (!live) return []
|
|
8767
|
+
return backgroundObligations.withTargetLane(live.key, async () => {
|
|
8768
|
+
await ready
|
|
8769
|
+
if (live.destroyed) return []
|
|
8770
|
+
const refs = inboundJournal.resolve(live.inboundCoverage)
|
|
8771
|
+
return refs.filter((ref) => isClaimedByCurrentTurn(live, inboundJournal.get(ref.inputId)))
|
|
8772
|
+
})
|
|
8773
|
+
}
|
|
8774
|
+
|
|
8775
|
+
const captureBackgroundResultCoverage: NonNullable<ChannelRouter['captureBackgroundResultCoverage']> = async (
|
|
8776
|
+
parentSessionId,
|
|
8777
|
+
) => {
|
|
8778
|
+
const live = [...liveSessions.values()].find(
|
|
8779
|
+
(candidate) => !candidate.destroyed && candidate.sessionId === parentSessionId,
|
|
8780
|
+
)
|
|
8781
|
+
if (!live) return []
|
|
8782
|
+
return backgroundObligations.withTargetLane(live.key, async () => {
|
|
8783
|
+
if (live.destroyed) return []
|
|
8784
|
+
const refs = await resolveBackgroundCoverage(live, live.backgroundCoverage, true)
|
|
8785
|
+
live.backgroundCoverage = refs.map((ref) => ref.obligationId)
|
|
8786
|
+
return refs
|
|
8787
|
+
})
|
|
8788
|
+
}
|
|
8789
|
+
|
|
7024
8790
|
const deliverCompletionReminder = (
|
|
7025
8791
|
live: LiveSession,
|
|
7026
8792
|
args: {
|
|
@@ -7031,6 +8797,7 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
7031
8797
|
durationMs: number
|
|
7032
8798
|
error?: string
|
|
7033
8799
|
hasRecoverableOutput?: boolean
|
|
8800
|
+
backgroundObligationId?: string
|
|
7034
8801
|
},
|
|
7035
8802
|
): { kind: 'delivered'; keyId: string } => {
|
|
7036
8803
|
const adapter = live.keyId.split(':', 1)[0] ?? ''
|
|
@@ -7044,7 +8811,16 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
7044
8811
|
channel: true,
|
|
7045
8812
|
adapter,
|
|
7046
8813
|
})
|
|
7047
|
-
|
|
8814
|
+
if (
|
|
8815
|
+
args.backgroundObligationId &&
|
|
8816
|
+
(live.backgroundCoverage.includes(args.backgroundObligationId) ||
|
|
8817
|
+
live.pendingSystemReminders.some((r) => r.backgroundObligationId === args.backgroundObligationId))
|
|
8818
|
+
)
|
|
8819
|
+
return { kind: 'delivered', keyId: live.keyId }
|
|
8820
|
+
live.pendingSystemReminders.push({
|
|
8821
|
+
...wakeupReminder(text),
|
|
8822
|
+
...(args.backgroundObligationId ? { backgroundObligationId: args.backgroundObligationId } : {}),
|
|
8823
|
+
})
|
|
7048
8824
|
// The reminder tells the agent to fetch this result now; clear the
|
|
7049
8825
|
// subagent_output window so an earlier premature-polling streak can't
|
|
7050
8826
|
// hard-block that legitimate fetch.
|
|
@@ -7061,7 +8837,7 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
7061
8837
|
return { kind: 'delivered', keyId: live.keyId }
|
|
7062
8838
|
}
|
|
7063
8839
|
|
|
7064
|
-
const injectSubagentCompletionReminder = (args: {
|
|
8840
|
+
const injectSubagentCompletionReminder = async (args: {
|
|
7065
8841
|
parentSessionId: string
|
|
7066
8842
|
subagent: string
|
|
7067
8843
|
taskId: string
|
|
@@ -7070,11 +8846,35 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
7070
8846
|
error?: string
|
|
7071
8847
|
hasRecoverableOutput?: boolean
|
|
7072
8848
|
channelKey?: { adapter: string; workspace: string; chat: string; thread: string | null }
|
|
7073
|
-
}): { kind: 'delivered'; keyId: string } | { kind: 'no-live-session' } => {
|
|
8849
|
+
}): Promise<{ kind: 'delivered'; keyId: string } | { kind: 'no-live-session' }> => {
|
|
8850
|
+
// Completion admission is independent of the presence or health of a parent.
|
|
8851
|
+
const existing = await backgroundObligations.lookup(args.parentSessionId, args.taskId)
|
|
8852
|
+
const row = existing
|
|
8853
|
+
? await backgroundObligations.withTargetLane(existing.target, async () => {
|
|
8854
|
+
await ready
|
|
8855
|
+
inboundJournal.assertAvailable()
|
|
8856
|
+
return backgroundObligations.resultReady(existing.obligationId)
|
|
8857
|
+
})
|
|
8858
|
+
: undefined
|
|
8859
|
+
if (existing && !row) return { kind: 'no-live-session' }
|
|
8860
|
+
const queueCompletion = (
|
|
8861
|
+
live: LiveSession,
|
|
8862
|
+
): Promise<{ kind: 'delivered'; keyId: string } | { kind: 'no-live-session' }> =>
|
|
8863
|
+
backgroundObligations.withTargetLane(live.key, async () => {
|
|
8864
|
+
await ready
|
|
8865
|
+
inboundJournal.assertAvailable()
|
|
8866
|
+
const current = row ? await backgroundObligations.get(row.obligationId) : undefined
|
|
8867
|
+
if (row && (!current || current.phase === 'closed' || current.transfer)) return { kind: 'no-live-session' }
|
|
8868
|
+
return deliverCompletionReminder(live, {
|
|
8869
|
+
...args,
|
|
8870
|
+
...(current ? { backgroundObligationId: current.obligationId } : {}),
|
|
8871
|
+
})
|
|
8872
|
+
})
|
|
7074
8873
|
for (const live of liveSessions.values()) {
|
|
7075
8874
|
if (live.destroyed) continue
|
|
7076
8875
|
if (live.sessionId !== args.parentSessionId) continue
|
|
7077
|
-
|
|
8876
|
+
if (row && channelKeyId(row.target) !== live.keyId) continue
|
|
8877
|
+
return queueCompletion(live)
|
|
7078
8878
|
}
|
|
7079
8879
|
// The exact parent session is gone. If the subagent was spawned from a
|
|
7080
8880
|
// channel session, the conversation may have rolled over
|
|
@@ -7089,9 +8889,83 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
7089
8889
|
logger.info(
|
|
7090
8890
|
`[channels] ${targetKeyId}: subagent-completion reminder rerouted to live successor (parent ${args.parentSessionId} gone) task=${args.taskId}`,
|
|
7091
8891
|
)
|
|
7092
|
-
|
|
8892
|
+
if (row && channelKeyId(row.target) !== successor.keyId) return { kind: 'no-live-session' }
|
|
8893
|
+
return queueCompletion(successor)
|
|
7093
8894
|
}
|
|
7094
8895
|
}
|
|
8896
|
+
// A reload retained this parent's deferred coverage and its successor has not
|
|
8897
|
+
// installed yet (or failed to). Queue the wake on that explicit handoff so the
|
|
8898
|
+
// successor adopts both together; no session is created here, and a result
|
|
8899
|
+
// without such a handoff stays owed for durable recovery as before.
|
|
8900
|
+
if (row) {
|
|
8901
|
+
const keyId = channelKeyId(row.target)
|
|
8902
|
+
const queued = await backgroundObligations.withTargetLane(row.target, async () => {
|
|
8903
|
+
await ready
|
|
8904
|
+
inboundJournal.assertAvailable()
|
|
8905
|
+
const current = await backgroundObligations.get(row.obligationId)
|
|
8906
|
+
const pending = pendingReloadHandoffs.get(keyId)
|
|
8907
|
+
if (
|
|
8908
|
+
!current ||
|
|
8909
|
+
current.phase === 'closed' ||
|
|
8910
|
+
current.transfer ||
|
|
8911
|
+
liveSessions.has(keyId) ||
|
|
8912
|
+
!pending?.coverage ||
|
|
8913
|
+
![pending.coverage.coverageDeferredParentSessionId, pending.coverage.ownerSessionId].includes(
|
|
8914
|
+
args.parentSessionId,
|
|
8915
|
+
)
|
|
8916
|
+
)
|
|
8917
|
+
return false
|
|
8918
|
+
if (pending.reminders.some((reminder) => reminder.backgroundObligationId === current.obligationId)) return true
|
|
8919
|
+
const reminders: PendingSystemReminder[] = [
|
|
8920
|
+
...pending.reminders,
|
|
8921
|
+
{
|
|
8922
|
+
...wakeupReminder(
|
|
8923
|
+
renderSubagentCompletionReminder({
|
|
8924
|
+
subagent: args.subagent,
|
|
8925
|
+
taskId: args.taskId,
|
|
8926
|
+
ok: args.ok,
|
|
8927
|
+
durationMs: args.durationMs,
|
|
8928
|
+
...(args.error !== undefined ? { error: args.error } : {}),
|
|
8929
|
+
...(args.hasRecoverableOutput === true ? { hasRecoverableOutput: true } : {}),
|
|
8930
|
+
channel: true,
|
|
8931
|
+
adapter: row.target.adapter,
|
|
8932
|
+
}),
|
|
8933
|
+
),
|
|
8934
|
+
backgroundObligationId: current.obligationId,
|
|
8935
|
+
},
|
|
8936
|
+
]
|
|
8937
|
+
// The same bounds as retention, checked on the candidate before anything
|
|
8938
|
+
// changes: a refused wake leaves the handoff exactly as it was and the
|
|
8939
|
+
// result owed for durable recovery.
|
|
8940
|
+
const estimatedBytes = estimateReloadHandoffBytes(
|
|
8941
|
+
pending.inbounds,
|
|
8942
|
+
pending.observed,
|
|
8943
|
+
reminders,
|
|
8944
|
+
pending.coverage,
|
|
8945
|
+
)
|
|
8946
|
+
const exceeded =
|
|
8947
|
+
pendingReloadHandoffItemCount({ ...pending, reminders }) > handoffRetryItemLimit
|
|
8948
|
+
? `per-key item limit ${handoffRetryItemLimit}`
|
|
8949
|
+
: estimatedBytes > handoffRetryByteLimit
|
|
8950
|
+
? `per-key byte limit ${handoffRetryByteLimit}`
|
|
8951
|
+
: pendingReloadHandoffTotalBytes() - pending.estimatedBytes + estimatedBytes >
|
|
8952
|
+
MAX_PENDING_RELOAD_HANDOFF_BYTES_TOTAL
|
|
8953
|
+
? `global byte limit ${MAX_PENDING_RELOAD_HANDOFF_BYTES_TOTAL}`
|
|
8954
|
+
: undefined
|
|
8955
|
+
if (exceeded !== undefined) {
|
|
8956
|
+
logger.error(
|
|
8957
|
+
`[channels] ${keyId}: subagent-completion reminder not held for reload successor (${exceeded} would be exceeded); result stays owed task=${args.taskId}`,
|
|
8958
|
+
)
|
|
8959
|
+
return false
|
|
8960
|
+
}
|
|
8961
|
+
pending.reminders = reminders
|
|
8962
|
+
pending.estimatedBytes = estimatedBytes
|
|
8963
|
+
forgetSharedLoopGuardTool(args.parentSessionId, SUBAGENT_OUTPUT_TOOL_NAME)
|
|
8964
|
+
logger.info(`[channels] ${keyId}: subagent-completion reminder held for reload successor task=${args.taskId}`)
|
|
8965
|
+
return true
|
|
8966
|
+
})
|
|
8967
|
+
if (queued) return { kind: 'delivered', keyId }
|
|
8968
|
+
}
|
|
7095
8969
|
return { kind: 'no-live-session' }
|
|
7096
8970
|
}
|
|
7097
8971
|
|
|
@@ -7124,6 +8998,7 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
7124
8998
|
prNumber: number
|
|
7125
8999
|
verdict: ReviewRoundOutcome
|
|
7126
9000
|
sessionId: string
|
|
9001
|
+
commitSha?: string
|
|
7127
9002
|
}): Promise<{ kind: 'completed' | 'no-round' }> => {
|
|
7128
9003
|
const chat = `pr:${args.prNumber}`
|
|
7129
9004
|
const publisher = Array.from(liveSessions.values()).find(
|
|
@@ -7153,6 +9028,14 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
7153
9028
|
return { kind: 'no-round' }
|
|
7154
9029
|
}
|
|
7155
9030
|
|
|
9031
|
+
// Checked before re-registration, which would evict the newer round that
|
|
9032
|
+
// replaced this one between the verdict landing and this observation.
|
|
9033
|
+
if (isGithubReviewRoundSuperseded(round, now)) {
|
|
9034
|
+
logger.warn(
|
|
9035
|
+
`[channels] github review round completion rejected pr=${args.workspace}#${args.prNumber} verdict=${args.verdict}: round superseded`,
|
|
9036
|
+
)
|
|
9037
|
+
return { kind: 'no-round' }
|
|
9038
|
+
}
|
|
7156
9039
|
const activeRound = registerGithubReviewRound(round, now(), now)
|
|
7157
9040
|
if (activeRound === null) {
|
|
7158
9041
|
logger.warn(
|
|
@@ -7169,7 +9052,7 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
7169
9052
|
)
|
|
7170
9053
|
return { kind: 'no-round' }
|
|
7171
9054
|
}
|
|
7172
|
-
if (!(await validateGithubReviewRound(activeRound, undefined, now))) {
|
|
9055
|
+
if (!(await validateGithubReviewRound(activeRound, undefined, now, args.commitSha ?? null))) {
|
|
7173
9056
|
resetGithubReviewRoundCompletion(activeRound, now)
|
|
7174
9057
|
persistMatchingGithubReviewRound(activeRound)
|
|
7175
9058
|
logger.warn(
|
|
@@ -7289,18 +9172,28 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
7289
9172
|
// event.sessionId), and confirmed to be the right github PR live session so a
|
|
7290
9173
|
// stray/mismatched signal can't suppress an unrelated turn's fallback. See
|
|
7291
9174
|
// `githubReviewOutputTurn`.
|
|
7292
|
-
const noteGithubReviewOutput = (args: {
|
|
9175
|
+
const noteGithubReviewOutput = async (args: {
|
|
7293
9176
|
sessionId: string
|
|
7294
9177
|
workspace: string
|
|
7295
9178
|
prNumber: number
|
|
7296
9179
|
state: ReviewOutputState
|
|
7297
|
-
|
|
9180
|
+
inboundCoverage?: InboundRef[]
|
|
9181
|
+
backgroundCoverage?: BackgroundObligationRef[]
|
|
9182
|
+
}): Promise<{ kind: 'stamped' | 'no-live-session' }> => {
|
|
7298
9183
|
const chat = `pr:${args.prNumber}`
|
|
7299
9184
|
for (const live of liveSessions.values()) {
|
|
7300
9185
|
if (live.destroyed) continue
|
|
7301
9186
|
if (live.sessionId !== args.sessionId) continue
|
|
7302
9187
|
if (live.key.adapter !== 'github') continue
|
|
7303
9188
|
if (live.key.workspace !== args.workspace || live.key.chat !== chat) continue
|
|
9189
|
+
// Only pre-publication coverage can be credited. An output-only recognizer
|
|
9190
|
+
// still stamps ordinary review accounting, but cannot borrow current debt.
|
|
9191
|
+
await recordLandedBackgroundResponse(
|
|
9192
|
+
live,
|
|
9193
|
+
'formal-review',
|
|
9194
|
+
args.backgroundCoverage ?? [],
|
|
9195
|
+
args.inboundCoverage ?? [],
|
|
9196
|
+
)
|
|
7304
9197
|
live.githubReviewOutputTurn = live.turnSeq
|
|
7305
9198
|
logger.info(`[channels] ${live.keyId}: github_review_output state=${args.state} turn=${live.turnSeq}`)
|
|
7306
9199
|
return { kind: 'stamped' }
|
|
@@ -7308,16 +9201,16 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
7308
9201
|
return { kind: 'no-live-session' }
|
|
7309
9202
|
}
|
|
7310
9203
|
|
|
7311
|
-
const markTurnSkipped = (args: {
|
|
9204
|
+
const markTurnSkipped = async (args: {
|
|
7312
9205
|
parentSessionId: string
|
|
7313
9206
|
reason: string
|
|
7314
|
-
}):
|
|
7315
|
-
| { kind: 'recorded'; keyId: string }
|
|
7316
|
-
|
|
7317
|
-
| { kind: 'no-live-session' } => {
|
|
9207
|
+
}): Promise<
|
|
9208
|
+
{ kind: 'recorded'; keyId: string } | { kind: 'recorded-after-send'; keyId: string } | { kind: 'no-live-session' }
|
|
9209
|
+
> => {
|
|
7318
9210
|
for (const live of liveSessions.values()) {
|
|
7319
9211
|
if (live.destroyed) continue
|
|
7320
9212
|
if (live.sessionId !== args.parentSessionId) continue
|
|
9213
|
+
await settleBackgroundCoverage(live, 'intentionally-suppressed', 'explicit-skip')
|
|
7321
9214
|
if (live.successfulChannelSends > live.successfulSendsAtTurnStart) {
|
|
7322
9215
|
// Reply-first skip ("acked, now going quiet"): accept as a terminal
|
|
7323
9216
|
// no-op, never stamp `skippedTurn`. The delivered reply stands and must
|
|
@@ -7383,6 +9276,27 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
7383
9276
|
})
|
|
7384
9277
|
}
|
|
7385
9278
|
|
|
9279
|
+
// Answers a failed turn with a reaction on its triggering message instead of
|
|
9280
|
+
// a chat line; in busy rooms the old text notices outnumbered real replies.
|
|
9281
|
+
// False when there is nothing reactable or the adapter refused, so the caller
|
|
9282
|
+
// falls back to its text notice and the user is never left with silence.
|
|
9283
|
+
const reactTurnFailed = async (live: LiveSession): Promise<boolean> => {
|
|
9284
|
+
const reactionRef = live.currentTurnReactionRef
|
|
9285
|
+
if (reactionRef === null) return false
|
|
9286
|
+
const result = await react({
|
|
9287
|
+
adapter: live.key.adapter,
|
|
9288
|
+
workspace: live.key.workspace,
|
|
9289
|
+
chat: live.key.chat,
|
|
9290
|
+
thread: live.key.thread,
|
|
9291
|
+
reactionRef,
|
|
9292
|
+
emoji: TURN_FAILED_REACTION_EMOJI,
|
|
9293
|
+
})
|
|
9294
|
+
if (!result.ok && result.code !== 'unsupported') {
|
|
9295
|
+
logger.info(`[channels] ${live.keyId}: turn-failed reaction failed, posting notice: ${result.error}`)
|
|
9296
|
+
}
|
|
9297
|
+
return result.ok
|
|
9298
|
+
}
|
|
9299
|
+
|
|
7386
9300
|
const armSilentTurnAck = (live: LiveSession, reason: SilentAckReason): void => {
|
|
7387
9301
|
// A GitHub review is user-facing output, so it always earns the 👀; deferred
|
|
7388
9302
|
// background work earns one only when the triggering message addressed the
|
|
@@ -7399,135 +9313,114 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
7399
9313
|
live.silentAckTurn = null
|
|
7400
9314
|
const reactionRef = live.currentTurnReactionRef
|
|
7401
9315
|
if (reactionRef === null) return
|
|
7402
|
-
|
|
7403
|
-
|
|
7404
|
-
|
|
7405
|
-
|
|
7406
|
-
|
|
7407
|
-
|
|
7408
|
-
|
|
7409
|
-
|
|
7410
|
-
|
|
7411
|
-
|
|
7412
|
-
|
|
7413
|
-
|
|
7414
|
-
`[channels] silent-ack-react failed reason=${reason} adapter=${live.key.adapter} chat=${live.key.chat}: ${result.error}`,
|
|
7415
|
-
)
|
|
7416
|
-
}
|
|
7417
|
-
})
|
|
7418
|
-
.catch((err) => {
|
|
7419
|
-
logger.info(
|
|
7420
|
-
`[channels] silent-ack-react threw reason=${reason} adapter=${live.key.adapter} chat=${live.key.chat}: ${describeError(err)}`,
|
|
7421
|
-
)
|
|
7422
|
-
})
|
|
7423
|
-
// Register the add promise (not its resolved ref) SYNCHRONOUSLY, so a later
|
|
7424
|
-
// replied-turn cleanup can never race ahead of a still-in-flight add.
|
|
7425
|
-
live.activeSilentAckReactions.push(addResult.then((r) => (r.ok ? (r.reactionRef ?? null) : null)).catch(() => null))
|
|
7426
|
-
}
|
|
7427
|
-
|
|
7428
|
-
// Retire every persistent silent-ack :eyes: outstanding in this live session
|
|
7429
|
-
// once the agent posts a genuine reply. Snapshot-and-clear the promise array
|
|
7430
|
-
// BEFORE awaiting, so a
|
|
7431
|
-
// concurrent later silent turn appending a fresh add is never swept by this
|
|
7432
|
-
// in-flight cleanup. Each entry is AWAITED to its resolved ref before removal,
|
|
7433
|
-
// so an add still in flight when the reply lands is still retired. Failures are
|
|
7434
|
-
// logged, never retried: stale emoji cleanup must never block routing.
|
|
7435
|
-
const dropSilentAckReactions = (live: LiveSession): Promise<void> => {
|
|
7436
|
-
const addPromises = live.activeSilentAckReactions
|
|
7437
|
-
if (addPromises.length === 0) return Promise.resolve()
|
|
7438
|
-
live.activeSilentAckReactions = []
|
|
7439
|
-
return Promise.all(
|
|
7440
|
-
addPromises.map((addPromise) =>
|
|
7441
|
-
addPromise
|
|
7442
|
-
.then((reactionRef) => {
|
|
7443
|
-
if (reactionRef === null) return undefined
|
|
7444
|
-
return removeReaction({
|
|
7445
|
-
adapter: live.key.adapter,
|
|
7446
|
-
workspace: live.key.workspace,
|
|
7447
|
-
chat: live.key.chat,
|
|
7448
|
-
thread: live.key.thread,
|
|
7449
|
-
reactionRef,
|
|
7450
|
-
})
|
|
7451
|
-
})
|
|
7452
|
-
.then((result) => {
|
|
7453
|
-
if (result && !result.ok && result.code !== 'unsupported' && result.code !== 'not-found') {
|
|
7454
|
-
logger.info(
|
|
7455
|
-
`[channels] silent-ack-unreact failed adapter=${live.key.adapter} chat=${live.key.chat}: ${result.error}`,
|
|
7456
|
-
)
|
|
7457
|
-
}
|
|
7458
|
-
})
|
|
7459
|
-
.catch((err) => {
|
|
7460
|
-
logger.info(
|
|
7461
|
-
`[channels] silent-ack-unreact threw adapter=${live.key.adapter} chat=${live.key.chat}: ${describeError(err)}`,
|
|
7462
|
-
)
|
|
7463
|
-
}),
|
|
7464
|
-
),
|
|
7465
|
-
).then(() => undefined)
|
|
9316
|
+
waitingReactions.acquire(
|
|
9317
|
+
{
|
|
9318
|
+
adapter: live.key.adapter,
|
|
9319
|
+
workspace: live.key.workspace,
|
|
9320
|
+
chat: live.key.chat,
|
|
9321
|
+
thread: live.key.thread,
|
|
9322
|
+
reactionRef,
|
|
9323
|
+
emoji: ENGAGE_REACTION_EMOJI,
|
|
9324
|
+
},
|
|
9325
|
+
true,
|
|
9326
|
+
{ conversation: live.key, kind: reason },
|
|
9327
|
+
)
|
|
7466
9328
|
}
|
|
7467
9329
|
|
|
9330
|
+
// The genuine-reply decision retires conversation-scoped acknowledgments,
|
|
9331
|
+
// including owners persisted by an earlier runtime. Cleanup cannot use this.
|
|
9332
|
+
const dropSilentAckReactions = (live: LiveSession): Promise<void> => waitingReactions.retireAcknowledgments(live.key)
|
|
9333
|
+
|
|
7468
9334
|
const reactOnContinuationWillingness = (live: LiveSession, reactionRef: ReactionRef): void => {
|
|
7469
|
-
const addResult = react({
|
|
7470
|
-
adapter: live.key.adapter,
|
|
7471
|
-
workspace: live.key.workspace,
|
|
7472
|
-
chat: live.key.chat,
|
|
7473
|
-
thread: live.key.thread,
|
|
7474
|
-
reactionRef,
|
|
7475
|
-
emoji: CONTINUATION_REACTION_EMOJI,
|
|
7476
|
-
})
|
|
7477
|
-
void addResult
|
|
7478
|
-
.then((result) => {
|
|
7479
|
-
if (!result.ok && result.code !== 'unsupported') {
|
|
7480
|
-
logger.info(
|
|
7481
|
-
`[channels] continuation-react failed adapter=${live.key.adapter} chat=${live.key.chat}: ${result.error}`,
|
|
7482
|
-
)
|
|
7483
|
-
}
|
|
7484
|
-
})
|
|
7485
|
-
.catch((err) => {
|
|
7486
|
-
logger.info(
|
|
7487
|
-
`[channels] continuation-react threw adapter=${live.key.adapter} chat=${live.key.chat}: ${describeError(err)}`,
|
|
7488
|
-
)
|
|
7489
|
-
})
|
|
7490
9335
|
live.activeContinuationReactions.push(
|
|
7491
|
-
|
|
9336
|
+
waitingReactions.acquire({
|
|
9337
|
+
adapter: live.key.adapter,
|
|
9338
|
+
workspace: live.key.workspace,
|
|
9339
|
+
chat: live.key.chat,
|
|
9340
|
+
thread: live.key.thread,
|
|
9341
|
+
reactionRef,
|
|
9342
|
+
emoji: CONTINUATION_REACTION_EMOJI,
|
|
9343
|
+
}),
|
|
7492
9344
|
)
|
|
7493
9345
|
}
|
|
7494
9346
|
|
|
7495
9347
|
const dropContinuationReactions = (live: LiveSession): Promise<void> => {
|
|
7496
|
-
const
|
|
7497
|
-
if (addPromises.length === 0) return Promise.resolve()
|
|
9348
|
+
const owners = live.activeContinuationReactions
|
|
7498
9349
|
live.activeContinuationReactions = []
|
|
7499
|
-
return Promise.all(
|
|
7500
|
-
|
|
7501
|
-
|
|
7502
|
-
|
|
7503
|
-
|
|
7504
|
-
|
|
7505
|
-
|
|
7506
|
-
|
|
7507
|
-
|
|
7508
|
-
|
|
7509
|
-
|
|
7510
|
-
|
|
7511
|
-
|
|
7512
|
-
|
|
7513
|
-
|
|
7514
|
-
|
|
7515
|
-
|
|
7516
|
-
|
|
7517
|
-
|
|
7518
|
-
|
|
7519
|
-
|
|
7520
|
-
|
|
7521
|
-
|
|
7522
|
-
|
|
7523
|
-
|
|
7524
|
-
|
|
7525
|
-
|
|
9350
|
+
return Promise.all(owners.map((owner) => waitingReactions.release(owner))).then(() => undefined)
|
|
9351
|
+
}
|
|
9352
|
+
const registerRecoveryAdapter = (adapter: ChannelKey['adapter'], callbacks: RecoveryAdapterCallbacks): void => {
|
|
9353
|
+
recoveryAdapters.set(adapter, callbacks)
|
|
9354
|
+
}
|
|
9355
|
+
const unregisterRecoveryAdapter = (adapter: ChannelKey['adapter'], callbacks: RecoveryAdapterCallbacks): void => {
|
|
9356
|
+
if (recoveryAdapters.get(adapter) === callbacks) recoveryAdapters.delete(adapter)
|
|
9357
|
+
}
|
|
9358
|
+
const activeTurnAccountIdentity = (live: LiveSession | undefined): string | undefined =>
|
|
9359
|
+
live && (live.promptInFlight || live.inboundCoverage.length > 0 || live.backgroundCoverage.length > 0)
|
|
9360
|
+
? live.turnAccountIdentity
|
|
9361
|
+
: undefined
|
|
9362
|
+
|
|
9363
|
+
const currentAccountIdentityFor = (key: Pick<ChannelKey, 'adapter' | 'workspace'>): string | undefined => {
|
|
9364
|
+
const callbacks = recoveryAdapters.get(key.adapter)
|
|
9365
|
+
if (callbacks) return callbacks.cachedAccountIdentity?.(key.workspace)
|
|
9366
|
+
return fallbackChannelAccountIdentity(
|
|
9367
|
+
key.adapter,
|
|
9368
|
+
key.workspace,
|
|
9369
|
+
selfIdentityResolvers.get(key.adapter)?.(key.workspace)?.id,
|
|
9370
|
+
)
|
|
9371
|
+
}
|
|
9372
|
+
|
|
9373
|
+
const getRecoveryAccountIdentity = async (
|
|
9374
|
+
adapter: ChannelKey['adapter'],
|
|
9375
|
+
workspace = '',
|
|
9376
|
+
): Promise<string | undefined> => {
|
|
9377
|
+
return currentAccountIdentityFor({ adapter, workspace })
|
|
9378
|
+
}
|
|
9379
|
+
const validateRecovery = async (record: RecoveryRecord): Promise<RecoveryFailure | undefined> => {
|
|
9380
|
+
const config = options.configForAdapter(record.target.adapter)
|
|
9381
|
+
if (!config || !config.enabled) {
|
|
9382
|
+
return { kind: 'configuration', safeReason: 'original adapter is not configured' }
|
|
9383
|
+
}
|
|
9384
|
+
if (
|
|
9385
|
+
record.principal.kind !== 'channel' ||
|
|
9386
|
+
!permissions.has({ ...record.principal, thread: record.target.thread }, CORE_PERMISSIONS.channelRespond)
|
|
9387
|
+
) {
|
|
9388
|
+
return { kind: 'permission', safeReason: 'original principal cannot respond or send' }
|
|
9389
|
+
}
|
|
9390
|
+
const callbacks = recoveryAdapters.get(record.target.adapter)
|
|
9391
|
+
const identity = await (callbacks
|
|
9392
|
+
? callbacks.accountIdentity(record.target.workspace)
|
|
9393
|
+
: getRecoveryAccountIdentity(record.target.adapter, record.target.workspace))
|
|
9394
|
+
if (identity === undefined) return { kind: 'unavailable', safeReason: 'adapter account is not ready' }
|
|
9395
|
+
if (record.accountIdentity === 'unbound-legacy' && record.boundAccountIdentity === undefined) return undefined
|
|
9396
|
+
if (identity !== (record.boundAccountIdentity ?? record.accountIdentity))
|
|
9397
|
+
return { kind: 'identity', safeReason: 'original account identity changed' }
|
|
9398
|
+
return undefined
|
|
9399
|
+
}
|
|
9400
|
+
const reconcileRecovery = async (record: RecoveryRecord): Promise<RecoveryReconcileResult> =>
|
|
9401
|
+
recoveryAdapters.get(record.target.adapter)?.reconcile(record) ?? { status: 'unreconcilable' }
|
|
9402
|
+
const recoveryRedelivery = (record: RecoveryRecord, eventKinds: readonly string[]): RecoveryRedelivery =>
|
|
9403
|
+
recoveryAdapters.get(record.target.adapter)?.redelivery?.(record, eventKinds) ?? 'not-redelivered'
|
|
9404
|
+
const setRecoveryStopHandler = (handler: (target: ChannelKey, parentSessionId: string) => Promise<void>): void => {
|
|
9405
|
+
recoveryStopHandler = handler
|
|
7526
9406
|
}
|
|
7527
9407
|
|
|
7528
9408
|
return {
|
|
7529
9409
|
route,
|
|
7530
9410
|
send,
|
|
9411
|
+
acceptBackgroundResponse,
|
|
9412
|
+
suppressUnstartedBackgroundResponse,
|
|
9413
|
+
attachBackgroundResultCoverage,
|
|
9414
|
+
captureBackgroundResultCoverage,
|
|
9415
|
+
captureInboundResultCoverage,
|
|
9416
|
+
captureTurnAccountIdentity,
|
|
9417
|
+
registerRecoveryAdapter,
|
|
9418
|
+
unregisterRecoveryAdapter,
|
|
9419
|
+
getRecoveryAccountIdentity,
|
|
9420
|
+
validateRecovery,
|
|
9421
|
+
reconcileRecovery,
|
|
9422
|
+
recoveryRedelivery,
|
|
9423
|
+
setRecoveryStopHandler,
|
|
7531
9424
|
getConsecutiveSendCount,
|
|
7532
9425
|
hasQualifyingWorkThisLogicalTurn,
|
|
7533
9426
|
getSendRate,
|
|
@@ -7540,6 +9433,12 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
7540
9433
|
registerRemoveReaction,
|
|
7541
9434
|
unregisterRemoveReaction,
|
|
7542
9435
|
removeReaction,
|
|
9436
|
+
registerPrepareOwnReaction,
|
|
9437
|
+
unregisterPrepareOwnReaction,
|
|
9438
|
+
registerRemoveOwnReaction,
|
|
9439
|
+
unregisterRemoveOwnReaction,
|
|
9440
|
+
removeOwnReaction,
|
|
9441
|
+
recoverWaitingReactions,
|
|
7543
9442
|
registerTyping,
|
|
7544
9443
|
unregisterTyping,
|
|
7545
9444
|
setTypingCapability,
|
|
@@ -7594,7 +9493,6 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
7594
9493
|
stop,
|
|
7595
9494
|
tearDownAllLive,
|
|
7596
9495
|
markRestartAbortForAllLive,
|
|
7597
|
-
writeInterruptedSubagentHandoff,
|
|
7598
9496
|
liveCount: () => liveSessions.size,
|
|
7599
9497
|
__testing: {
|
|
7600
9498
|
githubReviewRoundFor: (key: ChannelKey) => liveSessions.get(channelKeyId(key))?.githubReviewRound,
|
|
@@ -7675,6 +9573,22 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
7675
9573
|
},
|
|
7676
9574
|
typingHeartbeatIntervalFor,
|
|
7677
9575
|
runIdleGc,
|
|
9576
|
+
getRecoveryAccountingSnapshot: (key: ChannelKey) => {
|
|
9577
|
+
const live = liveSessions.get(channelKeyId(key))
|
|
9578
|
+
if (!live) return undefined
|
|
9579
|
+
return {
|
|
9580
|
+
successfulChannelSends: live.successfulChannelSends,
|
|
9581
|
+
promisedWorkOutstandingThisLogicalTurn: live.promisedWorkOutstandingThisLogicalTurn,
|
|
9582
|
+
lastSendLeafId: live.lastSendLeafId,
|
|
9583
|
+
skippedTurn: live.skippedTurn,
|
|
9584
|
+
consecutiveSends: [...live.consecutiveSends],
|
|
9585
|
+
lastSentText: [...live.lastSentText],
|
|
9586
|
+
policyDeniedToolSendsThisTurn: [...live.policyDeniedToolSendsThisTurn],
|
|
9587
|
+
continuationReactions: [...live.activeContinuationReactions],
|
|
9588
|
+
typingEpoch: live.typingEpoch,
|
|
9589
|
+
continueReplyTurn: live.continueReplyTurn,
|
|
9590
|
+
}
|
|
9591
|
+
},
|
|
7678
9592
|
getLiveOriginSnapshot: (key: ChannelKey) => {
|
|
7679
9593
|
const live = liveSessions.get(channelKeyId(key))
|
|
7680
9594
|
const origin = live?.originRef.current
|
|
@@ -7696,11 +9610,6 @@ export function createChannelRouter(options: CreateChannelRouterOptions): Channe
|
|
|
7696
9610
|
if (!live) return
|
|
7697
9611
|
live.pendingSystemReminders.push(retryReminder(text))
|
|
7698
9612
|
},
|
|
7699
|
-
enqueueUserInbound: (key: ChannelKey, event: InboundMessage): void => {
|
|
7700
|
-
const live = liveSessions.get(channelKeyId(key))
|
|
7701
|
-
if (!live) return
|
|
7702
|
-
enqueue(live, event, null)
|
|
7703
|
-
},
|
|
7704
9613
|
},
|
|
7705
9614
|
}
|
|
7706
9615
|
}
|
|
@@ -8506,7 +10415,7 @@ async function raceWithTimeout<T>(work: Promise<T>, ms: number, label: string):
|
|
|
8506
10415
|
function recoverableAssistantText(
|
|
8507
10416
|
session: AgentSession,
|
|
8508
10417
|
): { text: string; source: 'leaf' | 'mid-turn' | 'pre-tool' | 'length-leaf' } | null {
|
|
8509
|
-
const leaf = session
|
|
10418
|
+
const leaf = branchLeafEntry(session)
|
|
8510
10419
|
if (!leaf) return null
|
|
8511
10420
|
|
|
8512
10421
|
if (leaf.type === 'message' && leaf.message.role === 'assistant') {
|
|
@@ -8556,6 +10465,21 @@ function recoverableAssistantText(
|
|
|
8556
10465
|
return null
|
|
8557
10466
|
}
|
|
8558
10467
|
|
|
10468
|
+
// The session's last entry, skipping trailing pi `context_edit` entries. pi
|
|
10469
|
+
// 0.99's truncated-response and auto-retry recovery appends a `context_edit`
|
|
10470
|
+
// (`replacement: null`) after the attempt it omits from model projection, and
|
|
10471
|
+
// when that recovery does not go on to retry, the edit is the raw leaf. Every
|
|
10472
|
+
// turn-end check here keys on the last message's role and stopReason, so reading
|
|
10473
|
+
// the raw leaf turned a `length` truncation into a "no assistant message" turn:
|
|
10474
|
+
// the raised-budget empty-turn retry was skipped and the turn-ended notice posted.
|
|
10475
|
+
function branchLeafEntry(session: AgentSession): SessionEntry | undefined {
|
|
10476
|
+
let entry = session.sessionManager.getLeafEntry()
|
|
10477
|
+
for (let depth = 0; depth < 32 && entry?.type === 'context_edit'; depth++) {
|
|
10478
|
+
entry = entry.parentId ? session.sessionManager.getEntry(entry.parentId) : undefined
|
|
10479
|
+
}
|
|
10480
|
+
return entry
|
|
10481
|
+
}
|
|
10482
|
+
|
|
8559
10483
|
// The non-terminal stop reason when the leaf is an assistant message that did
|
|
8560
10484
|
// NOT cleanly finish — `length` (hit the token cap, the canonical kimi
|
|
8561
10485
|
// reasoning-loop), `error` (an upstream provider failure), or `aborted` (the
|
|
@@ -8565,7 +10489,7 @@ function recoverableAssistantText(
|
|
|
8565
10489
|
// diverts to the provider-error notice, `length` gets a raised retry budget,
|
|
8566
10490
|
// `aborted` retries under the default backstop.
|
|
8567
10491
|
function assistantLeafStopReason(session: AgentSession): 'length' | 'error' | 'aborted' | undefined {
|
|
8568
|
-
const leaf = session
|
|
10492
|
+
const leaf = branchLeafEntry(session)
|
|
8569
10493
|
if (!leaf || leaf.type !== 'message' || leaf.message.role !== 'assistant') return undefined
|
|
8570
10494
|
const stop = leaf.message.stopReason
|
|
8571
10495
|
if (stop === 'length' || stop === 'error' || stop === 'aborted') return stop
|
|
@@ -8585,7 +10509,7 @@ function assistantLeafStopReason(session: AgentSession): 'length' | 'error' | 'a
|
|
|
8585
10509
|
// to re-prompt a turn that strands mid-work after a `more_work_this_turn: true` status
|
|
8586
10510
|
// reply instead of ending in silence.
|
|
8587
10511
|
function leafIsStrandedToolUse(session: AgentSession): boolean {
|
|
8588
|
-
const leaf = session
|
|
10512
|
+
const leaf = branchLeafEntry(session)
|
|
8589
10513
|
if (!leaf || leaf.type !== 'message') return false
|
|
8590
10514
|
if (leaf.message.role === 'assistant') {
|
|
8591
10515
|
return leaf.message.stopReason === 'toolUse' && visibleAssistantText(leaf.message).trim() === ''
|
|
@@ -8669,7 +10593,7 @@ function repairDanglingToolUseBranch(session: AgentSession, timestamp: number):
|
|
|
8669
10593
|
// of this shape: `channel_reply({ more_work_this_turn: true })` via `continueReplyTurn`, and
|
|
8670
10594
|
// `channel_send` acks via the willingness phrase table.
|
|
8671
10595
|
function isFreshEmptyStopAfterSend(live: LiveSession): boolean {
|
|
8672
|
-
const leaf = live.session
|
|
10596
|
+
const leaf = branchLeafEntry(live.session)
|
|
8673
10597
|
if (!leaf || leaf.type !== 'message' || leaf.message.role !== 'assistant') return false
|
|
8674
10598
|
if (leaf.message.stopReason !== 'stop') return false
|
|
8675
10599
|
if (hasToolCall(leaf.message) || visibleAssistantText(leaf.message).trim() !== '') return false
|
|
@@ -8711,7 +10635,7 @@ function hasToolCall(message: AssistantMessage): boolean {
|
|
|
8711
10635
|
// `toolUse`, not `stop`. Depth-bounded like the other branch walks so a
|
|
8712
10636
|
// malformed session can't loop.
|
|
8713
10637
|
function attemptMadeToolCall(session: AgentSession): boolean {
|
|
8714
|
-
let cursor: SessionEntry | undefined = session
|
|
10638
|
+
let cursor: SessionEntry | undefined = branchLeafEntry(session)
|
|
8715
10639
|
for (let depth = 0; depth < 32 && cursor; depth++) {
|
|
8716
10640
|
if (cursor.type === 'message') {
|
|
8717
10641
|
if (cursor.message.role === 'user') return false
|