switchroom 0.19.41 → 0.19.43

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/dist/agent-scheduler/index.js +23 -3
  2. package/dist/auth-broker/index.js +54 -9
  3. package/dist/cli/notion-write-pretool.mjs +23 -3
  4. package/dist/cli/switchroom.js +1956 -900
  5. package/dist/host-control/main.js +227 -30
  6. package/dist/vault/approvals/kernel-server.js +54 -9
  7. package/dist/vault/broker/server.js +54 -9
  8. package/package.json +3 -3
  9. package/profiles/_base/start.sh.hbs +82 -4
  10. package/telegram-plugin/bridge/bridge.ts +1 -1
  11. package/telegram-plugin/card-layout.ts +63 -1
  12. package/telegram-plugin/dist/bridge/bridge.js +4 -1
  13. package/telegram-plugin/dist/gateway/gateway.js +1318 -693
  14. package/telegram-plugin/dist/server.js +5 -2
  15. package/telegram-plugin/flood-429-ledger.ts +5 -3
  16. package/telegram-plugin/flood-circuit-breaker.ts +11 -3
  17. package/telegram-plugin/format.ts +223 -13
  18. package/telegram-plugin/gateway/approval-callback-consume-record.test.ts +132 -0
  19. package/telegram-plugin/gateway/boot-card.ts +7 -2
  20. package/telegram-plugin/gateway/flood-reply-queue.ts +5 -0
  21. package/telegram-plugin/gateway/gateway.ts +56 -58
  22. package/telegram-plugin/gateway/handback-orphan-recovery.ts +69 -0
  23. package/telegram-plugin/gateway/handback-preturn-signal.ts +170 -32
  24. package/telegram-plugin/gateway/ipc-protocol.ts +31 -3
  25. package/telegram-plugin/gateway/ipc-server.ts +13 -4
  26. package/telegram-plugin/gateway/liveness-wiring.ts +12 -0
  27. package/telegram-plugin/gateway/model-command.ts +21 -109
  28. package/telegram-plugin/gateway/obligation-store.ts +55 -0
  29. package/telegram-plugin/gateway/outbound-send-path.ts +127 -11
  30. package/telegram-plugin/gateway/outbox-sweep.ts +371 -61
  31. package/telegram-plugin/gateway/rollout-status-edit.ts +175 -0
  32. package/telegram-plugin/gateway/stream-render.ts +62 -0
  33. package/telegram-plugin/gateway/update-announce.ts +11 -4
  34. package/telegram-plugin/hooks/audience-classify.d.mts +50 -0
  35. package/telegram-plugin/hooks/audience-classify.mjs +364 -0
  36. package/telegram-plugin/hooks/silent-end-interrupt-stop.mjs +86 -3
  37. package/telegram-plugin/hooks/silent-end-scan.mjs +80 -4
  38. package/telegram-plugin/outbox.ts +103 -2
  39. package/telegram-plugin/render/rich-render.ts +10 -2
  40. package/telegram-plugin/reply-quote.ts +150 -0
  41. package/telegram-plugin/retry-api-call.ts +254 -11
  42. package/telegram-plugin/shared/bot-runtime.ts +69 -34
  43. package/telegram-plugin/shared/gw-trace-gate.ts +14 -2
  44. package/telegram-plugin/silent-end.ts +34 -2
  45. package/telegram-plugin/status-no-truncate.ts +31 -16
  46. package/telegram-plugin/stream-controller.ts +44 -9
  47. package/telegram-plugin/stream-reply-handler.ts +5 -3
  48. package/telegram-plugin/tests/boot-card-render.test.ts +2 -1
  49. package/telegram-plugin/tests/card-budget-never-overflows.test.ts +165 -0
  50. package/telegram-plugin/tests/card-hard-truncate-entity-safe.test.ts +153 -0
  51. package/telegram-plugin/tests/card-variants.golden.txt +9 -9
  52. package/telegram-plugin/tests/flood-reply-queue.test.ts +89 -4
  53. package/telegram-plugin/tests/format-consistency.test.ts +160 -14
  54. package/telegram-plugin/tests/framework-fallback-drains-parked.test.ts +134 -0
  55. package/telegram-plugin/tests/handback-orphan-recovery.test.ts +131 -0
  56. package/telegram-plugin/tests/handback-preturn-signal.test.ts +212 -17
  57. package/telegram-plugin/tests/helpers/liveness-wiring-fixture.ts +3 -0
  58. package/telegram-plugin/tests/nested-indent-idiom.test.ts +162 -0
  59. package/telegram-plugin/tests/nested-prefix-indent.test.ts +115 -0
  60. package/telegram-plugin/tests/no-robust-api-call-factory.test.ts +62 -0
  61. package/telegram-plugin/tests/obligation-store.test.ts +51 -0
  62. package/telegram-plugin/tests/outbox-audience-3865.test.ts +579 -0
  63. package/telegram-plugin/tests/outbox-provenance-4141.test.ts +1126 -0
  64. package/telegram-plugin/tests/outbox-sweep-single-flight.test.ts +138 -0
  65. package/telegram-plugin/tests/pinned-card-collapse.test.ts +12 -10
  66. package/telegram-plugin/tests/plain-fallback-4096-cap.test.ts +509 -0
  67. package/telegram-plugin/tests/reply-quote-wire.test.ts +433 -0
  68. package/telegram-plugin/tests/retry-grammy-http-error.test.ts +404 -0
  69. package/telegram-plugin/tests/rollout-narration-edit-socket.test.ts +170 -0
  70. package/telegram-plugin/tests/rollout-status-edit-retry-policy.test.ts +366 -0
  71. package/telegram-plugin/tests/rollout-status-edit.test.ts +140 -0
  72. package/telegram-plugin/tests/rollout-status-wiring.test.ts +40 -7
  73. package/telegram-plugin/tests/tg-post-logger-error-shape.test.ts +4 -0
  74. package/telegram-plugin/tests/tg-post-retry-attribution.test.ts +196 -0
  75. package/telegram-plugin/tests/tool-activity-summary.test.ts +23 -17
  76. package/telegram-plugin/tool-activity-summary.ts +58 -30
  77. package/telegram-plugin/uat/scenarios/jtbd-foreground-subagent-activity-dm.test.ts +3 -2
  78. package/vendor/hindsight-memory/CHANGELOG.md +28 -0
  79. package/vendor/hindsight-memory/docs/measurements/subagent-volume-gate-3994.md +84 -0
  80. package/vendor/hindsight-memory/scripts/backfill_transcripts.py +254 -40
  81. package/vendor/hindsight-memory/scripts/lib/content.py +11 -2
  82. package/vendor/hindsight-memory/scripts/recall.py +18 -3
  83. package/vendor/hindsight-memory/scripts/subagent_retain.py +59 -38
  84. package/vendor/hindsight-memory/scripts/tests/data/replay_volume_gate_3994.py +295 -0
  85. package/vendor/hindsight-memory/scripts/tests/test_backfill_from_logs.py +109 -0
  86. package/vendor/hindsight-memory/scripts/tests/test_overlap_tokens.py +135 -0
  87. package/vendor/hindsight-memory/scripts/tests/test_recall_no_lexical_gate.py +20 -23
  88. package/vendor/hindsight-memory/scripts/tests/test_recall_query_shaping.py +48 -0
  89. package/vendor/hindsight-memory/scripts/tests/test_subagent_retain.py +94 -1
  90. package/vendor/hindsight-memory/scripts/tests/test_subagent_retain_learnings.py +98 -42
@@ -280,10 +280,8 @@ import { REPLY_TOOLS } from '../narrative-dedup.js'
280
280
  import { NarrativeFlushController, PENDING_NARRATIVE_FLUSH_MS } from '../narrative-flush.js'
281
281
  import { createTypingWrapper } from '../typing-wrap.js'
282
282
  import { createTurnTypingLoop } from './turn-typing-loop.js'
283
- import {
284
- createHandbackPreturnSignal,
285
- type PreTurnCardRecord,
286
- } from './handback-preturn-signal.js'
283
+ import { createHandbackPreturnSignal } from './handback-preturn-signal.js'
284
+ import { createHandbackOrphanRecovery } from './handback-orphan-recovery.js'
287
285
  import { deriveTurnId } from './derive-turn-id.js'
288
286
  import { createTypingEmitter, TYPING_REFRESH_MS } from '../typing-emitter.js'
289
287
  import { type DraftStreamHandle } from '../draft-stream.js'
@@ -457,7 +455,7 @@ import {
457
455
  type SendReplyGatewayDeps,
458
456
  type DeliverCapturedProseDeps,
459
457
  } from './outbound-send-path.js'
460
- import { handleSessionEvent as handleSessionEventCore } from './stream-render.js'
458
+ import { handleSessionEvent as handleSessionEventCore, drainParkedTurnStartsForChat } from './stream-render.js'
461
459
  import { createNarrativeLane } from './narrative-lane.js'
462
460
  import {
463
461
  parseAgentCallback,
@@ -654,6 +652,7 @@ import { startWebhookIngestServer } from './webhook-ingest-server.js'
654
652
  import { recordWebhookEvent } from '../../src/web/webhook-gateway-record.js'
655
653
 
656
654
  import { createIpcServer, type IpcClient, type IpcServer } from './ipc-server.js'
655
+ import { handleRolloutStatusEdit } from './rollout-status-edit.js'
657
656
  import { handleRequestDriveApproval } from './drive-write-approval.js'
658
657
  import { handleRequestMs365Approval } from './ms365-write-approval.js'
659
658
  import { buildDiffPreviewCard } from './diff-preview-card.js'
@@ -667,7 +666,7 @@ import {
667
666
  } from './obligation-ledger.js'
668
667
  // (#2996 P8 PR-C2: buildObligationRepresentInbound is now imported by
669
668
  // obligation-wiring.ts, where the sweep lives.)
670
- import { loadObligations, persistObligations } from './obligation-store.js'
669
+ import { loadObligations, persistObligations, removeUnmaintainedSnapshot } from './obligation-store.js'
671
670
  import { buildCapturedDeliverySnapshot, createCapturedResumeDispatcher } from './captured-answer-resume.js'
672
671
  import {
673
672
  loadStatusPins,
@@ -2804,7 +2803,7 @@ const obligationStoreFs = {
2804
2803
  writeFileSync: (p: string, d: string) => writeFileSync(p, d),
2805
2804
  renameSync: (a: string, b: string) => renameSync(a, b),
2806
2805
  existsSync: (p: string) => existsSync(p),
2807
- fsyncFileSync: fsyncPathSync, fsyncDirSync: fsyncPathSync,
2806
+ fsyncFileSync: fsyncPathSync, fsyncDirSync: fsyncPathSync, unlinkSync,
2808
2807
  }
2809
2808
  const obligationLedger = new ObligationLedger(OBLIGATION_REPRESENT_MAX, {
2810
2809
  onChange:
@@ -2822,7 +2821,7 @@ if (isGatewayMain && !STATIC && OBLIGATION_LEDGER_ENABLED) {
2822
2821
  `telegram gateway: obligation-ledger hydrated ${restored.length} open obligation(s) from ${OBLIGATION_STORE_PATH}\n`,
2823
2822
  )
2824
2823
  }
2825
- }
2824
+ } else if (isGatewayMain) removeUnmaintainedSnapshot(OBLIGATION_STORE_PATH, obligationStoreFs, `static=${STATIC} enabled=${OBLIGATION_LEDGER_ENABLED}`) // #4146: persistence is off, so a leftover snapshot must not read as "nobody is waiting" to the W1-d audience classifier (logic in obligation-store.ts)
2826
2825
  // Origin ids with an escalation send IN FLIGHT — prevents the 5s sweep from
2827
2826
  // firing a second concurrent send for the same obligation while the first is
2828
2827
  // still awaiting (an escalation that takes >5s, or repeated failures).
@@ -9669,14 +9668,14 @@ function gatewayLivenessWiringDeps() {
9669
9668
  turnLiveForItsTopic,
9670
9669
  endCurrentTurnForKey,
9671
9670
  // Getter, not an eager read: `pendingInboundBuffer` (const) is declared
9672
- // LATER in this module (~9863) than the module-load `startTimer` call that
9673
- // invokes this builder (isGatewayMain path). A by-value capture here is a
9674
- // temporal-dead-zone ReferenceError that crash-loops every gateway boot
9675
- // (#3392 regression). Deferring to a getter matching getInboundSpool /
9676
- // getTurnsDb / ipcServer — means the binding is only read when the
9677
- // fallback actually fires, well after module eval.
9671
+ // LATER than the module-load `startTimer` call that invokes this builder,
9672
+ // so a by-value capture is a temporal-dead-zone ReferenceError that
9673
+ // crash-loops boot (#3392). A getter (matching getInboundSpool / getTurnsDb
9674
+ // / ipcServer) is read only when the fallback fires, well after module eval.
9678
9675
  getPendingInboundBuffer: () => pendingInboundBuffer,
9679
9676
  trackRedeliveredInbound,
9677
+ // Two-state desync unwedge (#3927): drain the parked store the fallback else leaves latched — see liveness-wiring + stream-render drainParkedTurnStartsForChat.
9678
+ drainParkedTurnStarts: (chatId: string, threadId: number | null): number => drainParkedTurnStartsForChat(gatewayStreamRenderDeps(), chatId, threadId != null ? String(threadId) : null).length,
9680
9679
  closeActivityLane,
9681
9680
  closeProgressLane,
9682
9681
  // Stage B: escalate a mid-tool + marker-stale fallback to a real restart.
@@ -10052,7 +10051,7 @@ async function deliverCapturedProse(args: {
10052
10051
  originTurnId: string
10053
10052
  text: string
10054
10053
  /** Turn elapsed for the honest "(waited Ns)" apology clause; optional. */
10055
- turnDurationMs?: number
10054
+ turnDurationMs?: number; replyToolThrewThisTurn?: boolean // #4141: label prose recovered from a turn whose reply tool threw (see outbound-send-path.ts)
10056
10055
  }): Promise<void> {
10057
10056
  // #2996 P2: body moved VERBATIM to outbound-send-path.ts (single tested
10058
10057
  // send module). Injects the ONE `outboundDedup` instance (Amendment 1/9).
@@ -11447,28 +11446,31 @@ if (isGatewayMain) ipcServer = createIpcServer({
11447
11446
  }
11448
11447
  },
11449
11448
 
11450
- // #2726 Part 2 — edit the previously-posted rollout status message in place as
11451
- // later phases arrive. Fire-and-forget: an edit failure (incl. Telegram 429)
11452
- // is swallowed here and NEVER surfaced back toward the roll. hostd owns the
11453
- // debounce + 429 retry cadence; this handler is a thin edit relay.
11454
- onRolloutStatusEdit(_client: IpcClient, msg: RolloutStatusEditMessage) {
11455
- const self = process.env.SWITCHROOM_AGENT_NAME
11456
- if (self && msg.agentName !== self) {
11457
- process.stderr.write(
11458
- `telegram gateway: rollout_status_edit rejected — agent mismatch (${msg.agentName} != ${self})\n`,
11459
- )
11460
- return
11461
- }
11462
- const operator = loadAccess().allowFrom[0]
11463
- if (operator === undefined) {
11464
- process.stderr.write(`telegram gateway: rollout_status_editno operator chat (allowFrom empty)\n`)
11465
- return
11466
- }
11467
- void swallowingApiCall(
11468
- () =>
11469
- // allow-raw-bot-api: in-place edit of the ordinary status message.
11470
- bot.api.editMessageText(operator, msg.messageId, richMessage(msg.text), {}),
11471
- { chat_id: String(operator), verb: 'rollout-status-edit' },
11449
+ // #2726 Part 2 / #4065 — edit the previously-posted rollout status message in
11450
+ // place as later phases arrive, and REPLY with the outcome so hostd can tell
11451
+ // an applied edit from an edit into a deleted card (see rollout-status-edit.ts).
11452
+ // Still fire-and-forget toward the roll: hostd never blocks on the reply.
11453
+ onRolloutStatusEdit(client: IpcClient, msg: RolloutStatusEditMessage) {
11454
+ return handleRolloutStatusEdit(
11455
+ {
11456
+ selfAgentName: process.env.SWITCHROOM_AGENT_NAME,
11457
+ operatorChatId: () => loadAccess().allowFrom[0],
11458
+ editMessage: (chatId, messageId, text) =>
11459
+ robustApiCall(
11460
+ () =>
11461
+ // allow-raw-bot-api: in-place edit of the ordinary status message.
11462
+ bot.api.editMessageText(chatId, messageId, richMessage(text), {}),
11463
+ // BOTH opt-outs are load-bearing see "TWO swallows" in
11464
+ // rollout-status-edit.ts. A SHED edit and a SWALLOWED benign
11465
+ // `400 message to edit not found` each resolve WITHOUT throwing,
11466
+ // and the handler reads success from the absence of a throw.
11467
+ { chat_id: String(chatId), verb: 'rollout-status-edit',
11468
+ priorityClass: 'critical', rethrowBenign400: true },
11469
+ ),
11470
+ log: (m) => process.stderr.write(`telegram gateway: ${m}\n`),
11471
+ },
11472
+ client,
11473
+ msg,
11472
11474
  )
11473
11475
  },
11474
11476
 
@@ -13743,8 +13745,6 @@ if (isGatewayMain && !STATIC && FEED_HEARTBEAT_ENABLED) {
13743
13745
  // closes. Kill switch: SWITCHROOM_HANDBACK_PRETURN=0.
13744
13746
  const HANDBACK_PRETURN_ENABLED = !STATIC && process.env.SWITCHROOM_HANDBACK_PRETURN !== '0'
13745
13747
  const HANDBACK_PRETURN_HTML = '🤝 Reading the worker’s results…'
13746
- const HANDBACK_PRETURN_ORPHAN_HTML =
13747
- '🤝 A background worker finished, but the handback never started — it may need a nudge.'
13748
13748
 
13749
13749
  async function openHandbackPreTurnCard(
13750
13750
  chatId: string,
@@ -13774,24 +13774,20 @@ async function openHandbackPreTurnCard(
13774
13774
  }
13775
13775
  }
13776
13776
 
13777
- function finalizeHandbackPreTurnCard(record: PreTurnCardRecord): Promise<void> {
13778
- return robustApiCall(
13779
- () =>
13780
- bot.api.editMessageText(
13781
- record.chatId,
13782
- record.activityMessageId,
13783
- richMessage(HANDBACK_PRETURN_ORPHAN_HTML),
13784
- {},
13785
- ),
13786
- {
13787
- chat_id: record.chatId,
13788
- ...(record.threadId != null ? { threadId: record.threadId } : {}),
13789
- verb: 'handback-preturn.orphan-finalize',
13790
- },
13791
- )
13792
- .then(() => undefined)
13793
- .catch(() => undefined)
13794
- }
13777
+ // Deterministic recovery for a genuinely-orphaned handback: delete the frozen
13778
+ // pre-turn card, re-inject the handback through the pending-inbound buffer, and
13779
+ // past the retry cap emit fleet-health telemetry — never an operator-facing
13780
+ // "needs a nudge" card. See handback-orphan-recovery.ts.
13781
+ const handbackOrphanRecovery = createHandbackOrphanRecovery({
13782
+ deleteMessage: (chatId, messageId, threadId) =>
13783
+ robustApiCall(() => bot.api.deleteMessage(chatId, messageId), {
13784
+ chat_id: chatId,
13785
+ ...(threadId != null ? { threadId } : {}),
13786
+ verb: 'handback-preturn.orphan-delete',
13787
+ }),
13788
+ pushInbound: (inbound) =>
13789
+ pendingInboundBuffer.push(process.env.SWITCHROOM_AGENT_NAME ?? '', inbound),
13790
+ })
13795
13791
 
13796
13792
  const handbackPreturnSignal = createHandbackPreturnSignal({
13797
13793
  chatKey: (chatId, threadId) => chatKey(chatId, threadId) as string,
@@ -13799,7 +13795,9 @@ const handbackPreturnSignal = createHandbackPreturnSignal({
13799
13795
  startTypingLoop: (chatId, threadId) => startTurnTypingLoop(chatId, threadId),
13800
13796
  stopTypingLoop: (chatId, threadId) => stopTurnTypingLoop(chatId, threadId),
13801
13797
  openCard: openHandbackPreTurnCard,
13802
- finalizeCard: finalizeHandbackPreTurnCard,
13798
+ deleteCard: handbackOrphanRecovery.deleteCard,
13799
+ reinjectHandback: handbackOrphanRecovery.reinjectHandback,
13800
+ escalateOrphan: handbackOrphanRecovery.escalateOrphan,
13803
13801
  writeCardRecord: (record) => {
13804
13802
  if (!activityCardPersistEnabled) return
13805
13803
  writeActivityCardRecord(ACTIVITY_CARD_STORE_PATH, activityCardStoreFs, record)
@@ -0,0 +1,69 @@
1
+ /**
2
+ * handback-orphan-recovery.ts — the gateway-side EFFECTS for deterministic
3
+ * recovery of a genuinely-orphaned sub-agent handback.
4
+ *
5
+ * The decision ("is this an orphan, should it be re-injected, has it exhausted
6
+ * its retries") lives in `handback-preturn-signal.ts`. This module owns only the
7
+ * three side effects that decision drives, so they live in a module instead of
8
+ * inline in gateway.ts (switchroom#2996 anti-inflation ratchet — new logic goes
9
+ * into a seam, not into the 24k-line gateway):
10
+ *
11
+ * 1. `deleteCard` — DELETE (never edit) the frozen pre-turn card. The
12
+ * old behaviour rewrote it to an operator-facing string asking a human to
13
+ * manually nudge a system that must recover itself. That string is gone
14
+ * from the codebase; the card is removed outright.
15
+ * 2. `reinjectHandback` — push the handback back through the pending-inbound
16
+ * buffer (the same path live synthesis + boot replay use) so the idle drain
17
+ * re-delivers it and the machine re-processes it. The seam stamps the retry
18
+ * counter on `inbound.meta` before calling this, so the cap survives the
19
+ * round trip.
20
+ * 3. `escalateOrphan` — retries exhausted → fleet-health TELEMETRY, never a
21
+ * chat card. A structured line on the gateway log that the nightly
22
+ * fleet-health sensor (`src/fleet-health/scan.ts`) reads; the
23
+ * `handback orphan escalation` marker is greppable and carries the join key.
24
+ *
25
+ * Every effect is injected, so the contract is asserted by a unit test with spy
26
+ * transports (`tests/handback-orphan-recovery.test.ts`) rather than by reading
27
+ * gateway.ts.
28
+ */
29
+
30
+ import type { InboundMessage } from './ipc-protocol.js'
31
+ import type { HandbackOrphanEscalation, PreTurnCardRecord } from './handback-preturn-signal.js'
32
+
33
+ export interface HandbackOrphanRecoveryDeps {
34
+ /** Delete a message. The gateway supplies its `robustApiCall`-wrapped
35
+ * `bot.api.deleteMessage`; rejections are swallowed here (best-effort). */
36
+ deleteMessage: (chatId: string, messageId: number, threadId: number | null) => Promise<unknown>
37
+ /** The gateway's `pendingInboundBuffer.push`, pre-bound to this agent. */
38
+ pushInbound: (inbound: InboundMessage) => void
39
+ /** Telemetry sink. Defaults to the gateway log (stderr). */
40
+ writeLog?: (line: string) => void
41
+ }
42
+
43
+ /** Structured, greppable telemetry line for an exhausted-retry orphan. Exported
44
+ * so the test asserts the exact marker + join keys the fleet-health sensor
45
+ * greps for, not just that "something was logged". */
46
+ export function formatOrphanEscalation(esc: HandbackOrphanEscalation): string {
47
+ return (
48
+ `telegram gateway: handback orphan escalation key=${esc.statusKey} ` +
49
+ `turnId=${esc.adoptTurnId} reinjects=${esc.reinjectCount} ageMs=${esc.ageMs} ` +
50
+ `— deterministic recovery exhausted, no operator nudge issued\n`
51
+ )
52
+ }
53
+
54
+ export function createHandbackOrphanRecovery(deps: HandbackOrphanRecoveryDeps): {
55
+ deleteCard: (record: PreTurnCardRecord) => Promise<void>
56
+ reinjectHandback: (inbound: InboundMessage) => void
57
+ escalateOrphan: (escalation: HandbackOrphanEscalation) => void
58
+ } {
59
+ const writeLog = deps.writeLog ?? ((line: string) => void process.stderr.write(line))
60
+ return {
61
+ deleteCard: (record) =>
62
+ deps
63
+ .deleteMessage(record.chatId, record.activityMessageId, record.threadId)
64
+ .then(() => undefined)
65
+ .catch(() => undefined),
66
+ reinjectHandback: (inbound) => deps.pushInbound(inbound),
67
+ escalateOrphan: (esc) => writeLog(formatOrphanEscalation(esc)),
68
+ }
69
+ }
@@ -105,6 +105,29 @@ import type { InboundMessage } from './ipc-protocol.js'
105
105
  * the seam's `handleReaped` hook (stop typing loop + drop map entry). */
106
106
  export const PRETURN_TURNKEY_PREFIX = 'preturn:'
107
107
 
108
+ /** `meta` key carrying the deterministic-recovery re-injection counter across a
109
+ * re-injected handback's round trip through the pending-inbound buffer. Stamped
110
+ * on the inbound just before it is re-injected on a genuine orphan; read back
111
+ * when that same inbound is released again, so the retry cap survives the
112
+ * drop-and-recreate of the per-topic pre-turn entry. `meta` is
113
+ * `Record<string,string>`, so the count is stored as a decimal string. */
114
+ export const HANDBACK_REINJECT_COUNT_META_KEY = 'handbackReinjectCount'
115
+
116
+ /** Emitted (NOT to chat) when a genuine orphan has exhausted its re-injection
117
+ * retries — the deterministic-recovery escalation surface. The gateway routes
118
+ * this to fleet-health telemetry (a structured log line the nightly sensor
119
+ * reads), never to an operator-facing card: the system must recover itself, not
120
+ * ask a human to nudge it. */
121
+ export interface HandbackOrphanEscalation {
122
+ statusKey: string
123
+ chatId: string
124
+ threadId: number | null
125
+ adoptTurnId: string
126
+ reinjectCount: number
127
+ /** Age (ms) of the handback at escalation — release → escalation. */
128
+ ageMs: number
129
+ }
130
+
108
131
  /** True IFF `source` is the load-bearing subagent-handback source string. Kept
109
132
  * here next to the seam so a source-string regression is caught by this
110
133
  * module's own test, not only the inbound-builder's. */
@@ -162,9 +185,21 @@ export interface HandbackPreturnSignalDeps {
162
185
  * when the send failed / was suppressed (best-effort — a null just means no
163
186
  * card was painted, so nothing to adopt or reap). */
164
187
  openCard: (chatId: string, threadId: number | null) => Promise<number | null>
165
- /** Finalize (single honest edit) a frozen never-adopted pre-turn card on
166
- * orphan self-reap. Best-effort; failures swallowed by the caller. */
167
- finalizeCard: (record: PreTurnCardRecord) => void | Promise<void>
188
+ /** DELETE (not edit) a never-adopted pre-turn card. Used on the genuine-orphan
189
+ * reap so the frozen card is removed rather than rewritten to a stale
190
+ * user-facing "needs a nudge" message, and on the rare emit-race teardown of a
191
+ * card the adopting turn already owns. Best-effort; failures swallowed by the
192
+ * caller. */
193
+ deleteCard: (record: PreTurnCardRecord) => void | Promise<void>
194
+ /** Re-inject a genuinely-orphaned handback back through the pending-inbound
195
+ * buffer so the machine recovers itself — the deterministic alternative to
196
+ * asking the operator to nudge. The gateway's `pendingInboundBuffer.push`. The
197
+ * seam stamps `HANDBACK_REINJECT_COUNT_META_KEY` on `inbound.meta` first so the
198
+ * retry cap survives the round trip. */
199
+ reinjectHandback: (inbound: InboundMessage) => void
200
+ /** Escalate a genuine orphan that has exhausted its re-injection retries to
201
+ * fleet-health telemetry (NOT a chat card). */
202
+ escalateOrphan: (escalation: HandbackOrphanEscalation) => void
168
203
  /** Persist the durable card record (the gateway's `writeActivityCardRecord`,
169
204
  * scoped by `turnKey`). */
170
205
  writeCardRecord: (record: PreTurnCardRecord) => void
@@ -190,16 +225,18 @@ export interface HandbackPreturnSignalDeps {
190
225
  * handback is delivered into that single queue and only mints its adopting
191
226
  * turn when claude next goes idle, so while claude is busy the handback is
192
227
  * NOT orphaned — it is correctly enqueued BEHIND the in-flight turn (a long
193
- * parent turn can run many minutes). Finalizing it as an orphan in that
194
- * window posts a false "never started — needs a nudge" card. The reap
195
- * therefore DEFERS (re-arms) while this is true and finalizes only once
196
- * claude is idle and the handback still has not been adopted the genuine
197
- * bridge-death / dropped-inbound degenerate case the reap exists for.
198
- * Distinct from `hasLiveTurn`, which is PER-KEY and only governs the
199
- * typing-loop stop; this is the GLOBAL busy signal because the handback
200
- * queues behind whatever turn is running, regardless of topic. Optional;
201
- * defaults to "not busy" (immediate reap the pre-fix behaviour), so the
202
- * genuine-orphan path is unchanged. */
228
+ * parent turn can run many minutes). Reaping it as an orphan in that window
229
+ * used to post a false "never started — needs a nudge" card (the 3-cards-in-
230
+ * 4-min incident). The reap therefore DEFERS (re-arms) while this is true
231
+ * and, complementarily, while the handback was delivered but its turn has not
232
+ * started streaming yet (`deliveredAwaitingTurnStart`, the #4027 coverage
233
+ * gap) recovering only once claude is idle AND the delivered window has
234
+ * elapsed AND the handback still has not been adopted (the genuine bridge-
235
+ * death / dropped-inbound case). Recovery is deterministic (delete the frozen
236
+ * card + re-inject, then escalate), never an operator nudge. Distinct from
237
+ * `hasLiveTurn`, which is PER-KEY and only governs the typing-loop stop; this
238
+ * is the GLOBAL busy signal because the handback queues behind whatever turn
239
+ * is running, regardless of topic. Optional; defaults to "not busy". */
203
240
  isClaudeBusy?: () => boolean
204
241
  now?: () => number
205
242
  /** Debounce before painting the pre-turn card (kills sub-second flicker). */
@@ -207,6 +244,17 @@ export interface HandbackPreturnSignalDeps {
207
244
  /** Age after emit at which a never-adopted card self-reaps. Independent of
208
245
  * topic liveness. */
209
246
  adoptTimeoutMs?: number
247
+ /** TTL (ms) for the delivered-but-not-yet-streaming window. A released
248
+ * handback has been DELIVERED into claude's queue but its adopting turn may
249
+ * not have started streaming yet — the false-orphan window #4027 left open
250
+ * (idle machine + un-adopted = looked like an orphan). While within this TTL
251
+ * the reap DEFERS (`deliveredAwaitingTurnStart`); past it a truly-stuck
252
+ * delivery finally reaps for deterministic recovery rather than hanging
253
+ * forever. Defaults to 5 min. */
254
+ deliveredTtlMs?: number
255
+ /** Max deterministic re-injections of a genuine orphan before escalating to
256
+ * fleet-health telemetry. Defaults to 2. */
257
+ maxReinjects?: number
210
258
  /** Injected scheduler (mirrors bridge-dead-watchdog.ts): defaults to
211
259
  * setTimeout/clearTimeout. Injected so a runner-agnostic test can drive the
212
260
  * debounce + self-reap deterministically WITHOUT any vitest-only fake-timer
@@ -224,6 +272,18 @@ interface PreTurnEntry {
224
272
  adoptTurnId: string
225
273
  syntheticTurnKey: string
226
274
  startedAt: number
275
+ /** When the handback was RELEASED/DELIVERED for delivery (stamped at
276
+ * `noteHandbackRelease`). Drives `deliveredAwaitingTurnStart`: the reap must
277
+ * not treat a delivered-but-not-yet-streaming handback as an orphan within
278
+ * the delivered TTL. Equal to `startedAt` (release is the delivery point). */
279
+ deliveredAt: number
280
+ /** How many times this handback has already been deterministically re-injected
281
+ * (carried across the buffer round trip via `HANDBACK_REINJECT_COUNT_META_KEY`
282
+ * on the inbound). Governs the retry cap before fleet-health escalation. */
283
+ reinjectCount: number
284
+ /** The released handback inbound itself — retained so a genuine orphan can be
285
+ * re-injected through the pending-inbound buffer (deterministic recovery). */
286
+ inbound: InboundMessage
227
287
  pinned: boolean
228
288
  debounceTimer: unknown | null
229
289
  reapTimer: unknown | null
@@ -273,6 +333,8 @@ export function createHandbackPreturnSignal(
273
333
  const now = deps.now ?? (() => Date.now())
274
334
  const debounceMs = deps.debounceMs ?? 700
275
335
  const adoptTimeoutMs = deps.adoptTimeoutMs ?? 30_000
336
+ const deliveredTtlMs = deps.deliveredTtlMs ?? 300_000
337
+ const maxReinjects = deps.maxReinjects ?? 2
276
338
  const setTimer =
277
339
  deps.setTimer ??
278
340
  ((fn: () => void, ms: number) => {
@@ -333,9 +395,9 @@ export function createHandbackPreturnSignal(
333
395
  if (messageId == null) return // no card painted; the reap timer cleans up
334
396
  if (entry.consumed) {
335
397
  // Adopted/reaped while the send was in flight: the card is now the
336
- // turn's (adoption seeds a null id it can't use) — finalize+clear so
337
- // it never dangles. Rare; the debounce makes it unlikely.
338
- void deps.finalizeCard({
398
+ // turn's (adoption seeds a null id it can't use) — DELETE it so it
399
+ // never dangles. Rare; the debounce makes it unlikely.
400
+ void deps.deleteCard({
339
401
  turnKey: entry.syntheticTurnKey,
340
402
  chatId: entry.chatId,
341
403
  threadId: entry.threadId,
@@ -385,30 +447,52 @@ export function createHandbackPreturnSignal(
385
447
  deps.stopTypingLoop(entry.chatId, entry.threadId)
386
448
  }
387
449
 
450
+ /**
451
+ * The delivered-but-not-yet-streaming window (#4027 coverage gap). A released
452
+ * handback has been DELIVERED into claude's single input queue, but its
453
+ * adopting turn may not have started streaming yet — and in that stretch the
454
+ * machine reads as idle (`isClaudeBusy()` false) with the entry un-adopted,
455
+ * which looked exactly like a genuine orphan and produced the false "needs a
456
+ * nudge" card even after #4027 gated on `isMachineInTurn`. While the handback
457
+ * is within the delivered TTL and its turn has not started (we are in `reap`,
458
+ * so the entry is un-consumed = no adopting turn minted), it is NOT orphaned —
459
+ * defer. Past the TTL a truly-stuck delivery finally reaps rather than hanging
460
+ * forever, handing off to deterministic recovery.
461
+ */
462
+ function deliveredAwaitingTurnStart(entry: PreTurnEntry): boolean {
463
+ return now() - entry.deliveredAt < deliveredTtlMs
464
+ }
465
+
388
466
  function reap(entry: PreTurnEntry): void {
389
467
  entry.reapTimer = null
390
468
  if (entry.consumed) return
391
469
  // PRIMARY GATE (queue state, not a flat timeout): a released handback is
392
470
  // delivered into claude's SINGLE input queue and mints its adopting turn
393
- // only when claude next goes idle. If claude is STILL in a turn, this
394
- // handback is not orphaned it is correctly enqueued behind the in-flight
395
- // turn (a long parent turn can run many minutes). Finalizing now would post
396
- // a false "never started — needs a nudge" card (the 3-cards-in-4-min
397
- // incident). Defer: re-arm and re-check. We finalize only once claude is
398
- // idle AND the handback still has not been adopted (the genuine bridge-
399
- // death / dropped-inbound case). This cannot loop forever on a healthy
400
- // agent: claude processes its queue FIFO, so the handback is either adopted
401
- // (cancelling the reap) or claude eventually goes idle (letting it fire).
402
- if (deps.isClaudeBusy?.() === true) {
471
+ // only when claude next goes idle. If claude is STILL in a turn, OR the
472
+ // handback was delivered and its turn has simply not started streaming yet
473
+ // (within the delivered TTL the #4027 coverage gap), the handback is NOT
474
+ // orphaned. Reaping now would have posted a false "never started — needs a
475
+ // nudge" card (the 3-cards-in-4-min incident). Defer: re-arm and re-check.
476
+ // We proceed to recovery only once claude is idle AND the delivered window
477
+ // has elapsed AND the handback still has not been adopted (the genuine
478
+ // bridge-death / dropped-inbound case). This cannot loop forever: claude
479
+ // processes its queue FIFO (adoption cancels the reap), and the delivered
480
+ // TTL is bounded, so a stuck delivery eventually recovers deterministically.
481
+ if (deps.isClaudeBusy?.() === true || deliveredAwaitingTurnStart(entry)) {
403
482
  log(
404
483
  `handback-preturn-signal: reap deferred key=${entry.statusKey} ` +
405
- `(claude busy handback enqueued behind an in-flight turn, not orphaned)\n`,
484
+ `(${deps.isClaudeBusy?.() === true ? 'claude busy' : 'delivered, turn not yet streaming'}` +
485
+ ` — not orphaned)\n`,
406
486
  )
407
487
  entry.reapTimer = setTimer(() => reap(entry), adoptTimeoutMs)
408
488
  return
409
489
  }
410
490
  entry.consumed = true
411
- // Stop the forever-running typing loop and finalize the frozen card.
491
+ // GENUINE ORPHAN. Recover DETERMINISTICALLY never ask the operator to
492
+ // nudge. Stop the forever-running typing loop, DELETE the frozen card (do
493
+ // not leave a stale "needs a nudge" message), then re-inject the handback so
494
+ // the machine re-processes it itself. Cap the re-injections; past the cap
495
+ // escalate to fleet-health telemetry (NOT a chat card).
412
496
  stopTypingUnlessTurnLive(entry, 'orphan reap')
413
497
  if (entry.activityMessageId != null) {
414
498
  const record: PreTurnCardRecord = {
@@ -419,17 +503,61 @@ export function createHandbackPreturnSignal(
419
503
  startedAt: entry.startedAt,
420
504
  pinned: entry.pinned,
421
505
  }
422
- // Clear the durable record BEFORE the finalizing edit (at-most-once
423
- // idempotency guard, mirroring the activity-card-store reapers).
506
+ // Clear the durable record BEFORE the delete (at-most-once idempotency
507
+ // guard, mirroring the activity-card-store reapers).
424
508
  deps.clearCardRecord(entry.syntheticTurnKey, entry.activityMessageId)
425
- void Promise.resolve(deps.finalizeCard(record)).catch((err) => {
509
+ void Promise.resolve(deps.deleteCard(record)).catch((err) => {
426
510
  log(
427
- `handback-preturn-signal: orphan finalize failed key=${entry.statusKey}: ` +
511
+ `handback-preturn-signal: orphan card delete failed key=${entry.statusKey}: ` +
428
512
  `${err instanceof Error ? err.message : String(err)}\n`,
429
513
  )
430
514
  })
431
515
  }
516
+ // Drop the entry from the topic map BEFORE re-injecting so a redelivery that
517
+ // re-enters `noteHandbackRelease` (production: async buffer drain; possible
518
+ // synchronously) is not swallowed by the per-topic dedupe guard. The entry
519
+ // object stays valid for the captured re-inject below.
432
520
  dropEntry(entry)
521
+ if (entry.reinjectCount < maxReinjects) {
522
+ const nextCount = entry.reinjectCount + 1
523
+ // Stamp the counter on the inbound so it survives the buffer round trip
524
+ // (the re-injected inbound re-enters `noteHandbackRelease`, which reads it
525
+ // back — the retry cap outlives the drop-and-recreate of this entry).
526
+ entry.inbound.meta[HANDBACK_REINJECT_COUNT_META_KEY] = String(nextCount)
527
+ log(
528
+ `handback-preturn-signal: orphan reap key=${entry.statusKey} ` +
529
+ `turnId=${entry.adoptTurnId} — re-injecting handback (attempt ${nextCount}/${maxReinjects})\n`,
530
+ )
531
+ try {
532
+ deps.reinjectHandback(entry.inbound)
533
+ } catch (err) {
534
+ log(
535
+ `handback-preturn-signal: orphan re-inject failed key=${entry.statusKey}: ` +
536
+ `${err instanceof Error ? err.message : String(err)}\n`,
537
+ )
538
+ }
539
+ } else {
540
+ log(
541
+ `handback-preturn-signal: orphan reap key=${entry.statusKey} ` +
542
+ `turnId=${entry.adoptTurnId} — re-injection cap (${maxReinjects}) exhausted, ` +
543
+ `escalating to fleet-health telemetry\n`,
544
+ )
545
+ try {
546
+ deps.escalateOrphan({
547
+ statusKey: entry.statusKey,
548
+ chatId: entry.chatId,
549
+ threadId: entry.threadId,
550
+ adoptTurnId: entry.adoptTurnId,
551
+ reinjectCount: entry.reinjectCount,
552
+ ageMs: now() - entry.deliveredAt,
553
+ })
554
+ } catch (err) {
555
+ log(
556
+ `handback-preturn-signal: orphan escalation failed key=${entry.statusKey}: ` +
557
+ `${err instanceof Error ? err.message : String(err)}\n`,
558
+ )
559
+ }
560
+ }
433
561
  }
434
562
 
435
563
  return {
@@ -466,6 +594,12 @@ export function createHandbackPreturnSignal(
466
594
  )
467
595
  const startedAt = now()
468
596
  const syntheticTurnKey = `${PRETURN_TURNKEY_PREFIX}${statusKey}:${startedAt}`
597
+ // Carry the re-injection counter across the buffer round trip: a genuine
598
+ // orphan re-injects this inbound, which re-enters here with the counter
599
+ // stamped on `meta`, so the retry cap survives the entry's recreate.
600
+ const rawReinject = inbound.meta?.[HANDBACK_REINJECT_COUNT_META_KEY]
601
+ const parsedReinject = rawReinject != null ? Number.parseInt(rawReinject, 10) : 0
602
+ const reinjectCount = Number.isFinite(parsedReinject) && parsedReinject > 0 ? parsedReinject : 0
469
603
  const entry: PreTurnEntry = {
470
604
  statusKey,
471
605
  chatId,
@@ -473,6 +607,10 @@ export function createHandbackPreturnSignal(
473
607
  adoptTurnId,
474
608
  syntheticTurnKey,
475
609
  startedAt,
610
+ // Release IS the delivery point (this is the buffer-drain chokepoint).
611
+ deliveredAt: startedAt,
612
+ reinjectCount,
613
+ inbound,
476
614
  pinned: false,
477
615
  debounceTimer: null,
478
616
  reapTimer: null,
@@ -179,6 +179,7 @@ export type GatewayToClient =
179
179
  | Ms365ApprovalPostedEvent
180
180
  | ConfigApprovalResolvedEvent
181
181
  | RolloutStatusPostedEvent
182
+ | RolloutStatusEditedEvent
182
183
  | PendingPermissionStatusEvent
183
184
  | PreApprovedResultEvent;
184
185
 
@@ -567,9 +568,13 @@ export interface RolloutStatusPostMessage {
567
568
  /**
568
569
  * #2726 Part 2 — hostd asks the gateway to EDIT the rollout status message it
569
570
  * previously posted (identified by `messageId`, returned in the
570
- * `rollout_status_posted` reply). Best-effort, fire-and-forget: an edit failure
571
- * (incl. Telegram 429) is handled gateway-side and never surfaced back toward
572
- * the roll.
571
+ * `rollout_status_posted` reply). Best-effort toward the ROLL: hostd never
572
+ * blocks on it and Telegram 429s are retried gateway-side.
573
+ *
574
+ * #4065 — the gateway now replies with `rollout_status_edited` so hostd can
575
+ * tell "edited" from "edited into the void". A seeded-resume narrator holds a
576
+ * message_id it never posted itself; without the reply a stale id means the
577
+ * operator sees a frozen card for the rest of the roll.
573
578
  */
574
579
  export interface RolloutStatusEditMessage {
575
580
  type: "rollout_status_edit";
@@ -597,6 +602,29 @@ export interface RolloutStatusPostedEvent {
597
602
  reason?: string;
598
603
  }
599
604
 
605
+ /**
606
+ * #4065 — gateway → hostd reply after a `rollout_status_edit`, so a narrator
607
+ * editing a card it did NOT post (the seeded post-self-bump resume) can tell
608
+ * that the card is gone instead of editing into the void for the rest of the
609
+ * roll. Advisory only: hostd never blocks a roll on it, and a gateway that
610
+ * predates this event simply never sends one (hostd times out and behaves as
611
+ * before).
612
+ */
613
+ export interface RolloutStatusEditedEvent {
614
+ type: "rollout_status_edited";
615
+ requestId: string;
616
+ /** True when the live card now carries the requested body. */
617
+ ok: boolean;
618
+ /**
619
+ * True IFF the failure means the target message no longer exists / can no
620
+ * longer be edited. ONLY this justifies a re-post — a transient failure
621
+ * (429 past retries, network) leaves a perfectly good card in the chat.
622
+ */
623
+ gone?: boolean;
624
+ /** Diagnostic detail on failure. */
625
+ reason?: string;
626
+ }
627
+
600
628
  /**
601
629
  * Issue #2971 — sent by the wedge-watchdog (autoaccept-poll sidecar) over
602
630
  * the gateway UDS, BEFORE it would otherwise Esc a shape-persistent