typeclaw 0.47.2 → 0.48.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. package/package.json +1 -1
  2. package/src/agent/index.ts +1 -2
  3. package/src/agent/loop-guard.ts +71 -41
  4. package/src/agent/plugin-tools.ts +167 -50
  5. package/src/agent/system-prompt.ts +1 -1
  6. package/src/agent/turn-runner.ts +12 -2
  7. package/src/bundled-plugins/agent-browser/index.ts +0 -38
  8. package/src/bundled-plugins/agent-browser/skills/agent-browser/SKILL.md +1 -1
  9. package/src/bundled-plugins/guard/policies/non-workspace-write.ts +16 -1
  10. package/src/bundled-plugins/memory/append-tool.ts +75 -2
  11. package/src/bundled-plugins/memory/frontmatter.ts +13 -2
  12. package/src/bundled-plugins/memory/load-memory.ts +1 -1
  13. package/src/bundled-plugins/memory/memory-logger.ts +5 -1
  14. package/src/bundled-plugins/memory/secret-detector.ts +7 -0
  15. package/src/bundled-plugins/memory/vector/hybrid.ts +3 -1
  16. package/src/bundled-plugins/memory/vector/passages.ts +5 -1
  17. package/src/bundled-plugins/security/index.ts +148 -98
  18. package/src/bundled-plugins/security/policies/private-surface-read.ts +43 -28
  19. package/src/channels/adapters/discord-bot-edit.ts +3 -5
  20. package/src/channels/adapters/discord-bot-reactions.ts +4 -6
  21. package/src/channels/adapters/discord-bot-slash-commands.ts +3 -1
  22. package/src/channels/adapters/discord-bot.ts +20 -23
  23. package/src/channels/adapters/discord-edit.ts +3 -5
  24. package/src/channels/adapters/discord-reactions.ts +4 -6
  25. package/src/channels/adapters/discord.ts +1 -1
  26. package/src/channels/adapters/github/decoy-reviewer.ts +2 -1
  27. package/src/channels/adapters/github/history.ts +2 -1
  28. package/src/channels/adapters/github/inbound.ts +4 -11
  29. package/src/channels/adapters/github/index.ts +6 -9
  30. package/src/channels/adapters/github/outbound.ts +3 -6
  31. package/src/channels/adapters/github/reactions.ts +3 -2
  32. package/src/channels/adapters/github/reconcile-cooldown-store.ts +4 -6
  33. package/src/channels/adapters/github/reconcile-open-prs.ts +5 -4
  34. package/src/channels/adapters/github/recover-failed-deliveries.ts +2 -5
  35. package/src/channels/adapters/github/review-state.ts +3 -2
  36. package/src/channels/adapters/github/review-thread-resolver.ts +3 -2
  37. package/src/channels/adapters/github/webhook-register.ts +5 -8
  38. package/src/channels/adapters/instagram-channel-resolver.ts +3 -5
  39. package/src/channels/adapters/instagram.ts +1 -1
  40. package/src/channels/adapters/kakaotalk-author-resolver.ts +2 -5
  41. package/src/channels/adapters/kakaotalk-channel-resolver.ts +3 -5
  42. package/src/channels/adapters/kakaotalk-fetch-attachment.ts +2 -1
  43. package/src/channels/adapters/kakaotalk-membership.ts +2 -5
  44. package/src/channels/adapters/kakaotalk-typing.ts +1 -1
  45. package/src/channels/adapters/kakaotalk.ts +1 -1
  46. package/src/channels/adapters/line-channel-resolver.ts +3 -5
  47. package/src/channels/adapters/line.ts +1 -1
  48. package/src/channels/adapters/slack-bot-edit.ts +1 -1
  49. package/src/channels/adapters/slack-bot-reactions.ts +4 -6
  50. package/src/channels/adapters/slack-bot.ts +8 -8
  51. package/src/channels/adapters/slack-edit.ts +3 -5
  52. package/src/channels/adapters/slack-reactions.ts +4 -6
  53. package/src/channels/adapters/slack.ts +1 -1
  54. package/src/channels/adapters/teams-edit.ts +2 -5
  55. package/src/channels/adapters/teams.ts +1 -1
  56. package/src/channels/adapters/telegram-bot-edit.ts +4 -7
  57. package/src/channels/adapters/telegram-bot.ts +1 -1
  58. package/src/channels/adapters/webex-bot.ts +1 -1
  59. package/src/channels/adapters/webex-edit.ts +3 -5
  60. package/src/channels/adapters/webex.ts +1 -1
  61. package/src/channels/github-review-claim.ts +89 -4
  62. package/src/channels/github-token-bridge.ts +2 -1
  63. package/src/channels/manager.ts +478 -64
  64. package/src/channels/membership-cache.ts +2 -5
  65. package/src/channels/persistence.ts +3 -6
  66. package/src/channels/reloadable.ts +23 -6
  67. package/src/channels/router.ts +134 -56
  68. package/src/channels/tunnel-bridge.ts +2 -1
  69. package/src/channels/types.ts +6 -3
  70. package/src/cli/hostd.ts +56 -28
  71. package/src/config/config.ts +10 -0
  72. package/src/config/providers.ts +179 -0
  73. package/src/dependencies/bin-resolver.ts +88 -0
  74. package/src/dependencies/bins.ts +227 -0
  75. package/src/dependencies/index.ts +17 -0
  76. package/src/doctor/checks.ts +2 -0
  77. package/src/doctor/operational-incidents.ts +32 -0
  78. package/src/hostd/credential-rotation-apply.ts +94 -0
  79. package/src/init/models-dev.ts +17 -3
  80. package/src/init/validate-api-key.ts +26 -4
  81. package/src/operations/classify-tool-outcome.ts +151 -0
  82. package/src/operations/incidents.ts +274 -0
  83. package/src/operations/index.ts +3 -0
  84. package/src/operations/remediations.ts +41 -0
  85. package/src/reload/client.ts +8 -2
  86. package/src/reload/docker-exec-client.ts +10 -2
  87. package/src/reload/index.ts +1 -1
  88. package/src/reload/recover.ts +5 -3
  89. package/src/reload/registry.ts +5 -5
  90. package/src/reload/types.ts +9 -1
  91. package/src/run/index.ts +5 -0
  92. package/src/sandbox/dependency-bins.ts +125 -0
  93. package/src/sandbox/errors.ts +32 -0
  94. package/src/sandbox/hidden-paths.ts +4 -1
  95. package/src/sandbox/index.ts +8 -0
  96. package/src/sandbox/policy.ts +1 -1
  97. package/src/sandbox/privileged-runtime.ts +88 -7
  98. package/src/sandbox/runtime-private.ts +4 -0
  99. package/src/sandbox/writable-zones.ts +4 -4
  100. package/src/server/index.ts +7 -5
  101. package/src/shared/protocol.ts +6 -1
  102. package/typeclaw.schema.json +35 -5
  103. package/src/bundled-plugins/agent-browser/shim-install.ts +0 -179
  104. package/src/bundled-plugins/agent-browser/shim.ts +0 -108
  105. /package/src/channels/{adapters/describe-error.ts → describe-error.ts} +0 -0
@@ -26,6 +26,7 @@ import { createTeamsAdapter, type TeamsAdapter } from './adapters/teams'
26
26
  import { createTelegramBotAdapter, type TelegramBotAdapter } from './adapters/telegram-bot'
27
27
  import { createWebexAdapter, type WebexAdapter } from './adapters/webex'
28
28
  import { createWebexBotAdapter, type WebexBotAdapter } from './adapters/webex-bot'
29
+ import { describeError } from './describe-error'
29
30
  import type { GithubTokenBridge } from './github-token-bridge'
30
31
  import {
31
32
  createChannelRouter,
@@ -145,7 +146,14 @@ export type ChannelManagerOptions = {
145
146
  connectionRecovery?: {
146
147
  checkIntervalMs?: number
147
148
  disconnectedGraceMs?: number
149
+ // Base delay for the failed-adapter retry backoff (`retryBaseMs * 2^(n-1)`,
150
+ // capped at 15 minutes, jittered). Deliberately independent of
151
+ // `checkIntervalMs`: a retry deadline is only observed on a tick, so a tick
152
+ // as coarse as the base would round a jittered 30-36s deadline up to the
153
+ // 60s tick and silently double every early delay.
154
+ retryBaseMs?: number
148
155
  now?: () => number
156
+ random?: () => number
149
157
  setInterval?: (fn: () => void, ms: number) => unknown
150
158
  clearInterval?: (handle: unknown) => void
151
159
  }
@@ -156,7 +164,7 @@ export type ChannelManager = {
156
164
  start: () => Promise<void>
157
165
  stop: () => Promise<void>
158
166
  restartAdapter: (name: AdapterId) => Promise<void>
159
- reload: () => Promise<{ started: string[]; stopped: string[]; restartRequired: string[] }>
167
+ reload: (options?: ChannelReloadOptions) => Promise<ChannelReloadDiff>
160
168
  }
161
169
 
162
170
  type AnyAdapter =
@@ -187,6 +195,47 @@ type AdapterEntry = {
187
195
  recoveryRestartQueued: boolean
188
196
  }
189
197
 
198
+ type FailedAdapter = {
199
+ kind: 'start-failed' | 'missing-credentials'
200
+ attempts: number
201
+ failedSinceMs: number
202
+ nextAttemptAtMs: number
203
+ retryQueued: boolean
204
+ nextLogAtMs: number
205
+ }
206
+
207
+ export type AdapterRestartOutcome = 'restarted' | 'recovery-pending' | 'stop-failed' | 'skipped'
208
+
209
+ // `already-current` is not a restart outcome: the live adapter is already on the
210
+ // new credential, so there was nothing to apply. It still has to be reported,
211
+ // because a named adapter that reports nothing reads as "not applied" and sends
212
+ // the caller to its fallback.
213
+ export type CredentialApplyOutcome = AdapterRestartOutcome | 'already-current'
214
+
215
+ export type ChannelReloadOptions = { applyCredentialRotation?: AdapterId }
216
+
217
+ export type ChannelReloadDiff = {
218
+ started: string[]
219
+ stopped: string[]
220
+ restarted: string[]
221
+ restartRequired: string[]
222
+ credentialApply?: { adapter: AdapterId; outcome: CredentialApplyOutcome }
223
+ }
224
+
225
+ type ReloadAdapterOutcome = 'started' | 'stopped' | 'rotated' | 'already-current' | AdapterRestartOutcome | null
226
+
227
+ type StartAdapterResult =
228
+ | { status: 'started' }
229
+ | { status: 'disabled' }
230
+ | { status: 'blocked' }
231
+ | { status: 'aborted' }
232
+ | {
233
+ status: 'failed'
234
+ kind: FailedAdapter['kind']
235
+ detail: string
236
+ inputSignature: string
237
+ }
238
+
190
239
  export function createChannelManager(options: ChannelManagerOptions): ChannelManager {
191
240
  const logger = options.logger ?? consoleLogger
192
241
  const env = options.env ?? process.env
@@ -222,15 +271,20 @@ export function createChannelManager(options: ChannelManagerOptions): ChannelMan
222
271
  const createWebexBot = options.createWebexBotAdapter ?? createWebexBotAdapter
223
272
 
224
273
  const live = new Map<AdapterId, AdapterEntry>()
274
+ const failed = new Map<AdapterId, FailedAdapter>()
275
+ const failedInputSignatures = new WeakMap<FailedAdapter, string>()
225
276
  const perAdapterSerial = new Map<AdapterId, Promise<unknown>>()
226
277
  const recovery = options.connectionRecovery ?? {}
227
- const recoveryCheckIntervalMs = recovery.checkIntervalMs ?? 30_000
278
+ const recoveryCheckIntervalMs = recovery.checkIntervalMs ?? 5_000
228
279
  const recoveryDisconnectedGraceMs = recovery.disconnectedGraceMs ?? 90_000
229
280
  const recoveryNow = recovery.now ?? (() => Date.now())
281
+ const recoveryRandom = recovery.random ?? Math.random
230
282
  const recoverySetInterval = recovery.setInterval ?? ((fn: () => void, ms: number) => setInterval(fn, ms))
231
283
  const recoveryClearInterval =
232
284
  recovery.clearInterval ?? ((handle: unknown) => clearInterval(handle as ReturnType<typeof setInterval>))
233
285
  let recoveryTimer: unknown = null
286
+ let running = false
287
+ let lifecycleEpoch = 0
234
288
 
235
289
  const runSerially = <T>(name: AdapterId, op: () => Promise<T>): Promise<T> => {
236
290
  const prev = perAdapterSerial.get(name) ?? Promise.resolve()
@@ -407,50 +461,263 @@ export function createChannelManager(options: ChannelManagerOptions): ChannelMan
407
461
  return null
408
462
  }
409
463
 
410
- const startAdapter = async (name: AdapterId, cfg: ChannelAdapterConfig): Promise<boolean> => {
464
+ const buildFailedInputSignature = (
465
+ name: AdapterId,
466
+ cfg: ChannelAdapterConfig,
467
+ credentials = buildCredentialSignature(name),
468
+ ): string => JSON.stringify([cfg, credentials.signature, credentials.missing])
469
+
470
+ const cleanupPartialStart = async (adapter: AnyAdapter): Promise<void> => {
471
+ try {
472
+ await adapter.stop()
473
+ } catch {}
474
+ }
475
+
476
+ const startAdapterOnce = async (
477
+ name: AdapterId,
478
+ cfg: ChannelAdapterConfig,
479
+ canCommit: () => boolean,
480
+ ): Promise<StartAdapterResult | { status: 'credential-changed' }> => {
481
+ if (!canCommit()) return { status: 'aborted' }
411
482
  if (cfg.enabled === false) {
412
483
  logger.info(`[channels] adapter "${name}" is disabled; skipping`)
413
- return false
484
+ return { status: 'disabled' }
414
485
  }
415
- const { signature, missing } = buildCredentialSignature(name)
486
+ const credentials = buildCredentialSignature(name)
487
+ const { signature, missing } = credentials
416
488
  if (missing.length > 0) {
417
- logger.error(`[channels] adapter "${name}" missing credentials: ${missing.join(', ')}; skipping`)
418
- return false
489
+ return {
490
+ status: 'failed',
491
+ kind: 'missing-credentials',
492
+ detail: `missing credentials: ${missing.join(', ')}`,
493
+ inputSignature: buildFailedInputSignature(name, cfg, credentials),
494
+ }
419
495
  }
420
496
  const adapter = buildAdapter(name, cfg)
421
497
  if (adapter === null) {
422
498
  logger.error(`[channels] adapter "${name}" could not be constructed; skipping`)
423
- return false
499
+ return { status: 'blocked' }
424
500
  }
425
501
  try {
426
502
  await adapter.start()
503
+ if (!canCommit()) {
504
+ await cleanupPartialStart(adapter)
505
+ return { status: 'aborted' }
506
+ }
507
+ if (buildCredentialSignature(name).signature !== signature) {
508
+ await cleanupPartialStart(adapter)
509
+ return { status: 'credential-changed' }
510
+ }
427
511
  live.set(name, {
428
512
  adapter,
429
513
  credentialSignature: signature,
430
514
  disconnectedSinceMs: adapter.isConnected() ? null : recoveryNow(),
431
515
  recoveryRestartQueued: false,
432
516
  })
433
- logger.info(`[channels] adapter "${name}" started`)
434
- return true
517
+ return { status: 'started' }
435
518
  } catch (err) {
436
- logger.error(`[channels] adapter "${name}" failed to start: ${describe(err)}`)
437
- return false
519
+ await cleanupPartialStart(adapter)
520
+ if (!canCommit()) return { status: 'aborted' }
521
+ return {
522
+ status: 'failed',
523
+ kind: 'start-failed',
524
+ detail: `failed to start: ${describeError(err)}`,
525
+ inputSignature: buildFailedInputSignature(name, cfg, credentials),
526
+ }
527
+ }
528
+ }
529
+
530
+ // The adapter reads the credential file itself inside start(), so a write
531
+ // landing mid-start would leave `live` holding a signature that describes a
532
+ // different token than the one now in memory: the entry looks current while
533
+ // running a superseded credential, and the next reload sees no rotation to
534
+ // apply. Retry once, which covers a renewal write racing a start; a second
535
+ // change means a persistent concurrent writer, and spinning on it is worse
536
+ // than handing the adapter to supervision's backoff.
537
+ const startAdapter = async (
538
+ name: AdapterId,
539
+ cfg: ChannelAdapterConfig,
540
+ canCommit: () => boolean,
541
+ ): Promise<StartAdapterResult> => {
542
+ for (let attempt = 0; ; attempt += 1) {
543
+ const result = await startAdapterOnce(name, cfg, canCommit)
544
+ if (result.status !== 'credential-changed') return result
545
+ if (attempt >= 1) {
546
+ return {
547
+ status: 'failed',
548
+ kind: 'start-failed',
549
+ detail: 'credentials changed twice during start',
550
+ inputSignature: buildFailedInputSignature(name, cfg, buildCredentialSignature(name)),
551
+ }
552
+ }
553
+ }
554
+ }
555
+
556
+ const retryCapMs = 15 * 60 * 1_000
557
+ const retryHeartbeatMs = 6 * 60 * 60 * 1_000
558
+ const retryBaseMs = Math.max(1, recovery.retryBaseMs ?? 30_000)
559
+
560
+ const nominalRetryDelayMs = (attempts: number): number =>
561
+ Math.min(retryCapMs, retryBaseMs * 2 ** Math.min(30, attempts - 1))
562
+
563
+ const retryDelayMs = (attempts: number): number => {
564
+ const nominal = nominalRetryDelayMs(attempts)
565
+ const spread = nominal * 0.4
566
+ // A deadline is only ever observed on a supervision tick, which rounds it
567
+ // UP by as much as one interval. Narrow the jitter spread by that interval
568
+ // so the OBSERVED retry still lands inside the advertised ±20% window
569
+ // instead of up to a tick beyond it. When the tick is coarser than the
570
+ // window there is no room to compensate, so fall back to plain jitter and
571
+ // let quantization dominate rather than collapsing every delay to the floor.
572
+ const usableSpread = recoveryCheckIntervalMs < spread ? spread - recoveryCheckIntervalMs : spread
573
+ return Math.min(retryCapMs, Math.floor(nominal * 0.8 + recoveryRandom() * usableSpread))
574
+ }
575
+
576
+ const recordFailure = (
577
+ name: AdapterId,
578
+ cfg: ChannelAdapterConfig,
579
+ failure: Extract<StartAdapterResult, { status: 'failed' }>,
580
+ previous?: FailedAdapter,
581
+ ): FailedAdapter => {
582
+ const now = recoveryNow()
583
+ const continuing = previous !== undefined && failed.get(name) === previous
584
+ const attempts = continuing ? previous.attempts + 1 : 1
585
+ const delayMs = retryDelayMs(attempts)
586
+ const nominalDelayMs = nominalRetryDelayMs(attempts)
587
+ const previousNominalDelayMs = attempts > 1 ? nominalRetryDelayMs(attempts - 1) : 0
588
+ const entry: FailedAdapter = continuing
589
+ ? previous
590
+ : {
591
+ kind: failure.kind,
592
+ attempts,
593
+ failedSinceMs: now,
594
+ nextAttemptAtMs: now + delayMs,
595
+ retryQueued: false,
596
+ nextLogAtMs: Number.POSITIVE_INFINITY,
597
+ }
598
+
599
+ entry.kind = failure.kind
600
+ entry.attempts = attempts
601
+ entry.nextAttemptAtMs = now + delayMs
602
+ entry.retryQueued = false
603
+
604
+ const reachedCap = nominalDelayMs === retryCapMs
605
+ const justReachedCap = reachedCap && previousNominalDelayMs < retryCapMs
606
+ if (justReachedCap) entry.nextLogAtMs = now + retryHeartbeatMs
607
+
608
+ if (attempts === 1) {
609
+ if (reachedCap) entry.nextLogAtMs = now + retryHeartbeatMs
610
+ logger.error(`[channels] adapter "${name}" ${failure.detail} (attempt ${attempts}); retrying in ${delayMs}ms`)
611
+ } else if (nominalDelayMs > previousNominalDelayMs) {
612
+ logger.warn(
613
+ `[channels] adapter "${name}" still unavailable after ${Math.round(now - entry.failedSinceMs)}ms (attempt ${attempts}; ${failure.detail}); retrying in ${delayMs}ms`,
614
+ )
615
+ } else if (reachedCap && now >= entry.nextLogAtMs) {
616
+ logger.warn(
617
+ `[channels] adapter "${name}" still unavailable after ${Math.round(now - entry.failedSinceMs)}ms (attempt ${attempts}; ${failure.detail}); retrying in ${delayMs}ms`,
618
+ )
619
+ entry.nextLogAtMs = now + retryHeartbeatMs
620
+ }
621
+
622
+ failed.set(name, entry)
623
+ failedInputSignatures.set(entry, failure.inputSignature)
624
+ return entry
625
+ }
626
+
627
+ const applyStartResult = (
628
+ name: AdapterId,
629
+ cfg: ChannelAdapterConfig,
630
+ result: StartAdapterResult,
631
+ previousFailure?: FailedAdapter,
632
+ ): boolean => {
633
+ if (result.status === 'started') {
634
+ const recoveryState = failed.get(name) ?? previousFailure
635
+ failed.delete(name)
636
+ if (recoveryState === undefined) {
637
+ logger.info(`[channels] adapter "${name}" started`)
638
+ } else {
639
+ logger.info(
640
+ `[channels] adapter "${name}" recovered after ${Math.round(recoveryNow() - recoveryState.failedSinceMs)}ms (${recoveryState.attempts} failed attempts)`,
641
+ )
642
+ }
643
+ return true
644
+ }
645
+ if (result.status === 'failed') {
646
+ recordFailure(name, cfg, result, previousFailure)
647
+ } else if (previousFailure !== undefined && failed.get(name) === previousFailure) {
648
+ failed.delete(name)
438
649
  }
650
+ return false
651
+ }
652
+
653
+ const recordMissingCredentialFailure = (
654
+ name: AdapterId,
655
+ cfg: ChannelAdapterConfig,
656
+ credentials: { signature: string; missing: string[] },
657
+ ): void => {
658
+ recordFailure(
659
+ name,
660
+ cfg,
661
+ {
662
+ status: 'failed',
663
+ kind: 'missing-credentials',
664
+ detail: `missing credentials: ${credentials.missing.join(', ')}`,
665
+ inputSignature: buildFailedInputSignature(name, cfg, credentials),
666
+ },
667
+ failed.get(name),
668
+ )
669
+ }
670
+
671
+ const recordUnexpectedStartFailure = (
672
+ name: AdapterId,
673
+ cfg: ChannelAdapterConfig,
674
+ err: unknown,
675
+ previous?: FailedAdapter,
676
+ ): void => {
677
+ recordFailure(
678
+ name,
679
+ cfg,
680
+ {
681
+ status: 'failed',
682
+ kind: 'start-failed',
683
+ detail: `failed to start: ${describeError(err)}`,
684
+ inputSignature: buildFailedInputSignature(name, cfg),
685
+ },
686
+ previous,
687
+ )
439
688
  }
440
689
 
441
- const stopAdapter = async (name: AdapterId): Promise<void> => {
690
+ const stopAdapter = async (name: AdapterId): Promise<boolean> => {
442
691
  const entry = live.get(name)
443
- if (!entry) return
692
+ if (!entry) return true
444
693
  try {
445
694
  await entry.adapter.stop()
446
- live.delete(name)
695
+ if (live.get(name) === entry) live.delete(name)
447
696
  logger.info(`[channels] adapter "${name}" stopped`)
697
+ return true
448
698
  } catch (err) {
449
- logger.error(`[channels] adapter "${name}" failed to stop: ${describe(err)}`)
699
+ logger.error(`[channels] adapter "${name}" failed to stop: ${describeError(err)}`)
700
+ return false
450
701
  }
451
702
  }
452
703
 
453
- const checkConnectionRecovery = (): void => {
704
+ // Callers must already hold this adapter's `runSerially` barrier. Re-entering
705
+ // it here would queue behind the caller's own still-open turn and deadlock,
706
+ // which is exactly what happens if `reload` calls the public `restartAdapter`
707
+ // from inside its per-adapter block.
708
+ const restartAdapterLocked = async (name: AdapterId): Promise<AdapterRestartOutcome> => {
709
+ const currentCfg = options.channelsConfigRef()[name]
710
+ if (currentCfg === undefined || currentCfg.enabled === false) return 'skipped'
711
+ const previousFailure = failed.get(name)
712
+ failed.delete(name)
713
+ if (!(await stopAdapter(name))) return 'stop-failed'
714
+ const queuedEpoch = lifecycleEpoch
715
+ const result = await startAdapter(name, currentCfg, () => running && queuedEpoch === lifecycleEpoch)
716
+ return applyStartResult(name, currentCfg, result, previousFailure) ? 'restarted' : 'recovery-pending'
717
+ }
718
+
719
+ const superviseAdapters = (): void => {
720
+ if (!running) return
454
721
  const now = recoveryNow()
455
722
  for (const [name, entry] of live) {
456
723
  if (entry.adapter.isConnected()) {
@@ -469,27 +736,88 @@ export function createChannelManager(options: ChannelManagerOptions): ChannelMan
469
736
  logger.warn(
470
737
  `[channels] adapter "${name}" disconnected for ${Math.round(disconnectedForMs)}ms; restarting adapter`,
471
738
  )
739
+ const queuedEpoch = lifecycleEpoch
472
740
  void runSerially(name, async () => {
473
741
  try {
474
742
  const current = live.get(name)
475
- if (current !== entry) return
743
+ if (!running || queuedEpoch !== lifecycleEpoch || current !== entry) return
476
744
  const currentCfg = options.channelsConfigRef()[name]
477
745
  if (currentCfg === undefined || currentCfg.enabled === false) {
478
746
  logger.info(`[channels] recovery restart for "${name}" skipped; adapter no longer enabled`)
479
747
  return
480
748
  }
481
- await stopAdapter(name)
482
- await startAdapter(name, currentCfg)
749
+ const stopped = await stopAdapter(name)
750
+ if (!stopped || !running || queuedEpoch !== lifecycleEpoch) return
751
+ const latestCfg = options.channelsConfigRef()[name]
752
+ if (latestCfg === undefined || latestCfg.enabled === false) return
753
+ try {
754
+ const result = await startAdapter(name, latestCfg, () => {
755
+ const cfg = options.channelsConfigRef()[name]
756
+ return running && queuedEpoch === lifecycleEpoch && cfg !== undefined && cfg.enabled !== false
757
+ })
758
+ applyStartResult(name, latestCfg, result)
759
+ } catch (err) {
760
+ const cfg = options.channelsConfigRef()[name]
761
+ if (running && queuedEpoch === lifecycleEpoch && cfg !== undefined && cfg.enabled !== false) {
762
+ recordUnexpectedStartFailure(name, cfg, err)
763
+ }
764
+ }
483
765
  } finally {
484
766
  if (live.get(name) === entry) entry.recoveryRestartQueued = false
485
767
  }
768
+ }).catch((err) => {
769
+ logger.error(`[channels] adapter "${name}" recovery supervision failed: ${describeError(err)}`)
770
+ })
771
+ }
772
+
773
+ for (const [name, entry] of failed) {
774
+ if (entry.nextAttemptAtMs > now || entry.retryQueued) continue
775
+ entry.retryQueued = true
776
+ const queuedEpoch = lifecycleEpoch
777
+ void runSerially(name, async () => {
778
+ try {
779
+ if (!running || queuedEpoch !== lifecycleEpoch || failed.get(name) !== entry) return
780
+ const currentCfg = options.channelsConfigRef()[name]
781
+ if (currentCfg === undefined || currentCfg.enabled === false) {
782
+ if (failed.get(name) === entry) failed.delete(name)
783
+ return
784
+ }
785
+ try {
786
+ const result = await startAdapter(name, currentCfg, () => {
787
+ const cfg = options.channelsConfigRef()[name]
788
+ return (
789
+ running &&
790
+ queuedEpoch === lifecycleEpoch &&
791
+ failed.get(name) === entry &&
792
+ cfg !== undefined &&
793
+ cfg.enabled !== false
794
+ )
795
+ })
796
+ applyStartResult(name, currentCfg, result, entry)
797
+ } catch (err) {
798
+ const cfg = options.channelsConfigRef()[name]
799
+ if (
800
+ running &&
801
+ queuedEpoch === lifecycleEpoch &&
802
+ failed.get(name) === entry &&
803
+ cfg !== undefined &&
804
+ cfg.enabled !== false
805
+ ) {
806
+ recordUnexpectedStartFailure(name, cfg, err, entry)
807
+ }
808
+ }
809
+ } finally {
810
+ if (failed.get(name) === entry) entry.retryQueued = false
811
+ }
812
+ }).catch((err) => {
813
+ logger.error(`[channels] adapter "${name}" retry supervision failed: ${describeError(err)}`)
486
814
  })
487
815
  }
488
816
  }
489
817
 
490
818
  const startRecoveryTimer = (): void => {
491
819
  if (recoveryTimer !== null) return
492
- recoveryTimer = recoverySetInterval(checkConnectionRecovery, recoveryCheckIntervalMs)
820
+ recoveryTimer = recoverySetInterval(superviseAdapters, recoveryCheckIntervalMs)
493
821
  }
494
822
 
495
823
  const stopRecoveryTimer = (): void => {
@@ -503,6 +831,8 @@ export function createChannelManager(options: ChannelManagerOptions): ChannelMan
503
831
 
504
832
  async start(): Promise<void> {
505
833
  const cfg = options.channelsConfigRef()
834
+ running = true
835
+ const startedEpoch = lifecycleEpoch
506
836
  // Safe to fan out: `live` and every router registry are keyed by adapter
507
837
  // name, so concurrent starts never collide. Serial start would otherwise pay
508
838
  // the sum of each adapter's connect latency instead of just the slowest.
@@ -514,7 +844,12 @@ export function createChannelManager(options: ChannelManagerOptions): ChannelMan
514
844
  // uses this flag to answer "configured but unavailable" instead of the
515
845
  // misleading "not supported".
516
846
  router.setAdapterConfigured(name, adapterCfg.enabled !== false)
517
- return [runSerially(name, () => startAdapter(name, adapterCfg))]
847
+ return [
848
+ runSerially(name, async () => {
849
+ const result = await startAdapter(name, adapterCfg, () => running && startedEpoch === lifecycleEpoch)
850
+ return applyStartResult(name, adapterCfg, result)
851
+ }),
852
+ ]
518
853
  })
519
854
  // Await every launched start to settle BEFORE surfacing a failure.
520
855
  // `startAdapter` converts expected per-adapter failures to `false`, so a
@@ -524,81 +859,150 @@ export function createChannelManager(options: ChannelManagerOptions): ChannelMan
524
859
  // that the caller's subsequent `stop()` never sees. Settle all, then rethrow.
525
860
  const results = await Promise.allSettled(starts)
526
861
  const failure = results.find((r): r is PromiseRejectedResult => r.status === 'rejected')
862
+ // An unexpected throw leaves the manager half-built, so arming supervision
863
+ // would retry against a state the caller is about to tear down.
527
864
  if (failure !== undefined) throw failure.reason
528
865
  startRecoveryTimer()
529
866
  },
530
867
 
531
868
  async stop(): Promise<void> {
869
+ // Everything that gates a queued start must flip BEFORE the first await,
870
+ // or a retry already sitting in `perAdapterSerial` re-registers an adapter
871
+ // after teardown and leaks it past the caller's stop().
872
+ running = false
873
+ lifecycleEpoch += 1
532
874
  stopRecoveryTimer()
533
- for (const name of Array.from(live.keys())) await runSerially(name, () => stopAdapter(name))
875
+ failed.clear()
876
+ // Drain every id, not just the live ones: an in-flight retry for a
877
+ // currently-dead adapter has to be awaited too, and its `canCommit` will
878
+ // have aborted it by the time this barrier runs.
879
+ await Promise.all(ADAPTER_IDS.map((name) => runSerially(name, () => stopAdapter(name))))
534
880
  await router.stop()
535
881
  },
536
882
 
537
883
  async restartAdapter(name: AdapterId): Promise<void> {
538
884
  await runSerially(name, async () => {
539
- if (!live.has(name)) {
540
- logger.info(`[channels] restartAdapter('${name}'): adapter not live, skipping`)
541
- return
542
- }
543
885
  const currentCfg = options.channelsConfigRef()[name]
544
886
  if (currentCfg === undefined) {
545
887
  logger.info(`[channels] restartAdapter('${name}'): adapter config missing, skipping`)
546
888
  return
547
889
  }
548
- await stopAdapter(name)
549
- await startAdapter(name, currentCfg)
890
+ if (currentCfg.enabled === false) {
891
+ logger.info(`[channels] restartAdapter('${name}'): adapter is disabled, skipping`)
892
+ return
893
+ }
894
+ // Deliberately not gated on `live.has(name)`: the whole point is to let
895
+ // an operator revive an adapter that failed to start and is therefore
896
+ // absent from `live`.
897
+ await restartAdapterLocked(name)
550
898
  })
551
899
  },
552
900
 
553
- async reload(): Promise<{ started: string[]; stopped: string[]; restartRequired: string[] }> {
901
+ async reload(reloadOptions: ChannelReloadOptions = {}): Promise<ChannelReloadDiff> {
554
902
  const cfg = options.channelsConfigRef()
555
903
  const started: string[] = []
556
904
  const stopped: string[] = []
905
+ const restarted: string[] = []
557
906
  const restartRequired: string[] = []
907
+ let credentialApply: ChannelReloadDiff['credentialApply']
558
908
 
559
909
  for (const name of ADAPTER_IDS) {
560
910
  const desired = cfg[name]
561
- const current = live.get(name)
562
911
  if (desired === undefined || desired.enabled === false) {
563
912
  router.setAdapterConfigured(name, false)
564
- if (current) {
913
+ failed.delete(name)
914
+ if (live.has(name)) {
565
915
  await runSerially(name, () => stopAdapter(name))
566
916
  stopped.push(name)
567
917
  }
568
- } else if (!current) {
569
- router.setAdapterConfigured(name, true)
570
- const ok = await runSerially(name, () => startAdapter(name, desired))
571
- if (ok) started.push(name)
572
- } else {
573
- const { signature, missing } = buildCredentialSignature(name)
918
+ continue
919
+ }
920
+ router.setAdapterConfigured(name, true)
921
+ // The whole decision runs inside the barrier. Choosing a branch from
922
+ // `live.has(name)` out here races a queued retry: it can go live (or
923
+ // die) before the barrier admits us, and the branch we picked would
924
+ // then either start a second instance or skip credential
925
+ // reconciliation on an adapter that is now live with a stale token.
926
+ const outcome = await runSerially(name, async (): Promise<ReloadAdapterOutcome> => {
927
+ const latest = options.channelsConfigRef()[name]
928
+ if (latest === undefined || latest.enabled === false) return null
929
+ const credentials = buildCredentialSignature(name)
930
+ const { signature, missing } = credentials
931
+ const current = live.get(name)
932
+
574
933
  if (missing.length > 0) {
575
934
  // Required credentials disappeared (env vars removed from .env, or
576
935
  // KakaoTalk credentials removed from secrets.json). Continuing to use the
577
936
  // in-memory credentials would silently honor a credential the
578
937
  // operator explicitly removed, so stop the adapter instead of
579
938
  // waiting for a manual restart.
939
+ if (current === undefined) {
940
+ recordMissingCredentialFailure(name, latest, credentials)
941
+ return null
942
+ }
580
943
  logger.warn(
581
944
  `[channels] adapter "${name}" missing credentials after reload (${missing.join(', ')}); stopping`,
582
945
  )
583
- await runSerially(name, () => stopAdapter(name))
584
- stopped.push(name)
585
- } else if (signature !== current.credentialSignature) {
586
- const reason =
587
- name === 'kakaotalk' ||
588
- name === 'line' ||
589
- name === 'instagram' ||
590
- name === 'webex' ||
591
- name === 'teams' ||
592
- name === 'slack' ||
593
- name === 'discord'
594
- ? 'credential rotation'
595
- : 'token rotation'
596
- restartRequired.push(`${name} (${reason})`)
946
+ if (!(await stopAdapter(name))) return null
947
+ // Without this the adapter leaves `live` and never enters `failed`,
948
+ // so supervision loses it entirely and restoring the credential
949
+ // could not revive it without another manual reload.
950
+ recordMissingCredentialFailure(name, latest, credentials)
951
+ return 'stopped'
952
+ }
953
+
954
+ const named = reloadOptions.applyCredentialRotation === name
955
+
956
+ if (current !== undefined) {
957
+ if (signature === current.credentialSignature) return named ? 'already-current' : null
958
+ // Bouncing a live adapter drops the channel turns it is currently
959
+ // serving, so a plain operator reload only reports the rotation.
960
+ // Only a caller that names this exact adapter — hostd, right after
961
+ // it rewrote that credential on disk — gets the destructive path.
962
+ if (!named) return 'rotated'
963
+ return await restartAdapterLocked(name)
964
+ }
965
+
966
+ // A reload is the operator's signal that inputs changed, so a
967
+ // pending backoff timer is stale — retry now rather than making
968
+ // them wait out up to fifteen minutes.
969
+ const previousFailure = failed.get(name)
970
+ if (previousFailure !== undefined) {
971
+ const changed =
972
+ failedInputSignatures.get(previousFailure) !== buildFailedInputSignature(name, latest, credentials)
973
+ if (changed) failed.delete(name)
974
+ // A queued retry will read the new credential when it fires, so the
975
+ // rotation is on its way even though nothing happened right here.
976
+ else if (previousFailure.retryQueued) return named ? 'recovery-pending' : null
597
977
  }
978
+ const queuedEpoch = lifecycleEpoch
979
+ const result = await startAdapter(name, latest, () => running && queuedEpoch === lifecycleEpoch)
980
+ if (applyStartResult(name, latest, result, previousFailure)) return 'started'
981
+ return named ? 'recovery-pending' : null
982
+ })
983
+
984
+ if (outcome === 'started') {
985
+ started.push(name)
986
+ // Starting a down adapter applies the rotation just as much as
987
+ // bouncing a live one; reporting nothing here would read as a failure
988
+ // and send the caller to a container restart it does not need.
989
+ if (reloadOptions.applyCredentialRotation === name) {
990
+ credentialApply = { adapter: name, outcome: 'restarted' }
991
+ }
992
+ } else if (outcome === 'stopped') stopped.push(name)
993
+ else if (outcome === 'rotated') restartRequired.push(`${name} (${rotationReason(name)})`)
994
+ else if (outcome !== null) {
995
+ credentialApply = { adapter: name, outcome }
996
+ if (outcome === 'restarted') restarted.push(name)
997
+ // A stop that failed leaves the OLD credential live, so the rotation
998
+ // has not been applied and still owes a restart. `recovery-pending`
999
+ // does not: the adapter is already down and supervision is retrying
1000
+ // it with the new credential on backoff.
1001
+ else if (outcome === 'stop-failed') restartRequired.push(`${name} (${rotationReason(name)})`)
598
1002
  }
599
1003
  }
600
1004
 
601
- return { started, stopped, restartRequired }
1005
+ return { started, stopped, restarted, restartRequired, ...(credentialApply ? { credentialApply } : {}) }
602
1006
  },
603
1007
  }
604
1008
  }
@@ -644,7 +1048,7 @@ function buildDiscordSignature(agentDir: string): { signature: string; missing:
644
1048
  const digest = createHash('sha256').update(JSON.stringify(block)).digest('hex')
645
1049
  return { signature: `secrets.json#channels.discord@sha256:${digest}`, missing: [] }
646
1050
  } catch (err) {
647
- return { signature: '', missing: [`secrets.json#channels.discord (${describe(err)})`] }
1051
+ return { signature: '', missing: [`secrets.json#channels.discord (${describeError(err)})`] }
648
1052
  }
649
1053
  }
650
1054
 
@@ -670,7 +1074,7 @@ function buildSlackSignature(agentDir: string): { signature: string; missing: st
670
1074
  const digest = createHash('sha256').update(JSON.stringify(block)).digest('hex')
671
1075
  return { signature: `secrets.json#channels.slack@sha256:${digest}`, missing: [] }
672
1076
  } catch (err) {
673
- return { signature: '', missing: [`secrets.json#channels.slack (${describe(err)})`] }
1077
+ return { signature: '', missing: [`secrets.json#channels.slack (${describeError(err)})`] }
674
1078
  }
675
1079
  }
676
1080
 
@@ -696,7 +1100,7 @@ function buildWebexSignature(agentDir: string): { signature: string; missing: st
696
1100
  const digest = createHash('sha256').update(JSON.stringify(block)).digest('hex')
697
1101
  return { signature: `secrets.json#channels.webex@sha256:${digest}`, missing: [] }
698
1102
  } catch (err) {
699
- return { signature: '', missing: [`secrets.json#channels.webex (${describe(err)})`] }
1103
+ return { signature: '', missing: [`secrets.json#channels.webex (${describeError(err)})`] }
700
1104
  }
701
1105
  }
702
1106
 
@@ -712,6 +1116,20 @@ function createContainerTeamsCredentialStore(
712
1116
  })
713
1117
  }
714
1118
 
1119
+ // Personal-account adapters authenticate as a user, so their whole credential
1120
+ // block rotates; token-based bot adapters only swap an env token.
1121
+ function rotationReason(name: AdapterId): string {
1122
+ return name === 'kakaotalk' ||
1123
+ name === 'line' ||
1124
+ name === 'instagram' ||
1125
+ name === 'webex' ||
1126
+ name === 'teams' ||
1127
+ name === 'slack' ||
1128
+ name === 'discord'
1129
+ ? 'credential rotation'
1130
+ : 'token rotation'
1131
+ }
1132
+
715
1133
  function buildTeamsSignature(agentDir: string): { signature: string; missing: string[] } {
716
1134
  const path = join(agentDir, 'secrets.json')
717
1135
  try {
@@ -722,7 +1140,7 @@ function buildTeamsSignature(agentDir: string): { signature: string; missing: st
722
1140
  const digest = createHash('sha256').update(JSON.stringify(block)).digest('hex')
723
1141
  return { signature: `secrets.json#channels.teams@sha256:${digest}`, missing: [] }
724
1142
  } catch (err) {
725
- return { signature: '', missing: [`secrets.json#channels.teams (${describe(err)})`] }
1143
+ return { signature: '', missing: [`secrets.json#channels.teams (${describeError(err)})`] }
726
1144
  }
727
1145
  }
728
1146
 
@@ -748,7 +1166,7 @@ function buildKakaotalkSignature(agentDir: string): { signature: string; missing
748
1166
  const digest = createHash('sha256').update(JSON.stringify(block)).digest('hex')
749
1167
  return { signature: `secrets.json#channels.kakaotalk@sha256:${digest}`, missing: [] }
750
1168
  } catch (err) {
751
- return { signature: '', missing: [`secrets.json#channels.kakaotalk (${describe(err)})`] }
1169
+ return { signature: '', missing: [`secrets.json#channels.kakaotalk (${describeError(err)})`] }
752
1170
  }
753
1171
  }
754
1172
 
@@ -774,7 +1192,7 @@ function buildLineSignature(agentDir: string): { signature: string; missing: str
774
1192
  const digest = createHash('sha256').update(JSON.stringify(block)).digest('hex')
775
1193
  return { signature: `secrets.json#channels.line@sha256:${digest}`, missing: [] }
776
1194
  } catch (err) {
777
- return { signature: '', missing: [`secrets.json#channels.line (${describe(err)})`] }
1195
+ return { signature: '', missing: [`secrets.json#channels.line (${describeError(err)})`] }
778
1196
  }
779
1197
  }
780
1198
 
@@ -800,7 +1218,7 @@ function buildInstagramSignature(agentDir: string): { signature: string; missing
800
1218
  const digest = createHash('sha256').update(JSON.stringify(block)).digest('hex')
801
1219
  return { signature: `secrets.json#channels.instagram@sha256:${digest}`, missing: [] }
802
1220
  } catch (err) {
803
- return { signature: '', missing: [`secrets.json#channels.instagram (${describe(err)})`] }
1221
+ return { signature: '', missing: [`secrets.json#channels.instagram (${describeError(err)})`] }
804
1222
  }
805
1223
  }
806
1224
 
@@ -892,7 +1310,3 @@ function isTeamsCredentialBlock(value: unknown): value is { accounts: Record<str
892
1310
  typeof accounts === 'object' && accounts !== null && !Array.isArray(accounts) && Object.keys(accounts).length > 0
893
1311
  )
894
1312
  }
895
-
896
- function describe(err: unknown): string {
897
- return err instanceof Error ? err.message : String(err)
898
- }