switchroom 0.18.6 → 0.18.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. package/dist/agent-scheduler/index.js +1 -0
  2. package/dist/auth-broker/index.js +1 -0
  3. package/dist/cli/autoaccept-poll.js +140 -33
  4. package/dist/cli/notion-write-pretool.mjs +1 -0
  5. package/dist/cli/switchroom.js +1172 -812
  6. package/dist/host-control/main.js +2 -1
  7. package/dist/vault/approvals/kernel-server.js +1 -0
  8. package/dist/vault/broker/server.js +1 -0
  9. package/package.json +3 -3
  10. package/profiles/_base/cron-session.sh.hbs +55 -16
  11. package/profiles/_base/start.sh.hbs +146 -50
  12. package/profiles/default/CLAUDE.md.hbs +1 -1
  13. package/skills/switchroom-runtime/SKILL.md +2 -0
  14. package/telegram-plugin/dist/bridge/bridge.js +22 -0
  15. package/telegram-plugin/dist/gateway/gateway.js +2965 -862
  16. package/telegram-plugin/dist/server.js +24 -0
  17. package/telegram-plugin/flood-circuit-breaker.ts +123 -0
  18. package/telegram-plugin/gateway/activity-card-store.ts +63 -18
  19. package/telegram-plugin/gateway/always-allow-persist-queue.ts +438 -0
  20. package/telegram-plugin/gateway/approval-timeout-inbound-builders.ts +150 -0
  21. package/telegram-plugin/gateway/boot-card.ts +27 -0
  22. package/telegram-plugin/gateway/busy-ack.ts +106 -0
  23. package/telegram-plugin/gateway/clean-shutdown-marker.ts +68 -20
  24. package/telegram-plugin/gateway/gateway.ts +1618 -198
  25. package/telegram-plugin/gateway/inbound-spool.ts +2 -1
  26. package/telegram-plugin/gateway/inject-handler.test.ts +19 -0
  27. package/telegram-plugin/gateway/inject-handler.ts +17 -0
  28. package/telegram-plugin/gateway/ipc-protocol.ts +44 -2
  29. package/telegram-plugin/gateway/ipc-server.ts +40 -0
  30. package/telegram-plugin/gateway/mental-model-propose-diff.ts +61 -5
  31. package/telegram-plugin/gateway/model-command.ts +227 -54
  32. package/telegram-plugin/gateway/pending-card-expiry.ts +98 -0
  33. package/telegram-plugin/gateway/pending-card-store.ts +173 -0
  34. package/telegram-plugin/gateway/pending-inbound-buffer.ts +12 -2
  35. package/telegram-plugin/gateway/resume-inbound-builder.ts +240 -2
  36. package/telegram-plugin/gateway/session-model-file.ts +198 -0
  37. package/telegram-plugin/gateway/session-model-source.ts +73 -0
  38. package/telegram-plugin/gateway/status-pin-store.ts +82 -22
  39. package/telegram-plugin/gateway/worker-feed-dispatch.ts +24 -1
  40. package/telegram-plugin/gateway/worker-pin-reaper.ts +114 -0
  41. package/telegram-plugin/hooks/hooks.json +10 -10
  42. package/telegram-plugin/hooks/run-hook.sh +84 -0
  43. package/telegram-plugin/hooks/subagent-tracker-pretool.mjs +30 -7
  44. package/telegram-plugin/model-label.ts +69 -0
  45. package/telegram-plugin/model-unavailable.ts +26 -0
  46. package/telegram-plugin/operator-events.ts +24 -0
  47. package/telegram-plugin/permission-diff.ts +128 -0
  48. package/telegram-plugin/pty-partial-handler.ts +39 -0
  49. package/telegram-plugin/registry/subagents-schema.ts +80 -1
  50. package/telegram-plugin/registry/subagents.test.ts +90 -0
  51. package/telegram-plugin/render/rich-render.ts +79 -1
  52. package/telegram-plugin/retry-api-call.ts +62 -0
  53. package/telegram-plugin/session-tail.ts +28 -0
  54. package/telegram-plugin/shared/bot-runtime.ts +8 -1
  55. package/telegram-plugin/silence-poke.ts +14 -0
  56. package/telegram-plugin/silent-end.ts +49 -4
  57. package/telegram-plugin/stream-controller.ts +156 -38
  58. package/telegram-plugin/subagent-watcher.ts +222 -37
  59. package/telegram-plugin/tests/activity-card-store.test.ts +47 -2
  60. package/telegram-plugin/tests/always-allow-persist-queue.test.ts +529 -0
  61. package/telegram-plugin/tests/approval-card-restart-outcome.test.ts +218 -0
  62. package/telegram-plugin/tests/approval-timeout-inbound-builders.test.ts +94 -0
  63. package/telegram-plugin/tests/boot-card-flood-suppress.test.ts +111 -0
  64. package/telegram-plugin/tests/busy-ack-wiring.test.ts +118 -0
  65. package/telegram-plugin/tests/busy-ack.test.ts +121 -0
  66. package/telegram-plugin/tests/button-tap-turn-gated.test.ts +263 -0
  67. package/telegram-plugin/tests/flood-circuit-breaker.test.ts +74 -0
  68. package/telegram-plugin/tests/gateway-clean-shutdown-marker.test.ts +85 -27
  69. package/telegram-plugin/tests/gateway-session-model-relaunch.test.ts +179 -25
  70. package/telegram-plugin/tests/ipc-server-query-pending-permission.test.ts +157 -0
  71. package/telegram-plugin/tests/mental-model-name-entity-corruption.test.ts +119 -0
  72. package/telegram-plugin/tests/mental-model-propose-callback-gate.test.ts +8 -5
  73. package/telegram-plugin/tests/model-command.test.ts +203 -43
  74. package/telegram-plugin/tests/model-label.test.ts +64 -0
  75. package/telegram-plugin/tests/model-unavailable.test.ts +41 -0
  76. package/telegram-plugin/tests/operator-events.test.ts +1 -0
  77. package/telegram-plugin/tests/pending-card-durability-wiring.test.ts +202 -0
  78. package/telegram-plugin/tests/pending-card-expiry.test.ts +190 -0
  79. package/telegram-plugin/tests/pending-card-store.test.ts +173 -0
  80. package/telegram-plugin/tests/permission-diff.test.ts +111 -0
  81. package/telegram-plugin/tests/pty-partial-handler.test.ts +56 -0
  82. package/telegram-plugin/tests/render/render-outbound-chunks.test.ts +98 -0
  83. package/telegram-plugin/tests/resume-inbound-builder.test.ts +286 -0
  84. package/telegram-plugin/tests/retry-api-call.test.ts +59 -0
  85. package/telegram-plugin/tests/run-hook-wrapper.test.ts +132 -0
  86. package/telegram-plugin/tests/session-model-file.test.ts +132 -0
  87. package/telegram-plugin/tests/session-model-source.test.ts +67 -0
  88. package/telegram-plugin/tests/session-tail.test.ts +64 -0
  89. package/telegram-plugin/tests/silent-end.test.ts +46 -1
  90. package/telegram-plugin/tests/slot-banner-boot-recovery.test.ts +3 -3
  91. package/telegram-plugin/tests/status-pin-boot-recovery.test.ts +3 -3
  92. package/telegram-plugin/tests/status-pin-store.test.ts +62 -6
  93. package/telegram-plugin/tests/stream-controller-chunk-cap.test.ts +122 -0
  94. package/telegram-plugin/tests/subagent-tracker-hooks.test.ts +39 -0
  95. package/telegram-plugin/tests/subagent-watcher-boot-promotion-replay.test.ts +107 -4
  96. package/telegram-plugin/tests/subagent-watcher-handback-gaps.test.ts +42 -4
  97. package/telegram-plugin/tests/subagent-watcher-parent-turn-key.test.ts +47 -0
  98. package/telegram-plugin/tests/subagent-watcher-terminated-ids-cap.test.ts +150 -0
  99. package/telegram-plugin/tests/subagent-watcher.test.ts +54 -0
  100. package/telegram-plugin/tests/tool-activity-summary.test.ts +37 -0
  101. package/telegram-plugin/tests/typing-wrap.test.ts +23 -0
  102. package/telegram-plugin/tests/voice-send.test.ts +308 -0
  103. package/telegram-plugin/tests/worker-activity-feed.test.ts +11 -0
  104. package/telegram-plugin/tests/worker-feed-dispatch.test.ts +126 -0
  105. package/telegram-plugin/tests/worker-pin-reaper.test.ts +132 -0
  106. package/telegram-plugin/tool-activity-summary.ts +22 -2
  107. package/telegram-plugin/typing-wrap.ts +72 -25
  108. package/telegram-plugin/uat/scenarios/jtbd-deliberate-restart-resumes-dm.test.ts +118 -0
  109. package/telegram-plugin/uat/scenarios/jtbd-midflight-busy-ack-dm.test.ts +201 -0
  110. package/telegram-plugin/uat/scenarios/jtbd-worker-pin-lifecycle-dm.test.ts +208 -0
  111. package/telegram-plugin/uat/scenarios/vault-card-survives-gateway-restart-dm.test.ts +140 -0
  112. package/telegram-plugin/uat/scenarios/vault-deny-resumes-turn-dm.test.ts +84 -0
  113. package/telegram-plugin/uat/scenarios/vault-timeout-wakes-agent-dm.test.ts +91 -0
  114. package/telegram-plugin/voice-ondemand.ts +25 -1
  115. package/telegram-plugin/voice-send.ts +154 -0
  116. package/telegram-plugin/worker-activity-feed.ts +9 -0
@@ -1,10 +1,15 @@
1
1
  // Auto-wrap tool dispatch with a Telegram typing-indicator loop so the user
2
- // sees a live "agent is working" signal during the 3–30s gap where the
3
- // progress card is deliberately suppressed (its initialDelayMs is 3s).
4
- // The first tool call on a given (chat, thread) fires the typing loop
5
- // immediately so there's no silent dead window before the progress card
6
- // appears. Subsequent calls on the same lane honour the debounce to avoid
7
- // churn. Surface tools own their own loop — see isSurfaceTool.
2
+ // sees a live "agent is working" signal for the run of a turn. This is now
3
+ // the PRIMARY liveness signal for short tool-using turns, not a bridge to a
4
+ // soon-to-appear pinned progress card: the pinned card is retired
5
+ // (`unpinProgressCardForChat = null`, see
6
+ // `docs/diagrams/deterministic-status-anatomy.spec.md`), and the worker
7
+ // progress card only gates in at `elapsed >= 60s OR sub-agent appeared` — for
8
+ // most short turns it never appears at all. The first tool call on a given
9
+ // (chat, thread) fires the typing loop immediately so there's no silent dead
10
+ // window at turn start. Subsequent calls on the same lane honour the
11
+ // debounce to avoid churn. Surface tools own their own loop — see
12
+ // isSurfaceTool.
8
13
  //
9
14
  // Keying changed from `chatId` to `(chatId, threadId)` in PR3 of the
10
15
  // supergroup-mode rollout. In supergroup mode one agent owns many topics
@@ -38,17 +43,55 @@ export interface TypingWrapper {
38
43
  interface Entry {
39
44
  chatId: string
40
45
  threadId: number | null
46
+ lane: string
41
47
  timer: ReturnType<typeof setTimeout>
48
+ }
49
+
50
+ /**
51
+ * Per-lane bookkeeping: `count` is the number of outstanding (dispatched but
52
+ * not yet resulted) tool-uses on the lane; `started` is whether the typing
53
+ * loop is currently believed to be running for it. Ref-counted rather than a
54
+ * boolean-per-lane `Set` (pre-fix#7 shape) because two PARALLEL tool_use
55
+ * blocks on one lane must both hold the lane open: with a plain Set,
56
+ * `onToolResult` for the first one deleted the lane and stopped the loop
57
+ * while the second tool was still running, flickering the typing indicator
58
+ * off until the next tool re-fired it. Only the count reaching zero may stop
59
+ * the loop.
60
+ */
61
+ interface LaneState {
62
+ count: number
42
63
  started: boolean
43
64
  }
44
65
 
45
66
  export function createTypingWrapper(deps: TypingWrapperDeps): TypingWrapper {
46
67
  const debounceMs = deps.debounceMs ?? 500
47
68
  const pending = new Map<string, Entry>()
48
- // Track per-(chat,thread) lanes that already have an active typing loop
49
- // so the first tool call on a lane fires immediately while subsequent
50
- // calls on the same lane use the debounce.
51
- const activeLanes = new Set<string>()
69
+ const lanes = new Map<string, LaneState>()
70
+
71
+ function laneFor(lane: string): LaneState {
72
+ let s = lanes.get(lane)
73
+ if (s == null) {
74
+ s = { count: 0, started: false }
75
+ lanes.set(lane, s)
76
+ }
77
+ return s
78
+ }
79
+
80
+ // Decrement the lane's outstanding count. Returns true iff the count just
81
+ // reached zero AND the loop was running — i.e. iff the CALLER must now
82
+ // call `stopTypingLoop`. This is the ref-count release: the loop only
83
+ // stops once every outstanding tool-use on the lane has resulted.
84
+ function release(lane: string): boolean {
85
+ const s = lanes.get(lane)
86
+ if (s == null) return false
87
+ s.count = Math.max(0, s.count - 1)
88
+ if (s.count === 0) {
89
+ const wasStarted = s.started
90
+ lanes.delete(lane)
91
+ return wasStarted
92
+ }
93
+ return false
94
+ }
52
95
 
53
96
  return {
54
97
  onToolUse(toolUseId, chatId, toolName, threadId) {
@@ -60,19 +103,21 @@ export function createTypingWrapper(deps: TypingWrapperDeps): TypingWrapper {
60
103
  const prior = pending.get(toolUseId)
61
104
  if (prior) {
62
105
  clearTimeout(prior.timer)
63
- if (prior.started) deps.stopTypingLoop(prior.chatId, prior.threadId)
106
+ if (release(prior.lane)) deps.stopTypingLoop(prior.chatId, prior.threadId)
64
107
  pending.delete(toolUseId)
65
108
  }
66
- // First tool on this lane: fire immediately rather than waiting for
67
- // the debounce — this closes the silent dead window before the first
68
- // progress card appears.
69
- if (!activeLanes.has(lane)) {
109
+ const state = laneFor(lane)
110
+ state.count += 1
111
+ // First outstanding tool-use on this lane: fire immediately rather
112
+ // than waiting for the debounce — this closes the silent dead window
113
+ // at turn start.
114
+ if (state.count === 1) {
70
115
  deps.startTypingLoop(chatId, tid)
71
- activeLanes.add(lane)
116
+ state.started = true
72
117
  const entry: Entry = {
73
118
  chatId,
74
119
  threadId: tid,
75
- started: true,
120
+ lane,
76
121
  timer: setTimeout(() => {}, 0), // no-op sentinel
77
122
  }
78
123
  pending.set(toolUseId, entry)
@@ -81,10 +126,10 @@ export function createTypingWrapper(deps: TypingWrapperDeps): TypingWrapper {
81
126
  const entry: Entry = {
82
127
  chatId,
83
128
  threadId: tid,
84
- started: false,
129
+ lane,
85
130
  timer: setTimeout(() => {
86
131
  deps.startTypingLoop(chatId, tid)
87
- entry.started = true
132
+ state.started = true
88
133
  }, debounceMs),
89
134
  }
90
135
  pending.set(toolUseId, entry)
@@ -95,20 +140,22 @@ export function createTypingWrapper(deps: TypingWrapperDeps): TypingWrapper {
95
140
  const entry = pending.get(toolUseId)
96
141
  if (!entry) return
97
142
  clearTimeout(entry.timer)
98
- if (entry.started) {
99
- deps.stopTypingLoop(entry.chatId, entry.threadId)
100
- activeLanes.delete(chatKey(entry.chatId, entry.threadId) as string)
101
- }
143
+ if (release(entry.lane)) deps.stopTypingLoop(entry.chatId, entry.threadId)
102
144
  pending.delete(toolUseId)
103
145
  },
104
146
 
105
147
  drainAll() {
148
+ const stoppedLanes = new Set<string>()
106
149
  for (const entry of pending.values()) {
107
150
  clearTimeout(entry.timer)
108
- if (entry.started) deps.stopTypingLoop(entry.chatId, entry.threadId)
151
+ const state = lanes.get(entry.lane)
152
+ if (state?.started && !stoppedLanes.has(entry.lane)) {
153
+ deps.stopTypingLoop(entry.chatId, entry.threadId)
154
+ stoppedLanes.add(entry.lane)
155
+ }
109
156
  }
110
157
  pending.clear()
111
- activeLanes.clear()
158
+ lanes.clear()
112
159
  },
113
160
  }
114
161
  }
@@ -0,0 +1,118 @@
1
+ /**
2
+ * JTBD scenario named by `reference/jobs/survive-reboots-and-real-life.md`
3
+ * § Prove it — "Deliberate restart resumes, bounded (DM)".
4
+ *
5
+ * Contract under test (#2988): a DELIBERATE restart (operator
6
+ * `switchroom agent restart`, i.e. a clean SIGTERM shutdown with a
7
+ * clean-shutdown marker) that lands MID-TURN resumes the interrupted turn
8
+ * like a crash-class interruption. The quota-saving suppression applies only
9
+ * when nothing was in flight — it never swallows mid-turn work. And the
10
+ * resume is bounded: AT MOST ONCE per interruption (the loop-guard caps the
11
+ * resume chain), so we also assert no second resume-framed turn fires after
12
+ * the first completes.
13
+ *
14
+ * Sibling: `jtbd-interrupted-turn-resumes-dm.test.ts` (same interruption
15
+ * mechanics; this scenario adds the deliberate-restart class + the
16
+ * at-most-once bound). Decision-level twin: `decideBootResumeKind` cases in
17
+ * `tests/resume-inbound-builder.test.ts` (BOUNDED CHAIN) and
18
+ * `shouldSuppressBootResume` in `tests/gateway-clean-shutdown-marker.test.ts`.
19
+ *
20
+ * Self-skips green without NOPASSWD sudo.
21
+ */
22
+
23
+ import { describe, it, expect } from "vitest";
24
+ import { execSync, spawn } from "node:child_process";
25
+ import { spinUp } from "../harness.js";
26
+
27
+ const AGENT = "test-harness";
28
+ const MID_TURN_MS = 10_000; // let the turn get in-flight before the bounce
29
+ const RESUME_BUDGET_MS = 180_000; // boot + resume + reply
30
+ const QUIET_WINDOW_MS = 90_000; // after the resume completes, no second resume may fire
31
+ // Messages landing within this window of the FIRST resume-framed reply are
32
+ // treated as the same resumed turn (e.g. a progress line plus the final
33
+ // answer of one turn) — only a resume-framed message AFTER it counts as a
34
+ // second resume for the at-most-once bound.
35
+ const SAME_TURN_GRACE_MS = 20_000;
36
+
37
+ const RESUME_FRAMING = /resum|picking .*back|interrupted|cut off|just restarted/i;
38
+
39
+ function canShellSudo(): boolean {
40
+ try {
41
+ execSync("sudo -n true", { stdio: "ignore", timeout: 2_000 });
42
+ return true;
43
+ } catch {
44
+ return false;
45
+ }
46
+ }
47
+
48
+ function kickRestartDetached(name: string): void {
49
+ // A deliberate operator restart: clean SIGTERM path, clean-shutdown marker
50
+ // written — the exact class #2988 makes resume in-flight work through.
51
+ const child = spawn(
52
+ "sudo",
53
+ ["-n", "env", `PATH=${process.env.PATH}`, `HOME=${process.env.HOME}`,
54
+ "switchroom", "agent", "restart", name, "--force"],
55
+ { detached: true, stdio: "ignore" },
56
+ );
57
+ child.unref();
58
+ }
59
+
60
+ const sudoOk = canShellSudo();
61
+
62
+ (sudoOk ? describe : describe.skip)(
63
+ "uat: deliberate restart mid-turn resumes exactly once (DM, #2988)",
64
+ () => {
65
+ it(
66
+ "an operator restart mid-turn resumes the work once, and never a second time",
67
+ async () => {
68
+ const sc = await spinUp({ agent: AGENT, settleMs: 0 });
69
+ try {
70
+ // A task long enough to still be in flight at MID_TURN_MS.
71
+ await sc.sendDM(
72
+ `Please write a thorough, detailed ~300-word explanation of how a ` +
73
+ `player piano's pneumatic action reads a music roll, step by step. ` +
74
+ `Take your time and be complete.`,
75
+ );
76
+
77
+ await new Promise((r) => setTimeout(r, MID_TURN_MS));
78
+ kickRestartDetached(AGENT);
79
+
80
+ // 1. In-flight work resumes: quota-saving suppression must NOT
81
+ // swallow a turn that was mid-flight at the bounce. Edits are
82
+ // excluded — a progress-card edit must not satisfy this; only
83
+ // a genuinely NEW message from the resumed turn counts.
84
+ const reply = await sc.expectMessage(
85
+ (m) => !m.edited && RESUME_FRAMING.test(m.text),
86
+ { from: "bot", timeout: RESUME_BUDGET_MS },
87
+ );
88
+ expect(reply.text).toMatch(RESUME_FRAMING);
89
+ expect(reply.text.length).toBeGreaterThan(40);
90
+
91
+ // 2. Bounded chain: with the interruption resolved, no SECOND
92
+ // resume-framed turn may fire (at-most-once per interruption).
93
+ // Edits are excluded, and messages within SAME_TURN_GRACE_MS of
94
+ // the first reply are deduped as part of the SAME resumed turn
95
+ // (two resume-ish lines in one turn are not a second resume).
96
+ const firstReplyAt = reply.date.getTime();
97
+ let second: unknown = null;
98
+ try {
99
+ second = await sc.expectMessage(
100
+ (m) =>
101
+ !m.edited &&
102
+ m.messageId !== reply.messageId &&
103
+ m.date.getTime() > firstReplyAt + SAME_TURN_GRACE_MS &&
104
+ RESUME_FRAMING.test(m.text),
105
+ { from: "bot", timeout: QUIET_WINDOW_MS },
106
+ );
107
+ } catch {
108
+ // timeout is the PASS: no repeat resume.
109
+ }
110
+ expect(second).toBeNull();
111
+ } finally {
112
+ await sc.tearDown();
113
+ }
114
+ },
115
+ RESUME_BUDGET_MS + QUIET_WINDOW_MS + 120_000,
116
+ );
117
+ },
118
+ );
@@ -0,0 +1,201 @@
1
+ /**
2
+ * JTBD scenario — mid-flight busy ack (#2995).
3
+ *
4
+ * Serves: `reference/jobs/steer-or-queue-mid-flight.md` — the third
5
+ * mid-flight class (a quick question that neither steers nor queues
6
+ * work of its own) and its latency promise: a mid-flight message
7
+ * arriving while the agent sits inside ONE long blocking tool call
8
+ * gets a visible, silent ack naming the blocking activity within
9
+ * seconds — never minutes of dead air behind a `--watch`-style call.
10
+ *
11
+ * ## What the scenario drives
12
+ *
13
+ * 1. Prompt the agent into a deliberately slow BLOCKING foreground
14
+ * Bash (`sleep`) — explicitly instructed NOT to background it.
15
+ * 2. Wait past the busy-ack step-age threshold (12s), then send a
16
+ * trivial mid-turn status question. It buffers (unmarked mid-turn
17
+ * follow-up = queue).
18
+ * 3. Assert a bot message matching the deterministic busy-ack shape
19
+ * ("⏳ Queued — currently inside `…`") lands within ACK_SLA_MS of
20
+ * the ping, and is SILENT (disable_notification).
21
+ * 4. Assert the real answer to the question arrives later (after the
22
+ * blocking step returns and the buffer flushes).
23
+ * 5. Assert the busy-ack card was cleaned up (deleted) once the
24
+ * answer landed — never a dangling "Queued" line.
25
+ *
26
+ * Negative case: a ping during a SHORT blocking step (well under the 12s
27
+ * threshold, and finished before the deferred re-check could ever see a
28
+ * 12s-old step) gets NO busy-ack card at any point — an ack that fires
29
+ * when the agent was seconds from answering is the job spec's explicit
30
+ * Bad bullet. (A young-step ping during a LONG step DOES get a deferred
31
+ * card once the step ages past the threshold — that is the re-check
32
+ * behaviour, covered by the positive case's latency budget.)
33
+ *
34
+ * ## Conventions
35
+ *
36
+ * - mtcute harness (`spinUp`), skip discipline like siblings: env
37
+ * handled by load-env; the suite hard-fails without creds by design
38
+ * (CI path-gates it to the uat-host runner).
39
+ * - `!m.edited` filters on every observation (PR #2991 lesson — the
40
+ * live draft/feed edits stream constantly; edits must never satisfy
41
+ * a fresh-message expectation).
42
+ */
43
+
44
+ import { describe, expect, it } from "vitest";
45
+ import { spinUp } from "../harness.js";
46
+ import type { ObservedMessage } from "../driver.js";
47
+
48
+ const AGENT = "test-harness";
49
+
50
+ // The gateway constant is 12s (BUSY_ACK_STEP_AGE_THRESHOLD_MS). The
51
+ // warmup is anchored to the SEND, not to observed tool start — the agent
52
+ // needs a few seconds to open the turn and enter the Bash step — so it
53
+ // carries a large margin: even if the tool starts ~15s after the send,
54
+ // the step is comfortably past the threshold when the ping lands.
55
+ const STEP_WARMUP_MS = 35_000;
56
+
57
+ // Latency promise: user-visible ack within seconds of the ping. The card
58
+ // is posted synchronously by the gateway (no model). When the ping lands
59
+ // past the threshold the card is immediate; if the step turns out to be
60
+ // just under, the deferred re-check posts it within (threshold − age).
61
+ // 10s + jitter headroom covers both.
62
+ const ACK_SLA_MS = 10_000;
63
+
64
+ // The blocking sleep. Long enough that warmup + ping + ack all land while
65
+ // the step is still open (even with a late tool start); short enough to
66
+ // keep wall-clock sane.
67
+ const SLEEP_SECS = 120;
68
+
69
+ const BUSY_ACK_RE = /⏳ Queued — currently inside `.+`.*I'll answer when this step finishes/s;
70
+
71
+ const SLOW_PROMPT =
72
+ `For a latency test, run this exact command in the FOREGROUND with your ` +
73
+ `Bash tool (do NOT use run_in_background, do NOT split it up): ` +
74
+ `\`sleep ${SLEEP_SECS} && echo done\`. When it completes, reply with ` +
75
+ `exactly: SLEEP_DONE. Do not send any message before it completes.`;
76
+
77
+ const PING = "Quick question while you work: are you still there? One word is fine.";
78
+
79
+ describe("uat: mid-flight busy ack during a long blocking tool call (#2995)", () => {
80
+ it(
81
+ "a mid-turn ping behind a long blocking Bash gets a silent Queued ack naming the activity, then the real answer, then cleanup",
82
+ async () => {
83
+ const sc = await spinUp({ agent: AGENT });
84
+ try {
85
+ await sc.sendDM(SLOW_PROMPT);
86
+
87
+ // Let the blocking step age past the busy-ack threshold.
88
+ await new Promise((r) => setTimeout(r, STEP_WARMUP_MS));
89
+
90
+ const pingAt = Date.now();
91
+ await sc.sendDM(PING);
92
+
93
+ // The deterministic busy-ack must land within the SLA, silent,
94
+ // naming the blocking activity. `!m.edited` — feed/draft edits
95
+ // must never satisfy this.
96
+ const ack = await sc.expectMessage(
97
+ (m: ObservedMessage) => m.fromBot && !m.edited && BUSY_ACK_RE.test(m.text),
98
+ { from: "bot", timeout: ACK_SLA_MS + 5_000 },
99
+ );
100
+ const ackLatency = Date.now() - pingAt;
101
+ expect(
102
+ ackLatency,
103
+ `busy-ack landed ${ackLatency}ms after the ping — the <10s ` +
104
+ `latency promise for a mid-flight quick question is broken.`,
105
+ ).toBeLessThan(ACK_SLA_MS + 5_000);
106
+ expect(
107
+ ack.silent,
108
+ `busy-ack pinged the user's device (silent=false) — the card ` +
109
+ `must carry disable_notification:true.`,
110
+ ).toBe(true);
111
+ // Names the blocking activity (the Bash sleep), not a generic line.
112
+ expect(ack.text).toMatch(/`.*(Bash|sleep).*`/i);
113
+
114
+ // The real answer to the ping arrives AFTER the blocking step
115
+ // returns and the buffer flushes. Any fresh non-card bot message
116
+ // that isn't the busy-ack or the SLEEP_DONE completion counts —
117
+ // the SLEEP_DONE exclusion is explicit (it answers the FIRST
118
+ // prompt, not the ping).
119
+ const answer = await sc.expectMessage(
120
+ (m: ObservedMessage) =>
121
+ m.fromBot &&
122
+ !m.edited &&
123
+ m.messageId !== ack.messageId &&
124
+ !BUSY_ACK_RE.test(m.text) &&
125
+ !/SLEEP_DONE/i.test(m.text) &&
126
+ m.text.trim().length > 0,
127
+ { from: "bot", timeout: (SLEEP_SECS + 90) * 1000 },
128
+ );
129
+ expect(answer.text.trim().length).toBeGreaterThan(0);
130
+
131
+ // Cleanup: once answers land and the turns wrap, the busy-ack
132
+ // card is deleted (delete-on-answer lifecycle). Poll briefly —
133
+ // the reap is fire-and-forget after the answer/turn-end.
134
+ const cleanupDeadline = Date.now() + 60_000;
135
+ let cardGone = false;
136
+ while (Date.now() < cleanupDeadline) {
137
+ const still = await sc.driver.getMessage(sc.botUserId, ack.messageId);
138
+ if (still == null) {
139
+ cardGone = true;
140
+ break;
141
+ }
142
+ await new Promise((r) => setTimeout(r, 3_000));
143
+ }
144
+ expect(
145
+ cardGone,
146
+ `busy-ack card (msg ${ack.messageId}) still present 60s after ` +
147
+ `the answer — a stale "Queued" line is dangling in the chat.`,
148
+ ).toBe(true);
149
+ } finally {
150
+ await sc.tearDown();
151
+ }
152
+ },
153
+ // warmup + sleep + flush + answer + cleanup poll + settle headroom
154
+ (STEP_WARMUP_MS + SLEEP_SECS * 1000 + 200_000),
155
+ );
156
+
157
+ it(
158
+ "negative: a ping during a SHORT step (finishes under the threshold) never gets a busy-ack card",
159
+ async () => {
160
+ const sc = await spinUp({ agent: AGENT });
161
+ try {
162
+ // An 8s sleep can NEVER age past the 12s threshold — neither the
163
+ // direct evaluation nor the deferred re-check may produce a card
164
+ // (the re-check re-reads live state and finds the step gone).
165
+ await sc.sendDM(
166
+ `For a latency test, run this exact command in the FOREGROUND ` +
167
+ `with your Bash tool (do NOT use run_in_background): ` +
168
+ `\`sleep 8 && echo done\`. When it completes, reply with ` +
169
+ `exactly: SHORT_SLEEP_DONE. Do not send any message before it completes.`,
170
+ );
171
+
172
+ // Give the agent a beat to enter the Bash step, then ping while
173
+ // the short step is (at most) a few seconds old.
174
+ await new Promise((r) => setTimeout(r, 4_000));
175
+ await sc.sendDM("Still with me?");
176
+
177
+ // Watch through the step end + the widest possible re-check window
178
+ // + the answer. No message matching the busy-ack shape may appear.
179
+ let prematureAck: ObservedMessage | null = null;
180
+ try {
181
+ prematureAck = await sc.expectMessage(
182
+ (m: ObservedMessage) => m.fromBot && !m.edited && BUSY_ACK_RE.test(m.text),
183
+ { from: "bot", timeout: 45_000 },
184
+ );
185
+ } catch {
186
+ // timeout = the correct outcome (no ack, ever)
187
+ }
188
+ expect(
189
+ prematureAck,
190
+ `a busy-ack fired for a step that never reached the threshold ` +
191
+ `(${JSON.stringify(prematureAck?.text.slice(0, 120))}) — the ` +
192
+ `agent was seconds from answering; the threshold/re-check ` +
193
+ `gates are not holding.`,
194
+ ).toBeNull();
195
+ } finally {
196
+ await sc.tearDown();
197
+ }
198
+ },
199
+ 150_000,
200
+ );
201
+ });
@@ -0,0 +1,208 @@
1
+ /**
2
+ * JTBD — worker-pin lifecycle + the restart rule (#3001, DM).
3
+ *
4
+ * The job spec (`reference/jobs/know-what-my-agent-is-doing.md`) sanctions
5
+ * exactly one pin: silently pinning the already-rendered status message while
6
+ * its work is in flight, auto-unpinned when the work completes. #3001's
7
+ * operator report was the failure mode: worker (`wk:`) pins with no
8
+ * mid-session reaper and a forfeit-on-failure boot sweep left a long tail of
9
+ * stale pinned `🛠 Worker` messages. The restart rule this scenario pins:
10
+ * **agent restart = reset any pinned message whose work didn't finish.**
11
+ *
12
+ * Two acceptance criteria, shared setup (a long background dispatch):
13
+ *
14
+ * AC-1 — lifecycle: the `🛠 Worker` feed message is PINNED while the
15
+ * background worker runs, and UNPINNED after it completes (the
16
+ * normal completion unpin — no restart involved).
17
+ * AC-2 — restart rule: with a SECOND dispatch pinned mid-flight, a forced
18
+ * gateway restart orphans the pin; the next boot's sweep
19
+ * (statusPinBootCleanup, reading status-pins.json) unpins it. The
20
+ * restart half self-skips green without NOPASSWD sudo, mirroring
21
+ * jtbd-message-during-restart-dm.
22
+ *
23
+ * Observation shape: pins/unpins are SERVICE events, not message sends or
24
+ * edits — the worker card EDITS in place the whole time it is pinned, so a
25
+ * message-observer predicate would fire on edits and prove nothing. We
26
+ * assert via `driver.observePins` (raw `updatePinnedMessages`), keyed to the
27
+ * exact pinned messageId, and explicitly ignore `m.edited` traffic.
28
+ *
29
+ * Env note: requires the harness agent's default
30
+ * `pin_status_while_working` (ON) and the worker activity feed enabled —
31
+ * same env as bg-sub-agent-dispatch-dm (SETUP.md §6).
32
+ */
33
+
34
+ import { describe, it, expect } from "vitest";
35
+ import { execSync, spawn } from "node:child_process";
36
+ import { spinUp } from "../harness.js";
37
+ import type { ObservedPin } from "../driver.js";
38
+ import { WORKER_FEED_RE } from "../assertions.js";
39
+
40
+ const AGENT = "test-harness";
41
+
42
+ // Same deterministic Option-1 dispatch prompt shape as
43
+ // bg-sub-agent-dispatch-dm: this scenario verifies the PIN infra, not the
44
+ // model's delegation judgment. ~60s of paced background work — long enough
45
+ // for the feed to paint (8s first-paint floor) and the wk: pin to land.
46
+ function bgDispatchPrompt(steps: number): string {
47
+ return (
48
+ `Use the Agent tool with subagent_type "general-purpose" and ` +
49
+ `run_in_background: true to dispatch a worker with this exact task: ` +
50
+ `"Do ${steps} steps, ONE AT A TIME, k = 1 through ${steps}. Before each ` +
51
+ `step write a brief one-sentence narration, then run \`sleep 2\` via the ` +
52
+ `Bash tool, then run \`echo step-k\` via the Bash tool (substitute the ` +
53
+ `real number for k). Run every sleep and every echo as its OWN separate ` +
54
+ `Bash call — never batch or chain them with && — and narrate before ` +
55
+ `each. Do not stop early; complete all ${steps} steps." After ` +
56
+ `dispatching, send a brief reply saying you've kicked off the worker.`
57
+ );
58
+ }
59
+
60
+ /** Wait for the next pin/unpin event matching `pred`. observePins yields
61
+ * service-level pin transitions only — message EDITS never surface here,
62
+ * which is exactly why this scenario observes pins, not messages. */
63
+ async function nextPinEvent(
64
+ iter: AsyncIterator<ObservedPin>,
65
+ pred: (p: ObservedPin) => boolean,
66
+ timeoutMs: number,
67
+ what: string,
68
+ ): Promise<ObservedPin> {
69
+ const deadline = Date.now() + timeoutMs;
70
+ while (Date.now() < deadline) {
71
+ const remaining = deadline - Date.now();
72
+ const race = await Promise.race([
73
+ iter.next(),
74
+ new Promise<"timeout">((r) => setTimeout(() => r("timeout"), remaining)),
75
+ ]);
76
+ if (race === "timeout") break;
77
+ if (race.done === true) break;
78
+ if (pred(race.value)) return race.value;
79
+ }
80
+ throw new Error(`nextPinEvent: no ${what} within ${timeoutMs}ms`);
81
+ }
82
+
83
+ function canShellSudo(): boolean {
84
+ try {
85
+ execSync("sudo -n true", { stdio: "ignore", timeout: 2_000 });
86
+ return true;
87
+ } catch {
88
+ return false;
89
+ }
90
+ }
91
+
92
+ /** Kick a marker-safe restart and return immediately (detached) — same shape
93
+ * as jtbd-message-during-restart-dm. */
94
+ function kickRestartDetached(name: string): void {
95
+ const child = spawn(
96
+ "sudo",
97
+ ["-n", "env", `PATH=${process.env.PATH}`, `HOME=${process.env.HOME}`,
98
+ "switchroom", "agent", "restart", name, "--force"],
99
+ { detached: true, stdio: "ignore" },
100
+ );
101
+ child.unref();
102
+ }
103
+
104
+ const sudoOk = canShellSudo();
105
+
106
+ describe("uat: worker-pin lifecycle + restart rule (#3001, DM)", () => {
107
+ it(
108
+ "AC-1: the worker feed message is pinned while the bg dispatch runs and unpinned after completion",
109
+ async () => {
110
+ const sc = await spinUp({ agent: AGENT });
111
+ const pinIter = sc.driver
112
+ .observePins(sc.botUserId)
113
+ [Symbol.asyncIterator]();
114
+ try {
115
+ await sc.sendDM(bgDispatchPrompt(10));
116
+
117
+ // Parent ack — the dispatch happened.
118
+ await sc.expectMessage(/.+/, { from: "bot", timeout: 45_000 });
119
+
120
+ // The wk: pin lands once the worker feed paints (~8s first-paint) —
121
+ // a PIN service event, not a send/edit.
122
+ const pinned = await nextPinEvent(
123
+ pinIter,
124
+ (p) => p.pinned,
125
+ 90_000,
126
+ "worker pin",
127
+ );
128
+ expect(pinned.messageId).toBeGreaterThan(0);
129
+
130
+ // Sanity: what got pinned is the 🛠 Worker feed message (fetch by id
131
+ // — the card keeps EDITING in place; we never assert on edits).
132
+ const msg = await sc.driver.getMessage(sc.botUserId, pinned.messageId);
133
+ expect(msg, "pinned message vanished").not.toBeNull();
134
+ expect(msg!.text).toMatch(WORKER_FEED_RE);
135
+
136
+ // Completion unpins it. Budget mirrors bg-sub-agent-dispatch-dm's
137
+ // done-window: worker (~60s) + watcher stall detection (60s) + slack.
138
+ const unpinned = await nextPinEvent(
139
+ pinIter,
140
+ (p) => !p.pinned && p.messageId === pinned.messageId,
141
+ 240_000,
142
+ `unpin of worker pin msg=${pinned.messageId}`,
143
+ );
144
+ expect(unpinned.pinned).toBe(false);
145
+ } finally {
146
+ await pinIter.return?.();
147
+ await sc.tearDown();
148
+ }
149
+ },
150
+ // 45s ack + 90s pin + 240s unpin + spinUp/teardown slack.
151
+ 420_000,
152
+ );
153
+
154
+ (sudoOk ? it : it.skip)(
155
+ "AC-2 (restart rule): a worker pin orphaned by a forced gateway restart is unpinned by the next boot's sweep",
156
+ async () => {
157
+ const sc = await spinUp({ agent: AGENT });
158
+ const pinIter = sc.driver
159
+ .observePins(sc.botUserId)
160
+ [Symbol.asyncIterator]();
161
+ try {
162
+ // Longer dispatch (~2min of steps) so the restart reliably lands
163
+ // while the worker — and its pin — are still in flight.
164
+ await sc.sendDM(bgDispatchPrompt(30));
165
+ await sc.expectMessage(/.+/, { from: "bot", timeout: 45_000 });
166
+
167
+ // Latch onto the 🛠 Worker feed pin specifically — the parent turn's
168
+ // own fg: status pin can land first, and a bare `p.pinned` predicate
169
+ // would seize it and then wait on the wrong messageId's unpin. Keep
170
+ // consuming pin events until one resolves to a worker-feed message
171
+ // (AC-1's sanity fetch, applied inside the wait loop).
172
+ const pinDeadline = Date.now() + 90_000;
173
+ let pinned: ObservedPin | null = null;
174
+ while (pinned == null) {
175
+ const candidate = await nextPinEvent(
176
+ pinIter,
177
+ (p) => p.pinned,
178
+ Math.max(1, pinDeadline - Date.now()),
179
+ "worker-feed pin",
180
+ );
181
+ const msg = await sc.driver
182
+ .getMessage(sc.botUserId, candidate.messageId)
183
+ .catch(() => null);
184
+ if (msg != null && WORKER_FEED_RE.test(msg.text)) pinned = candidate;
185
+ }
186
+
187
+ // Force the restart MID-dispatch: the wk: claim is persisted in
188
+ // status-pins.json; the dying session never runs its completion
189
+ // unpin. The next boot's statusPinBootCleanup must unpin it.
190
+ kickRestartDetached(AGENT);
191
+
192
+ const unpinned = await nextPinEvent(
193
+ pinIter,
194
+ (p) => !p.pinned && p.messageId === pinned.messageId,
195
+ // Restart (docker recreate + gateway boot + startup mutex + boot
196
+ // sweep) comfortably inside this window.
197
+ 240_000,
198
+ `boot-sweep unpin of orphaned worker pin msg=${pinned.messageId}`,
199
+ );
200
+ expect(unpinned.pinned).toBe(false);
201
+ } finally {
202
+ await pinIter.return?.();
203
+ await sc.tearDown();
204
+ }
205
+ },
206
+ 420_000,
207
+ );
208
+ });