switchroom 0.18.7 → 0.18.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. package/README.md +2 -2
  2. package/dist/cli/switchroom.js +905 -758
  3. package/dist/host-control/main.js +1 -1
  4. package/package.json +1 -1
  5. package/profiles/_base/start.sh.hbs +111 -34
  6. package/skills/switchroom-runtime/SKILL.md +2 -0
  7. package/telegram-plugin/dist/gateway/gateway.js +46273 -44324
  8. package/telegram-plugin/flood-circuit-breaker.ts +123 -0
  9. package/telegram-plugin/gateway/activity-card-store.ts +63 -18
  10. package/telegram-plugin/gateway/approval-card-stores.ts +99 -0
  11. package/telegram-plugin/gateway/boot-card.ts +27 -0
  12. package/telegram-plugin/gateway/bot-commands-ops-info.ts +194 -0
  13. package/telegram-plugin/gateway/busy-ack.ts +106 -0
  14. package/telegram-plugin/gateway/callback-query-handlers.ts +2660 -0
  15. package/telegram-plugin/gateway/gateway.ts +1169 -3043
  16. package/telegram-plugin/gateway/inbound-delivery-machine-dispatch.ts +181 -23
  17. package/telegram-plugin/gateway/inbound-delivery-machine.ts +8 -0
  18. package/telegram-plugin/gateway/mental-model-propose-diff.ts +61 -5
  19. package/telegram-plugin/gateway/model-command.ts +23 -11
  20. package/telegram-plugin/gateway/outbound-send-path.ts +375 -0
  21. package/telegram-plugin/gateway/pending-state-stores.ts +106 -0
  22. package/telegram-plugin/gateway/register-bot-commands.ts +30 -0
  23. package/telegram-plugin/gateway/session-model-file.ts +198 -0
  24. package/telegram-plugin/gateway/status-pin-store.ts +82 -22
  25. package/telegram-plugin/gateway/worker-pin-reaper.ts +114 -0
  26. package/telegram-plugin/hooks/hooks.json +10 -10
  27. package/telegram-plugin/hooks/run-hook.sh +84 -0
  28. package/telegram-plugin/model-unavailable.ts +26 -0
  29. package/telegram-plugin/pty-partial-handler.ts +39 -0
  30. package/telegram-plugin/render/rich-render.ts +79 -1
  31. package/telegram-plugin/retry-api-call.ts +62 -0
  32. package/telegram-plugin/shared/bot-runtime.ts +8 -1
  33. package/telegram-plugin/silence-poke.ts +14 -0
  34. package/telegram-plugin/stream-controller.ts +156 -38
  35. package/telegram-plugin/tests/activity-card-store.test.ts +47 -2
  36. package/telegram-plugin/tests/approval-card-restart-outcome.test.ts +218 -0
  37. package/telegram-plugin/tests/approval-card-stores.test.ts +124 -0
  38. package/telegram-plugin/tests/boot-card-flood-suppress.test.ts +111 -0
  39. package/telegram-plugin/tests/busy-ack-wiring.test.ts +118 -0
  40. package/telegram-plugin/tests/busy-ack.test.ts +121 -0
  41. package/telegram-plugin/tests/callback-query-handlers.test.ts +701 -0
  42. package/telegram-plugin/tests/emission-determinism-wiring.test.ts +11 -4
  43. package/telegram-plugin/tests/fixtures/cutover-killswitch-probe.ts +75 -0
  44. package/telegram-plugin/tests/flood-circuit-breaker.test.ts +74 -0
  45. package/telegram-plugin/tests/gateway-outbound-redact.test.ts +5 -1
  46. package/telegram-plugin/tests/gateway-session-model-relaunch.test.ts +177 -25
  47. package/telegram-plugin/tests/inbound-delivery-cutover-flip.test.ts +418 -0
  48. package/telegram-plugin/tests/inbound-delivery-dispatch-equivalence.test.ts +348 -0
  49. package/telegram-plugin/tests/inbound-delivery-machine-dispatch.test.ts +141 -52
  50. package/telegram-plugin/tests/mental-model-name-entity-corruption.test.ts +119 -0
  51. package/telegram-plugin/tests/mental-model-propose-callback-gate.test.ts +8 -1
  52. package/telegram-plugin/tests/model-command.test.ts +2 -2
  53. package/telegram-plugin/tests/model-unavailable.test.ts +41 -0
  54. package/telegram-plugin/tests/outbound-send-chunks.test.ts +304 -0
  55. package/telegram-plugin/tests/outbound-send-path.test.ts +222 -0
  56. package/telegram-plugin/tests/pending-card-durability-wiring.test.ts +34 -15
  57. package/telegram-plugin/tests/pending-state-stores.test.ts +235 -0
  58. package/telegram-plugin/tests/pty-partial-handler.test.ts +56 -0
  59. package/telegram-plugin/tests/render/render-outbound-chunks.test.ts +98 -0
  60. package/telegram-plugin/tests/retry-api-call.test.ts +59 -0
  61. package/telegram-plugin/tests/run-hook-wrapper.test.ts +132 -0
  62. package/telegram-plugin/tests/session-model-file.test.ts +132 -0
  63. package/telegram-plugin/tests/slot-banner-boot-recovery.test.ts +3 -3
  64. package/telegram-plugin/tests/status-pin-boot-recovery.test.ts +3 -3
  65. package/telegram-plugin/tests/status-pin-store.test.ts +62 -6
  66. package/telegram-plugin/tests/stream-controller-chunk-cap.test.ts +122 -0
  67. package/telegram-plugin/tests/turn-flush-safety.test.ts +18 -4
  68. package/telegram-plugin/tests/vault-approval-posture.test.ts +15 -7
  69. package/telegram-plugin/tests/vault-grant-auto-resume.test.ts +8 -4
  70. package/telegram-plugin/tests/vault-grant-union.test.ts +8 -4
  71. package/telegram-plugin/tests/vault-grant-wizard.test.ts +8 -1
  72. package/telegram-plugin/tests/vault-grants-revoke.test.ts +8 -1
  73. package/telegram-plugin/tests/vault-key-regex-allows-slash.test.ts +8 -4
  74. package/telegram-plugin/tests/vault-request-access-tool.test.ts +8 -4
  75. package/telegram-plugin/tests/vault-request-access-unlock-resume.test.ts +8 -4
  76. package/telegram-plugin/tests/voice-send.test.ts +308 -0
  77. package/telegram-plugin/tests/worker-pin-reaper.test.ts +132 -0
  78. package/telegram-plugin/uat/scenarios/jtbd-deliberate-restart-resumes-dm.test.ts +118 -0
  79. package/telegram-plugin/uat/scenarios/jtbd-midflight-busy-ack-dm.test.ts +201 -0
  80. package/telegram-plugin/uat/scenarios/jtbd-worker-pin-lifecycle-dm.test.ts +208 -0
  81. package/telegram-plugin/uat/scenarios/vault-card-survives-gateway-restart-dm.test.ts +140 -0
  82. package/telegram-plugin/uat/scenarios/vault-deny-resumes-turn-dm.test.ts +84 -0
  83. package/telegram-plugin/uat/scenarios/vault-timeout-wakes-agent-dm.test.ts +91 -0
  84. package/telegram-plugin/voice-ondemand.ts +25 -1
  85. package/telegram-plugin/voice-send.ts +154 -0
@@ -0,0 +1,132 @@
1
+ import { describe, it, expect } from "vitest";
2
+ import {
3
+ decideWorkerPinReaps,
4
+ workerAgentIdOfPinKey,
5
+ WORKER_PIN_TTL_MS_DEFAULT,
6
+ type WorkerPinCandidate,
7
+ } from "../gateway/worker-pin-reaper.js";
8
+
9
+ const NOW = 1_750_000_000_000;
10
+ const TTL = WORKER_PIN_TTL_MS_DEFAULT;
11
+
12
+ function pin(over: Partial<WorkerPinCandidate> = {}): WorkerPinCandidate {
13
+ return { pinKey: "wk:agent-1", chatId: "123", pinnedAt: NOW - 60_000, ...over };
14
+ }
15
+
16
+ describe("workerAgentIdOfPinKey", () => {
17
+ it("extracts the agentId from a wk: key", () => {
18
+ expect(workerAgentIdOfPinKey("wk:abc123")).toBe("abc123");
19
+ });
20
+ it("returns null for non-worker keys (fg:/banner:/tool:) and empty ids", () => {
21
+ expect(workerAgentIdOfPinKey("fg:123:5")).toBeNull();
22
+ expect(workerAgentIdOfPinKey("banner:owner")).toBeNull();
23
+ expect(workerAgentIdOfPinKey("tool:123:9")).toBeNull();
24
+ expect(workerAgentIdOfPinKey("wk:")).toBeNull();
25
+ });
26
+ });
27
+
28
+ describe("decideWorkerPinReaps (#3001 mid-session wk: sweep)", () => {
29
+ it("reaps a pin whose worker is terminal, however young the pin is", () => {
30
+ const reaps = decideWorkerPinReaps({
31
+ pins: [pin({ pinnedAt: NOW - 1_000 })],
32
+ statusOf: () => 'terminal' as const,
33
+ ttlMs: TTL,
34
+ now: NOW,
35
+ });
36
+ expect(reaps).toHaveLength(1);
37
+ expect(reaps[0].reason).toBe("terminal");
38
+ expect(reaps[0].pinKey).toBe("wk:agent-1");
39
+ });
40
+
41
+ it("reaps a non-terminal pin once it exceeds the TTL", () => {
42
+ const reaps = decideWorkerPinReaps({
43
+ pins: [pin({ pinnedAt: NOW - TTL })],
44
+ statusOf: () => 'unknown' as const,
45
+ ttlMs: TTL,
46
+ now: NOW,
47
+ });
48
+ expect(reaps).toHaveLength(1);
49
+ expect(reaps[0].reason).toBe("ttl");
50
+ });
51
+
52
+ it("NEVER touches a pin younger than the TTL when the registry cannot vouch (unknown)", () => {
53
+ const reaps = decideWorkerPinReaps({
54
+ pins: [pin({ pinnedAt: NOW - TTL + 1 })],
55
+ statusOf: () => 'unknown' as const,
56
+ ttlMs: TTL,
57
+ now: NOW,
58
+ });
59
+ expect(reaps).toHaveLength(0);
60
+ });
61
+
62
+ it("ignores non-worker keys entirely (fg:/banner:/tool: are not its job)", () => {
63
+ const reaps = decideWorkerPinReaps({
64
+ pins: [
65
+ pin({ pinKey: "fg:123:5", pinnedAt: NOW - 10 * TTL }),
66
+ pin({ pinKey: "banner:owner", pinnedAt: NOW - 10 * TTL }),
67
+ pin({ pinKey: "tool:123:9", pinnedAt: NOW - 10 * TTL }),
68
+ ],
69
+ statusOf: () => 'terminal' as const,
70
+ ttlMs: TTL,
71
+ now: NOW,
72
+ });
73
+ expect(reaps).toHaveLength(0);
74
+ });
75
+
76
+ it("skips a candidate with no chat id (cannot unpin without one)", () => {
77
+ const reaps = decideWorkerPinReaps({
78
+ pins: [pin({ chatId: "", pinnedAt: NOW - 10 * TTL })],
79
+ statusOf: () => 'terminal' as const,
80
+ ttlMs: TTL,
81
+ now: NOW,
82
+ });
83
+ expect(reaps).toHaveLength(0);
84
+ });
85
+
86
+ it("a registry-confirmed RUNNING worker keeps its pin PAST the TTL (no unpin/re-pin churn on healthy long workers)", () => {
87
+ const reaps = decideWorkerPinReaps({
88
+ pins: [pin({ pinnedAt: NOW - 10 * TTL })],
89
+ statusOf: () => 'running' as const,
90
+ ttlMs: TTL,
91
+ now: NOW,
92
+ });
93
+ expect(reaps).toHaveLength(0);
94
+ });
95
+
96
+ it("status is consulted per-agent: terminal reaps young, running exempts old, unknown falls to the TTL gate", () => {
97
+ const seen: string[] = [];
98
+ const reaps = decideWorkerPinReaps({
99
+ pins: [
100
+ pin({ pinKey: "wk:done", pinnedAt: NOW - 1_000 }),
101
+ pin({ pinKey: "wk:running-old", pinnedAt: NOW - TTL - 1 }),
102
+ pin({ pinKey: "wk:unknown-young", pinnedAt: NOW - 1_000 }),
103
+ pin({ pinKey: "wk:unknown-old", pinnedAt: NOW - TTL - 1 }),
104
+ ],
105
+ statusOf: (agentId) => {
106
+ seen.push(agentId);
107
+ if (agentId === "done") return "terminal";
108
+ if (agentId === "running-old") return "running";
109
+ return "unknown";
110
+ },
111
+ ttlMs: TTL,
112
+ now: NOW,
113
+ });
114
+ expect(seen).toEqual(["done", "running-old", "unknown-young", "unknown-old"]);
115
+ expect(reaps.map((r) => [r.pinKey, r.reason])).toEqual([
116
+ ["wk:done", "terminal"],
117
+ ["wk:unknown-old", "ttl"],
118
+ ]);
119
+ });
120
+
121
+ it("registry hiccup ('unknown') never unpins a young pin — the caller's contract on statusOf degrade", () => {
122
+ // The gateway wraps its DB lookup and degrades to 'unknown'. This pins the
123
+ // pure module's side of the contract: unknown + young pin = untouched.
124
+ const reaps = decideWorkerPinReaps({
125
+ pins: [pin()],
126
+ statusOf: () => 'unknown' as const,
127
+ ttlMs: TTL,
128
+ now: NOW,
129
+ });
130
+ expect(reaps).toHaveLength(0);
131
+ });
132
+ });
@@ -0,0 +1,118 @@
1
+ /**
2
+ * JTBD scenario named by `reference/jobs/survive-reboots-and-real-life.md`
3
+ * § Prove it — "Deliberate restart resumes, bounded (DM)".
4
+ *
5
+ * Contract under test (#2988): a DELIBERATE restart (operator
6
+ * `switchroom agent restart`, i.e. a clean SIGTERM shutdown with a
7
+ * clean-shutdown marker) that lands MID-TURN resumes the interrupted turn
8
+ * like a crash-class interruption. The quota-saving suppression applies only
9
+ * when nothing was in flight — it never swallows mid-turn work. And the
10
+ * resume is bounded: AT MOST ONCE per interruption (the loop-guard caps the
11
+ * resume chain), so we also assert no second resume-framed turn fires after
12
+ * the first completes.
13
+ *
14
+ * Sibling: `jtbd-interrupted-turn-resumes-dm.test.ts` (same interruption
15
+ * mechanics; this scenario adds the deliberate-restart class + the
16
+ * at-most-once bound). Decision-level twin: `decideBootResumeKind` cases in
17
+ * `tests/resume-inbound-builder.test.ts` (BOUNDED CHAIN) and
18
+ * `shouldSuppressBootResume` in `tests/gateway-clean-shutdown-marker.test.ts`.
19
+ *
20
+ * Self-skips green without NOPASSWD sudo.
21
+ */
22
+
23
+ import { describe, it, expect } from "vitest";
24
+ import { execSync, spawn } from "node:child_process";
25
+ import { spinUp } from "../harness.js";
26
+
27
+ const AGENT = "test-harness";
28
+ const MID_TURN_MS = 10_000; // let the turn get in-flight before the bounce
29
+ const RESUME_BUDGET_MS = 180_000; // boot + resume + reply
30
+ const QUIET_WINDOW_MS = 90_000; // after the resume completes, no second resume may fire
31
+ // Messages landing within this window of the FIRST resume-framed reply are
32
+ // treated as the same resumed turn (e.g. a progress line plus the final
33
+ // answer of one turn) — only a resume-framed message AFTER it counts as a
34
+ // second resume for the at-most-once bound.
35
+ const SAME_TURN_GRACE_MS = 20_000;
36
+
37
+ const RESUME_FRAMING = /resum|picking .*back|interrupted|cut off|just restarted/i;
38
+
39
+ function canShellSudo(): boolean {
40
+ try {
41
+ execSync("sudo -n true", { stdio: "ignore", timeout: 2_000 });
42
+ return true;
43
+ } catch {
44
+ return false;
45
+ }
46
+ }
47
+
48
+ function kickRestartDetached(name: string): void {
49
+ // A deliberate operator restart: clean SIGTERM path, clean-shutdown marker
50
+ // written — the exact class #2988 makes resume in-flight work through.
51
+ const child = spawn(
52
+ "sudo",
53
+ ["-n", "env", `PATH=${process.env.PATH}`, `HOME=${process.env.HOME}`,
54
+ "switchroom", "agent", "restart", name, "--force"],
55
+ { detached: true, stdio: "ignore" },
56
+ );
57
+ child.unref();
58
+ }
59
+
60
+ const sudoOk = canShellSudo();
61
+
62
+ (sudoOk ? describe : describe.skip)(
63
+ "uat: deliberate restart mid-turn resumes exactly once (DM, #2988)",
64
+ () => {
65
+ it(
66
+ "an operator restart mid-turn resumes the work once, and never a second time",
67
+ async () => {
68
+ const sc = await spinUp({ agent: AGENT, settleMs: 0 });
69
+ try {
70
+ // A task long enough to still be in flight at MID_TURN_MS.
71
+ await sc.sendDM(
72
+ `Please write a thorough, detailed ~300-word explanation of how a ` +
73
+ `player piano's pneumatic action reads a music roll, step by step. ` +
74
+ `Take your time and be complete.`,
75
+ );
76
+
77
+ await new Promise((r) => setTimeout(r, MID_TURN_MS));
78
+ kickRestartDetached(AGENT);
79
+
80
+ // 1. In-flight work resumes: quota-saving suppression must NOT
81
+ // swallow a turn that was mid-flight at the bounce. Edits are
82
+ // excluded — a progress-card edit must not satisfy this; only
83
+ // a genuinely NEW message from the resumed turn counts.
84
+ const reply = await sc.expectMessage(
85
+ (m) => !m.edited && RESUME_FRAMING.test(m.text),
86
+ { from: "bot", timeout: RESUME_BUDGET_MS },
87
+ );
88
+ expect(reply.text).toMatch(RESUME_FRAMING);
89
+ expect(reply.text.length).toBeGreaterThan(40);
90
+
91
+ // 2. Bounded chain: with the interruption resolved, no SECOND
92
+ // resume-framed turn may fire (at-most-once per interruption).
93
+ // Edits are excluded, and messages within SAME_TURN_GRACE_MS of
94
+ // the first reply are deduped as part of the SAME resumed turn
95
+ // (two resume-ish lines in one turn are not a second resume).
96
+ const firstReplyAt = reply.date.getTime();
97
+ let second: unknown = null;
98
+ try {
99
+ second = await sc.expectMessage(
100
+ (m) =>
101
+ !m.edited &&
102
+ m.messageId !== reply.messageId &&
103
+ m.date.getTime() > firstReplyAt + SAME_TURN_GRACE_MS &&
104
+ RESUME_FRAMING.test(m.text),
105
+ { from: "bot", timeout: QUIET_WINDOW_MS },
106
+ );
107
+ } catch {
108
+ // timeout is the PASS: no repeat resume.
109
+ }
110
+ expect(second).toBeNull();
111
+ } finally {
112
+ await sc.tearDown();
113
+ }
114
+ },
115
+ RESUME_BUDGET_MS + QUIET_WINDOW_MS + 120_000,
116
+ );
117
+ },
118
+ );
@@ -0,0 +1,201 @@
1
+ /**
2
+ * JTBD scenario — mid-flight busy ack (#2995).
3
+ *
4
+ * Serves: `reference/jobs/steer-or-queue-mid-flight.md` — the third
5
+ * mid-flight class (a quick question that neither steers nor queues
6
+ * work of its own) and its latency promise: a mid-flight message
7
+ * arriving while the agent sits inside ONE long blocking tool call
8
+ * gets a visible, silent ack naming the blocking activity within
9
+ * seconds — never minutes of dead air behind a `--watch`-style call.
10
+ *
11
+ * ## What the scenario drives
12
+ *
13
+ * 1. Prompt the agent into a deliberately slow BLOCKING foreground
14
+ * Bash (`sleep`) — explicitly instructed NOT to background it.
15
+ * 2. Wait past the busy-ack step-age threshold (12s), then send a
16
+ * trivial mid-turn status question. It buffers (unmarked mid-turn
17
+ * follow-up = queue).
18
+ * 3. Assert a bot message matching the deterministic busy-ack shape
19
+ * ("⏳ Queued — currently inside `…`") lands within ACK_SLA_MS of
20
+ * the ping, and is SILENT (disable_notification).
21
+ * 4. Assert the real answer to the question arrives later (after the
22
+ * blocking step returns and the buffer flushes).
23
+ * 5. Assert the busy-ack card was cleaned up (deleted) once the
24
+ * answer landed — never a dangling "Queued" line.
25
+ *
26
+ * Negative case: a ping during a SHORT blocking step (well under the 12s
27
+ * threshold, and finished before the deferred re-check could ever see a
28
+ * 12s-old step) gets NO busy-ack card at any point — an ack that fires
29
+ * when the agent was seconds from answering is the job spec's explicit
30
+ * Bad bullet. (A young-step ping during a LONG step DOES get a deferred
31
+ * card once the step ages past the threshold — that is the re-check
32
+ * behaviour, covered by the positive case's latency budget.)
33
+ *
34
+ * ## Conventions
35
+ *
36
+ * - mtcute harness (`spinUp`), skip discipline like siblings: env
37
+ * handled by load-env; the suite hard-fails without creds by design
38
+ * (CI path-gates it to the uat-host runner).
39
+ * - `!m.edited` filters on every observation (PR #2991 lesson — the
40
+ * live draft/feed edits stream constantly; edits must never satisfy
41
+ * a fresh-message expectation).
42
+ */
43
+
44
+ import { describe, expect, it } from "vitest";
45
+ import { spinUp } from "../harness.js";
46
+ import type { ObservedMessage } from "../driver.js";
47
+
48
+ const AGENT = "test-harness";
49
+
50
+ // The gateway constant is 12s (BUSY_ACK_STEP_AGE_THRESHOLD_MS). The
51
+ // warmup is anchored to the SEND, not to observed tool start — the agent
52
+ // needs a few seconds to open the turn and enter the Bash step — so it
53
+ // carries a large margin: even if the tool starts ~15s after the send,
54
+ // the step is comfortably past the threshold when the ping lands.
55
+ const STEP_WARMUP_MS = 35_000;
56
+
57
+ // Latency promise: user-visible ack within seconds of the ping. The card
58
+ // is posted synchronously by the gateway (no model). When the ping lands
59
+ // past the threshold the card is immediate; if the step turns out to be
60
+ // just under, the deferred re-check posts it within (threshold − age).
61
+ // 10s + jitter headroom covers both.
62
+ const ACK_SLA_MS = 10_000;
63
+
64
+ // The blocking sleep. Long enough that warmup + ping + ack all land while
65
+ // the step is still open (even with a late tool start); short enough to
66
+ // keep wall-clock sane.
67
+ const SLEEP_SECS = 120;
68
+
69
+ const BUSY_ACK_RE = /⏳ Queued — currently inside `.+`.*I'll answer when this step finishes/s;
70
+
71
+ const SLOW_PROMPT =
72
+ `For a latency test, run this exact command in the FOREGROUND with your ` +
73
+ `Bash tool (do NOT use run_in_background, do NOT split it up): ` +
74
+ `\`sleep ${SLEEP_SECS} && echo done\`. When it completes, reply with ` +
75
+ `exactly: SLEEP_DONE. Do not send any message before it completes.`;
76
+
77
+ const PING = "Quick question while you work: are you still there? One word is fine.";
78
+
79
+ describe("uat: mid-flight busy ack during a long blocking tool call (#2995)", () => {
80
+ it(
81
+ "a mid-turn ping behind a long blocking Bash gets a silent Queued ack naming the activity, then the real answer, then cleanup",
82
+ async () => {
83
+ const sc = await spinUp({ agent: AGENT });
84
+ try {
85
+ await sc.sendDM(SLOW_PROMPT);
86
+
87
+ // Let the blocking step age past the busy-ack threshold.
88
+ await new Promise((r) => setTimeout(r, STEP_WARMUP_MS));
89
+
90
+ const pingAt = Date.now();
91
+ await sc.sendDM(PING);
92
+
93
+ // The deterministic busy-ack must land within the SLA, silent,
94
+ // naming the blocking activity. `!m.edited` — feed/draft edits
95
+ // must never satisfy this.
96
+ const ack = await sc.expectMessage(
97
+ (m: ObservedMessage) => m.fromBot && !m.edited && BUSY_ACK_RE.test(m.text),
98
+ { from: "bot", timeout: ACK_SLA_MS + 5_000 },
99
+ );
100
+ const ackLatency = Date.now() - pingAt;
101
+ expect(
102
+ ackLatency,
103
+ `busy-ack landed ${ackLatency}ms after the ping — the <10s ` +
104
+ `latency promise for a mid-flight quick question is broken.`,
105
+ ).toBeLessThan(ACK_SLA_MS + 5_000);
106
+ expect(
107
+ ack.silent,
108
+ `busy-ack pinged the user's device (silent=false) — the card ` +
109
+ `must carry disable_notification:true.`,
110
+ ).toBe(true);
111
+ // Names the blocking activity (the Bash sleep), not a generic line.
112
+ expect(ack.text).toMatch(/`.*(Bash|sleep).*`/i);
113
+
114
+ // The real answer to the ping arrives AFTER the blocking step
115
+ // returns and the buffer flushes. Any fresh non-card bot message
116
+ // that isn't the busy-ack or the SLEEP_DONE completion counts —
117
+ // the SLEEP_DONE exclusion is explicit (it answers the FIRST
118
+ // prompt, not the ping).
119
+ const answer = await sc.expectMessage(
120
+ (m: ObservedMessage) =>
121
+ m.fromBot &&
122
+ !m.edited &&
123
+ m.messageId !== ack.messageId &&
124
+ !BUSY_ACK_RE.test(m.text) &&
125
+ !/SLEEP_DONE/i.test(m.text) &&
126
+ m.text.trim().length > 0,
127
+ { from: "bot", timeout: (SLEEP_SECS + 90) * 1000 },
128
+ );
129
+ expect(answer.text.trim().length).toBeGreaterThan(0);
130
+
131
+ // Cleanup: once answers land and the turns wrap, the busy-ack
132
+ // card is deleted (delete-on-answer lifecycle). Poll briefly —
133
+ // the reap is fire-and-forget after the answer/turn-end.
134
+ const cleanupDeadline = Date.now() + 60_000;
135
+ let cardGone = false;
136
+ while (Date.now() < cleanupDeadline) {
137
+ const still = await sc.driver.getMessage(sc.botUserId, ack.messageId);
138
+ if (still == null) {
139
+ cardGone = true;
140
+ break;
141
+ }
142
+ await new Promise((r) => setTimeout(r, 3_000));
143
+ }
144
+ expect(
145
+ cardGone,
146
+ `busy-ack card (msg ${ack.messageId}) still present 60s after ` +
147
+ `the answer — a stale "Queued" line is dangling in the chat.`,
148
+ ).toBe(true);
149
+ } finally {
150
+ await sc.tearDown();
151
+ }
152
+ },
153
+ // warmup + sleep + flush + answer + cleanup poll + settle headroom
154
+ (STEP_WARMUP_MS + SLEEP_SECS * 1000 + 200_000),
155
+ );
156
+
157
+ it(
158
+ "negative: a ping during a SHORT step (finishes under the threshold) never gets a busy-ack card",
159
+ async () => {
160
+ const sc = await spinUp({ agent: AGENT });
161
+ try {
162
+ // An 8s sleep can NEVER age past the 12s threshold — neither the
163
+ // direct evaluation nor the deferred re-check may produce a card
164
+ // (the re-check re-reads live state and finds the step gone).
165
+ await sc.sendDM(
166
+ `For a latency test, run this exact command in the FOREGROUND ` +
167
+ `with your Bash tool (do NOT use run_in_background): ` +
168
+ `\`sleep 8 && echo done\`. When it completes, reply with ` +
169
+ `exactly: SHORT_SLEEP_DONE. Do not send any message before it completes.`,
170
+ );
171
+
172
+ // Give the agent a beat to enter the Bash step, then ping while
173
+ // the short step is (at most) a few seconds old.
174
+ await new Promise((r) => setTimeout(r, 4_000));
175
+ await sc.sendDM("Still with me?");
176
+
177
+ // Watch through the step end + the widest possible re-check window
178
+ // + the answer. No message matching the busy-ack shape may appear.
179
+ let prematureAck: ObservedMessage | null = null;
180
+ try {
181
+ prematureAck = await sc.expectMessage(
182
+ (m: ObservedMessage) => m.fromBot && !m.edited && BUSY_ACK_RE.test(m.text),
183
+ { from: "bot", timeout: 45_000 },
184
+ );
185
+ } catch {
186
+ // timeout = the correct outcome (no ack, ever)
187
+ }
188
+ expect(
189
+ prematureAck,
190
+ `a busy-ack fired for a step that never reached the threshold ` +
191
+ `(${JSON.stringify(prematureAck?.text.slice(0, 120))}) — the ` +
192
+ `agent was seconds from answering; the threshold/re-check ` +
193
+ `gates are not holding.`,
194
+ ).toBeNull();
195
+ } finally {
196
+ await sc.tearDown();
197
+ }
198
+ },
199
+ 150_000,
200
+ );
201
+ });
@@ -0,0 +1,208 @@
1
+ /**
2
+ * JTBD — worker-pin lifecycle + the restart rule (#3001, DM).
3
+ *
4
+ * The job spec (`reference/jobs/know-what-my-agent-is-doing.md`) sanctions
5
+ * exactly one pin: silently pinning the already-rendered status message while
6
+ * its work is in flight, auto-unpinned when the work completes. #3001's
7
+ * operator report was the failure mode: worker (`wk:`) pins with no
8
+ * mid-session reaper and a forfeit-on-failure boot sweep left a long tail of
9
+ * stale pinned `🛠 Worker` messages. The restart rule this scenario pins:
10
+ * **agent restart = reset any pinned message whose work didn't finish.**
11
+ *
12
+ * Two acceptance criteria, shared setup (a long background dispatch):
13
+ *
14
+ * AC-1 — lifecycle: the `🛠 Worker` feed message is PINNED while the
15
+ * background worker runs, and UNPINNED after it completes (the
16
+ * normal completion unpin — no restart involved).
17
+ * AC-2 — restart rule: with a SECOND dispatch pinned mid-flight, a forced
18
+ * gateway restart orphans the pin; the next boot's sweep
19
+ * (statusPinBootCleanup, reading status-pins.json) unpins it. The
20
+ * restart half self-skips green without NOPASSWD sudo, mirroring
21
+ * jtbd-message-during-restart-dm.
22
+ *
23
+ * Observation shape: pins/unpins are SERVICE events, not message sends or
24
+ * edits — the worker card EDITS in place the whole time it is pinned, so a
25
+ * message-observer predicate would fire on edits and prove nothing. We
26
+ * assert via `driver.observePins` (raw `updatePinnedMessages`), keyed to the
27
+ * exact pinned messageId, and explicitly ignore `m.edited` traffic.
28
+ *
29
+ * Env note: requires the harness agent's default
30
+ * `pin_status_while_working` (ON) and the worker activity feed enabled —
31
+ * same env as bg-sub-agent-dispatch-dm (SETUP.md §6).
32
+ */
33
+
34
+ import { describe, it, expect } from "vitest";
35
+ import { execSync, spawn } from "node:child_process";
36
+ import { spinUp } from "../harness.js";
37
+ import type { ObservedPin } from "../driver.js";
38
+ import { WORKER_FEED_RE } from "../assertions.js";
39
+
40
+ const AGENT = "test-harness";
41
+
42
+ // Same deterministic Option-1 dispatch prompt shape as
43
+ // bg-sub-agent-dispatch-dm: this scenario verifies the PIN infra, not the
44
+ // model's delegation judgment. ~60s of paced background work — long enough
45
+ // for the feed to paint (8s first-paint floor) and the wk: pin to land.
46
+ function bgDispatchPrompt(steps: number): string {
47
+ return (
48
+ `Use the Agent tool with subagent_type "general-purpose" and ` +
49
+ `run_in_background: true to dispatch a worker with this exact task: ` +
50
+ `"Do ${steps} steps, ONE AT A TIME, k = 1 through ${steps}. Before each ` +
51
+ `step write a brief one-sentence narration, then run \`sleep 2\` via the ` +
52
+ `Bash tool, then run \`echo step-k\` via the Bash tool (substitute the ` +
53
+ `real number for k). Run every sleep and every echo as its OWN separate ` +
54
+ `Bash call — never batch or chain them with && — and narrate before ` +
55
+ `each. Do not stop early; complete all ${steps} steps." After ` +
56
+ `dispatching, send a brief reply saying you've kicked off the worker.`
57
+ );
58
+ }
59
+
60
+ /** Wait for the next pin/unpin event matching `pred`. observePins yields
61
+ * service-level pin transitions only — message EDITS never surface here,
62
+ * which is exactly why this scenario observes pins, not messages. */
63
+ async function nextPinEvent(
64
+ iter: AsyncIterator<ObservedPin>,
65
+ pred: (p: ObservedPin) => boolean,
66
+ timeoutMs: number,
67
+ what: string,
68
+ ): Promise<ObservedPin> {
69
+ const deadline = Date.now() + timeoutMs;
70
+ while (Date.now() < deadline) {
71
+ const remaining = deadline - Date.now();
72
+ const race = await Promise.race([
73
+ iter.next(),
74
+ new Promise<"timeout">((r) => setTimeout(() => r("timeout"), remaining)),
75
+ ]);
76
+ if (race === "timeout") break;
77
+ if (race.done === true) break;
78
+ if (pred(race.value)) return race.value;
79
+ }
80
+ throw new Error(`nextPinEvent: no ${what} within ${timeoutMs}ms`);
81
+ }
82
+
83
+ function canShellSudo(): boolean {
84
+ try {
85
+ execSync("sudo -n true", { stdio: "ignore", timeout: 2_000 });
86
+ return true;
87
+ } catch {
88
+ return false;
89
+ }
90
+ }
91
+
92
+ /** Kick a marker-safe restart and return immediately (detached) — same shape
93
+ * as jtbd-message-during-restart-dm. */
94
+ function kickRestartDetached(name: string): void {
95
+ const child = spawn(
96
+ "sudo",
97
+ ["-n", "env", `PATH=${process.env.PATH}`, `HOME=${process.env.HOME}`,
98
+ "switchroom", "agent", "restart", name, "--force"],
99
+ { detached: true, stdio: "ignore" },
100
+ );
101
+ child.unref();
102
+ }
103
+
104
+ const sudoOk = canShellSudo();
105
+
106
+ describe("uat: worker-pin lifecycle + restart rule (#3001, DM)", () => {
107
+ it(
108
+ "AC-1: the worker feed message is pinned while the bg dispatch runs and unpinned after completion",
109
+ async () => {
110
+ const sc = await spinUp({ agent: AGENT });
111
+ const pinIter = sc.driver
112
+ .observePins(sc.botUserId)
113
+ [Symbol.asyncIterator]();
114
+ try {
115
+ await sc.sendDM(bgDispatchPrompt(10));
116
+
117
+ // Parent ack — the dispatch happened.
118
+ await sc.expectMessage(/.+/, { from: "bot", timeout: 45_000 });
119
+
120
+ // The wk: pin lands once the worker feed paints (~8s first-paint) —
121
+ // a PIN service event, not a send/edit.
122
+ const pinned = await nextPinEvent(
123
+ pinIter,
124
+ (p) => p.pinned,
125
+ 90_000,
126
+ "worker pin",
127
+ );
128
+ expect(pinned.messageId).toBeGreaterThan(0);
129
+
130
+ // Sanity: what got pinned is the 🛠 Worker feed message (fetch by id
131
+ // — the card keeps EDITING in place; we never assert on edits).
132
+ const msg = await sc.driver.getMessage(sc.botUserId, pinned.messageId);
133
+ expect(msg, "pinned message vanished").not.toBeNull();
134
+ expect(msg!.text).toMatch(WORKER_FEED_RE);
135
+
136
+ // Completion unpins it. Budget mirrors bg-sub-agent-dispatch-dm's
137
+ // done-window: worker (~60s) + watcher stall detection (60s) + slack.
138
+ const unpinned = await nextPinEvent(
139
+ pinIter,
140
+ (p) => !p.pinned && p.messageId === pinned.messageId,
141
+ 240_000,
142
+ `unpin of worker pin msg=${pinned.messageId}`,
143
+ );
144
+ expect(unpinned.pinned).toBe(false);
145
+ } finally {
146
+ await pinIter.return?.();
147
+ await sc.tearDown();
148
+ }
149
+ },
150
+ // 45s ack + 90s pin + 240s unpin + spinUp/teardown slack.
151
+ 420_000,
152
+ );
153
+
154
+ (sudoOk ? it : it.skip)(
155
+ "AC-2 (restart rule): a worker pin orphaned by a forced gateway restart is unpinned by the next boot's sweep",
156
+ async () => {
157
+ const sc = await spinUp({ agent: AGENT });
158
+ const pinIter = sc.driver
159
+ .observePins(sc.botUserId)
160
+ [Symbol.asyncIterator]();
161
+ try {
162
+ // Longer dispatch (~2min of steps) so the restart reliably lands
163
+ // while the worker — and its pin — are still in flight.
164
+ await sc.sendDM(bgDispatchPrompt(30));
165
+ await sc.expectMessage(/.+/, { from: "bot", timeout: 45_000 });
166
+
167
+ // Latch onto the 🛠 Worker feed pin specifically — the parent turn's
168
+ // own fg: status pin can land first, and a bare `p.pinned` predicate
169
+ // would seize it and then wait on the wrong messageId's unpin. Keep
170
+ // consuming pin events until one resolves to a worker-feed message
171
+ // (AC-1's sanity fetch, applied inside the wait loop).
172
+ const pinDeadline = Date.now() + 90_000;
173
+ let pinned: ObservedPin | null = null;
174
+ while (pinned == null) {
175
+ const candidate = await nextPinEvent(
176
+ pinIter,
177
+ (p) => p.pinned,
178
+ Math.max(1, pinDeadline - Date.now()),
179
+ "worker-feed pin",
180
+ );
181
+ const msg = await sc.driver
182
+ .getMessage(sc.botUserId, candidate.messageId)
183
+ .catch(() => null);
184
+ if (msg != null && WORKER_FEED_RE.test(msg.text)) pinned = candidate;
185
+ }
186
+
187
+ // Force the restart MID-dispatch: the wk: claim is persisted in
188
+ // status-pins.json; the dying session never runs its completion
189
+ // unpin. The next boot's statusPinBootCleanup must unpin it.
190
+ kickRestartDetached(AGENT);
191
+
192
+ const unpinned = await nextPinEvent(
193
+ pinIter,
194
+ (p) => !p.pinned && p.messageId === pinned.messageId,
195
+ // Restart (docker recreate + gateway boot + startup mutex + boot
196
+ // sweep) comfortably inside this window.
197
+ 240_000,
198
+ `boot-sweep unpin of orphaned worker pin msg=${pinned.messageId}`,
199
+ );
200
+ expect(unpinned.pinned).toBe(false);
201
+ } finally {
202
+ await pinIter.return?.();
203
+ await sc.tearDown();
204
+ }
205
+ },
206
+ 420_000,
207
+ );
208
+ });