switchroom 0.18.7 → 0.18.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/cli/switchroom.js +905 -758
- package/dist/host-control/main.js +1 -1
- package/package.json +1 -1
- package/profiles/_base/start.sh.hbs +111 -34
- package/skills/switchroom-runtime/SKILL.md +2 -0
- package/telegram-plugin/dist/gateway/gateway.js +46273 -44324
- package/telegram-plugin/flood-circuit-breaker.ts +123 -0
- package/telegram-plugin/gateway/activity-card-store.ts +63 -18
- package/telegram-plugin/gateway/approval-card-stores.ts +99 -0
- package/telegram-plugin/gateway/boot-card.ts +27 -0
- package/telegram-plugin/gateway/bot-commands-ops-info.ts +194 -0
- package/telegram-plugin/gateway/busy-ack.ts +106 -0
- package/telegram-plugin/gateway/callback-query-handlers.ts +2660 -0
- package/telegram-plugin/gateway/gateway.ts +1169 -3043
- package/telegram-plugin/gateway/inbound-delivery-machine-dispatch.ts +181 -23
- package/telegram-plugin/gateway/inbound-delivery-machine.ts +8 -0
- package/telegram-plugin/gateway/mental-model-propose-diff.ts +61 -5
- package/telegram-plugin/gateway/model-command.ts +23 -11
- package/telegram-plugin/gateway/outbound-send-path.ts +375 -0
- package/telegram-plugin/gateway/pending-state-stores.ts +106 -0
- package/telegram-plugin/gateway/register-bot-commands.ts +30 -0
- package/telegram-plugin/gateway/session-model-file.ts +198 -0
- package/telegram-plugin/gateway/status-pin-store.ts +82 -22
- package/telegram-plugin/gateway/worker-pin-reaper.ts +114 -0
- package/telegram-plugin/hooks/hooks.json +10 -10
- package/telegram-plugin/hooks/run-hook.sh +84 -0
- package/telegram-plugin/model-unavailable.ts +26 -0
- package/telegram-plugin/pty-partial-handler.ts +39 -0
- package/telegram-plugin/render/rich-render.ts +79 -1
- package/telegram-plugin/retry-api-call.ts +62 -0
- package/telegram-plugin/shared/bot-runtime.ts +8 -1
- package/telegram-plugin/silence-poke.ts +14 -0
- package/telegram-plugin/stream-controller.ts +156 -38
- package/telegram-plugin/tests/activity-card-store.test.ts +47 -2
- package/telegram-plugin/tests/approval-card-restart-outcome.test.ts +218 -0
- package/telegram-plugin/tests/approval-card-stores.test.ts +124 -0
- package/telegram-plugin/tests/boot-card-flood-suppress.test.ts +111 -0
- package/telegram-plugin/tests/busy-ack-wiring.test.ts +118 -0
- package/telegram-plugin/tests/busy-ack.test.ts +121 -0
- package/telegram-plugin/tests/callback-query-handlers.test.ts +701 -0
- package/telegram-plugin/tests/emission-determinism-wiring.test.ts +11 -4
- package/telegram-plugin/tests/fixtures/cutover-killswitch-probe.ts +75 -0
- package/telegram-plugin/tests/flood-circuit-breaker.test.ts +74 -0
- package/telegram-plugin/tests/gateway-outbound-redact.test.ts +5 -1
- package/telegram-plugin/tests/gateway-session-model-relaunch.test.ts +177 -25
- package/telegram-plugin/tests/inbound-delivery-cutover-flip.test.ts +418 -0
- package/telegram-plugin/tests/inbound-delivery-dispatch-equivalence.test.ts +348 -0
- package/telegram-plugin/tests/inbound-delivery-machine-dispatch.test.ts +141 -52
- package/telegram-plugin/tests/mental-model-name-entity-corruption.test.ts +119 -0
- package/telegram-plugin/tests/mental-model-propose-callback-gate.test.ts +8 -1
- package/telegram-plugin/tests/model-command.test.ts +2 -2
- package/telegram-plugin/tests/model-unavailable.test.ts +41 -0
- package/telegram-plugin/tests/outbound-send-chunks.test.ts +304 -0
- package/telegram-plugin/tests/outbound-send-path.test.ts +222 -0
- package/telegram-plugin/tests/pending-card-durability-wiring.test.ts +34 -15
- package/telegram-plugin/tests/pending-state-stores.test.ts +235 -0
- package/telegram-plugin/tests/pty-partial-handler.test.ts +56 -0
- package/telegram-plugin/tests/render/render-outbound-chunks.test.ts +98 -0
- package/telegram-plugin/tests/retry-api-call.test.ts +59 -0
- package/telegram-plugin/tests/run-hook-wrapper.test.ts +132 -0
- package/telegram-plugin/tests/session-model-file.test.ts +132 -0
- package/telegram-plugin/tests/slot-banner-boot-recovery.test.ts +3 -3
- package/telegram-plugin/tests/status-pin-boot-recovery.test.ts +3 -3
- package/telegram-plugin/tests/status-pin-store.test.ts +62 -6
- package/telegram-plugin/tests/stream-controller-chunk-cap.test.ts +122 -0
- package/telegram-plugin/tests/turn-flush-safety.test.ts +18 -4
- package/telegram-plugin/tests/vault-approval-posture.test.ts +15 -7
- package/telegram-plugin/tests/vault-grant-auto-resume.test.ts +8 -4
- package/telegram-plugin/tests/vault-grant-union.test.ts +8 -4
- package/telegram-plugin/tests/vault-grant-wizard.test.ts +8 -1
- package/telegram-plugin/tests/vault-grants-revoke.test.ts +8 -1
- package/telegram-plugin/tests/vault-key-regex-allows-slash.test.ts +8 -4
- package/telegram-plugin/tests/vault-request-access-tool.test.ts +8 -4
- package/telegram-plugin/tests/vault-request-access-unlock-resume.test.ts +8 -4
- package/telegram-plugin/tests/voice-send.test.ts +308 -0
- package/telegram-plugin/tests/worker-pin-reaper.test.ts +132 -0
- package/telegram-plugin/uat/scenarios/jtbd-deliberate-restart-resumes-dm.test.ts +118 -0
- package/telegram-plugin/uat/scenarios/jtbd-midflight-busy-ack-dm.test.ts +201 -0
- package/telegram-plugin/uat/scenarios/jtbd-worker-pin-lifecycle-dm.test.ts +208 -0
- package/telegram-plugin/uat/scenarios/vault-card-survives-gateway-restart-dm.test.ts +140 -0
- package/telegram-plugin/uat/scenarios/vault-deny-resumes-turn-dm.test.ts +84 -0
- package/telegram-plugin/uat/scenarios/vault-timeout-wakes-agent-dm.test.ts +91 -0
- package/telegram-plugin/voice-ondemand.ts +25 -1
- package/telegram-plugin/voice-send.ts +154 -0
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
import { describe, it, expect } from "vitest";
|
|
2
|
+
import {
|
|
3
|
+
decideWorkerPinReaps,
|
|
4
|
+
workerAgentIdOfPinKey,
|
|
5
|
+
WORKER_PIN_TTL_MS_DEFAULT,
|
|
6
|
+
type WorkerPinCandidate,
|
|
7
|
+
} from "../gateway/worker-pin-reaper.js";
|
|
8
|
+
|
|
9
|
+
const NOW = 1_750_000_000_000;
|
|
10
|
+
const TTL = WORKER_PIN_TTL_MS_DEFAULT;
|
|
11
|
+
|
|
12
|
+
function pin(over: Partial<WorkerPinCandidate> = {}): WorkerPinCandidate {
|
|
13
|
+
return { pinKey: "wk:agent-1", chatId: "123", pinnedAt: NOW - 60_000, ...over };
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
describe("workerAgentIdOfPinKey", () => {
|
|
17
|
+
it("extracts the agentId from a wk: key", () => {
|
|
18
|
+
expect(workerAgentIdOfPinKey("wk:abc123")).toBe("abc123");
|
|
19
|
+
});
|
|
20
|
+
it("returns null for non-worker keys (fg:/banner:/tool:) and empty ids", () => {
|
|
21
|
+
expect(workerAgentIdOfPinKey("fg:123:5")).toBeNull();
|
|
22
|
+
expect(workerAgentIdOfPinKey("banner:owner")).toBeNull();
|
|
23
|
+
expect(workerAgentIdOfPinKey("tool:123:9")).toBeNull();
|
|
24
|
+
expect(workerAgentIdOfPinKey("wk:")).toBeNull();
|
|
25
|
+
});
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
describe("decideWorkerPinReaps (#3001 mid-session wk: sweep)", () => {
|
|
29
|
+
it("reaps a pin whose worker is terminal, however young the pin is", () => {
|
|
30
|
+
const reaps = decideWorkerPinReaps({
|
|
31
|
+
pins: [pin({ pinnedAt: NOW - 1_000 })],
|
|
32
|
+
statusOf: () => 'terminal' as const,
|
|
33
|
+
ttlMs: TTL,
|
|
34
|
+
now: NOW,
|
|
35
|
+
});
|
|
36
|
+
expect(reaps).toHaveLength(1);
|
|
37
|
+
expect(reaps[0].reason).toBe("terminal");
|
|
38
|
+
expect(reaps[0].pinKey).toBe("wk:agent-1");
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
it("reaps a non-terminal pin once it exceeds the TTL", () => {
|
|
42
|
+
const reaps = decideWorkerPinReaps({
|
|
43
|
+
pins: [pin({ pinnedAt: NOW - TTL })],
|
|
44
|
+
statusOf: () => 'unknown' as const,
|
|
45
|
+
ttlMs: TTL,
|
|
46
|
+
now: NOW,
|
|
47
|
+
});
|
|
48
|
+
expect(reaps).toHaveLength(1);
|
|
49
|
+
expect(reaps[0].reason).toBe("ttl");
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
it("NEVER touches a pin younger than the TTL when the registry cannot vouch (unknown)", () => {
|
|
53
|
+
const reaps = decideWorkerPinReaps({
|
|
54
|
+
pins: [pin({ pinnedAt: NOW - TTL + 1 })],
|
|
55
|
+
statusOf: () => 'unknown' as const,
|
|
56
|
+
ttlMs: TTL,
|
|
57
|
+
now: NOW,
|
|
58
|
+
});
|
|
59
|
+
expect(reaps).toHaveLength(0);
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
it("ignores non-worker keys entirely (fg:/banner:/tool: are not its job)", () => {
|
|
63
|
+
const reaps = decideWorkerPinReaps({
|
|
64
|
+
pins: [
|
|
65
|
+
pin({ pinKey: "fg:123:5", pinnedAt: NOW - 10 * TTL }),
|
|
66
|
+
pin({ pinKey: "banner:owner", pinnedAt: NOW - 10 * TTL }),
|
|
67
|
+
pin({ pinKey: "tool:123:9", pinnedAt: NOW - 10 * TTL }),
|
|
68
|
+
],
|
|
69
|
+
statusOf: () => 'terminal' as const,
|
|
70
|
+
ttlMs: TTL,
|
|
71
|
+
now: NOW,
|
|
72
|
+
});
|
|
73
|
+
expect(reaps).toHaveLength(0);
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
it("skips a candidate with no chat id (cannot unpin without one)", () => {
|
|
77
|
+
const reaps = decideWorkerPinReaps({
|
|
78
|
+
pins: [pin({ chatId: "", pinnedAt: NOW - 10 * TTL })],
|
|
79
|
+
statusOf: () => 'terminal' as const,
|
|
80
|
+
ttlMs: TTL,
|
|
81
|
+
now: NOW,
|
|
82
|
+
});
|
|
83
|
+
expect(reaps).toHaveLength(0);
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
it("a registry-confirmed RUNNING worker keeps its pin PAST the TTL (no unpin/re-pin churn on healthy long workers)", () => {
|
|
87
|
+
const reaps = decideWorkerPinReaps({
|
|
88
|
+
pins: [pin({ pinnedAt: NOW - 10 * TTL })],
|
|
89
|
+
statusOf: () => 'running' as const,
|
|
90
|
+
ttlMs: TTL,
|
|
91
|
+
now: NOW,
|
|
92
|
+
});
|
|
93
|
+
expect(reaps).toHaveLength(0);
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
it("status is consulted per-agent: terminal reaps young, running exempts old, unknown falls to the TTL gate", () => {
|
|
97
|
+
const seen: string[] = [];
|
|
98
|
+
const reaps = decideWorkerPinReaps({
|
|
99
|
+
pins: [
|
|
100
|
+
pin({ pinKey: "wk:done", pinnedAt: NOW - 1_000 }),
|
|
101
|
+
pin({ pinKey: "wk:running-old", pinnedAt: NOW - TTL - 1 }),
|
|
102
|
+
pin({ pinKey: "wk:unknown-young", pinnedAt: NOW - 1_000 }),
|
|
103
|
+
pin({ pinKey: "wk:unknown-old", pinnedAt: NOW - TTL - 1 }),
|
|
104
|
+
],
|
|
105
|
+
statusOf: (agentId) => {
|
|
106
|
+
seen.push(agentId);
|
|
107
|
+
if (agentId === "done") return "terminal";
|
|
108
|
+
if (agentId === "running-old") return "running";
|
|
109
|
+
return "unknown";
|
|
110
|
+
},
|
|
111
|
+
ttlMs: TTL,
|
|
112
|
+
now: NOW,
|
|
113
|
+
});
|
|
114
|
+
expect(seen).toEqual(["done", "running-old", "unknown-young", "unknown-old"]);
|
|
115
|
+
expect(reaps.map((r) => [r.pinKey, r.reason])).toEqual([
|
|
116
|
+
["wk:done", "terminal"],
|
|
117
|
+
["wk:unknown-old", "ttl"],
|
|
118
|
+
]);
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
it("registry hiccup ('unknown') never unpins a young pin — the caller's contract on statusOf degrade", () => {
|
|
122
|
+
// The gateway wraps its DB lookup and degrades to 'unknown'. This pins the
|
|
123
|
+
// pure module's side of the contract: unknown + young pin = untouched.
|
|
124
|
+
const reaps = decideWorkerPinReaps({
|
|
125
|
+
pins: [pin()],
|
|
126
|
+
statusOf: () => 'unknown' as const,
|
|
127
|
+
ttlMs: TTL,
|
|
128
|
+
now: NOW,
|
|
129
|
+
});
|
|
130
|
+
expect(reaps).toHaveLength(0);
|
|
131
|
+
});
|
|
132
|
+
});
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* JTBD scenario named by `reference/jobs/survive-reboots-and-real-life.md`
|
|
3
|
+
* § Prove it — "Deliberate restart resumes, bounded (DM)".
|
|
4
|
+
*
|
|
5
|
+
* Contract under test (#2988): a DELIBERATE restart (operator
|
|
6
|
+
* `switchroom agent restart`, i.e. a clean SIGTERM shutdown with a
|
|
7
|
+
* clean-shutdown marker) that lands MID-TURN resumes the interrupted turn
|
|
8
|
+
* like a crash-class interruption. The quota-saving suppression applies only
|
|
9
|
+
* when nothing was in flight — it never swallows mid-turn work. And the
|
|
10
|
+
* resume is bounded: AT MOST ONCE per interruption (the loop-guard caps the
|
|
11
|
+
* resume chain), so we also assert no second resume-framed turn fires after
|
|
12
|
+
* the first completes.
|
|
13
|
+
*
|
|
14
|
+
* Sibling: `jtbd-interrupted-turn-resumes-dm.test.ts` (same interruption
|
|
15
|
+
* mechanics; this scenario adds the deliberate-restart class + the
|
|
16
|
+
* at-most-once bound). Decision-level twin: `decideBootResumeKind` cases in
|
|
17
|
+
* `tests/resume-inbound-builder.test.ts` (BOUNDED CHAIN) and
|
|
18
|
+
* `shouldSuppressBootResume` in `tests/gateway-clean-shutdown-marker.test.ts`.
|
|
19
|
+
*
|
|
20
|
+
* Self-skips green without NOPASSWD sudo.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
import { describe, it, expect } from "vitest";
|
|
24
|
+
import { execSync, spawn } from "node:child_process";
|
|
25
|
+
import { spinUp } from "../harness.js";
|
|
26
|
+
|
|
27
|
+
const AGENT = "test-harness";
|
|
28
|
+
const MID_TURN_MS = 10_000; // let the turn get in-flight before the bounce
|
|
29
|
+
const RESUME_BUDGET_MS = 180_000; // boot + resume + reply
|
|
30
|
+
const QUIET_WINDOW_MS = 90_000; // after the resume completes, no second resume may fire
|
|
31
|
+
// Messages landing within this window of the FIRST resume-framed reply are
|
|
32
|
+
// treated as the same resumed turn (e.g. a progress line plus the final
|
|
33
|
+
// answer of one turn) — only a resume-framed message AFTER it counts as a
|
|
34
|
+
// second resume for the at-most-once bound.
|
|
35
|
+
const SAME_TURN_GRACE_MS = 20_000;
|
|
36
|
+
|
|
37
|
+
const RESUME_FRAMING = /resum|picking .*back|interrupted|cut off|just restarted/i;
|
|
38
|
+
|
|
39
|
+
function canShellSudo(): boolean {
|
|
40
|
+
try {
|
|
41
|
+
execSync("sudo -n true", { stdio: "ignore", timeout: 2_000 });
|
|
42
|
+
return true;
|
|
43
|
+
} catch {
|
|
44
|
+
return false;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function kickRestartDetached(name: string): void {
|
|
49
|
+
// A deliberate operator restart: clean SIGTERM path, clean-shutdown marker
|
|
50
|
+
// written — the exact class #2988 makes resume in-flight work through.
|
|
51
|
+
const child = spawn(
|
|
52
|
+
"sudo",
|
|
53
|
+
["-n", "env", `PATH=${process.env.PATH}`, `HOME=${process.env.HOME}`,
|
|
54
|
+
"switchroom", "agent", "restart", name, "--force"],
|
|
55
|
+
{ detached: true, stdio: "ignore" },
|
|
56
|
+
);
|
|
57
|
+
child.unref();
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
const sudoOk = canShellSudo();
|
|
61
|
+
|
|
62
|
+
(sudoOk ? describe : describe.skip)(
|
|
63
|
+
"uat: deliberate restart mid-turn resumes exactly once (DM, #2988)",
|
|
64
|
+
() => {
|
|
65
|
+
it(
|
|
66
|
+
"an operator restart mid-turn resumes the work once, and never a second time",
|
|
67
|
+
async () => {
|
|
68
|
+
const sc = await spinUp({ agent: AGENT, settleMs: 0 });
|
|
69
|
+
try {
|
|
70
|
+
// A task long enough to still be in flight at MID_TURN_MS.
|
|
71
|
+
await sc.sendDM(
|
|
72
|
+
`Please write a thorough, detailed ~300-word explanation of how a ` +
|
|
73
|
+
`player piano's pneumatic action reads a music roll, step by step. ` +
|
|
74
|
+
`Take your time and be complete.`,
|
|
75
|
+
);
|
|
76
|
+
|
|
77
|
+
await new Promise((r) => setTimeout(r, MID_TURN_MS));
|
|
78
|
+
kickRestartDetached(AGENT);
|
|
79
|
+
|
|
80
|
+
// 1. In-flight work resumes: quota-saving suppression must NOT
|
|
81
|
+
// swallow a turn that was mid-flight at the bounce. Edits are
|
|
82
|
+
// excluded — a progress-card edit must not satisfy this; only
|
|
83
|
+
// a genuinely NEW message from the resumed turn counts.
|
|
84
|
+
const reply = await sc.expectMessage(
|
|
85
|
+
(m) => !m.edited && RESUME_FRAMING.test(m.text),
|
|
86
|
+
{ from: "bot", timeout: RESUME_BUDGET_MS },
|
|
87
|
+
);
|
|
88
|
+
expect(reply.text).toMatch(RESUME_FRAMING);
|
|
89
|
+
expect(reply.text.length).toBeGreaterThan(40);
|
|
90
|
+
|
|
91
|
+
// 2. Bounded chain: with the interruption resolved, no SECOND
|
|
92
|
+
// resume-framed turn may fire (at-most-once per interruption).
|
|
93
|
+
// Edits are excluded, and messages within SAME_TURN_GRACE_MS of
|
|
94
|
+
// the first reply are deduped as part of the SAME resumed turn
|
|
95
|
+
// (two resume-ish lines in one turn are not a second resume).
|
|
96
|
+
const firstReplyAt = reply.date.getTime();
|
|
97
|
+
let second: unknown = null;
|
|
98
|
+
try {
|
|
99
|
+
second = await sc.expectMessage(
|
|
100
|
+
(m) =>
|
|
101
|
+
!m.edited &&
|
|
102
|
+
m.messageId !== reply.messageId &&
|
|
103
|
+
m.date.getTime() > firstReplyAt + SAME_TURN_GRACE_MS &&
|
|
104
|
+
RESUME_FRAMING.test(m.text),
|
|
105
|
+
{ from: "bot", timeout: QUIET_WINDOW_MS },
|
|
106
|
+
);
|
|
107
|
+
} catch {
|
|
108
|
+
// timeout is the PASS: no repeat resume.
|
|
109
|
+
}
|
|
110
|
+
expect(second).toBeNull();
|
|
111
|
+
} finally {
|
|
112
|
+
await sc.tearDown();
|
|
113
|
+
}
|
|
114
|
+
},
|
|
115
|
+
RESUME_BUDGET_MS + QUIET_WINDOW_MS + 120_000,
|
|
116
|
+
);
|
|
117
|
+
},
|
|
118
|
+
);
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* JTBD scenario — mid-flight busy ack (#2995).
|
|
3
|
+
*
|
|
4
|
+
* Serves: `reference/jobs/steer-or-queue-mid-flight.md` — the third
|
|
5
|
+
* mid-flight class (a quick question that neither steers nor queues
|
|
6
|
+
* work of its own) and its latency promise: a mid-flight message
|
|
7
|
+
* arriving while the agent sits inside ONE long blocking tool call
|
|
8
|
+
* gets a visible, silent ack naming the blocking activity within
|
|
9
|
+
* seconds — never minutes of dead air behind a `--watch`-style call.
|
|
10
|
+
*
|
|
11
|
+
* ## What the scenario drives
|
|
12
|
+
*
|
|
13
|
+
* 1. Prompt the agent into a deliberately slow BLOCKING foreground
|
|
14
|
+
* Bash (`sleep`) — explicitly instructed NOT to background it.
|
|
15
|
+
* 2. Wait past the busy-ack step-age threshold (12s), then send a
|
|
16
|
+
* trivial mid-turn status question. It buffers (unmarked mid-turn
|
|
17
|
+
* follow-up = queue).
|
|
18
|
+
* 3. Assert a bot message matching the deterministic busy-ack shape
|
|
19
|
+
* ("⏳ Queued — currently inside `…`") lands within ACK_SLA_MS of
|
|
20
|
+
* the ping, and is SILENT (disable_notification).
|
|
21
|
+
* 4. Assert the real answer to the question arrives later (after the
|
|
22
|
+
* blocking step returns and the buffer flushes).
|
|
23
|
+
* 5. Assert the busy-ack card was cleaned up (deleted) once the
|
|
24
|
+
* answer landed — never a dangling "Queued" line.
|
|
25
|
+
*
|
|
26
|
+
* Negative case: a ping during a SHORT blocking step (well under the 12s
|
|
27
|
+
* threshold, and finished before the deferred re-check could ever see a
|
|
28
|
+
* 12s-old step) gets NO busy-ack card at any point — an ack that fires
|
|
29
|
+
* when the agent was seconds from answering is the job spec's explicit
|
|
30
|
+
* Bad bullet. (A young-step ping during a LONG step DOES get a deferred
|
|
31
|
+
* card once the step ages past the threshold — that is the re-check
|
|
32
|
+
* behaviour, covered by the positive case's latency budget.)
|
|
33
|
+
*
|
|
34
|
+
* ## Conventions
|
|
35
|
+
*
|
|
36
|
+
* - mtcute harness (`spinUp`), skip discipline like siblings: env
|
|
37
|
+
* handled by load-env; the suite hard-fails without creds by design
|
|
38
|
+
* (CI path-gates it to the uat-host runner).
|
|
39
|
+
* - `!m.edited` filters on every observation (PR #2991 lesson — the
|
|
40
|
+
* live draft/feed edits stream constantly; edits must never satisfy
|
|
41
|
+
* a fresh-message expectation).
|
|
42
|
+
*/
|
|
43
|
+
|
|
44
|
+
import { describe, expect, it } from "vitest";
|
|
45
|
+
import { spinUp } from "../harness.js";
|
|
46
|
+
import type { ObservedMessage } from "../driver.js";
|
|
47
|
+
|
|
48
|
+
const AGENT = "test-harness";
|
|
49
|
+
|
|
50
|
+
// The gateway constant is 12s (BUSY_ACK_STEP_AGE_THRESHOLD_MS). The
|
|
51
|
+
// warmup is anchored to the SEND, not to observed tool start — the agent
|
|
52
|
+
// needs a few seconds to open the turn and enter the Bash step — so it
|
|
53
|
+
// carries a large margin: even if the tool starts ~15s after the send,
|
|
54
|
+
// the step is comfortably past the threshold when the ping lands.
|
|
55
|
+
const STEP_WARMUP_MS = 35_000;
|
|
56
|
+
|
|
57
|
+
// Latency promise: user-visible ack within seconds of the ping. The card
|
|
58
|
+
// is posted synchronously by the gateway (no model). When the ping lands
|
|
59
|
+
// past the threshold the card is immediate; if the step turns out to be
|
|
60
|
+
// just under, the deferred re-check posts it within (threshold − age).
|
|
61
|
+
// 10s + jitter headroom covers both.
|
|
62
|
+
const ACK_SLA_MS = 10_000;
|
|
63
|
+
|
|
64
|
+
// The blocking sleep. Long enough that warmup + ping + ack all land while
|
|
65
|
+
// the step is still open (even with a late tool start); short enough to
|
|
66
|
+
// keep wall-clock sane.
|
|
67
|
+
const SLEEP_SECS = 120;
|
|
68
|
+
|
|
69
|
+
const BUSY_ACK_RE = /⏳ Queued — currently inside `.+`.*I'll answer when this step finishes/s;
|
|
70
|
+
|
|
71
|
+
const SLOW_PROMPT =
|
|
72
|
+
`For a latency test, run this exact command in the FOREGROUND with your ` +
|
|
73
|
+
`Bash tool (do NOT use run_in_background, do NOT split it up): ` +
|
|
74
|
+
`\`sleep ${SLEEP_SECS} && echo done\`. When it completes, reply with ` +
|
|
75
|
+
`exactly: SLEEP_DONE. Do not send any message before it completes.`;
|
|
76
|
+
|
|
77
|
+
const PING = "Quick question while you work: are you still there? One word is fine.";
|
|
78
|
+
|
|
79
|
+
describe("uat: mid-flight busy ack during a long blocking tool call (#2995)", () => {
|
|
80
|
+
it(
|
|
81
|
+
"a mid-turn ping behind a long blocking Bash gets a silent Queued ack naming the activity, then the real answer, then cleanup",
|
|
82
|
+
async () => {
|
|
83
|
+
const sc = await spinUp({ agent: AGENT });
|
|
84
|
+
try {
|
|
85
|
+
await sc.sendDM(SLOW_PROMPT);
|
|
86
|
+
|
|
87
|
+
// Let the blocking step age past the busy-ack threshold.
|
|
88
|
+
await new Promise((r) => setTimeout(r, STEP_WARMUP_MS));
|
|
89
|
+
|
|
90
|
+
const pingAt = Date.now();
|
|
91
|
+
await sc.sendDM(PING);
|
|
92
|
+
|
|
93
|
+
// The deterministic busy-ack must land within the SLA, silent,
|
|
94
|
+
// naming the blocking activity. `!m.edited` — feed/draft edits
|
|
95
|
+
// must never satisfy this.
|
|
96
|
+
const ack = await sc.expectMessage(
|
|
97
|
+
(m: ObservedMessage) => m.fromBot && !m.edited && BUSY_ACK_RE.test(m.text),
|
|
98
|
+
{ from: "bot", timeout: ACK_SLA_MS + 5_000 },
|
|
99
|
+
);
|
|
100
|
+
const ackLatency = Date.now() - pingAt;
|
|
101
|
+
expect(
|
|
102
|
+
ackLatency,
|
|
103
|
+
`busy-ack landed ${ackLatency}ms after the ping — the <10s ` +
|
|
104
|
+
`latency promise for a mid-flight quick question is broken.`,
|
|
105
|
+
).toBeLessThan(ACK_SLA_MS + 5_000);
|
|
106
|
+
expect(
|
|
107
|
+
ack.silent,
|
|
108
|
+
`busy-ack pinged the user's device (silent=false) — the card ` +
|
|
109
|
+
`must carry disable_notification:true.`,
|
|
110
|
+
).toBe(true);
|
|
111
|
+
// Names the blocking activity (the Bash sleep), not a generic line.
|
|
112
|
+
expect(ack.text).toMatch(/`.*(Bash|sleep).*`/i);
|
|
113
|
+
|
|
114
|
+
// The real answer to the ping arrives AFTER the blocking step
|
|
115
|
+
// returns and the buffer flushes. Any fresh non-card bot message
|
|
116
|
+
// that isn't the busy-ack or the SLEEP_DONE completion counts —
|
|
117
|
+
// the SLEEP_DONE exclusion is explicit (it answers the FIRST
|
|
118
|
+
// prompt, not the ping).
|
|
119
|
+
const answer = await sc.expectMessage(
|
|
120
|
+
(m: ObservedMessage) =>
|
|
121
|
+
m.fromBot &&
|
|
122
|
+
!m.edited &&
|
|
123
|
+
m.messageId !== ack.messageId &&
|
|
124
|
+
!BUSY_ACK_RE.test(m.text) &&
|
|
125
|
+
!/SLEEP_DONE/i.test(m.text) &&
|
|
126
|
+
m.text.trim().length > 0,
|
|
127
|
+
{ from: "bot", timeout: (SLEEP_SECS + 90) * 1000 },
|
|
128
|
+
);
|
|
129
|
+
expect(answer.text.trim().length).toBeGreaterThan(0);
|
|
130
|
+
|
|
131
|
+
// Cleanup: once answers land and the turns wrap, the busy-ack
|
|
132
|
+
// card is deleted (delete-on-answer lifecycle). Poll briefly —
|
|
133
|
+
// the reap is fire-and-forget after the answer/turn-end.
|
|
134
|
+
const cleanupDeadline = Date.now() + 60_000;
|
|
135
|
+
let cardGone = false;
|
|
136
|
+
while (Date.now() < cleanupDeadline) {
|
|
137
|
+
const still = await sc.driver.getMessage(sc.botUserId, ack.messageId);
|
|
138
|
+
if (still == null) {
|
|
139
|
+
cardGone = true;
|
|
140
|
+
break;
|
|
141
|
+
}
|
|
142
|
+
await new Promise((r) => setTimeout(r, 3_000));
|
|
143
|
+
}
|
|
144
|
+
expect(
|
|
145
|
+
cardGone,
|
|
146
|
+
`busy-ack card (msg ${ack.messageId}) still present 60s after ` +
|
|
147
|
+
`the answer — a stale "Queued" line is dangling in the chat.`,
|
|
148
|
+
).toBe(true);
|
|
149
|
+
} finally {
|
|
150
|
+
await sc.tearDown();
|
|
151
|
+
}
|
|
152
|
+
},
|
|
153
|
+
// warmup + sleep + flush + answer + cleanup poll + settle headroom
|
|
154
|
+
(STEP_WARMUP_MS + SLEEP_SECS * 1000 + 200_000),
|
|
155
|
+
);
|
|
156
|
+
|
|
157
|
+
it(
|
|
158
|
+
"negative: a ping during a SHORT step (finishes under the threshold) never gets a busy-ack card",
|
|
159
|
+
async () => {
|
|
160
|
+
const sc = await spinUp({ agent: AGENT });
|
|
161
|
+
try {
|
|
162
|
+
// An 8s sleep can NEVER age past the 12s threshold — neither the
|
|
163
|
+
// direct evaluation nor the deferred re-check may produce a card
|
|
164
|
+
// (the re-check re-reads live state and finds the step gone).
|
|
165
|
+
await sc.sendDM(
|
|
166
|
+
`For a latency test, run this exact command in the FOREGROUND ` +
|
|
167
|
+
`with your Bash tool (do NOT use run_in_background): ` +
|
|
168
|
+
`\`sleep 8 && echo done\`. When it completes, reply with ` +
|
|
169
|
+
`exactly: SHORT_SLEEP_DONE. Do not send any message before it completes.`,
|
|
170
|
+
);
|
|
171
|
+
|
|
172
|
+
// Give the agent a beat to enter the Bash step, then ping while
|
|
173
|
+
// the short step is (at most) a few seconds old.
|
|
174
|
+
await new Promise((r) => setTimeout(r, 4_000));
|
|
175
|
+
await sc.sendDM("Still with me?");
|
|
176
|
+
|
|
177
|
+
// Watch through the step end + the widest possible re-check window
|
|
178
|
+
// + the answer. No message matching the busy-ack shape may appear.
|
|
179
|
+
let prematureAck: ObservedMessage | null = null;
|
|
180
|
+
try {
|
|
181
|
+
prematureAck = await sc.expectMessage(
|
|
182
|
+
(m: ObservedMessage) => m.fromBot && !m.edited && BUSY_ACK_RE.test(m.text),
|
|
183
|
+
{ from: "bot", timeout: 45_000 },
|
|
184
|
+
);
|
|
185
|
+
} catch {
|
|
186
|
+
// timeout = the correct outcome (no ack, ever)
|
|
187
|
+
}
|
|
188
|
+
expect(
|
|
189
|
+
prematureAck,
|
|
190
|
+
`a busy-ack fired for a step that never reached the threshold ` +
|
|
191
|
+
`(${JSON.stringify(prematureAck?.text.slice(0, 120))}) — the ` +
|
|
192
|
+
`agent was seconds from answering; the threshold/re-check ` +
|
|
193
|
+
`gates are not holding.`,
|
|
194
|
+
).toBeNull();
|
|
195
|
+
} finally {
|
|
196
|
+
await sc.tearDown();
|
|
197
|
+
}
|
|
198
|
+
},
|
|
199
|
+
150_000,
|
|
200
|
+
);
|
|
201
|
+
});
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* JTBD — worker-pin lifecycle + the restart rule (#3001, DM).
|
|
3
|
+
*
|
|
4
|
+
* The job spec (`reference/jobs/know-what-my-agent-is-doing.md`) sanctions
|
|
5
|
+
* exactly one pin: silently pinning the already-rendered status message while
|
|
6
|
+
* its work is in flight, auto-unpinned when the work completes. #3001's
|
|
7
|
+
* operator report was the failure mode: worker (`wk:`) pins with no
|
|
8
|
+
* mid-session reaper and a forfeit-on-failure boot sweep left a long tail of
|
|
9
|
+
* stale pinned `🛠 Worker` messages. The restart rule this scenario pins:
|
|
10
|
+
* **agent restart = reset any pinned message whose work didn't finish.**
|
|
11
|
+
*
|
|
12
|
+
* Two acceptance criteria, shared setup (a long background dispatch):
|
|
13
|
+
*
|
|
14
|
+
* AC-1 — lifecycle: the `🛠 Worker` feed message is PINNED while the
|
|
15
|
+
* background worker runs, and UNPINNED after it completes (the
|
|
16
|
+
* normal completion unpin — no restart involved).
|
|
17
|
+
* AC-2 — restart rule: with a SECOND dispatch pinned mid-flight, a forced
|
|
18
|
+
* gateway restart orphans the pin; the next boot's sweep
|
|
19
|
+
* (statusPinBootCleanup, reading status-pins.json) unpins it. The
|
|
20
|
+
* restart half self-skips green without NOPASSWD sudo, mirroring
|
|
21
|
+
* jtbd-message-during-restart-dm.
|
|
22
|
+
*
|
|
23
|
+
* Observation shape: pins/unpins are SERVICE events, not message sends or
|
|
24
|
+
* edits — the worker card EDITS in place the whole time it is pinned, so a
|
|
25
|
+
* message-observer predicate would fire on edits and prove nothing. We
|
|
26
|
+
* assert via `driver.observePins` (raw `updatePinnedMessages`), keyed to the
|
|
27
|
+
* exact pinned messageId, and explicitly ignore `m.edited` traffic.
|
|
28
|
+
*
|
|
29
|
+
* Env note: requires the harness agent's default
|
|
30
|
+
* `pin_status_while_working` (ON) and the worker activity feed enabled —
|
|
31
|
+
* same env as bg-sub-agent-dispatch-dm (SETUP.md §6).
|
|
32
|
+
*/
|
|
33
|
+
|
|
34
|
+
import { describe, it, expect } from "vitest";
|
|
35
|
+
import { execSync, spawn } from "node:child_process";
|
|
36
|
+
import { spinUp } from "../harness.js";
|
|
37
|
+
import type { ObservedPin } from "../driver.js";
|
|
38
|
+
import { WORKER_FEED_RE } from "../assertions.js";
|
|
39
|
+
|
|
40
|
+
const AGENT = "test-harness";
|
|
41
|
+
|
|
42
|
+
// Same deterministic Option-1 dispatch prompt shape as
|
|
43
|
+
// bg-sub-agent-dispatch-dm: this scenario verifies the PIN infra, not the
|
|
44
|
+
// model's delegation judgment. ~60s of paced background work — long enough
|
|
45
|
+
// for the feed to paint (8s first-paint floor) and the wk: pin to land.
|
|
46
|
+
function bgDispatchPrompt(steps: number): string {
|
|
47
|
+
return (
|
|
48
|
+
`Use the Agent tool with subagent_type "general-purpose" and ` +
|
|
49
|
+
`run_in_background: true to dispatch a worker with this exact task: ` +
|
|
50
|
+
`"Do ${steps} steps, ONE AT A TIME, k = 1 through ${steps}. Before each ` +
|
|
51
|
+
`step write a brief one-sentence narration, then run \`sleep 2\` via the ` +
|
|
52
|
+
`Bash tool, then run \`echo step-k\` via the Bash tool (substitute the ` +
|
|
53
|
+
`real number for k). Run every sleep and every echo as its OWN separate ` +
|
|
54
|
+
`Bash call — never batch or chain them with && — and narrate before ` +
|
|
55
|
+
`each. Do not stop early; complete all ${steps} steps." After ` +
|
|
56
|
+
`dispatching, send a brief reply saying you've kicked off the worker.`
|
|
57
|
+
);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** Wait for the next pin/unpin event matching `pred`. observePins yields
|
|
61
|
+
* service-level pin transitions only — message EDITS never surface here,
|
|
62
|
+
* which is exactly why this scenario observes pins, not messages. */
|
|
63
|
+
async function nextPinEvent(
|
|
64
|
+
iter: AsyncIterator<ObservedPin>,
|
|
65
|
+
pred: (p: ObservedPin) => boolean,
|
|
66
|
+
timeoutMs: number,
|
|
67
|
+
what: string,
|
|
68
|
+
): Promise<ObservedPin> {
|
|
69
|
+
const deadline = Date.now() + timeoutMs;
|
|
70
|
+
while (Date.now() < deadline) {
|
|
71
|
+
const remaining = deadline - Date.now();
|
|
72
|
+
const race = await Promise.race([
|
|
73
|
+
iter.next(),
|
|
74
|
+
new Promise<"timeout">((r) => setTimeout(() => r("timeout"), remaining)),
|
|
75
|
+
]);
|
|
76
|
+
if (race === "timeout") break;
|
|
77
|
+
if (race.done === true) break;
|
|
78
|
+
if (pred(race.value)) return race.value;
|
|
79
|
+
}
|
|
80
|
+
throw new Error(`nextPinEvent: no ${what} within ${timeoutMs}ms`);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function canShellSudo(): boolean {
|
|
84
|
+
try {
|
|
85
|
+
execSync("sudo -n true", { stdio: "ignore", timeout: 2_000 });
|
|
86
|
+
return true;
|
|
87
|
+
} catch {
|
|
88
|
+
return false;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** Kick a marker-safe restart and return immediately (detached) — same shape
|
|
93
|
+
* as jtbd-message-during-restart-dm. */
|
|
94
|
+
function kickRestartDetached(name: string): void {
|
|
95
|
+
const child = spawn(
|
|
96
|
+
"sudo",
|
|
97
|
+
["-n", "env", `PATH=${process.env.PATH}`, `HOME=${process.env.HOME}`,
|
|
98
|
+
"switchroom", "agent", "restart", name, "--force"],
|
|
99
|
+
{ detached: true, stdio: "ignore" },
|
|
100
|
+
);
|
|
101
|
+
child.unref();
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
const sudoOk = canShellSudo();
|
|
105
|
+
|
|
106
|
+
describe("uat: worker-pin lifecycle + restart rule (#3001, DM)", () => {
|
|
107
|
+
it(
|
|
108
|
+
"AC-1: the worker feed message is pinned while the bg dispatch runs and unpinned after completion",
|
|
109
|
+
async () => {
|
|
110
|
+
const sc = await spinUp({ agent: AGENT });
|
|
111
|
+
const pinIter = sc.driver
|
|
112
|
+
.observePins(sc.botUserId)
|
|
113
|
+
[Symbol.asyncIterator]();
|
|
114
|
+
try {
|
|
115
|
+
await sc.sendDM(bgDispatchPrompt(10));
|
|
116
|
+
|
|
117
|
+
// Parent ack — the dispatch happened.
|
|
118
|
+
await sc.expectMessage(/.+/, { from: "bot", timeout: 45_000 });
|
|
119
|
+
|
|
120
|
+
// The wk: pin lands once the worker feed paints (~8s first-paint) —
|
|
121
|
+
// a PIN service event, not a send/edit.
|
|
122
|
+
const pinned = await nextPinEvent(
|
|
123
|
+
pinIter,
|
|
124
|
+
(p) => p.pinned,
|
|
125
|
+
90_000,
|
|
126
|
+
"worker pin",
|
|
127
|
+
);
|
|
128
|
+
expect(pinned.messageId).toBeGreaterThan(0);
|
|
129
|
+
|
|
130
|
+
// Sanity: what got pinned is the 🛠 Worker feed message (fetch by id
|
|
131
|
+
// — the card keeps EDITING in place; we never assert on edits).
|
|
132
|
+
const msg = await sc.driver.getMessage(sc.botUserId, pinned.messageId);
|
|
133
|
+
expect(msg, "pinned message vanished").not.toBeNull();
|
|
134
|
+
expect(msg!.text).toMatch(WORKER_FEED_RE);
|
|
135
|
+
|
|
136
|
+
// Completion unpins it. Budget mirrors bg-sub-agent-dispatch-dm's
|
|
137
|
+
// done-window: worker (~60s) + watcher stall detection (60s) + slack.
|
|
138
|
+
const unpinned = await nextPinEvent(
|
|
139
|
+
pinIter,
|
|
140
|
+
(p) => !p.pinned && p.messageId === pinned.messageId,
|
|
141
|
+
240_000,
|
|
142
|
+
`unpin of worker pin msg=${pinned.messageId}`,
|
|
143
|
+
);
|
|
144
|
+
expect(unpinned.pinned).toBe(false);
|
|
145
|
+
} finally {
|
|
146
|
+
await pinIter.return?.();
|
|
147
|
+
await sc.tearDown();
|
|
148
|
+
}
|
|
149
|
+
},
|
|
150
|
+
// 45s ack + 90s pin + 240s unpin + spinUp/teardown slack.
|
|
151
|
+
420_000,
|
|
152
|
+
);
|
|
153
|
+
|
|
154
|
+
(sudoOk ? it : it.skip)(
|
|
155
|
+
"AC-2 (restart rule): a worker pin orphaned by a forced gateway restart is unpinned by the next boot's sweep",
|
|
156
|
+
async () => {
|
|
157
|
+
const sc = await spinUp({ agent: AGENT });
|
|
158
|
+
const pinIter = sc.driver
|
|
159
|
+
.observePins(sc.botUserId)
|
|
160
|
+
[Symbol.asyncIterator]();
|
|
161
|
+
try {
|
|
162
|
+
// Longer dispatch (~2min of steps) so the restart reliably lands
|
|
163
|
+
// while the worker — and its pin — are still in flight.
|
|
164
|
+
await sc.sendDM(bgDispatchPrompt(30));
|
|
165
|
+
await sc.expectMessage(/.+/, { from: "bot", timeout: 45_000 });
|
|
166
|
+
|
|
167
|
+
// Latch onto the 🛠 Worker feed pin specifically — the parent turn's
|
|
168
|
+
// own fg: status pin can land first, and a bare `p.pinned` predicate
|
|
169
|
+
// would seize it and then wait on the wrong messageId's unpin. Keep
|
|
170
|
+
// consuming pin events until one resolves to a worker-feed message
|
|
171
|
+
// (AC-1's sanity fetch, applied inside the wait loop).
|
|
172
|
+
const pinDeadline = Date.now() + 90_000;
|
|
173
|
+
let pinned: ObservedPin | null = null;
|
|
174
|
+
while (pinned == null) {
|
|
175
|
+
const candidate = await nextPinEvent(
|
|
176
|
+
pinIter,
|
|
177
|
+
(p) => p.pinned,
|
|
178
|
+
Math.max(1, pinDeadline - Date.now()),
|
|
179
|
+
"worker-feed pin",
|
|
180
|
+
);
|
|
181
|
+
const msg = await sc.driver
|
|
182
|
+
.getMessage(sc.botUserId, candidate.messageId)
|
|
183
|
+
.catch(() => null);
|
|
184
|
+
if (msg != null && WORKER_FEED_RE.test(msg.text)) pinned = candidate;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
// Force the restart MID-dispatch: the wk: claim is persisted in
|
|
188
|
+
// status-pins.json; the dying session never runs its completion
|
|
189
|
+
// unpin. The next boot's statusPinBootCleanup must unpin it.
|
|
190
|
+
kickRestartDetached(AGENT);
|
|
191
|
+
|
|
192
|
+
const unpinned = await nextPinEvent(
|
|
193
|
+
pinIter,
|
|
194
|
+
(p) => !p.pinned && p.messageId === pinned.messageId,
|
|
195
|
+
// Restart (docker recreate + gateway boot + startup mutex + boot
|
|
196
|
+
// sweep) comfortably inside this window.
|
|
197
|
+
240_000,
|
|
198
|
+
`boot-sweep unpin of orphaned worker pin msg=${pinned.messageId}`,
|
|
199
|
+
);
|
|
200
|
+
expect(unpinned.pinned).toBe(false);
|
|
201
|
+
} finally {
|
|
202
|
+
await pinIter.return?.();
|
|
203
|
+
await sc.tearDown();
|
|
204
|
+
}
|
|
205
|
+
},
|
|
206
|
+
420_000,
|
|
207
|
+
);
|
|
208
|
+
});
|