switchroom 0.16.38 → 0.16.47
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-scheduler/index.js +8 -2
- package/dist/auth-broker/index.js +7 -1
- package/dist/cli/notion-write-pretool.mjs +7 -1
- package/dist/cli/switchroom.js +1259 -375
- package/dist/cli/ui/index.html +877 -214
- package/dist/host-control/main.js +116 -84
- package/dist/vault/approvals/kernel-server.js +8 -2
- package/dist/vault/broker/server.js +8 -2
- package/examples/minimal.yaml +1 -1
- package/examples/switchroom.yaml +1 -1
- package/package.json +2 -2
- package/profiles/_shared/reply-discipline.md.hbs +9 -0
- package/skills/switchroom-status/SKILL.md +1 -1
- package/telegram-plugin/bridge/bridge.ts +2 -1
- package/telegram-plugin/card-format.ts +7 -1
- package/telegram-plugin/dist/bridge/bridge.js +20 -2
- package/telegram-plugin/dist/gateway/gateway.js +2197 -964
- package/telegram-plugin/dist/server.js +20 -2
- package/telegram-plugin/format.ts +305 -31
- package/telegram-plugin/gateway/gateway.ts +310 -70
- package/telegram-plugin/gateway/model-command.ts +173 -19
- package/telegram-plugin/hooks/tool-label-pretool.d.mts +12 -0
- package/telegram-plugin/hooks/tool-label-pretool.mjs +54 -16
- package/telegram-plugin/package.json +1 -1
- package/telegram-plugin/session-tail.ts +47 -1
- package/telegram-plugin/stream-reply-handler.ts +19 -1
- package/telegram-plugin/tests/always-allow-grant.test.ts +34 -2
- package/telegram-plugin/tests/card-format.test.ts +28 -0
- package/telegram-plugin/tests/claude-code-event-contract.test.ts +151 -0
- package/telegram-plugin/tests/format-consistency.test.ts +223 -0
- package/telegram-plugin/tests/formatting-parse-regression.test.ts +272 -0
- package/telegram-plugin/tests/formatting-torture-set.ts +218 -0
- package/telegram-plugin/tests/model-command.test.ts +213 -47
- package/telegram-plugin/tests/paragraph-normalizer.test.ts +203 -21
- package/telegram-plugin/tests/rich-markdown-oracle.ts +469 -0
- package/telegram-plugin/tests/session-tail.test.ts +91 -0
- package/telegram-plugin/tests/status-vocabulary-unification.test.ts +125 -0
- package/telegram-plugin/tests/telegram-format.test.ts +33 -8
- package/telegram-plugin/tests/text-voice-scrub.test.ts +142 -22
- package/telegram-plugin/tests/tool-activity-summary.test.ts +6 -1
- package/telegram-plugin/tests/tts-normalize.test.ts +242 -0
- package/telegram-plugin/tests/vault-request-access-tool.test.ts +24 -0
- package/telegram-plugin/tests/vault-request-access-unlock-resume.test.ts +46 -0
- package/telegram-plugin/tests/voice-ondemand.test.ts +99 -2
- package/telegram-plugin/tests/voice-presynth.test.ts +437 -0
- package/telegram-plugin/tests/worker-activity-feed.test.ts +49 -0
- package/telegram-plugin/text-voice-scrub.ts +68 -18
- package/telegram-plugin/tool-activity-summary.ts +20 -108
- package/telegram-plugin/tts-normalize.ts +377 -0
- package/telegram-plugin/uat/driver.ts +472 -22
- package/telegram-plugin/uat/scenarios/jtbd-model-litellm-sr-dm.test.ts +34 -14
- package/telegram-plugin/uat/scenarios/jtbd-multipart-render-dm.test.ts +169 -0
- package/telegram-plugin/uat/scenarios/jtbd-narration-intent-dm.test.ts +134 -0
- package/telegram-plugin/uat/scenarios/jtbd-rich-formatting-render-dm.test.ts +254 -0
- package/telegram-plugin/uat/scenarios/jtbd-status-phase-transitions-dm.test.ts +109 -0
- package/telegram-plugin/uat/uat-driver.test.ts +297 -0
- package/telegram-plugin/voice-ondemand.ts +161 -10
- package/telegram-plugin/voice-presynth.ts +242 -0
- package/telegram-plugin/worker-activity-feed.ts +9 -1
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* JTBD scenario — a reply too long for one rich message renders correctly
|
|
3
|
+
* across BOTH chunks.
|
|
4
|
+
*
|
|
5
|
+
* Serves the same trust contract as `jtbd-rich-formatting-render-dm.test.ts`
|
|
6
|
+
* (`reference/jobs/know-what-my-agent-is-doing.md`): what the agent posts must
|
|
7
|
+
* render on the user's phone. That sibling proves single-message render
|
|
8
|
+
* fidelity. THIS scenario proves the MULTI-PART path: when a reply exceeds the
|
|
9
|
+
* Bot API 10.1 rich-message wire cap (`RICH_MESSAGE_MAX_CHARS` = 32768,
|
|
10
|
+
* `telegram-plugin/format.ts`), the gateway splits it into ordered chunks
|
|
11
|
+
* (`splitMarkdownChunks`) — and each chunk must still be a self-contained,
|
|
12
|
+
* correctly-parsed rich message. A split that bisects a formatting construct
|
|
13
|
+
* (an unterminated bold, a half-open code fence) produces a chunk Telegram
|
|
14
|
+
* renders wrong; this closes that loop against Telegram's REAL renderer via
|
|
15
|
+
* the mtcute driver, the same way the single-message scenario does.
|
|
16
|
+
*
|
|
17
|
+
* NOTE: switchroom has NO streaming-edit reply path — a long reply is sent as
|
|
18
|
+
* discrete chunk messages, not progressive edits. This scenario asserts on
|
|
19
|
+
* those discrete parts. It is deliberately NOT a streaming test.
|
|
20
|
+
*
|
|
21
|
+
* ## What it asserts
|
|
22
|
+
*
|
|
23
|
+
* 1. TWO distinct bot messages arrive (the reply chunked — not one).
|
|
24
|
+
* 2. The FIRST part carries the head marker and renders a formatting entity
|
|
25
|
+
* (bold) — formatting survived in chunk 1.
|
|
26
|
+
* 3. The LAST part carries the tail marker and renders a formatting entity
|
|
27
|
+
* (bold) — formatting survived across the split into chunk 2.
|
|
28
|
+
* 4. Neither part decoded as the `\x01` unsupported-media sentinel.
|
|
29
|
+
*
|
|
30
|
+
* ## Status: ARMED, not yet wired into the uat-gate
|
|
31
|
+
*
|
|
32
|
+
* A first live attempt on the uat-host (PR #2745) timed out at the HEAD-marker
|
|
33
|
+
* poll: coaxing a model turn to emit ~40k chars of verbatim padding is not
|
|
34
|
+
* reliable — the agent summarizes/truncates or blows the turn budget, so no
|
|
35
|
+
* >32768-char reply is produced and the gateway never chunks. The chunker
|
|
36
|
+
* itself (`splitMarkdownChunks`) is unit-tested deterministically in
|
|
37
|
+
* `telegram-plugin/tests/`; what this scenario needs to prove live is the
|
|
38
|
+
* end-to-end SEND of an already-oversized reply, which a prompt can't force.
|
|
39
|
+
*
|
|
40
|
+
* So this scenario is deliberately NOT in the `uat-gate` step list — it would
|
|
41
|
+
* red the gate on the model's refusal to pad, not on a real render regression.
|
|
42
|
+
* It self-skips green in ordinary CI (no driver creds) and is kept as the
|
|
43
|
+
* armed harness for a future deterministic oversized-send hook (e.g. a driver
|
|
44
|
+
* that posts a pre-composed >32k body directly, bypassing the model turn).
|
|
45
|
+
*
|
|
46
|
+
* ## Self-skip
|
|
47
|
+
*
|
|
48
|
+
* Same gating as every scenario here: needs the driver session
|
|
49
|
+
* (`TELEGRAM_UAT_DRIVER_SESSION`, gated to the CI uat-host). Self-skips green
|
|
50
|
+
* when creds are absent.
|
|
51
|
+
*/
|
|
52
|
+
|
|
53
|
+
import { describe, it, expect } from "vitest";
|
|
54
|
+
import { spinUp } from "../harness.js";
|
|
55
|
+
import type { ObservedMessage } from "../driver.js";
|
|
56
|
+
import { RICH_MESSAGE_MAX_CHARS } from "../../format.js";
|
|
57
|
+
|
|
58
|
+
const AGENT = "test-harness";
|
|
59
|
+
|
|
60
|
+
const HAS_DRIVER_CREDS =
|
|
61
|
+
(process.env.TELEGRAM_UAT_DRIVER_SESSION ?? "").length > 0 &&
|
|
62
|
+
(process.env.TELEGRAM_API_HASH ?? "").length > 0 &&
|
|
63
|
+
Number.isFinite(Number.parseInt(process.env.TELEGRAM_API_ID ?? "", 10)) &&
|
|
64
|
+
(process.env.TELEGRAM_TEST_BOT_USERNAME ?? "").length > 0;
|
|
65
|
+
|
|
66
|
+
// Head + tail markers, uppercase+digit so a voice/dash scrub leaves them
|
|
67
|
+
// intact. HEAD must land in chunk 1, TAIL in the final chunk.
|
|
68
|
+
const HEAD = "MULTIHEAD3";
|
|
69
|
+
const TAIL = "MULTITAIL9";
|
|
70
|
+
|
|
71
|
+
// We need the composed reply to exceed the 32768-char cap so the gateway
|
|
72
|
+
// chunks it. Asking a model to emit ~40k literal chars is flaky (it'll
|
|
73
|
+
// summarize or truncate). Instead we ask it to REPEAT a fixed, cheap-to-
|
|
74
|
+
// generate padding block a deterministic number of times — a task an agent
|
|
75
|
+
// reliably obeys because it's mechanical, not generative.
|
|
76
|
+
const PAD_LINE = "The quick brown fox jumps over the lazy dog. ";
|
|
77
|
+
// Overshoot the cap by a comfortable margin so a slightly-short reply still
|
|
78
|
+
// splits: (cap / line) rounded up, times a 1.3 safety factor.
|
|
79
|
+
const REPEATS = Math.ceil((RICH_MESSAGE_MAX_CHARS / PAD_LINE.length) * 1.3);
|
|
80
|
+
|
|
81
|
+
const PROMPT = [
|
|
82
|
+
`I need a LONG reply to test message chunking. Do EXACTLY this, nothing else:`,
|
|
83
|
+
``,
|
|
84
|
+
`1. Start your reply with: ${HEAD}: **head bold marker**`,
|
|
85
|
+
`2. Then output this exact sentence ${REPEATS} times, each on its own line:`,
|
|
86
|
+
` "${PAD_LINE.trim()}"`,
|
|
87
|
+
`3. End your reply with: ${TAIL}: **tail bold marker**`,
|
|
88
|
+
``,
|
|
89
|
+
`Do not summarize or shorten. Emit all ${REPEATS} repetitions verbatim.`,
|
|
90
|
+
].join("\n");
|
|
91
|
+
|
|
92
|
+
function kinds(msg: ObservedMessage): Set<string> {
|
|
93
|
+
return new Set(msg.entities.map((e) => e.kind));
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
(HAS_DRIVER_CREDS ? describe : describe.skip)(
|
|
97
|
+
"uat: a long reply chunks over the wire cap and both parts render",
|
|
98
|
+
() => {
|
|
99
|
+
it(
|
|
100
|
+
"first + last chunk each decode to real text with formatting intact",
|
|
101
|
+
async () => {
|
|
102
|
+
const sc = await spinUp({ agent: AGENT });
|
|
103
|
+
try {
|
|
104
|
+
await sc.sendDM(PROMPT);
|
|
105
|
+
|
|
106
|
+
// Chunk 1: the part carrying the HEAD marker.
|
|
107
|
+
const first = await sc.expectMessage(
|
|
108
|
+
(m: ObservedMessage) => m.text.includes(HEAD) || m.text === "\x01",
|
|
109
|
+
{ from: "bot", timeout: 120_000 },
|
|
110
|
+
);
|
|
111
|
+
// Chunk 2 (final): the part carrying the TAIL marker. This is a
|
|
112
|
+
// DIFFERENT message than `first` — if the reply hadn't chunked,
|
|
113
|
+
// TAIL would be in the same message and this poll would time out,
|
|
114
|
+
// failing loudly (which is the correct signal: no split happened).
|
|
115
|
+
const last = await sc.expectMessage(
|
|
116
|
+
(m: ObservedMessage) =>
|
|
117
|
+
(m.text.includes(TAIL) || m.text === "\x01") &&
|
|
118
|
+
m.messageId !== first.messageId,
|
|
119
|
+
{ from: "bot", timeout: 120_000 },
|
|
120
|
+
);
|
|
121
|
+
|
|
122
|
+
// (4) Neither part is the unsupported-media sentinel.
|
|
123
|
+
expect(
|
|
124
|
+
first.text,
|
|
125
|
+
"first chunk decoded as \\x01 sentinel — rich decode failed",
|
|
126
|
+
).not.toBe("\x01");
|
|
127
|
+
expect(
|
|
128
|
+
last.text,
|
|
129
|
+
"last chunk decoded as \\x01 sentinel — rich decode failed",
|
|
130
|
+
).not.toBe("\x01");
|
|
131
|
+
|
|
132
|
+
// (1) They are genuinely two distinct messages.
|
|
133
|
+
expect(
|
|
134
|
+
last.messageId,
|
|
135
|
+
"TAIL marker landed in the SAME message as HEAD — the reply did " +
|
|
136
|
+
"not chunk (was it under the 32768-char cap?)",
|
|
137
|
+
).not.toBe(first.messageId);
|
|
138
|
+
|
|
139
|
+
// (2) Head marker + a bold entity present in chunk 1.
|
|
140
|
+
expect(first.text).toContain(HEAD);
|
|
141
|
+
expect(
|
|
142
|
+
kinds(first).has("bold"),
|
|
143
|
+
"no bold entity in the first chunk — formatting stripped on the " +
|
|
144
|
+
"wire for chunk 1",
|
|
145
|
+
).toBe(true);
|
|
146
|
+
|
|
147
|
+
// (3) Tail marker + a bold entity present in the final chunk — the
|
|
148
|
+
// load-bearing assertion: formatting survived the split boundary.
|
|
149
|
+
expect(last.text).toContain(TAIL);
|
|
150
|
+
expect(
|
|
151
|
+
kinds(last).has("bold"),
|
|
152
|
+
"no bold entity in the last chunk — a split boundary corrupted " +
|
|
153
|
+
"the formatting of chunk 2",
|
|
154
|
+
).toBe(true);
|
|
155
|
+
|
|
156
|
+
console.info(
|
|
157
|
+
`[uat] multipart render: first msg=${first.messageId} ` +
|
|
158
|
+
`(${first.text.length} chars, kinds=${[...kinds(first)].join(",")}) ` +
|
|
159
|
+
`last msg=${last.messageId} ` +
|
|
160
|
+
`(${last.text.length} chars, kinds=${[...kinds(last)].join(",")})`,
|
|
161
|
+
);
|
|
162
|
+
} finally {
|
|
163
|
+
await sc.tearDown();
|
|
164
|
+
}
|
|
165
|
+
},
|
|
166
|
+
180_000,
|
|
167
|
+
);
|
|
168
|
+
},
|
|
169
|
+
);
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* JTBD scenario — narration-quality: a plain-language intent line before
|
|
3
|
+
* a silent tool stretch.
|
|
4
|
+
*
|
|
5
|
+
* Serves "know what my agent is actually doing"
|
|
6
|
+
* (`reference/jobs/know-what-my-agent-is-doing.md`) beat 3, and the
|
|
7
|
+
* thinking-visibility work (Option 2). With extended-thinking text
|
|
8
|
+
* server-redacted for the current flagship models (they emit
|
|
9
|
+
* `thinking` → `tool_use` → `reply` with no interstitial prose, see the
|
|
10
|
+
* PIVOT in `reference/rfcs/draft-mirror-preview.md`), the model's OWN
|
|
11
|
+
* plain-language intent narration is the only compliant carrier for
|
|
12
|
+
* "what am I doing / why" during a silent tool burst. The pacing prompt
|
|
13
|
+
* (scaffold.ts `TELEGRAM_GUIDANCE` beat 2 + the per-turn `<turn-pacing>`
|
|
14
|
+
* directive) now instructs the model to drop ONE intent line as ordinary
|
|
15
|
+
* working text before a tool stretch. This scenario pins the CARRIER: a
|
|
16
|
+
* leading `text` block surfaces as working narration (not suppressed as a
|
|
17
|
+
* draft-of-reply, not sent as a chat message) BEFORE the silent tool
|
|
18
|
+
* stretch, and the Bash `description` is the visible activity label.
|
|
19
|
+
*
|
|
20
|
+
* Why a projection-kernel scenario, not a live mtcute harness: the
|
|
21
|
+
* intent line lives in the ephemeral compose-area draft, and mtcute
|
|
22
|
+
* "CANNOT observe drafts or reactions" (CLAUDE.md, UAT caveat) — a live
|
|
23
|
+
* harness structurally cannot see this surface. The same reasoning made
|
|
24
|
+
* `jtbd-reflective-status-reaction-dm` a controller-level scenario. The
|
|
25
|
+
* deterministic assertion here (over the exact JSONL the model emits) is
|
|
26
|
+
* strictly more powerful than a flaky "did the model narrate this run"
|
|
27
|
+
* live check. The prompt that MAKES the model narrate is pinned by
|
|
28
|
+
* `tests/scaffold.test.ts` ("intent-narration carrier").
|
|
29
|
+
*/
|
|
30
|
+
|
|
31
|
+
import { describe, it, expect } from "vitest";
|
|
32
|
+
import { projectTranscriptLine } from "../../session-tail.js";
|
|
33
|
+
import { isDraftOfReply } from "../../narrative-dedup.js";
|
|
34
|
+
import { toolLabel } from "../../tool-labels.js";
|
|
35
|
+
|
|
36
|
+
/** The plain-language intent the pacing prompt asks for before a burst. */
|
|
37
|
+
const INTENT =
|
|
38
|
+
"Now checking the gateway logs to find why the turn stalled.";
|
|
39
|
+
|
|
40
|
+
/** A multi-tool assistant message: intent line, then a SILENT tool burst
|
|
41
|
+
* (no reply among the tools). This is the exact shape beat 2 describes —
|
|
42
|
+
* one intent line, then heads-down work. */
|
|
43
|
+
function multiToolTurnLine(): string {
|
|
44
|
+
return JSON.stringify({
|
|
45
|
+
type: "assistant",
|
|
46
|
+
message: {
|
|
47
|
+
content: [
|
|
48
|
+
{ type: "text", text: INTENT },
|
|
49
|
+
{
|
|
50
|
+
type: "tool_use",
|
|
51
|
+
id: "toolu_01",
|
|
52
|
+
name: "Bash",
|
|
53
|
+
input: {
|
|
54
|
+
command: "grep -n stall /var/log/switchroom/gateway.log | tail",
|
|
55
|
+
description: "Search the gateway log for the stall",
|
|
56
|
+
},
|
|
57
|
+
},
|
|
58
|
+
{
|
|
59
|
+
type: "tool_use",
|
|
60
|
+
id: "toolu_02",
|
|
61
|
+
name: "Read",
|
|
62
|
+
input: { file_path: "/state/agent/home/gateway.ts" },
|
|
63
|
+
},
|
|
64
|
+
{
|
|
65
|
+
type: "tool_use",
|
|
66
|
+
id: "toolu_03",
|
|
67
|
+
name: "Grep",
|
|
68
|
+
input: { pattern: "activeTurnStartedAt", path: "src/" },
|
|
69
|
+
},
|
|
70
|
+
],
|
|
71
|
+
},
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
describe("uat-jtbd: narration intent before a silent tool stretch", () => {
|
|
76
|
+
it("a multi-tool turn surfaces at least one plain-language intent line before the burst", () => {
|
|
77
|
+
const events = projectTranscriptLine(multiToolTurnLine());
|
|
78
|
+
|
|
79
|
+
// The intent narration is projected as a `text` event.
|
|
80
|
+
const textEvents = events.filter((e) => e.kind === "text");
|
|
81
|
+
expect(textEvents.length).toBeGreaterThanOrEqual(1);
|
|
82
|
+
const intentEv = textEvents[0];
|
|
83
|
+
expect(intentEv.kind === "text" && intentEv.text).toBe(INTENT);
|
|
84
|
+
|
|
85
|
+
// It is PLAIN LANGUAGE, not a raw tool name / debug dump. (Bad-list:
|
|
86
|
+
// "calling Bash", "Read(x)", raw commands must never be the carrier.)
|
|
87
|
+
const t = (intentEv as { text: string }).text;
|
|
88
|
+
expect(t).not.toMatch(/calling \w+|Read\(|Bash\(|grep -n|\bnull\b/);
|
|
89
|
+
expect(t.length).toBeLessThan(160); // one line, phone-readable
|
|
90
|
+
|
|
91
|
+
// The intent line comes BEFORE the silent stretch: its projected
|
|
92
|
+
// position precedes every tool_use event in source order.
|
|
93
|
+
const firstToolIdx = events.findIndex((e) => e.kind === "tool_use");
|
|
94
|
+
const intentIdx = events.findIndex((e) => e.kind === "text");
|
|
95
|
+
expect(intentIdx).toBeGreaterThanOrEqual(0);
|
|
96
|
+
expect(firstToolIdx).toBeGreaterThan(intentIdx);
|
|
97
|
+
|
|
98
|
+
// And it really is a SILENT stretch — a burst of >1 tool with no reply
|
|
99
|
+
// tool interleaved. The narration is the ONLY signal the user gets.
|
|
100
|
+
const toolCount = events.filter((e) => e.kind === "tool_use").length;
|
|
101
|
+
expect(toolCount).toBeGreaterThan(1);
|
|
102
|
+
const replyToolFired = events.some(
|
|
103
|
+
(e) => e.kind === "tool_use" && /^(reply|stream_reply)$/.test(e.toolName),
|
|
104
|
+
);
|
|
105
|
+
expect(replyToolFired).toBe(false);
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
it("the intent line is working-narration (surfaced), not a draft-of-reply (suppressed)", () => {
|
|
109
|
+
// The reducer-side dedup gate SUPPRESSES a text block only when it is
|
|
110
|
+
// the model composing its answer just before `reply`. An intent line
|
|
111
|
+
// that names what it's about to DO shares no prefix with the eventual
|
|
112
|
+
// answer, so the gate SHOWS it. Pin both sides.
|
|
113
|
+
const laterAnswer =
|
|
114
|
+
"The turn stalled because the typing loop was never cleared on error.";
|
|
115
|
+
expect(isDraftOfReply(INTENT, laterAnswer)).toBe(false);
|
|
116
|
+
|
|
117
|
+
// Control: a genuine draft-then-send IS suppressed, so we know the gate
|
|
118
|
+
// is live and the assertion above isn't vacuously true.
|
|
119
|
+
const draft = laterAnswer + " (composing…)";
|
|
120
|
+
expect(isDraftOfReply(draft, laterAnswer)).toBe(true);
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
it("the Bash description — not the raw command — is the visible activity label", () => {
|
|
124
|
+
// The compose-area draft renders `input.description` for Bash (never
|
|
125
|
+
// the raw `command`). This is why the prompt tells the model to always
|
|
126
|
+
// give Bash a plain-English description: it IS the visible carrier.
|
|
127
|
+
const label = toolLabel("Bash", {
|
|
128
|
+
command: "grep -n stall /var/log/switchroom/gateway.log | tail",
|
|
129
|
+
description: "Search the gateway log for the stall",
|
|
130
|
+
});
|
|
131
|
+
expect(label).toBe("Search the gateway log for the stall");
|
|
132
|
+
expect(label).not.toContain("grep");
|
|
133
|
+
});
|
|
134
|
+
});
|
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* JTBD scenario — the real account SEES rendered rich formatting.
|
|
3
|
+
*
|
|
4
|
+
* Serves: `reference/jobs/know-what-my-agent-is-doing.md` — the trust
|
|
5
|
+
* contract that what the agent posts renders correctly on the user's
|
|
6
|
+
* phone. Phase 1 (PR #2741, `formatting-parse-regression.test.ts`) pins
|
|
7
|
+
* this at the PARSE level (our emitted markdown parses to the intended
|
|
8
|
+
* entity structure, checked by an independent oracle). This scenario is
|
|
9
|
+
* the higher-fidelity RENDER-level layer: it drives a REAL agent turn
|
|
10
|
+
* through a REAL bot into a REAL chat, then reads back — via the mtcute
|
|
11
|
+
* MTProto driver — the entity structure Telegram ACTUALLY parsed and will
|
|
12
|
+
* render. It closes the loop the unit suite can't: it proves Telegram's
|
|
13
|
+
* own renderer agrees with our oracle, not just that our oracle agrees
|
|
14
|
+
* with our formatter.
|
|
15
|
+
*
|
|
16
|
+
* ## Why this needs the mtcute upgrade (issue #2739)
|
|
17
|
+
*
|
|
18
|
+
* On the pinned mtcute (0.27.x) a Bot API 10.1 rich message
|
|
19
|
+
* (`sendMessage` with `{ markdown }`) decoded as `messageMediaUnsupported`
|
|
20
|
+
* with empty text — so the driver saw NEITHER text NOR entities and
|
|
21
|
+
* substituted a `\x01` sentinel. With mtcute >=0.30 the message decodes to
|
|
22
|
+
* real text + entities, and `toObserved()` now surfaces both
|
|
23
|
+
* `ObservedMessage.entities` and the `t.me` permalink (`.link`). This test
|
|
24
|
+
* asserts on that real, decoded structure. If the sentinel ever reappears
|
|
25
|
+
* (`text === "\x01"`, `entities === []`) the assertions fail LOUDLY rather
|
|
26
|
+
* than silently pass — it is the regression gate for the decode itself.
|
|
27
|
+
*
|
|
28
|
+
* ## What it asserts
|
|
29
|
+
*
|
|
30
|
+
* The agent is asked to reply with a small, unambiguous formatting sample
|
|
31
|
+
* (bold / italic / inline code / a fenced code block / an inline link)
|
|
32
|
+
* anchored by a stable marker token so the reply is findable in the stream.
|
|
33
|
+
* We then assert, on the observed bot reply:
|
|
34
|
+
*
|
|
35
|
+
* 1. The body decoded to real text (NOT the `\x01` unsupported-media
|
|
36
|
+
* sentinel) — the core decode regression gate.
|
|
37
|
+
* 2. Telegram parsed the expected entity KINDS (membership, not exact
|
|
38
|
+
* spans — a model reply is not byte-deterministic, so we mirror
|
|
39
|
+
* Phase 1's "these entities are PRESENT" philosophy rather than an
|
|
40
|
+
* exact-echo assertion).
|
|
41
|
+
* 3. The link entity carries the right destination URL.
|
|
42
|
+
* 4. A `t.me/c/<chat>/<msgid>` (or `t.me/<user>`) permalink is captured
|
|
43
|
+
* and logged for human eyeball reference (best-effort — private DMs
|
|
44
|
+
* don't support message links, so this is logged, not asserted).
|
|
45
|
+
*
|
|
46
|
+
* ## Self-skip
|
|
47
|
+
*
|
|
48
|
+
* Like every scenario in this dir, it needs the driver session
|
|
49
|
+
* (`TELEGRAM_UAT_DRIVER_SESSION`, vault key `telegram-uat-driver-session`,
|
|
50
|
+
* gated to the CI uat-host runner / test-harness agent). It self-skips
|
|
51
|
+
* green when those creds are absent so a fork PR / un-wired host never
|
|
52
|
+
* reds — the `uat/**` tree is excluded from gating CI anyway; the
|
|
53
|
+
* `uat-gate` sentinel owns the real run.
|
|
54
|
+
*/
|
|
55
|
+
|
|
56
|
+
import { describe, it, expect } from "vitest";
|
|
57
|
+
import { spinUp } from "../harness.js";
|
|
58
|
+
import type { ObservedMessage } from "../driver.js";
|
|
59
|
+
|
|
60
|
+
const AGENT = "test-harness";
|
|
61
|
+
|
|
62
|
+
// The driver session is the load-bearing credential. Absent it, spinUp()
|
|
63
|
+
// throws in resolveConfig — so guard at describe level and self-skip green.
|
|
64
|
+
const HAS_DRIVER_CREDS =
|
|
65
|
+
(process.env.TELEGRAM_UAT_DRIVER_SESSION ?? "").length > 0 &&
|
|
66
|
+
(process.env.TELEGRAM_API_HASH ?? "").length > 0 &&
|
|
67
|
+
Number.isFinite(Number.parseInt(process.env.TELEGRAM_API_ID ?? "", 10)) &&
|
|
68
|
+
(process.env.TELEGRAM_TEST_BOT_USERNAME ?? "").length > 0;
|
|
69
|
+
|
|
70
|
+
// A stable marker so the reply is unambiguously findable in the observed
|
|
71
|
+
// stream (the agent is told to include it verbatim). Uppercase + digit so
|
|
72
|
+
// it survives any voice/dash scrub untouched.
|
|
73
|
+
const MARKER = "RICHFMT7";
|
|
74
|
+
|
|
75
|
+
// The formatting sample the agent is asked to reproduce. Kept small and
|
|
76
|
+
// unambiguous: one entity of each kind we care about, plus a link with a
|
|
77
|
+
// known destination.
|
|
78
|
+
const LINK_URL = "https://github.com/switchroom/switchroom";
|
|
79
|
+
const SAMPLE_LINES = [
|
|
80
|
+
`Reply with EXACTLY this, including the marker ${MARKER} verbatim, and NOTHING else:`,
|
|
81
|
+
"",
|
|
82
|
+
`${MARKER}: here is **a bold phrase**, some *italic words*, an inline \`code_token\`, ` +
|
|
83
|
+
`some ~~struck text~~, a ||hidden spoiler||, and a link to [the repo](${LINK_URL}).`,
|
|
84
|
+
"",
|
|
85
|
+
"## A heading line",
|
|
86
|
+
"",
|
|
87
|
+
"> a blockquoted line",
|
|
88
|
+
"",
|
|
89
|
+
"- first bullet",
|
|
90
|
+
"- second bullet",
|
|
91
|
+
"",
|
|
92
|
+
"1. ordered one",
|
|
93
|
+
"2. ordered two",
|
|
94
|
+
"",
|
|
95
|
+
"Then, on new lines, a fenced code block:",
|
|
96
|
+
"```bash",
|
|
97
|
+
'echo "hello from the torture set"',
|
|
98
|
+
"```",
|
|
99
|
+
].join("\n");
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* mtcute entity kinds we HARD-ASSERT Telegram parsed from the sample. Phase 2
|
|
103
|
+
* extends the original bold/italic/code/pre/text_link set with strikethrough
|
|
104
|
+
* and blockquote — both confirmed to survive the live send→IV→decode round
|
|
105
|
+
* trip on the uat-host (PR #2745 first live run: present set was
|
|
106
|
+
* `bold, italic, code, strikethrough, text_link, blockquote, pre`).
|
|
107
|
+
*
|
|
108
|
+
* `spoiler` is deliberately NOT hard-asserted: on the live wire the `||…||`
|
|
109
|
+
* span did NOT surface as a `spoiler`/`textMarked` entity (the model either
|
|
110
|
+
* dropped the syntax or Telegram's chat-message GFM parser doesn't map it the
|
|
111
|
+
* way the IV table-of-contents `textMarked` node does). Rather than red the
|
|
112
|
+
* whole render gate on a construct the round trip doesn't reliably produce, we
|
|
113
|
+
* observe spoiler softly (logged below). The decoder's `textMarked → spoiler`
|
|
114
|
+
* mapping is still pinned deterministically by the hosted unit suite.
|
|
115
|
+
*
|
|
116
|
+
* Lists and dividers carry no first-class Bot API entity (they render as
|
|
117
|
+
* bulleted/numbered/rule TEXT), so they are asserted on `reply.text` below.
|
|
118
|
+
*/
|
|
119
|
+
const EXPECTED_KINDS = [
|
|
120
|
+
"bold",
|
|
121
|
+
"italic",
|
|
122
|
+
"code",
|
|
123
|
+
"pre",
|
|
124
|
+
"text_link",
|
|
125
|
+
"strikethrough",
|
|
126
|
+
"blockquote",
|
|
127
|
+
] as const;
|
|
128
|
+
|
|
129
|
+
/** Soft-observed only (see note above) — logged, never fails the gate. */
|
|
130
|
+
const SOFT_KINDS = ["spoiler"] as const;
|
|
131
|
+
|
|
132
|
+
function kindsPresent(msg: ObservedMessage): Set<string> {
|
|
133
|
+
return new Set(msg.entities.map((e) => e.kind));
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
(HAS_DRIVER_CREDS ? describe : describe.skip)(
|
|
137
|
+
"uat: real account sees rendered rich formatting (entities + permalink)",
|
|
138
|
+
() => {
|
|
139
|
+
it(
|
|
140
|
+
"bot reply decodes to real text + expected entity kinds; permalink captured",
|
|
141
|
+
async () => {
|
|
142
|
+
const sc = await spinUp({ agent: AGENT });
|
|
143
|
+
try {
|
|
144
|
+
await sc.sendDM(SAMPLE_LINES);
|
|
145
|
+
|
|
146
|
+
// Find the bot reply carrying our marker. The reply may be
|
|
147
|
+
// streamed/split; we want the chunk that actually holds the
|
|
148
|
+
// formatting sample (the one with the marker token).
|
|
149
|
+
const reply = await sc.expectMessage(
|
|
150
|
+
(m: ObservedMessage) =>
|
|
151
|
+
m.text.includes(MARKER) || m.text === "\x01",
|
|
152
|
+
{ from: "bot", timeout: 90_000 },
|
|
153
|
+
);
|
|
154
|
+
|
|
155
|
+
// (1) Decode regression gate: the body must be REAL text, not the
|
|
156
|
+
// unsupported-media sentinel. If this fails, mtcute is not
|
|
157
|
+
// decoding the rich message — the whole point of the upgrade.
|
|
158
|
+
expect(
|
|
159
|
+
reply.text,
|
|
160
|
+
"bot reply decoded as the \\x01 unsupported-media sentinel — " +
|
|
161
|
+
"mtcute is NOT decoding the Bot API rich message (issue #2739 regression)",
|
|
162
|
+
).not.toBe("\x01");
|
|
163
|
+
expect(reply.text).toContain(MARKER);
|
|
164
|
+
|
|
165
|
+
// (2) Telegram must have parsed at least SOME entities — a reply
|
|
166
|
+
// that carries the marker but zero entities means formatting was
|
|
167
|
+
// stripped on the wire (or decode silently dropped entities).
|
|
168
|
+
const present = kindsPresent(reply);
|
|
169
|
+
expect(
|
|
170
|
+
reply.entities.length,
|
|
171
|
+
`observed reply had zero entities (kinds=${[...present].join(",")}) — ` +
|
|
172
|
+
"formatting did not survive the render round-trip",
|
|
173
|
+
).toBeGreaterThan(0);
|
|
174
|
+
|
|
175
|
+
// (3) Entity-kind membership. We assert PRESENCE of each expected
|
|
176
|
+
// kind, not exact spans (model replies aren't byte-deterministic,
|
|
177
|
+
// and streaming may split the sample across chunks — this mirrors
|
|
178
|
+
// Phase 1's membership philosophy). A missing kind is a real
|
|
179
|
+
// render-fidelity regression for that construct.
|
|
180
|
+
const missing = EXPECTED_KINDS.filter((k) => !present.has(k));
|
|
181
|
+
expect(
|
|
182
|
+
missing,
|
|
183
|
+
`expected entity kinds missing from the rendered reply: ${missing.join(", ")} ` +
|
|
184
|
+
`(present: ${[...present].join(", ")})`,
|
|
185
|
+
).toEqual([]);
|
|
186
|
+
|
|
187
|
+
// Soft-observed constructs (e.g. spoiler) — logged, never fail the
|
|
188
|
+
// gate. See EXPECTED_KINDS note for why spoiler is soft.
|
|
189
|
+
for (const k of SOFT_KINDS) {
|
|
190
|
+
console.info(
|
|
191
|
+
`[uat] soft-observed construct '${k}': ` +
|
|
192
|
+
(present.has(k) ? "PRESENT on the wire" : "absent (expected — not gated)"),
|
|
193
|
+
);
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
// (4) The link entity must carry the right destination.
|
|
197
|
+
const link = reply.entities.find((e) => e.kind === "text_link");
|
|
198
|
+
expect(link, "no text_link entity found in the reply").toBeDefined();
|
|
199
|
+
expect(link?.url).toBe(LINK_URL);
|
|
200
|
+
|
|
201
|
+
// (5) Lists and dividers have NO first-class Bot API entity — the
|
|
202
|
+
// decoder renders them into TEXT (bulleted "• ", numbered "N. ").
|
|
203
|
+
// Assert the decoded body carries a bullet and an ordinal marker so
|
|
204
|
+
// a regression that drops list structure (back to bare inline text)
|
|
205
|
+
// reds here. Model replies aren't byte-deterministic, so we check
|
|
206
|
+
// for the STRUCTURE markers, not exact list wording.
|
|
207
|
+
expect(
|
|
208
|
+
reply.text,
|
|
209
|
+
"decoded reply carried no bullet marker — pageBlockList decode " +
|
|
210
|
+
"regressed to flat text",
|
|
211
|
+
).toContain("• ");
|
|
212
|
+
expect(
|
|
213
|
+
reply.text,
|
|
214
|
+
"decoded reply carried no ordinal marker — pageBlockOrderedList " +
|
|
215
|
+
"decode regressed to flat text",
|
|
216
|
+
).toMatch(/\b1\.\s/);
|
|
217
|
+
|
|
218
|
+
// Permalink capture — best-effort human-eyeball reference. Private
|
|
219
|
+
// DMs don't support message links (mtcute's .link getter throws
|
|
220
|
+
// there; toObserved swallows it), so we LOG rather than assert.
|
|
221
|
+
if (reply.link) {
|
|
222
|
+
console.info(
|
|
223
|
+
`[uat] rich-formatting reply permalink: ${reply.link} ` +
|
|
224
|
+
`(chat=${reply.chatId} msg=${reply.messageId})`,
|
|
225
|
+
);
|
|
226
|
+
} else {
|
|
227
|
+
console.info(
|
|
228
|
+
`[uat] rich-formatting reply had no permalink ` +
|
|
229
|
+
`(expected for a private DM) — chat=${reply.chatId} msg=${reply.messageId}`,
|
|
230
|
+
);
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
// Forensic dump of the exact entity structure Telegram returned —
|
|
234
|
+
// this is the render-fidelity evidence the parse-level suite can't
|
|
235
|
+
// produce (it's what Telegram PARSED, not what we EMITTED).
|
|
236
|
+
console.info(
|
|
237
|
+
"[uat] rendered entity structure: " +
|
|
238
|
+
JSON.stringify(
|
|
239
|
+
reply.entities.map((e) => ({
|
|
240
|
+
kind: e.kind,
|
|
241
|
+
text: e.text,
|
|
242
|
+
...(e.url ? { url: e.url } : {}),
|
|
243
|
+
...(e.language ? { language: e.language } : {}),
|
|
244
|
+
})),
|
|
245
|
+
),
|
|
246
|
+
);
|
|
247
|
+
} finally {
|
|
248
|
+
await sc.tearDown();
|
|
249
|
+
}
|
|
250
|
+
},
|
|
251
|
+
120_000,
|
|
252
|
+
);
|
|
253
|
+
},
|
|
254
|
+
);
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* JTBD scenario — the status reaction moves through DISTINCT phases across
|
|
3
|
+
* a turn: acknowledged → thinking → working/editing → done.
|
|
4
|
+
*
|
|
5
|
+
* Serves "know what my agent is actually doing"
|
|
6
|
+
* (`reference/jobs/know-what-my-agent-is-doing.md`): the "Good looks like"
|
|
7
|
+
* line — "that signal distinguishes phases at a glance: acknowledged,
|
|
8
|
+
* thinking or working, actively editing. It stays present from receipt to
|
|
9
|
+
* turn end and never disappears mid-turn." The reflective controller test
|
|
10
|
+
* (`jtbd-reflective-status-reaction-dm`) pins the #1713 non-event/
|
|
11
|
+
* bidirectional rules; THIS scenario pins the complementary contract the
|
|
12
|
+
* job cares about most: over a realistic full turn the phases actually
|
|
13
|
+
* FIRE, in order, and are visibly DIFFERENT emoji families (not one
|
|
14
|
+
* undifferentiated "working" blob — an explicit bad-list item).
|
|
15
|
+
*
|
|
16
|
+
* Simulated (not live mtcute): real Bot-API reactions expose only the
|
|
17
|
+
* CURRENT emoji, never the transition trail, so the controller-level
|
|
18
|
+
* assertion over the whole sequence is strictly more powerful — the same
|
|
19
|
+
* reasoning documented in `jtbd-reflective-status-reaction-dm`.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
|
|
23
|
+
import { StatusReactionController } from "../../status-reactions.js";
|
|
24
|
+
|
|
25
|
+
async function flush(): Promise<void> {
|
|
26
|
+
for (let i = 0; i < 8; i++) await Promise.resolve();
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
// Phase families the JOB cares about (job spec "Good looks like"). Emoji
|
|
30
|
+
// membership is pinned by status-reactions.ts; we assert the family a
|
|
31
|
+
// glance would read, so a palette retune that preserves the phase meaning
|
|
32
|
+
// doesn't false-fail this behavioural scenario.
|
|
33
|
+
const ACK = new Set(["👀", "🤔", "🤓"]); // received / acknowledged
|
|
34
|
+
const THINKING = new Set(["🤔", "🤓"]); // generating
|
|
35
|
+
const WORKING = new Set(["✍", "⚡", "👌", "👨💻", "🗜"]); // tools / editing / coding
|
|
36
|
+
const DONE = new Set(["👍", "💯", "🎉"]); // turn over
|
|
37
|
+
|
|
38
|
+
describe("uat-jtbd: status reaction phase transitions across a turn", () => {
|
|
39
|
+
beforeEach(() => vi.useFakeTimers());
|
|
40
|
+
afterEach(() => vi.useRealTimers());
|
|
41
|
+
|
|
42
|
+
it("acknowledged → thinking → working/editing → done all fire, in order, as distinct phases", async () => {
|
|
43
|
+
const calls: string[] = [];
|
|
44
|
+
const ctrl = new StatusReactionController(async (e: string) => {
|
|
45
|
+
calls.push(e);
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
// A realistic turn: inbound arrives, model thinks, then reads/edits
|
|
49
|
+
// (a silent tool stretch), then the turn ends with an answer.
|
|
50
|
+
ctrl.setQueued(); // acknowledged
|
|
51
|
+
await flush();
|
|
52
|
+
|
|
53
|
+
ctrl.setThinking(); // thinking
|
|
54
|
+
vi.advanceTimersByTime(3500);
|
|
55
|
+
await flush();
|
|
56
|
+
|
|
57
|
+
ctrl.setTool("Read"); // working — coding family
|
|
58
|
+
vi.advanceTimersByTime(3500);
|
|
59
|
+
await flush();
|
|
60
|
+
|
|
61
|
+
ctrl.setTool("Edit"); // still working — actively editing
|
|
62
|
+
vi.advanceTimersByTime(3500);
|
|
63
|
+
await flush();
|
|
64
|
+
|
|
65
|
+
ctrl.finalize("done"); // turn_end — done
|
|
66
|
+
await flush();
|
|
67
|
+
|
|
68
|
+
// Each phase fired at least once.
|
|
69
|
+
expect(calls.some((e) => ACK.has(e))).toBe(true);
|
|
70
|
+
expect(calls.some((e) => THINKING.has(e))).toBe(true);
|
|
71
|
+
expect(calls.some((e) => WORKING.has(e))).toBe(true);
|
|
72
|
+
expect(calls.some((e) => DONE.has(e))).toBe(true);
|
|
73
|
+
|
|
74
|
+
// They fired IN ORDER: first ack, a working beat before the terminal,
|
|
75
|
+
// and done strictly last.
|
|
76
|
+
const firstWorkingIdx = calls.findIndex((e) => WORKING.has(e));
|
|
77
|
+
const doneIdx = calls.findIndex((e) => DONE.has(e));
|
|
78
|
+
expect(firstWorkingIdx).toBeGreaterThan(0); // ack precedes working
|
|
79
|
+
expect(doneIdx).toBe(calls.length - 1); // done is terminal + last
|
|
80
|
+
expect(firstWorkingIdx).toBeLessThan(doneIdx); // work before done
|
|
81
|
+
|
|
82
|
+
// The phases are VISIBLY DIFFERENT — thinking and working are not the
|
|
83
|
+
// same emoji (the "one undifferentiated working signal" bad-list item).
|
|
84
|
+
const thinkingEmoji = calls.find((e) => THINKING.has(e))!;
|
|
85
|
+
const workingEmoji = calls.find((e) => WORKING.has(e))!;
|
|
86
|
+
expect(thinkingEmoji).not.toBe(workingEmoji);
|
|
87
|
+
|
|
88
|
+
// Liveness never disappears mid-turn: 👍 appears exactly once, only at
|
|
89
|
+
// the end — the signal is present from receipt to terminal state.
|
|
90
|
+
expect(calls.filter((e) => DONE.has(e)).length).toBe(1);
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
it("a trivial turn still acknowledges before it finishes (never a silent gap)", async () => {
|
|
94
|
+
// "A trivial ask just gets the answer" — but even then the inbound is
|
|
95
|
+
// acknowledged (👀) before the turn finalizes; the ack is never skipped.
|
|
96
|
+
const calls: string[] = [];
|
|
97
|
+
const ctrl = new StatusReactionController(async (e: string) => {
|
|
98
|
+
calls.push(e);
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
ctrl.setQueued();
|
|
102
|
+
await flush();
|
|
103
|
+
ctrl.finalize("done");
|
|
104
|
+
await flush();
|
|
105
|
+
|
|
106
|
+
expect(ACK.has(calls[0])).toBe(true);
|
|
107
|
+
expect(DONE.has(calls[calls.length - 1])).toBe(true);
|
|
108
|
+
});
|
|
109
|
+
});
|