switchroom 0.18.7 → 0.18.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/cli/switchroom.js +905 -758
- package/dist/host-control/main.js +1 -1
- package/package.json +1 -1
- package/profiles/_base/start.sh.hbs +111 -34
- package/skills/switchroom-runtime/SKILL.md +2 -0
- package/telegram-plugin/dist/gateway/gateway.js +46273 -44324
- package/telegram-plugin/flood-circuit-breaker.ts +123 -0
- package/telegram-plugin/gateway/activity-card-store.ts +63 -18
- package/telegram-plugin/gateway/approval-card-stores.ts +99 -0
- package/telegram-plugin/gateway/boot-card.ts +27 -0
- package/telegram-plugin/gateway/bot-commands-ops-info.ts +194 -0
- package/telegram-plugin/gateway/busy-ack.ts +106 -0
- package/telegram-plugin/gateway/callback-query-handlers.ts +2660 -0
- package/telegram-plugin/gateway/gateway.ts +1169 -3043
- package/telegram-plugin/gateway/inbound-delivery-machine-dispatch.ts +181 -23
- package/telegram-plugin/gateway/inbound-delivery-machine.ts +8 -0
- package/telegram-plugin/gateway/mental-model-propose-diff.ts +61 -5
- package/telegram-plugin/gateway/model-command.ts +23 -11
- package/telegram-plugin/gateway/outbound-send-path.ts +375 -0
- package/telegram-plugin/gateway/pending-state-stores.ts +106 -0
- package/telegram-plugin/gateway/register-bot-commands.ts +30 -0
- package/telegram-plugin/gateway/session-model-file.ts +198 -0
- package/telegram-plugin/gateway/status-pin-store.ts +82 -22
- package/telegram-plugin/gateway/worker-pin-reaper.ts +114 -0
- package/telegram-plugin/hooks/hooks.json +10 -10
- package/telegram-plugin/hooks/run-hook.sh +84 -0
- package/telegram-plugin/model-unavailable.ts +26 -0
- package/telegram-plugin/pty-partial-handler.ts +39 -0
- package/telegram-plugin/render/rich-render.ts +79 -1
- package/telegram-plugin/retry-api-call.ts +62 -0
- package/telegram-plugin/shared/bot-runtime.ts +8 -1
- package/telegram-plugin/silence-poke.ts +14 -0
- package/telegram-plugin/stream-controller.ts +156 -38
- package/telegram-plugin/tests/activity-card-store.test.ts +47 -2
- package/telegram-plugin/tests/approval-card-restart-outcome.test.ts +218 -0
- package/telegram-plugin/tests/approval-card-stores.test.ts +124 -0
- package/telegram-plugin/tests/boot-card-flood-suppress.test.ts +111 -0
- package/telegram-plugin/tests/busy-ack-wiring.test.ts +118 -0
- package/telegram-plugin/tests/busy-ack.test.ts +121 -0
- package/telegram-plugin/tests/callback-query-handlers.test.ts +701 -0
- package/telegram-plugin/tests/emission-determinism-wiring.test.ts +11 -4
- package/telegram-plugin/tests/fixtures/cutover-killswitch-probe.ts +75 -0
- package/telegram-plugin/tests/flood-circuit-breaker.test.ts +74 -0
- package/telegram-plugin/tests/gateway-outbound-redact.test.ts +5 -1
- package/telegram-plugin/tests/gateway-session-model-relaunch.test.ts +177 -25
- package/telegram-plugin/tests/inbound-delivery-cutover-flip.test.ts +418 -0
- package/telegram-plugin/tests/inbound-delivery-dispatch-equivalence.test.ts +348 -0
- package/telegram-plugin/tests/inbound-delivery-machine-dispatch.test.ts +141 -52
- package/telegram-plugin/tests/mental-model-name-entity-corruption.test.ts +119 -0
- package/telegram-plugin/tests/mental-model-propose-callback-gate.test.ts +8 -1
- package/telegram-plugin/tests/model-command.test.ts +2 -2
- package/telegram-plugin/tests/model-unavailable.test.ts +41 -0
- package/telegram-plugin/tests/outbound-send-chunks.test.ts +304 -0
- package/telegram-plugin/tests/outbound-send-path.test.ts +222 -0
- package/telegram-plugin/tests/pending-card-durability-wiring.test.ts +34 -15
- package/telegram-plugin/tests/pending-state-stores.test.ts +235 -0
- package/telegram-plugin/tests/pty-partial-handler.test.ts +56 -0
- package/telegram-plugin/tests/render/render-outbound-chunks.test.ts +98 -0
- package/telegram-plugin/tests/retry-api-call.test.ts +59 -0
- package/telegram-plugin/tests/run-hook-wrapper.test.ts +132 -0
- package/telegram-plugin/tests/session-model-file.test.ts +132 -0
- package/telegram-plugin/tests/slot-banner-boot-recovery.test.ts +3 -3
- package/telegram-plugin/tests/status-pin-boot-recovery.test.ts +3 -3
- package/telegram-plugin/tests/status-pin-store.test.ts +62 -6
- package/telegram-plugin/tests/stream-controller-chunk-cap.test.ts +122 -0
- package/telegram-plugin/tests/turn-flush-safety.test.ts +18 -4
- package/telegram-plugin/tests/vault-approval-posture.test.ts +15 -7
- package/telegram-plugin/tests/vault-grant-auto-resume.test.ts +8 -4
- package/telegram-plugin/tests/vault-grant-union.test.ts +8 -4
- package/telegram-plugin/tests/vault-grant-wizard.test.ts +8 -1
- package/telegram-plugin/tests/vault-grants-revoke.test.ts +8 -1
- package/telegram-plugin/tests/vault-key-regex-allows-slash.test.ts +8 -4
- package/telegram-plugin/tests/vault-request-access-tool.test.ts +8 -4
- package/telegram-plugin/tests/vault-request-access-unlock-resume.test.ts +8 -4
- package/telegram-plugin/tests/voice-send.test.ts +308 -0
- package/telegram-plugin/tests/worker-pin-reaper.test.ts +132 -0
- package/telegram-plugin/uat/scenarios/jtbd-deliberate-restart-resumes-dm.test.ts +118 -0
- package/telegram-plugin/uat/scenarios/jtbd-midflight-busy-ack-dm.test.ts +201 -0
- package/telegram-plugin/uat/scenarios/jtbd-worker-pin-lifecycle-dm.test.ts +208 -0
- package/telegram-plugin/uat/scenarios/vault-card-survives-gateway-restart-dm.test.ts +140 -0
- package/telegram-plugin/uat/scenarios/vault-deny-resumes-turn-dm.test.ts +84 -0
- package/telegram-plugin/uat/scenarios/vault-timeout-wakes-agent-dm.test.ts +91 -0
- package/telegram-plugin/voice-ondemand.ts +25 -1
- package/telegram-plugin/voice-send.ts +154 -0
|
@@ -32,6 +32,7 @@ export type PtyPartialAction =
|
|
|
32
32
|
| 'dedup-skip' // same text as the previous partial; no-op
|
|
33
33
|
| 'update-existing' // pushed into an already-live stream
|
|
34
34
|
| 'update-new' // created a new stream and pushed into it
|
|
35
|
+
| 'error-suppressed' // raw API-error TUI line; dropped (issue #2922 Bug 3)
|
|
35
36
|
|
|
36
37
|
export interface PtyHandlerState {
|
|
37
38
|
/**
|
|
@@ -84,6 +85,32 @@ export interface PtyHandlerDeps {
|
|
|
84
85
|
writeError?: (line: string) => void
|
|
85
86
|
}
|
|
86
87
|
|
|
88
|
+
/**
|
|
89
|
+
* Detect a raw API-error line scraped from Claude Code's TUI — issue #2922
|
|
90
|
+
* Bug 3. When the model 429s / errors, the CLI renders an `API Error: … ·
|
|
91
|
+
* b'{"type":"error",…}'` line into the terminal; the PTY tail would otherwise
|
|
92
|
+
* scrape it as the assistant reply and relay the raw bytes verbatim to chat.
|
|
93
|
+
* These lines are suppressed here so the model-unavailable operator-event
|
|
94
|
+
* pipeline owns the user-facing rendering (a clean ⚠️ card), not the raw tail.
|
|
95
|
+
*
|
|
96
|
+
* Kept deliberately tight (anchored error markers, not any mention of "error")
|
|
97
|
+
* so genuine assistant text that happens to discuss errors is NOT swallowed.
|
|
98
|
+
*/
|
|
99
|
+
export function looksLikeRawApiError(text: string): boolean {
|
|
100
|
+
if (typeof text !== 'string' || text.length === 0) return false
|
|
101
|
+
const lower = text.toLowerCase()
|
|
102
|
+
return (
|
|
103
|
+
lower.includes('api error:')
|
|
104
|
+
|| lower.includes('"type":"error"')
|
|
105
|
+
|| lower.includes("'type': 'error'")
|
|
106
|
+
|| lower.includes('rate_limit_error')
|
|
107
|
+
|| lower.includes('overloaded_error')
|
|
108
|
+
|| lower.includes('"is_error":true')
|
|
109
|
+
// The CLI's Python-style raw-body render: · b'{...}'
|
|
110
|
+
|| / b'\{/.test(text)
|
|
111
|
+
)
|
|
112
|
+
}
|
|
113
|
+
|
|
87
114
|
function streamKey(chatId: string, threadId?: number): string {
|
|
88
115
|
// Canonical chat-key derivation lives in gateway/chat-key.ts — keep this
|
|
89
116
|
// expression in lockstep (treats 0/null/undefined the same). See #1564.
|
|
@@ -96,6 +123,13 @@ function streamKey(chatId: string, threadId?: number): string {
|
|
|
96
123
|
*
|
|
97
124
|
* Returns the action taken. All state mutation happens through the
|
|
98
125
|
* supplied `state` object so callers can inspect before/after.
|
|
126
|
+
*
|
|
127
|
+
* NOTE on `looksLikeRawApiError` suppression (#2922 Bug 3): this is
|
|
128
|
+
* *preview-only* scope. PTY partials are the live-streaming terminal tail,
|
|
129
|
+
* never the authoritative reply (that flows through the operator-event /
|
|
130
|
+
* reply pipelines). So the worst case of an over-eager match here is a
|
|
131
|
+
* missing streaming flicker for one snapshot — NOT a dropped user answer.
|
|
132
|
+
* That asymmetry is why the matcher can stay aggressive on raw error shapes.
|
|
99
133
|
*/
|
|
100
134
|
export function handlePtyPartialPure(
|
|
101
135
|
text: string,
|
|
@@ -132,6 +166,11 @@ export function handlePtyPartialPure(
|
|
|
132
166
|
|
|
133
167
|
if (suppressed) return 'suppressed'
|
|
134
168
|
|
|
169
|
+
// Drop raw API-error TUI lines so they never leak to chat as the reply —
|
|
170
|
+
// the model-unavailable card (operator-event pipeline) renders these
|
|
171
|
+
// instead. See issue #2922 Bug 3.
|
|
172
|
+
if (looksLikeRawApiError(text)) return 'error-suppressed'
|
|
173
|
+
|
|
135
174
|
if (state.lastPtyPreviewByChat.get(sKey) === text) return 'dedup-skip'
|
|
136
175
|
|
|
137
176
|
const isFirst = !state.lastPtyPreviewByChat.has(sKey)
|
|
@@ -25,7 +25,18 @@
|
|
|
25
25
|
|
|
26
26
|
import { parse } from "./parse.js";
|
|
27
27
|
import { renderSafe, type RenderResult } from "./render.js";
|
|
28
|
-
import { RICH_MESSAGE_MAX_CHARS } from "../format.js";
|
|
28
|
+
import { RICH_MESSAGE_MAX_CHARS, splitMarkdownChunks } from "../format.js";
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* The legacy plain-text `sendMessage` / `editMessageText` wire cap (4096
|
|
32
|
+
* UTF-16 units). It does NOT apply to the rich path (`sendRichMessage`, up
|
|
33
|
+
* to `RICH_MESSAGE_MAX_CHARS`), but it DOES apply the moment `renderSafe`
|
|
34
|
+
* degrades a body to `mode: "plain"` — the send path then routes it through
|
|
35
|
+
* the plain `sendMessage` endpoint (see stream-controller `doSend`/`doEdit`).
|
|
36
|
+
* A degraded plain body larger than this is rejected by Telegram with
|
|
37
|
+
* `message is too long`, so `renderOutboundChunks` caps plain pieces here.
|
|
38
|
+
*/
|
|
39
|
+
export const PLAIN_TEXT_MAX_CHARS = 4096;
|
|
29
40
|
|
|
30
41
|
/** Parse the `SWITCHROOM_RICH_RENDER` flag value. Default OFF; accepts the
|
|
31
42
|
* same truthy tokens as the other switchroom env flags. Pure so the default
|
|
@@ -70,3 +81,70 @@ export function maybeRenderOutbound(
|
|
|
70
81
|
if (!richRenderEnabled(env)) return { text, mode: "markdown", degradations: [] };
|
|
71
82
|
return renderOutbound(text, maxLen);
|
|
72
83
|
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Flag-gated, CAP-ENFORCING transform for the live send path.
|
|
87
|
+
*
|
|
88
|
+
* `maybeRenderOutbound` returns ONE `RenderResult` and can only ever fit a
|
|
89
|
+
* body into a single wire message. But `renderSafe`'s markdown re-escaping
|
|
90
|
+
* (GFM-special chars gain a leading `\`) GROWS a chunk: a raw body sized just
|
|
91
|
+
* under `maxLen` can escape PAST it. `renderSafe` handles that by degrading
|
|
92
|
+
* the WHOLE document to `mode: "plain"` (raw source, no rich wrapper) — it
|
|
93
|
+
* never re-splits a multi-block body. The send path then ships that plain
|
|
94
|
+
* body through the 4096-capped `sendMessage` endpoint, so a ~32k plain body
|
|
95
|
+
* is rejected by Telegram (`message is too long`) and the answer is dropped.
|
|
96
|
+
*
|
|
97
|
+
* This function closes that gap: it returns an ARRAY of pieces, EACH of which
|
|
98
|
+
* is guaranteed to fit its own wire cap —
|
|
99
|
+
* - a `markdown` piece is `<= maxLen` (`renderSafe`'s own guarantee), and
|
|
100
|
+
* - a `plain` piece is `<= PLAIN_TEXT_MAX_CHARS` (the plain endpoint's cap).
|
|
101
|
+
* Pieces are cut only at `splitMarkdownChunks`' safe boundaries, so a fenced
|
|
102
|
+
* code block or a table row is NEVER bisected. Re-splitting the RAW source and
|
|
103
|
+
* re-rendering each piece also RECOVERS rich formatting the whole-document
|
|
104
|
+
* plain degradation would have thrown away: the smaller pieces individually
|
|
105
|
+
* escape under `maxLen` and come back as `markdown`.
|
|
106
|
+
*
|
|
107
|
+
* - flag OFF (default): `[{ text, mode: "markdown", degradations: [] }]` —
|
|
108
|
+
* a single passthrough piece, identical to `maybeRenderOutbound`.
|
|
109
|
+
* - flag ON, body fits: `[renderSafe(...)]` — a single piece, identical to
|
|
110
|
+
* `maybeRenderOutbound` (byte-for-byte for the common case).
|
|
111
|
+
* - flag ON, body oversize: 2+ cap-respecting pieces in send order.
|
|
112
|
+
*/
|
|
113
|
+
export function renderOutboundChunks(
|
|
114
|
+
text: string,
|
|
115
|
+
env: NodeJS.ProcessEnv = process.env,
|
|
116
|
+
maxLen: number = RICH_MESSAGE_MAX_CHARS,
|
|
117
|
+
plainMax: number = PLAIN_TEXT_MAX_CHARS,
|
|
118
|
+
): RenderResult[] {
|
|
119
|
+
if (!richRenderEnabled(env)) {
|
|
120
|
+
return [{ text, mode: "markdown", degradations: [] }];
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
const whole = renderOutbound(text, maxLen);
|
|
124
|
+
// `markdown` mode from renderSafe is already `<= maxLen`; a `plain` result
|
|
125
|
+
// that fits the plain cap is a single deliverable piece too. Common case.
|
|
126
|
+
if (whole.mode === "markdown" && whole.text.length <= maxLen) return [whole];
|
|
127
|
+
if (whole.mode === "plain" && whole.text.length <= plainMax) return [whole];
|
|
128
|
+
|
|
129
|
+
// Oversize: re-split the RAW source at safe boundaries and re-render each
|
|
130
|
+
// piece. Each sub-piece is `<= maxLen` in raw form, so it usually escapes
|
|
131
|
+
// back under `maxLen` and renders as rich markdown; a piece whose escaped
|
|
132
|
+
// form STILL overflows (or an indivisible plain blob) is emitted as plain,
|
|
133
|
+
// further split to `plainMax` so it fits the plain endpoint.
|
|
134
|
+
const out: RenderResult[] = [];
|
|
135
|
+
for (const rawPiece of splitMarkdownChunks(text, maxLen)) {
|
|
136
|
+
const rendered = renderOutbound(rawPiece, maxLen);
|
|
137
|
+
if (rendered.mode === "markdown" && rendered.text.length <= maxLen) {
|
|
138
|
+
out.push(rendered);
|
|
139
|
+
continue;
|
|
140
|
+
}
|
|
141
|
+
// Plain piece — cap at the plain-endpoint limit. `renderSafe` returned the
|
|
142
|
+
// raw source for a plain result, so split that source (safe boundaries,
|
|
143
|
+
// never bisecting a fence/table) into `<= plainMax` slices.
|
|
144
|
+
for (const plainPiece of splitMarkdownChunks(rendered.text, plainMax)) {
|
|
145
|
+
out.push({ text: plainPiece, mode: "plain", degradations: rendered.degradations });
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
// Defensive: a degenerate input that split to nothing still yields one piece.
|
|
149
|
+
return out.length > 0 ? out : [whole];
|
|
150
|
+
}
|
|
@@ -63,8 +63,48 @@ export interface RetryApiCallConfig {
|
|
|
63
63
|
observer?: RetryObserver
|
|
64
64
|
/** Optional log sink for flood-wait / network lines. */
|
|
65
65
|
log?: (line: string) => void
|
|
66
|
+
/**
|
|
67
|
+
* Fires whenever a Telegram 429 flood-wait is observed, BEFORE the sleep.
|
|
68
|
+
* The circuit-breaker (#2923) persists the flood-wait window here so that
|
|
69
|
+
* a container restart during an active ban can suppress non-essential
|
|
70
|
+
* sends (boot cards) instead of feeding the same per-bot-token flood
|
|
71
|
+
* counter and prolonging the ban. Best-effort; a throw here is swallowed.
|
|
72
|
+
*/
|
|
73
|
+
onFloodWait?: (retryAfterSec: number) => void
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* True when the thrown error is a LOCAL resource-exhaustion failure —
|
|
78
|
+
* ENOSPC (disk/tmpfs full), EDQUOT (quota), EIO, or ENOMEM — rather than a
|
|
79
|
+
* remote Telegram API failure. Issue #2923: an agent's tmpfs filling up
|
|
80
|
+
* wedges the outbound send's local staging step; retrying that as if it
|
|
81
|
+
* were a transient REMOTE failure hammers the Bot API and trips a per-bot
|
|
82
|
+
* flood ban. A local disk error must NOT drive remote retries — surface it
|
|
83
|
+
* as a distinct, non-retryable degraded state instead.
|
|
84
|
+
*/
|
|
85
|
+
export function isLocalResourceError(err: unknown): boolean {
|
|
86
|
+
const code = (err as { code?: unknown })?.code
|
|
87
|
+
if (typeof code === 'string' && ['ENOSPC', 'EDQUOT', 'EIO', 'ENOMEM'].includes(code)) {
|
|
88
|
+
return true
|
|
89
|
+
}
|
|
90
|
+
const msg = err instanceof Error ? err.message : String(err ?? '')
|
|
91
|
+
return (
|
|
92
|
+
// Word-boundaried so a substring can't false-match; covers the same set
|
|
93
|
+
// as the errno code list above (ENOSPC/EDQUOT/EIO/ENOMEM).
|
|
94
|
+
/\b(ENOSPC|EDQUOT|EIO|ENOMEM)\b/.test(msg) ||
|
|
95
|
+
/no space left on device/i.test(msg) ||
|
|
96
|
+
/disk quota exceeded/i.test(msg)
|
|
97
|
+
)
|
|
66
98
|
}
|
|
67
99
|
|
|
100
|
+
/**
|
|
101
|
+
* Marker error thrown when `retryApiCall` refuses to retry a LOCAL
|
|
102
|
+
* resource-exhaustion failure (#2923). Callers can detect this to surface a
|
|
103
|
+
* "degraded: local disk full" state rather than treating it like a remote
|
|
104
|
+
* send failure worth retrying.
|
|
105
|
+
*/
|
|
106
|
+
export const LOCAL_RESOURCE_EXHAUSTED = 'LOCAL_RESOURCE_EXHAUSTED'
|
|
107
|
+
|
|
68
108
|
const DEFAULT_SLEEP = (ms: number) => new Promise<void>((r) => setTimeout(r, ms))
|
|
69
109
|
|
|
70
110
|
/**
|
|
@@ -83,6 +123,7 @@ export function createRetryApiCall(
|
|
|
83
123
|
const sleep = config.sleep ?? DEFAULT_SLEEP
|
|
84
124
|
const observer = config.observer
|
|
85
125
|
const log = config.log
|
|
126
|
+
const onFloodWait = config.onFloodWait
|
|
86
127
|
|
|
87
128
|
return async function retryApiCall<T>(
|
|
88
129
|
fn: () => Promise<T>,
|
|
@@ -96,12 +137,33 @@ export function createRetryApiCall(
|
|
|
96
137
|
const msg = err instanceof Error ? err.message : String(err)
|
|
97
138
|
const desc = isGrammyErr ? (err as GrammyError).description : msg
|
|
98
139
|
|
|
140
|
+
// LOCAL resource exhaustion (#2923) — ENOSPC/EDQUOT/EIO/ENOMEM. This
|
|
141
|
+
// is a LOCAL disk/memory failure, not a remote API failure: retrying
|
|
142
|
+
// it in a tight loop is exactly what tripped the per-bot flood ban.
|
|
143
|
+
// Do NOT retry. Throw a distinct, non-retryable marker so the caller
|
|
144
|
+
// surfaces a degraded "local disk full" state and backs off hard.
|
|
145
|
+
if (isLocalResourceError(err)) {
|
|
146
|
+
log?.(
|
|
147
|
+
`telegram gateway: LOCAL resource exhaustion (${(err as { code?: string }).code ?? 'disk/mem'}) — ` +
|
|
148
|
+
`not retrying the send (would feed a flood ban); surfacing degraded state\n`,
|
|
149
|
+
)
|
|
150
|
+
observer?.onGiveUp?.({ attempts: attempt + 1, error: err })
|
|
151
|
+
throw Object.assign(new Error(LOCAL_RESOURCE_EXHAUSTED), { original: err })
|
|
152
|
+
}
|
|
153
|
+
|
|
99
154
|
// Flood-wait — sleep retry_after and try again.
|
|
100
155
|
if (isGrammyErr && (err as GrammyError).error_code === 429) {
|
|
101
156
|
const retryAfter = Number(
|
|
102
157
|
(err as GrammyError).parameters?.retry_after ?? 5,
|
|
103
158
|
)
|
|
104
159
|
const delayMs = retryAfter * 1000
|
|
160
|
+
// Persist the flood window so a restart during the ban can suppress
|
|
161
|
+
// non-essential sends instead of extending it (#2923 circuit breaker).
|
|
162
|
+
try {
|
|
163
|
+
onFloodWait?.(retryAfter)
|
|
164
|
+
} catch {
|
|
165
|
+
/* best-effort — never let the breaker hook break the retry path */
|
|
166
|
+
}
|
|
105
167
|
log?.(`telegram gateway: 429 rate limited, waiting ${retryAfter}s\n`)
|
|
106
168
|
observer?.onRetry?.({ attempt, reason: 'flood_wait', delayMs })
|
|
107
169
|
await sleep(delayMs)
|
|
@@ -29,6 +29,7 @@ import { createHash } from 'crypto'
|
|
|
29
29
|
import { AsyncLocalStorage } from 'async_hooks'
|
|
30
30
|
import { clearStaleTelegramPollingState } from '../startup-reset.js'
|
|
31
31
|
import { createRetryApiCall } from '../retry-api-call.js'
|
|
32
|
+
import { makeFloodWaitRecorder } from '../flood-circuit-breaker.js'
|
|
32
33
|
import { RICH_MESSAGE_MAX_CHARS } from '../format.js'
|
|
33
34
|
|
|
34
35
|
// ─── tg-post tag plumbing ─────────────────────────────────────────────────
|
|
@@ -149,9 +150,15 @@ export function installTgPostLogger(bot: Bot): void {
|
|
|
149
150
|
* Usage:
|
|
150
151
|
* const robustApiCall = createRobustApiCall()
|
|
151
152
|
*/
|
|
152
|
-
export function createRobustApiCall() {
|
|
153
|
+
export function createRobustApiCall(opts: { floodStatePath?: string } = {}) {
|
|
153
154
|
return createRetryApiCall({
|
|
154
155
|
log: (line) => process.stderr.write(line),
|
|
156
|
+
// #2923: persist every observed 429 flood-wait window so a restart during
|
|
157
|
+
// the ban can suppress non-essential sends (boot cards) instead of feeding
|
|
158
|
+
// the per-bot flood counter and prolonging the ban.
|
|
159
|
+
...(opts.floodStatePath
|
|
160
|
+
? { onFloodWait: makeFloodWaitRecorder(opts.floodStatePath) }
|
|
161
|
+
: {}),
|
|
155
162
|
})
|
|
156
163
|
}
|
|
157
164
|
|
|
@@ -423,6 +423,20 @@ export function formatFrameworkFallbackText(
|
|
|
423
423
|
return null
|
|
424
424
|
}
|
|
425
425
|
|
|
426
|
+
/**
|
|
427
|
+
* #2995 — the LONGEST-running in-flight tool for a turn key, or null when
|
|
428
|
+
* none is tracked. Read-only accessor over `inFlightTools` for the
|
|
429
|
+
* mid-flight busy-ack: the gateway needs the blocking step's name/label
|
|
430
|
+
* and its age to decide whether (and how) to ack a buffered mid-turn
|
|
431
|
+
* inbound. Does not touch the silence clock.
|
|
432
|
+
*/
|
|
433
|
+
export function longestInFlightTool(key: string, now: number): ToolSnapshot | null {
|
|
434
|
+
const s = state.get(key)
|
|
435
|
+
if (s == null) return null
|
|
436
|
+
const snaps = snapshotInFlight(s, now)
|
|
437
|
+
return snaps.length > 0 ? snaps[0]! : null
|
|
438
|
+
}
|
|
439
|
+
|
|
426
440
|
/** Snapshot in-flight tools sorted longest-running first — for the honest
|
|
427
441
|
* floor/fallback message body. */
|
|
428
442
|
function snapshotInFlight(s: SilencePokeState, now: number): ToolSnapshot[] {
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
|
|
22
22
|
import { createDraftStream, type DraftStreamHandle } from './draft-stream.js'
|
|
23
23
|
import { richMessage, isParseEntitiesError } from './rich-send.js'
|
|
24
|
-
import {
|
|
24
|
+
import { renderOutboundChunks } from './render/rich-render.js'
|
|
25
25
|
|
|
26
26
|
/**
|
|
27
27
|
* Minimal bot.api surface the controller needs. Real callers pass grammy's
|
|
@@ -223,81 +223,199 @@ export function createStreamController(cfg: StreamControllerConfig): DraftStream
|
|
|
223
223
|
// `format:'text'` stream (plain sendMessage, no rich wrapper).
|
|
224
224
|
// sendRichMessage does NOT accept link_preview_options (rich messages
|
|
225
225
|
// control previews via entity detection), so strip it for the rich path.
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
226
|
+
// Render the outbound body into 1+ wire-cap-respecting pieces. The common
|
|
227
|
+
// case is a SINGLE piece whose output is identical to the pre-existing
|
|
228
|
+
// `maybeRenderOutbound` path; a body whose markdown-escaping pushed the
|
|
229
|
+
// rendered form past the cap splits into several pieces, each of which fits
|
|
230
|
+
// its own wire cap and never bisects a fenced block / table row (see
|
|
231
|
+
// `renderOutboundChunks`). Flag OFF (default) and a literal `format:'text'`
|
|
232
|
+
// stream both yield a single passthrough piece — no behavioural change.
|
|
233
|
+
const renderPieces = (text: string): { text: string; rich: boolean }[] => {
|
|
234
|
+
if (literalText) return [{ text, rich: false }]
|
|
235
|
+
// A `plain`-mode piece (oversized/unsafe content renderSafe declined to
|
|
236
|
+
// emit as rich) sends WITHOUT the rich wrapper.
|
|
237
|
+
return renderOutboundChunks(text).map((r) => ({ text: r.text, rich: r.mode !== 'plain' }))
|
|
238
|
+
}
|
|
239
|
+
// Send ONE rendered piece. Rich pieces go through sendRichMessage (with
|
|
240
|
+
// link_preview_options stripped — rich messages control previews via entity
|
|
241
|
+
// detection); plain pieces (and literal streams) through sendMessage.
|
|
242
|
+
const sendPiece = (piece: { text: string; rich: boolean }, opts: StreamSendOpts) => {
|
|
243
|
+
if (!piece.rich) return bot.api.sendMessage(chatId, piece.text, opts)
|
|
234
244
|
const richOpts = { ...opts }
|
|
235
245
|
delete richOpts.link_preview_options
|
|
236
|
-
return bot.api.sendRichMessage(chatId, richMessage(
|
|
246
|
+
return bot.api.sendRichMessage(chatId, richMessage(piece.text), richOpts)
|
|
237
247
|
}
|
|
238
|
-
const
|
|
239
|
-
if (
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
248
|
+
const editPiece = (id: number, piece: { text: string; rich: boolean }, opts: StreamSendOpts) => {
|
|
249
|
+
if (!piece.rich) return bot.api.editMessageText(chatId, id, piece.text, opts)
|
|
250
|
+
return bot.api.editMessageText(chatId, id, richMessage(piece.text), opts)
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
// Overflow-tail bookkeeping, shared across the send + edit closures for the
|
|
254
|
+
// whole stream lifetime. A body large enough to split into several
|
|
255
|
+
// wire-cap pieces anchors on piece[0] (edited in place by draft-stream) and
|
|
256
|
+
// parks pieces[1..n] as follow-up messages. The draft-stream edit callback
|
|
257
|
+
// fires on EVERY throttled flush as a streamed answer grows, so we MUST NOT
|
|
258
|
+
// re-send those tails each tick (that flooded the chat with duplicates — the
|
|
259
|
+
// blocker this fix closes). Instead we remember each tail's message_id the
|
|
260
|
+
// first time it is emitted and edit it in place on later flushes; a tail that
|
|
261
|
+
// did not exist on a prior flush (the piece count grew) is sent fresh once.
|
|
262
|
+
// End state after finalize: anchor + one message per tail piece, no dupes.
|
|
263
|
+
const tailIds: number[] = []
|
|
264
|
+
const tailLastText: string[] = []
|
|
265
|
+
|
|
266
|
+
// Emit or update a single tail piece (0-based index `ti` = piece index - 1).
|
|
267
|
+
// Sends a fresh message the first time; edits in place (skipping unchanged
|
|
268
|
+
// text) thereafter. A non-parse failure is logged as a partial-delivery
|
|
269
|
+
// warning and swallowed so the remaining tail pieces still get a chance to
|
|
270
|
+
// land — never a silent drop, never an abort of pieces K..N (concern C1).
|
|
271
|
+
const upsertTail = async (ti: number, piece: { text: string; rich: boolean }): Promise<void> => {
|
|
272
|
+
const existingId = tailIds[ti]
|
|
273
|
+
if (existingId != null) {
|
|
274
|
+
if (tailLastText[ti] === piece.text) return // unchanged — skip the API call
|
|
275
|
+
try {
|
|
276
|
+
await retry(() => editPiece(existingId, piece, baseOpts), { threadId, chat_id: chatId })
|
|
277
|
+
tailLastText[ti] = piece.text
|
|
278
|
+
onEdit?.(existingId, piece.text.length)
|
|
279
|
+
} catch (err) {
|
|
280
|
+
if (!literalText && piece.rich && isParseEntitiesError(err)) {
|
|
281
|
+
warn?.(
|
|
282
|
+
`stream-controller: tail-piece #${ti + 1} edit parse-entities rejected — retrying same id=${existingId} as plain text (${err instanceof Error ? err.message : String(err)})`,
|
|
283
|
+
)
|
|
284
|
+
await retry(
|
|
285
|
+
() => bot.api.editMessageText(chatId, existingId, piece.text, baseOpts),
|
|
286
|
+
{ threadId, chat_id: chatId },
|
|
287
|
+
)
|
|
288
|
+
tailLastText[ti] = piece.text
|
|
289
|
+
onEdit?.(existingId, piece.text.length)
|
|
290
|
+
} else {
|
|
291
|
+
// Best-effort continue: leave tailLastText[ti] stale so the next
|
|
292
|
+
// flush retries this piece, and surface the partial delivery loudly.
|
|
293
|
+
warn?.(
|
|
294
|
+
`stream-controller: tail-piece #${ti + 1} edit FAILED (id=${existingId}) — partial delivery, this piece may be stale (${err instanceof Error ? err.message : String(err)})`,
|
|
295
|
+
)
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
return
|
|
299
|
+
}
|
|
300
|
+
// First emission of this tail piece → a fresh follow-up message.
|
|
301
|
+
try {
|
|
302
|
+
const sent = await retry(() => sendPiece(piece, sendOpts), { threadId, chat_id: chatId })
|
|
303
|
+
tailIds[ti] = sent.message_id
|
|
304
|
+
tailLastText[ti] = piece.text
|
|
305
|
+
onSend?.(sent.message_id, piece.text.length)
|
|
306
|
+
} catch (err) {
|
|
307
|
+
if (!literalText && piece.rich && isParseEntitiesError(err)) {
|
|
308
|
+
warn?.(
|
|
309
|
+
`stream-controller: tail-piece #${ti + 1} send parse-entities rejected — sending as plain text (${err instanceof Error ? err.message : String(err)})`,
|
|
310
|
+
)
|
|
311
|
+
const sent = await retry(
|
|
312
|
+
() => bot.api.sendMessage(chatId, piece.text, sendOpts),
|
|
313
|
+
{ threadId, chat_id: chatId },
|
|
314
|
+
)
|
|
315
|
+
tailIds[ti] = sent.message_id
|
|
316
|
+
tailLastText[ti] = piece.text
|
|
317
|
+
onSend?.(sent.message_id, piece.text.length)
|
|
318
|
+
} else {
|
|
319
|
+
// Best-effort continue: no id recorded, so the next flush re-attempts
|
|
320
|
+
// this piece rather than silently dropping pieces K..N (concern C1).
|
|
321
|
+
warn?.(
|
|
322
|
+
`stream-controller: tail-piece #${ti + 1} send FAILED — partial delivery, this and later pieces may be missing this flush (${err instanceof Error ? err.message : String(err)})`,
|
|
323
|
+
)
|
|
324
|
+
}
|
|
325
|
+
}
|
|
243
326
|
}
|
|
244
327
|
|
|
245
328
|
return createDraftStream(
|
|
246
329
|
async (text) => {
|
|
330
|
+
// Render → 1+ cap-respecting pieces. The FIRST piece's message_id anchors
|
|
331
|
+
// the stream (later edits target it); any overflow pieces are parked as
|
|
332
|
+
// follow-up messages via upsertTail. For the common single-piece case
|
|
333
|
+
// this is exactly one send.
|
|
334
|
+
const pieces = renderPieces(text)
|
|
335
|
+
const head = pieces[0]
|
|
336
|
+
let anchorId: number | undefined
|
|
247
337
|
try {
|
|
248
338
|
const sent = await retry(
|
|
249
|
-
() =>
|
|
339
|
+
() => sendPiece(head, sendOpts),
|
|
250
340
|
{ threadId, chat_id: chatId },
|
|
251
341
|
)
|
|
252
|
-
|
|
253
|
-
return sent.message_id
|
|
342
|
+
anchorId = sent.message_id
|
|
254
343
|
} catch (err) {
|
|
255
|
-
if (!literalText && isParseEntitiesError(err)) {
|
|
256
|
-
//
|
|
257
|
-
//
|
|
258
|
-
//
|
|
259
|
-
// (no rich wrapper, so the parser never runs)
|
|
260
|
-
// recovery — see issue #657. The raw markdown source is itself
|
|
261
|
-
// readable, so we send it verbatim.
|
|
344
|
+
if (!literalText && head.rich && isParseEntitiesError(err)) {
|
|
345
|
+
// Piece rejected because its markdown couldn't be parsed. There is
|
|
346
|
+
// no message_id to edit (the send 400'd before any message was
|
|
347
|
+
// created), so recover with a single fresh PLAIN send of the same
|
|
348
|
+
// body (no rich wrapper, so the parser never runs) — see issue #657.
|
|
262
349
|
warn?.(
|
|
263
350
|
`stream-controller: send parse-entities rejected — retrying once as plain text (${err instanceof Error ? err.message : String(err)})`,
|
|
264
351
|
)
|
|
352
|
+
// Resend the RAW source verbatim (readable, and the exact
|
|
353
|
+
// pre-existing #657 contract) rather than the escaped/rendered
|
|
354
|
+
// body. For a single-piece stream (the common case) this is the
|
|
355
|
+
// whole message; for a rare oversize split the head falls back to
|
|
356
|
+
// its own body and each tail to its own body below.
|
|
357
|
+
const fallbackBody = pieces.length === 1 ? text : head.text
|
|
265
358
|
const sent = await retry(
|
|
266
|
-
() => bot.api.sendMessage(chatId,
|
|
359
|
+
() => bot.api.sendMessage(chatId, fallbackBody, sendOpts),
|
|
267
360
|
{ threadId, chat_id: chatId },
|
|
268
361
|
)
|
|
269
|
-
|
|
270
|
-
|
|
362
|
+
anchorId = sent.message_id
|
|
363
|
+
} else {
|
|
364
|
+
throw err
|
|
271
365
|
}
|
|
272
|
-
throw err
|
|
273
366
|
}
|
|
367
|
+
// C2: report the ACTUAL emitted anchor-piece length, not the full body
|
|
368
|
+
// length — the anchor only holds the head piece, and the tails report
|
|
369
|
+
// their own lengths via upsertTail.
|
|
370
|
+
onSend?.(anchorId as number, head.text.length)
|
|
371
|
+
// Park overflow pieces (first-time send records their ids for reuse).
|
|
372
|
+
for (let pi = 1; pi < pieces.length; pi++) {
|
|
373
|
+
await upsertTail(pi - 1, pieces[pi])
|
|
374
|
+
}
|
|
375
|
+
return anchorId as number
|
|
274
376
|
},
|
|
275
377
|
async (id, text) => {
|
|
378
|
+
const pieces = renderPieces(text)
|
|
379
|
+
const head = pieces[0]
|
|
380
|
+
// Edit the anchor message in place with the FIRST piece.
|
|
276
381
|
try {
|
|
277
382
|
await retry(
|
|
278
|
-
() =>
|
|
383
|
+
() => editPiece(id, head, baseOpts),
|
|
279
384
|
{ threadId, chat_id: chatId },
|
|
280
385
|
)
|
|
281
|
-
|
|
386
|
+
// C2: report the actual head-piece length, not the full body length.
|
|
387
|
+
onEdit?.(id, head.text.length)
|
|
282
388
|
} catch (err) {
|
|
283
|
-
if (!literalText && isParseEntitiesError(err)) {
|
|
389
|
+
if (!literalText && head.rich && isParseEntitiesError(err)) {
|
|
284
390
|
// Edit rejected because the markdown couldn't be parsed — DO NOT
|
|
285
391
|
// send a fresh message. The whole point of issue #657 is that the
|
|
286
392
|
// previous implementation sent a duplicate message every time a
|
|
287
393
|
// parse rejection fired. Retry the edit on the SAME message_id as
|
|
288
|
-
// PLAIN text (no rich wrapper, so the parser never runs).
|
|
289
|
-
// markdown source is itself readable, so we send it verbatim.
|
|
394
|
+
// PLAIN text (no rich wrapper, so the parser never runs).
|
|
290
395
|
warn?.(
|
|
291
396
|
`stream-controller: edit parse-entities rejected — retrying same id=${id} as plain text (${err instanceof Error ? err.message : String(err)})`,
|
|
292
397
|
)
|
|
398
|
+
// Re-edit the SAME id with the RAW source verbatim (the exact #657
|
|
399
|
+
// contract). For a single-piece stream (common case) this is the
|
|
400
|
+
// whole body; a rare oversize split edits the head piece's body.
|
|
401
|
+
const fallbackBody = pieces.length === 1 ? text : head.text
|
|
293
402
|
await retry(
|
|
294
|
-
() => bot.api.editMessageText(chatId, id,
|
|
403
|
+
() => bot.api.editMessageText(chatId, id, fallbackBody, baseOpts),
|
|
295
404
|
{ threadId, chat_id: chatId },
|
|
296
405
|
)
|
|
297
|
-
onEdit?.(id, text.length)
|
|
298
|
-
|
|
406
|
+
onEdit?.(id, head.text.length)
|
|
407
|
+
} else {
|
|
408
|
+
throw err
|
|
299
409
|
}
|
|
300
|
-
|
|
410
|
+
}
|
|
411
|
+
// Oversize tail: the anchor message holds only the first piece. On EVERY
|
|
412
|
+
// edit flush we UPDATE the parked tail messages in place (or send a tail
|
|
413
|
+
// that only just came into existence) — we never re-send tails already
|
|
414
|
+
// emitted on a prior flush. This is the fix for the duplicate-flood
|
|
415
|
+
// blocker: the previous code re-sent pieces[1..n] as brand-new messages
|
|
416
|
+
// on each throttled edit tick.
|
|
417
|
+
for (let pi = 1; pi < pieces.length; pi++) {
|
|
418
|
+
await upsertTail(pi - 1, pieces[pi])
|
|
301
419
|
}
|
|
302
420
|
},
|
|
303
421
|
{
|
|
@@ -30,6 +30,7 @@
|
|
|
30
30
|
*/
|
|
31
31
|
import { describe, it, expect } from 'vitest'
|
|
32
32
|
import {
|
|
33
|
+
BOOT_UNPIN_MAX_ATTEMPTS,
|
|
33
34
|
loadActivityCards,
|
|
34
35
|
persistActivityCards,
|
|
35
36
|
writeActivityCardRecord,
|
|
@@ -314,9 +315,10 @@ describe('runActivityCardBootReaper — transport-boundary outcome tests', () =>
|
|
|
314
315
|
expect(loadActivityCards(PATH, fs)).toEqual([])
|
|
315
316
|
})
|
|
316
317
|
|
|
317
|
-
it('a failing unpin
|
|
318
|
+
it('retry-safe (#3001): a failing unpin is non-fatal, does not re-run the edit, and RETAINS the record for a next-boot unpin retry', async () => {
|
|
318
319
|
const { fs } = memFs()
|
|
319
|
-
|
|
320
|
+
const rec = card()
|
|
321
|
+
writeActivityCardRecord(PATH, fs, rec)
|
|
320
322
|
let editCalls = 0
|
|
321
323
|
const res = await runActivityCardBootReaper({
|
|
322
324
|
path: PATH,
|
|
@@ -328,9 +330,52 @@ describe('runActivityCardBootReaper — transport-boundary outcome tests', () =>
|
|
|
328
330
|
unpinCard: async () => {
|
|
329
331
|
throw new Error('message to unpin not found')
|
|
330
332
|
},
|
|
333
|
+
log: () => {},
|
|
331
334
|
})
|
|
332
335
|
expect(res).toEqual({ finalized: 1, vanished: 0, unpinned: 0, total: 1 })
|
|
333
336
|
expect(editCalls).toBe(1)
|
|
337
|
+
// The record is retained (not forfeited) so the next boot retries the
|
|
338
|
+
// idempotent unpin — flagged so it can never re-run the finalize edit.
|
|
339
|
+
expect(loadActivityCards(PATH, fs)).toEqual([
|
|
340
|
+
{ ...rec, finalizeAttempted: true, unpinAttempts: 1 },
|
|
341
|
+
])
|
|
342
|
+
|
|
343
|
+
// Second boot: the retained record retries ONLY the unpin (edit stays
|
|
344
|
+
// at-most-once); on success the record is finally dropped.
|
|
345
|
+
let secondBootEdits = 0
|
|
346
|
+
const res2 = await runActivityCardBootReaper({
|
|
347
|
+
path: PATH,
|
|
348
|
+
fs,
|
|
349
|
+
finalizeCard: async () => {
|
|
350
|
+
secondBootEdits++
|
|
351
|
+
return { ok: true }
|
|
352
|
+
},
|
|
353
|
+
unpinCard: async () => ({ ok: true }),
|
|
354
|
+
log: () => {},
|
|
355
|
+
})
|
|
356
|
+
expect(secondBootEdits).toBe(0)
|
|
357
|
+
expect(res2).toEqual({ finalized: 0, vanished: 0, unpinned: 1, total: 1 })
|
|
358
|
+
expect(loadActivityCards(PATH, fs)).toEqual([])
|
|
359
|
+
})
|
|
360
|
+
|
|
361
|
+
it('retry-safe (#3001): the unpin retry is forfeited at BOOT_UNPIN_MAX_ATTEMPTS', async () => {
|
|
362
|
+
const { fs } = memFs()
|
|
363
|
+
writeActivityCardRecord(
|
|
364
|
+
PATH,
|
|
365
|
+
fs,
|
|
366
|
+
card({ finalizeAttempted: true, unpinAttempts: BOOT_UNPIN_MAX_ATTEMPTS - 1 }),
|
|
367
|
+
)
|
|
368
|
+
const res = await runActivityCardBootReaper({
|
|
369
|
+
path: PATH,
|
|
370
|
+
fs,
|
|
371
|
+
finalizeCard: async () => ({ ok: true }),
|
|
372
|
+
unpinCard: async () => {
|
|
373
|
+
throw new Error('still no rights')
|
|
374
|
+
},
|
|
375
|
+
log: () => {},
|
|
376
|
+
})
|
|
377
|
+
// Final attempt failed — record forfeited so it cannot re-fail every boot.
|
|
378
|
+
expect(res).toEqual({ finalized: 0, vanished: 0, unpinned: 0, total: 1 })
|
|
334
379
|
expect(loadActivityCards(PATH, fs)).toEqual([])
|
|
335
380
|
})
|
|
336
381
|
|