switchroom 0.18.7 → 0.18.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/switchroom.js +905 -758
- package/dist/host-control/main.js +1 -1
- package/package.json +1 -1
- package/profiles/_base/start.sh.hbs +111 -34
- package/skills/switchroom-runtime/SKILL.md +2 -0
- package/telegram-plugin/dist/gateway/gateway.js +1403 -657
- package/telegram-plugin/flood-circuit-breaker.ts +123 -0
- package/telegram-plugin/gateway/activity-card-store.ts +63 -18
- package/telegram-plugin/gateway/boot-card.ts +27 -0
- package/telegram-plugin/gateway/busy-ack.ts +106 -0
- package/telegram-plugin/gateway/gateway.ts +564 -85
- package/telegram-plugin/gateway/mental-model-propose-diff.ts +61 -5
- package/telegram-plugin/gateway/model-command.ts +23 -11
- package/telegram-plugin/gateway/session-model-file.ts +198 -0
- package/telegram-plugin/gateway/status-pin-store.ts +82 -22
- package/telegram-plugin/gateway/worker-pin-reaper.ts +114 -0
- package/telegram-plugin/hooks/hooks.json +10 -10
- package/telegram-plugin/hooks/run-hook.sh +84 -0
- package/telegram-plugin/model-unavailable.ts +26 -0
- package/telegram-plugin/pty-partial-handler.ts +39 -0
- package/telegram-plugin/render/rich-render.ts +79 -1
- package/telegram-plugin/retry-api-call.ts +62 -0
- package/telegram-plugin/shared/bot-runtime.ts +8 -1
- package/telegram-plugin/silence-poke.ts +14 -0
- package/telegram-plugin/stream-controller.ts +156 -38
- package/telegram-plugin/tests/activity-card-store.test.ts +47 -2
- package/telegram-plugin/tests/approval-card-restart-outcome.test.ts +218 -0
- package/telegram-plugin/tests/boot-card-flood-suppress.test.ts +111 -0
- package/telegram-plugin/tests/busy-ack-wiring.test.ts +118 -0
- package/telegram-plugin/tests/busy-ack.test.ts +121 -0
- package/telegram-plugin/tests/flood-circuit-breaker.test.ts +74 -0
- package/telegram-plugin/tests/gateway-session-model-relaunch.test.ts +177 -25
- package/telegram-plugin/tests/mental-model-name-entity-corruption.test.ts +119 -0
- package/telegram-plugin/tests/model-command.test.ts +2 -2
- package/telegram-plugin/tests/model-unavailable.test.ts +41 -0
- package/telegram-plugin/tests/pty-partial-handler.test.ts +56 -0
- package/telegram-plugin/tests/render/render-outbound-chunks.test.ts +98 -0
- package/telegram-plugin/tests/retry-api-call.test.ts +59 -0
- package/telegram-plugin/tests/run-hook-wrapper.test.ts +132 -0
- package/telegram-plugin/tests/session-model-file.test.ts +132 -0
- package/telegram-plugin/tests/slot-banner-boot-recovery.test.ts +3 -3
- package/telegram-plugin/tests/status-pin-boot-recovery.test.ts +3 -3
- package/telegram-plugin/tests/status-pin-store.test.ts +62 -6
- package/telegram-plugin/tests/stream-controller-chunk-cap.test.ts +122 -0
- package/telegram-plugin/tests/voice-send.test.ts +308 -0
- package/telegram-plugin/tests/worker-pin-reaper.test.ts +132 -0
- package/telegram-plugin/uat/scenarios/jtbd-deliberate-restart-resumes-dm.test.ts +118 -0
- package/telegram-plugin/uat/scenarios/jtbd-midflight-busy-ack-dm.test.ts +201 -0
- package/telegram-plugin/uat/scenarios/jtbd-worker-pin-lifecycle-dm.test.ts +208 -0
- package/telegram-plugin/uat/scenarios/vault-card-survives-gateway-restart-dm.test.ts +140 -0
- package/telegram-plugin/uat/scenarios/vault-deny-resumes-turn-dm.test.ts +84 -0
- package/telegram-plugin/uat/scenarios/vault-timeout-wakes-agent-dm.test.ts +91 -0
- package/telegram-plugin/voice-ondemand.ts +25 -1
- package/telegram-plugin/voice-send.ts +154 -0
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
"hooks": [
|
|
6
6
|
{
|
|
7
7
|
"type": "command",
|
|
8
|
-
"command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/secret-guard-pretool.mjs\"",
|
|
8
|
+
"command": "sh \"${CLAUDE_PLUGIN_ROOT}/hooks/run-hook.sh\" node \"${CLAUDE_PLUGIN_ROOT}/hooks/secret-guard-pretool.mjs\"",
|
|
9
9
|
"timeout": 10
|
|
10
10
|
}
|
|
11
11
|
]
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
"hooks": [
|
|
15
15
|
{
|
|
16
16
|
"type": "command",
|
|
17
|
-
"command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/sentinel-reply-guard-pretool.mjs\"",
|
|
17
|
+
"command": "sh \"${CLAUDE_PLUGIN_ROOT}/hooks/run-hook.sh\" node \"${CLAUDE_PLUGIN_ROOT}/hooks/sentinel-reply-guard-pretool.mjs\"",
|
|
18
18
|
"timeout": 5
|
|
19
19
|
}
|
|
20
20
|
]
|
|
@@ -24,7 +24,7 @@
|
|
|
24
24
|
"hooks": [
|
|
25
25
|
{
|
|
26
26
|
"type": "command",
|
|
27
|
-
"command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/subagent-tracker-pretool.mjs\"",
|
|
27
|
+
"command": "sh \"${CLAUDE_PLUGIN_ROOT}/hooks/run-hook.sh\" node \"${CLAUDE_PLUGIN_ROOT}/hooks/subagent-tracker-pretool.mjs\"",
|
|
28
28
|
"timeout": 10
|
|
29
29
|
}
|
|
30
30
|
]
|
|
@@ -33,7 +33,7 @@
|
|
|
33
33
|
"hooks": [
|
|
34
34
|
{
|
|
35
35
|
"type": "command",
|
|
36
|
-
"command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/tool-label-pretool.mjs\"",
|
|
36
|
+
"command": "sh \"${CLAUDE_PLUGIN_ROOT}/hooks/run-hook.sh\" node \"${CLAUDE_PLUGIN_ROOT}/hooks/tool-label-pretool.mjs\"",
|
|
37
37
|
"timeout": 5
|
|
38
38
|
}
|
|
39
39
|
]
|
|
@@ -43,7 +43,7 @@
|
|
|
43
43
|
"hooks": [
|
|
44
44
|
{
|
|
45
45
|
"type": "command",
|
|
46
|
-
"command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/repo-context-pretool.mjs\"",
|
|
46
|
+
"command": "sh \"${CLAUDE_PLUGIN_ROOT}/hooks/run-hook.sh\" node \"${CLAUDE_PLUGIN_ROOT}/hooks/repo-context-pretool.mjs\"",
|
|
47
47
|
"timeout": 5
|
|
48
48
|
}
|
|
49
49
|
]
|
|
@@ -55,7 +55,7 @@
|
|
|
55
55
|
"hooks": [
|
|
56
56
|
{
|
|
57
57
|
"type": "command",
|
|
58
|
-
"command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/subagent-tracker-posttool.mjs\"",
|
|
58
|
+
"command": "sh \"${CLAUDE_PLUGIN_ROOT}/hooks/run-hook.sh\" node \"${CLAUDE_PLUGIN_ROOT}/hooks/subagent-tracker-posttool.mjs\"",
|
|
59
59
|
"timeout": 10
|
|
60
60
|
}
|
|
61
61
|
]
|
|
@@ -65,7 +65,7 @@
|
|
|
65
65
|
"hooks": [
|
|
66
66
|
{
|
|
67
67
|
"type": "command",
|
|
68
|
-
"command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/sandbox-hint-posttool.mjs\"",
|
|
68
|
+
"command": "sh \"${CLAUDE_PLUGIN_ROOT}/hooks/run-hook.sh\" node \"${CLAUDE_PLUGIN_ROOT}/hooks/sandbox-hint-posttool.mjs\"",
|
|
69
69
|
"timeout": 3
|
|
70
70
|
}
|
|
71
71
|
]
|
|
@@ -76,7 +76,7 @@
|
|
|
76
76
|
"hooks": [
|
|
77
77
|
{
|
|
78
78
|
"type": "command",
|
|
79
|
-
"command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/secret-scrub-stop.mjs\"",
|
|
79
|
+
"command": "sh \"${CLAUDE_PLUGIN_ROOT}/hooks/run-hook.sh\" node \"${CLAUDE_PLUGIN_ROOT}/hooks/secret-scrub-stop.mjs\"",
|
|
80
80
|
"timeout": 15,
|
|
81
81
|
"async": true
|
|
82
82
|
}
|
|
@@ -86,7 +86,7 @@
|
|
|
86
86
|
"hooks": [
|
|
87
87
|
{
|
|
88
88
|
"type": "command",
|
|
89
|
-
"command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/silent-end-interrupt-stop.mjs\"",
|
|
89
|
+
"command": "sh \"${CLAUDE_PLUGIN_ROOT}/hooks/run-hook.sh\" node \"${CLAUDE_PLUGIN_ROOT}/hooks/silent-end-interrupt-stop.mjs\"",
|
|
90
90
|
"timeout": 5
|
|
91
91
|
}
|
|
92
92
|
]
|
|
@@ -95,7 +95,7 @@
|
|
|
95
95
|
"hooks": [
|
|
96
96
|
{
|
|
97
97
|
"type": "command",
|
|
98
|
-
"command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/tool-label-stop.mjs\"",
|
|
98
|
+
"command": "sh \"${CLAUDE_PLUGIN_ROOT}/hooks/run-hook.sh\" node \"${CLAUDE_PLUGIN_ROOT}/hooks/tool-label-stop.mjs\"",
|
|
99
99
|
"timeout": 5,
|
|
100
100
|
"async": true
|
|
101
101
|
}
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
#!/bin/sh
|
|
2
|
+
# run-hook.sh — resilient launcher for Claude Code Node hooks (issue #2555).
|
|
3
|
+
#
|
|
4
|
+
# Under cgroup memory-ceiling pressure (the cgroup pinned at memory.max with
|
|
5
|
+
# reclaim lagging, page cache ~= cap), a freshly-spawned Node process can abort
|
|
6
|
+
# at STARTUP with exit 134 (SIGABRT) inside the libuv threadpool constructor —
|
|
7
|
+
# `Assertion failed: (0) == (uv_thread_create(...))` — BEFORE any hook code
|
|
8
|
+
# runs. It is a transient allocation failure, not a real hook error, but it
|
|
9
|
+
# surfaces a 🔴 issues card and skips the hook's work on that one tool call.
|
|
10
|
+
#
|
|
11
|
+
# This wrapper makes the invocation tolerant:
|
|
12
|
+
# 1. Shrink the libuv threadpool to 1 so Node needs the fewest possible
|
|
13
|
+
# thread-stack mmaps at startup (minimises the failure window).
|
|
14
|
+
# 2. Capture the hook payload from stdin ONCE and replay it on each attempt.
|
|
15
|
+
# The abort can land AFTER Node started draining the pipe, so a naive
|
|
16
|
+
# retry would feed the second attempt EMPTY stdin — a secret scanner would
|
|
17
|
+
# then scan nothing. Replaying the captured payload keeps the retry
|
|
18
|
+
# faithful.
|
|
19
|
+
# 3. Retry ONCE on a 134 abort after a brief backoff.
|
|
20
|
+
# 4. If it STILL aborts:
|
|
21
|
+
# - for SECURITY-CRITICAL hooks (secret-guard / secret-scrub) FAIL
|
|
22
|
+
# CLOSED — propagate 134 so the runtime cards it. 134 is a generic
|
|
23
|
+
# SIGABRT (assertion / OOM / any abort), not memory-pressure-specific,
|
|
24
|
+
# so silently returning 0 for a genuinely broken scanner would be a
|
|
25
|
+
# silent security BYPASS. Those hooks must never fail open.
|
|
26
|
+
# - for all other hooks, SKIP CLEANLY (exit 0) with a single stderr
|
|
27
|
+
# warn. A skipped label/context hook on one call is the documented,
|
|
28
|
+
# accepted degradation; a crash-card storm under memory pressure is
|
|
29
|
+
# not.
|
|
30
|
+
#
|
|
31
|
+
# Any non-134 exit status is passed through unchanged — real hook decisions
|
|
32
|
+
# (block/allow/non-zero) are never masked. `sleep 0.15` uses a fractional
|
|
33
|
+
# second (supported by GNU/BusyBox sleep, both present in the agent image).
|
|
34
|
+
#
|
|
35
|
+
# Usage (from hooks.json):
|
|
36
|
+
# sh "${CLAUDE_PLUGIN_ROOT}/hooks/run-hook.sh" node "${CLAUDE_PLUGIN_ROOT}/hooks/<name>.mjs"
|
|
37
|
+
|
|
38
|
+
export UV_THREADPOOL_SIZE="${UV_THREADPOOL_SIZE:-1}"
|
|
39
|
+
|
|
40
|
+
# Identify the hook script (the .mjs argument) to decide fail-open vs closed.
|
|
41
|
+
hook_path=""
|
|
42
|
+
for a in "$@"; do
|
|
43
|
+
case "$a" in
|
|
44
|
+
*.mjs) hook_path="$a" ;;
|
|
45
|
+
esac
|
|
46
|
+
done
|
|
47
|
+
hook_name=$(basename "$hook_path" 2>/dev/null)
|
|
48
|
+
|
|
49
|
+
# Security-critical hooks MUST fail closed on a persistent abort.
|
|
50
|
+
fail_closed=0
|
|
51
|
+
case "$hook_name" in
|
|
52
|
+
secret-guard-pretool.mjs | secret-scrub-stop.mjs) fail_closed=1 ;;
|
|
53
|
+
esac
|
|
54
|
+
|
|
55
|
+
# Capture stdin once so both attempts see the SAME payload. Claude Code feeds
|
|
56
|
+
# the hook a single JSON object on stdin and closes it, so `cat` returns at
|
|
57
|
+
# EOF. (Command substitution strips trailing newlines, which JSON parsing does
|
|
58
|
+
# not care about.)
|
|
59
|
+
payload=$(cat)
|
|
60
|
+
|
|
61
|
+
run_hook() {
|
|
62
|
+
printf '%s' "$payload" | "$@"
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
run_hook "$@"
|
|
66
|
+
status=$?
|
|
67
|
+
if [ "$status" -ne 134 ]; then
|
|
68
|
+
exit "$status"
|
|
69
|
+
fi
|
|
70
|
+
|
|
71
|
+
# Transient thread-create abort — brief backoff, then retry once with the
|
|
72
|
+
# faithfully-replayed payload.
|
|
73
|
+
sleep 0.15
|
|
74
|
+
run_hook "$@"
|
|
75
|
+
status=$?
|
|
76
|
+
if [ "$status" -eq 134 ]; then
|
|
77
|
+
if [ "$fail_closed" -eq 1 ]; then
|
|
78
|
+
echo "run-hook: security hook '$hook_name' aborted twice with exit 134 — FAILING CLOSED (not skipping) (#2555)" >&2
|
|
79
|
+
exit 134
|
|
80
|
+
fi
|
|
81
|
+
echo "run-hook: '$hook_name' aborted twice with exit 134 (uv_thread_create under memory pressure) — skipping hook cleanly (#2555)" >&2
|
|
82
|
+
exit 0
|
|
83
|
+
fi
|
|
84
|
+
exit "$status"
|
|
@@ -69,6 +69,32 @@ export function detectModelUnavailable(
|
|
|
69
69
|
const sample = stderr.length > 16_384 ? stderr.slice(0, 16_384) : stderr
|
|
70
70
|
const lower = sample.toLowerCase()
|
|
71
71
|
|
|
72
|
+
// ── 0. Transient / server-side 429 (NOT account quota) — issue #2922 ────
|
|
73
|
+
// Anthropic emits a `rate_limit_error` whose message explicitly negates the
|
|
74
|
+
// account-quota reading: "Server is temporarily limiting requests (not your
|
|
75
|
+
// usage limit)". A negation-blind substring match on "usage limit" (step 1
|
|
76
|
+
// below) would misclassify this as `quota_exhausted`, firing a phantom fleet
|
|
77
|
+
// failover that self-cancels and leaves the turn dead. These are upstream
|
|
78
|
+
// throttles Claude Code retries internally with backoff — classify them as
|
|
79
|
+
// `overload` (the calm rate-limit path) BEFORE the quota substrings run, so
|
|
80
|
+
// the negation is honoured and no failover is announced.
|
|
81
|
+
const transientUpstreamSignals = [
|
|
82
|
+
'not your usage limit',
|
|
83
|
+
'not your account',
|
|
84
|
+
"not your account's",
|
|
85
|
+
'temporarily limiting requests',
|
|
86
|
+
'temporarily rate',
|
|
87
|
+
'server is temporarily',
|
|
88
|
+
'would exceed your account’s rate limit',
|
|
89
|
+
"would exceed your account's rate limit",
|
|
90
|
+
]
|
|
91
|
+
if (transientUpstreamSignals.some(s => lower.includes(s))) {
|
|
92
|
+
const resetAt = parseResetTime(sample)
|
|
93
|
+
return resetAt !== undefined
|
|
94
|
+
? { kind: 'overload', resetAt, raw: stderr }
|
|
95
|
+
: { kind: 'overload', raw: stderr }
|
|
96
|
+
}
|
|
97
|
+
|
|
72
98
|
// ── 1. Quota / billing exhaustion ──────────────────────────────────────
|
|
73
99
|
const quotaSignals = [
|
|
74
100
|
'out of extra usage',
|
|
@@ -32,6 +32,7 @@ export type PtyPartialAction =
|
|
|
32
32
|
| 'dedup-skip' // same text as the previous partial; no-op
|
|
33
33
|
| 'update-existing' // pushed into an already-live stream
|
|
34
34
|
| 'update-new' // created a new stream and pushed into it
|
|
35
|
+
| 'error-suppressed' // raw API-error TUI line; dropped (issue #2922 Bug 3)
|
|
35
36
|
|
|
36
37
|
export interface PtyHandlerState {
|
|
37
38
|
/**
|
|
@@ -84,6 +85,32 @@ export interface PtyHandlerDeps {
|
|
|
84
85
|
writeError?: (line: string) => void
|
|
85
86
|
}
|
|
86
87
|
|
|
88
|
+
/**
|
|
89
|
+
* Detect a raw API-error line scraped from Claude Code's TUI — issue #2922
|
|
90
|
+
* Bug 3. When the model 429s / errors, the CLI renders an `API Error: … ·
|
|
91
|
+
* b'{"type":"error",…}'` line into the terminal; the PTY tail would otherwise
|
|
92
|
+
* scrape it as the assistant reply and relay the raw bytes verbatim to chat.
|
|
93
|
+
* These lines are suppressed here so the model-unavailable operator-event
|
|
94
|
+
* pipeline owns the user-facing rendering (a clean ⚠️ card), not the raw tail.
|
|
95
|
+
*
|
|
96
|
+
* Kept deliberately tight (anchored error markers, not any mention of "error")
|
|
97
|
+
* so genuine assistant text that happens to discuss errors is NOT swallowed.
|
|
98
|
+
*/
|
|
99
|
+
export function looksLikeRawApiError(text: string): boolean {
|
|
100
|
+
if (typeof text !== 'string' || text.length === 0) return false
|
|
101
|
+
const lower = text.toLowerCase()
|
|
102
|
+
return (
|
|
103
|
+
lower.includes('api error:')
|
|
104
|
+
|| lower.includes('"type":"error"')
|
|
105
|
+
|| lower.includes("'type': 'error'")
|
|
106
|
+
|| lower.includes('rate_limit_error')
|
|
107
|
+
|| lower.includes('overloaded_error')
|
|
108
|
+
|| lower.includes('"is_error":true')
|
|
109
|
+
// The CLI's Python-style raw-body render: · b'{...}'
|
|
110
|
+
|| / b'\{/.test(text)
|
|
111
|
+
)
|
|
112
|
+
}
|
|
113
|
+
|
|
87
114
|
function streamKey(chatId: string, threadId?: number): string {
|
|
88
115
|
// Canonical chat-key derivation lives in gateway/chat-key.ts — keep this
|
|
89
116
|
// expression in lockstep (treats 0/null/undefined the same). See #1564.
|
|
@@ -96,6 +123,13 @@ function streamKey(chatId: string, threadId?: number): string {
|
|
|
96
123
|
*
|
|
97
124
|
* Returns the action taken. All state mutation happens through the
|
|
98
125
|
* supplied `state` object so callers can inspect before/after.
|
|
126
|
+
*
|
|
127
|
+
* NOTE on `looksLikeRawApiError` suppression (#2922 Bug 3): this is
|
|
128
|
+
* *preview-only* scope. PTY partials are the live-streaming terminal tail,
|
|
129
|
+
* never the authoritative reply (that flows through the operator-event /
|
|
130
|
+
* reply pipelines). So the worst case of an over-eager match here is a
|
|
131
|
+
* missing streaming flicker for one snapshot — NOT a dropped user answer.
|
|
132
|
+
* That asymmetry is why the matcher can stay aggressive on raw error shapes.
|
|
99
133
|
*/
|
|
100
134
|
export function handlePtyPartialPure(
|
|
101
135
|
text: string,
|
|
@@ -132,6 +166,11 @@ export function handlePtyPartialPure(
|
|
|
132
166
|
|
|
133
167
|
if (suppressed) return 'suppressed'
|
|
134
168
|
|
|
169
|
+
// Drop raw API-error TUI lines so they never leak to chat as the reply —
|
|
170
|
+
// the model-unavailable card (operator-event pipeline) renders these
|
|
171
|
+
// instead. See issue #2922 Bug 3.
|
|
172
|
+
if (looksLikeRawApiError(text)) return 'error-suppressed'
|
|
173
|
+
|
|
135
174
|
if (state.lastPtyPreviewByChat.get(sKey) === text) return 'dedup-skip'
|
|
136
175
|
|
|
137
176
|
const isFirst = !state.lastPtyPreviewByChat.has(sKey)
|
|
@@ -25,7 +25,18 @@
|
|
|
25
25
|
|
|
26
26
|
import { parse } from "./parse.js";
|
|
27
27
|
import { renderSafe, type RenderResult } from "./render.js";
|
|
28
|
-
import { RICH_MESSAGE_MAX_CHARS } from "../format.js";
|
|
28
|
+
import { RICH_MESSAGE_MAX_CHARS, splitMarkdownChunks } from "../format.js";
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* The legacy plain-text `sendMessage` / `editMessageText` wire cap (4096
|
|
32
|
+
* UTF-16 units). It does NOT apply to the rich path (`sendRichMessage`, up
|
|
33
|
+
* to `RICH_MESSAGE_MAX_CHARS`), but it DOES apply the moment `renderSafe`
|
|
34
|
+
* degrades a body to `mode: "plain"` — the send path then routes it through
|
|
35
|
+
* the plain `sendMessage` endpoint (see stream-controller `doSend`/`doEdit`).
|
|
36
|
+
* A degraded plain body larger than this is rejected by Telegram with
|
|
37
|
+
* `message is too long`, so `renderOutboundChunks` caps plain pieces here.
|
|
38
|
+
*/
|
|
39
|
+
export const PLAIN_TEXT_MAX_CHARS = 4096;
|
|
29
40
|
|
|
30
41
|
/** Parse the `SWITCHROOM_RICH_RENDER` flag value. Default OFF; accepts the
|
|
31
42
|
* same truthy tokens as the other switchroom env flags. Pure so the default
|
|
@@ -70,3 +81,70 @@ export function maybeRenderOutbound(
|
|
|
70
81
|
if (!richRenderEnabled(env)) return { text, mode: "markdown", degradations: [] };
|
|
71
82
|
return renderOutbound(text, maxLen);
|
|
72
83
|
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Flag-gated, CAP-ENFORCING transform for the live send path.
|
|
87
|
+
*
|
|
88
|
+
* `maybeRenderOutbound` returns ONE `RenderResult` and can only ever fit a
|
|
89
|
+
* body into a single wire message. But `renderSafe`'s markdown re-escaping
|
|
90
|
+
* (GFM-special chars gain a leading `\`) GROWS a chunk: a raw body sized just
|
|
91
|
+
* under `maxLen` can escape PAST it. `renderSafe` handles that by degrading
|
|
92
|
+
* the WHOLE document to `mode: "plain"` (raw source, no rich wrapper) — it
|
|
93
|
+
* never re-splits a multi-block body. The send path then ships that plain
|
|
94
|
+
* body through the 4096-capped `sendMessage` endpoint, so a ~32k plain body
|
|
95
|
+
* is rejected by Telegram (`message is too long`) and the answer is dropped.
|
|
96
|
+
*
|
|
97
|
+
* This function closes that gap: it returns an ARRAY of pieces, EACH of which
|
|
98
|
+
* is guaranteed to fit its own wire cap —
|
|
99
|
+
* - a `markdown` piece is `<= maxLen` (`renderSafe`'s own guarantee), and
|
|
100
|
+
* - a `plain` piece is `<= PLAIN_TEXT_MAX_CHARS` (the plain endpoint's cap).
|
|
101
|
+
* Pieces are cut only at `splitMarkdownChunks`' safe boundaries, so a fenced
|
|
102
|
+
* code block or a table row is NEVER bisected. Re-splitting the RAW source and
|
|
103
|
+
* re-rendering each piece also RECOVERS rich formatting the whole-document
|
|
104
|
+
* plain degradation would have thrown away: the smaller pieces individually
|
|
105
|
+
* escape under `maxLen` and come back as `markdown`.
|
|
106
|
+
*
|
|
107
|
+
* - flag OFF (default): `[{ text, mode: "markdown", degradations: [] }]` —
|
|
108
|
+
* a single passthrough piece, identical to `maybeRenderOutbound`.
|
|
109
|
+
* - flag ON, body fits: `[renderSafe(...)]` — a single piece, identical to
|
|
110
|
+
* `maybeRenderOutbound` (byte-for-byte for the common case).
|
|
111
|
+
* - flag ON, body oversize: 2+ cap-respecting pieces in send order.
|
|
112
|
+
*/
|
|
113
|
+
export function renderOutboundChunks(
|
|
114
|
+
text: string,
|
|
115
|
+
env: NodeJS.ProcessEnv = process.env,
|
|
116
|
+
maxLen: number = RICH_MESSAGE_MAX_CHARS,
|
|
117
|
+
plainMax: number = PLAIN_TEXT_MAX_CHARS,
|
|
118
|
+
): RenderResult[] {
|
|
119
|
+
if (!richRenderEnabled(env)) {
|
|
120
|
+
return [{ text, mode: "markdown", degradations: [] }];
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
const whole = renderOutbound(text, maxLen);
|
|
124
|
+
// `markdown` mode from renderSafe is already `<= maxLen`; a `plain` result
|
|
125
|
+
// that fits the plain cap is a single deliverable piece too. Common case.
|
|
126
|
+
if (whole.mode === "markdown" && whole.text.length <= maxLen) return [whole];
|
|
127
|
+
if (whole.mode === "plain" && whole.text.length <= plainMax) return [whole];
|
|
128
|
+
|
|
129
|
+
// Oversize: re-split the RAW source at safe boundaries and re-render each
|
|
130
|
+
// piece. Each sub-piece is `<= maxLen` in raw form, so it usually escapes
|
|
131
|
+
// back under `maxLen` and renders as rich markdown; a piece whose escaped
|
|
132
|
+
// form STILL overflows (or an indivisible plain blob) is emitted as plain,
|
|
133
|
+
// further split to `plainMax` so it fits the plain endpoint.
|
|
134
|
+
const out: RenderResult[] = [];
|
|
135
|
+
for (const rawPiece of splitMarkdownChunks(text, maxLen)) {
|
|
136
|
+
const rendered = renderOutbound(rawPiece, maxLen);
|
|
137
|
+
if (rendered.mode === "markdown" && rendered.text.length <= maxLen) {
|
|
138
|
+
out.push(rendered);
|
|
139
|
+
continue;
|
|
140
|
+
}
|
|
141
|
+
// Plain piece — cap at the plain-endpoint limit. `renderSafe` returned the
|
|
142
|
+
// raw source for a plain result, so split that source (safe boundaries,
|
|
143
|
+
// never bisecting a fence/table) into `<= plainMax` slices.
|
|
144
|
+
for (const plainPiece of splitMarkdownChunks(rendered.text, plainMax)) {
|
|
145
|
+
out.push({ text: plainPiece, mode: "plain", degradations: rendered.degradations });
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
// Defensive: a degenerate input that split to nothing still yields one piece.
|
|
149
|
+
return out.length > 0 ? out : [whole];
|
|
150
|
+
}
|
|
@@ -63,8 +63,48 @@ export interface RetryApiCallConfig {
|
|
|
63
63
|
observer?: RetryObserver
|
|
64
64
|
/** Optional log sink for flood-wait / network lines. */
|
|
65
65
|
log?: (line: string) => void
|
|
66
|
+
/**
|
|
67
|
+
* Fires whenever a Telegram 429 flood-wait is observed, BEFORE the sleep.
|
|
68
|
+
* The circuit-breaker (#2923) persists the flood-wait window here so that
|
|
69
|
+
* a container restart during an active ban can suppress non-essential
|
|
70
|
+
* sends (boot cards) instead of feeding the same per-bot-token flood
|
|
71
|
+
* counter and prolonging the ban. Best-effort; a throw here is swallowed.
|
|
72
|
+
*/
|
|
73
|
+
onFloodWait?: (retryAfterSec: number) => void
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* True when the thrown error is a LOCAL resource-exhaustion failure —
|
|
78
|
+
* ENOSPC (disk/tmpfs full), EDQUOT (quota), EIO, or ENOMEM — rather than a
|
|
79
|
+
* remote Telegram API failure. Issue #2923: an agent's tmpfs filling up
|
|
80
|
+
* wedges the outbound send's local staging step; retrying that as if it
|
|
81
|
+
* were a transient REMOTE failure hammers the Bot API and trips a per-bot
|
|
82
|
+
* flood ban. A local disk error must NOT drive remote retries — surface it
|
|
83
|
+
* as a distinct, non-retryable degraded state instead.
|
|
84
|
+
*/
|
|
85
|
+
export function isLocalResourceError(err: unknown): boolean {
|
|
86
|
+
const code = (err as { code?: unknown })?.code
|
|
87
|
+
if (typeof code === 'string' && ['ENOSPC', 'EDQUOT', 'EIO', 'ENOMEM'].includes(code)) {
|
|
88
|
+
return true
|
|
89
|
+
}
|
|
90
|
+
const msg = err instanceof Error ? err.message : String(err ?? '')
|
|
91
|
+
return (
|
|
92
|
+
// Word-boundaried so a substring can't false-match; covers the same set
|
|
93
|
+
// as the errno code list above (ENOSPC/EDQUOT/EIO/ENOMEM).
|
|
94
|
+
/\b(ENOSPC|EDQUOT|EIO|ENOMEM)\b/.test(msg) ||
|
|
95
|
+
/no space left on device/i.test(msg) ||
|
|
96
|
+
/disk quota exceeded/i.test(msg)
|
|
97
|
+
)
|
|
66
98
|
}
|
|
67
99
|
|
|
100
|
+
/**
|
|
101
|
+
* Marker error thrown when `retryApiCall` refuses to retry a LOCAL
|
|
102
|
+
* resource-exhaustion failure (#2923). Callers can detect this to surface a
|
|
103
|
+
* "degraded: local disk full" state rather than treating it like a remote
|
|
104
|
+
* send failure worth retrying.
|
|
105
|
+
*/
|
|
106
|
+
export const LOCAL_RESOURCE_EXHAUSTED = 'LOCAL_RESOURCE_EXHAUSTED'
|
|
107
|
+
|
|
68
108
|
const DEFAULT_SLEEP = (ms: number) => new Promise<void>((r) => setTimeout(r, ms))
|
|
69
109
|
|
|
70
110
|
/**
|
|
@@ -83,6 +123,7 @@ export function createRetryApiCall(
|
|
|
83
123
|
const sleep = config.sleep ?? DEFAULT_SLEEP
|
|
84
124
|
const observer = config.observer
|
|
85
125
|
const log = config.log
|
|
126
|
+
const onFloodWait = config.onFloodWait
|
|
86
127
|
|
|
87
128
|
return async function retryApiCall<T>(
|
|
88
129
|
fn: () => Promise<T>,
|
|
@@ -96,12 +137,33 @@ export function createRetryApiCall(
|
|
|
96
137
|
const msg = err instanceof Error ? err.message : String(err)
|
|
97
138
|
const desc = isGrammyErr ? (err as GrammyError).description : msg
|
|
98
139
|
|
|
140
|
+
// LOCAL resource exhaustion (#2923) — ENOSPC/EDQUOT/EIO/ENOMEM. This
|
|
141
|
+
// is a LOCAL disk/memory failure, not a remote API failure: retrying
|
|
142
|
+
// it in a tight loop is exactly what tripped the per-bot flood ban.
|
|
143
|
+
// Do NOT retry. Throw a distinct, non-retryable marker so the caller
|
|
144
|
+
// surfaces a degraded "local disk full" state and backs off hard.
|
|
145
|
+
if (isLocalResourceError(err)) {
|
|
146
|
+
log?.(
|
|
147
|
+
`telegram gateway: LOCAL resource exhaustion (${(err as { code?: string }).code ?? 'disk/mem'}) — ` +
|
|
148
|
+
`not retrying the send (would feed a flood ban); surfacing degraded state\n`,
|
|
149
|
+
)
|
|
150
|
+
observer?.onGiveUp?.({ attempts: attempt + 1, error: err })
|
|
151
|
+
throw Object.assign(new Error(LOCAL_RESOURCE_EXHAUSTED), { original: err })
|
|
152
|
+
}
|
|
153
|
+
|
|
99
154
|
// Flood-wait — sleep retry_after and try again.
|
|
100
155
|
if (isGrammyErr && (err as GrammyError).error_code === 429) {
|
|
101
156
|
const retryAfter = Number(
|
|
102
157
|
(err as GrammyError).parameters?.retry_after ?? 5,
|
|
103
158
|
)
|
|
104
159
|
const delayMs = retryAfter * 1000
|
|
160
|
+
// Persist the flood window so a restart during the ban can suppress
|
|
161
|
+
// non-essential sends instead of extending it (#2923 circuit breaker).
|
|
162
|
+
try {
|
|
163
|
+
onFloodWait?.(retryAfter)
|
|
164
|
+
} catch {
|
|
165
|
+
/* best-effort — never let the breaker hook break the retry path */
|
|
166
|
+
}
|
|
105
167
|
log?.(`telegram gateway: 429 rate limited, waiting ${retryAfter}s\n`)
|
|
106
168
|
observer?.onRetry?.({ attempt, reason: 'flood_wait', delayMs })
|
|
107
169
|
await sleep(delayMs)
|
|
@@ -29,6 +29,7 @@ import { createHash } from 'crypto'
|
|
|
29
29
|
import { AsyncLocalStorage } from 'async_hooks'
|
|
30
30
|
import { clearStaleTelegramPollingState } from '../startup-reset.js'
|
|
31
31
|
import { createRetryApiCall } from '../retry-api-call.js'
|
|
32
|
+
import { makeFloodWaitRecorder } from '../flood-circuit-breaker.js'
|
|
32
33
|
import { RICH_MESSAGE_MAX_CHARS } from '../format.js'
|
|
33
34
|
|
|
34
35
|
// ─── tg-post tag plumbing ─────────────────────────────────────────────────
|
|
@@ -149,9 +150,15 @@ export function installTgPostLogger(bot: Bot): void {
|
|
|
149
150
|
* Usage:
|
|
150
151
|
* const robustApiCall = createRobustApiCall()
|
|
151
152
|
*/
|
|
152
|
-
export function createRobustApiCall() {
|
|
153
|
+
export function createRobustApiCall(opts: { floodStatePath?: string } = {}) {
|
|
153
154
|
return createRetryApiCall({
|
|
154
155
|
log: (line) => process.stderr.write(line),
|
|
156
|
+
// #2923: persist every observed 429 flood-wait window so a restart during
|
|
157
|
+
// the ban can suppress non-essential sends (boot cards) instead of feeding
|
|
158
|
+
// the per-bot flood counter and prolonging the ban.
|
|
159
|
+
...(opts.floodStatePath
|
|
160
|
+
? { onFloodWait: makeFloodWaitRecorder(opts.floodStatePath) }
|
|
161
|
+
: {}),
|
|
155
162
|
})
|
|
156
163
|
}
|
|
157
164
|
|
|
@@ -423,6 +423,20 @@ export function formatFrameworkFallbackText(
|
|
|
423
423
|
return null
|
|
424
424
|
}
|
|
425
425
|
|
|
426
|
+
/**
|
|
427
|
+
* #2995 — the LONGEST-running in-flight tool for a turn key, or null when
|
|
428
|
+
* none is tracked. Read-only accessor over `inFlightTools` for the
|
|
429
|
+
* mid-flight busy-ack: the gateway needs the blocking step's name/label
|
|
430
|
+
* and its age to decide whether (and how) to ack a buffered mid-turn
|
|
431
|
+
* inbound. Does not touch the silence clock.
|
|
432
|
+
*/
|
|
433
|
+
export function longestInFlightTool(key: string, now: number): ToolSnapshot | null {
|
|
434
|
+
const s = state.get(key)
|
|
435
|
+
if (s == null) return null
|
|
436
|
+
const snaps = snapshotInFlight(s, now)
|
|
437
|
+
return snaps.length > 0 ? snaps[0]! : null
|
|
438
|
+
}
|
|
439
|
+
|
|
426
440
|
/** Snapshot in-flight tools sorted longest-running first — for the honest
|
|
427
441
|
* floor/fallback message body. */
|
|
428
442
|
function snapshotInFlight(s: SilencePokeState, now: number): ToolSnapshot[] {
|