switchroom 0.18.11 → 0.18.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-scheduler/index.js +37 -5
- package/dist/auth-broker/index.js +116 -78
- package/dist/cli/hindsight-mental-model-pretool.mjs +39 -0
- package/dist/cli/ms-365-write-pretool.mjs +31 -8
- package/dist/cli/notion-write-pretool.mjs +38 -6
- package/dist/cli/skill-validate-pretool.mjs +144 -2847
- package/dist/cli/switchroom.js +3529 -4543
- package/dist/cli/ui/index.html +163 -17
- package/dist/host-control/main.js +605 -2847
- package/dist/vault/approvals/kernel-server.js +120 -13
- package/dist/vault/broker/server.js +259 -157
- package/package.json +3 -4
- package/profiles/_base/start.sh.hbs +65 -0
- package/profiles/_shared/vault-protocol.md.hbs +3 -1
- package/profiles/coding/CLAUDE.md.hbs +1 -1
- package/profiles/default/CLAUDE.md.hbs +2 -2
- package/profiles/executive-assistant/CLAUDE.md.hbs +1 -1
- package/profiles/health-coach/CLAUDE.md.hbs +1 -1
- package/telegram-plugin/bridge/bridge.ts +37 -0
- package/telegram-plugin/bridge/inbound-dedup.ts +101 -0
- package/telegram-plugin/dist/bridge/bridge.js +122 -4
- package/telegram-plugin/dist/gateway/gateway.js +4213 -3288
- package/telegram-plugin/dist/server.js +139 -5
- package/telegram-plugin/flood-circuit-breaker.ts +493 -21
- package/telegram-plugin/format.ts +19 -0
- package/telegram-plugin/gateway/approval-hold.ts +602 -0
- package/telegram-plugin/gateway/auth-command.ts +92 -2
- package/telegram-plugin/gateway/auth-loopback-relay.ts +670 -0
- package/telegram-plugin/gateway/boot-card.ts +12 -5
- package/telegram-plugin/gateway/callback-query-handlers.ts +88 -1
- package/telegram-plugin/gateway/config-approval-handler.ts +6 -1
- package/telegram-plugin/gateway/disconnect-flush.ts +19 -0
- package/telegram-plugin/gateway/dm-pin-sweep.test.ts +251 -0
- package/telegram-plugin/gateway/dm-pin-sweep.ts +178 -0
- package/telegram-plugin/gateway/gateway.ts +1695 -230
- package/telegram-plugin/gateway/hostd-dispatch.ts +23 -0
- package/telegram-plugin/gateway/idle-clear.ts +90 -6
- package/telegram-plugin/gateway/inbound-delivery-machine-shadow.ts +26 -5
- package/telegram-plugin/gateway/inject-handler.ts +8 -0
- package/telegram-plugin/gateway/ipc-protocol.ts +46 -3
- package/telegram-plugin/gateway/ipc-server.ts +43 -0
- package/telegram-plugin/gateway/mental-model-propose-resolve.ts +145 -37
- package/telegram-plugin/gateway/model-command.ts +9 -3
- package/telegram-plugin/gateway/pending-session-command.ts +13 -1
- package/telegram-plugin/gateway/permission-ttl-sweep.ts +66 -0
- package/telegram-plugin/gateway/pre-approval-check.ts +74 -0
- package/telegram-plugin/gateway/queued-card-store.ts +217 -0
- package/telegram-plugin/gateway/session-model-file.ts +26 -1
- package/telegram-plugin/gateway/turn-end-gate-backstop.ts +59 -0
- package/telegram-plugin/gateway/turn-end-gate.ts +95 -0
- package/telegram-plugin/gateway/turn-typing-loop.ts +10 -2
- package/telegram-plugin/gateway/unhandled-rejection-policy.ts +13 -0
- package/telegram-plugin/history.ts +51 -0
- package/telegram-plugin/hooks/dispatch-claim-scan.mjs +259 -0
- package/telegram-plugin/hooks/dispatch-claim-stop.mjs +129 -0
- package/telegram-plugin/hooks/hooks.json +9 -0
- package/telegram-plugin/inline-keyboard-callbacks.ts +303 -2
- package/telegram-plugin/model-unavailable.ts +41 -11
- package/telegram-plugin/operator-events.ts +23 -0
- package/telegram-plugin/outbound-field-redact.ts +69 -0
- package/telegram-plugin/package.json +0 -1
- package/telegram-plugin/permission-rule.ts +1 -0
- package/telegram-plugin/permission-title.ts +1 -0
- package/telegram-plugin/render/render.ts +32 -14
- package/telegram-plugin/retry-api-call.ts +212 -2
- package/telegram-plugin/scoped-approval.ts +11 -2
- package/telegram-plugin/secret-detect/chunker.ts +18 -4
- package/telegram-plugin/secret-detect/index.ts +12 -56
- package/telegram-plugin/send-gate-degraded.test.ts +574 -0
- package/telegram-plugin/send-gate-observability.test.ts +470 -0
- package/telegram-plugin/send-gate-observability.ts +355 -0
- package/telegram-plugin/send-gate.test.ts +717 -0
- package/telegram-plugin/send-gate.ts +1056 -0
- package/telegram-plugin/session-tail.ts +82 -7
- package/telegram-plugin/shared/bot-runtime.ts +17 -5
- package/telegram-plugin/shared/gw-trace-gate.ts +105 -0
- package/telegram-plugin/status-pin-driver.ts +52 -7
- package/telegram-plugin/status-pin.ts +81 -0
- package/telegram-plugin/subagent-watcher.ts +173 -18
- package/telegram-plugin/tests/activity-card-wiring.test.ts +18 -5
- package/telegram-plugin/tests/approval-hold-harness.ts +425 -0
- package/telegram-plugin/tests/approval-hold-outcome.test.ts +327 -0
- package/telegram-plugin/tests/approval-hold-record.test.ts +531 -0
- package/telegram-plugin/tests/approval-hold-redeliver.test.ts +602 -0
- package/telegram-plugin/tests/auth-loopback-relay.test.ts +533 -0
- package/telegram-plugin/tests/boot-card-flood-suppress.test.ts +53 -7
- package/telegram-plugin/tests/busy-key-reaper.test.ts +1 -0
- package/telegram-plugin/tests/callback-query-handlers.test.ts +65 -0
- package/telegram-plugin/tests/dispatch-claim-scan.test.ts +250 -0
- package/telegram-plugin/tests/flood-breaker-blindness.test.ts +213 -0
- package/telegram-plugin/tests/flood-windows-persistence.test.ts +224 -0
- package/telegram-plugin/tests/gateway-boot-marker-clear.test.ts +3 -3
- package/telegram-plugin/tests/gateway-disconnect-flush.test.ts +29 -1
- package/telegram-plugin/tests/gateway-loopback-paste-redact.test.ts +66 -0
- package/telegram-plugin/tests/gateway-outbound-redact.test.ts +57 -0
- package/telegram-plugin/tests/gw-trace-gate.test.ts +105 -0
- package/telegram-plugin/tests/history.test.ts +115 -0
- package/telegram-plugin/tests/idle-clear.test.ts +233 -3
- package/telegram-plugin/tests/inbound-dedup.test.ts +93 -0
- package/telegram-plugin/tests/inbound-message-types.test.ts +5 -1
- package/telegram-plugin/tests/inline-keyboard-callbacks.test.ts +448 -0
- package/telegram-plugin/tests/ipc-server-check-pre-approved.test.ts +194 -0
- package/telegram-plugin/tests/mental-model-propose-resolve.test.ts +123 -0
- package/telegram-plugin/tests/missed-approvals-wiring.test.ts +1 -1
- package/telegram-plugin/tests/model-command.test.ts +14 -0
- package/telegram-plugin/tests/operator-events-session-tail.test.ts +74 -0
- package/telegram-plugin/tests/outbound-field-redact.test.ts +107 -0
- package/telegram-plugin/tests/pending-session-command.test.ts +21 -0
- package/telegram-plugin/tests/permission-card-routing.test.ts +30 -5
- package/telegram-plugin/tests/permission-no-repeat-wiring.test.ts +8 -7
- package/telegram-plugin/tests/permission-rearm-wiring.test.ts +1 -1
- package/telegram-plugin/tests/pre-approval-check.test.ts +148 -0
- package/telegram-plugin/tests/queued-card-store.test.ts +232 -0
- package/telegram-plugin/tests/reaction-flush-turn-gated.test.ts +100 -0
- package/telegram-plugin/tests/reaction-gate-routing.test.ts +173 -0
- package/telegram-plugin/tests/render/render.test.ts +88 -0
- package/telegram-plugin/tests/retry-api-call.test.ts +398 -0
- package/telegram-plugin/tests/scoped-approval.test.ts +27 -0
- package/telegram-plugin/tests/secret-detect-chunk-overlap.test.ts +65 -0
- package/telegram-plugin/tests/secret-detect-oauth-code.test.ts +5 -4
- package/telegram-plugin/tests/session-model-file.test.ts +50 -0
- package/telegram-plugin/tests/session-tail-sidecar-reap.test.ts +268 -0
- package/telegram-plugin/tests/status-pin-boot-recovery.test.ts +35 -14
- package/telegram-plugin/tests/status-pin.test.ts +275 -1
- package/telegram-plugin/tests/subagent-watcher-deferral-log-ratelimit.test.ts +316 -0
- package/telegram-plugin/tests/subagent-watcher-fd-leak.test.ts +275 -0
- package/telegram-plugin/tests/turn-end-gate-backstop.test.ts +92 -0
- package/telegram-plugin/tests/turn-end-gate.test.ts +137 -0
- package/telegram-plugin/tests/typing-emitter.test.ts +586 -0
- package/telegram-plugin/tests/unhandled-rejection-policy.test.ts +20 -0
- package/telegram-plugin/tests/worktree-watch-cwds.test.ts +215 -1
- package/telegram-plugin/typing-emitter.ts +224 -0
- package/telegram-plugin/uat/scenarios/jtbd-feel-like-a-colleague-dm.test.ts +136 -0
- package/telegram-plugin/welcome-text.ts +42 -0
- package/telegram-plugin/worktree-watch-cwds.ts +194 -5
- package/vendor/hindsight-memory/scripts/drain_pending.py +22 -6
- package/vendor/hindsight-memory/scripts/lib/client.py +12 -5
- package/vendor/hindsight-memory/scripts/lib/directives.py +38 -3
- package/vendor/hindsight-memory/scripts/lib/pending.py +36 -9
- package/vendor/hindsight-memory/scripts/session_end.py +14 -3
- package/vendor/hindsight-memory/scripts/session_start.py +21 -0
- package/vendor/hindsight-memory/scripts/tests/test_directives.py +38 -0
- package/vendor/hindsight-memory/tests/test_drain_pending.py +68 -0
- package/vendor/hindsight-memory/tests/test_pending.py +44 -0
- package/vendor/hindsight-memory/tests/test_session_end_pending.py +38 -0
- package/vendor/hindsight-memory/tests/test_session_start_drain.py +155 -0
- package/telegram-plugin/channel-envelope-safety.test.ts +0 -56
- package/telegram-plugin/channel-envelope-safety.ts +0 -56
- package/telegram-plugin/secret-detect/secretlint-source.ts +0 -95
- package/telegram-plugin/tests/secret-detect-secretlint.test.ts +0 -105
|
@@ -41,7 +41,8 @@ function isMultiAgentEnabled(env: NodeJS.ProcessEnv = process.env): boolean {
|
|
|
41
41
|
return env.PROGRESS_CARD_MULTI_AGENT !== '0'
|
|
42
42
|
}
|
|
43
43
|
import { classifyClaudeError, type OperatorEventKind } from './operator-events.js'
|
|
44
|
-
import {
|
|
44
|
+
import { isTransientUpstreamSignal } from './model-unavailable.js'
|
|
45
|
+
import { createToolLabelSidecar, type ToolLabelSidecar, type SidecarOptions } from './tool-label-sidecar.js'
|
|
45
46
|
import { isModelSentinel } from './model-label.js'
|
|
46
47
|
|
|
47
48
|
/** Match Claude Code's cli.js VX() function. */
|
|
@@ -668,13 +669,26 @@ export function detectErrorInTranscriptLine(
|
|
|
668
669
|
typeof obj.apiErrorStatus === 'number' ? obj.apiErrorStatus : null
|
|
669
670
|
const errStr = typeof obj.error === 'string' ? obj.error : ''
|
|
670
671
|
const text = extractAssistantText(obj)
|
|
671
|
-
// A 429 in this shape is a subscription usage-limit
|
|
672
|
-
// a reset time) — classify it quota-exhausted so the operator
|
|
673
|
-
// resolves to an auto-fallback-eligible kind
|
|
672
|
+
// A 429 in this shape is USUALLY a subscription usage-limit wall (it
|
|
673
|
+
// carries a reset time) — classify it quota-exhausted so the operator
|
|
674
|
+
// event resolves to an auto-fallback-eligible kind that always shows the
|
|
675
|
+
// "model unavailable" card. BUT Anthropic also emits a 429 for a TRANSIENT
|
|
676
|
+
// per-account burst / RPM throttle whose wording explicitly negates the
|
|
677
|
+
// account-quota reading ("This request would exceed your account's rate
|
|
678
|
+
// limit … not your usage limit"). That is a self-healing few-second throttle
|
|
679
|
+
// Claude Code retries internally — blanket-labeling it quota-exhausted fired
|
|
680
|
+
// a false scary card on the fleet (carrie incident, 2026-07-12). So a 429 is
|
|
681
|
+
// only quota-exhausted when it LACKS an explicit transient-burst marker;
|
|
682
|
+
// with one, classify it rate-limited so it takes the calm path (no card, no
|
|
683
|
+
// failover). Keyed on the explicit transient NEGATION (canonical list in
|
|
684
|
+
// model-unavailable.ts) — an ambiguous 429 that merely says "limit" stays
|
|
685
|
+
// quota-exhausted, biasing toward surfacing a real wall. Other statuses fall
|
|
674
686
|
// through to the shared classifier.
|
|
675
687
|
const kind: OperatorEventKind =
|
|
676
688
|
status === 429
|
|
677
|
-
?
|
|
689
|
+
? isTransientUpstreamSignal(`${text}\n${errStr}`)
|
|
690
|
+
? 'rate-limited'
|
|
691
|
+
: 'quota-exhausted'
|
|
678
692
|
: classifyClaudeError({ type: errStr, status, message: text })
|
|
679
693
|
// An `isApiErrorMessage` line is Claude surfacing the failure to the
|
|
680
694
|
// user — terminal by construction (Claude writes this shape only
|
|
@@ -781,6 +795,13 @@ export interface SessionTailConfig {
|
|
|
781
795
|
claudeHome?: string
|
|
782
796
|
/** How often to re-scan for a new active session file (ms). Default 500. */
|
|
783
797
|
rescanIntervalMs?: number
|
|
798
|
+
/**
|
|
799
|
+
* Idle window before an inactive sub-agent FSWatcher (and its PreToolUse
|
|
800
|
+
* sidecar) is reaped, in ms. Defaults to 5 minutes — well past the 99th-
|
|
801
|
+
* percentile sub-agent completion time. Exposed only so tests can drive the
|
|
802
|
+
* reap deterministically without a 5-minute wall-clock wait.
|
|
803
|
+
*/
|
|
804
|
+
subTailIdleReapMs?: number
|
|
784
805
|
/** Optional logger. */
|
|
785
806
|
log?: (msg: string) => void
|
|
786
807
|
/** Called for each parsed event. */
|
|
@@ -791,6 +812,18 @@ export interface SessionTailConfig {
|
|
|
791
812
|
* TODO(Phase 4b): wire this to the gateway's emitOperatorEvent pipeline.
|
|
792
813
|
*/
|
|
793
814
|
onOperatorEvent?: (event: TailOperatorEvent) => void
|
|
815
|
+
/**
|
|
816
|
+
* PreToolUse sidecar factory. Defaults to the real `createToolLabelSidecar`;
|
|
817
|
+
* production never sets this. It exists as a dependency-injection seam so the
|
|
818
|
+
* M1 FD-leak reap test can drive a fake sidecar per session WITHOUT
|
|
819
|
+
* `vi.mock`-ing the shared `tool-label-sidecar` module: bun's `vi.mock` is
|
|
820
|
+
* process-global (not file-scoped like vitest), and the CI bun-test shard
|
|
821
|
+
* runs the whole `tests/` dir in ONE process, so a module-mock here would
|
|
822
|
+
* leak into the real `tool-label-sidecar.test.ts` suite and break it. This
|
|
823
|
+
* mirrors the repo's bun-safe injection precedent (`vault-write-posture`'s
|
|
824
|
+
* optional `deps` param).
|
|
825
|
+
*/
|
|
826
|
+
createSidecar?: (opts: SidecarOptions) => ToolLabelSidecar
|
|
794
827
|
}
|
|
795
828
|
|
|
796
829
|
export interface SessionTailHandle {
|
|
@@ -887,6 +920,7 @@ export function startSessionTail(config: SessionTailConfig): SessionTailHandle {
|
|
|
887
920
|
// $TELEGRAM_STATE_DIR/tool-labels-<session_id>.jsonl. Each sub-agent
|
|
888
921
|
// has its OWN sessionId (its jsonl filename stem), so we key by that.
|
|
889
922
|
const sidecars = new Map<string, ToolLabelSidecar>()
|
|
923
|
+
const createSidecar = config.createSidecar ?? createToolLabelSidecar
|
|
890
924
|
const stateDirForSidecar = process.env.TELEGRAM_STATE_DIR ?? null
|
|
891
925
|
function sessionIdForFile(file: string | null): string | null {
|
|
892
926
|
if (!file) return null
|
|
@@ -898,7 +932,7 @@ export function startSessionTail(config: SessionTailConfig): SessionTailHandle {
|
|
|
898
932
|
const existing = sidecars.get(sessionId)
|
|
899
933
|
if (existing) return existing
|
|
900
934
|
try {
|
|
901
|
-
const s =
|
|
935
|
+
const s = createSidecar({ stateDir: stateDirForSidecar, sessionId })
|
|
902
936
|
sidecars.set(sessionId, s)
|
|
903
937
|
// Real-time draft-mirror source: emit a `tool_label` event the moment
|
|
904
938
|
// the hook writes a label (flush-independent), so the gateway can
|
|
@@ -913,6 +947,22 @@ export function startSessionTail(config: SessionTailConfig): SessionTailHandle {
|
|
|
913
947
|
return null
|
|
914
948
|
}
|
|
915
949
|
}
|
|
950
|
+
/**
|
|
951
|
+
* M1 FD-leak fix: stop and forget the PreToolUse sidecar for a session that
|
|
952
|
+
* has ended (a rotated-away parent session, or a reaped sub-agent). Each
|
|
953
|
+
* sidecar holds its own stat-poll timer (and, on real fs, a file handle);
|
|
954
|
+
* pre-fix they were only reaped in `stop()`, so every session rotation
|
|
955
|
+
* (`/clear`, compaction → new sessionId) and every finished sub-agent leaked
|
|
956
|
+
* one for the gateway's life. Idempotent — a no-op when the key is absent.
|
|
957
|
+
*/
|
|
958
|
+
function stopSidecar(sessionId: string | null): void {
|
|
959
|
+
if (!sessionId) return
|
|
960
|
+
const s = sidecars.get(sessionId)
|
|
961
|
+
if (!s) return
|
|
962
|
+
try { s.stop() } catch { /* ignore */ }
|
|
963
|
+
sidecars.delete(sessionId)
|
|
964
|
+
}
|
|
965
|
+
|
|
916
966
|
function decorate(ev: SessionEvent, sessionId: string | null): SessionEvent {
|
|
917
967
|
if (!sessionId) return ev
|
|
918
968
|
if (ev.kind !== 'tool_use' && ev.kind !== 'sub_agent_tool_use') return ev
|
|
@@ -1024,6 +1074,19 @@ export function startSessionTail(config: SessionTailConfig): SessionTailHandle {
|
|
|
1024
1074
|
try { watcher.close() } catch { /* ignore */ }
|
|
1025
1075
|
watcher = null
|
|
1026
1076
|
}
|
|
1077
|
+
// M1 FD-leak fix: we are rotating the PARENT tail off `currentFile`; its
|
|
1078
|
+
// PreToolUse sidecar is no longer needed (its watcher just closed). Reap it
|
|
1079
|
+
// so `/clear`- and compaction-driven session rotations don't accumulate one
|
|
1080
|
+
// idle sidecar poll-timer per rotation. Parent session ids are the JSONL
|
|
1081
|
+
// stem (`<uuid>`); sub-agent sidecars are keyed by `agent-<id>` stems and
|
|
1082
|
+
// owned by their sub-tail (reaped in `reapIdleSubTails`), so this never
|
|
1083
|
+
// stops a sidecar a live sub-tail still depends on. A later re-attach to
|
|
1084
|
+
// this same file transparently recreates the sidecar via `ensureSidecar`.
|
|
1085
|
+
const rotatedAwaySid = sessionIdForFile(currentFile)
|
|
1086
|
+
const nextSid = sessionIdForFile(file)
|
|
1087
|
+
if (rotatedAwaySid != null && rotatedAwaySid !== nextSid) {
|
|
1088
|
+
stopSidecar(rotatedAwaySid)
|
|
1089
|
+
}
|
|
1027
1090
|
currentFile = file
|
|
1028
1091
|
const prior = fileCursors.get(file)
|
|
1029
1092
|
if (prior != null) {
|
|
@@ -1113,7 +1176,12 @@ export function startSessionTail(config: SessionTailConfig): SessionTailHandle {
|
|
|
1113
1176
|
* very-long task (rescanSubagents picks the file back up on the
|
|
1114
1177
|
* next tick if it grows).
|
|
1115
1178
|
*/
|
|
1116
|
-
|
|
1179
|
+
// Floor-clamp the reap window: a 0 / negative override would make
|
|
1180
|
+
// `reapIdleSubTails` treat every live sub-tail as instantly idle
|
|
1181
|
+
// (cutoff = Date.now() - 0 ≥ lastActivityAt), reaping every live
|
|
1182
|
+
// sidecar on the first tick. Only tests set this today, but the clamp
|
|
1183
|
+
// makes the footgun unreachable — the smallest sane window is 1s.
|
|
1184
|
+
const IDLE_FSWATCH_TTL_MS = Math.max(1000, config.subTailIdleReapMs ?? 5 * 60 * 1000)
|
|
1117
1185
|
|
|
1118
1186
|
function readSub(t: SubTail): void {
|
|
1119
1187
|
if (stopped) return
|
|
@@ -1254,6 +1322,13 @@ export function startSessionTail(config: SessionTailConfig): SessionTailHandle {
|
|
|
1254
1322
|
try { t.watcher.close() } catch { /* ignore */ }
|
|
1255
1323
|
t.watcher = null
|
|
1256
1324
|
}
|
|
1325
|
+
// M1 FD-leak fix: reap the sub-agent's PreToolUse sidecar alongside its
|
|
1326
|
+
// file watcher. Sub-agent sidecars are keyed by the sub file's stem
|
|
1327
|
+
// (`agent-<id>`), created lazily by `decorate` while reading the sub
|
|
1328
|
+
// JSONL. Pre-fix `reapIdleSubTails` closed the sub-tail watcher but left
|
|
1329
|
+
// the sidecar (and its poll timer) alive until `stop()`, so a long-lived
|
|
1330
|
+
// agent leaked one sidecar per finished sub-agent.
|
|
1331
|
+
stopSidecar(sessionIdForFile(t.file))
|
|
1257
1332
|
subTails.delete(file)
|
|
1258
1333
|
log?.(`session-tail: reaped idle sub ${t.agentId} (${file})`)
|
|
1259
1334
|
}
|
|
@@ -29,8 +29,9 @@ import { createHash } from 'crypto'
|
|
|
29
29
|
import { AsyncLocalStorage } from 'async_hooks'
|
|
30
30
|
import { clearStaleTelegramPollingState } from '../startup-reset.js'
|
|
31
31
|
import { createRetryApiCall } from '../retry-api-call.js'
|
|
32
|
-
import { makeFloodWaitRecorder } from '../flood-circuit-breaker.js'
|
|
32
|
+
import { makeFloodWaitRecorder, makeFloodWaitProbe } from '../flood-circuit-breaker.js'
|
|
33
33
|
import { RICH_MESSAGE_MAX_CHARS } from '../format.js'
|
|
34
|
+
import { shouldEmitTgPost } from './gw-trace-gate.js'
|
|
34
35
|
|
|
35
36
|
// ─── tg-post tag plumbing ─────────────────────────────────────────────────
|
|
36
37
|
|
|
@@ -115,9 +116,14 @@ export function installTgPostLogger(bot: Bot): void {
|
|
|
115
116
|
const tagSuffix = formatTgPostTags(_getTgPostTags())
|
|
116
117
|
try {
|
|
117
118
|
const res = await prev(method, payload, signal)
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
119
|
+
// #3025: suppress zero-signal per-poll heartbeats (getUpdates/getMe
|
|
120
|
+
// status=ok, one line per ~30s long-poll tick) unless the operator
|
|
121
|
+
// set SWITCHROOM_GW_TRACE. Errors and all other methods still log.
|
|
122
|
+
if (shouldEmitTgPost(method, 'ok')) {
|
|
123
|
+
process.stderr.write(
|
|
124
|
+
`tg-post method=${method} chat=${chat} thread=${thread} parse_mode=${parseMode} bytes=${bytes} hash=${hash} status=ok err=- code=- desc=-${tagSuffix}\n`,
|
|
125
|
+
)
|
|
126
|
+
}
|
|
121
127
|
return res
|
|
122
128
|
} catch (err) {
|
|
123
129
|
const errClass = err instanceof GrammyError
|
|
@@ -156,8 +162,14 @@ export function createRobustApiCall(opts: { floodStatePath?: string } = {}) {
|
|
|
156
162
|
// #2923: persist every observed 429 flood-wait window so a restart during
|
|
157
163
|
// the ban can suppress non-essential sends (boot cards) instead of feeding
|
|
158
164
|
// the per-bot flood counter and prolonging the ban.
|
|
165
|
+
// #3084: and refuse to issue a call while that window is still open —
|
|
166
|
+
// otherwise the card heartbeats re-drive a request into the ban every
|
|
167
|
+
// few seconds once the policy stops sleeping long waits.
|
|
159
168
|
...(opts.floodStatePath
|
|
160
|
-
? {
|
|
169
|
+
? {
|
|
170
|
+
onFloodWait: makeFloodWaitRecorder(opts.floodStatePath),
|
|
171
|
+
floodWaitRemainingMs: makeFloodWaitProbe(opts.floodStatePath),
|
|
172
|
+
}
|
|
161
173
|
: {}),
|
|
162
174
|
})
|
|
163
175
|
}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Gateway trace verbosity gate (#3025).
|
|
3
|
+
*
|
|
4
|
+
* The gateway emits two families of high-frequency stderr trace lines that
|
|
5
|
+
* are pure per-poll noise on a healthy, idle agent:
|
|
6
|
+
*
|
|
7
|
+
* - `tg-post method=getUpdates ... status=ok` / `method=getMe ... status=ok`
|
|
8
|
+
* — one line per long-poll tick (~every 30s) from the tg-post
|
|
9
|
+
* observability transformer in `bot-runtime.ts`.
|
|
10
|
+
* - `gw-trace shadow event=tick effects=[] global=bridge_alive_idle ...`
|
|
11
|
+
* — one line per delivery-machine TTL tick (~every 30s) from
|
|
12
|
+
* `inbound-delivery-machine-shadow.ts`.
|
|
13
|
+
*
|
|
14
|
+
* With no log rotation these ungated lines grew gateway-supervisor.log to
|
|
15
|
+
* 555MB on clerk / 162MB on klanker and filled the host disk. They carry
|
|
16
|
+
* zero signal when nothing happened: a successful empty long-poll and an
|
|
17
|
+
* idle no-op tick.
|
|
18
|
+
*
|
|
19
|
+
* This module centralises the "should I emit a zero-signal trace line?"
|
|
20
|
+
* decision behind a single debug env flag so operators can turn the full
|
|
21
|
+
* firehose back on when actually debugging the delivery/poll path.
|
|
22
|
+
*
|
|
23
|
+
* Flag: set `SWITCHROOM_GW_TRACE=1` (or `true`/`yes`/`on`, any case) to
|
|
24
|
+
* restore every trace line unconditionally. Unset / any other value → zero-signal polling
|
|
25
|
+
* success + idle-tick lines are suppressed, while ALL error lines, all
|
|
26
|
+
* non-poll methods (sendMessage, editMessageText, ...), and every tick or
|
|
27
|
+
* event that produced a real effect or state change are still emitted.
|
|
28
|
+
*
|
|
29
|
+
* Read once at module init — the flag is a boot-time debug switch, not a
|
|
30
|
+
* hot-reloadable knob, so a cached read keeps the per-line cost at a
|
|
31
|
+
* single boolean check.
|
|
32
|
+
*/
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Pure env-flag parse — exported so tests can exercise the parsing
|
|
36
|
+
* directly. Kept separate from the module-init read because the dual
|
|
37
|
+
* vitest/bun runner boundary means module-cache tricks
|
|
38
|
+
* (`vi.resetModules()` + re-import) don't work under bun's vitest shim;
|
|
39
|
+
* flag variants must be testable via explicit arguments.
|
|
40
|
+
*/
|
|
41
|
+
export function computeGwTraceVerbose(flag: string | undefined): boolean {
|
|
42
|
+
const v = (flag ?? '').trim().toLowerCase()
|
|
43
|
+
return v === '1' || v === 'true' || v === 'yes' || v === 'on'
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** True when the operator opted into the full gateway trace firehose. */
|
|
47
|
+
export const gwTraceVerbose: boolean = computeGwTraceVerbose(
|
|
48
|
+
process.env.SWITCHROOM_GW_TRACE,
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Bot API methods whose successful long-poll ticks are zero-signal:
|
|
53
|
+
* `getUpdates` is the long-poll itself and `getMe` is the periodic
|
|
54
|
+
* identity refresh. A `status=ok` on either says only "polling is alive",
|
|
55
|
+
* which the gateway already reports once at startup. Errors on these
|
|
56
|
+
* methods are NOT suppressed (see `shouldEmitTgPost`).
|
|
57
|
+
*/
|
|
58
|
+
const ZERO_SIGNAL_POLL_METHODS = new Set(['getUpdates', 'getMe'])
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Decide whether a `tg-post` line should be written.
|
|
62
|
+
*
|
|
63
|
+
* Suppressed (unless `SWITCHROOM_GW_TRACE` is set):
|
|
64
|
+
* - `status=ok` for `getUpdates` / `getMe` — the per-poll heartbeat.
|
|
65
|
+
*
|
|
66
|
+
* Always emitted:
|
|
67
|
+
* - any `status=err` (real failures — the whole point of the log),
|
|
68
|
+
* - every other method's `ok` line (sendMessage/editMessageText/... are
|
|
69
|
+
* genuine outbound observability, #656/#657).
|
|
70
|
+
*/
|
|
71
|
+
export function shouldEmitTgPost(
|
|
72
|
+
method: string,
|
|
73
|
+
status: 'ok' | 'err',
|
|
74
|
+
verbose: boolean = gwTraceVerbose,
|
|
75
|
+
): boolean {
|
|
76
|
+
if (verbose) return true
|
|
77
|
+
if (status === 'err') return true
|
|
78
|
+
return !ZERO_SIGNAL_POLL_METHODS.has(method)
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Decide whether a `gw-trace shadow` line should be written.
|
|
83
|
+
*
|
|
84
|
+
* Suppressed (unless `SWITCHROOM_GW_TRACE` is set):
|
|
85
|
+
* - `event=tick` that produced NO effects AND left the machine in an
|
|
86
|
+
* idle global state (`bridge_alive_idle` / `bridge_dead`) — the
|
|
87
|
+
* no-op TTL heartbeat.
|
|
88
|
+
*
|
|
89
|
+
* Always emitted:
|
|
90
|
+
* - any non-tick event (real inbound/turn/bridge activity),
|
|
91
|
+
* - any tick that produced effects (e.g. a TTL turn expiry) or that
|
|
92
|
+
* landed the machine in `bridge_alive_in_turn`.
|
|
93
|
+
*/
|
|
94
|
+
export function shouldEmitShadowTrace(
|
|
95
|
+
eventKind: string,
|
|
96
|
+
effectCount: number,
|
|
97
|
+
globalKind: string,
|
|
98
|
+
verbose: boolean = gwTraceVerbose,
|
|
99
|
+
): boolean {
|
|
100
|
+
if (verbose) return true
|
|
101
|
+
if (eventKind !== 'tick') return true
|
|
102
|
+
if (effectCount > 0) return true
|
|
103
|
+
const idle = globalKind === 'bridge_alive_idle' || globalKind === 'bridge_dead'
|
|
104
|
+
return !idle
|
|
105
|
+
}
|
|
@@ -29,8 +29,8 @@
|
|
|
29
29
|
* job spec.
|
|
30
30
|
*/
|
|
31
31
|
|
|
32
|
-
import type { PinState, DesiredPin } from './status-pin.js'
|
|
33
|
-
import { decidePinAction } from './status-pin.js'
|
|
32
|
+
import type { PinState, DesiredPin, PinRightsCache } from './status-pin.js'
|
|
33
|
+
import { decidePinAction, isPinRightsError } from './status-pin.js'
|
|
34
34
|
|
|
35
35
|
/** Minimal subset of grammy's `bot.api` the pin driver depends on.
|
|
36
36
|
* Lets tests swap in a fake without dragging in the full Bot type. */
|
|
@@ -55,6 +55,16 @@ export interface ReconcilePinArgs {
|
|
|
55
55
|
desired: DesiredPin
|
|
56
56
|
/** Optional API-failure observer. Default: silent. */
|
|
57
57
|
onError?: (phase: 'pin' | 'unpin', err: unknown) => void
|
|
58
|
+
/** Optional per-process rights-aware negative cache (issue #3024). When a
|
|
59
|
+
* pin attempt fails with the permanent "not enough rights" 400, the chat is
|
|
60
|
+
* recorded and all subsequent pin attempts in it are skipped (no API call,
|
|
61
|
+
* no log). Omit to disable the cache entirely (the pre-#3024 behaviour). */
|
|
62
|
+
rightsCache?: PinRightsCache
|
|
63
|
+
/** Called EXACTLY ONCE per chat, the first time that chat is recorded as
|
|
64
|
+
* pin-incapable. Lets the caller emit a single warn line instead of the
|
|
65
|
+
* per-attempt `status-pin pin failed` spam. Only fires when `rightsCache`
|
|
66
|
+
* is supplied. */
|
|
67
|
+
onPinRightsDisabled?: (chatId: string) => void
|
|
58
68
|
}
|
|
59
69
|
|
|
60
70
|
/**
|
|
@@ -77,10 +87,26 @@ export async function reconcilePin(
|
|
|
77
87
|
if (action.kind === 'noop') return args.prevState
|
|
78
88
|
|
|
79
89
|
if (action.kind === 'unpin') {
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
90
|
+
// Skip the unpin API call in a chat the bot can't manage pins in — the
|
|
91
|
+
// call would fail with the same rights 400 and spam the log. The claim is
|
|
92
|
+
// dropped either way (below), so skipping is safe.
|
|
93
|
+
if (!args.rightsCache?.isBlocked(args.chatId)) {
|
|
94
|
+
try {
|
|
95
|
+
await args.api.unpinChatMessage(args.chatId, action.messageId)
|
|
96
|
+
} catch (err) {
|
|
97
|
+
// Symmetric with the pin path below: a permanent rights 400 on UNPIN
|
|
98
|
+
// (rights revoked mid-session after we pinned) also enters the
|
|
99
|
+
// negative cache and logs once via onPinRightsDisabled — otherwise
|
|
100
|
+
// every later unpin attempt would burn an API call and spam
|
|
101
|
+
// `status-pin unpin failed` per attempt, the exact class this cache
|
|
102
|
+
// exists to kill (#3073 review finding). Claim is dropped regardless.
|
|
103
|
+
if (args.rightsCache && isPinRightsError(err)) {
|
|
104
|
+
const firstTime = args.rightsCache.block(args.chatId)
|
|
105
|
+
if (firstTime) args.onPinRightsDisabled?.(args.chatId)
|
|
106
|
+
} else {
|
|
107
|
+
args.onError?.('unpin', err)
|
|
108
|
+
}
|
|
109
|
+
}
|
|
84
110
|
}
|
|
85
111
|
// Drop the claim regardless of the unpin outcome. A stuck claim would
|
|
86
112
|
// leave a permanent pin on a crash / out-of-band unpin — the exact
|
|
@@ -89,14 +115,33 @@ export async function reconcilePin(
|
|
|
89
115
|
}
|
|
90
116
|
|
|
91
117
|
// action.kind === 'pin' — pin an EXISTING message, silently.
|
|
118
|
+
// Rights-aware negative cache (issue #3024): if a prior attempt in this chat
|
|
119
|
+
// already failed with the permanent "not enough rights" 400, skip silently —
|
|
120
|
+
// no API call, no log. Don't claim the message (the pin never happened), so a
|
|
121
|
+
// later reconcile after a restart (cache cleared) can retry.
|
|
122
|
+
if (args.rightsCache?.isBlocked(args.chatId)) {
|
|
123
|
+
return args.prevState
|
|
124
|
+
}
|
|
92
125
|
try {
|
|
93
126
|
await args.api.pinChatMessage(args.chatId, action.messageId, {
|
|
94
127
|
disable_notification: true,
|
|
95
128
|
})
|
|
96
129
|
} catch (err) {
|
|
97
|
-
|
|
130
|
+
// A permanent pin-rights failure enters the negative cache and logs ONCE
|
|
131
|
+
// (via onPinRightsDisabled) — every subsequent attempt in this chat is then
|
|
132
|
+
// skipped above. Transient failures (429 / 5xx / network) are NOT cached
|
|
133
|
+
// and route through onError as before, preserving retry behaviour.
|
|
134
|
+
if (args.rightsCache && isPinRightsError(err)) {
|
|
135
|
+
const firstTime = args.rightsCache.block(args.chatId)
|
|
136
|
+
if (firstTime) args.onPinRightsDisabled?.(args.chatId)
|
|
137
|
+
} else {
|
|
138
|
+
args.onError?.('pin', err)
|
|
139
|
+
}
|
|
98
140
|
// Don't claim a message we failed to pin — the next reconcile retries.
|
|
99
141
|
return args.prevState
|
|
100
142
|
}
|
|
143
|
+
// Pin succeeded — if this chat was previously cached as pin-incapable, rights
|
|
144
|
+
// were granted since; forget it so we resume normal behaviour immediately.
|
|
145
|
+
args.rightsCache?.clear(args.chatId)
|
|
101
146
|
return { messageId: action.messageId }
|
|
102
147
|
}
|
|
@@ -74,3 +74,84 @@ export function decidePinAction(
|
|
|
74
74
|
// re-pins the new one on the next reconcile.
|
|
75
75
|
return { kind: 'unpin', messageId: prev.messageId }
|
|
76
76
|
}
|
|
77
|
+
|
|
78
|
+
/** Extract a lowercased human description from any thrown value, preferring
|
|
79
|
+
* grammy's structured `.description` (the wire text Telegram returned) and
|
|
80
|
+
* falling back to `.message` / String(). Kept dependency-free — matches on
|
|
81
|
+
* the wire text, never on a grammy class import. */
|
|
82
|
+
function errorDescription(err: unknown): string {
|
|
83
|
+
if (err != null && typeof err === 'object') {
|
|
84
|
+
const o = err as { description?: unknown; message?: unknown }
|
|
85
|
+
if (typeof o.description === 'string' && o.description.length > 0) {
|
|
86
|
+
return o.description.toLowerCase()
|
|
87
|
+
}
|
|
88
|
+
if (typeof o.message === 'string' && o.message.length > 0) {
|
|
89
|
+
return o.message.toLowerCase()
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
return String(err).toLowerCase()
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* True when a pin/unpin failure is the PERMANENT "the bot lacks pin rights in
|
|
97
|
+
* this chat" class — Telegram returns this as a 400 (not a 403):
|
|
98
|
+
* "not enough rights to manage pinned messages in the chat". This is the one
|
|
99
|
+
* error class that will NOT self-heal on retry: the bot is simply not an admin
|
|
100
|
+
* (or lacks the "Pin messages" right) in that supergroup, and every subsequent
|
|
101
|
+
* pin attempt in the same chat fails identically until an admin grants the
|
|
102
|
+
* right (which only takes effect after a gateway restart re-reads chat perms).
|
|
103
|
+
*
|
|
104
|
+
* Transient classes (429 flood-wait, 5xx, network HttpError) are deliberately
|
|
105
|
+
* EXCLUDED — they must keep the existing retry behaviour, never enter the
|
|
106
|
+
* negative cache. Only this permanent-rights class is cacheable.
|
|
107
|
+
*
|
|
108
|
+
* The single source of truth for the pin-rights concept: the gateway's
|
|
109
|
+
* `unhandled-rejection-policy.ts` and the various edit-error classifiers each
|
|
110
|
+
* fold `not enough rights` into a broader boolean for their own purpose; this
|
|
111
|
+
* is the one helper dedicated to the pin negative-cache decision.
|
|
112
|
+
*/
|
|
113
|
+
export function isPinRightsError(err: unknown): boolean {
|
|
114
|
+
return errorDescription(err).includes('not enough rights')
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* Per-process, per-chat negative cache for chats the bot cannot pin in.
|
|
119
|
+
*
|
|
120
|
+
* When an auto status-pin attempt fails with the permanent pin-rights 400
|
|
121
|
+
* (`isPinRightsError`), the chat is recorded here as pin-incapable so the
|
|
122
|
+
* driver skips every subsequent auto-pin attempt in that chat — no wasted API
|
|
123
|
+
* call, no repeated `status-pin pin failed` log line (issue #3024: marko logged
|
|
124
|
+
* 41 identical pin-rights rejections in 48h, one per attempt).
|
|
125
|
+
*
|
|
126
|
+
* Deliberately IN-MEMORY / per-boot only: pin rights may be granted later, and
|
|
127
|
+
* Telegram surfaces the new permission to the bot only on a fresh chat-member
|
|
128
|
+
* fetch — a gateway restart. Clearing the cache on restart is therefore the
|
|
129
|
+
* correct re-enable trigger. An explicit `pin_message` tool success in a chat
|
|
130
|
+
* also clears its entry (rights were granted mid-session).
|
|
131
|
+
*
|
|
132
|
+
* Scope: the AUTO status-pin path only. The explicit `pin_message` MCP tool
|
|
133
|
+
* never consults this cache — it always attempts and surfaces the error to the
|
|
134
|
+
* agent as a normal tool error.
|
|
135
|
+
*/
|
|
136
|
+
export class PinRightsCache {
|
|
137
|
+
private readonly blocked = new Set<string>()
|
|
138
|
+
|
|
139
|
+
/** True when auto-pin should be skipped for this chat (already known bad). */
|
|
140
|
+
isBlocked(chatId: string): boolean {
|
|
141
|
+
return this.blocked.has(chatId)
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/** Record a chat as pin-incapable. Returns `true` only the FIRST time a chat
|
|
145
|
+
* is added, so the caller can log the warn line exactly once per chat. */
|
|
146
|
+
block(chatId: string): boolean {
|
|
147
|
+
if (this.blocked.has(chatId)) return false
|
|
148
|
+
this.blocked.add(chatId)
|
|
149
|
+
return true
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/** Forget a chat (rights re-granted, e.g. an explicit pin later succeeded).
|
|
153
|
+
* Returns `true` if an entry was actually removed. */
|
|
154
|
+
clear(chatId: string): boolean {
|
|
155
|
+
return this.blocked.delete(chatId)
|
|
156
|
+
}
|
|
157
|
+
}
|