switchroom 0.18.17 → 0.18.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-scheduler/index.js +13 -0
- package/dist/auth-broker/index.js +13 -0
- package/dist/cli/notion-write-pretool.mjs +13 -0
- package/dist/cli/switchroom.js +605 -479
- package/dist/host-control/main.js +17 -1
- package/dist/vault/approvals/kernel-server.js +13 -0
- package/dist/vault/broker/server.js +13 -0
- package/package.json +1 -1
- package/telegram-plugin/bridge/bridge.ts +7 -1
- package/telegram-plugin/dist/bridge/bridge.js +26 -1
- package/telegram-plugin/dist/gateway/gateway.js +1544 -619
- package/telegram-plugin/dist/server.js +32 -1
- package/telegram-plugin/fleet-fallback-resume.ts +26 -3
- package/telegram-plugin/format.ts +137 -213
- package/telegram-plugin/gateway/approval-hold.ts +49 -0
- package/telegram-plugin/gateway/bridge-dead-watchdog.ts +61 -18
- package/telegram-plugin/gateway/gateway.ts +399 -85
- package/telegram-plugin/gateway/linear-activity.ts +20 -4
- package/telegram-plugin/gateway/outbound-send-path.ts +9 -7
- package/telegram-plugin/gateway/premium-recovery-wiring.ts +122 -0
- package/telegram-plugin/gateway/session-model-file.ts +103 -0
- package/telegram-plugin/gateway/tier-downgrade-wiring.ts +121 -0
- package/telegram-plugin/gateway/unhandled-rejection-policy.ts +14 -1
- package/telegram-plugin/llm-error-present.ts +474 -0
- package/telegram-plugin/operator-events.ts +7 -1
- package/telegram-plugin/permission-title.ts +172 -10
- package/telegram-plugin/premium-recovery.ts +101 -0
- package/telegram-plugin/raw-error-scrub.ts +73 -0
- package/telegram-plugin/retry-api-call.ts +8 -2
- package/telegram-plugin/send-gate-degraded.test.ts +152 -1
- package/telegram-plugin/send-gate-observability.test.ts +140 -0
- package/telegram-plugin/send-gate-observability.ts +65 -20
- package/telegram-plugin/send-gate.test.ts +143 -1
- package/telegram-plugin/send-gate.ts +212 -19
- package/telegram-plugin/session-tail.ts +16 -0
- package/telegram-plugin/shared/local-time.ts +69 -0
- package/telegram-plugin/stream-reply-handler.ts +5 -14
- package/telegram-plugin/tests/approval-hold-harness.ts +6 -6
- package/telegram-plugin/tests/approval-hold-outcome.test.ts +10 -2
- package/telegram-plugin/tests/bridge-dead-watchdog.test.ts +61 -0
- package/telegram-plugin/tests/fleet-fallback-resume.test.ts +39 -0
- package/telegram-plugin/tests/flood-windows-persistence.test.ts +3 -2
- package/telegram-plugin/tests/format-consistency.test.ts +68 -53
- package/telegram-plugin/tests/formatting-parse-regression.test.ts +5 -6
- package/telegram-plugin/tests/formatting-torture-set.ts +1 -1
- package/telegram-plugin/tests/gateway-outbound-redact.test.ts +26 -0
- package/telegram-plugin/tests/linear-create-issue.test.ts +30 -2
- package/telegram-plugin/tests/llm-error-present.test.ts +481 -0
- package/telegram-plugin/tests/outbound-send-path.test.ts +4 -3
- package/telegram-plugin/tests/paragraph-normalizer.test.ts +42 -100
- package/telegram-plugin/tests/permission-title.test.ts +167 -4
- package/telegram-plugin/tests/premium-recovery-wiring.test.ts +150 -0
- package/telegram-plugin/tests/premium-recovery.test.ts +165 -0
- package/telegram-plugin/tests/reaction-gate-routing.test.ts +6 -1
- package/telegram-plugin/tests/retry-api-call.test.ts +21 -0
- package/telegram-plugin/tests/stream-reply-handler.test.ts +9 -12
- package/telegram-plugin/tests/telegram-format.test.ts +86 -31
- package/telegram-plugin/tests/tier-downgrade-wiring.test.ts +165 -0
- package/telegram-plugin/tests/tier-downgrade.test.ts +141 -0
- package/telegram-plugin/tests/turn-flush-safety.test.ts +17 -21
- package/telegram-plugin/tests/unhandled-rejection-policy.test.ts +27 -1
- package/telegram-plugin/tests/worker-activity-feed.test.ts +5 -2
- package/telegram-plugin/tests/worker-feed-coalesce.test.ts +492 -0
- package/telegram-plugin/tier-downgrade.ts +198 -0
- package/telegram-plugin/tool-activity-summary.ts +99 -0
- package/telegram-plugin/turn-flush-safety.ts +4 -3
- package/telegram-plugin/worker-activity-feed.ts +509 -409
|
@@ -184,6 +184,7 @@ import {
|
|
|
184
184
|
selectHeldForRedelivery,
|
|
185
185
|
holdReasonFor,
|
|
186
186
|
heldRetryBackoffMs,
|
|
187
|
+
applyDeliveredHoldReset,
|
|
187
188
|
type UndeliverableMark,
|
|
188
189
|
} from './approval-hold.js'
|
|
189
190
|
import { isTelegramReplyTool, isTelegramSurfaceTool } from '../tool-names.js'
|
|
@@ -207,7 +208,7 @@ import {
|
|
|
207
208
|
isPhotoDimensionRejectError,
|
|
208
209
|
isFloodWaitActiveError,
|
|
209
210
|
} from '../retry-api-call.js'
|
|
210
|
-
import { createSendGate,
|
|
211
|
+
import { createSendGate, sendGateConfigFromEnv } from '../send-gate.js'
|
|
211
212
|
import { createStatsLogger, createFloodWindowObserver } from '../send-gate-observability.js'
|
|
212
213
|
import { classifyPhotoFile, rerouteResultSuffix } from '../photo-precheck.js'
|
|
213
214
|
import { installTgPostLogger, withTgPostTags } from '../shared/bot-runtime.js'
|
|
@@ -314,6 +315,11 @@ import {
|
|
|
314
315
|
type OperatorEventKind,
|
|
315
316
|
} from '../operator-events.js'
|
|
316
317
|
import { recordOperatorEvent } from '../operator-events-history.js'
|
|
318
|
+
import {
|
|
319
|
+
parseLlmError,
|
|
320
|
+
renderLlmErrorSafe,
|
|
321
|
+
decideErrorSurface,
|
|
322
|
+
} from '../llm-error-present.js'
|
|
317
323
|
import {
|
|
318
324
|
formatModelUnavailableCard,
|
|
319
325
|
resolveModelUnavailableFromOperatorEvent,
|
|
@@ -342,7 +348,7 @@ const REPLY_TO_TEXT_MAX = 200
|
|
|
342
348
|
// #1161 silent-end fallback text now lives in ../silent-end.ts
|
|
343
349
|
// (`silentEndFallbackText`, imported above) so the transport-boundary
|
|
344
350
|
// tests exercise the real string — see PR #2892.
|
|
345
|
-
import { splitMarkdownChunks, repairEscapedWhitespace, normalizeParagraphBreaks,
|
|
351
|
+
import { splitMarkdownChunks, repairEscapedWhitespace, normalizeParagraphBreaks, normalizePunctuation, stripExcessBold, escapeMarkdown, hardenCardBreaks, RICH_MESSAGE_MAX_CHARS } from '../format.js'
|
|
346
352
|
import { richMessage } from '../rich-send.js'
|
|
347
353
|
import { scrubVoice } from '../text-voice-scrub.js'
|
|
348
354
|
import {
|
|
@@ -468,7 +474,13 @@ import {
|
|
|
468
474
|
readConfiguredDefaultModel,
|
|
469
475
|
writeSessionEffortFile,
|
|
470
476
|
clearSessionEffortFile,
|
|
477
|
+
writePremiumRecoveryFile,
|
|
478
|
+
readPremiumRecoveryFile,
|
|
479
|
+
clearPremiumRecoveryFile,
|
|
471
480
|
} from './session-model-file.js'
|
|
481
|
+
import { runTierDowngrade } from './tier-downgrade-wiring.js'
|
|
482
|
+
import { runPremiumRecoveryPing } from './premium-recovery-wiring.js'
|
|
483
|
+
import { decidePremiumRecovery } from '../premium-recovery.js'
|
|
472
484
|
import { discoverModels, selectModel } from '../../src/agents/model-picker.js'
|
|
473
485
|
import { resolveMainModel, SWITCHROOM_DEFAULT_THINKING_EFFORT } from '../../src/agents/scaffold.js'
|
|
474
486
|
import {
|
|
@@ -5468,8 +5480,14 @@ const recordFloodWindow = makeFloodWindowRecorder(FLOOD_WINDOWS_PATH)
|
|
|
5468
5480
|
// ban never resends into an open window. onWindowOpen write-throughs every
|
|
5469
5481
|
// runtime-opened window to FLOOD_WINDOWS_PATH; bootRamp starts the global
|
|
5470
5482
|
// bucket at half capacity for 10s to absorb the boot-card burst.
|
|
5483
|
+
// Resolve enabled + the tunable rate limits (channels.telegram.send_gate.* →
|
|
5484
|
+
// SWITCHROOM_TG_SEND_GATE_* env) ONCE at boot. Unset knobs are absent, so
|
|
5485
|
+
// createSendGate applies SEND_GATE_DEFAULTS — omitting config = today's exact
|
|
5486
|
+
// behaviour. The SWITCHROOM_TELEGRAM_SEND_GATE break-glass valve still wins on
|
|
5487
|
+
// `enabled` when explicitly set (see sendGateConfigFromEnv precedence).
|
|
5488
|
+
const sendGateConfig = sendGateConfigFromEnv()
|
|
5471
5489
|
const sendGate = createSendGate({
|
|
5472
|
-
|
|
5490
|
+
...sendGateConfig,
|
|
5473
5491
|
initialWindows: loadInitialFloodWindows(FLOOD_STATE_PATH, FLOOD_WINDOWS_PATH, Date.now()),
|
|
5474
5492
|
bootRamp: {},
|
|
5475
5493
|
onWindowOpen: (scopeKey, untilTs) => recordFloodWindow(scopeKey, untilTs),
|
|
@@ -5482,15 +5500,20 @@ const sendGate = createSendGate({
|
|
|
5482
5500
|
const probeFloodWaitRemainingMs = makeFloodWaitProbe(FLOOD_STATE_PATH)
|
|
5483
5501
|
const rawRobustApiCall = createRetryApiCall({
|
|
5484
5502
|
log: (line) => process.stderr.write(line),
|
|
5485
|
-
onFloodWait: (retryAfterSec) => {
|
|
5503
|
+
onFloodWait: (retryAfterSec, opts) => {
|
|
5486
5504
|
// #2923/#3094 — persist the single-object global window (probe reads this).
|
|
5487
5505
|
makeFloodWaitRecorder(FLOOD_STATE_PATH)(retryAfterSec)
|
|
5488
|
-
// #3084 PR 2 — also open
|
|
5489
|
-
// for the ban's duration even on a SHORT (slept-and-retried)
|
|
5490
|
-
// throws FLOOD_WAIT_ACTIVE.
|
|
5491
|
-
//
|
|
5506
|
+
// #3084 PR 2 / #3111 — also open SCOPE-PRECISE send-gate window(s) so cosmetic
|
|
5507
|
+
// traffic sheds for the ban's duration even on a SHORT (slept-and-retried)
|
|
5508
|
+
// 429 that never throws FLOOD_WAIT_ACTIVE. The retry policy now passes the
|
|
5509
|
+
// call's `opts` (#3111) so this opens the FINEST scope the 429 implies —
|
|
5510
|
+
// `chat:`/`group:`/`msg-edit:` for a chat-bound call, `global` only when the
|
|
5511
|
+
// call carries no chat scope (genuinely global) — instead of a blanket
|
|
5512
|
+
// `global` window that would suppress unrelated chats. This mirrors the
|
|
5513
|
+
// gate's own FLOOD_WAIT_ACTIVE catch, which uses the same scope-precise
|
|
5514
|
+
// opener for LONG bans.
|
|
5492
5515
|
try {
|
|
5493
|
-
sendGate.
|
|
5516
|
+
sendGate.openScopedFloodWindows(opts, Date.now() + Math.max(0, retryAfterSec) * 1000)
|
|
5494
5517
|
} catch {
|
|
5495
5518
|
/* best-effort — never let the window hook break the retry path */
|
|
5496
5519
|
}
|
|
@@ -5634,6 +5657,9 @@ const sendGateStatsLogger = createStatsLogger({
|
|
|
5634
5657
|
const floodWindowObserver = createFloodWindowObserver({
|
|
5635
5658
|
clock: { now: () => Date.now(), sleep: (ms) => new Promise((r) => setTimeout(r, ms)) },
|
|
5636
5659
|
log: (line) => process.stderr.write(line),
|
|
5660
|
+
// Render the operator-facing flood alerts in the configured local timezone
|
|
5661
|
+
// (same env the config cascade bakes into every agent — see timezone.ts).
|
|
5662
|
+
tz: process.env.SWITCHROOM_TIMEZONE ?? process.env.TZ ?? 'UTC',
|
|
5637
5663
|
stats: () => sendGate.stats(),
|
|
5638
5664
|
readWindows: (now) => readFloodWindows(FLOOD_WINDOWS_PATH, now),
|
|
5639
5665
|
markAlerted: (scopeKey, alertedAt) =>
|
|
@@ -5654,7 +5680,7 @@ const floodWindowObserver = createFloodWindowObserver({
|
|
|
5654
5680
|
)
|
|
5655
5681
|
},
|
|
5656
5682
|
})
|
|
5657
|
-
if (
|
|
5683
|
+
if (sendGateConfig.enabled) {
|
|
5658
5684
|
const observeTimer = setInterval(() => {
|
|
5659
5685
|
try {
|
|
5660
5686
|
sendGateStatsLogger.tick()
|
|
@@ -6726,10 +6752,19 @@ function isAutoFallbackCooldownActive(_agentName: string, now: number): boolean
|
|
|
6726
6752
|
|
|
6727
6753
|
async function editCardExpired(chatId: string, messageId: number | undefined, body: string): Promise<void> {
|
|
6728
6754
|
if (messageId == null) return
|
|
6729
|
-
|
|
6730
|
-
|
|
6731
|
-
|
|
6732
|
-
|
|
6755
|
+
// #3084 bypass audit: this best-effort card-expiry strip (reaper + lazy
|
|
6756
|
+
// sweeps, no grammy ctx) previously fired a RAW `lockedBot.api.editMessageText`
|
|
6757
|
+
// OUTSIDE the send gate — a residual flood vector (a 429 here never reached
|
|
6758
|
+
// `onFloodWait`, so the breaker stayed blind). Routed through `robustApiCall`
|
|
6759
|
+
// (chat-lock → send-gate → retry/breaker) as a `cosmetic` edit carrying
|
|
6760
|
+
// messageId+editPayload so it paces/sheds under pressure and its 429 records
|
|
6761
|
+
// the flood window. Dropping reply_markup strips the stale keyboard atomically
|
|
6762
|
+
// with the text edit; message-id-targeted so no thread to lose.
|
|
6763
|
+
const text = richMessage(body)
|
|
6764
|
+
await robustApiCall(
|
|
6765
|
+
() => lockedBot.api.editMessageText(chatId, messageId, text, { reply_markup: { inline_keyboard: [] } }),
|
|
6766
|
+
{ chat_id: chatId, verb: 'card-expired.strip', priorityClass: 'cosmetic', messageId, editPayload: body },
|
|
6767
|
+
).catch(() => {})
|
|
6733
6768
|
}
|
|
6734
6769
|
|
|
6735
6770
|
function recordMissedApproval(opts: {
|
|
@@ -7073,20 +7108,13 @@ function postPermissionCard(
|
|
|
7073
7108
|
const landedThreadId = sent.message_thread_id ?? undefined
|
|
7074
7109
|
live.cards.push({ chatId, messageId: sent.message_id, threadId: landedThreadId })
|
|
7075
7110
|
// The card LANDED — the operator can see and tap it, so the block is over.
|
|
7076
|
-
// Drop the hold mark and reconcile the off-Telegram
|
|
7077
|
-
//
|
|
7078
|
-
//
|
|
7079
|
-
|
|
7080
|
-
|
|
7081
|
-
|
|
7082
|
-
|
|
7083
|
-
// the agent asked; the TTL measures how long the operator had to
|
|
7084
|
-
// answer. Until this instant they had NOTHING to answer — the card did
|
|
7085
|
-
// not exist in any chat. Without the reset, a card held through a 4.6h
|
|
7086
|
-
// ban lands already-expired against a 60-min TTL and the very next
|
|
7087
|
-
// reaper tick auto-denies it: we would have MOVED the silent denial,
|
|
7088
|
-
// not removed it.
|
|
7089
|
-
live.startedAt = Date.now()
|
|
7111
|
+
// Drop the hold mark, reset the TTL clock, and reconcile the off-Telegram
|
|
7112
|
+
// surface — as ONE shared decision (`applyDeliveredHoldReset`, #3128) that
|
|
7113
|
+
// the outcome test's harness ALSO drives, so deleting the `startedAt` reset
|
|
7114
|
+
// turns the behavioural `(d2)` test red instead of only a fragile grep.
|
|
7115
|
+
// PR 3: the reset is load-bearing — the TTL measures how long the operator
|
|
7116
|
+
// had to answer, and until this moment they had nothing to answer.
|
|
7117
|
+
if (applyDeliveredHoldReset(live, Date.now())) {
|
|
7090
7118
|
reconcileBlockedApprovals()
|
|
7091
7119
|
process.stderr.write(
|
|
7092
7120
|
`telegram gateway: permission-card RE-DELIVERED request=${requestId} ` +
|
|
@@ -7662,6 +7690,21 @@ const inboundCoalescer = createInboundCoalescer<CoalescePayload>({
|
|
|
7662
7690
|
function emitGatewayOperatorEvent(event: OperatorEvent): void {
|
|
7663
7691
|
const { agent, kind } = event
|
|
7664
7692
|
|
|
7693
|
+
// #llm-error-surfacing FIX 2 (secret leak): the operator-event cards are sent
|
|
7694
|
+
// via a raw bot.api.sendRichMessage that BYPASSES the normal outbound redact
|
|
7695
|
+
// chokepoint (normalizeOutboundBody → redact, outbound-send-path.ts). The only
|
|
7696
|
+
// scrub the renderers apply is stripRawErrorBytes — a JSON-SHAPE scrub, NOT a
|
|
7697
|
+
// secret scrubber — so a bearer token / `sk-…` key / url-embedded credential
|
|
7698
|
+
// smuggled in an error `detail` would reach the operator card verbatim on the
|
|
7699
|
+
// credentials-expired / credit-exhausted / unknown-4xx paths. Redact the detail
|
|
7700
|
+
// ONCE here, up front, through the SAME redact() the reply path uses — and
|
|
7701
|
+
// crucially BEFORE renderOperatorEvent runs escapeMarkdown on it (redacting the
|
|
7702
|
+
// already-escaped text would let url-query-param secrets slip past url-redact,
|
|
7703
|
+
// exactly the order the outbound pipeline documents: redact before markdown).
|
|
7704
|
+
// Doing it at the top also scrubs the recorded operator-event history and the
|
|
7705
|
+
// 429 metrics — defense in depth, no secret survives in ANY downstream sink.
|
|
7706
|
+
event = { ...event, detail: redactOutboundText(event.detail, 'operator_event') }
|
|
7707
|
+
|
|
7665
7708
|
// ── 429 throttle tier (operator spec: "retry in place under 5 min, else
|
|
7666
7709
|
// mark + failover, honest reset messaging") ────────────────────────────
|
|
7667
7710
|
// A terminal TRANSIENT ACCOUNT-scoped 429 — kind `rate-limited` carrying
|
|
@@ -7910,6 +7953,35 @@ function emitGatewayOperatorEvent(event: OperatorEvent): void {
|
|
|
7910
7953
|
// the old account fails — honest reset messaging on the enriched card.
|
|
7911
7954
|
void fireFleetAutoFallback(agent, untilMs, modelUnavailable.resetAt)
|
|
7912
7955
|
}
|
|
7956
|
+
} else if (kind === 'rate-limited' || kind === 'unknown-5xx') {
|
|
7957
|
+
// #llm-error-surfacing — surface #2 (the "🚦 Rate limited" operator card).
|
|
7958
|
+
// The transient rate-limit / overload family used to render the raw
|
|
7959
|
+
// synthetic-error `detail` (bytes and all) via renderOperatorEvent. Route
|
|
7960
|
+
// it through the humanized card instead: JSON-stripped coreText + reset in
|
|
7961
|
+
// LOCAL time. The ErrorPresenceGate is the cross-surface dedup authority —
|
|
7962
|
+
// the reply/done-card surfaces are already suppressed at the transcript
|
|
7963
|
+
// source (session-tail), so the operator card is the sole renderer and
|
|
7964
|
+
// wins the claim; a redundant burst within the collapse window is
|
|
7965
|
+
// suppressed here in addition to the existing 5-min per-kind cooldown.
|
|
7966
|
+
const parsed = parseLlmError(event.detail)
|
|
7967
|
+
const now = Date.now()
|
|
7968
|
+
if (decideErrorSurface(parsed, agent, { claim: true, now }) === 'suppress') {
|
|
7969
|
+
process.stderr.write(
|
|
7970
|
+
`telegram gateway: operator-event collapsed (error-presence-gate) agent=${agent} kind=${kind}\n`,
|
|
7971
|
+
)
|
|
7972
|
+
return
|
|
7973
|
+
}
|
|
7974
|
+
const tz = process.env.SWITCHROOM_TIMEZONE ?? process.env.TZ ?? 'UTC'
|
|
7975
|
+
// #llm-error-surfacing FIX 3 (crash guard): renderLlmErrorSafe wraps the
|
|
7976
|
+
// tz-formatting render — an invalid IANA `SWITCHROOM_TIMEZONE`/`TZ` throws a
|
|
7977
|
+
// RangeError out of Intl.DateTimeFormat (local-time.ts's "never throws" claim
|
|
7978
|
+
// does NOT hold for construction-time zone validation). Pre-fix this branch
|
|
7979
|
+
// had no guard, so a bad tz crashed the whole operator-event turn; now it
|
|
7980
|
+
// degrades to a minimal tz-free line. There are no action buttons on this
|
|
7981
|
+
// card (FIX 1) — the humanized text carries any recommendation inline.
|
|
7982
|
+
const r = renderLlmErrorSafe(parsed, agent, tz, new Date(now))
|
|
7983
|
+
renderedText = r.text
|
|
7984
|
+
renderedKeyboard = undefined
|
|
7913
7985
|
} else {
|
|
7914
7986
|
try {
|
|
7915
7987
|
const r = renderOperatorEvent(event)
|
|
@@ -8578,6 +8650,17 @@ async function runMidSessionCardReaper(): Promise<void> {
|
|
|
8578
8650
|
const reaps = decideWorkerPinReaps({
|
|
8579
8651
|
pins: candidates,
|
|
8580
8652
|
statusOf: (agentId) => {
|
|
8653
|
+
// #3207: coalesced feed pins are GROUP-level (`wk:group:<feedKey>`),
|
|
8654
|
+
// so `workerAgentIdOfPinKey` yields `group:<feedKey>`, not a real
|
|
8655
|
+
// jsonl agent id. Vouch for these off the live feed instead of the
|
|
8656
|
+
// registry: a group the feed still tracks is 'running' (exempt from
|
|
8657
|
+
// the TTL); a lingering group pin the feed no longer knows about is a
|
|
8658
|
+
// missed unpin → 'terminal' (reap now). The feed's own group-empty
|
|
8659
|
+
// unpin is the primary path; this is the missed-unpin backstop.
|
|
8660
|
+
if (agentId.startsWith('group:')) {
|
|
8661
|
+
const feedKey = agentId.slice('group:'.length)
|
|
8662
|
+
return workerActivityFeed?.hasRunningInFeed(feedKey) ? 'running' : 'terminal'
|
|
8663
|
+
}
|
|
8581
8664
|
if (turnsDb == null) return 'unknown'
|
|
8582
8665
|
try {
|
|
8583
8666
|
const row = getSubagentByJsonlId(turnsDb, agentId)
|
|
@@ -8736,34 +8819,14 @@ async function reconcileStatusPinInner(
|
|
|
8736
8819
|
}
|
|
8737
8820
|
}
|
|
8738
8821
|
|
|
8739
|
-
|
|
8740
|
-
|
|
8741
|
-
|
|
8742
|
-
|
|
8743
|
-
|
|
8744
|
-
|
|
8745
|
-
|
|
8746
|
-
|
|
8747
|
-
function reconcileWorkerPin(
|
|
8748
|
-
agentId: string,
|
|
8749
|
-
chatId: string | null,
|
|
8750
|
-
running: boolean,
|
|
8751
|
-
): void {
|
|
8752
|
-
if (!PIN_STATUS_WHILE_WORKING) return
|
|
8753
|
-
const key = `wk:${agentId}`
|
|
8754
|
-
if (!running) {
|
|
8755
|
-
// Unpin: recover the chat we pinned in (caller may not have it at
|
|
8756
|
-
// completion). No-op when nothing was pinned for this worker.
|
|
8757
|
-
const unpinChat = chatId ?? statusPinChatIds.get(key)
|
|
8758
|
-
if (unpinChat == null) return
|
|
8759
|
-
void reconcileStatusPin(key, unpinChat, { pinned: false })
|
|
8760
|
-
return
|
|
8761
|
-
}
|
|
8762
|
-
if (chatId == null) return
|
|
8763
|
-
const messageId = workerActivityFeed?.messageIdOf(agentId) ?? null
|
|
8764
|
-
if (messageId == null) return // no message painted yet — nothing to pin
|
|
8765
|
-
void reconcileStatusPin(key, chatId, { pinned: true, messageId })
|
|
8766
|
-
}
|
|
8822
|
+
// #3207: the per-worker `reconcileWorkerPin(agentId, …)` (keyed `wk:<agentId>`)
|
|
8823
|
+
// was removed. Now that background workers COALESCE into one shared message per
|
|
8824
|
+
// chat/thread, the pin is driven at the GROUP level by the feed itself
|
|
8825
|
+
// (`reconcilePin` → `wk:group:<feedKey>`, wired at createWorkerActivityFeed):
|
|
8826
|
+
// it pins when the group's first worker paints and unpins only when the group
|
|
8827
|
+
// empties. A per-worker unpin used to physically unpin a message a sibling
|
|
8828
|
+
// still needed, after which the survivor's re-pin NO-OP'd (its claim still
|
|
8829
|
+
// named that id) — leaving live work unpinned (review blocker).
|
|
8767
8830
|
|
|
8768
8831
|
/** Unpin every owned status pin — used by the pre-restart sweep so a
|
|
8769
8832
|
* crash / interrupt never leaves a permanent pin behind. Best-effort;
|
|
@@ -10347,6 +10410,13 @@ const bridgeDeadWatchdog = createBridgeDeadWatchdog({
|
|
|
10347
10410
|
// consecutive-escalation count. At the cap, arm() stands down loudly
|
|
10348
10411
|
// instead of restart-looping a deterministically-failing bridge.
|
|
10349
10412
|
priorStreak: bridgeDeadPriorStreak,
|
|
10413
|
+
// #3086 — only THIS gateway's own primary bridge (registering under
|
|
10414
|
+
// $SWITCHROOM_AGENT_NAME) drives the watchdog. A secondary/relay client
|
|
10415
|
+
// (e.g. `overlord-relay`) registering a different name into this socket
|
|
10416
|
+
// is named + non-cron but must NOT mark the (still-alive) primary bridge
|
|
10417
|
+
// dead when it disconnects. Empty string ⟹ fall back to the pre-#3086
|
|
10418
|
+
// "any named non-cron client" test inside the watchdog.
|
|
10419
|
+
selfAgentName: process.env.SWITCHROOM_AGENT_NAME ?? '',
|
|
10350
10420
|
})
|
|
10351
10421
|
if (BRIDGE_DEAD_ESCALATION_ENABLED) {
|
|
10352
10422
|
bridgeDeadWatchdog.arm()
|
|
@@ -10386,10 +10456,11 @@ const ipcServer: IpcServer = createIpcServer({
|
|
|
10386
10456
|
: []
|
|
10387
10457
|
// #3038 — a REAL (named, non-cron) bridge registered: stand the
|
|
10388
10458
|
// bridge-dead watchdog down. Anonymous clients (recall.py, mcp
|
|
10389
|
-
// handshakes)
|
|
10390
|
-
//
|
|
10391
|
-
//
|
|
10392
|
-
//
|
|
10459
|
+
// handshakes), cron-session bridges, and secondary/relay clients
|
|
10460
|
+
// registering a name other than this gateway's own agent (#3086) must
|
|
10461
|
+
// NOT satisfy it — the watchdog gates on the identity INTERNALLY
|
|
10462
|
+
// (isRealBridgeIdentity vs selfAgentName), so this call is safe wherever
|
|
10463
|
+
// it sits relative to the cron early-return above (#3038 review finding 5).
|
|
10393
10464
|
bridgeDeadWatchdog.noteBridgeRegistered(client.agentName)
|
|
10394
10465
|
client.send({ type: 'status', status: 'agent_connected' })
|
|
10395
10466
|
|
|
@@ -10583,8 +10654,10 @@ const ipcServer: IpcServer = createIpcServer({
|
|
|
10583
10654
|
// #3038 — the real bridge went away mid-life. Re-arm the grace
|
|
10584
10655
|
// window: a normal claude restart re-registers within seconds and
|
|
10585
10656
|
// stands it down; a bridge that died for good escalates once (the
|
|
10586
|
-
// once-per-boot fuse inside the watchdog caps it). Cron/anonymous
|
|
10587
|
-
// identities
|
|
10657
|
+
// once-per-boot fuse inside the watchdog caps it). Cron/anonymous and
|
|
10658
|
+
// secondary/relay identities (a name other than this gateway's own
|
|
10659
|
+
// agent, #3086) are ignored inside the watchdog itself (finding 5) —
|
|
10660
|
+
// so a transient relay disconnect never bounces a healthy container.
|
|
10588
10661
|
if (BRIDGE_DEAD_ESCALATION_ENABLED) bridgeDeadWatchdog.noteBridgeDisconnected(client.agentName)
|
|
10589
10662
|
}
|
|
10590
10663
|
|
|
@@ -14948,9 +15021,11 @@ async function executeEditMessage(args: Record<string, unknown>): Promise<unknow
|
|
|
14948
15021
|
// secret into a live bubble or the history row. Mask before scrub/send.
|
|
14949
15022
|
editRawText = redactOutboundText(editRawText, 'edit_message')
|
|
14950
15023
|
// Fleet-wide consistent formatting (same order as the reply path: redact
|
|
14951
|
-
// first so secrets are matched literally, then normalize
|
|
14952
|
-
// the
|
|
14953
|
-
|
|
15024
|
+
// first so secrets are matched literally, then normalize). No paragraph
|
|
15025
|
+
// spacer pass — the NBSP spacer was removed in the #2669 follow-up because it
|
|
15026
|
+
// double-gapped every paragraph; the rich renderer already shows `\n\n` as
|
|
15027
|
+
// one blank line.
|
|
15028
|
+
if (!editLiteralText) editRawText = stripExcessBold(normalizePunctuation(editRawText))
|
|
14954
15029
|
// Voice scrub (#1683): same em-dash scrub as the reply path. Edits
|
|
14955
15030
|
// are how silent-anchor and progress-update mutate already-sent
|
|
14956
15031
|
// bubbles, so without this an edit can re-introduce dashes the
|
|
@@ -17386,9 +17461,9 @@ function handleSessionEvent(ev: SessionEvent): void {
|
|
|
17386
17461
|
// breaks into GFM hard breaks so the Bot API 10.1 rich path doesn't
|
|
17387
17462
|
// collapse them (lists/tables/code left untouched). Runs BEFORE the
|
|
17388
17463
|
// redact/scrub below, exactly as reply orders it (repair → normalize →
|
|
17389
|
-
// redact → scrub), so masking sees the repaired text.
|
|
17390
|
-
//
|
|
17391
|
-
//
|
|
17464
|
+
// redact → scrub), so masking sees the repaired text. Paragraph gaps
|
|
17465
|
+
// are the plain `\n\n` normalizeParagraphBreaks guarantees — no spacer
|
|
17466
|
+
// pass runs on the send side any more (removed in the #2669 follow-up).
|
|
17392
17467
|
capturedText = normalizeParagraphBreaks(repairEscapedWhitespace(capturedText))
|
|
17393
17468
|
// Component 3 — origin-thread backstop. `chatId`/`threadId` are
|
|
17394
17469
|
// captured from the turn atom (turn.sessionChatId/sessionThreadId)
|
|
@@ -17532,13 +17607,12 @@ function handleSessionEvent(ev: SessionEvent): void {
|
|
|
17532
17607
|
link_preview_options: { is_disabled: true },
|
|
17533
17608
|
}
|
|
17534
17609
|
const limit = RICH_MESSAGE_MAX_CHARS
|
|
17535
|
-
//
|
|
17536
|
-
//
|
|
17537
|
-
//
|
|
17538
|
-
//
|
|
17539
|
-
//
|
|
17540
|
-
|
|
17541
|
-
const renderedText = addParagraphSpacers(capturedText)
|
|
17610
|
+
// The `\n\n` block joins from turn-flush-safety.ts render as normal
|
|
17611
|
+
// single blank lines under the Bot API 10.1 rich GFM path, so no
|
|
17612
|
+
// spacer pass runs before splitting (the NBSP spacer was removed in
|
|
17613
|
+
// the #2669 follow-up — it double-gapped every paragraph). Mirrors
|
|
17614
|
+
// executeReply, which now also sends the normalized text as-is.
|
|
17615
|
+
const renderedText = capturedText
|
|
17542
17616
|
const htmlChunks = splitMarkdownChunks(renderedText, limit)
|
|
17543
17617
|
const sentIds: number[] = []
|
|
17544
17618
|
try {
|
|
@@ -21939,6 +22013,10 @@ function recordTypedModelSwitch(
|
|
|
21939
22013
|
}
|
|
21940
22014
|
if (!reply.selectedModel) return ''
|
|
21941
22015
|
sessionModelSource.setOverride(reply.selectedModel)
|
|
22016
|
+
// A manual /model apply to the dropped premium clears any pending recovery
|
|
22017
|
+
// marker so the "available again" ping can't still fire after the user has
|
|
22018
|
+
// already switched back themselves.
|
|
22019
|
+
clearPremiumRecoveryOnManualSwitch(reply.selectedModel)
|
|
21942
22020
|
return ''
|
|
21943
22021
|
}
|
|
21944
22022
|
|
|
@@ -21967,6 +22045,11 @@ function recordModelMenuSideEffects(
|
|
|
21967
22045
|
// (recommended)" selection clears any leftover carrier.
|
|
21968
22046
|
if (outcome.selectedModel) {
|
|
21969
22047
|
sessionModelSource.setOverride(outcome.selectedModel)
|
|
22048
|
+
// Clear a pending premium-recovery marker when the tap re-selects the
|
|
22049
|
+
// dropped premium (menu OR the recovery ping's own switch-back button):
|
|
22050
|
+
// no stale "available again" ping once we're back on it. Idempotent — the
|
|
22051
|
+
// ping-send path already consumed the marker, so this is a no-op there.
|
|
22052
|
+
clearPremiumRecoveryOnManualSwitch(outcome.selectedModel)
|
|
21970
22053
|
}
|
|
21971
22054
|
if (outcome.clearedDefault) {
|
|
21972
22055
|
const smDir = resolveAgentDirFromEnv()
|
|
@@ -23546,6 +23629,173 @@ function broadcastFleetFallbackFailure(triggerAgent: string, reason: string): vo
|
|
|
23546
23629
|
}
|
|
23547
23630
|
}
|
|
23548
23631
|
|
|
23632
|
+
/**
|
|
23633
|
+
* Broadcast a status notice to every authorized chat (system notice posture:
|
|
23634
|
+
* silenced ping, wrapped send). Shared by the tier-downgrade notices below.
|
|
23635
|
+
*/
|
|
23636
|
+
function broadcastTierNotice(markdown: string): void {
|
|
23637
|
+
const access = loadAccess()
|
|
23638
|
+
if (access.allowFrom.length === 0) return
|
|
23639
|
+
for (const chat_id of access.allowFrom) {
|
|
23640
|
+
void swallowingApiCall(
|
|
23641
|
+
// allow-raw-bot-api: wrapped in swallowingApiCall (retry policy)
|
|
23642
|
+
() => bot.api.sendRichMessage(chat_id, richMessage(markdown), { disable_notification: true }),
|
|
23643
|
+
{ chat_id: String(chat_id), verb: 'tier-downgrade:notify' },
|
|
23644
|
+
)
|
|
23645
|
+
}
|
|
23646
|
+
}
|
|
23647
|
+
|
|
23648
|
+
/**
|
|
23649
|
+
* MODEL-TIER downgrade failover (second recovery tier). Consulted ONLY from the
|
|
23650
|
+
* `all-blocked` branch of doFireFleetAutoFallback — i.e. AFTER account-swap has
|
|
23651
|
+
* been tried and found no account still serving the walled premium model
|
|
23652
|
+
* (precedence A: account-swap first). When a PREMIUM /model override is active
|
|
23653
|
+
* (distinct from the configured default), downgrade to the configured default
|
|
23654
|
+
* and resume the dead turn via a self-restart, rather than let the turn stall.
|
|
23655
|
+
*
|
|
23656
|
+
* - NO automatic return to the premium model. The /model override is
|
|
23657
|
+
* session-scoped and in-memory only (recordTypedModelSwitch writes no
|
|
23658
|
+
* carrier), so it dies on the downgrade SIGTERM. The consume-once
|
|
23659
|
+
* `.session-model` carrier written here targets the CONFIGURED DEFAULT:
|
|
23660
|
+
* start.sh applies+deletes it on the resume boot, and every later restart
|
|
23661
|
+
* also boots the default. The premium tier is never restored on its own —
|
|
23662
|
+
* the user must re-issue `/model <premium>`, which the notice says plainly.
|
|
23663
|
+
* - Effort is NATIVE: no `.session-effort` carrier is written, so the restart
|
|
23664
|
+
* sheds any live /effort override and the downgraded default boots at the
|
|
23665
|
+
* configured `thinking_effort` (the fleet `low` pin, #1978).
|
|
23666
|
+
* - Loop-bounded by the NATURAL on-default guard: after the downgrade boot the
|
|
23667
|
+
* session runs the configured default (override gone), so a re-entry returns
|
|
23668
|
+
* `skip` ('on-default') and never re-downgrades — even if the default is
|
|
23669
|
+
* itself walled (then the normal all-blocked card fires, no loop). Pacing
|
|
23670
|
+
* reuses the fleetFallbackResumeGate single-flight + 3h staleness.
|
|
23671
|
+
*
|
|
23672
|
+
* Returns:
|
|
23673
|
+
* 'downgraded' — carrier written, latch armed, restart fired; caller must
|
|
23674
|
+
* NOT also emit the all-blocked give-up card (we are
|
|
23675
|
+
* recovering, not giving up).
|
|
23676
|
+
* 'restart-pending' — a resume restart was ALREADY armed in this process (a
|
|
23677
|
+
* concurrent turn's downgrade or account-swap); that
|
|
23678
|
+
* restart replays the interrupted turn, so caller must NOT
|
|
23679
|
+
* emit a give-up card (avoids the contradictory "could not
|
|
23680
|
+
* be recovered" race).
|
|
23681
|
+
* 'skip' — not applicable (on the configured default, unresolved
|
|
23682
|
+
* default, or the turn is too stale to resume); caller
|
|
23683
|
+
* falls through to the all-blocked card.
|
|
23684
|
+
*/
|
|
23685
|
+
function maybeTierDowngrade(triggerAgent: string): 'downgraded' | 'restart-pending' | 'skip' {
|
|
23686
|
+
// Thin adapter: the order-sensitive glue lives in runTierDowngrade
|
|
23687
|
+
// (tier-downgrade-wiring.ts) behind injected deps so it is unit-testable
|
|
23688
|
+
// without importing the gateway. This wires the real seams.
|
|
23689
|
+
return runTierDowngrade(triggerAgent, {
|
|
23690
|
+
getAgentDir: () => resolveAgentDirFromEnv() ?? null,
|
|
23691
|
+
getConfiguredDefault: () => {
|
|
23692
|
+
const dir = resolveAgentDirFromEnv()
|
|
23693
|
+
if (!dir) return null
|
|
23694
|
+
return resolveMainModel(readConfiguredDefaultModel(dir) ?? undefined)
|
|
23695
|
+
},
|
|
23696
|
+
getSessionOverride: () => sessionModelSource.getOverride(),
|
|
23697
|
+
resolve: (t) => resolveMainModel(t),
|
|
23698
|
+
// PEEK the resume gate WITHOUT arming (single-flight within this process +
|
|
23699
|
+
// 3h staleness), the same gate the account-swap resume path uses.
|
|
23700
|
+
peekResumeGate: () => fleetFallbackResumeGate.peek(newestActiveTurnStartedAtMs()),
|
|
23701
|
+
writeCarrier: (dir, toModel, cfg) => writeSessionModelFile(dir, toModel, cfg),
|
|
23702
|
+
armResumeGate: () => fleetFallbackResumeGate.arm(),
|
|
23703
|
+
// Records the DROPPED premium token + the notice's allowFrom chats; start.sh
|
|
23704
|
+
// never consumes `.premium-recovery`, so it survives the self-restart.
|
|
23705
|
+
writeRecoveryMarker: (dir, premiumModel) => {
|
|
23706
|
+
const chats = loadAccess().allowFrom.map((c) => String(c))
|
|
23707
|
+
if (chats.length > 0) writePremiumRecoveryFile(dir, premiumModel, chats)
|
|
23708
|
+
},
|
|
23709
|
+
broadcastNotice: (md) => broadcastTierNotice(md),
|
|
23710
|
+
selfRestart: (agent) => triggerSelfRestart(agent, 'tier-downgrade-resume'),
|
|
23711
|
+
selfAgent: (t) => process.env.SWITCHROOM_AGENT_NAME ?? t,
|
|
23712
|
+
log: (msg) => process.stderr.write(`telegram gateway: ${msg}\n`),
|
|
23713
|
+
})
|
|
23714
|
+
}
|
|
23715
|
+
|
|
23716
|
+
/**
|
|
23717
|
+
* "Premium model recovered" ping (tier-downgrade companion). Consulted from the
|
|
23718
|
+
* gateway's existing `runQuotaWatch` tick (every 15 min, on the cheap
|
|
23719
|
+
* `list-state` IPC read it already does — NO new poller, NO broker change).
|
|
23720
|
+
* When a `.premium-recovery` marker is pending (a downgrade dropped a premium
|
|
23721
|
+
* `/model` selection fleet-wide) AND the broker's live-authoritative per-account
|
|
23722
|
+
* eligibility shows the premium tier servable again — at least one account
|
|
23723
|
+
* neither `exhausted` nor `premium_walled` — fire EXACTLY ONE ping to the
|
|
23724
|
+
* recorded chats with a one-tap "switch back" button, then consume the marker.
|
|
23725
|
+
*
|
|
23726
|
+
* DETERMINISTIC: `exhausted` / `premium_walled` are the broker's own verdicts
|
|
23727
|
+
* (`isAccountExhausted` / `isAccountPremiumWalled` → `isModelTierWalled`, a pure
|
|
23728
|
+
* timestamp compare). No model judgement, no clock of our own.
|
|
23729
|
+
*
|
|
23730
|
+
* AT-MOST-ONCE: a fleet-wide `claim-notification` gates against a bounce or a
|
|
23731
|
+
* concurrent tick, and the marker is cleared BEFORE the send (never-storm). The
|
|
23732
|
+
* button routes through the SAME session-scoped `/model` apply path as the
|
|
23733
|
+
* model menu (`mdl:alias:<premium>` → handleModelMenuCallback +
|
|
23734
|
+
* recordModelMenuSideEffects) — it does not bypass it.
|
|
23735
|
+
*/
|
|
23736
|
+
async function maybePremiumRecoveryPing(
|
|
23737
|
+
brokerClient: NonNullable<Awaited<ReturnType<typeof getAuthBrokerClient>>>,
|
|
23738
|
+
accounts: ReadonlyArray<{ exhausted: boolean; premium_walled?: boolean }>,
|
|
23739
|
+
): Promise<void> {
|
|
23740
|
+
// Thin adapter: the order-sensitive never-storm glue lives in
|
|
23741
|
+
// runPremiumRecoveryPing (premium-recovery-wiring.ts) behind injected deps so
|
|
23742
|
+
// it is unit-testable without importing the gateway. This wires the real seams
|
|
23743
|
+
// (marker FS, fleet claim, send) — behaviour is preserved exactly.
|
|
23744
|
+
return runPremiumRecoveryPing({
|
|
23745
|
+
getAgentDir: () => resolveAgentDirFromEnv() ?? null,
|
|
23746
|
+
readMarker: (dir) => readPremiumRecoveryFile(dir),
|
|
23747
|
+
clearMarker: (dir) => clearPremiumRecoveryFile(dir),
|
|
23748
|
+
getAgent: () => getMyAgentName(),
|
|
23749
|
+
// The pure recovery predicate, bound to THIS tick's live per-account
|
|
23750
|
+
// eligibility (the broker's own verdicts; `premium_walled` may be absent).
|
|
23751
|
+
decide: () =>
|
|
23752
|
+
decidePremiumRecovery({
|
|
23753
|
+
hasMarker: true,
|
|
23754
|
+
accounts: accounts.map((a) => ({
|
|
23755
|
+
exhausted: a.exhausted,
|
|
23756
|
+
premiumWalled: a.premium_walled === true,
|
|
23757
|
+
})),
|
|
23758
|
+
}).fire,
|
|
23759
|
+
// Fail-open (claimQuotaNotification returns true on any broker error) — a
|
|
23760
|
+
// convenience ping degrades to at-least-once, never lost.
|
|
23761
|
+
claimNotification: (key) => claimQuotaNotification(brokerClient, key),
|
|
23762
|
+
fallbackChats: () => loadAccess().allowFrom.map((c) => String(c)),
|
|
23763
|
+
sendToChat: (chat_id, ping, keyboard) => {
|
|
23764
|
+
void swallowingApiCall(
|
|
23765
|
+
// allow-raw-bot-api: wrapped in swallowingApiCall (retry policy)
|
|
23766
|
+
() =>
|
|
23767
|
+
bot.api.sendRichMessage(chat_id, richMessage(ping.text), {
|
|
23768
|
+
disable_notification: true,
|
|
23769
|
+
reply_markup: keyboard,
|
|
23770
|
+
}),
|
|
23771
|
+
{ chat_id: String(chat_id), verb: 'premium-recovery:notify' },
|
|
23772
|
+
)
|
|
23773
|
+
},
|
|
23774
|
+
log: (msg) => process.stderr.write(`telegram gateway: ${msg}\n`),
|
|
23775
|
+
})
|
|
23776
|
+
}
|
|
23777
|
+
|
|
23778
|
+
/**
|
|
23779
|
+
* Clear a pending premium-recovery marker when the user MANUALLY re-selects the
|
|
23780
|
+
* dropped premium model before the recovery ping fires — no stale "available
|
|
23781
|
+
* again" ping after they've already switched back. Called from every `/model`
|
|
23782
|
+
* apply chokepoint (typed + menu/callback). Matches on the canonicalized token
|
|
23783
|
+
* so an alias vs resolved-id spelling of the same model still clears it.
|
|
23784
|
+
*/
|
|
23785
|
+
function clearPremiumRecoveryOnManualSwitch(appliedToken: string | null | undefined): void {
|
|
23786
|
+
if (appliedToken == null || appliedToken.length === 0) return
|
|
23787
|
+
const agentDir = resolveAgentDirFromEnv()
|
|
23788
|
+
if (!agentDir) return
|
|
23789
|
+
const marker = readPremiumRecoveryFile(agentDir)
|
|
23790
|
+
if (marker == null) return
|
|
23791
|
+
if (resolveMainModel(appliedToken) === resolveMainModel(marker.premiumModel)) {
|
|
23792
|
+
clearPremiumRecoveryFile(agentDir)
|
|
23793
|
+
process.stderr.write(
|
|
23794
|
+
`telegram gateway: [premium-recovery] marker cleared — user manually re-issued /model ${marker.premiumModel}\n`,
|
|
23795
|
+
)
|
|
23796
|
+
}
|
|
23797
|
+
}
|
|
23798
|
+
|
|
23549
23799
|
/** Returns true iff the dispatcher actually performed a swap (and the
|
|
23550
23800
|
* user-visible announcement was broadcast). False on no-op /
|
|
23551
23801
|
* error / idempotent-skip — caller uses this to decide whether to
|
|
@@ -23628,6 +23878,21 @@ async function doFireFleetAutoFallback(
|
|
|
23628
23878
|
if (outcome.kind === 'switched') {
|
|
23629
23879
|
fallbackAllBlockedNoticeState = { lastSentAtMs: 0 }
|
|
23630
23880
|
} else if (outcome.kind === 'all-blocked') {
|
|
23881
|
+
// ── Second recovery tier: MODEL-TIER downgrade (precedence A) ──────────
|
|
23882
|
+
// Account-swap just came back all-blocked (no account still serves the
|
|
23883
|
+
// walled model). Before giving up, if a PREMIUM /model override is active,
|
|
23884
|
+
// downgrade to the configured default and resume the dead turn rather than
|
|
23885
|
+
// stall. Only reachable here — a `switched` outcome never enters this
|
|
23886
|
+
// branch, so a model throttled on one account is always recovered by
|
|
23887
|
+
// account-swap first, never by a downgrade. Loop-bounded by the natural
|
|
23888
|
+
// on-default guard; effort revert is native (see maybeTierDowngrade).
|
|
23889
|
+
const tier = maybeTierDowngrade(triggerAgent)
|
|
23890
|
+
if (tier === 'downgraded' || tier === 'restart-pending') {
|
|
23891
|
+
// Either this turn armed a downgrade restart, or a concurrent turn
|
|
23892
|
+
// already armed a resume restart. Do NOT emit the all-blocked give-up
|
|
23893
|
+
// card — a restart is coming that replays the interrupted turn.
|
|
23894
|
+
return false
|
|
23895
|
+
}
|
|
23631
23896
|
const verdict = evaluateAllBlockedNotice(fallbackAllBlockedNoticeState, Date.now())
|
|
23632
23897
|
if (!verdict.send) {
|
|
23633
23898
|
process.stderr.write(
|
|
@@ -23860,6 +24125,16 @@ async function runQuotaWatch(opts: { bootTick?: boolean } = {}): Promise<void> {
|
|
|
23860
24125
|
return // No accounts — nothing to watch.
|
|
23861
24126
|
}
|
|
23862
24127
|
|
|
24128
|
+
// Premium-model recovery ping (tier-downgrade companion). Reuses THIS tick's
|
|
24129
|
+
// list-state read (no extra IPC): if a downgrade left a `.premium-recovery`
|
|
24130
|
+
// marker and the broker now shows the premium tier servable again, ping once
|
|
24131
|
+
// with a one-tap switch-back button. At-most-once + fleet-claim-deduped.
|
|
24132
|
+
await maybePremiumRecoveryPing(brokerClient, listStateData.accounts).catch((err) => {
|
|
24133
|
+
process.stderr.write(
|
|
24134
|
+
`telegram gateway: [premium-recovery] ping check failed (non-fatal): ${(err as Error)?.message ?? err}\n`,
|
|
24135
|
+
)
|
|
24136
|
+
})
|
|
24137
|
+
|
|
23863
24138
|
// Build AccountSnapshot[] from cached broker state only — no live probe.
|
|
23864
24139
|
// Accounts with null last_quota produce quota=null snapshots; classifyHealth
|
|
23865
24140
|
// returns 'unknown'; evaluateQuotaWatchAccount skips — no false alarms.
|
|
@@ -29100,6 +29375,12 @@ void (async () => {
|
|
|
29100
29375
|
// supersedes the coarse 5-min bucket relay below to avoid
|
|
29101
29376
|
// double-surfacing the same progress beat.
|
|
29102
29377
|
const workerFeedEnabled = isWorkerActivityFeedEnabled(process.env.SWITCHROOM_WORKER_ACTIVITY_FEED)
|
|
29378
|
+
// Combined-feed row cap (channels.telegram.worker_feed.max_rows).
|
|
29379
|
+
// Unset / non-positive → the feed's built-in default (8).
|
|
29380
|
+
const workerFeedMaxRows = (() => {
|
|
29381
|
+
const raw = Number(process.env.SWITCHROOM_TG_WORKER_FEED_MAX_ROWS)
|
|
29382
|
+
return Number.isInteger(raw) && raw > 0 ? raw : undefined
|
|
29383
|
+
})()
|
|
29103
29384
|
// Model A — foreground sub-agent nesting in the parent's live
|
|
29104
29385
|
// activity draft. ON by default; this edits the SAME activity-
|
|
29105
29386
|
// summary message the tool_label feed already owns (not the
|
|
@@ -29157,6 +29438,32 @@ void (async () => {
|
|
|
29157
29438
|
// every ~6s for the whole ban (the worker-feed shed-contract bug:
|
|
29158
29439
|
// 565 `sent.message_id` crashes in one 6h ban).
|
|
29159
29440
|
floodWaitRemainingMs: probeFloodWaitRemainingMs,
|
|
29441
|
+
// #3084 follow-up: 2+ background workers in one chat/thread coalesce
|
|
29442
|
+
// into ONE combined feed message. `maxRows` caps the visible rows
|
|
29443
|
+
// (compact `+M more working…` spill) so the body stays under the
|
|
29444
|
+
// rich-message wire ceiling (STATUS_CARD_CHAR_BUDGET). Sourced from
|
|
29445
|
+
// channels.telegram.worker_feed.max_rows via the config cascade
|
|
29446
|
+
// (scaffold emits SWITCHROOM_TG_WORKER_FEED_MAX_ROWS); unset → 8.
|
|
29447
|
+
maxRows: workerFeedMaxRows,
|
|
29448
|
+
// #3207 review: GROUP-level status pin. Workers now coalesce into
|
|
29449
|
+
// ONE shared message, so the pin must follow the GROUP lifecycle,
|
|
29450
|
+
// not a single worker's — otherwise a sibling's finish unpins a
|
|
29451
|
+
// message the survivors still need and the survivor's re-pin NOOPs
|
|
29452
|
+
// (its claim still names that id), leaving live work unpinned. The
|
|
29453
|
+
// feed drives this: pin `wk:group:<feedKey>` when the group first
|
|
29454
|
+
// paints, unpin only when the group empties (messageId === null).
|
|
29455
|
+
reconcilePin: ({ feedKey, chatId, messageId }) => {
|
|
29456
|
+
if (!PIN_STATUS_WHILE_WORKING) return
|
|
29457
|
+
const key = `wk:group:${feedKey}`
|
|
29458
|
+
if (messageId != null) {
|
|
29459
|
+
void reconcileStatusPin(key, chatId, { pinned: true, messageId })
|
|
29460
|
+
} else {
|
|
29461
|
+
const unpinChat = chatId || statusPinChatIds.get(key)
|
|
29462
|
+
if (unpinChat != null && unpinChat.length > 0) {
|
|
29463
|
+
void reconcileStatusPin(key, unpinChat, { pinned: false })
|
|
29464
|
+
}
|
|
29465
|
+
}
|
|
29466
|
+
},
|
|
29160
29467
|
log: (msg) => process.stderr.write(`telegram gateway: ${msg}\n`),
|
|
29161
29468
|
})
|
|
29162
29469
|
subagentWatcher = startSubagentWatcher({
|
|
@@ -29273,9 +29580,9 @@ void (async () => {
|
|
|
29273
29580
|
// anything. The actual repaint + re-pin happen DOWNSTREAM on the
|
|
29274
29581
|
// next replayed `running` cue: the watcher's re-registration
|
|
29275
29582
|
// replays `onProgress`, which calls `workerActivityFeed.update()`
|
|
29276
|
-
// (now un-gated) to first-paint a FRESH
|
|
29277
|
-
//
|
|
29278
|
-
//
|
|
29583
|
+
// (now un-gated) to first-paint a FRESH message, and the feed's
|
|
29584
|
+
// own `reconcilePin` (#3207) re-pins that message at the GROUP
|
|
29585
|
+
// level (`wk:group:<feedKey>`). Clearing the gate
|
|
29279
29586
|
// here FIRST is the ordering requirement — without it those first
|
|
29280
29587
|
// replayed ticks would be swallowed by the finalized gate and no
|
|
29281
29588
|
// new card would ever paint. Net effect restores the operator
|
|
@@ -29381,7 +29688,7 @@ void (async () => {
|
|
|
29381
29688
|
// card keeps the model tag even with no live entry.
|
|
29382
29689
|
model: dispatch.feedModel ?? undefined,
|
|
29383
29690
|
})
|
|
29384
|
-
|
|
29691
|
+
// #3207: group-level pin dropped by the feed on group-empty.
|
|
29385
29692
|
}
|
|
29386
29693
|
return
|
|
29387
29694
|
}
|
|
@@ -29459,8 +29766,9 @@ void (async () => {
|
|
|
29459
29766
|
state: outcome === 'failed' ? 'failed' : 'done',
|
|
29460
29767
|
model: dispatch.feedModel ?? undefined,
|
|
29461
29768
|
})
|
|
29462
|
-
//
|
|
29463
|
-
|
|
29769
|
+
// #3207: the group-level pin is dropped by the feed itself
|
|
29770
|
+
// when this group's LAST worker finishes (reconcilePin →
|
|
29771
|
+
// `wk:group:<feedKey>`); no per-worker unpin here.
|
|
29464
29772
|
}
|
|
29465
29773
|
return
|
|
29466
29774
|
}
|
|
@@ -29479,8 +29787,8 @@ void (async () => {
|
|
|
29479
29787
|
state: outcome === 'failed' ? 'failed' : 'done',
|
|
29480
29788
|
model: dispatch.feedModel ?? undefined,
|
|
29481
29789
|
})
|
|
29482
|
-
//
|
|
29483
|
-
|
|
29790
|
+
// #3207: the group-level pin is dropped by the feed itself
|
|
29791
|
+
// when this group's LAST worker finishes; no per-worker unpin.
|
|
29484
29792
|
}
|
|
29485
29793
|
|
|
29486
29794
|
const handbackOrigin = resolveSubagentOriginChat(agentId)
|
|
@@ -29647,7 +29955,10 @@ void (async () => {
|
|
|
29647
29955
|
model: feedModel,
|
|
29648
29956
|
},
|
|
29649
29957
|
wk.threadId,
|
|
29650
|
-
)
|
|
29958
|
+
)
|
|
29959
|
+
// #3207: the feed pins the group's shared message itself when
|
|
29960
|
+
// it first paints (reconcilePin → `wk:group:<feedKey>`); no
|
|
29961
|
+
// per-worker pin chained here.
|
|
29651
29962
|
return
|
|
29652
29963
|
}
|
|
29653
29964
|
if (surface !== 'nest') return // 'skip' — orphan-status off
|
|
@@ -29789,7 +30100,10 @@ void (async () => {
|
|
|
29789
30100
|
model: feedModel,
|
|
29790
30101
|
},
|
|
29791
30102
|
wk.threadId,
|
|
29792
|
-
)
|
|
30103
|
+
)
|
|
30104
|
+
// #3207: the feed pins the group's shared message itself when
|
|
30105
|
+
// it first paints (reconcilePin → `wk:group:<feedKey>`); no
|
|
30106
|
+
// per-worker pin chained here.
|
|
29793
30107
|
return
|
|
29794
30108
|
}
|
|
29795
30109
|
|