switchroom 0.19.25 → 0.19.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/git-agent-attribution-hook.sh +144 -0
- package/dist/agent-scheduler/index.js +61 -2
- package/dist/auth-broker/index.js +125 -8
- package/dist/cli/notion-write-pretool.mjs +61 -2
- package/dist/cli/switchroom.js +2347 -1104
- package/dist/host-control/main.js +126 -9
- package/dist/vault/approvals/kernel-server.js +124 -8
- package/dist/vault/broker/server.js +124 -8
- package/package.json +6 -2
- package/profiles/_base/cron-session.sh.hbs +14 -0
- package/profiles/_base/start.sh.hbs +145 -4
- package/telegram-plugin/card-layout.ts +328 -0
- package/telegram-plugin/dist/bridge/bridge.js +93 -1
- package/telegram-plugin/dist/gateway/gateway.js +2213 -1204
- package/telegram-plugin/dist/server.js +96 -1
- package/telegram-plugin/edit-flood-fuse.ts +637 -56
- package/telegram-plugin/flood-429-ledger.ts +526 -0
- package/telegram-plugin/flood-circuit-breaker.ts +18 -0
- package/telegram-plugin/gateway/flood-reply-queue.ts +168 -0
- package/telegram-plugin/gateway/gateway.ts +103 -112
- package/telegram-plugin/gateway/narrative-lane.ts +14 -0
- package/telegram-plugin/gateway/outbound-send-path.ts +36 -0
- package/telegram-plugin/gateway/outbox-sweep.ts +183 -6
- package/telegram-plugin/gateway/periodic-sweep-guard.ts +86 -0
- package/telegram-plugin/gateway/pinned-message-handler.ts +12 -16
- package/telegram-plugin/gateway/status-pin-retarget.ts +180 -0
- package/telegram-plugin/gateway/status-pin-store.ts +58 -9
- package/telegram-plugin/gateway/worker-pin-reaper.ts +56 -7
- package/telegram-plugin/llm-error-present.ts +61 -2
- package/telegram-plugin/model-unavailable.ts +8 -0
- package/telegram-plugin/operator-events.ts +72 -5
- package/telegram-plugin/outbound-class.ts +81 -0
- package/telegram-plugin/provider-credit.ts +237 -0
- package/telegram-plugin/scripts/bun-test-ci.sh +36 -6
- package/telegram-plugin/send-gate.ts +24 -2
- package/telegram-plugin/status-no-truncate.ts +11 -0
- package/telegram-plugin/status-pin-driver.ts +33 -17
- package/telegram-plugin/status-pin.ts +51 -5
- package/telegram-plugin/tests/card-golden.test.ts +69 -0
- package/telegram-plugin/tests/card-lifecycle-render.test.ts +362 -0
- package/telegram-plugin/tests/card-type-distinguishability.test.ts +291 -0
- package/telegram-plugin/tests/card-variants.golden.txt +211 -0
- package/telegram-plugin/tests/card-variants.ts +366 -0
- package/telegram-plugin/tests/edit-flood-fuse-ban-awareness.test.ts +316 -0
- package/telegram-plugin/tests/edit-flood-fuse-default-deny.test.ts +319 -0
- package/telegram-plugin/tests/edit-flood-fuse.test.ts +11 -2
- package/telegram-plugin/tests/feed-edit-rate-ceiling.test.ts +462 -0
- package/telegram-plugin/tests/fixtures/real-429-stream.ts +220 -0
- package/telegram-plugin/tests/flood-429-ledger.test.ts +278 -0
- package/telegram-plugin/tests/flood-429-recorder-wiring.test.ts +128 -0
- package/telegram-plugin/tests/flood-reply-queue.test.ts +418 -0
- package/telegram-plugin/tests/outbox-sweep-flood-breaker.test.ts +221 -0
- package/telegram-plugin/tests/periodic-sweep-guard.test.ts +151 -0
- package/telegram-plugin/tests/pinned-card-collapse.test.ts +24 -18
- package/telegram-plugin/tests/pinned-message-handler.test.ts +15 -15
- package/telegram-plugin/tests/provider-credit-402.test.ts +243 -0
- package/telegram-plugin/tests/status-pin-api.test.ts +11 -11
- package/telegram-plugin/tests/status-pin-boot-recovery.test.ts +36 -37
- package/telegram-plugin/tests/status-pin-lifecycle.test.ts +602 -0
- package/telegram-plugin/tests/status-pin-retarget.test.ts +244 -0
- package/telegram-plugin/tests/status-pin-service-message-suppression.test.ts +7 -3
- package/telegram-plugin/tests/status-pin-shutdown-wiring.test.ts +94 -0
- package/telegram-plugin/tests/status-pin-store.test.ts +179 -64
- package/telegram-plugin/tests/status-pin.test.ts +184 -7
- package/telegram-plugin/tests/test-runner-coverage.test.ts +133 -0
- package/telegram-plugin/tests/worker-activity-feed.test.ts +12 -10
- package/telegram-plugin/tests/worker-feed-coalesce.test.ts +29 -19
- package/telegram-plugin/tests/worker-feed-pin-persistence.test.ts +56 -59
- package/telegram-plugin/tests/worker-feed-terminal-edit-class.test.ts +335 -0
- package/telegram-plugin/tests/worker-visibility-prose-silent-harness.test.ts +1 -1
- package/telegram-plugin/tier-downgrade.ts +3 -2
- package/telegram-plugin/tool-activity-summary.ts +239 -322
- package/telegram-plugin/uat/assertions.ts +33 -3
- package/telegram-plugin/uat/feed-matcher.test.ts +36 -0
- package/telegram-plugin/uat/scenarios/jtbd-liveness-narration-channel.test.ts +9 -2
- package/telegram-plugin/uat/scenarios/jtbd-liveness-narration-dm.test.ts +9 -2
- package/telegram-plugin/worker-activity-feed.ts +109 -30
- package/vendor/hindsight-memory/CLAUDE.md +45 -0
- package/vendor/hindsight-memory/scripts/lib/config.py +33 -0
- package/vendor/hindsight-memory/scripts/recall.py +176 -7
- package/vendor/hindsight-memory/scripts/tests/test_config_recall_passthrough_env.py +170 -0
- package/vendor/hindsight-memory/scripts/tests/test_recall_min_score.py +464 -0
- package/vendor/hindsight-memory/scripts/tests/test_recall_request_timeout.py +241 -0
- package/vendor/hindsight-memory/settings.json +1 -1
|
@@ -52,6 +52,7 @@ import {
|
|
|
52
52
|
} from '../format.js'
|
|
53
53
|
import { richMessage } from '../rich-send.js'
|
|
54
54
|
import { journalExternalDelivery } from './outbox-sweep.js'
|
|
55
|
+
import { queueFloodBlockedReply } from './flood-reply-queue.js'
|
|
55
56
|
import { resolveChatIdFallback } from './chat-id-fallback.js'
|
|
56
57
|
import { isFinalAnswerReply, isSubstantiveFinalReply, shouldJournalReplySiteDelivery } from '../final-answer-detect.js'
|
|
57
58
|
import { decideOverPing, type OverPingDecision } from '../over-ping-safety-net.js'
|
|
@@ -1960,6 +1961,41 @@ export async function sendReply(
|
|
|
1960
1961
|
threadId = _sendResult.threadId
|
|
1961
1962
|
previewMessageId = _sendResult.previewMessageId
|
|
1962
1963
|
} catch (err) {
|
|
1964
|
+
// #3861 — a flood-blocked reply is QUEUED, never discarded. Pre-fix this
|
|
1965
|
+
// catch re-wrapped `FLOOD_WAIT_ACTIVE` into a string and threw the answer
|
|
1966
|
+
// away: `retryApiCall` fails fast BEFORE the wire while a long window is
|
|
1967
|
+
// open, so during a multi-hour ban EVERY answer the agent composed for the
|
|
1968
|
+
// operator was lost — the one content class the durable outbox did not
|
|
1969
|
+
// cover. The undelivered text goes into the same outbox the sweep drains
|
|
1970
|
+
// (it defers while the window is open, #3854) under a content-derived
|
|
1971
|
+
// nonce, so a caller retry cannot double-deliver. `cosmetic` sends are
|
|
1972
|
+
// still dropped by `queueFloodBlockedReply`; this site is `critical`.
|
|
1973
|
+
const queued = queueFloodBlockedReply({
|
|
1974
|
+
err,
|
|
1975
|
+
chatId: chat_id,
|
|
1976
|
+
threadId: threadId ?? null,
|
|
1977
|
+
// Partial failure: queue only the chunks that never landed, so the user
|
|
1978
|
+
// does not receive the already-delivered prefix a second time.
|
|
1979
|
+
text: sentIds.length === 0 ? text : chunks.slice(sentIds.length).join('\n\n'),
|
|
1980
|
+
priorityClass: 'critical',
|
|
1981
|
+
originChatId: turn?.sessionChatId ?? null,
|
|
1982
|
+
originThreadId: turn?.sessionThreadId ?? null,
|
|
1983
|
+
})
|
|
1984
|
+
if (queued != null) {
|
|
1985
|
+
process.stderr.write(
|
|
1986
|
+
`telegram gateway: reply flood-deferred — queued to outbox nonce=${queued.turnNonce} ` +
|
|
1987
|
+
`chat=${chat_id} chars=${text.length} sent=${sentIds.length}/${chunks.length}\n`,
|
|
1988
|
+
)
|
|
1989
|
+
// The obligation is DISCHARGED by the durable enqueue. Without this the
|
|
1990
|
+
// obligation tracker keeps re-prompting the agent to reply every few
|
|
1991
|
+
// minutes for the whole ban — dozens of forced turns it physically
|
|
1992
|
+
// cannot satisfy (and pre-#3861 each attempt destroyed its own answer).
|
|
1993
|
+
if (turn != null && isSubstantiveFinalReply({ text, disableNotification: modelDisableNotification })) {
|
|
1994
|
+
closeObligationOnSubstantiveReply(args, turn, replyRoutedOriginTurn)
|
|
1995
|
+
}
|
|
1996
|
+
// (the `finally` below stops the typing loop on this return path too)
|
|
1997
|
+
return { content: [{ type: 'text', text: queued.notice }] }
|
|
1998
|
+
}
|
|
1963
1999
|
const msg = err instanceof Error ? err.message : String(err)
|
|
1964
2000
|
throw new Error(`reply failed after ${sentIds.length} of ${chunks.length} chunk(s) sent: ${msg}`)
|
|
1965
2001
|
} finally {
|
|
@@ -45,6 +45,11 @@ import {
|
|
|
45
45
|
import { isShownBlock } from '../shown-ledger.js'
|
|
46
46
|
import { resolveSubagentOriginTurnKey } from '../registry/subagents-schema.js'
|
|
47
47
|
import { createRetryApiCall, retryWithThreadFallback } from '../retry-api-call.js'
|
|
48
|
+
import {
|
|
49
|
+
floodStatePath,
|
|
50
|
+
makeFloodWaitProbe,
|
|
51
|
+
makeFloodWaitRecorder,
|
|
52
|
+
} from '../flood-circuit-breaker.js'
|
|
48
53
|
|
|
49
54
|
export interface OutboxSweepDeps {
|
|
50
55
|
/** Deliver `text` to the chat. Resolves to the primary message id (best-effort). */
|
|
@@ -67,12 +72,39 @@ export interface OutboxSweepDeps {
|
|
|
67
72
|
now?: () => number
|
|
68
73
|
log?: (line: string) => void
|
|
69
74
|
quietMs?: number
|
|
75
|
+
/**
|
|
76
|
+
* Remaining ms of a KNOWN-OPEN Telegram flood window, or 0 when none — the
|
|
77
|
+
* SAME persisted breaker state (`flood-wait.json`) `robustApiCall` consults
|
|
78
|
+
* (`makeFloodWaitProbe`). When it reports an open window the sweep does not
|
|
79
|
+
* scan, claim, or send at all.
|
|
80
|
+
*
|
|
81
|
+
* Why the sweep needs its own check on top of the retry policy's: the retry
|
|
82
|
+
* policy only short-circuits windows LONGER than its in-process sleep ceiling
|
|
83
|
+
* (120s), and even then it does so per-CALL — it still costs a claim/release
|
|
84
|
+
* cycle and a log line every tick. The outbox is a safety net with no latency
|
|
85
|
+
* SLA; deferring the whole sweep while ANY window is open is strictly correct
|
|
86
|
+
* and is the only thing that makes "does not hit the wire" true.
|
|
87
|
+
*
|
|
88
|
+
* FAILS OPEN (a throwing probe ⇒ 0) for the same reason `makeFloodWaitProbe`
|
|
89
|
+
* does: a broken marker file must never permanently strand the outbox.
|
|
90
|
+
*/
|
|
91
|
+
floodWaitRemainingMs?: () => number
|
|
70
92
|
}
|
|
71
93
|
|
|
72
94
|
export interface OutboxSweepSummary {
|
|
73
95
|
scanned: number
|
|
74
96
|
delivered: number
|
|
75
97
|
skipped: number
|
|
98
|
+
/**
|
|
99
|
+
* Records whose send THREW this sweep (claim released, retried next tick).
|
|
100
|
+
* The tick-level backoff keys off this: a 5s fixed retry against a failing
|
|
101
|
+
* Telegram is how the 2026-07-27 sweep hit an open 4.4h ban 228 times.
|
|
102
|
+
*/
|
|
103
|
+
sendFailures: number
|
|
104
|
+
/** True when the whole sweep was skipped because a flood window is open. */
|
|
105
|
+
floodDeferred?: boolean
|
|
106
|
+
/** Remaining ms of the window that caused `floodDeferred`. */
|
|
107
|
+
floodRemainingMs?: number
|
|
76
108
|
}
|
|
77
109
|
|
|
78
110
|
/**
|
|
@@ -81,7 +113,20 @@ export interface OutboxSweepSummary {
|
|
|
81
113
|
export async function sweepOutbox(deps: OutboxSweepDeps): Promise<OutboxSweepSummary> {
|
|
82
114
|
const now = deps.now?.() ?? Date.now()
|
|
83
115
|
const log = deps.log ?? (() => {})
|
|
84
|
-
const summary: OutboxSweepSummary = { scanned: 0, delivered: 0, skipped: 0 }
|
|
116
|
+
const summary: OutboxSweepSummary = { scanned: 0, delivered: 0, skipped: 0, sendFailures: 0 }
|
|
117
|
+
|
|
118
|
+
// #3853 — do not sweep INTO a known-open flood window. Every record is
|
|
119
|
+
// preserved on disk and re-scanned on the next tick after the window closes,
|
|
120
|
+
// so this loses no delivery; issuing the send would only feed the ban.
|
|
121
|
+
const remainingMs = probeFloodWindow(deps.floodWaitRemainingMs)
|
|
122
|
+
if (remainingMs > 0) {
|
|
123
|
+
summary.floodDeferred = true
|
|
124
|
+
summary.floodRemainingMs = remainingMs
|
|
125
|
+
// NOT logged here: a 4.4h ban is ~3181 ticks, and one line per tick just
|
|
126
|
+
// moves the flood from the wire to the disk. `startOutboxSweep` logs the
|
|
127
|
+
// deferral at most once per DEFER_LOG_INTERVAL_MS.
|
|
128
|
+
return summary
|
|
129
|
+
}
|
|
85
130
|
|
|
86
131
|
// Re-queue crashed claims first (claim-then-crash before send).
|
|
87
132
|
reclaimStaleSending(deps.stateDir, now)
|
|
@@ -90,9 +135,27 @@ export async function sweepOutbox(deps: OutboxSweepDeps): Promise<OutboxSweepSum
|
|
|
90
135
|
if (pending.length === 0) return summary
|
|
91
136
|
const deliveredNonces = readDeliveredNonces(deps.stateDir)
|
|
92
137
|
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
138
|
+
// #3861 — deliver in CAPTURE order. `listPendingRecords` returns readdir
|
|
139
|
+
// order, which is filesystem-dependent (hash order on most Linux filesystems,
|
|
140
|
+
// i.e. effectively the nonce's hash). That was harmless while the outbox held
|
|
141
|
+
// at most one stray handback per turn; it is not harmless now that a
|
|
142
|
+
// multi-hour flood window can park a whole conversation's worth of queued
|
|
143
|
+
// replies, which would then land shuffled. Sort by `createdAt`, tie-broken by
|
|
144
|
+
// nonce so the order is total and deterministic.
|
|
145
|
+
const records = pending
|
|
146
|
+
.map((fileName) => readOutboxRecord(fileName, deps.stateDir))
|
|
147
|
+
.filter((r): r is OutboxRecord => r != null)
|
|
148
|
+
.sort((a, b) =>
|
|
149
|
+
a.createdAt !== b.createdAt
|
|
150
|
+
? a.createdAt - b.createdAt
|
|
151
|
+
: a.turnNonce < b.turnNonce
|
|
152
|
+
? -1
|
|
153
|
+
: a.turnNonce > b.turnNonce
|
|
154
|
+
? 1
|
|
155
|
+
: 0,
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
for (const record of records) {
|
|
96
159
|
summary.scanned++
|
|
97
160
|
|
|
98
161
|
// Resolve destination (anchor → registry chain → per-session origin). Fails
|
|
@@ -209,12 +272,24 @@ export async function sweepOutbox(deps: OutboxSweepDeps): Promise<OutboxSweepSum
|
|
|
209
272
|
// preserved on disk; no loss.
|
|
210
273
|
releaseClaim(record.turnNonce, deps.stateDir)
|
|
211
274
|
summary.skipped++
|
|
275
|
+
summary.sendFailures++
|
|
212
276
|
log(`outbox-sweep: send failed nonce=${record.turnNonce}: ${(err as Error).message} — will retry\n`)
|
|
213
277
|
}
|
|
214
278
|
}
|
|
215
279
|
return summary
|
|
216
280
|
}
|
|
217
281
|
|
|
282
|
+
/** Read the flood probe, failing OPEN on any throw. */
|
|
283
|
+
function probeFloodWindow(probe: (() => number) | undefined): number {
|
|
284
|
+
if (probe == null) return 0
|
|
285
|
+
try {
|
|
286
|
+
const ms = probe()
|
|
287
|
+
return Number.isFinite(ms) && ms > 0 ? ms : 0
|
|
288
|
+
} catch {
|
|
289
|
+
return 0
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
|
|
218
293
|
/**
|
|
219
294
|
* Parse a registry `turn_key` (`chatId:threadId`, `_` → null thread) into a
|
|
220
295
|
* routable chat. Exported for the gateway wiring + tests.
|
|
@@ -230,6 +305,59 @@ export function chatFromTurnKey(turnKey: string): { chatId: string; threadId: nu
|
|
|
230
305
|
/** How often the sweep ticks (aligned with the delivery-confirm sweep cadence). */
|
|
231
306
|
export const OUTBOX_SWEEP_INTERVAL_MS = 5_000
|
|
232
307
|
|
|
308
|
+
/** Ceiling on the exponential send-failure backoff. */
|
|
309
|
+
export const OUTBOX_SWEEP_BACKOFF_MAX_MS = 5 * 60_000
|
|
310
|
+
|
|
311
|
+
/**
|
|
312
|
+
* Minimum gap between two "flood window still open" log lines. A 4.4h ban is
|
|
313
|
+
* ~3181 ticks; logging each one just relocates the flood from the wire to the
|
|
314
|
+
* disk, so the deferral is reported once a minute instead.
|
|
315
|
+
*/
|
|
316
|
+
export const OUTBOX_SWEEP_DEFER_LOG_INTERVAL_MS = 60_000
|
|
317
|
+
|
|
318
|
+
/**
|
|
319
|
+
* Exponential backoff over the fixed 5s sweep tick (#3853).
|
|
320
|
+
*
|
|
321
|
+
* The tick interval alone is not a retry policy. On 2026-07-27 the sweep held
|
|
322
|
+
* ONE undeliverable record and re-issued its `sendMessage` every 5s for the
|
|
323
|
+
* whole 4.4h ban — 228 requests into a window the breaker already knew was
|
|
324
|
+
* open, each one answered with the same counting-down `retry_after`. The flood
|
|
325
|
+
* probe (above) closes the case where the breaker HAS a window recorded; this
|
|
326
|
+
* closes the case where it does not — a persistent failure of any other kind
|
|
327
|
+
* must not be re-driven at 12 requests/minute forever.
|
|
328
|
+
*
|
|
329
|
+
* Pure and clock-injected so the schedule is asserted in tests rather than
|
|
330
|
+
* inferred from timer behaviour.
|
|
331
|
+
*/
|
|
332
|
+
export function createSweepBackoff(cfg: { baseMs?: number; maxMs?: number } = {}): {
|
|
333
|
+
ready: (now: number) => boolean
|
|
334
|
+
noteFailure: (now: number) => number
|
|
335
|
+
noteSuccess: () => void
|
|
336
|
+
failures: () => number
|
|
337
|
+
} {
|
|
338
|
+
const baseMs = cfg.baseMs ?? OUTBOX_SWEEP_INTERVAL_MS
|
|
339
|
+
const maxMs = cfg.maxMs ?? OUTBOX_SWEEP_BACKOFF_MAX_MS
|
|
340
|
+
let failures = 0
|
|
341
|
+
let nextAllowedAt = 0
|
|
342
|
+
return {
|
|
343
|
+
ready: (now) => now >= nextAllowedAt,
|
|
344
|
+
noteFailure: (now) => {
|
|
345
|
+
failures++
|
|
346
|
+
// 5s, 10s, 20s, … capped. `failures - 1` so the FIRST failure still
|
|
347
|
+
// retries at the normal tick cadence — a one-off transient send error
|
|
348
|
+
// should not delay a real answer.
|
|
349
|
+
const delay = Math.min(maxMs, baseMs * Math.pow(2, failures - 1))
|
|
350
|
+
nextAllowedAt = now + delay
|
|
351
|
+
return delay
|
|
352
|
+
},
|
|
353
|
+
noteSuccess: () => {
|
|
354
|
+
failures = 0
|
|
355
|
+
nextAllowedAt = 0
|
|
356
|
+
},
|
|
357
|
+
failures: () => failures,
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
|
|
233
361
|
/**
|
|
234
362
|
* Own the heartbeat-tick outbox sweep entirely, keeping the timer / chunking /
|
|
235
363
|
* turn-key parse / dedup / registry-chain wiring OUT of `gateway.ts` (the line
|
|
@@ -336,21 +464,43 @@ export function startOutboxSweep(deps: {
|
|
|
336
464
|
text: string,
|
|
337
465
|
) => OutboxDeliveryMarkup | undefined
|
|
338
466
|
log?: (line: string) => void
|
|
467
|
+
/** Test seam — override the flood probe (default: the persisted breaker). */
|
|
468
|
+
floodWaitRemainingMs?: () => number
|
|
469
|
+
/** Test seam — override the tick backoff. */
|
|
470
|
+
backoff?: ReturnType<typeof createSweepBackoff>
|
|
339
471
|
}): ReturnType<typeof setInterval> | undefined {
|
|
340
472
|
if (!deps.isGatewayMain || process.env.SWITCHROOM_TG_OUTBOX_DELIVERY === '0') return undefined
|
|
341
|
-
|
|
473
|
+
// #3853 — the sweep is a full outbound machine and must consult the SAME
|
|
474
|
+
// persisted flood breaker every other outbound path does. Before this it was
|
|
475
|
+
// the one `createRetryApiCall` wiring with neither hook: it could not see an
|
|
476
|
+
// open ban (no `floodWaitRemainingMs`) and could not record one it caused (no
|
|
477
|
+
// `onFloodWait`). `scripts/check-retry-flood-hooks.mjs` now fails lint if any
|
|
478
|
+
// wiring omits either.
|
|
479
|
+
const floodProbe =
|
|
480
|
+
deps.floodWaitRemainingMs ?? makeFloodWaitProbe(floodStatePath(deps.stateDir))
|
|
481
|
+
const recordFloodWait = makeFloodWaitRecorder(floodStatePath(deps.stateDir))
|
|
482
|
+
const retry = createRetryApiCall({
|
|
483
|
+
log: deps.log,
|
|
484
|
+
floodWaitRemainingMs: floodProbe,
|
|
485
|
+
onFloodWait: (retryAfterSec) => recordFloodWait(retryAfterSec),
|
|
486
|
+
})
|
|
342
487
|
const send = createOutboxSend({
|
|
343
488
|
getBot: deps.getBot,
|
|
344
489
|
retry,
|
|
345
490
|
...(deps.resolveReplyMarkup != null ? { resolveReplyMarkup: deps.resolveReplyMarkup } : {}),
|
|
346
491
|
})
|
|
492
|
+
const backoff = deps.backoff ?? createSweepBackoff()
|
|
493
|
+
let lastDeferLogAt = 0
|
|
347
494
|
const tick = () => {
|
|
348
495
|
const bot = deps.getBot()
|
|
349
496
|
if (bot == null) return
|
|
497
|
+
const now = Date.now()
|
|
498
|
+
if (!backoff.ready(now)) return
|
|
350
499
|
void sweepOutbox({
|
|
351
500
|
stateDir: deps.stateDir,
|
|
352
501
|
log: deps.log,
|
|
353
502
|
send,
|
|
503
|
+
floodWaitRemainingMs: floodProbe,
|
|
354
504
|
textAlreadyDelivered: (chatId, threadId, text) => deps.dedupCheck(chatId, threadId ?? undefined, text),
|
|
355
505
|
registryChainLookup: (taskId) => {
|
|
356
506
|
const db = deps.getTurnsDb()
|
|
@@ -358,7 +508,34 @@ export function startOutboxSweep(deps: {
|
|
|
358
508
|
const turnKey = resolveSubagentOriginTurnKey(db, taskId)
|
|
359
509
|
return turnKey == null ? null : chatFromTurnKey(turnKey)
|
|
360
510
|
},
|
|
361
|
-
})
|
|
511
|
+
})
|
|
512
|
+
.then((summary) => {
|
|
513
|
+
// A flood-deferred sweep is neither success nor failure: it never
|
|
514
|
+
// reached the wire, so it must not clear a backoff earned by real
|
|
515
|
+
// failures, and it must not deepen one either.
|
|
516
|
+
if (summary.floodDeferred === true) {
|
|
517
|
+
const at = Date.now()
|
|
518
|
+
if (at - lastDeferLogAt >= OUTBOX_SWEEP_DEFER_LOG_INTERVAL_MS) {
|
|
519
|
+
lastDeferLogAt = at
|
|
520
|
+
deps.log?.(
|
|
521
|
+
`outbox-sweep: deferred — Telegram flood window still open for ` +
|
|
522
|
+
`${Math.ceil((summary.floodRemainingMs ?? 0) / 1000)}s; not issuing any ` +
|
|
523
|
+
`send (would feed the ban)\n`,
|
|
524
|
+
)
|
|
525
|
+
}
|
|
526
|
+
return
|
|
527
|
+
}
|
|
528
|
+
if (summary.sendFailures > 0) {
|
|
529
|
+
const delay = backoff.noteFailure(Date.now())
|
|
530
|
+
deps.log?.(
|
|
531
|
+
`outbox-sweep: ${summary.sendFailures} send failure(s) — backing off ` +
|
|
532
|
+
`${Math.round(delay / 1000)}s before the next sweep (attempt ${backoff.failures()})\n`,
|
|
533
|
+
)
|
|
534
|
+
} else {
|
|
535
|
+
backoff.noteSuccess()
|
|
536
|
+
}
|
|
537
|
+
})
|
|
538
|
+
.catch((err) => deps.log?.(`outbox-sweep: tick failed: ${(err as Error).message}\n`))
|
|
362
539
|
}
|
|
363
540
|
const timer = setInterval(tick, OUTBOX_SWEEP_INTERVAL_MS)
|
|
364
541
|
timer.unref?.()
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* periodic-sweep-guard.ts — single-flight wrapper for an interval-driven sweep.
|
|
3
|
+
*
|
|
4
|
+
* Why this exists
|
|
5
|
+
* ---------------
|
|
6
|
+
* `runMidSessionCardReaper` (gateway.ts) is dispatched from a `setInterval` as
|
|
7
|
+
* `void run()`, every 5 minutes by default. Its body awaits REAL Telegram calls:
|
|
8
|
+
* `robustApiCall` finalize edits for orphaned activity cards, and
|
|
9
|
+
* `reconcileStatusPin` unpins for stale `wk:` pins. Both retry with backoff and
|
|
10
|
+
* both can park behind a flood-wait — this gateway took a 62-minute flood ban on
|
|
11
|
+
* 2026-07-25.
|
|
12
|
+
*
|
|
13
|
+
* With no guard, every tick started ANOTHER concurrent pass. During a stall that
|
|
14
|
+
* is unbounded fan-out: N passes reading the same `status-pins.json`, deciding
|
|
15
|
+
* the same reaps, and firing duplicate unpins into the very rate limit that was
|
|
16
|
+
* slowing them down. Concurrent passes also race each other in the store-orphan
|
|
17
|
+
* branch, which unpins and drops the row OUTSIDE the per-key reconcile lock.
|
|
18
|
+
*
|
|
19
|
+
* The guard is deliberately SKIP, not queue: a tick arriving while a pass is
|
|
20
|
+
* running is dropped. The reaper is a backstop with no deadline, and the next
|
|
21
|
+
* tick is at most one interval away — queuing would just rebuild the same
|
|
22
|
+
* pile-up with extra latency.
|
|
23
|
+
*
|
|
24
|
+
* It also absorbs a throw out of the pass. The gateway's `unhandledRejection`
|
|
25
|
+
* handler crashes the process, and a fire-and-forget cosmetic sweep must never
|
|
26
|
+
* be able to do that (the same rule `reconcileStatusPin` follows after a benign
|
|
27
|
+
* pin-rights 400 took the whole gateway down, marko 2026-07-01).
|
|
28
|
+
*/
|
|
29
|
+
|
|
30
|
+
export interface PeriodicSweepGuard {
|
|
31
|
+
/** Run the sweep unless a pass is already in flight. Never rejects. */
|
|
32
|
+
tick(): Promise<void>
|
|
33
|
+
/** True while a pass is in flight. Test/introspection seam. */
|
|
34
|
+
isRunning(): boolean
|
|
35
|
+
/** Passes skipped because one was already running. Test/introspection seam. */
|
|
36
|
+
skipped(): number
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Wrap `run` so at most one pass is in flight at a time.
|
|
41
|
+
*
|
|
42
|
+
* `onSkip` / `onError` are optional observers (the gateway logs to stderr);
|
|
43
|
+
* both are themselves absorbed, so a broken logger cannot take down the one
|
|
44
|
+
* function whose job is to make the sweep un-crashable.
|
|
45
|
+
*/
|
|
46
|
+
export function createPeriodicSweepGuard(args: {
|
|
47
|
+
run: () => Promise<void>
|
|
48
|
+
onSkip?: () => void
|
|
49
|
+
onError?: (err: unknown) => void
|
|
50
|
+
}): PeriodicSweepGuard {
|
|
51
|
+
let running = false
|
|
52
|
+
let skippedCount = 0
|
|
53
|
+
|
|
54
|
+
const notify = (fn: (() => void) | undefined): void => {
|
|
55
|
+
if (fn == null) return
|
|
56
|
+
try {
|
|
57
|
+
fn()
|
|
58
|
+
} catch {
|
|
59
|
+
/* an observer must never break the guard */
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
return {
|
|
64
|
+
async tick(): Promise<void> {
|
|
65
|
+
if (running) {
|
|
66
|
+
skippedCount += 1
|
|
67
|
+
notify(args.onSkip)
|
|
68
|
+
return
|
|
69
|
+
}
|
|
70
|
+
running = true
|
|
71
|
+
try {
|
|
72
|
+
await args.run()
|
|
73
|
+
} catch (err) {
|
|
74
|
+
notify(args.onError == null ? undefined : () => args.onError?.(err))
|
|
75
|
+
} finally {
|
|
76
|
+
running = false
|
|
77
|
+
}
|
|
78
|
+
},
|
|
79
|
+
isRunning(): boolean {
|
|
80
|
+
return running
|
|
81
|
+
},
|
|
82
|
+
skipped(): number {
|
|
83
|
+
return skippedCount
|
|
84
|
+
},
|
|
85
|
+
}
|
|
86
|
+
}
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* silent status-pin fires. The pin call passes `disable_notification: true`,
|
|
6
6
|
* which kills the PUSH notification but NOT the in-chat service message — so we
|
|
7
7
|
* delete that service message as it lands, but ONLY for pins we own (tracked in
|
|
8
|
-
* `
|
|
8
|
+
* `statusPinClaims`). Manual/operator pins are never silent and are never
|
|
9
9
|
* touched. Only silent status pins reach here as an OUR-pin match, so the
|
|
10
10
|
* ownership check is the guard.
|
|
11
11
|
*
|
|
@@ -21,13 +21,12 @@
|
|
|
21
21
|
|
|
22
22
|
import type { Context, Filter } from 'grammy'
|
|
23
23
|
import { pinnedMessageIsOurs, type TrackedStatusPin } from './status-pin-store.js'
|
|
24
|
-
import type {
|
|
24
|
+
import type { StatusPinClaim } from './status-pin-retarget.js'
|
|
25
25
|
|
|
26
26
|
export interface PinnedMessageHandlerDeps {
|
|
27
|
-
/** Live pinKey→
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
statusPinChatIds: ReadonlyMap<string, string>
|
|
27
|
+
/** Live pinKey→claim registry of pins we own. ONE record per key (#3809):
|
|
28
|
+
* the chat id is a field of the claim, not a parallel map that can diverge. */
|
|
29
|
+
statusPinClaims: ReadonlyMap<string, StatusPinClaim>
|
|
31
30
|
/**
|
|
32
31
|
* Delete the pin service message. Bound in gateway.ts through
|
|
33
32
|
* `robustApiCall(() => lockedBot.api.deleteMessage(...))` so the raw Bot API
|
|
@@ -50,16 +49,13 @@ export async function handlePinnedMessage(
|
|
|
50
49
|
// Chat-scoped ownership (see pinnedMessageIsOurs): the match requires BOTH
|
|
51
50
|
// the messageId AND that the tracked entry lives in THIS chat, so a pin id
|
|
52
51
|
// colliding across chats can't delete a foreign (e.g. operator-manual) pin
|
|
53
|
-
// notice.
|
|
54
|
-
//
|
|
55
|
-
const trackedPins = (): TrackedStatusPin[] =>
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
}
|
|
61
|
-
return out
|
|
62
|
-
}
|
|
52
|
+
// notice. Each claim carries its own chat id (#3809), so the match can never
|
|
53
|
+
// silently degrade to messageId-only because a companion map went missing.
|
|
54
|
+
const trackedPins = (): TrackedStatusPin[] =>
|
|
55
|
+
[...deps.statusPinClaims.values()].map((c) => ({
|
|
56
|
+
chatId: c.chatId,
|
|
57
|
+
messageId: c.messageId,
|
|
58
|
+
}))
|
|
63
59
|
const isOurs = () => pinnedMessageIsOurs(trackedPins(), chatId, pinnedId)
|
|
64
60
|
|
|
65
61
|
if (!isOurs()) {
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* status-pin-retarget.ts — orchestrates a status-pin RETARGET as two persisted
|
|
3
|
+
* legs, so a single-shot caller still ends up with the new surface pinned.
|
|
4
|
+
*
|
|
5
|
+
* The defect this closes
|
|
6
|
+
* ----------------------
|
|
7
|
+
* `decidePinAction(prev, { pinned: true, messageId: NEW })` with a claim on a
|
|
8
|
+
* DIFFERENT message used to report a bare `unpin`, documented as "a subsequent
|
|
9
|
+
* desired-pin re-pins the new one on the next reconcile". That is true only for
|
|
10
|
+
* callers that RE-DRIVE the reconcile:
|
|
11
|
+
*
|
|
12
|
+
* - the worker-activity feed does (`syncPin` runs on every steady-state edit,
|
|
13
|
+
* `worker-activity-feed.ts`), so its group-message rotation converges;
|
|
14
|
+
* - the FOREGROUND activity card does NOT. `narrative-lane.ts` reconciles
|
|
15
|
+
* `fg:<statusKey>` exactly once, inside the `activityMessageId == null`
|
|
16
|
+
* OPEN branch; the `else` (edit) branch never touches the pin, and turn-end
|
|
17
|
+
* only ever unpins (`turn-end.ts`).
|
|
18
|
+
*
|
|
19
|
+
* So when the ack-first reopen path (`feed-reopen-gate.ts`) nulls
|
|
20
|
+
* `activityMessageId` mid-turn and a FRESH card opens, that one reconcile
|
|
21
|
+
* unpinned the old card and stopped. The live card ran unpinned for the rest of
|
|
22
|
+
* the turn, and — because the retarget mapped to a `clear` persist op — the
|
|
23
|
+
* durable `status-pins.json` row was deleted too, so neither the mid-session
|
|
24
|
+
* reaper nor the next boot's sweep had anything left to reconcile from.
|
|
25
|
+
*
|
|
26
|
+
* Why TWO legs and not one
|
|
27
|
+
* ------------------------
|
|
28
|
+
* `reconcileAndPersistStatusPin` carries the persist-BEFORE-pin contract for a
|
|
29
|
+
* single op. Running the retarget as one `pin`-op leg would write a pending row
|
|
30
|
+
* naming the NEW id before the OLD one is unpinned — a crash in that window
|
|
31
|
+
* erases the only durable record of a message that is still pinned. Splitting
|
|
32
|
+
* it keeps each leg's crash-safety exactly as designed:
|
|
33
|
+
*
|
|
34
|
+
* leg 1 unpin the stale claim, with the row still naming the OLD id;
|
|
35
|
+
* leg 2 pin the new message from a null claim, with its own pending row.
|
|
36
|
+
*
|
|
37
|
+
* Failure semantics are inherited from `executePinLeg`, not re-invented here: a
|
|
38
|
+
* leg-1 result that is NON-NULL means the unpin was never confirmed (#3664
|
|
39
|
+
* Defect B — the old message is provably still pinned and the claim was
|
|
40
|
+
* deliberately retained). Pinning the new message then would leave two pins in
|
|
41
|
+
* the chat with a durable record of one, so leg 2 is skipped and the retained
|
|
42
|
+
* claim is left for the next reconcile / the mid-session reaper / the boot
|
|
43
|
+
* sweep to retry.
|
|
44
|
+
*
|
|
45
|
+
* THE SINGLE PATH (#3831)
|
|
46
|
+
* -----------------------
|
|
47
|
+
* This module is the ONLY decider. `status-pin-driver.ts` used to call
|
|
48
|
+
* `decidePinAction` itself and carry its OWN copy of the expansion above — a
|
|
49
|
+
* copy that recursed without the per-leg persistence, so which of the two
|
|
50
|
+
* implementations ran decided whether the durable row survived. That copy is
|
|
51
|
+
* deleted, and the driver's parameter type (`PinLegAction`) can no longer even
|
|
52
|
+
* express a repin, so a second one cannot grow back.
|
|
53
|
+
*
|
|
54
|
+
* Dependency-free apart from the pure decision, so it is provable in isolation
|
|
55
|
+
* (`telegram-plugin/tests/status-pin-retarget.test.ts`); the gateway owns the
|
|
56
|
+
* wiring of `runPin` / `commit`.
|
|
57
|
+
*/
|
|
58
|
+
|
|
59
|
+
import type { DesiredPin, PinLegAction, PinState } from '../status-pin.js'
|
|
60
|
+
import { decidePinAction } from '../status-pin.js'
|
|
61
|
+
import type { StatusPinPersistOp, StatusPinStoreFsSeam } from './status-pin-store.js'
|
|
62
|
+
import { reconcileAndPersistStatusPin } from './status-pin-store.js'
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* ONE in-memory claim for a pin key — the message we believe is pinned, the
|
|
66
|
+
* chat it lives in, and when the claim was FIRST taken (the reaper's TTL age).
|
|
67
|
+
*
|
|
68
|
+
* #3809 — these were three parallel `Map`s (`statusPinState` / `statusPinChatIds`
|
|
69
|
+
* / `statusPinPinnedAt`) written together but able to diverge. A `wk:` claim
|
|
70
|
+
* whose chatId entry went missing was skipped by the mid-session reaper on
|
|
71
|
+
* EVERY pass (it cannot unpin without a chat) and excluded from the durable
|
|
72
|
+
* store-orphan net (which skips keys that DO have an in-memory claim) — so it
|
|
73
|
+
* was permanently unreapable until the next boot, silently. Collapsing the
|
|
74
|
+
* three maps into one record makes the invariant STRUCTURAL: a claim without a
|
|
75
|
+
* chat id cannot be represented, so that gap cannot be reintroduced.
|
|
76
|
+
*/
|
|
77
|
+
export interface StatusPinClaim {
|
|
78
|
+
/** Telegram message_id we believe is pinned for this key. */
|
|
79
|
+
messageId: number
|
|
80
|
+
/** Chat the pin lives in. Never empty — the reconcile refuses an empty chat. */
|
|
81
|
+
chatId: string
|
|
82
|
+
/** Wall-clock ms the claim was FIRST taken (survives a same-id re-pin). */
|
|
83
|
+
pinnedAt: number
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
export interface StatusPinReconcileArgs {
|
|
87
|
+
pinKey: string
|
|
88
|
+
chatId: string
|
|
89
|
+
/** The claim held for this key when the reconcile started. */
|
|
90
|
+
prev: PinState | null
|
|
91
|
+
/** What the caller wants pinned for this key now. */
|
|
92
|
+
desired: DesiredPin
|
|
93
|
+
/** Durable-store binding, or null when persistence is off (STATIC). */
|
|
94
|
+
persist: { path: string; fs: StatusPinStoreFsSeam } | null
|
|
95
|
+
/**
|
|
96
|
+
* Execute ONE already-decided leg against Telegram (the gateway binds
|
|
97
|
+
* `executePinLeg`). Must never throw — API errors are absorbed inside.
|
|
98
|
+
*/
|
|
99
|
+
runPin: (action: PinLegAction, from: PinState | null) => Promise<PinState | null>
|
|
100
|
+
/** The single claim registry, keyed by pinKey. */
|
|
101
|
+
claims: Map<string, StatusPinClaim>
|
|
102
|
+
/** Injectable clock for the `pinnedAt` stamp (tests). */
|
|
103
|
+
now?: () => number
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* Drive `prev → desired` for one pin key, expanding a RETARGET into its two
|
|
108
|
+
* legs. Every other action (pin / unpin / noop) is a single leg, unchanged.
|
|
109
|
+
*
|
|
110
|
+
* Each leg is committed to the claim registry as it lands, so the in-memory
|
|
111
|
+
* claim never lags the durable row — including the intermediate cleared state
|
|
112
|
+
* between a retarget's two legs, which is what lets a concurrent reader (the
|
|
113
|
+
* `wk:` reaper) see an honest snapshot rather than a claim on an unpinned id.
|
|
114
|
+
*
|
|
115
|
+
* Callers must hold the per-key reconcile lock (`withPinReconcileLock`); `prev`
|
|
116
|
+
* is read once by the caller and both legs run inside that critical section.
|
|
117
|
+
*/
|
|
118
|
+
export async function runStatusPinReconcile(args: StatusPinReconcileArgs): Promise<void> {
|
|
119
|
+
const { pinKey, chatId, prev, desired, persist, runPin, claims } = args
|
|
120
|
+
const now = args.now ?? Date.now
|
|
121
|
+
|
|
122
|
+
// Publish a leg's outcome into the single claim registry. `pinnedAt` is
|
|
123
|
+
// carried forward when a claim already exists so it keeps the FIRST-taken
|
|
124
|
+
// timestamp; clearing the claim drops it, so a re-pin ages fresh.
|
|
125
|
+
const commit = (next: PinState | null): void => {
|
|
126
|
+
if (next == null) {
|
|
127
|
+
claims.delete(pinKey)
|
|
128
|
+
return
|
|
129
|
+
}
|
|
130
|
+
const pinnedAt = claims.get(pinKey)?.pinnedAt ?? now()
|
|
131
|
+
claims.set(pinKey, { messageId: next.messageId, chatId, pinnedAt })
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
// One leg, with the persist ordering its action requires. Only a fresh `pin`
|
|
135
|
+
// opens the crash window that persist-BEFORE-pin closes; unpin / noop / re-pin
|
|
136
|
+
// of the same id clear or leave the row and are safe to persist after.
|
|
137
|
+
//
|
|
138
|
+
// The leg action is decided ONCE, below, and threaded through to both the
|
|
139
|
+
// persist-op mapping and the driver — the driver no longer re-derives it
|
|
140
|
+
// (#3831).
|
|
141
|
+
const runLeg = (
|
|
142
|
+
legAction: PinLegAction,
|
|
143
|
+
from: PinState | null,
|
|
144
|
+
): Promise<PinState | null> => {
|
|
145
|
+
if (persist == null) return runPin(legAction, from)
|
|
146
|
+
const op: StatusPinPersistOp =
|
|
147
|
+
legAction.kind === 'pin'
|
|
148
|
+
? { kind: 'pin', messageId: legAction.messageId }
|
|
149
|
+
: { kind: 'clear' }
|
|
150
|
+
return reconcileAndPersistStatusPin({
|
|
151
|
+
path: persist.path,
|
|
152
|
+
fs: persist.fs,
|
|
153
|
+
pinKey,
|
|
154
|
+
chatId,
|
|
155
|
+
op,
|
|
156
|
+
applyPin: () => runPin(legAction, from),
|
|
157
|
+
now: now(),
|
|
158
|
+
})
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
const action = decidePinAction(prev, desired)
|
|
162
|
+
|
|
163
|
+
// THE retarget expansion. This is the only copy in the codebase; the driver's
|
|
164
|
+
// parameter type (`PinLegAction`) cannot express a repin, so a second one
|
|
165
|
+
// cannot be written there again (#3831).
|
|
166
|
+
if (action.kind === 'repin') {
|
|
167
|
+
const afterUnpin = await runLeg(
|
|
168
|
+
{ kind: 'unpin', messageId: action.unpinMessageId },
|
|
169
|
+
prev,
|
|
170
|
+
)
|
|
171
|
+
commit(afterUnpin)
|
|
172
|
+
// Never-confirmed unpin (#3664 Defect B): keep the retained claim, skip the
|
|
173
|
+
// pin leg. See the docblock.
|
|
174
|
+
if (afterUnpin != null) return
|
|
175
|
+
commit(await runLeg({ kind: 'pin', messageId: action.pinMessageId }, null))
|
|
176
|
+
return
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
commit(await runLeg(action, prev))
|
|
180
|
+
}
|