switchroom 0.19.26 → 0.19.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/git-agent-attribution-hook.sh +144 -0
- package/dist/agent-scheduler/index.js +60 -2
- package/dist/auth-broker/index.js +244 -13
- package/dist/cli/autoaccept-poll.js +225 -17
- package/dist/cli/notion-write-pretool.mjs +60 -2
- package/dist/cli/switchroom.js +2843 -1220
- package/dist/host-control/main.js +245 -14
- package/dist/vault/approvals/kernel-server.js +242 -13
- package/dist/vault/broker/server.js +242 -13
- package/package.json +7 -2
- package/profiles/_base/cron-session.sh.hbs +8 -0
- package/profiles/_base/start.sh.hbs +175 -15
- package/telegram-plugin/card-layout.ts +328 -0
- package/telegram-plugin/dist/bridge/bridge.js +94 -1
- package/telegram-plugin/dist/gateway/gateway.js +2544 -1182
- package/telegram-plugin/dist/server.js +97 -1
- package/telegram-plugin/edit-flood-fuse.ts +841 -57
- package/telegram-plugin/flood-429-ledger.ts +526 -0
- package/telegram-plugin/flood-circuit-breaker.ts +18 -0
- package/telegram-plugin/gateway/callback-query-handlers.ts +6 -0
- package/telegram-plugin/gateway/flood-reply-queue.ts +168 -0
- package/telegram-plugin/gateway/gateway.ts +67 -70
- package/telegram-plugin/gateway/mcp-failure-hook.ts +74 -0
- package/telegram-plugin/gateway/narrative-lane.ts +14 -0
- package/telegram-plugin/gateway/outbound-send-path.ts +36 -0
- package/telegram-plugin/gateway/outbox-sweep.ts +183 -6
- package/telegram-plugin/gateway/pinned-message-handler.ts +12 -16
- package/telegram-plugin/gateway/status-pin-retarget.ts +72 -36
- package/telegram-plugin/gateway/status-pin-store.ts +58 -9
- package/telegram-plugin/gateway/worker-pin-reaper.ts +56 -7
- package/telegram-plugin/inline-keyboard-callbacks.ts +202 -21
- package/telegram-plugin/llm-error-present.ts +61 -2
- package/telegram-plugin/mcp-credential-failure.ts +459 -0
- package/telegram-plugin/model-unavailable.ts +8 -0
- package/telegram-plugin/operator-events.ts +110 -5
- package/telegram-plugin/outbound-class.ts +81 -0
- package/telegram-plugin/provider-credit.ts +237 -0
- package/telegram-plugin/scripts/bun-test-ci.sh +36 -6
- package/telegram-plugin/send-gate.ts +24 -2
- package/telegram-plugin/status-no-truncate.ts +10 -48
- package/telegram-plugin/status-pin-driver.ts +33 -45
- package/telegram-plugin/status-pin.ts +18 -1
- package/telegram-plugin/tests/card-golden.test.ts +69 -0
- package/telegram-plugin/tests/card-lifecycle-render.test.ts +362 -0
- package/telegram-plugin/tests/card-type-distinguishability.test.ts +187 -164
- package/telegram-plugin/tests/card-variants.golden.txt +211 -0
- package/telegram-plugin/tests/card-variants.ts +366 -0
- package/telegram-plugin/tests/edit-flood-fuse-ban-awareness.test.ts +373 -0
- package/telegram-plugin/tests/edit-flood-fuse-default-deny.test.ts +319 -0
- package/telegram-plugin/tests/edit-flood-fuse-reply-reserve.test.ts +340 -0
- package/telegram-plugin/tests/edit-flood-fuse.test.ts +11 -2
- package/telegram-plugin/tests/feed-edit-rate-ceiling.test.ts +462 -0
- package/telegram-plugin/tests/finalize-callback-flood-policy.test.ts +298 -0
- package/telegram-plugin/tests/finalize-callback.test.ts +41 -8
- package/telegram-plugin/tests/fixtures/real-429-stream.ts +220 -0
- package/telegram-plugin/tests/flood-429-ledger.test.ts +278 -0
- package/telegram-plugin/tests/flood-429-recorder-wiring.test.ts +128 -0
- package/telegram-plugin/tests/flood-reply-queue.test.ts +418 -0
- package/telegram-plugin/tests/mcp-credential-failure.test.ts +310 -0
- package/telegram-plugin/tests/outbox-sweep-flood-breaker.test.ts +221 -0
- package/telegram-plugin/tests/pinned-card-collapse.test.ts +19 -24
- package/telegram-plugin/tests/pinned-message-handler.test.ts +15 -15
- package/telegram-plugin/tests/provider-credit-402.test.ts +243 -0
- package/telegram-plugin/tests/status-pin-api.test.ts +11 -11
- package/telegram-plugin/tests/status-pin-boot-recovery.test.ts +36 -37
- package/telegram-plugin/tests/status-pin-lifecycle.test.ts +602 -0
- package/telegram-plugin/tests/status-pin-retarget.test.ts +90 -62
- package/telegram-plugin/tests/status-pin-service-message-suppression.test.ts +7 -3
- package/telegram-plugin/tests/status-pin-store.test.ts +109 -60
- package/telegram-plugin/tests/status-pin.test.ts +56 -5
- package/telegram-plugin/tests/test-runner-coverage.test.ts +133 -0
- package/telegram-plugin/tests/worker-activity-feed.test.ts +12 -10
- package/telegram-plugin/tests/worker-feed-coalesce.test.ts +23 -29
- package/telegram-plugin/tests/worker-feed-pin-persistence.test.ts +56 -59
- package/telegram-plugin/tests/worker-feed-terminal-edit-class.test.ts +335 -0
- package/telegram-plugin/tests/worker-visibility-prose-silent-harness.test.ts +1 -1
- package/telegram-plugin/tool-activity-summary.ts +239 -365
- package/telegram-plugin/uat/assertions.ts +22 -11
- package/telegram-plugin/uat/feed-matcher.test.ts +24 -17
- package/telegram-plugin/worker-activity-feed.ts +105 -47
- package/vendor/hindsight-memory/CLAUDE.md +45 -0
- package/vendor/hindsight-memory/scripts/drain_pending.py +433 -11
- package/vendor/hindsight-memory/scripts/lib/config.py +33 -0
- package/vendor/hindsight-memory/scripts/lib/pending.py +193 -28
- package/vendor/hindsight-memory/scripts/recall.py +176 -7
- package/vendor/hindsight-memory/scripts/tests/test_config_recall_passthrough_env.py +170 -0
- package/vendor/hindsight-memory/scripts/tests/test_drain_circuit_breaker.py +401 -0
- package/vendor/hindsight-memory/scripts/tests/test_drain_serialisation.py +286 -0
- package/vendor/hindsight-memory/scripts/tests/test_pending_drops.py +817 -8
- package/vendor/hindsight-memory/scripts/tests/test_recall_min_score.py +464 -0
- package/vendor/hindsight-memory/settings.json +1 -1
- package/vendor/hindsight-memory/tests/test_hooks.py +11 -2
|
@@ -52,6 +52,7 @@ import {
|
|
|
52
52
|
} from '../format.js'
|
|
53
53
|
import { richMessage } from '../rich-send.js'
|
|
54
54
|
import { journalExternalDelivery } from './outbox-sweep.js'
|
|
55
|
+
import { queueFloodBlockedReply } from './flood-reply-queue.js'
|
|
55
56
|
import { resolveChatIdFallback } from './chat-id-fallback.js'
|
|
56
57
|
import { isFinalAnswerReply, isSubstantiveFinalReply, shouldJournalReplySiteDelivery } from '../final-answer-detect.js'
|
|
57
58
|
import { decideOverPing, type OverPingDecision } from '../over-ping-safety-net.js'
|
|
@@ -1960,6 +1961,41 @@ export async function sendReply(
|
|
|
1960
1961
|
threadId = _sendResult.threadId
|
|
1961
1962
|
previewMessageId = _sendResult.previewMessageId
|
|
1962
1963
|
} catch (err) {
|
|
1964
|
+
// #3861 — a flood-blocked reply is QUEUED, never discarded. Pre-fix this
|
|
1965
|
+
// catch re-wrapped `FLOOD_WAIT_ACTIVE` into a string and threw the answer
|
|
1966
|
+
// away: `retryApiCall` fails fast BEFORE the wire while a long window is
|
|
1967
|
+
// open, so during a multi-hour ban EVERY answer the agent composed for the
|
|
1968
|
+
// operator was lost — the one content class the durable outbox did not
|
|
1969
|
+
// cover. The undelivered text goes into the same outbox the sweep drains
|
|
1970
|
+
// (it defers while the window is open, #3854) under a content-derived
|
|
1971
|
+
// nonce, so a caller retry cannot double-deliver. `cosmetic` sends are
|
|
1972
|
+
// still dropped by `queueFloodBlockedReply`; this site is `critical`.
|
|
1973
|
+
const queued = queueFloodBlockedReply({
|
|
1974
|
+
err,
|
|
1975
|
+
chatId: chat_id,
|
|
1976
|
+
threadId: threadId ?? null,
|
|
1977
|
+
// Partial failure: queue only the chunks that never landed, so the user
|
|
1978
|
+
// does not receive the already-delivered prefix a second time.
|
|
1979
|
+
text: sentIds.length === 0 ? text : chunks.slice(sentIds.length).join('\n\n'),
|
|
1980
|
+
priorityClass: 'critical',
|
|
1981
|
+
originChatId: turn?.sessionChatId ?? null,
|
|
1982
|
+
originThreadId: turn?.sessionThreadId ?? null,
|
|
1983
|
+
})
|
|
1984
|
+
if (queued != null) {
|
|
1985
|
+
process.stderr.write(
|
|
1986
|
+
`telegram gateway: reply flood-deferred — queued to outbox nonce=${queued.turnNonce} ` +
|
|
1987
|
+
`chat=${chat_id} chars=${text.length} sent=${sentIds.length}/${chunks.length}\n`,
|
|
1988
|
+
)
|
|
1989
|
+
// The obligation is DISCHARGED by the durable enqueue. Without this the
|
|
1990
|
+
// obligation tracker keeps re-prompting the agent to reply every few
|
|
1991
|
+
// minutes for the whole ban — dozens of forced turns it physically
|
|
1992
|
+
// cannot satisfy (and pre-#3861 each attempt destroyed its own answer).
|
|
1993
|
+
if (turn != null && isSubstantiveFinalReply({ text, disableNotification: modelDisableNotification })) {
|
|
1994
|
+
closeObligationOnSubstantiveReply(args, turn, replyRoutedOriginTurn)
|
|
1995
|
+
}
|
|
1996
|
+
// (the `finally` below stops the typing loop on this return path too)
|
|
1997
|
+
return { content: [{ type: 'text', text: queued.notice }] }
|
|
1998
|
+
}
|
|
1963
1999
|
const msg = err instanceof Error ? err.message : String(err)
|
|
1964
2000
|
throw new Error(`reply failed after ${sentIds.length} of ${chunks.length} chunk(s) sent: ${msg}`)
|
|
1965
2001
|
} finally {
|
|
@@ -45,6 +45,11 @@ import {
|
|
|
45
45
|
import { isShownBlock } from '../shown-ledger.js'
|
|
46
46
|
import { resolveSubagentOriginTurnKey } from '../registry/subagents-schema.js'
|
|
47
47
|
import { createRetryApiCall, retryWithThreadFallback } from '../retry-api-call.js'
|
|
48
|
+
import {
|
|
49
|
+
floodStatePath,
|
|
50
|
+
makeFloodWaitProbe,
|
|
51
|
+
makeFloodWaitRecorder,
|
|
52
|
+
} from '../flood-circuit-breaker.js'
|
|
48
53
|
|
|
49
54
|
export interface OutboxSweepDeps {
|
|
50
55
|
/** Deliver `text` to the chat. Resolves to the primary message id (best-effort). */
|
|
@@ -67,12 +72,39 @@ export interface OutboxSweepDeps {
|
|
|
67
72
|
now?: () => number
|
|
68
73
|
log?: (line: string) => void
|
|
69
74
|
quietMs?: number
|
|
75
|
+
/**
|
|
76
|
+
* Remaining ms of a KNOWN-OPEN Telegram flood window, or 0 when none — the
|
|
77
|
+
* SAME persisted breaker state (`flood-wait.json`) `robustApiCall` consults
|
|
78
|
+
* (`makeFloodWaitProbe`). When it reports an open window the sweep does not
|
|
79
|
+
* scan, claim, or send at all.
|
|
80
|
+
*
|
|
81
|
+
* Why the sweep needs its own check on top of the retry policy's: the retry
|
|
82
|
+
* policy only short-circuits windows LONGER than its in-process sleep ceiling
|
|
83
|
+
* (120s), and even then it does so per-CALL — it still costs a claim/release
|
|
84
|
+
* cycle and a log line every tick. The outbox is a safety net with no latency
|
|
85
|
+
* SLA; deferring the whole sweep while ANY window is open is strictly correct
|
|
86
|
+
* and is the only thing that makes "does not hit the wire" true.
|
|
87
|
+
*
|
|
88
|
+
* FAILS OPEN (a throwing probe ⇒ 0) for the same reason `makeFloodWaitProbe`
|
|
89
|
+
* does: a broken marker file must never permanently strand the outbox.
|
|
90
|
+
*/
|
|
91
|
+
floodWaitRemainingMs?: () => number
|
|
70
92
|
}
|
|
71
93
|
|
|
72
94
|
export interface OutboxSweepSummary {
|
|
73
95
|
scanned: number
|
|
74
96
|
delivered: number
|
|
75
97
|
skipped: number
|
|
98
|
+
/**
|
|
99
|
+
* Records whose send THREW this sweep (claim released, retried next tick).
|
|
100
|
+
* The tick-level backoff keys off this: a 5s fixed retry against a failing
|
|
101
|
+
* Telegram is how the 2026-07-27 sweep hit an open 4.4h ban 228 times.
|
|
102
|
+
*/
|
|
103
|
+
sendFailures: number
|
|
104
|
+
/** True when the whole sweep was skipped because a flood window is open. */
|
|
105
|
+
floodDeferred?: boolean
|
|
106
|
+
/** Remaining ms of the window that caused `floodDeferred`. */
|
|
107
|
+
floodRemainingMs?: number
|
|
76
108
|
}
|
|
77
109
|
|
|
78
110
|
/**
|
|
@@ -81,7 +113,20 @@ export interface OutboxSweepSummary {
|
|
|
81
113
|
export async function sweepOutbox(deps: OutboxSweepDeps): Promise<OutboxSweepSummary> {
|
|
82
114
|
const now = deps.now?.() ?? Date.now()
|
|
83
115
|
const log = deps.log ?? (() => {})
|
|
84
|
-
const summary: OutboxSweepSummary = { scanned: 0, delivered: 0, skipped: 0 }
|
|
116
|
+
const summary: OutboxSweepSummary = { scanned: 0, delivered: 0, skipped: 0, sendFailures: 0 }
|
|
117
|
+
|
|
118
|
+
// #3853 — do not sweep INTO a known-open flood window. Every record is
|
|
119
|
+
// preserved on disk and re-scanned on the next tick after the window closes,
|
|
120
|
+
// so this loses no delivery; issuing the send would only feed the ban.
|
|
121
|
+
const remainingMs = probeFloodWindow(deps.floodWaitRemainingMs)
|
|
122
|
+
if (remainingMs > 0) {
|
|
123
|
+
summary.floodDeferred = true
|
|
124
|
+
summary.floodRemainingMs = remainingMs
|
|
125
|
+
// NOT logged here: a 4.4h ban is ~3181 ticks, and one line per tick just
|
|
126
|
+
// moves the flood from the wire to the disk. `startOutboxSweep` logs the
|
|
127
|
+
// deferral at most once per DEFER_LOG_INTERVAL_MS.
|
|
128
|
+
return summary
|
|
129
|
+
}
|
|
85
130
|
|
|
86
131
|
// Re-queue crashed claims first (claim-then-crash before send).
|
|
87
132
|
reclaimStaleSending(deps.stateDir, now)
|
|
@@ -90,9 +135,27 @@ export async function sweepOutbox(deps: OutboxSweepDeps): Promise<OutboxSweepSum
|
|
|
90
135
|
if (pending.length === 0) return summary
|
|
91
136
|
const deliveredNonces = readDeliveredNonces(deps.stateDir)
|
|
92
137
|
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
138
|
+
// #3861 — deliver in CAPTURE order. `listPendingRecords` returns readdir
|
|
139
|
+
// order, which is filesystem-dependent (hash order on most Linux filesystems,
|
|
140
|
+
// i.e. effectively the nonce's hash). That was harmless while the outbox held
|
|
141
|
+
// at most one stray handback per turn; it is not harmless now that a
|
|
142
|
+
// multi-hour flood window can park a whole conversation's worth of queued
|
|
143
|
+
// replies, which would then land shuffled. Sort by `createdAt`, tie-broken by
|
|
144
|
+
// nonce so the order is total and deterministic.
|
|
145
|
+
const records = pending
|
|
146
|
+
.map((fileName) => readOutboxRecord(fileName, deps.stateDir))
|
|
147
|
+
.filter((r): r is OutboxRecord => r != null)
|
|
148
|
+
.sort((a, b) =>
|
|
149
|
+
a.createdAt !== b.createdAt
|
|
150
|
+
? a.createdAt - b.createdAt
|
|
151
|
+
: a.turnNonce < b.turnNonce
|
|
152
|
+
? -1
|
|
153
|
+
: a.turnNonce > b.turnNonce
|
|
154
|
+
? 1
|
|
155
|
+
: 0,
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
for (const record of records) {
|
|
96
159
|
summary.scanned++
|
|
97
160
|
|
|
98
161
|
// Resolve destination (anchor → registry chain → per-session origin). Fails
|
|
@@ -209,12 +272,24 @@ export async function sweepOutbox(deps: OutboxSweepDeps): Promise<OutboxSweepSum
|
|
|
209
272
|
// preserved on disk; no loss.
|
|
210
273
|
releaseClaim(record.turnNonce, deps.stateDir)
|
|
211
274
|
summary.skipped++
|
|
275
|
+
summary.sendFailures++
|
|
212
276
|
log(`outbox-sweep: send failed nonce=${record.turnNonce}: ${(err as Error).message} — will retry\n`)
|
|
213
277
|
}
|
|
214
278
|
}
|
|
215
279
|
return summary
|
|
216
280
|
}
|
|
217
281
|
|
|
282
|
+
/** Read the flood probe, failing OPEN on any throw. */
|
|
283
|
+
function probeFloodWindow(probe: (() => number) | undefined): number {
|
|
284
|
+
if (probe == null) return 0
|
|
285
|
+
try {
|
|
286
|
+
const ms = probe()
|
|
287
|
+
return Number.isFinite(ms) && ms > 0 ? ms : 0
|
|
288
|
+
} catch {
|
|
289
|
+
return 0
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
|
|
218
293
|
/**
|
|
219
294
|
* Parse a registry `turn_key` (`chatId:threadId`, `_` → null thread) into a
|
|
220
295
|
* routable chat. Exported for the gateway wiring + tests.
|
|
@@ -230,6 +305,59 @@ export function chatFromTurnKey(turnKey: string): { chatId: string; threadId: nu
|
|
|
230
305
|
/** How often the sweep ticks (aligned with the delivery-confirm sweep cadence). */
|
|
231
306
|
export const OUTBOX_SWEEP_INTERVAL_MS = 5_000
|
|
232
307
|
|
|
308
|
+
/** Ceiling on the exponential send-failure backoff. */
|
|
309
|
+
export const OUTBOX_SWEEP_BACKOFF_MAX_MS = 5 * 60_000
|
|
310
|
+
|
|
311
|
+
/**
|
|
312
|
+
* Minimum gap between two "flood window still open" log lines. A 4.4h ban is
|
|
313
|
+
* ~3181 ticks; logging each one just relocates the flood from the wire to the
|
|
314
|
+
* disk, so the deferral is reported once a minute instead.
|
|
315
|
+
*/
|
|
316
|
+
export const OUTBOX_SWEEP_DEFER_LOG_INTERVAL_MS = 60_000
|
|
317
|
+
|
|
318
|
+
/**
|
|
319
|
+
* Exponential backoff over the fixed 5s sweep tick (#3853).
|
|
320
|
+
*
|
|
321
|
+
* The tick interval alone is not a retry policy. On 2026-07-27 the sweep held
|
|
322
|
+
* ONE undeliverable record and re-issued its `sendMessage` every 5s for the
|
|
323
|
+
* whole 4.4h ban — 228 requests into a window the breaker already knew was
|
|
324
|
+
* open, each one answered with the same counting-down `retry_after`. The flood
|
|
325
|
+
* probe (above) closes the case where the breaker HAS a window recorded; this
|
|
326
|
+
* closes the case where it does not — a persistent failure of any other kind
|
|
327
|
+
* must not be re-driven at 12 requests/minute forever.
|
|
328
|
+
*
|
|
329
|
+
* Pure and clock-injected so the schedule is asserted in tests rather than
|
|
330
|
+
* inferred from timer behaviour.
|
|
331
|
+
*/
|
|
332
|
+
export function createSweepBackoff(cfg: { baseMs?: number; maxMs?: number } = {}): {
|
|
333
|
+
ready: (now: number) => boolean
|
|
334
|
+
noteFailure: (now: number) => number
|
|
335
|
+
noteSuccess: () => void
|
|
336
|
+
failures: () => number
|
|
337
|
+
} {
|
|
338
|
+
const baseMs = cfg.baseMs ?? OUTBOX_SWEEP_INTERVAL_MS
|
|
339
|
+
const maxMs = cfg.maxMs ?? OUTBOX_SWEEP_BACKOFF_MAX_MS
|
|
340
|
+
let failures = 0
|
|
341
|
+
let nextAllowedAt = 0
|
|
342
|
+
return {
|
|
343
|
+
ready: (now) => now >= nextAllowedAt,
|
|
344
|
+
noteFailure: (now) => {
|
|
345
|
+
failures++
|
|
346
|
+
// 5s, 10s, 20s, … capped. `failures - 1` so the FIRST failure still
|
|
347
|
+
// retries at the normal tick cadence — a one-off transient send error
|
|
348
|
+
// should not delay a real answer.
|
|
349
|
+
const delay = Math.min(maxMs, baseMs * Math.pow(2, failures - 1))
|
|
350
|
+
nextAllowedAt = now + delay
|
|
351
|
+
return delay
|
|
352
|
+
},
|
|
353
|
+
noteSuccess: () => {
|
|
354
|
+
failures = 0
|
|
355
|
+
nextAllowedAt = 0
|
|
356
|
+
},
|
|
357
|
+
failures: () => failures,
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
|
|
233
361
|
/**
|
|
234
362
|
* Own the heartbeat-tick outbox sweep entirely, keeping the timer / chunking /
|
|
235
363
|
* turn-key parse / dedup / registry-chain wiring OUT of `gateway.ts` (the line
|
|
@@ -336,21 +464,43 @@ export function startOutboxSweep(deps: {
|
|
|
336
464
|
text: string,
|
|
337
465
|
) => OutboxDeliveryMarkup | undefined
|
|
338
466
|
log?: (line: string) => void
|
|
467
|
+
/** Test seam — override the flood probe (default: the persisted breaker). */
|
|
468
|
+
floodWaitRemainingMs?: () => number
|
|
469
|
+
/** Test seam — override the tick backoff. */
|
|
470
|
+
backoff?: ReturnType<typeof createSweepBackoff>
|
|
339
471
|
}): ReturnType<typeof setInterval> | undefined {
|
|
340
472
|
if (!deps.isGatewayMain || process.env.SWITCHROOM_TG_OUTBOX_DELIVERY === '0') return undefined
|
|
341
|
-
|
|
473
|
+
// #3853 — the sweep is a full outbound machine and must consult the SAME
|
|
474
|
+
// persisted flood breaker every other outbound path does. Before this it was
|
|
475
|
+
// the one `createRetryApiCall` wiring with neither hook: it could not see an
|
|
476
|
+
// open ban (no `floodWaitRemainingMs`) and could not record one it caused (no
|
|
477
|
+
// `onFloodWait`). `scripts/check-retry-flood-hooks.mjs` now fails lint if any
|
|
478
|
+
// wiring omits either.
|
|
479
|
+
const floodProbe =
|
|
480
|
+
deps.floodWaitRemainingMs ?? makeFloodWaitProbe(floodStatePath(deps.stateDir))
|
|
481
|
+
const recordFloodWait = makeFloodWaitRecorder(floodStatePath(deps.stateDir))
|
|
482
|
+
const retry = createRetryApiCall({
|
|
483
|
+
log: deps.log,
|
|
484
|
+
floodWaitRemainingMs: floodProbe,
|
|
485
|
+
onFloodWait: (retryAfterSec) => recordFloodWait(retryAfterSec),
|
|
486
|
+
})
|
|
342
487
|
const send = createOutboxSend({
|
|
343
488
|
getBot: deps.getBot,
|
|
344
489
|
retry,
|
|
345
490
|
...(deps.resolveReplyMarkup != null ? { resolveReplyMarkup: deps.resolveReplyMarkup } : {}),
|
|
346
491
|
})
|
|
492
|
+
const backoff = deps.backoff ?? createSweepBackoff()
|
|
493
|
+
let lastDeferLogAt = 0
|
|
347
494
|
const tick = () => {
|
|
348
495
|
const bot = deps.getBot()
|
|
349
496
|
if (bot == null) return
|
|
497
|
+
const now = Date.now()
|
|
498
|
+
if (!backoff.ready(now)) return
|
|
350
499
|
void sweepOutbox({
|
|
351
500
|
stateDir: deps.stateDir,
|
|
352
501
|
log: deps.log,
|
|
353
502
|
send,
|
|
503
|
+
floodWaitRemainingMs: floodProbe,
|
|
354
504
|
textAlreadyDelivered: (chatId, threadId, text) => deps.dedupCheck(chatId, threadId ?? undefined, text),
|
|
355
505
|
registryChainLookup: (taskId) => {
|
|
356
506
|
const db = deps.getTurnsDb()
|
|
@@ -358,7 +508,34 @@ export function startOutboxSweep(deps: {
|
|
|
358
508
|
const turnKey = resolveSubagentOriginTurnKey(db, taskId)
|
|
359
509
|
return turnKey == null ? null : chatFromTurnKey(turnKey)
|
|
360
510
|
},
|
|
361
|
-
})
|
|
511
|
+
})
|
|
512
|
+
.then((summary) => {
|
|
513
|
+
// A flood-deferred sweep is neither success nor failure: it never
|
|
514
|
+
// reached the wire, so it must not clear a backoff earned by real
|
|
515
|
+
// failures, and it must not deepen one either.
|
|
516
|
+
if (summary.floodDeferred === true) {
|
|
517
|
+
const at = Date.now()
|
|
518
|
+
if (at - lastDeferLogAt >= OUTBOX_SWEEP_DEFER_LOG_INTERVAL_MS) {
|
|
519
|
+
lastDeferLogAt = at
|
|
520
|
+
deps.log?.(
|
|
521
|
+
`outbox-sweep: deferred — Telegram flood window still open for ` +
|
|
522
|
+
`${Math.ceil((summary.floodRemainingMs ?? 0) / 1000)}s; not issuing any ` +
|
|
523
|
+
`send (would feed the ban)\n`,
|
|
524
|
+
)
|
|
525
|
+
}
|
|
526
|
+
return
|
|
527
|
+
}
|
|
528
|
+
if (summary.sendFailures > 0) {
|
|
529
|
+
const delay = backoff.noteFailure(Date.now())
|
|
530
|
+
deps.log?.(
|
|
531
|
+
`outbox-sweep: ${summary.sendFailures} send failure(s) — backing off ` +
|
|
532
|
+
`${Math.round(delay / 1000)}s before the next sweep (attempt ${backoff.failures()})\n`,
|
|
533
|
+
)
|
|
534
|
+
} else {
|
|
535
|
+
backoff.noteSuccess()
|
|
536
|
+
}
|
|
537
|
+
})
|
|
538
|
+
.catch((err) => deps.log?.(`outbox-sweep: tick failed: ${(err as Error).message}\n`))
|
|
362
539
|
}
|
|
363
540
|
const timer = setInterval(tick, OUTBOX_SWEEP_INTERVAL_MS)
|
|
364
541
|
timer.unref?.()
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* silent status-pin fires. The pin call passes `disable_notification: true`,
|
|
6
6
|
* which kills the PUSH notification but NOT the in-chat service message — so we
|
|
7
7
|
* delete that service message as it lands, but ONLY for pins we own (tracked in
|
|
8
|
-
* `
|
|
8
|
+
* `statusPinClaims`). Manual/operator pins are never silent and are never
|
|
9
9
|
* touched. Only silent status pins reach here as an OUR-pin match, so the
|
|
10
10
|
* ownership check is the guard.
|
|
11
11
|
*
|
|
@@ -21,13 +21,12 @@
|
|
|
21
21
|
|
|
22
22
|
import type { Context, Filter } from 'grammy'
|
|
23
23
|
import { pinnedMessageIsOurs, type TrackedStatusPin } from './status-pin-store.js'
|
|
24
|
-
import type {
|
|
24
|
+
import type { StatusPinClaim } from './status-pin-retarget.js'
|
|
25
25
|
|
|
26
26
|
export interface PinnedMessageHandlerDeps {
|
|
27
|
-
/** Live pinKey→
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
statusPinChatIds: ReadonlyMap<string, string>
|
|
27
|
+
/** Live pinKey→claim registry of pins we own. ONE record per key (#3809):
|
|
28
|
+
* the chat id is a field of the claim, not a parallel map that can diverge. */
|
|
29
|
+
statusPinClaims: ReadonlyMap<string, StatusPinClaim>
|
|
31
30
|
/**
|
|
32
31
|
* Delete the pin service message. Bound in gateway.ts through
|
|
33
32
|
* `robustApiCall(() => lockedBot.api.deleteMessage(...))` so the raw Bot API
|
|
@@ -50,16 +49,13 @@ export async function handlePinnedMessage(
|
|
|
50
49
|
// Chat-scoped ownership (see pinnedMessageIsOurs): the match requires BOTH
|
|
51
50
|
// the messageId AND that the tracked entry lives in THIS chat, so a pin id
|
|
52
51
|
// colliding across chats can't delete a foreign (e.g. operator-manual) pin
|
|
53
|
-
// notice.
|
|
54
|
-
//
|
|
55
|
-
const trackedPins = (): TrackedStatusPin[] =>
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
}
|
|
61
|
-
return out
|
|
62
|
-
}
|
|
52
|
+
// notice. Each claim carries its own chat id (#3809), so the match can never
|
|
53
|
+
// silently degrade to messageId-only because a companion map went missing.
|
|
54
|
+
const trackedPins = (): TrackedStatusPin[] =>
|
|
55
|
+
[...deps.statusPinClaims.values()].map((c) => ({
|
|
56
|
+
chatId: c.chatId,
|
|
57
|
+
messageId: c.messageId,
|
|
58
|
+
}))
|
|
63
59
|
const isOurs = () => pinnedMessageIsOurs(trackedPins(), chatId, pinnedId)
|
|
64
60
|
|
|
65
61
|
if (!isOurs()) {
|
|
@@ -34,7 +34,7 @@
|
|
|
34
34
|
* leg 1 unpin the stale claim, with the row still naming the OLD id;
|
|
35
35
|
* leg 2 pin the new message from a null claim, with its own pending row.
|
|
36
36
|
*
|
|
37
|
-
* Failure semantics are inherited from `
|
|
37
|
+
* Failure semantics are inherited from `executePinLeg`, not re-invented here: a
|
|
38
38
|
* leg-1 result that is NON-NULL means the unpin was never confirmed (#3664
|
|
39
39
|
* Defect B — the old message is provably still pinned and the claim was
|
|
40
40
|
* deliberately retained). Pinning the new message then would leave two pins in
|
|
@@ -42,24 +42,45 @@
|
|
|
42
42
|
* claim is left for the next reconcile / the mid-session reaper / the boot
|
|
43
43
|
* sweep to retry.
|
|
44
44
|
*
|
|
45
|
+
* THE SINGLE PATH (#3831)
|
|
46
|
+
* -----------------------
|
|
47
|
+
* This module is the ONLY decider. `status-pin-driver.ts` used to call
|
|
48
|
+
* `decidePinAction` itself and carry its OWN copy of the expansion above — a
|
|
49
|
+
* copy that recursed without the per-leg persistence, so which of the two
|
|
50
|
+
* implementations ran decided whether the durable row survived. That copy is
|
|
51
|
+
* deleted, and the driver's parameter type (`PinLegAction`) can no longer even
|
|
52
|
+
* express a repin, so a second one cannot grow back.
|
|
53
|
+
*
|
|
45
54
|
* Dependency-free apart from the pure decision, so it is provable in isolation
|
|
46
55
|
* (`telegram-plugin/tests/status-pin-retarget.test.ts`); the gateway owns the
|
|
47
|
-
* wiring of `
|
|
56
|
+
* wiring of `runPin` / `commit`.
|
|
48
57
|
*/
|
|
49
58
|
|
|
50
|
-
import type { DesiredPin, PinState } from '../status-pin.js'
|
|
59
|
+
import type { DesiredPin, PinLegAction, PinState } from '../status-pin.js'
|
|
51
60
|
import { decidePinAction } from '../status-pin.js'
|
|
52
61
|
import type { StatusPinPersistOp, StatusPinStoreFsSeam } from './status-pin-store.js'
|
|
53
62
|
import { reconcileAndPersistStatusPin } from './status-pin-store.js'
|
|
54
63
|
|
|
55
|
-
/**
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
64
|
+
/**
|
|
65
|
+
* ONE in-memory claim for a pin key — the message we believe is pinned, the
|
|
66
|
+
* chat it lives in, and when the claim was FIRST taken (the reaper's TTL age).
|
|
67
|
+
*
|
|
68
|
+
* #3809 — these were three parallel `Map`s (`statusPinState` / `statusPinChatIds`
|
|
69
|
+
* / `statusPinPinnedAt`) written together but able to diverge. A `wk:` claim
|
|
70
|
+
* whose chatId entry went missing was skipped by the mid-session reaper on
|
|
71
|
+
* EVERY pass (it cannot unpin without a chat) and excluded from the durable
|
|
72
|
+
* store-orphan net (which skips keys that DO have an in-memory claim) — so it
|
|
73
|
+
* was permanently unreapable until the next boot, silently. Collapsing the
|
|
74
|
+
* three maps into one record makes the invariant STRUCTURAL: a claim without a
|
|
75
|
+
* chat id cannot be represented, so that gap cannot be reintroduced.
|
|
76
|
+
*/
|
|
77
|
+
export interface StatusPinClaim {
|
|
78
|
+
/** Telegram message_id we believe is pinned for this key. */
|
|
79
|
+
messageId: number
|
|
80
|
+
/** Chat the pin lives in. Never empty — the reconcile refuses an empty chat. */
|
|
81
|
+
chatId: string
|
|
82
|
+
/** Wall-clock ms the claim was FIRST taken (survives a same-id re-pin). */
|
|
83
|
+
pinnedAt: number
|
|
63
84
|
}
|
|
64
85
|
|
|
65
86
|
export interface StatusPinReconcileArgs {
|
|
@@ -72,48 +93,56 @@ export interface StatusPinReconcileArgs {
|
|
|
72
93
|
/** Durable-store binding, or null when persistence is off (STATIC). */
|
|
73
94
|
persist: { path: string; fs: StatusPinStoreFsSeam } | null
|
|
74
95
|
/**
|
|
75
|
-
* Execute
|
|
76
|
-
* `
|
|
96
|
+
* Execute ONE already-decided leg against Telegram (the gateway binds
|
|
97
|
+
* `executePinLeg`). Must never throw — API errors are absorbed inside.
|
|
77
98
|
*/
|
|
78
|
-
runPin: (from: PinState | null
|
|
79
|
-
|
|
99
|
+
runPin: (action: PinLegAction, from: PinState | null) => Promise<PinState | null>
|
|
100
|
+
/** The single claim registry, keyed by pinKey. */
|
|
101
|
+
claims: Map<string, StatusPinClaim>
|
|
102
|
+
/** Injectable clock for the `pinnedAt` stamp (tests). */
|
|
103
|
+
now?: () => number
|
|
80
104
|
}
|
|
81
105
|
|
|
82
106
|
/**
|
|
83
107
|
* Drive `prev → desired` for one pin key, expanding a RETARGET into its two
|
|
84
108
|
* legs. Every other action (pin / unpin / noop) is a single leg, unchanged.
|
|
85
109
|
*
|
|
86
|
-
* Each leg is committed to the
|
|
87
|
-
* never lags the durable row — including the intermediate cleared state
|
|
88
|
-
* a retarget's two legs, which is what lets a concurrent reader (the
|
|
89
|
-
* reaper) see an honest snapshot rather than a claim on an unpinned id.
|
|
110
|
+
* Each leg is committed to the claim registry as it lands, so the in-memory
|
|
111
|
+
* claim never lags the durable row — including the intermediate cleared state
|
|
112
|
+
* between a retarget's two legs, which is what lets a concurrent reader (the
|
|
113
|
+
* `wk:` reaper) see an honest snapshot rather than a claim on an unpinned id.
|
|
90
114
|
*
|
|
91
115
|
* Callers must hold the per-key reconcile lock (`withPinReconcileLock`); `prev`
|
|
92
116
|
* is read once by the caller and both legs run inside that critical section.
|
|
93
117
|
*/
|
|
94
118
|
export async function runStatusPinReconcile(args: StatusPinReconcileArgs): Promise<void> {
|
|
95
|
-
const { pinKey, chatId, prev, desired, persist, runPin,
|
|
119
|
+
const { pinKey, chatId, prev, desired, persist, runPin, claims } = args
|
|
120
|
+
const now = args.now ?? Date.now
|
|
96
121
|
|
|
97
|
-
// Publish a leg's outcome. `pinnedAt` is
|
|
98
|
-
//
|
|
122
|
+
// Publish a leg's outcome into the single claim registry. `pinnedAt` is
|
|
123
|
+
// carried forward when a claim already exists so it keeps the FIRST-taken
|
|
124
|
+
// timestamp; clearing the claim drops it, so a re-pin ages fresh.
|
|
99
125
|
const commit = (next: PinState | null): void => {
|
|
100
126
|
if (next == null) {
|
|
101
|
-
|
|
102
|
-
registries.chatIds.delete(pinKey)
|
|
103
|
-
registries.pinnedAt.delete(pinKey)
|
|
127
|
+
claims.delete(pinKey)
|
|
104
128
|
return
|
|
105
129
|
}
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
if (!registries.pinnedAt.has(pinKey)) registries.pinnedAt.set(pinKey, Date.now())
|
|
130
|
+
const pinnedAt = claims.get(pinKey)?.pinnedAt ?? now()
|
|
131
|
+
claims.set(pinKey, { messageId: next.messageId, chatId, pinnedAt })
|
|
109
132
|
}
|
|
110
133
|
|
|
111
134
|
// One leg, with the persist ordering its action requires. Only a fresh `pin`
|
|
112
135
|
// opens the crash window that persist-BEFORE-pin closes; unpin / noop / re-pin
|
|
113
136
|
// of the same id clear or leave the row and are safe to persist after.
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
137
|
+
//
|
|
138
|
+
// The leg action is decided ONCE, below, and threaded through to both the
|
|
139
|
+
// persist-op mapping and the driver — the driver no longer re-derives it
|
|
140
|
+
// (#3831).
|
|
141
|
+
const runLeg = (
|
|
142
|
+
legAction: PinLegAction,
|
|
143
|
+
from: PinState | null,
|
|
144
|
+
): Promise<PinState | null> => {
|
|
145
|
+
if (persist == null) return runPin(legAction, from)
|
|
117
146
|
const op: StatusPinPersistOp =
|
|
118
147
|
legAction.kind === 'pin'
|
|
119
148
|
? { kind: 'pin', messageId: legAction.messageId }
|
|
@@ -124,21 +153,28 @@ export async function runStatusPinReconcile(args: StatusPinReconcileArgs): Promi
|
|
|
124
153
|
pinKey,
|
|
125
154
|
chatId,
|
|
126
155
|
op,
|
|
127
|
-
applyPin: () => runPin(
|
|
156
|
+
applyPin: () => runPin(legAction, from),
|
|
157
|
+
now: now(),
|
|
128
158
|
})
|
|
129
159
|
}
|
|
130
160
|
|
|
131
161
|
const action = decidePinAction(prev, desired)
|
|
132
162
|
|
|
133
|
-
|
|
134
|
-
|
|
163
|
+
// THE retarget expansion. This is the only copy in the codebase; the driver's
|
|
164
|
+
// parameter type (`PinLegAction`) cannot express a repin, so a second one
|
|
165
|
+
// cannot be written there again (#3831).
|
|
166
|
+
if (action.kind === 'repin') {
|
|
167
|
+
const afterUnpin = await runLeg(
|
|
168
|
+
{ kind: 'unpin', messageId: action.unpinMessageId },
|
|
169
|
+
prev,
|
|
170
|
+
)
|
|
135
171
|
commit(afterUnpin)
|
|
136
172
|
// Never-confirmed unpin (#3664 Defect B): keep the retained claim, skip the
|
|
137
173
|
// pin leg. See the docblock.
|
|
138
174
|
if (afterUnpin != null) return
|
|
139
|
-
commit(await runLeg(
|
|
175
|
+
commit(await runLeg({ kind: 'pin', messageId: action.pinMessageId }, null))
|
|
140
176
|
return
|
|
141
177
|
}
|
|
142
178
|
|
|
143
|
-
commit(await runLeg(
|
|
179
|
+
commit(await runLeg(action, prev))
|
|
144
180
|
}
|