switchroom 0.19.41 → 0.19.43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-scheduler/index.js +23 -3
- package/dist/auth-broker/index.js +54 -9
- package/dist/cli/notion-write-pretool.mjs +23 -3
- package/dist/cli/switchroom.js +1956 -900
- package/dist/host-control/main.js +227 -30
- package/dist/vault/approvals/kernel-server.js +54 -9
- package/dist/vault/broker/server.js +54 -9
- package/package.json +3 -3
- package/profiles/_base/start.sh.hbs +82 -4
- package/telegram-plugin/bridge/bridge.ts +1 -1
- package/telegram-plugin/card-layout.ts +63 -1
- package/telegram-plugin/dist/bridge/bridge.js +4 -1
- package/telegram-plugin/dist/gateway/gateway.js +1318 -693
- package/telegram-plugin/dist/server.js +5 -2
- package/telegram-plugin/flood-429-ledger.ts +5 -3
- package/telegram-plugin/flood-circuit-breaker.ts +11 -3
- package/telegram-plugin/format.ts +223 -13
- package/telegram-plugin/gateway/approval-callback-consume-record.test.ts +132 -0
- package/telegram-plugin/gateway/boot-card.ts +7 -2
- package/telegram-plugin/gateway/flood-reply-queue.ts +5 -0
- package/telegram-plugin/gateway/gateway.ts +56 -58
- package/telegram-plugin/gateway/handback-orphan-recovery.ts +69 -0
- package/telegram-plugin/gateway/handback-preturn-signal.ts +170 -32
- package/telegram-plugin/gateway/ipc-protocol.ts +31 -3
- package/telegram-plugin/gateway/ipc-server.ts +13 -4
- package/telegram-plugin/gateway/liveness-wiring.ts +12 -0
- package/telegram-plugin/gateway/model-command.ts +21 -109
- package/telegram-plugin/gateway/obligation-store.ts +55 -0
- package/telegram-plugin/gateway/outbound-send-path.ts +127 -11
- package/telegram-plugin/gateway/outbox-sweep.ts +371 -61
- package/telegram-plugin/gateway/rollout-status-edit.ts +175 -0
- package/telegram-plugin/gateway/stream-render.ts +62 -0
- package/telegram-plugin/gateway/update-announce.ts +11 -4
- package/telegram-plugin/hooks/audience-classify.d.mts +50 -0
- package/telegram-plugin/hooks/audience-classify.mjs +364 -0
- package/telegram-plugin/hooks/silent-end-interrupt-stop.mjs +86 -3
- package/telegram-plugin/hooks/silent-end-scan.mjs +80 -4
- package/telegram-plugin/outbox.ts +103 -2
- package/telegram-plugin/render/rich-render.ts +10 -2
- package/telegram-plugin/reply-quote.ts +150 -0
- package/telegram-plugin/retry-api-call.ts +254 -11
- package/telegram-plugin/shared/bot-runtime.ts +69 -34
- package/telegram-plugin/shared/gw-trace-gate.ts +14 -2
- package/telegram-plugin/silent-end.ts +34 -2
- package/telegram-plugin/status-no-truncate.ts +31 -16
- package/telegram-plugin/stream-controller.ts +44 -9
- package/telegram-plugin/stream-reply-handler.ts +5 -3
- package/telegram-plugin/tests/boot-card-render.test.ts +2 -1
- package/telegram-plugin/tests/card-budget-never-overflows.test.ts +165 -0
- package/telegram-plugin/tests/card-hard-truncate-entity-safe.test.ts +153 -0
- package/telegram-plugin/tests/card-variants.golden.txt +9 -9
- package/telegram-plugin/tests/flood-reply-queue.test.ts +89 -4
- package/telegram-plugin/tests/format-consistency.test.ts +160 -14
- package/telegram-plugin/tests/framework-fallback-drains-parked.test.ts +134 -0
- package/telegram-plugin/tests/handback-orphan-recovery.test.ts +131 -0
- package/telegram-plugin/tests/handback-preturn-signal.test.ts +212 -17
- package/telegram-plugin/tests/helpers/liveness-wiring-fixture.ts +3 -0
- package/telegram-plugin/tests/nested-indent-idiom.test.ts +162 -0
- package/telegram-plugin/tests/nested-prefix-indent.test.ts +115 -0
- package/telegram-plugin/tests/no-robust-api-call-factory.test.ts +62 -0
- package/telegram-plugin/tests/obligation-store.test.ts +51 -0
- package/telegram-plugin/tests/outbox-audience-3865.test.ts +579 -0
- package/telegram-plugin/tests/outbox-provenance-4141.test.ts +1126 -0
- package/telegram-plugin/tests/outbox-sweep-single-flight.test.ts +138 -0
- package/telegram-plugin/tests/pinned-card-collapse.test.ts +12 -10
- package/telegram-plugin/tests/plain-fallback-4096-cap.test.ts +509 -0
- package/telegram-plugin/tests/reply-quote-wire.test.ts +433 -0
- package/telegram-plugin/tests/retry-grammy-http-error.test.ts +404 -0
- package/telegram-plugin/tests/rollout-narration-edit-socket.test.ts +170 -0
- package/telegram-plugin/tests/rollout-status-edit-retry-policy.test.ts +366 -0
- package/telegram-plugin/tests/rollout-status-edit.test.ts +140 -0
- package/telegram-plugin/tests/rollout-status-wiring.test.ts +40 -7
- package/telegram-plugin/tests/tg-post-logger-error-shape.test.ts +4 -0
- package/telegram-plugin/tests/tg-post-retry-attribution.test.ts +196 -0
- package/telegram-plugin/tests/tool-activity-summary.test.ts +23 -17
- package/telegram-plugin/tool-activity-summary.ts +58 -30
- package/telegram-plugin/uat/scenarios/jtbd-foreground-subagent-activity-dm.test.ts +3 -2
- package/vendor/hindsight-memory/CHANGELOG.md +28 -0
- package/vendor/hindsight-memory/docs/measurements/subagent-volume-gate-3994.md +84 -0
- package/vendor/hindsight-memory/scripts/backfill_transcripts.py +254 -40
- package/vendor/hindsight-memory/scripts/lib/content.py +11 -2
- package/vendor/hindsight-memory/scripts/recall.py +18 -3
- package/vendor/hindsight-memory/scripts/subagent_retain.py +59 -38
- package/vendor/hindsight-memory/scripts/tests/data/replay_volume_gate_3994.py +295 -0
- package/vendor/hindsight-memory/scripts/tests/test_backfill_from_logs.py +109 -0
- package/vendor/hindsight-memory/scripts/tests/test_overlap_tokens.py +135 -0
- package/vendor/hindsight-memory/scripts/tests/test_recall_no_lexical_gate.py +20 -23
- package/vendor/hindsight-memory/scripts/tests/test_recall_query_shaping.py +48 -0
- package/vendor/hindsight-memory/scripts/tests/test_subagent_retain.py +94 -1
- package/vendor/hindsight-memory/scripts/tests/test_subagent_retain_learnings.py +98 -42
|
@@ -26,6 +26,7 @@
|
|
|
26
26
|
*/
|
|
27
27
|
|
|
28
28
|
import {
|
|
29
|
+
OUTBOX_MAX_AGE_MS,
|
|
29
30
|
OUTBOX_QUIET_MS,
|
|
30
31
|
appendDelivered,
|
|
31
32
|
claimRecord,
|
|
@@ -42,9 +43,16 @@ import {
|
|
|
42
43
|
sha256Hex,
|
|
43
44
|
type OutboxRecord,
|
|
44
45
|
} from '../outbox.js'
|
|
46
|
+
import {
|
|
47
|
+
AUDIENCE_INTERNAL,
|
|
48
|
+
formatInternalSuppression,
|
|
49
|
+
formatReplyThrowFraming,
|
|
50
|
+
resolveRecordAudience,
|
|
51
|
+
shouldSuppressForAudience,
|
|
52
|
+
} from '../hooks/audience-classify.mjs'
|
|
45
53
|
import { isShownBlock } from '../shown-ledger.js'
|
|
46
54
|
import { richMessage, isParseEntitiesError } from '../rich-send.js'
|
|
47
|
-
import { splitMarkdownChunks } from '../format.js'
|
|
55
|
+
import { splitMarkdownChunks, splitPlainTextToCap } from '../format.js'
|
|
48
56
|
import { resolveSubagentOriginTurnKey } from '../registry/subagents-schema.js'
|
|
49
57
|
import { createRetryApiCall, retryWithThreadFallback } from '../retry-api-call.js'
|
|
50
58
|
import {
|
|
@@ -124,6 +132,36 @@ export interface OutboxSweepDeps {
|
|
|
124
132
|
* does: a broken marker file must never permanently strand the outbox.
|
|
125
133
|
*/
|
|
126
134
|
floodWaitRemainingMs?: () => number
|
|
135
|
+
/**
|
|
136
|
+
* W1-d (#3865) kill switch for the audience gate. Defaults to the env read
|
|
137
|
+
* `SWITCHROOM_TG_OUTBOX_AUDIENCE_GATE !== '0'` (ON by default; `=0` restores
|
|
138
|
+
* the pre-change leak). Injected rather than read at module scope so the
|
|
139
|
+
* revert-check test can flip it deterministically inside one process — a
|
|
140
|
+
* module-level const would be captured at import and the revert-check could
|
|
141
|
+
* not run at all.
|
|
142
|
+
*/
|
|
143
|
+
audienceGateEnabled?: () => boolean
|
|
144
|
+
/**
|
|
145
|
+
* W1-d (#3865) structured-telemetry sink for a suppressed `'internal'`
|
|
146
|
+
* record. Mirrors `escalateOrphan` (#4104): a SEPARATE, non-chat channel, so
|
|
147
|
+
* the escalation path for internal prose is structurally incapable of
|
|
148
|
+
* becoming a message. Defaults to `log`.
|
|
149
|
+
*/
|
|
150
|
+
escalateInternalSuppression?: (line: string) => void
|
|
151
|
+
/**
|
|
152
|
+
* W1-d follow-up (#4141) kill switch for the reply-throw provenance banner.
|
|
153
|
+
* Defaults to `SWITCHROOM_TG_OUTBOX_PROVENANCE_FRAMING !== '0'` (ON). Same
|
|
154
|
+
* injected-seam reasoning as {@link audienceGateEnabled}: a module-scope env
|
|
155
|
+
* read would be captured at import and the revert-check could not run.
|
|
156
|
+
*/
|
|
157
|
+
provenanceFramingEnabled?: () => boolean
|
|
158
|
+
/**
|
|
159
|
+
* W1-d follow-up (#4141) telemetry sink for a FRAMED delivery. Separate from
|
|
160
|
+
* {@link escalateInternalSuppression} because the two outcomes are different
|
|
161
|
+
* (withheld vs. delivered-with-provenance) and must be greppable apart.
|
|
162
|
+
* Defaults to `log`.
|
|
163
|
+
*/
|
|
164
|
+
logProvenanceFraming?: (line: string) => void
|
|
127
165
|
}
|
|
128
166
|
|
|
129
167
|
export interface OutboxSweepSummary {
|
|
@@ -140,6 +178,18 @@ export interface OutboxSweepSummary {
|
|
|
140
178
|
floodDeferred?: boolean
|
|
141
179
|
/** Remaining ms of the window that caused `floodDeferred`. */
|
|
142
180
|
floodRemainingMs?: number
|
|
181
|
+
/**
|
|
182
|
+
* W1-d (#3865): records withheld from the chat this sweep because their
|
|
183
|
+
* audience is `'internal'`. Counted separately from `skipped` so "we
|
|
184
|
+
* suppressed a leak" is never confused with "we deferred a delivery".
|
|
185
|
+
*/
|
|
186
|
+
audienceSuppressed?: number
|
|
187
|
+
/**
|
|
188
|
+
* W1-d follow-up (#4141): deliveries this sweep that carried the reply-throw
|
|
189
|
+
* provenance banner. These ARE counted in `delivered` — framing is a
|
|
190
|
+
* delivery, not a skip.
|
|
191
|
+
*/
|
|
192
|
+
provenanceFramed?: number
|
|
143
193
|
}
|
|
144
194
|
|
|
145
195
|
/**
|
|
@@ -190,9 +240,66 @@ export async function sweepOutbox(deps: OutboxSweepDeps): Promise<OutboxSweepSum
|
|
|
190
240
|
: 0,
|
|
191
241
|
)
|
|
192
242
|
|
|
243
|
+
const audienceGateEnabled = deps.audienceGateEnabled?.() ??
|
|
244
|
+
process.env.SWITCHROOM_TG_OUTBOX_AUDIENCE_GATE !== '0'
|
|
245
|
+
const escalateInternalSuppression = deps.escalateInternalSuppression ?? log
|
|
246
|
+
const provenanceFramingEnabled = deps.provenanceFramingEnabled?.() ??
|
|
247
|
+
process.env.SWITCHROOM_TG_OUTBOX_PROVENANCE_FRAMING !== '0'
|
|
248
|
+
const logProvenanceFraming = deps.logProvenanceFraming ?? log
|
|
249
|
+
|
|
193
250
|
for (const record of records) {
|
|
194
251
|
summary.scanned++
|
|
195
252
|
|
|
253
|
+
// ── W1-d (#3865): AUDIENCE GATE ─────────────────────────────────────────
|
|
254
|
+
//
|
|
255
|
+
// Placed at ENTRY SELECTION — before routing, before the claim, before any
|
|
256
|
+
// adapter is consulted — and NOT inside the send adapter. That placement is
|
|
257
|
+
// the whole design: W1-b replaces the send adapter wholesale, and a gate
|
|
258
|
+
// living inside the adapter would leave with it. Here, an `'internal'`
|
|
259
|
+
// record simply never becomes a send candidate, whatever is wired below.
|
|
260
|
+
//
|
|
261
|
+
// Terminal, not deferred: we journal the nonce and delete the record, so a
|
|
262
|
+
// second tick has nothing to look at and the decision survives a restart.
|
|
263
|
+
// The escalation is STRUCTURED TELEMETRY on a non-chat channel — internal
|
|
264
|
+
// prose has exactly one exit from this pipeline, and it is not Telegram.
|
|
265
|
+
if (shouldSuppressForAudience(record, { gateEnabled: audienceGateEnabled })) {
|
|
266
|
+
summary.skipped++
|
|
267
|
+
summary.audienceSuppressed = (summary.audienceSuppressed ?? 0) + 1
|
|
268
|
+
const ageMs = now - record.createdAt
|
|
269
|
+
appendDelivered(
|
|
270
|
+
{
|
|
271
|
+
turnNonce: record.turnNonce,
|
|
272
|
+
textSha256: record.textSha256,
|
|
273
|
+
ts: now,
|
|
274
|
+
// The record is TERMINALLY resolved by the sweep machine — the same
|
|
275
|
+
// provenance a delivery would carry, so every exactly-once guard
|
|
276
|
+
// (`backstopAlreadyDelivered`, the turn-flush backstop) also treats
|
|
277
|
+
// this turn as settled and no other machine re-delivers the prose.
|
|
278
|
+
deliverySource: 'sweep',
|
|
279
|
+
replyAlreadyDeliveredThisTurn: record.replyAlreadyDeliveredThisTurn === true,
|
|
280
|
+
audience: AUDIENCE_INTERNAL,
|
|
281
|
+
// The honest terminal state: withheld by design. Recorded as an
|
|
282
|
+
// explicit named outcome rather than as an absence, so suppression
|
|
283
|
+
// reads as telemetry and never as a delivery failure.
|
|
284
|
+
suppressedAudience: AUDIENCE_INTERNAL,
|
|
285
|
+
},
|
|
286
|
+
deps.stateDir,
|
|
287
|
+
)
|
|
288
|
+
clearOutboxRecord(record.turnNonce, deps.stateDir)
|
|
289
|
+
escalateInternalSuppression(
|
|
290
|
+
formatInternalSuppression({
|
|
291
|
+
turnNonce: record.turnNonce,
|
|
292
|
+
turnId: turnIdFromNonce(record.turnNonce),
|
|
293
|
+
chatId: record.chatId,
|
|
294
|
+
textSha256: record.textSha256,
|
|
295
|
+
ageMs,
|
|
296
|
+
source: record.source,
|
|
297
|
+
pastWindow: ageMs > OUTBOX_MAX_AGE_MS,
|
|
298
|
+
}),
|
|
299
|
+
)
|
|
300
|
+
continue
|
|
301
|
+
}
|
|
302
|
+
|
|
196
303
|
// Resolve destination (anchor → registry chain → per-session origin). Fails
|
|
197
304
|
// CLOSED (null → held) rather than routing to an arbitrary chat (F2).
|
|
198
305
|
const resolved = resolveOutboxChat(record, {
|
|
@@ -218,6 +325,22 @@ export async function sweepOutbox(deps: OutboxSweepDeps): Promise<OutboxSweepSum
|
|
|
218
325
|
// progress card for this turn (durable shown-ledger). The invariant forbids
|
|
219
326
|
// a second, out-of-process delivery of an ephemeral-shown block.
|
|
220
327
|
shownLedgerHit: isShownBlock(record.turnNonce, record.text, deps.stateDir),
|
|
328
|
+
// ── W1-d follow-up (#4141): PROVENANCE FRAMING ───────────────────────
|
|
329
|
+
//
|
|
330
|
+
// The residual case #4140 deliberately leaves open: the reply tool threw
|
|
331
|
+
// on a FOREGROUND turn, so the inbound obligation is still OPEN, the
|
|
332
|
+
// record classifies `'user'`, and the captured trailing prose delivers
|
|
333
|
+
// presented as if it were the answer.
|
|
334
|
+
//
|
|
335
|
+
// We do NOT suppress here, and that is the design, not a compromise.
|
|
336
|
+
// Someone is provably still waiting; there is no deterministic signal
|
|
337
|
+
// that separates "working notes" from "the answer re-written as plain
|
|
338
|
+
// prose after the tool errored" (see `audience-classify.mjs`). Dropping
|
|
339
|
+
// the text would trade a visible wrong-content failure for an invisible
|
|
340
|
+
// no-answer one — the exact severity-3 class this whole line of work
|
|
341
|
+
// exists to prevent. So we state the provenance instead, and the human
|
|
342
|
+
// (who can tell the difference, unlike any classifier here) decides.
|
|
343
|
+
provenanceFraming: provenanceFramingEnabled,
|
|
221
344
|
})
|
|
222
345
|
|
|
223
346
|
if (decision.action !== 'send' && decision.action !== 'send-delayed') {
|
|
@@ -293,9 +416,28 @@ export async function sweepOutbox(deps: OutboxSweepDeps): Promise<OutboxSweepSum
|
|
|
293
416
|
// turn) is provable from the journal alone.
|
|
294
417
|
deliverySource: 'sweep',
|
|
295
418
|
replyAlreadyDeliveredThisTurn: record.replyAlreadyDeliveredThisTurn === true,
|
|
419
|
+
// #4141: the durable terminal stamp for a framed delivery. Written on
|
|
420
|
+
// the same line as the delivery it describes, so "what did the user
|
|
421
|
+
// actually see" is answerable from the journal alone after the logs
|
|
422
|
+
// have rotated.
|
|
423
|
+
...(decision.framedProvenance != null
|
|
424
|
+
? { framedProvenance: decision.framedProvenance }
|
|
425
|
+
: {}),
|
|
296
426
|
},
|
|
297
427
|
deps.stateDir,
|
|
298
428
|
)
|
|
429
|
+
if (decision.framedProvenance != null) {
|
|
430
|
+
summary.provenanceFramed = (summary.provenanceFramed ?? 0) + 1
|
|
431
|
+
logProvenanceFraming(
|
|
432
|
+
formatReplyThrowFraming({
|
|
433
|
+
turnNonce: record.turnNonce,
|
|
434
|
+
turnId: turnIdFromNonce(record.turnNonce),
|
|
435
|
+
chatId: resolvedChat.chatId,
|
|
436
|
+
textSha256: record.textSha256,
|
|
437
|
+
source: record.source,
|
|
438
|
+
}),
|
|
439
|
+
)
|
|
440
|
+
}
|
|
299
441
|
// Persist parity (mirrors outbound-send-path.ts + backstop-delivery.ts):
|
|
300
442
|
// record the delivered chunk ids + texts to history so a net-delivered
|
|
301
443
|
// answer is not silently absent from history.db. Best-effort — a missing
|
|
@@ -330,6 +472,20 @@ export async function sweepOutbox(deps: OutboxSweepDeps): Promise<OutboxSweepSum
|
|
|
330
472
|
return summary
|
|
331
473
|
}
|
|
332
474
|
|
|
475
|
+
/**
|
|
476
|
+
* W1-d (#3865): recover the gateway `turnId` from an envelope-derived nonce.
|
|
477
|
+
*
|
|
478
|
+
* `deriveTurnNonce` builds `${chatId}:${threadId||'_'}#${messageId}` — byte-
|
|
479
|
+
* identical to the gateway's `deriveTurnId` — so an envelope-bearing nonce IS
|
|
480
|
+
* the turn id and `extractTurnId` in `src/fleet-health/detect.ts` can join the
|
|
481
|
+
* suppression telemetry back to the turn record. The sha256 fallback nonce
|
|
482
|
+
* (envelope-less anchors) carries no turn id; return null rather than emit a
|
|
483
|
+
* hash that looks like one.
|
|
484
|
+
*/
|
|
485
|
+
export function turnIdFromNonce(nonce: string): string | null {
|
|
486
|
+
return /^-?\d+:[^#]*#\d+$/.test(nonce) ? nonce : null
|
|
487
|
+
}
|
|
488
|
+
|
|
333
489
|
/** Read the flood probe, failing OPEN on any throw. */
|
|
334
490
|
function probeFloodWindow(probe: (() => number) | undefined): number {
|
|
335
491
|
if (probe == null) return 0
|
|
@@ -460,22 +616,91 @@ type OutboxSendBot = {
|
|
|
460
616
|
* caught here — it propagates so `retryWithThreadFallback` and the sweep's
|
|
461
617
|
* claim-release retry handle it, exactly as before.
|
|
462
618
|
*/
|
|
619
|
+
/**
|
|
620
|
+
* #4043: the plain fallback is RE-CAPPED to `PLAIN_TEXT_MAX_CHARS` (4096). The
|
|
621
|
+
* caller splits at the RICH cap (32768 — `splitMarkdownChunks`' default), so an
|
|
622
|
+
* oversized chunk that parse-rejected used to be re-sent to the plain endpoint
|
|
623
|
+
* whole, get `message is too long`, throw, release the claim, and retry on the
|
|
624
|
+
* NEXT tick forever: the record could never be delivered and never cleared.
|
|
625
|
+
* Returns one entry per message actually landed (the plain fallback can land
|
|
626
|
+
* several) so the caller's history / journal bookkeeping stays aligned with the
|
|
627
|
+
* wire. A trailing `reply_markup` (the Listen button) rides the FINAL piece
|
|
628
|
+
* only, exactly as it rides the final chunk.
|
|
629
|
+
*/
|
|
630
|
+
/**
|
|
631
|
+
* Resume state for ONE chunk, owned by the caller and therefore SURVIVING a
|
|
632
|
+
* re-invocation of the send closure.
|
|
633
|
+
*
|
|
634
|
+
* The caller wraps this whole function — the multi-send plain-fallback loop
|
|
635
|
+
* included — in `retryWithThreadFallback(deps.retry, …)`, and
|
|
636
|
+
* `createRetryApiCall` re-invokes the closure it was given after a flood wait
|
|
637
|
+
* (`../retry-api-call.ts`). Without resume state, a 429 on plain piece 4 of 5
|
|
638
|
+
* meant the retry re-ran the closure from the top: the rich send
|
|
639
|
+
* DETERMINISTICALLY parse-rejects again and pieces 1-3 are re-delivered to the
|
|
640
|
+
* user's chat — repeating every attempt until the whole loop happens to
|
|
641
|
+
* succeed. A mid-loop 429 is likely precisely in the flood conditions the
|
|
642
|
+
* sweep runs under, so this is the common case, not the exotic one.
|
|
643
|
+
*
|
|
644
|
+
* Carrying landed-piece state (rather than wrapping each `sendMessage` in its
|
|
645
|
+
* own nested `deps.retry`) is what makes the send IDEMPOTENT under ANY
|
|
646
|
+
* re-invocation — the retry policy's flood re-drive AND
|
|
647
|
+
* `retryWithThreadFallback`'s thread-drop re-drive — instead of only the one
|
|
648
|
+
* we thought of. Nesting a retry inside the outer retry would instead multiply
|
|
649
|
+
* the attempt and flood-sleep budgets (maxRetries²) at exactly the moment the
|
|
650
|
+
* policy exists to issue LESS, and would still re-send pieces 1-3 whenever the
|
|
651
|
+
* inner budget was exhausted and the error bubbled to the outer wrapper.
|
|
652
|
+
*/
|
|
653
|
+
interface ChunkSendState {
|
|
654
|
+
/** The rich send already parse-rejected — skip straight to the plain loop. */
|
|
655
|
+
parseRejected: boolean
|
|
656
|
+
/** Plain pieces already ON THE WIRE, in order. Never re-sent. */
|
|
657
|
+
landed: Array<{ message_id?: number; text: string }>
|
|
658
|
+
}
|
|
659
|
+
|
|
660
|
+
function newChunkSendState(): ChunkSendState {
|
|
661
|
+
return { parseRejected: false, landed: [] }
|
|
662
|
+
}
|
|
663
|
+
|
|
463
664
|
async function sendChunkRich(
|
|
464
665
|
bot: OutboxSendBot,
|
|
465
666
|
chatId: string,
|
|
466
667
|
chunk: string,
|
|
467
668
|
opts: object,
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
669
|
+
state: ChunkSendState = newChunkSendState(),
|
|
670
|
+
): Promise<Array<{ message_id?: number; text: string }>> {
|
|
671
|
+
// A re-invocation after the rich send already parse-rejected must NOT
|
|
672
|
+
// re-issue it: the reject is deterministic (same body, same renderer), so
|
|
673
|
+
// the only thing another attempt buys is one more 400 against a
|
|
674
|
+
// flood-sensitive endpoint.
|
|
675
|
+
if (!state.parseRejected) {
|
|
676
|
+
try {
|
|
677
|
+
// allow-raw-bot-api: caller (createOutboxSend) invokes this inside retryWithThreadFallback + createRetryApiCall
|
|
678
|
+
const sent = await bot.api.sendRichMessage(chatId, richMessage(chunk), opts)
|
|
679
|
+
return [{ message_id: sent?.message_id, text: chunk }]
|
|
680
|
+
} catch (err) {
|
|
681
|
+
if (!isParseEntitiesError(err)) throw err
|
|
682
|
+
state.parseRejected = true
|
|
476
683
|
}
|
|
477
|
-
throw err
|
|
478
684
|
}
|
|
685
|
+
|
|
686
|
+
const pieces = splitPlainTextToCap(chunk)
|
|
687
|
+
const withoutMarkup = { ...(opts as Record<string, unknown>) }
|
|
688
|
+
delete withoutMarkup.reply_markup
|
|
689
|
+
// Resume at the first piece that has NOT landed. On a first invocation
|
|
690
|
+
// `landed` is empty and this is the plain 0..n loop.
|
|
691
|
+
for (let p = state.landed.length; p < pieces.length; p++) {
|
|
692
|
+
const isFinalPiece = p === pieces.length - 1
|
|
693
|
+
// allow-raw-bot-api: parse-reject plain fallback, still inside the sweep's retryWithThreadFallback wrapper
|
|
694
|
+
const sent = await bot.api.sendMessage(
|
|
695
|
+
chatId,
|
|
696
|
+
pieces[p],
|
|
697
|
+
isFinalPiece ? opts : withoutMarkup,
|
|
698
|
+
)
|
|
699
|
+
// Recorded BEFORE the next iteration can throw, so a mid-loop failure
|
|
700
|
+
// leaves the state pointing at the first UNSENT piece.
|
|
701
|
+
state.landed.push({ message_id: sent?.message_id, text: pieces[p] })
|
|
702
|
+
}
|
|
703
|
+
return state.landed
|
|
479
704
|
}
|
|
480
705
|
|
|
481
706
|
/**
|
|
@@ -521,6 +746,10 @@ export function createOutboxSend(deps: {
|
|
|
521
746
|
for (let idx = 0; idx < chunks.length; idx++) {
|
|
522
747
|
const chunk = chunks[idx]
|
|
523
748
|
const isLast = idx === chunks.length - 1
|
|
749
|
+
// Per-chunk resume state, OUTSIDE the closure the retry policy
|
|
750
|
+
// re-invokes: a flood wait mid plain-fallback loop must resume at the
|
|
751
|
+
// first unsent piece, not re-deliver the ones already in the chat.
|
|
752
|
+
const chunkState = newChunkSendState()
|
|
524
753
|
const res = await retryWithThreadFallback(
|
|
525
754
|
deps.retry,
|
|
526
755
|
(tid) => {
|
|
@@ -528,12 +757,17 @@ export function createOutboxSend(deps: {
|
|
|
528
757
|
// Button on the LAST chunk only (final visible message).
|
|
529
758
|
const opts =
|
|
530
759
|
isLast && replyMarkup != null ? { ...base, reply_markup: replyMarkup } : base
|
|
531
|
-
return sendChunkRich(bot, chatId, chunk, opts)
|
|
760
|
+
return sendChunkRich(bot, chatId, chunk, opts, chunkState)
|
|
532
761
|
},
|
|
533
762
|
{ threadId: threadId ?? undefined, chat_id: chatId, verb: 'outbox-sweep.sendMessage' },
|
|
534
763
|
)
|
|
535
|
-
|
|
536
|
-
|
|
764
|
+
// One entry per message that actually landed — the #4043 plain fallback
|
|
765
|
+
// can split ONE rich chunk into several 4096-capped plain messages, and
|
|
766
|
+
// history/backstop bookkeeping must reflect the wire, not the pre-split
|
|
767
|
+
// chunk.
|
|
768
|
+
for (const sent of res ?? []) {
|
|
769
|
+
if (sent.message_id != null) landed.push({ messageId: sent.message_id, text: sent.text })
|
|
770
|
+
}
|
|
537
771
|
}
|
|
538
772
|
return {
|
|
539
773
|
messageId: landed.length > 0 ? landed[landed.length - 1]!.messageId : undefined,
|
|
@@ -542,6 +776,98 @@ export function createOutboxSend(deps: {
|
|
|
542
776
|
}
|
|
543
777
|
}
|
|
544
778
|
|
|
779
|
+
/**
|
|
780
|
+
* Build the heartbeat tick that {@link startOutboxSweep} hands to `setInterval`,
|
|
781
|
+
* with the SINGLE-FLIGHT guard (#3864).
|
|
782
|
+
*
|
|
783
|
+
* The tick fires every `OUTBOX_SWEEP_INTERVAL_MS` and the sweep it starts is
|
|
784
|
+
* `void`-ed, so a sweep that outlives its interval used to have the NEXT tick
|
|
785
|
+
* start a second, concurrent sweep body against the same outbox directory. The
|
|
786
|
+
* per-record rename mutex (`claimRecord`, `../outbox.ts`) keeps that from
|
|
787
|
+
* double-sending a single record, but it is the only thing that does: the two
|
|
788
|
+
* bodies still race the same journal / dedup / backoff state, and every
|
|
789
|
+
* additional overlapping tick multiplies the wire pressure at exactly the
|
|
790
|
+
* moment (a slow or flood-throttled Telegram) when the sweep must issue LESS.
|
|
791
|
+
* `inFlight` makes overlap unrepresentable rather than merely survivable.
|
|
792
|
+
*
|
|
793
|
+
* Extracted + exported so the guard is unit-testable without a live gateway,
|
|
794
|
+
* a timer, or the filesystem: pass a `runSweep` you control and drive `tick()`
|
|
795
|
+
* directly.
|
|
796
|
+
*/
|
|
797
|
+
export function createOutboxSweepTick(deps: {
|
|
798
|
+
/** Nothing to sweep until the bot exists (unchanged pre-guard behaviour). */
|
|
799
|
+
getBot: () => unknown
|
|
800
|
+
backoff: ReturnType<typeof createSweepBackoff>
|
|
801
|
+
/** Run ONE sweep body. Wired by {@link startOutboxSweep} to {@link sweepOutbox}. */
|
|
802
|
+
runSweep: () => Promise<OutboxSweepSummary>
|
|
803
|
+
log?: (line: string) => void
|
|
804
|
+
/** Test seam — clock used for the backoff gate and the defer-log throttle. */
|
|
805
|
+
now?: () => number
|
|
806
|
+
}): (() => void) & { inFlight: () => boolean } {
|
|
807
|
+
const now = deps.now ?? (() => Date.now())
|
|
808
|
+
let lastDeferLogAt = 0
|
|
809
|
+
let inFlight = false
|
|
810
|
+
let overlapSkips = 0
|
|
811
|
+
const tick = () => {
|
|
812
|
+
if (deps.getBot() == null) return
|
|
813
|
+
if (!deps.backoff.ready(now())) return
|
|
814
|
+
// #3864 — a sweep is still running; skip this tick entirely rather than
|
|
815
|
+
// starting a second concurrent body. Skipping is free: the outbox is a
|
|
816
|
+
// durable safety net with no latency SLA, and the in-flight sweep will
|
|
817
|
+
// pick up anything this tick would have.
|
|
818
|
+
if (inFlight) {
|
|
819
|
+
overlapSkips++
|
|
820
|
+
if (overlapSkips === 1) {
|
|
821
|
+
deps.log?.(
|
|
822
|
+
`outbox-sweep: tick skipped — the previous sweep is still in flight ` +
|
|
823
|
+
`(single-flight guard; no concurrent sweep started)\n`,
|
|
824
|
+
)
|
|
825
|
+
}
|
|
826
|
+
return
|
|
827
|
+
}
|
|
828
|
+
inFlight = true
|
|
829
|
+
void deps
|
|
830
|
+
.runSweep()
|
|
831
|
+
.then((summary) => {
|
|
832
|
+
// A flood-deferred sweep is neither success nor failure: it never
|
|
833
|
+
// reached the wire, so it must not clear a backoff earned by real
|
|
834
|
+
// failures, and it must not deepen one either.
|
|
835
|
+
if (summary.floodDeferred === true) {
|
|
836
|
+
const at = now()
|
|
837
|
+
if (at - lastDeferLogAt >= OUTBOX_SWEEP_DEFER_LOG_INTERVAL_MS) {
|
|
838
|
+
lastDeferLogAt = at
|
|
839
|
+
deps.log?.(
|
|
840
|
+
`outbox-sweep: deferred — Telegram flood window still open for ` +
|
|
841
|
+
`${Math.ceil((summary.floodRemainingMs ?? 0) / 1000)}s; not issuing any ` +
|
|
842
|
+
`send (would feed the ban)\n`,
|
|
843
|
+
)
|
|
844
|
+
}
|
|
845
|
+
return
|
|
846
|
+
}
|
|
847
|
+
if (summary.sendFailures > 0) {
|
|
848
|
+
const delay = deps.backoff.noteFailure(now())
|
|
849
|
+
deps.log?.(
|
|
850
|
+
`outbox-sweep: ${summary.sendFailures} send failure(s) — backing off ` +
|
|
851
|
+
`${Math.round(delay / 1000)}s before the next sweep (attempt ${deps.backoff.failures()})\n`,
|
|
852
|
+
)
|
|
853
|
+
} else {
|
|
854
|
+
deps.backoff.noteSuccess()
|
|
855
|
+
}
|
|
856
|
+
})
|
|
857
|
+
.catch((err) => deps.log?.(`outbox-sweep: tick failed: ${(err as Error).message}\n`))
|
|
858
|
+
.finally(() => {
|
|
859
|
+
inFlight = false
|
|
860
|
+
if (overlapSkips > 0) {
|
|
861
|
+
deps.log?.(
|
|
862
|
+
`outbox-sweep: sweep finished; ${overlapSkips} tick(s) were skipped while it ran\n`,
|
|
863
|
+
)
|
|
864
|
+
overlapSkips = 0
|
|
865
|
+
}
|
|
866
|
+
})
|
|
867
|
+
}
|
|
868
|
+
return Object.assign(tick, { inFlight: () => inFlight })
|
|
869
|
+
}
|
|
870
|
+
|
|
545
871
|
export function startOutboxSweep(deps: {
|
|
546
872
|
isGatewayMain: boolean
|
|
547
873
|
stateDir: string
|
|
@@ -587,54 +913,26 @@ export function startOutboxSweep(deps: {
|
|
|
587
913
|
...(deps.resolveReplyMarkup != null ? { resolveReplyMarkup: deps.resolveReplyMarkup } : {}),
|
|
588
914
|
})
|
|
589
915
|
const backoff = deps.backoff ?? createSweepBackoff()
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
},
|
|
609
|
-
|
|
610
|
-
.then((summary) => {
|
|
611
|
-
// A flood-deferred sweep is neither success nor failure: it never
|
|
612
|
-
// reached the wire, so it must not clear a backoff earned by real
|
|
613
|
-
// failures, and it must not deepen one either.
|
|
614
|
-
if (summary.floodDeferred === true) {
|
|
615
|
-
const at = Date.now()
|
|
616
|
-
if (at - lastDeferLogAt >= OUTBOX_SWEEP_DEFER_LOG_INTERVAL_MS) {
|
|
617
|
-
lastDeferLogAt = at
|
|
618
|
-
deps.log?.(
|
|
619
|
-
`outbox-sweep: deferred — Telegram flood window still open for ` +
|
|
620
|
-
`${Math.ceil((summary.floodRemainingMs ?? 0) / 1000)}s; not issuing any ` +
|
|
621
|
-
`send (would feed the ban)\n`,
|
|
622
|
-
)
|
|
623
|
-
}
|
|
624
|
-
return
|
|
625
|
-
}
|
|
626
|
-
if (summary.sendFailures > 0) {
|
|
627
|
-
const delay = backoff.noteFailure(Date.now())
|
|
628
|
-
deps.log?.(
|
|
629
|
-
`outbox-sweep: ${summary.sendFailures} send failure(s) — backing off ` +
|
|
630
|
-
`${Math.round(delay / 1000)}s before the next sweep (attempt ${backoff.failures()})\n`,
|
|
631
|
-
)
|
|
632
|
-
} else {
|
|
633
|
-
backoff.noteSuccess()
|
|
634
|
-
}
|
|
635
|
-
})
|
|
636
|
-
.catch((err) => deps.log?.(`outbox-sweep: tick failed: ${(err as Error).message}\n`))
|
|
637
|
-
}
|
|
916
|
+
const tick = createOutboxSweepTick({
|
|
917
|
+
getBot: deps.getBot,
|
|
918
|
+
backoff,
|
|
919
|
+
log: deps.log,
|
|
920
|
+
runSweep: () =>
|
|
921
|
+
sweepOutbox({
|
|
922
|
+
stateDir: deps.stateDir,
|
|
923
|
+
log: deps.log,
|
|
924
|
+
send,
|
|
925
|
+
...(deps.recordOutbound != null ? { recordOutbound: deps.recordOutbound } : {}),
|
|
926
|
+
floodWaitRemainingMs: floodProbe,
|
|
927
|
+
textAlreadyDelivered: (chatId, threadId, text) => deps.dedupCheck(chatId, threadId ?? undefined, text),
|
|
928
|
+
registryChainLookup: (taskId) => {
|
|
929
|
+
const db = deps.getTurnsDb()
|
|
930
|
+
if (db == null) return null
|
|
931
|
+
const turnKey = resolveSubagentOriginTurnKey(db, taskId)
|
|
932
|
+
return turnKey == null ? null : chatFromTurnKey(turnKey)
|
|
933
|
+
},
|
|
934
|
+
}),
|
|
935
|
+
})
|
|
638
936
|
const timer = setInterval(tick, OUTBOX_SWEEP_INTERVAL_MS)
|
|
639
937
|
timer.unref?.()
|
|
640
938
|
return timer
|
|
@@ -674,6 +972,14 @@ export function journalExternalDelivery(
|
|
|
674
972
|
* the flush send and its journal write), not just via the in-memory dedup.
|
|
675
973
|
*/
|
|
676
974
|
deliverySource?: 'sweep' | 'reply-tool' | 'flush'
|
|
975
|
+
/**
|
|
976
|
+
* #4141: this delivery carried the reply-throw provenance banner. Durable
|
|
977
|
+
* terminal stamp mirroring the sweep's own `DeliveredEntry.framedProvenance`,
|
|
978
|
+
* so "was the recipient told where this text came from?" is answerable from
|
|
979
|
+
* the journal alone — on EITHER delivery path (sweep or captured-prose
|
|
980
|
+
* bridge), hours later.
|
|
981
|
+
*/
|
|
982
|
+
framedProvenance?: 'reply-throw'
|
|
677
983
|
},
|
|
678
984
|
stateDir?: string,
|
|
679
985
|
now: number = Date.now(),
|
|
@@ -683,6 +989,9 @@ export function journalExternalDelivery(
|
|
|
683
989
|
appendDelivered(
|
|
684
990
|
{
|
|
685
991
|
turnNonce: nonce,
|
|
992
|
+
// Hashed on the RAW text, deliberately: the banner is presentation, and
|
|
993
|
+
// the journal key must stay stable across framed/unframed so
|
|
994
|
+
// exactly-once-among-backstops keeps working (#4141).
|
|
686
995
|
textSha256: sha256Hex(args.text),
|
|
687
996
|
tgMessageId: args.tgMessageId,
|
|
688
997
|
ts: now,
|
|
@@ -690,6 +999,7 @@ export function journalExternalDelivery(
|
|
|
690
999
|
...(args.replyAlreadyDeliveredThisTurn == null
|
|
691
1000
|
? {}
|
|
692
1001
|
: { replyAlreadyDeliveredThisTurn: args.replyAlreadyDeliveredThisTurn }),
|
|
1002
|
+
...(args.framedProvenance == null ? {} : { framedProvenance: args.framedProvenance }),
|
|
693
1003
|
},
|
|
694
1004
|
stateDir,
|
|
695
1005
|
)
|