switchroom 0.19.11 → 0.19.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/switchroom.js +1 -1
- package/dist/host-control/main.js +1 -1
- package/package.json +1 -1
- package/profiles/_base/cron-session.sh.hbs +5 -0
- package/profiles/_base/start.sh.hbs +11 -0
- package/telegram-plugin/dist/gateway/gateway.js +1382 -1032
- package/telegram-plugin/final-answer-detect.ts +23 -0
- package/telegram-plugin/gateway/gateway.ts +29 -36
- package/telegram-plugin/gateway/outbound-send-path.ts +109 -8
- package/telegram-plugin/gateway/outbox-sweep.ts +282 -0
- package/telegram-plugin/gateway/stream-render.ts +26 -50
- package/telegram-plugin/hooks/silent-end-interrupt-stop.mjs +94 -2
- package/telegram-plugin/hooks/silent-end-scan.mjs +279 -0
- package/telegram-plugin/outbox.ts +472 -0
- package/telegram-plugin/scripts/bun-test-ci.sh +91 -0
- package/telegram-plugin/temporal-normalize.ts +347 -0
- package/telegram-plugin/tests/gateway-outbound-redact.test.ts +22 -12
- package/telegram-plugin/tests/outbound-send-path.test.ts +191 -0
- package/telegram-plugin/tests/outbox-capture-scan.test.ts +222 -0
- package/telegram-plugin/tests/outbox-delivery.test.ts +278 -0
- package/telegram-plugin/tests/outbox-hook-capture.test.ts +112 -0
- package/telegram-plugin/tests/silent-end-interrupt-stop-integration.test.ts +60 -63
- package/telegram-plugin/tests/silent-end.test.ts +21 -28
- package/telegram-plugin/tests/temporal-normalize.test.ts +222 -0
- package/telegram-plugin/tests/turn-flush-safety.test.ts +23 -18
|
@@ -120,3 +120,26 @@ export function isSubstantiveFinalReply(input: FinalAnswerReplyInput): boolean {
|
|
|
120
120
|
if (input.text.length >= FINAL_ANSWER_MIN_CHARS) return true
|
|
121
121
|
return false
|
|
122
122
|
}
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* F3 gate: should the legacy reply-site (`outbound-send-path.ts` sendReply)
|
|
126
|
+
* journal THIS reply's delivery under the turn nonce for the outbox sweep?
|
|
127
|
+
*
|
|
128
|
+
* This is deliberately `isSubstantiveFinalReply`, NOT `isFinalAnswerReply`.
|
|
129
|
+
* At the reply site the journaled text is the REPLY text — a DIFFERENT string
|
|
130
|
+
* from the trailing prose the Stop hook may capture under the SAME turn nonce
|
|
131
|
+
* (unlike the silent-anchor flush / captured-prose bridge, where the journaled
|
|
132
|
+
* text IS the turn's trailing content and a loose gate is loss-safe).
|
|
133
|
+
*
|
|
134
|
+
* `isFinalAnswerReply`'s ping clause classifies a short pinging interim ack
|
|
135
|
+
* ("On it — digging in", `disable_notification` omitted) as final. If such an
|
|
136
|
+
* ack journaled the turn nonce, and the model then ended the turn with
|
|
137
|
+
* gateway-invisible trailing prose (the real answer) captured under that same
|
|
138
|
+
* nonce, the sweep would hit `skip-journaled` and delete the real answer —
|
|
139
|
+
* silent loss, the exact incident this outbox exists to prevent. Requiring a
|
|
140
|
+
* SUBSTANTIVE reply (`done` or ≥200 chars) to journal keeps the double-post
|
|
141
|
+
* guard for genuine answers while never letting a pinging ack poison the nonce.
|
|
142
|
+
*/
|
|
143
|
+
export function shouldJournalReplySiteDelivery(input: FinalAnswerReplyInput): boolean {
|
|
144
|
+
return isSubstantiveFinalReply(input)
|
|
145
|
+
}
|
|
@@ -443,6 +443,7 @@ import { richMessage } from '../rich-send.js'
|
|
|
443
443
|
import { decideRedeliver, decideRedeliverCapture } from './redelivery-decision.js'
|
|
444
444
|
import { scrubVoice } from '../text-voice-scrub.js'
|
|
445
445
|
import {
|
|
446
|
+
normalizeOutboundBody,
|
|
446
447
|
sendReplyChunks,
|
|
447
448
|
sendReply,
|
|
448
449
|
deliverCapturedProse as deliverCapturedProseCore,
|
|
@@ -904,6 +905,7 @@ import {
|
|
|
904
905
|
TURN_ACTIVE_IDLE_SWEEP_MS,
|
|
905
906
|
} from './turn-active-marker.js'
|
|
906
907
|
import { startGatewayHeartbeat } from './gateway-heartbeat.js'
|
|
908
|
+
import { startOutboxSweep } from './outbox-sweep.js'
|
|
907
909
|
import {
|
|
908
910
|
VERSION,
|
|
909
911
|
COMMIT_SHA,
|
|
@@ -9822,20 +9824,15 @@ function trackRedeliveredInbound(merged: InboundMessage): void {
|
|
|
9822
9824
|
// `runDeliveryConfirmSweep` / `redeliverStrandedInbound` / `sweepSuspendedTargets`
|
|
9823
9825
|
// moved VERBATIM; the gateway keeps the deps builder, thin wrappers, and the
|
|
9824
9826
|
// `setInterval` registration below (P0c: modules export the tick body; the
|
|
9825
|
-
// gateway owns the timer).
|
|
9826
|
-
// (the design's C1 deps shape — never the machine's eager in-turn state).
|
|
9827
|
+
// gateway owns the timer). Idle gate crosses as `getCurrentTurnNull` ONLY.
|
|
9827
9828
|
|
|
9828
9829
|
/** Live gateway deps for the extracted delivery-confirm sweep wiring. */
|
|
9829
9830
|
function gatewayDeliveryConfirmDeps() {
|
|
9830
9831
|
return {
|
|
9831
|
-
DELIVERY_CONFIRM_ENABLED,
|
|
9832
|
-
DELIVERY_CONFIRM_TIMEOUT_MS,
|
|
9832
|
+
DELIVERY_CONFIRM_ENABLED, DELIVERY_CONFIRM_TIMEOUT_MS,
|
|
9833
9833
|
getCurrentTurnNull: (): boolean => currentTurn == null,
|
|
9834
9834
|
sendToAgent: (agent: string, m: InboundMessage): boolean => ipcServer.sendToAgent(agent, m),
|
|
9835
|
-
deliveryQueue,
|
|
9836
|
-
pendingInboundBuffer,
|
|
9837
|
-
pendingPermissions,
|
|
9838
|
-
pendingAskUser,
|
|
9835
|
+
deliveryQueue, pendingInboundBuffer, pendingPermissions, pendingAskUser,
|
|
9839
9836
|
}
|
|
9840
9837
|
}
|
|
9841
9838
|
export type DeliveryConfirmWiringDeps = ReturnType<typeof gatewayDeliveryConfirmDeps>
|
|
@@ -9852,6 +9849,8 @@ function runDeliveryConfirmSweep(): void {
|
|
|
9852
9849
|
const _deliveryConfirmSweep = isGatewayMain ? setInterval(runDeliveryConfirmSweep, DELIVERY_CONFIRM_SWEEP_MS) : undefined
|
|
9853
9850
|
_deliveryConfirmSweep?.unref?.()
|
|
9854
9851
|
|
|
9852
|
+
startOutboxSweep({ isGatewayMain, stateDir: STATE_DIR, getBot: () => bot, getTurnsDb: () => turnsDb, dedupCheck: (c, t, x) => outboundDedup.check(c, t, x, Date.now()) != null, log: (l) => process.stderr.write(l) }) // outbox: single deliverer for Stop-hook-captured prose (../outbox.ts)
|
|
9853
|
+
|
|
9855
9854
|
// #1445 cross-turn pending-async ambient. When a turn ends after the
|
|
9856
9855
|
// model dispatched background async work (Agent / Task / Bash run-in-
|
|
9857
9856
|
// background) and the model has stopped speaking, keep editing the
|
|
@@ -13303,34 +13302,26 @@ async function executeEditMessage(args: Record<string, unknown>): Promise<unknow
|
|
|
13303
13302
|
// Single rich-markdown path (#2669): `format:'text'` edits as a literal
|
|
13304
13303
|
// plain string; everything else edits via the rich-markdown path.
|
|
13305
13304
|
const editLiteralText = editFormat === 'text'
|
|
13306
|
-
//
|
|
13307
|
-
//
|
|
13308
|
-
//
|
|
13309
|
-
|
|
13310
|
-
|
|
13311
|
-
//
|
|
13312
|
-
|
|
13313
|
-
|
|
13314
|
-
|
|
13315
|
-
|
|
13316
|
-
|
|
13317
|
-
|
|
13318
|
-
|
|
13319
|
-
|
|
13320
|
-
|
|
13321
|
-
|
|
13322
|
-
|
|
13323
|
-
|
|
13324
|
-
|
|
13325
|
-
|
|
13326
|
-
editRawText = scrub.scrubbed
|
|
13327
|
-
emitRuntimeMetric({
|
|
13328
|
-
kind: 'voice_scrub_applied',
|
|
13329
|
-
chatKey: statusKey(String(args.chat_id ?? ''), undefined),
|
|
13330
|
-
replaced: scrub.replaced,
|
|
13331
|
-
site: 'edit_message',
|
|
13332
|
-
})
|
|
13333
|
-
}
|
|
13305
|
+
// #3501: route through the single shared outbound seam (repair → paragraph-
|
|
13306
|
+
// break → redact → punctuation/bold → temporal → voice scrub) instead of the
|
|
13307
|
+
// former hand-mirrored inline pipeline. The edit path's deviations are options:
|
|
13308
|
+
// literalText skips paragraph/punctuation/bold/spacers + temporal (a literal
|
|
13309
|
+
// edit lands byte-for-byte); addSpacers folds the idempotent U+00A0 paragraph
|
|
13310
|
+
// spacer into the rich formatting step. tz/nowMs drive the temporal pass.
|
|
13311
|
+
const _editNorm = normalizeOutboundBody(args.text as string, 'edit_message', redactOutboundText, {
|
|
13312
|
+
literalText: editLiteralText,
|
|
13313
|
+
addSpacers: !editLiteralText,
|
|
13314
|
+
tz: resolveEnvTimezone(),
|
|
13315
|
+
nowMs: Date.now(),
|
|
13316
|
+
})
|
|
13317
|
+
let editRawText = _editNorm.text
|
|
13318
|
+
if (_editNorm.voiceReplaced > 0) {
|
|
13319
|
+
emitRuntimeMetric({
|
|
13320
|
+
kind: 'voice_scrub_applied',
|
|
13321
|
+
chatKey: statusKey(String(args.chat_id ?? ''), undefined),
|
|
13322
|
+
replaced: _editNorm.voiceReplaced,
|
|
13323
|
+
site: 'edit_message',
|
|
13324
|
+
})
|
|
13334
13325
|
}
|
|
13335
13326
|
const edited = await robustApiCall(
|
|
13336
13327
|
() => lockedBot.api.editMessageText(
|
|
@@ -14795,6 +14786,8 @@ export async function handleInbound(
|
|
|
14795
14786
|
{
|
|
14796
14787
|
const inboundChatId = ctx.chat?.id
|
|
14797
14788
|
if (inboundChatId != null) void dmPinSweeper.sweep(String(inboundChatId))
|
|
14789
|
+
// Outbox envelope-less routing (F2) is scoped to each record's OWN stamped
|
|
14790
|
+
// per-session origin chat (captured at Stop), not a gateway-global stamp.
|
|
14798
14791
|
}
|
|
14799
14792
|
|
|
14800
14793
|
// Capture wall-clock receive time for inbound_ack metric (#203).
|
|
@@ -33,6 +33,8 @@ import {
|
|
|
33
33
|
RICH_MESSAGE_MAX_CHARS,
|
|
34
34
|
} from '../format.js'
|
|
35
35
|
import { scrubVoice } from '../text-voice-scrub.js'
|
|
36
|
+
import { normalizeTemporal } from '../temporal-normalize.js'
|
|
37
|
+
import { resolveEnvTimezone } from '../shared/local-time.js'
|
|
36
38
|
import { isMessageTooLongError, isHtmlParseRejectError } from '../retry-api-call.js'
|
|
37
39
|
|
|
38
40
|
// ── send-orchestration façade imports (#2996 P2) ──
|
|
@@ -49,8 +51,9 @@ import {
|
|
|
49
51
|
escapeMarkdown,
|
|
50
52
|
} from '../format.js'
|
|
51
53
|
import { richMessage } from '../rich-send.js'
|
|
54
|
+
import { journalExternalDelivery } from './outbox-sweep.js'
|
|
52
55
|
import { resolveChatIdFallback } from './chat-id-fallback.js'
|
|
53
|
-
import { isFinalAnswerReply, isSubstantiveFinalReply } from '../final-answer-detect.js'
|
|
56
|
+
import { isFinalAnswerReply, isSubstantiveFinalReply, shouldJournalReplySiteDelivery } from '../final-answer-detect.js'
|
|
54
57
|
import { decideOverPing, type OverPingDecision } from '../over-ping-safety-net.js'
|
|
55
58
|
import { decideSilentReplyAnchor } from '../silent-reply-anchor.js'
|
|
56
59
|
import {
|
|
@@ -114,16 +117,59 @@ export interface NormalizeOutboundResult {
|
|
|
114
117
|
}
|
|
115
118
|
|
|
116
119
|
/**
|
|
117
|
-
*
|
|
118
|
-
*
|
|
119
|
-
*
|
|
120
|
+
* Per-site shape knobs for {@link normalizeOutboundBody}. The reply path (the
|
|
121
|
+
* canonical caller) passes none — the defaults reproduce its exact pre-#3501
|
|
122
|
+
* byte output. The edit_message and turn-flush sites, which used to hand-mirror
|
|
123
|
+
* this pipeline inline, pass these to reproduce their two small deviations:
|
|
124
|
+
*
|
|
125
|
+
* - `literalText` (edit_message `format:'text'`): a literal edit must land
|
|
126
|
+
* byte-for-byte as authored, so it SKIPS paragraph-break promotion and the
|
|
127
|
+
* punctuation/bold/spacer formatting entirely — only the whitespace repair,
|
|
128
|
+
* the secret redact, and the voice scrub still run (they are safety
|
|
129
|
+
* transforms that must apply even to literal edits, matching the former
|
|
130
|
+
* inline `executeEditMessage` order).
|
|
131
|
+
* - `addSpacers` (edit_message non-literal): the edit path folds the
|
|
132
|
+
* idempotent U+00A0 paragraph spacer INTO this transform
|
|
133
|
+
* (`addParagraphSpacers(stripExcessBold(normalizePunctuation(…)))`),
|
|
134
|
+
* whereas the reply/turn-flush paths add spacers separately downstream (or
|
|
135
|
+
* not at all). Setting this reproduces that inline behaviour exactly.
|
|
136
|
+
*/
|
|
137
|
+
export interface NormalizeOutboundOptions {
|
|
138
|
+
/** Literal edit: skip paragraph-break + punctuation/bold/spacer formatting. */
|
|
139
|
+
literalText?: boolean
|
|
140
|
+
/** Fold `addParagraphSpacers` into the formatting step (edit_message path). */
|
|
141
|
+
addSpacers?: boolean
|
|
142
|
+
/**
|
|
143
|
+
* Agent-configured IANA timezone for the temporal-normalization pass (#3501).
|
|
144
|
+
* When BOTH `tz` and `nowMs` are supplied (callers pass `resolveEnvTimezone()`
|
|
145
|
+
* and `Date.now()`), the seam rewrites UTC/Zulu datetimes to local wall clock
|
|
146
|
+
* and corrects relative-day words against the current date in `tz`. Omitted →
|
|
147
|
+
* the temporal pass is a no-op (keeps the module pure/clock-free by default).
|
|
148
|
+
* NEVER fires on a literal edit (`literalText`) — a literal edit lands
|
|
149
|
+
* byte-for-byte as authored.
|
|
150
|
+
*/
|
|
151
|
+
tz?: string
|
|
152
|
+
/** Epoch-ms "now" for the temporal pass. Required alongside `tz` to fire. */
|
|
153
|
+
nowMs?: number
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/**
|
|
157
|
+
* Stage 1 — the deterministic outbound text transform. This is the SINGLE
|
|
158
|
+
* shared seam for every outbound prose send (#3501): the reply path, the
|
|
159
|
+
* edit_message path, and the turn-flush backstop all route through it instead
|
|
160
|
+
* of hand-mirroring the pipeline inline, so a new outbound transform is added
|
|
161
|
+
* in exactly one place.
|
|
120
162
|
*
|
|
121
163
|
* 1. repairEscapedWhitespace — undo LLM JSON-escape bungles
|
|
122
164
|
* 2. normalizeParagraphBreaks — promote lone prose breaks to GFM hard breaks
|
|
165
|
+
* (skipped for a literal edit)
|
|
123
166
|
* 3. redact (injected) — outbound secret scrub (#2044), BEFORE the
|
|
124
167
|
* punctuation/bold normalizers so a secret with
|
|
125
168
|
* an em-dash or `**` is matched literally
|
|
126
169
|
* 4. stripExcessBold∘normalizePunctuation — fleet-consistent formatting
|
|
170
|
+
* (optionally wrapped in addParagraphSpacers for
|
|
171
|
+
* the edit path; skipped entirely for a literal
|
|
172
|
+
* edit)
|
|
127
173
|
* 5. scrubVoice — em/en dash → comma/period (#1683)
|
|
128
174
|
*
|
|
129
175
|
* The order is load-bearing and MUST NOT change (each step's comment in the
|
|
@@ -133,10 +179,25 @@ export function normalizeOutboundBody(
|
|
|
133
179
|
rawText: string,
|
|
134
180
|
site: string,
|
|
135
181
|
redact: RedactFn,
|
|
182
|
+
opts: NormalizeOutboundOptions = {},
|
|
136
183
|
): NormalizeOutboundResult {
|
|
137
|
-
|
|
184
|
+
const { literalText = false, addSpacers = false, tz, nowMs } = opts
|
|
185
|
+
let text = repairEscapedWhitespace(rawText)
|
|
186
|
+
if (!literalText) text = normalizeParagraphBreaks(text)
|
|
138
187
|
text = redact(text, site)
|
|
139
|
-
|
|
188
|
+
if (!literalText) {
|
|
189
|
+
let formatted = stripExcessBold(normalizePunctuation(text))
|
|
190
|
+
if (addSpacers) formatted = addParagraphSpacers(formatted)
|
|
191
|
+
text = formatted
|
|
192
|
+
// Temporal normalization (#3501): UTC/Zulu → local wall clock, then
|
|
193
|
+
// relative-day accuracy, resolved in the agent's configured tz. Runs AFTER
|
|
194
|
+
// punctuation/bold (so masking sees final markdown) and BEFORE the voice
|
|
195
|
+
// scrub (so a freshly-rewritten "Thu 23 Jul 7:15 pm AEST" is never mangled
|
|
196
|
+
// by the em/en-dash scrub). Never on a literal edit. Pure / never-throws.
|
|
197
|
+
if (tz != null && nowMs != null) {
|
|
198
|
+
text = normalizeTemporal(text, tz, nowMs)
|
|
199
|
+
}
|
|
200
|
+
}
|
|
140
201
|
let voiceReplaced = 0
|
|
141
202
|
const scrub = scrubVoice(text)
|
|
142
203
|
if (scrub.replaced > 0) {
|
|
@@ -793,7 +854,10 @@ export async function sendReply(
|
|
|
793
854
|
// retries see the scrubbed dedup key). The metric side effect (fired on a
|
|
794
855
|
// non-zero voice-scrub replacement) stays here — the pure module returns the
|
|
795
856
|
// replacement count and the gateway emits.
|
|
796
|
-
const _normalized = normalizeOutboundBody(rawText, 'reply', redactOutboundText
|
|
857
|
+
const _normalized = normalizeOutboundBody(rawText, 'reply', redactOutboundText, {
|
|
858
|
+
tz: resolveEnvTimezone(),
|
|
859
|
+
nowMs: Date.now(),
|
|
860
|
+
})
|
|
797
861
|
let text = _normalized.text
|
|
798
862
|
if (_normalized.voiceReplaced > 0) {
|
|
799
863
|
emitRuntimeMetric({
|
|
@@ -1733,6 +1797,16 @@ export async function sendReply(
|
|
|
1733
1797
|
Date.now(),
|
|
1734
1798
|
turn?.registryKey ?? null,
|
|
1735
1799
|
)
|
|
1800
|
+
// F1: a FINAL-answer silent-anchor edit journals this legacy delivery
|
|
1801
|
+
// under the shared nonce (turn.turnId === deriveTurnId === the hook's
|
|
1802
|
+
// deriveTurnNonce `${chatKey}#${messageId}` for a gateway-visible turn)
|
|
1803
|
+
// and drops any hook-captured record, so the sweep never re-sends after
|
|
1804
|
+
// the in-memory dedup TTL evicts / a restart clears it / the captured
|
|
1805
|
+
// text differs. Gated so an interim-ack edit never journals the turn
|
|
1806
|
+
// nonce (which would suppress a later genuine answer for the turn).
|
|
1807
|
+
if (isFinalAnswerReply({ text: decision.mergedText, disableNotification: modelDisableNotification })) {
|
|
1808
|
+
journalExternalDelivery({ turnNonce: turn?.turnId ?? null, text: decision.mergedText, tgMessageId: decision.messageId })
|
|
1809
|
+
}
|
|
1736
1810
|
|
|
1737
1811
|
silentAnchorEditDone = true
|
|
1738
1812
|
} catch (err) {
|
|
@@ -2275,7 +2349,31 @@ export async function sendReply(
|
|
|
2275
2349
|
// calls with this same content within DEFAULT_DEDUP_TTL_MS will
|
|
2276
2350
|
// be suppressed.
|
|
2277
2351
|
if (sentIds.length > 0) {
|
|
2278
|
-
|
|
2352
|
+
const t = getCurrentTurn()
|
|
2353
|
+
outboundDedup.record(chat_id, threadId, text, Date.now(), t?.registryKey ?? null)
|
|
2354
|
+
// F1: a SUBSTANTIVE-final reply journals + clears under the shared nonce
|
|
2355
|
+
// (turn.turnId === deriveTurnId, the hook's deriveTurnNonce), so the sweep
|
|
2356
|
+
// never re-posts this turn's answer.
|
|
2357
|
+
//
|
|
2358
|
+
// F3: gate on `isSubstantiveFinalReply`, NOT `isFinalAnswerReply`. Unlike
|
|
2359
|
+
// the flush site (~1741) and the captured-prose bridge (~2407) — where the
|
|
2360
|
+
// journaled text IS this turn's trailing content, so a loose gate is
|
|
2361
|
+
// loss-safe — here the journaled text is the REPLY text, a DIFFERENT string
|
|
2362
|
+
// from the trailing prose the hook captures under the same turn nonce.
|
|
2363
|
+
// `isFinalAnswerReply`'s ping clause (`disableNotification` falsy — the
|
|
2364
|
+
// tool's default, routinely omitted by models) classifies a short pinging
|
|
2365
|
+
// interim ack ("On it — digging in") as final. Journaling on that ack would
|
|
2366
|
+
// poison the turn nonce: the model then ends the turn with gateway-invisible
|
|
2367
|
+
// trailing prose (the real answer), the Stop hook captures it under the SAME
|
|
2368
|
+
// nonce, and the sweep hits `skip-journaled` → `clearOutboxRecord` → the
|
|
2369
|
+
// real answer is silently destroyed — the exact incident class this outbox
|
|
2370
|
+
// exists to prevent. `isSubstantiveFinalReply` (`done === true ||
|
|
2371
|
+
// length ≥ 200`, no ping path) still journals a genuine answer (so the sweep
|
|
2372
|
+
// won't double-post it) while an interim ack never journals (so a later
|
|
2373
|
+
// genuinely-undelivered final answer is delivered by the sweep).
|
|
2374
|
+
if (shouldJournalReplySiteDelivery({ text: rawText, disableNotification: modelDisableNotification })) {
|
|
2375
|
+
journalExternalDelivery({ turnNonce: t?.turnId ?? null, text, tgMessageId: sentIds[sentIds.length - 1] })
|
|
2376
|
+
}
|
|
2279
2377
|
}
|
|
2280
2378
|
return { content: [{ type: 'text', text: result }] }
|
|
2281
2379
|
}
|
|
@@ -2383,6 +2481,9 @@ export async function deliverCapturedProse(
|
|
|
2383
2481
|
// Record what we just sent so a late reply / stream_reply retry with the
|
|
2384
2482
|
// same content is deduped at its send site (the #546 dedup cache).
|
|
2385
2483
|
outboundDedup.record(chatId, threadId, text, now, registryKey)
|
|
2484
|
+
// F1: captured-prose delivery journals + clears under the shared nonce
|
|
2485
|
+
// (`originTurnId` === turn.turnId === deriveTurnId, the hook's nonce).
|
|
2486
|
+
journalExternalDelivery({ turnNonce: originTurnId, text, tgMessageId: sentIds[sentIds.length - 1] })
|
|
2386
2487
|
process.stderr.write(
|
|
2387
2488
|
`telegram gateway: captured-prose delivery — sent ${out.length} chars recovered from ` +
|
|
2388
2489
|
`transcript scan (chat=${chatId} origin=${originTurnId})\n`,
|
|
@@ -0,0 +1,282 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* outbox-sweep.ts — the single deliverer for the guaranteed-final-message
|
|
3
|
+
* outbox. Runs on the gateway heartbeat tick, independent of turn lifecycle,
|
|
4
|
+
* so it covers turn CLASSES the gateway never sees a CurrentTurn for
|
|
5
|
+
* (`<task-notification>` handbacks, background-worker completions, unknown
|
|
6
|
+
* future wake shapes). See `../outbox.ts` for the record/nonce/journal model.
|
|
7
|
+
*
|
|
8
|
+
* Exactly-once (H1): every record is claimed with a rename-mutex, guarded by the
|
|
9
|
+
* shared delivered-keys journal (same turnNonce every delivering machine uses),
|
|
10
|
+
* and by the in-memory text `outboundDedup` cache. Delivered nonce is journaled
|
|
11
|
+
* AFTER a successful send. When a legacy machine (turn-flush / reply / captured
|
|
12
|
+
* prose) delivered the same turn's answer first, its journal write under the
|
|
13
|
+
* SAME nonce makes the sweep skip-journaled; and the sweep's own skip-dedup
|
|
14
|
+
* branch JOURNALS the nonce and DELETES the record (see below) so a text-dedup
|
|
15
|
+
* hit can never re-fire after the in-memory cache's TTL evicts. A crash between
|
|
16
|
+
* send and journal is caught next boot by the journal + text-dedup (never a
|
|
17
|
+
* loss, at most one duplicate).
|
|
18
|
+
*
|
|
19
|
+
* Routing (H3/F2): `resolveOutboxChat` runs the injected transitive registry-chain
|
|
20
|
+
* lookup then the record's OWN stamped per-session origin chat for envelope-less
|
|
21
|
+
* records — never a gateway-global last-inbound fallback (cross-chat leak). It
|
|
22
|
+
* FAILS CLOSED (holds the record) when neither resolves.
|
|
23
|
+
*
|
|
24
|
+
* The orchestration takes injected IO deps so it is unit-testable without a live
|
|
25
|
+
* gateway; `gateway.ts` wires the real send / dedup / registry lookups.
|
|
26
|
+
*/
|
|
27
|
+
|
|
28
|
+
import {
|
|
29
|
+
OUTBOX_QUIET_MS,
|
|
30
|
+
appendDelivered,
|
|
31
|
+
claimRecord,
|
|
32
|
+
clearOutboxRecord,
|
|
33
|
+
decideOutboxSweep,
|
|
34
|
+
extractTaskId,
|
|
35
|
+
listPendingRecords,
|
|
36
|
+
readDeliveredNonces,
|
|
37
|
+
readOutboxRecord,
|
|
38
|
+
reclaimStaleSending,
|
|
39
|
+
releaseClaim,
|
|
40
|
+
removeClaimed,
|
|
41
|
+
resolveOutboxChat,
|
|
42
|
+
sha256Hex,
|
|
43
|
+
type OutboxRecord,
|
|
44
|
+
} from '../outbox.js'
|
|
45
|
+
import { resolveSubagentOriginTurnKey } from '../registry/subagents-schema.js'
|
|
46
|
+
import { createRetryApiCall, retryWithThreadFallback } from '../retry-api-call.js'
|
|
47
|
+
|
|
48
|
+
export interface OutboxSweepDeps {
|
|
49
|
+
/** Deliver `text` to the chat. Resolves to the primary message id (best-effort). */
|
|
50
|
+
send: (
|
|
51
|
+
chatId: string,
|
|
52
|
+
threadId: number | null,
|
|
53
|
+
text: string,
|
|
54
|
+
) => Promise<number | undefined>
|
|
55
|
+
/** Has this exact text already been delivered to this chat/thread recently? */
|
|
56
|
+
textAlreadyDelivered: (chatId: string, threadId: number | null, text: string) => boolean
|
|
57
|
+
/**
|
|
58
|
+
* Transitive registry-chain lookup (H3): resolve the originating chat for a
|
|
59
|
+
* task-notification / chained-dispatch anchor from its `<task-id>`. Null when
|
|
60
|
+
* the chain doesn't resolve.
|
|
61
|
+
*/
|
|
62
|
+
registryChainLookup?: (
|
|
63
|
+
taskId: string,
|
|
64
|
+
) => { chatId: string; threadId: number | null } | null
|
|
65
|
+
stateDir?: string
|
|
66
|
+
now?: () => number
|
|
67
|
+
log?: (line: string) => void
|
|
68
|
+
quietMs?: number
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export interface OutboxSweepSummary {
|
|
72
|
+
scanned: number
|
|
73
|
+
delivered: number
|
|
74
|
+
skipped: number
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Sweep the outbox once. Idempotent; safe to call every heartbeat tick.
|
|
79
|
+
*/
|
|
80
|
+
export async function sweepOutbox(deps: OutboxSweepDeps): Promise<OutboxSweepSummary> {
|
|
81
|
+
const now = deps.now?.() ?? Date.now()
|
|
82
|
+
const log = deps.log ?? (() => {})
|
|
83
|
+
const summary: OutboxSweepSummary = { scanned: 0, delivered: 0, skipped: 0 }
|
|
84
|
+
|
|
85
|
+
// Re-queue crashed claims first (claim-then-crash before send).
|
|
86
|
+
reclaimStaleSending(deps.stateDir, now)
|
|
87
|
+
|
|
88
|
+
const pending = listPendingRecords(deps.stateDir)
|
|
89
|
+
if (pending.length === 0) return summary
|
|
90
|
+
const deliveredNonces = readDeliveredNonces(deps.stateDir)
|
|
91
|
+
|
|
92
|
+
for (const fileName of pending) {
|
|
93
|
+
const record = readOutboxRecord(fileName, deps.stateDir)
|
|
94
|
+
if (record == null) continue
|
|
95
|
+
summary.scanned++
|
|
96
|
+
|
|
97
|
+
// Resolve destination (anchor → registry chain → per-session origin). Fails
|
|
98
|
+
// CLOSED (null → held) rather than routing to an arbitrary chat (F2).
|
|
99
|
+
const resolved = resolveOutboxChat(record, {
|
|
100
|
+
registryChainLookup: (anchorContent) => {
|
|
101
|
+
const taskId = extractTaskId(anchorContent)
|
|
102
|
+
if (taskId == null || deps.registryChainLookup == null) return null
|
|
103
|
+
return deps.registryChainLookup(taskId)
|
|
104
|
+
},
|
|
105
|
+
})
|
|
106
|
+
|
|
107
|
+
const routePrefix = resolved?.via === 'origin' ? '(from background task) ' : ''
|
|
108
|
+
const decision = decideOutboxSweep({
|
|
109
|
+
record,
|
|
110
|
+
now,
|
|
111
|
+
deliveredNonces,
|
|
112
|
+
textAlreadyDelivered:
|
|
113
|
+
resolved != null &&
|
|
114
|
+
deps.textAlreadyDelivered(resolved.chatId, resolved.threadId, record.text),
|
|
115
|
+
routable: resolved != null,
|
|
116
|
+
routePrefix,
|
|
117
|
+
quietMs: deps.quietMs ?? OUTBOX_QUIET_MS,
|
|
118
|
+
})
|
|
119
|
+
|
|
120
|
+
if (decision.action !== 'send' && decision.action !== 'send-delayed') {
|
|
121
|
+
summary.skipped++
|
|
122
|
+
if (decision.action === 'skip-journaled') {
|
|
123
|
+
// Already delivered under this nonce by another machine → drop the
|
|
124
|
+
// pending record. clearOutboxRecord unlinks the `.json` (the pending
|
|
125
|
+
// file) AND any `.sending` — the pre-fix `removeClaimed` only unlinked
|
|
126
|
+
// `.sending`, leaking the `.json` to be rescanned forever (S4).
|
|
127
|
+
clearOutboxRecord(record.turnNonce, deps.stateDir)
|
|
128
|
+
} else if (decision.action === 'skip-dedup') {
|
|
129
|
+
// F1: the identical text was already delivered by the legacy flush/reply
|
|
130
|
+
// (in-memory `outboundDedup` hit). JOURNAL the nonce and DELETE the
|
|
131
|
+
// record NOW, so once that in-memory cache's TTL evicts (~60s) the sweep
|
|
132
|
+
// cannot resurrect and re-send this turn's answer. Deterministic
|
|
133
|
+
// exactly-once, independent of cache lifetime and surviving a restart
|
|
134
|
+
// (the record is gone from disk, the nonce is durably journaled).
|
|
135
|
+
appendDelivered(
|
|
136
|
+
{ turnNonce: record.turnNonce, textSha256: record.textSha256, ts: now },
|
|
137
|
+
deps.stateDir,
|
|
138
|
+
)
|
|
139
|
+
clearOutboxRecord(record.turnNonce, deps.stateDir)
|
|
140
|
+
}
|
|
141
|
+
continue
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
// Claim (rename mutex). If lost, another sweep/machine has it.
|
|
145
|
+
const claimed = claimRecord(record.turnNonce, deps.stateDir)
|
|
146
|
+
if (claimed == null) {
|
|
147
|
+
summary.skipped++
|
|
148
|
+
continue
|
|
149
|
+
}
|
|
150
|
+
const resolvedChat = resolved! // routable implies non-null
|
|
151
|
+
|
|
152
|
+
try {
|
|
153
|
+
const messageId = await deps.send(
|
|
154
|
+
resolvedChat.chatId,
|
|
155
|
+
resolvedChat.threadId,
|
|
156
|
+
decision.text ?? record.text,
|
|
157
|
+
)
|
|
158
|
+
appendDelivered(
|
|
159
|
+
{ turnNonce: record.turnNonce, textSha256: record.textSha256, tgMessageId: messageId, ts: now },
|
|
160
|
+
deps.stateDir,
|
|
161
|
+
)
|
|
162
|
+
removeClaimed(record.turnNonce, deps.stateDir)
|
|
163
|
+
summary.delivered++
|
|
164
|
+
log(
|
|
165
|
+
`outbox-sweep: delivered nonce=${record.turnNonce} via=${resolvedChat.via} ` +
|
|
166
|
+
`source=${record.source} chars=${record.text.length}${decision.action === 'send-delayed' ? ' (delayed)' : ''}\n`,
|
|
167
|
+
)
|
|
168
|
+
} catch (err) {
|
|
169
|
+
// Send failed — release the claim so the next tick retries. The record is
|
|
170
|
+
// preserved on disk; no loss.
|
|
171
|
+
releaseClaim(record.turnNonce, deps.stateDir)
|
|
172
|
+
summary.skipped++
|
|
173
|
+
log(`outbox-sweep: send failed nonce=${record.turnNonce}: ${(err as Error).message} — will retry\n`)
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
return summary
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/**
|
|
180
|
+
* Parse a registry `turn_key` (`chatId:threadId`, `_` → null thread) into a
|
|
181
|
+
* routable chat. Exported for the gateway wiring + tests.
|
|
182
|
+
*/
|
|
183
|
+
export function chatFromTurnKey(turnKey: string): { chatId: string; threadId: number | null } {
|
|
184
|
+
const idx = turnKey.indexOf(':')
|
|
185
|
+
const chatId = idx === -1 ? turnKey : turnKey.slice(0, idx)
|
|
186
|
+
const tRaw = idx === -1 ? '_' : turnKey.slice(idx + 1)
|
|
187
|
+
const t = tRaw === '_' || tRaw === '' ? null : Number(tRaw)
|
|
188
|
+
return { chatId, threadId: Number.isFinite(t as number) ? t : null }
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/** How often the sweep ticks (aligned with the delivery-confirm sweep cadence). */
|
|
192
|
+
export const OUTBOX_SWEEP_INTERVAL_MS = 5_000
|
|
193
|
+
|
|
194
|
+
/**
|
|
195
|
+
* Own the heartbeat-tick outbox sweep entirely, keeping the timer / chunking /
|
|
196
|
+
* turn-key parse / dedup / registry-chain wiring OUT of `gateway.ts` (the line
|
|
197
|
+
* ratchet). The gateway passes only the primitives that must come from its
|
|
198
|
+
* scope: lazy getters for `bot` / `turnsDb` (both assigned late), its live
|
|
199
|
+
* `OutboundDedupCache`, and the state dir. Returns the interval handle (unref'd)
|
|
200
|
+
* or `undefined` when disabled / not the main process.
|
|
201
|
+
*
|
|
202
|
+
* Delivery reuses `bot.api.sendMessage` and chunks to Telegram's 4096-char
|
|
203
|
+
* ceiling so an oversized record can never wedge the sweep in a permanent
|
|
204
|
+
* retry loop. Routing runs the transitive registry chain
|
|
205
|
+
* (`resolveSubagentOriginTurnKey`) then the last-real-inbound fallback (H3).
|
|
206
|
+
* Kill switch: `SWITCHROOM_TG_OUTBOX_DELIVERY=0`.
|
|
207
|
+
*/
|
|
208
|
+
export function startOutboxSweep(deps: {
|
|
209
|
+
isGatewayMain: boolean
|
|
210
|
+
stateDir: string
|
|
211
|
+
getBot: () => { api: { sendMessage: (chatId: string, text: string, opts: object) => Promise<{ message_id?: number }> } } | undefined
|
|
212
|
+
getTurnsDb: () => Parameters<typeof resolveSubagentOriginTurnKey>[0] | null
|
|
213
|
+
dedupCheck: (chatId: string, threadId: number | undefined, text: string) => boolean
|
|
214
|
+
log?: (line: string) => void
|
|
215
|
+
}): ReturnType<typeof setInterval> | undefined {
|
|
216
|
+
if (!deps.isGatewayMain || process.env.SWITCHROOM_TG_OUTBOX_DELIVERY === '0') return undefined
|
|
217
|
+
const retry = createRetryApiCall({ log: deps.log })
|
|
218
|
+
const tick = () => {
|
|
219
|
+
const bot = deps.getBot()
|
|
220
|
+
if (bot == null) return
|
|
221
|
+
void sweepOutbox({
|
|
222
|
+
stateDir: deps.stateDir,
|
|
223
|
+
log: deps.log,
|
|
224
|
+
send: async (chatId, threadId, text) => {
|
|
225
|
+
// Chunk to Telegram's 4096-char ceiling; each chunk goes through the
|
|
226
|
+
// standard retry / flood-wait / thread-fallback wrapper. A thrown send
|
|
227
|
+
// propagates so the sweep releases the claim and retries next tick (the
|
|
228
|
+
// record is never journaled → never lost).
|
|
229
|
+
let lastId: number | undefined
|
|
230
|
+
for (let i = 0; i < text.length; i += 4000) {
|
|
231
|
+
const chunk = text.slice(i, i + 4000)
|
|
232
|
+
const res = await retryWithThreadFallback(
|
|
233
|
+
retry,
|
|
234
|
+
(tid) => bot.api.sendMessage(chatId, chunk, tid != null ? { message_thread_id: tid } : {}),
|
|
235
|
+
{ threadId: threadId ?? undefined, chat_id: chatId, verb: 'outbox-sweep.sendMessage' },
|
|
236
|
+
)
|
|
237
|
+
lastId = res?.message_id
|
|
238
|
+
}
|
|
239
|
+
return lastId
|
|
240
|
+
},
|
|
241
|
+
textAlreadyDelivered: (chatId, threadId, text) => deps.dedupCheck(chatId, threadId ?? undefined, text),
|
|
242
|
+
registryChainLookup: (taskId) => {
|
|
243
|
+
const db = deps.getTurnsDb()
|
|
244
|
+
if (db == null) return null
|
|
245
|
+
const turnKey = resolveSubagentOriginTurnKey(db, taskId)
|
|
246
|
+
return turnKey == null ? null : chatFromTurnKey(turnKey)
|
|
247
|
+
},
|
|
248
|
+
}).catch((err) => deps.log?.(`outbox-sweep: tick failed: ${(err as Error).message}\n`))
|
|
249
|
+
}
|
|
250
|
+
const timer = setInterval(tick, OUTBOX_SWEEP_INTERVAL_MS)
|
|
251
|
+
timer.unref?.()
|
|
252
|
+
return timer
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
/**
|
|
256
|
+
* Journal a delivery made by a NON-sweep machine (legacy turn-flush /
|
|
257
|
+
* captured-prose bridge / exhausted fallback / final-answer reply send) under
|
|
258
|
+
* the shared nonce, AND drop any pending outbox record for it — the reply-path
|
|
259
|
+
* clear-by-nonce (H1/F1). Wired at every legacy delivery site with the gateway's
|
|
260
|
+
* own `deriveTurnId` nonce (`turn.turnId`), which is byte-identical to the
|
|
261
|
+
* hook's `deriveTurnNonce` `${chatKey}#${messageId}` for a gateway-visible turn, so:
|
|
262
|
+
* - the journal write makes a later sweep skip-journaled even after the
|
|
263
|
+
* in-memory `outboundDedup` TTL has evicted or the gateway restarted, and
|
|
264
|
+
* - `clearOutboxRecord` removes the hook's captured record immediately when
|
|
265
|
+
* the flush text differs from the captured text (text-dedup would miss).
|
|
266
|
+
* Best-effort; never throws. A null/empty nonce is ignored (nothing to journal).
|
|
267
|
+
*/
|
|
268
|
+
export function journalExternalDelivery(
|
|
269
|
+
args: { turnNonce: string | null; text: string; tgMessageId?: number },
|
|
270
|
+
stateDir?: string,
|
|
271
|
+
now: number = Date.now(),
|
|
272
|
+
): void {
|
|
273
|
+
const nonce = args.turnNonce
|
|
274
|
+
if (nonce == null || nonce === '') return
|
|
275
|
+
appendDelivered(
|
|
276
|
+
{ turnNonce: nonce, textSha256: sha256Hex(args.text), tgMessageId: args.tgMessageId, ts: now },
|
|
277
|
+
stateDir,
|
|
278
|
+
)
|
|
279
|
+
clearOutboxRecord(nonce, stateDir)
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
export type { OutboxRecord }
|