switchroom 0.14.64 → 0.14.66
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/switchroom.js +3 -3
- package/package.json +1 -1
- package/telegram-plugin/dist/gateway/gateway.js +109 -20
- package/telegram-plugin/gateway/answer-thread-resolve.test.ts +85 -0
- package/telegram-plugin/gateway/answer-thread-resolve.ts +30 -4
- package/telegram-plugin/gateway/gateway.ts +174 -19
- package/telegram-plugin/gateway/source-message-id.test.ts +32 -0
- package/telegram-plugin/gateway/source-message-id.ts +41 -0
- package/telegram-plugin/gateway/status-surface-log.test.ts +98 -0
- package/telegram-plugin/gateway/status-surface-log.ts +102 -0
- package/telegram-plugin/silence-poke.ts +47 -0
- package/telegram-plugin/tests/multitopic-routing-wiring.test.ts +4 -2
- package/telegram-plugin/tests/silence-poke.test.ts +69 -1
- package/telegram-plugin/tests/worker-activity-feed.test.ts +61 -0
- package/telegram-plugin/worker-activity-feed.ts +15 -0
package/dist/cli/switchroom.js
CHANGED
|
@@ -49452,8 +49452,8 @@ var {
|
|
|
49452
49452
|
} = import__.default;
|
|
49453
49453
|
|
|
49454
49454
|
// src/build-info.ts
|
|
49455
|
-
var VERSION = "0.14.
|
|
49456
|
-
var COMMIT_SHA = "
|
|
49455
|
+
var VERSION = "0.14.66";
|
|
49456
|
+
var COMMIT_SHA = "0f4f029d";
|
|
49457
49457
|
|
|
49458
49458
|
// src/cli/agent.ts
|
|
49459
49459
|
init_source();
|
|
@@ -52027,7 +52027,7 @@ function buildSettingsHooksBlock(p) {
|
|
|
52027
52027
|
|
|
52028
52028
|
` + 'Do NOT send a trailing confirmation after your answer \u2014 no "Done.", ' + '"Sent.", "Hope that helps." as a separate message once you have ' + "already replied. Your answer is the last thing the user should " + `see; a follow-up "Done." is dead-air clutter (and the user's ` + `device already pinged on the answer). Stop after the answer.
|
|
52029
52029
|
|
|
52030
|
-
` + 'CRITICAL: "answer" means a call to the reply tool ' + "(mcp__switchroom-telegram__reply, or stream_reply with done=true). " + "Your terminal/transcript text is NEVER delivered to Telegram \u2014 the " + "user sees only what you send through the reply tool. After a long " + "tool sequence (scheduling, multi-step research, sub-agent handback), " + "do not let your closing narration stand as the answer: end the turn " + "by passing that narration to the reply tool. No reply tool call = the " + "user got nothing, however much text you wrote.</turn-pacing>";
|
|
52030
|
+
` + 'CRITICAL: "answer" means a call to the reply tool ' + "(mcp__switchroom-telegram__reply, or stream_reply with done=true). " + "Your terminal/transcript text is NEVER delivered to Telegram \u2014 the " + "user sees only what you send through the reply tool. After a long " + "tool sequence (scheduling, multi-step research, sub-agent handback), " + "do not let your closing narration stand as the answer: end the turn " + "by passing that narration to the reply tool. No reply tool call = the " + "user got nothing, however much text you wrote. Call the reply tool as " + "your FIRST action when you have the answer \u2014 do not write it out as " + "transcript text first and call reply afterward: a framework backstop " + "flushes unsent text after a delay and then your real reply lands late " + "and out of order.</turn-pacing>";
|
|
52031
52031
|
const switchroomUserPromptSubmit = [
|
|
52032
52032
|
...useHotReloadStable ? [
|
|
52033
52033
|
{
|
package/package.json
CHANGED
|
@@ -32622,6 +32622,7 @@ function createWorkerActivityFeed(opts) {
|
|
|
32622
32622
|
h.messageId = sent.message_id;
|
|
32623
32623
|
h.lastBody = body;
|
|
32624
32624
|
h.lastEditAt = nowFn();
|
|
32625
|
+
log(`worker-feed: paint agent=${h.agentId} chat=${h.chatId} ` + `thread=${h.threadId ?? "-"} msgId=${h.messageId} bytes=${body.length}`);
|
|
32625
32626
|
} catch (err) {
|
|
32626
32627
|
noteRateLimited(h, err, "send");
|
|
32627
32628
|
log(`worker-feed: send failed: ${err.message}`);
|
|
@@ -32636,6 +32637,7 @@ function createWorkerActivityFeed(opts) {
|
|
|
32636
32637
|
await opts.bot.editMessageText(h.chatId, h.messageId, body, sendOptsFor(h));
|
|
32637
32638
|
h.lastBody = body;
|
|
32638
32639
|
h.lastEditAt = nowFn();
|
|
32640
|
+
log(`worker-feed: edit agent=${h.agentId} chat=${h.chatId} ` + `thread=${h.threadId ?? "-"} msgId=${h.messageId} bytes=${body.length}`);
|
|
32639
32641
|
} catch (err) {
|
|
32640
32642
|
noteRateLimited(h, err, "edit");
|
|
32641
32643
|
log(`worker-feed: edit failed, will re-post: ${err.message}`);
|
|
@@ -32656,6 +32658,7 @@ function createWorkerActivityFeed(opts) {
|
|
|
32656
32658
|
await opts.bot.editMessageText(h.chatId, h.messageId, body, sendOptsFor(h));
|
|
32657
32659
|
h.lastBody = body;
|
|
32658
32660
|
h.lastEditAt = nowFn();
|
|
32661
|
+
log(`worker-feed: finish agent=${h.agentId} chat=${h.chatId} ` + `thread=${h.threadId ?? "-"} msgId=${h.messageId} state=${view.state} bytes=${body.length}`);
|
|
32659
32662
|
} catch (err) {
|
|
32660
32663
|
noteRateLimited(h, err, "finish");
|
|
32661
32664
|
log(`worker-feed: finish edit failed: ${err.message}`);
|
|
@@ -32674,6 +32677,7 @@ function createWorkerActivityFeed(opts) {
|
|
|
32674
32677
|
let h = handles.get(agentId);
|
|
32675
32678
|
if (h == null) {
|
|
32676
32679
|
h = {
|
|
32680
|
+
agentId,
|
|
32677
32681
|
chatId,
|
|
32678
32682
|
threadId,
|
|
32679
32683
|
messageId: null,
|
|
@@ -32708,6 +32712,38 @@ function createWorkerActivityFeed(opts) {
|
|
|
32708
32712
|
};
|
|
32709
32713
|
}
|
|
32710
32714
|
|
|
32715
|
+
// gateway/status-surface-log.ts
|
|
32716
|
+
function formatTurnLifecycle(action, reason, t, now) {
|
|
32717
|
+
const ageMs = action === "clear" ? Math.max(0, now - t.startedAt) : 0;
|
|
32718
|
+
return `turn-lifecycle ${action} reason=${reason} turnId=${t.turnId} ` + `chat=${t.sessionChatId} thread=${t.sessionThreadId ?? "-"} ` + `tools=${t.toolCallCount} activityMsgId=${t.activityMessageId ?? "none"} ` + `feedOpened=${t.activityEverOpened} drainFailures=${t.activityDrainFailures} ` + `replyCalled=${t.replyCalled} finalAnswer=${t.finalAnswerDelivered} age_ms=${ageMs}`;
|
|
32719
|
+
}
|
|
32720
|
+
function detectStatusSurfaceDegraded(t) {
|
|
32721
|
+
if (t.toolCallCount === 0)
|
|
32722
|
+
return null;
|
|
32723
|
+
if (t.activityEverOpened)
|
|
32724
|
+
return null;
|
|
32725
|
+
if (t.activityDrainFailures === 0)
|
|
32726
|
+
return null;
|
|
32727
|
+
return {
|
|
32728
|
+
reason: "feed-never-opened",
|
|
32729
|
+
detail: `tools=${t.toolCallCount} drainFailures=${t.activityDrainFailures} ` + `activityMsgId=none \u2014 the live activity feed failed every send this turn ` + `(card was dark despite tool work)`
|
|
32730
|
+
};
|
|
32731
|
+
}
|
|
32732
|
+
|
|
32733
|
+
// gateway/source-message-id.ts
|
|
32734
|
+
var MAX_TELEGRAM_MESSAGE_ID = 2 ** 31;
|
|
32735
|
+
function parseSourceMessageId(raw) {
|
|
32736
|
+
if (raw == null)
|
|
32737
|
+
return null;
|
|
32738
|
+
const s = String(raw);
|
|
32739
|
+
if (!/^\d+$/.test(s))
|
|
32740
|
+
return null;
|
|
32741
|
+
const n = Number(s);
|
|
32742
|
+
if (!Number.isSafeInteger(n) || n <= 0 || n >= MAX_TELEGRAM_MESSAGE_ID)
|
|
32743
|
+
return null;
|
|
32744
|
+
return n;
|
|
32745
|
+
}
|
|
32746
|
+
|
|
32711
32747
|
// tool-names.ts
|
|
32712
32748
|
var TELEGRAM_TOOL_PREFIX_RE = /^mcp__[^_].*?telegram__/;
|
|
32713
32749
|
function stripPrefix(toolName) {
|
|
@@ -39067,6 +39103,9 @@ function tick(now) {
|
|
|
39067
39103
|
if (silence < 0)
|
|
39068
39104
|
continue;
|
|
39069
39105
|
if (!s.fallbackFired && silence >= thresholds.fallback) {
|
|
39106
|
+
if (activeDeps.deferFallbackWhileToolInFlight === true && s.inFlightTools.size > 0 && silence < (thresholds.fallbackHardCeiling ?? Number.POSITIVE_INFINITY)) {
|
|
39107
|
+
continue;
|
|
39108
|
+
}
|
|
39070
39109
|
s.fallbackFired = true;
|
|
39071
39110
|
const { chatId, threadId } = parseKey(key);
|
|
39072
39111
|
const recentThinking = s.lastThinkingAt != null && now - s.lastThinkingAt < 30000;
|
|
@@ -47897,6 +47936,10 @@ function resolveAnswerThreadId(input) {
|
|
|
47897
47936
|
return input.explicitThreadId;
|
|
47898
47937
|
if (input.originResolved)
|
|
47899
47938
|
return input.originThreadId;
|
|
47939
|
+
if (input.liveThreadId != null)
|
|
47940
|
+
return input.liveThreadId;
|
|
47941
|
+
if (input.lastEndedResolvedForChat)
|
|
47942
|
+
return input.lastEndedThreadIdForChat;
|
|
47900
47943
|
return input.liveThreadId;
|
|
47901
47944
|
}
|
|
47902
47945
|
|
|
@@ -52720,11 +52763,11 @@ function sweepStaleTurnActiveMarker(stateDir, opts) {
|
|
|
52720
52763
|
}
|
|
52721
52764
|
|
|
52722
52765
|
// ../src/build-info.ts
|
|
52723
|
-
var VERSION = "0.14.
|
|
52724
|
-
var COMMIT_SHA = "
|
|
52725
|
-
var COMMIT_DATE = "2026-06-
|
|
52726
|
-
var LATEST_PR =
|
|
52727
|
-
var COMMITS_AHEAD_OF_TAG =
|
|
52766
|
+
var VERSION = "0.14.66";
|
|
52767
|
+
var COMMIT_SHA = "0f4f029d";
|
|
52768
|
+
var COMMIT_DATE = "2026-06-05T07:05:45Z";
|
|
52769
|
+
var LATEST_PR = 2167;
|
|
52770
|
+
var COMMITS_AHEAD_OF_TAG = 2;
|
|
52728
52771
|
|
|
52729
52772
|
// gateway/boot-version.ts
|
|
52730
52773
|
function formatRelativeAgo(iso) {
|
|
@@ -54022,6 +54065,33 @@ function findTurnByOriginId(originTurnId) {
|
|
|
54022
54065
|
return currentTurn;
|
|
54023
54066
|
return recentTurnsById.get(originTurnId) ?? null;
|
|
54024
54067
|
}
|
|
54068
|
+
var LATE_REPLY_TOPIC_RECOVERY_ENABLED = process.env.SWITCHROOM_LATE_REPLY_TOPIC_RECOVERY !== "0";
|
|
54069
|
+
function findLatestEndedTurnForChat(chatId) {
|
|
54070
|
+
let latest = null;
|
|
54071
|
+
for (const t of recentTurnsById.values()) {
|
|
54072
|
+
if (t.sessionChatId === chatId)
|
|
54073
|
+
latest = t;
|
|
54074
|
+
}
|
|
54075
|
+
return latest;
|
|
54076
|
+
}
|
|
54077
|
+
function resolveAnswerThreadWithLog(chatId, explicitThreadId, originTurn, liveTurn, surface) {
|
|
54078
|
+
const recovered = LATE_REPLY_TOPIC_RECOVERY_ENABLED && explicitThreadId == null && originTurn == null && liveTurn?.sessionThreadId == null ? findLatestEndedTurnForChat(chatId) : null;
|
|
54079
|
+
const threadId = resolveAnswerThreadId({
|
|
54080
|
+
explicitThreadId,
|
|
54081
|
+
originResolved: originTurn != null,
|
|
54082
|
+
originThreadId: originTurn?.sessionThreadId,
|
|
54083
|
+
liveThreadId: liveTurn?.sessionThreadId,
|
|
54084
|
+
lastEndedResolvedForChat: recovered != null,
|
|
54085
|
+
lastEndedThreadIdForChat: recovered?.sessionThreadId
|
|
54086
|
+
});
|
|
54087
|
+
const via = explicitThreadId != null ? "explicit" : originTurn != null ? "origin" : liveTurn?.sessionThreadId != null ? "live" : recovered != null ? "recovered" : "none";
|
|
54088
|
+
const ownerTurn = originTurn ?? recovered ?? liveTurn;
|
|
54089
|
+
const isSupergroup = chatId.startsWith("-100");
|
|
54090
|
+
const unrouted = isSupergroup && threadId == null;
|
|
54091
|
+
process.stderr.write(`telegram gateway: reply-route surface=${surface} chat=${chatId} resolved_thread=${threadId ?? "-"} via=${via} late=${liveTurn == null} originTurn=${ownerTurn?.turnId ?? "-"} origin_thread=${ownerTurn?.sessionThreadId ?? "-"}` + (via === "recovered" ? " RECOVERED" : "") + (unrouted ? " UNROUTED(supergroup\u2192no-topic)" : "") + `
|
|
54092
|
+
`);
|
|
54093
|
+
return threadId;
|
|
54094
|
+
}
|
|
54025
54095
|
function closeObligationOnSubstantiveReply(args, liveTurn) {
|
|
54026
54096
|
if (!OBLIGATION_LEDGER_ENABLED)
|
|
54027
54097
|
return;
|
|
@@ -54267,6 +54337,13 @@ function endCurrentTurnAtomic(turn) {
|
|
|
54267
54337
|
if (currentTurn !== turn)
|
|
54268
54338
|
return;
|
|
54269
54339
|
currentTurn = null;
|
|
54340
|
+
process.stderr.write(`telegram gateway: ${formatTurnLifecycle("clear", "turn_end", turn, Date.now())}
|
|
54341
|
+
`);
|
|
54342
|
+
const degraded = detectStatusSurfaceDegraded(turn);
|
|
54343
|
+
if (degraded != null) {
|
|
54344
|
+
process.stderr.write(`telegram gateway: status-surface DEGRADED reason=${degraded.reason} turnId=${turn.turnId} chat=${turn.sessionChatId} thread=${turn.sessionThreadId ?? "-"} ${degraded.detail}
|
|
54345
|
+
`);
|
|
54346
|
+
}
|
|
54270
54347
|
if (OBLIGATION_LEDGER_ENABLED) {
|
|
54271
54348
|
if (turn.finalAnswerDelivered) {
|
|
54272
54349
|
obligationLedger.close(turn.turnId);
|
|
@@ -54344,7 +54421,11 @@ async function postCompactCard(occ, cap) {
|
|
|
54344
54421
|
const chatId = loadAccess().allowFrom[0];
|
|
54345
54422
|
if (!chatId)
|
|
54346
54423
|
return;
|
|
54347
|
-
const threadId =
|
|
54424
|
+
const threadId = topicForRecipient({
|
|
54425
|
+
recipientChatId: chatId,
|
|
54426
|
+
resolvedTopic: resolveAgentOutboundTopic({ kind: "compact-watchdog" }) ?? chatThreadMap.get(chatId),
|
|
54427
|
+
supergroupChatId: resolveAgentSupergroupChatId()
|
|
54428
|
+
});
|
|
54348
54429
|
const text = `\uD83D\uDDDC\uFE0F <b>Context compaction</b>
|
|
54349
54430
|
` + `Working context hit ~${occ.toLocaleString()} tokens (cap ${cap.toLocaleString()}) \u2014 running <code>/compact</code>. ` + `Older detail moves to Hindsight; I'll confirm here once the context has shrunk (may take a turn or two).`;
|
|
54350
54431
|
const sent = await swallowingApiCall(() => bot.api.sendMessage(chatId, text, {
|
|
@@ -55169,7 +55250,19 @@ function ensureIssuesCard(chatId, threadId) {
|
|
|
55169
55250
|
}
|
|
55170
55251
|
}
|
|
55171
55252
|
var inFlightUpdate = null;
|
|
55253
|
+
function parsePositiveMsEnv(name, fallbackMs) {
|
|
55254
|
+
const raw = process.env[name];
|
|
55255
|
+
if (raw == null || raw === "")
|
|
55256
|
+
return fallbackMs;
|
|
55257
|
+
const n = Number(raw);
|
|
55258
|
+
return Number.isFinite(n) && n > 0 ? Math.floor(n) : fallbackMs;
|
|
55259
|
+
}
|
|
55260
|
+
var SILENCE_FALLBACK_MS = parsePositiveMsEnv("SWITCHROOM_SILENCE_FALLBACK_MS", 300000);
|
|
55261
|
+
var SILENCE_FALLBACK_HARD_MS = parsePositiveMsEnv("SWITCHROOM_SILENCE_FALLBACK_HARD_MS", 900000);
|
|
55262
|
+
var SILENCE_DEFER_INFLIGHT_TOOLS = process.env.SWITCHROOM_SILENCE_DEFER_INFLIGHT_TOOLS === "1";
|
|
55172
55263
|
startTimer({
|
|
55264
|
+
thresholdsMs: { fallback: SILENCE_FALLBACK_MS, fallbackHardCeiling: SILENCE_FALLBACK_HARD_MS },
|
|
55265
|
+
deferFallbackWhileToolInFlight: SILENCE_DEFER_INFLIGHT_TOOLS,
|
|
55173
55266
|
emitMetric: (event) => {
|
|
55174
55267
|
emitRuntimeMetric(event);
|
|
55175
55268
|
},
|
|
@@ -56229,12 +56322,7 @@ ${url}`;
|
|
|
56229
56322
|
if (TURN_ORIGIN_ROUTING_ENABLED) {
|
|
56230
56323
|
const explicit = args.message_thread_id != null ? Number(args.message_thread_id) : undefined;
|
|
56231
56324
|
const originTurn = findTurnByOriginId(args.origin_turn_id);
|
|
56232
|
-
threadId =
|
|
56233
|
-
explicitThreadId: Number.isFinite(explicit) ? explicit : undefined,
|
|
56234
|
-
originResolved: originTurn != null,
|
|
56235
|
-
originThreadId: originTurn?.sessionThreadId,
|
|
56236
|
-
liveThreadId: turn?.sessionThreadId
|
|
56237
|
-
});
|
|
56325
|
+
threadId = resolveAnswerThreadWithLog(chat_id, Number.isFinite(explicit) ? explicit : undefined, originTurn, turn, "reply");
|
|
56238
56326
|
} else {
|
|
56239
56327
|
threadId = resolveThreadId(chat_id, args.message_thread_id ?? (turn?.sessionThreadId != null ? turn.sessionThreadId : undefined));
|
|
56240
56328
|
}
|
|
@@ -56594,12 +56682,7 @@ async function executeStreamReply(args) {
|
|
|
56594
56682
|
let injected;
|
|
56595
56683
|
if (TURN_ORIGIN_ROUTING_ENABLED) {
|
|
56596
56684
|
const originTurn = findTurnByOriginId(args.origin_turn_id);
|
|
56597
|
-
injected =
|
|
56598
|
-
explicitThreadId: undefined,
|
|
56599
|
-
originResolved: originTurn != null,
|
|
56600
|
-
originThreadId: originTurn?.sessionThreadId,
|
|
56601
|
-
liveThreadId: turn?.sessionThreadId
|
|
56602
|
-
});
|
|
56685
|
+
injected = resolveAnswerThreadWithLog(String(args.chat_id), undefined, originTurn, turn, "stream_reply");
|
|
56603
56686
|
} else {
|
|
56604
56687
|
injected = turn?.sessionThreadId;
|
|
56605
56688
|
}
|
|
@@ -57760,6 +57843,7 @@ async function drainActivitySummary(turn) {
|
|
|
57760
57843
|
...replyAnchor
|
|
57761
57844
|
}), { chat_id: chat, ...thread != null ? { threadId: thread } : {}, verb: "activity-summary.send" });
|
|
57762
57845
|
turn.activityMessageId = sent.message_id;
|
|
57846
|
+
turn.activityEverOpened = true;
|
|
57763
57847
|
} else {
|
|
57764
57848
|
const id = turn.activityMessageId;
|
|
57765
57849
|
await robustApiCall(() => bot.api.editMessageText(chat, id, html, { parse_mode: "HTML" }), { chat_id: chat, ...thread != null ? { threadId: thread } : {}, verb: "activity-summary.edit" });
|
|
@@ -57768,7 +57852,8 @@ async function drainActivitySummary(turn) {
|
|
|
57768
57852
|
} catch (err) {
|
|
57769
57853
|
const msg = err instanceof Error ? err.message : String(err);
|
|
57770
57854
|
if (!msg.includes("message is not modified")) {
|
|
57771
|
-
|
|
57855
|
+
turn.activityDrainFailures += 1;
|
|
57856
|
+
process.stderr.write(`telegram gateway: activity-summary drain failed: ${msg} (chat=${chat} thread=${thread ?? "-"} replyAnchor=${turn.sourceMessageId ?? "none"} everOpened=${turn.activityEverOpened} failures=${turn.activityDrainFailures})
|
|
57772
57857
|
`);
|
|
57773
57858
|
}
|
|
57774
57859
|
turn.activityLastSentRender = target;
|
|
@@ -57855,7 +57940,7 @@ function handleSessionEvent(ev) {
|
|
|
57855
57940
|
const next = {
|
|
57856
57941
|
sessionChatId: ev.chatId,
|
|
57857
57942
|
sessionThreadId: enqThreadIdNum,
|
|
57858
|
-
sourceMessageId:
|
|
57943
|
+
sourceMessageId: parseSourceMessageId(ev.messageId),
|
|
57859
57944
|
startedAt,
|
|
57860
57945
|
gatewayReceiveAt: startedAt,
|
|
57861
57946
|
replyCalled: false,
|
|
@@ -57876,12 +57961,16 @@ function handleSessionEvent(ev) {
|
|
|
57876
57961
|
activityInFlight: null,
|
|
57877
57962
|
activityPendingRender: null,
|
|
57878
57963
|
activityLastSentRender: null,
|
|
57964
|
+
activityEverOpened: false,
|
|
57965
|
+
activityDrainFailures: 0,
|
|
57879
57966
|
mirrorLines: [],
|
|
57880
57967
|
foregroundSubAgents: new Map,
|
|
57881
57968
|
answerStream: null,
|
|
57882
57969
|
isDm: isDmChatId(ev.chatId)
|
|
57883
57970
|
};
|
|
57884
57971
|
currentTurn = next;
|
|
57972
|
+
process.stderr.write(`telegram gateway: ${formatTurnLifecycle("set", "enqueue", next, startedAt)}
|
|
57973
|
+
`);
|
|
57885
57974
|
rememberRecentTurn(next);
|
|
57886
57975
|
promoteQueuedStatus(ev.chatId, enqThreadIdNum);
|
|
57887
57976
|
if (DELIVERY_CONFIRM_ENABLED) {
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import { describe, it, expect } from 'vitest'
|
|
2
|
+
import { resolveAnswerThreadId } from './answer-thread-resolve.js'
|
|
3
|
+
|
|
4
|
+
describe('resolveAnswerThreadId — precedence', () => {
|
|
5
|
+
it('(1) explicit model thread wins over everything', () => {
|
|
6
|
+
expect(
|
|
7
|
+
resolveAnswerThreadId({
|
|
8
|
+
explicitThreadId: 7,
|
|
9
|
+
originResolved: true,
|
|
10
|
+
originThreadId: 3,
|
|
11
|
+
liveThreadId: 4,
|
|
12
|
+
lastEndedResolvedForChat: true,
|
|
13
|
+
lastEndedThreadIdForChat: 9,
|
|
14
|
+
}),
|
|
15
|
+
).toBe(7)
|
|
16
|
+
})
|
|
17
|
+
|
|
18
|
+
it('(2) origin turn thread wins over the live turn (the Brevo→Meta fix)', () => {
|
|
19
|
+
expect(
|
|
20
|
+
resolveAnswerThreadId({ originResolved: true, originThreadId: 3, liveThreadId: 4 }),
|
|
21
|
+
).toBe(3)
|
|
22
|
+
})
|
|
23
|
+
|
|
24
|
+
it('(2) a DM origin (resolved, thread undefined) pins to undefined, not the live thread', () => {
|
|
25
|
+
expect(
|
|
26
|
+
resolveAnswerThreadId({ originResolved: true, originThreadId: undefined, liveThreadId: 4 }),
|
|
27
|
+
).toBeUndefined()
|
|
28
|
+
})
|
|
29
|
+
|
|
30
|
+
it('(3) no origin → falls back to the live turn thread (legacy #1664)', () => {
|
|
31
|
+
expect(
|
|
32
|
+
resolveAnswerThreadId({ originResolved: false, liveThreadId: 4 }),
|
|
33
|
+
).toBe(4)
|
|
34
|
+
})
|
|
35
|
+
|
|
36
|
+
// ── tier (4): late-reply topic recovery (2026-06-05) ──────────────────────
|
|
37
|
+
it('(4) no explicit, no origin, NO live turn → recovers the most-recent ended turn thread', () => {
|
|
38
|
+
// The marko bug: a reply that fired after the orphaned-reply backstop ended
|
|
39
|
+
// its turn. Pre-fix this returned undefined (General); now it recovers topic 3.
|
|
40
|
+
expect(
|
|
41
|
+
resolveAnswerThreadId({
|
|
42
|
+
originResolved: false,
|
|
43
|
+
liveThreadId: undefined,
|
|
44
|
+
lastEndedResolvedForChat: true,
|
|
45
|
+
lastEndedThreadIdForChat: 3,
|
|
46
|
+
}),
|
|
47
|
+
).toBe(3)
|
|
48
|
+
})
|
|
49
|
+
|
|
50
|
+
it('(4) a recovered DM turn (ended, thread undefined) stays threadless', () => {
|
|
51
|
+
expect(
|
|
52
|
+
resolveAnswerThreadId({
|
|
53
|
+
originResolved: false,
|
|
54
|
+
liveThreadId: undefined,
|
|
55
|
+
lastEndedResolvedForChat: true,
|
|
56
|
+
lastEndedThreadIdForChat: undefined,
|
|
57
|
+
}),
|
|
58
|
+
).toBeUndefined()
|
|
59
|
+
})
|
|
60
|
+
|
|
61
|
+
it('(4) recovery does NOT override a live turn — live thread still wins at tier 3', () => {
|
|
62
|
+
expect(
|
|
63
|
+
resolveAnswerThreadId({
|
|
64
|
+
originResolved: false,
|
|
65
|
+
liveThreadId: 4,
|
|
66
|
+
lastEndedResolvedForChat: true,
|
|
67
|
+
lastEndedThreadIdForChat: 3,
|
|
68
|
+
}),
|
|
69
|
+
).toBe(4)
|
|
70
|
+
})
|
|
71
|
+
|
|
72
|
+
it('(4) no recovery candidate → legacy result (undefined), unchanged', () => {
|
|
73
|
+
expect(
|
|
74
|
+
resolveAnswerThreadId({
|
|
75
|
+
originResolved: false,
|
|
76
|
+
liveThreadId: undefined,
|
|
77
|
+
lastEndedResolvedForChat: false,
|
|
78
|
+
}),
|
|
79
|
+
).toBeUndefined()
|
|
80
|
+
})
|
|
81
|
+
|
|
82
|
+
it('pure DM (every tier undefined) → undefined', () => {
|
|
83
|
+
expect(resolveAnswerThreadId({ originResolved: false })).toBeUndefined()
|
|
84
|
+
})
|
|
85
|
+
})
|
|
@@ -26,10 +26,14 @@
|
|
|
26
26
|
* 3. Else the LIVE turn's thread — but ONLY when the live turn IS the
|
|
27
27
|
* origin turn (no flip happened) OR no origin turn could be resolved
|
|
28
28
|
* at all (origin id absent/unknown; legacy / pre-stamp path).
|
|
29
|
-
* 4. Else (
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
*
|
|
29
|
+
* 4. Else (no explicit, no origin echoed, no live turn) — a LATE reply that
|
|
30
|
+
* fired after its turn already ended (the orphaned-reply backstop case) —
|
|
31
|
+
* recover the origin topic from the most-recently-ended turn for this
|
|
32
|
+
* chat. Without this, such a reply defaults to the main chat (General in a
|
|
33
|
+
* supergroup) and its answer vanishes from the topic the user is reading
|
|
34
|
+
* (the 2026-06-05 marko triage). Still NOT the `chatThreadMap` last-seen
|
|
35
|
+
* heuristic — the recovered turn is the chat's own most-recent turn, not
|
|
36
|
+
* whichever topic last received any message.
|
|
33
37
|
*
|
|
34
38
|
* The `chatThreadMap` last-seen fallback is preserved for NON-answer
|
|
35
39
|
* surfaces (`send_typing`, `forward_message`, `progress_update`) by NOT
|
|
@@ -53,6 +57,20 @@ export interface AnswerThreadInput {
|
|
|
53
57
|
* (no live turn, or a DM live turn). The legacy (#1664) fallback when
|
|
54
58
|
* no origin turn is resolvable. */
|
|
55
59
|
liveThreadId?: number | undefined
|
|
60
|
+
/**
|
|
61
|
+
* Late-reply topic recovery (2026-06-05). Thread of the most-recently-ended
|
|
62
|
+
* turn for THIS chat (from `recentTurnsById`), used as a deterministic
|
|
63
|
+
* fallback when the model echoed no `origin_turn_id` AND there is no live
|
|
64
|
+
* turn — the late-reply-after-turn-end case. Without it, a reply that fires
|
|
65
|
+
* after the orphaned-reply backstop closed its turn defaults to the main chat
|
|
66
|
+
* (General topic in a supergroup), so its answer vanishes from the topic the
|
|
67
|
+
* user is reading. Only consulted at tier (4); a DM origin yields undefined,
|
|
68
|
+
* which is correct.
|
|
69
|
+
*/
|
|
70
|
+
lastEndedThreadIdForChat?: number | undefined
|
|
71
|
+
/** Whether a recently-ended turn exists for this chat — distinguishes
|
|
72
|
+
* "ended turn exists, DM (thread undefined)" from "no ended turn at all". */
|
|
73
|
+
lastEndedResolvedForChat?: boolean
|
|
56
74
|
}
|
|
57
75
|
|
|
58
76
|
/**
|
|
@@ -75,5 +93,13 @@ export function resolveAnswerThreadId(input: AnswerThreadInput): number | undefi
|
|
|
75
93
|
if (input.originResolved) return input.originThreadId
|
|
76
94
|
// (3) no origin resolved (legacy / pre-stamp / evicted) → fall back to
|
|
77
95
|
// the live turn's thread, the existing turn-pinned behaviour (#1664).
|
|
96
|
+
if (input.liveThreadId != null) return input.liveThreadId
|
|
97
|
+
// (4) no explicit, no origin echoed, no live turn — a LATE reply that fired
|
|
98
|
+
// after its turn already ended (the orphaned-reply backstop case).
|
|
99
|
+
// Recover the origin topic from the most-recently-ended turn for this
|
|
100
|
+
// chat so the answer lands in the topic it belongs to instead of
|
|
101
|
+
// defaulting to the main chat (General). When no ended turn is known,
|
|
102
|
+
// fall through to liveThreadId (undefined) — the legacy result.
|
|
103
|
+
if (input.lastEndedResolvedForChat) return input.lastEndedThreadIdForChat
|
|
78
104
|
return input.liveThreadId
|
|
79
105
|
}
|
|
@@ -65,6 +65,8 @@ import {
|
|
|
65
65
|
import { StatusReactionController } from '../status-reactions.js'
|
|
66
66
|
import { DeferredDoneReactions } from '../reaction-defer.js'
|
|
67
67
|
import { createWorkerActivityFeed, isWorkerActivityFeedEnabled } from '../worker-activity-feed.js'
|
|
68
|
+
import { formatTurnLifecycle, detectStatusSurfaceDegraded } from './status-surface-log.js'
|
|
69
|
+
import { parseSourceMessageId } from './source-message-id.js'
|
|
68
70
|
import { isTelegramReplyTool, isTelegramSurfaceTool } from '../tool-names.js'
|
|
69
71
|
import { appendActivityLabel, renderActivityFeedWithNested } from '../tool-activity-summary.js'
|
|
70
72
|
import { toolLabel } from '../tool-labels.js'
|
|
@@ -1798,6 +1800,14 @@ type CurrentTurn = {
|
|
|
1798
1800
|
activityInFlight: Promise<void> | null
|
|
1799
1801
|
activityPendingRender: string | null
|
|
1800
1802
|
activityLastSentRender: string | null
|
|
1803
|
+
// Status-surface observability. `activityEverOpened` is sticky-true once the
|
|
1804
|
+
// feed posts its first message — unlike `activityMessageId`, it is NOT nulled
|
|
1805
|
+
// by `clearActivitySummary`, so the turn-end DEGRADED check can tell "feed
|
|
1806
|
+
// never opened" (the resume-400 signature) from "feed finalized + cleared".
|
|
1807
|
+
// `activityDrainFailures` counts real activity-feed send/edit failures this
|
|
1808
|
+
// turn (429s + "message is not modified" excluded). Both reset per turn.
|
|
1809
|
+
activityEverOpened: boolean
|
|
1810
|
+
activityDrainFailures: number
|
|
1801
1811
|
// Wall-clock anchor for the newest in-progress feed step — set each time a
|
|
1802
1812
|
// tool_label re-renders the feed. The heartbeat (`feedHeartbeatTick`) reads
|
|
1803
1813
|
// it to show a climbing " · Ns" elapsed on the live line so a long single
|
|
@@ -1878,6 +1888,83 @@ function findTurnByOriginId(originTurnId: string | null | undefined): CurrentTur
|
|
|
1878
1888
|
return recentTurnsById.get(originTurnId) ?? null
|
|
1879
1889
|
}
|
|
1880
1890
|
|
|
1891
|
+
// Late-reply topic recovery (2026-06-05 marko triage). Default ON; kill switch
|
|
1892
|
+
// SWITCHROOM_LATE_REPLY_TOPIC_RECOVERY=0 restores the legacy behaviour (a late
|
|
1893
|
+
// reply with no echoed origin and no live turn defaults to General).
|
|
1894
|
+
const LATE_REPLY_TOPIC_RECOVERY_ENABLED =
|
|
1895
|
+
process.env.SWITCHROOM_LATE_REPLY_TOPIC_RECOVERY !== '0'
|
|
1896
|
+
|
|
1897
|
+
/**
|
|
1898
|
+
* The most-recently-started turn for a chat from the bounded recently-ended
|
|
1899
|
+
* registry — the deterministic fallback for a LATE answer reply when the model
|
|
1900
|
+
* echoed no `origin_turn_id` and `currentTurn` has already cleared. Iterates in
|
|
1901
|
+
* insertion order so the last match is the most recent turn for that chat.
|
|
1902
|
+
* Returns null when the chat has no remembered turn (so the caller keeps the
|
|
1903
|
+
* legacy result). NB: this is the chat's own most-recent TURN, not the
|
|
1904
|
+
* `chatThreadMap` last-seen-any-message heuristic that caused the wrong-topic
|
|
1905
|
+
* bug — a late reply almost always belongs to the turn that just ended.
|
|
1906
|
+
*/
|
|
1907
|
+
function findLatestEndedTurnForChat(chatId: string): CurrentTurn | null {
|
|
1908
|
+
let latest: CurrentTurn | null = null
|
|
1909
|
+
for (const t of recentTurnsById.values()) {
|
|
1910
|
+
if (t.sessionChatId === chatId) latest = t
|
|
1911
|
+
}
|
|
1912
|
+
return latest
|
|
1913
|
+
}
|
|
1914
|
+
|
|
1915
|
+
/**
|
|
1916
|
+
* Resolve the answer-reply thread AND emit `reply-route` telemetry. The
|
|
1917
|
+
* 2026-06-05 triage showed reply routing was the blind spot: `reply: invoked`
|
|
1918
|
+
* logged only chat + char count, so a late reply landing in the wrong topic was
|
|
1919
|
+
* invisible without hand-correlating raw tg-post threads against turn-lifecycle
|
|
1920
|
+
* timestamps. This wrapper logs, per reply: which precedence tier won (`via`),
|
|
1921
|
+
* the resolved thread, the origin turn + its thread, and whether the reply was
|
|
1922
|
+
* late (turn already ended). `via=recovered` marks a late reply this fix saved
|
|
1923
|
+
* from General; `UNROUTED` flags a supergroup reply that still resolved to no
|
|
1924
|
+
* topic (the residual gap to watch).
|
|
1925
|
+
*/
|
|
1926
|
+
function resolveAnswerThreadWithLog(
|
|
1927
|
+
chatId: string,
|
|
1928
|
+
explicitThreadId: number | undefined,
|
|
1929
|
+
originTurn: CurrentTurn | null,
|
|
1930
|
+
liveTurn: CurrentTurn | null,
|
|
1931
|
+
surface: 'reply' | 'stream_reply',
|
|
1932
|
+
): number | undefined {
|
|
1933
|
+
const recovered =
|
|
1934
|
+
LATE_REPLY_TOPIC_RECOVERY_ENABLED &&
|
|
1935
|
+
explicitThreadId == null &&
|
|
1936
|
+
originTurn == null &&
|
|
1937
|
+
liveTurn?.sessionThreadId == null
|
|
1938
|
+
? findLatestEndedTurnForChat(chatId)
|
|
1939
|
+
: null
|
|
1940
|
+
const threadId = resolveAnswerThreadId({
|
|
1941
|
+
explicitThreadId,
|
|
1942
|
+
originResolved: originTurn != null,
|
|
1943
|
+
originThreadId: originTurn?.sessionThreadId,
|
|
1944
|
+
liveThreadId: liveTurn?.sessionThreadId,
|
|
1945
|
+
lastEndedResolvedForChat: recovered != null,
|
|
1946
|
+
lastEndedThreadIdForChat: recovered?.sessionThreadId,
|
|
1947
|
+
})
|
|
1948
|
+
const via =
|
|
1949
|
+
explicitThreadId != null ? 'explicit'
|
|
1950
|
+
: originTurn != null ? 'origin'
|
|
1951
|
+
: liveTurn?.sessionThreadId != null ? 'live'
|
|
1952
|
+
: recovered != null ? 'recovered'
|
|
1953
|
+
: 'none'
|
|
1954
|
+
const ownerTurn = originTurn ?? recovered ?? liveTurn
|
|
1955
|
+
const isSupergroup = chatId.startsWith('-100')
|
|
1956
|
+
const unrouted = isSupergroup && threadId == null
|
|
1957
|
+
process.stderr.write(
|
|
1958
|
+
`telegram gateway: reply-route surface=${surface} chat=${chatId} ` +
|
|
1959
|
+
`resolved_thread=${threadId ?? '-'} via=${via} late=${liveTurn == null} ` +
|
|
1960
|
+
`originTurn=${ownerTurn?.turnId ?? '-'} origin_thread=${ownerTurn?.sessionThreadId ?? '-'}` +
|
|
1961
|
+
(via === 'recovered' ? ' RECOVERED' : '') +
|
|
1962
|
+
(unrouted ? ' UNROUTED(supergroup→no-topic)' : '') +
|
|
1963
|
+
'\n',
|
|
1964
|
+
)
|
|
1965
|
+
return threadId
|
|
1966
|
+
}
|
|
1967
|
+
|
|
1881
1968
|
/**
|
|
1882
1969
|
* PR2 obligation-ledger CLOSE. Called when a SUBSTANTIVE final answer lands
|
|
1883
1970
|
* (not a bare interim ack — using finalAnswerSubstantive, the #2141 signal): the
|
|
@@ -2488,6 +2575,20 @@ function releaseTurnBufferGate(key: string, endingTurn?: CurrentTurn): void {
|
|
|
2488
2575
|
function endCurrentTurnAtomic(turn: CurrentTurn): void {
|
|
2489
2576
|
if (currentTurn !== turn) return
|
|
2490
2577
|
currentTurn = null
|
|
2578
|
+
// Status-surface observability: one line at every turn CLEAR (with how far
|
|
2579
|
+
// the turn got), plus a DEGRADED warning when the turn did tool work but the
|
|
2580
|
+
// live feed never opened because its sends failed (the resume-400 signature).
|
|
2581
|
+
process.stderr.write(
|
|
2582
|
+
`telegram gateway: ${formatTurnLifecycle('clear', 'turn_end', turn, Date.now())}\n`,
|
|
2583
|
+
)
|
|
2584
|
+
const degraded = detectStatusSurfaceDegraded(turn)
|
|
2585
|
+
if (degraded != null) {
|
|
2586
|
+
process.stderr.write(
|
|
2587
|
+
`telegram gateway: status-surface DEGRADED reason=${degraded.reason} ` +
|
|
2588
|
+
`turnId=${turn.turnId} chat=${turn.sessionChatId} ` +
|
|
2589
|
+
`thread=${turn.sessionThreadId ?? '-'} ${degraded.detail}\n`,
|
|
2590
|
+
)
|
|
2591
|
+
}
|
|
2491
2592
|
// PR2 obligation-ledger CLOSE-at-turn-end. Close the ended turn's obligation
|
|
2492
2593
|
// when it delivered a final answer. finalAnswerDelivered is the right signal
|
|
2493
2594
|
// HERE (not isSubstantiveFinalReply at reply-time): a SHORT genuine answer
|
|
@@ -2658,9 +2759,18 @@ async function postCompactCard(occ: number, cap: number): Promise<void> {
|
|
|
2658
2759
|
// instead of conversation lanes. Fleet/DM agents fall through to
|
|
2659
2760
|
// the existing chatThreadMap last-seen-thread fallback (no
|
|
2660
2761
|
// observable change).
|
|
2661
|
-
|
|
2662
|
-
|
|
2663
|
-
|
|
2762
|
+
// The compact-watchdog topic is valid ONLY in the agent's supergroup;
|
|
2763
|
+
// attaching it to an operator DM recipient 400s "message thread not found"
|
|
2764
|
+
// and the notice silently vanishes (the marko #2096 class — proactiveCompact
|
|
2765
|
+
// was the one operator-send still missing this guard, 2026-06-05). DM
|
|
2766
|
+
// recipients get a thread-less send; the supergroup owner keeps the lane.
|
|
2767
|
+
const threadId = topicForRecipient({
|
|
2768
|
+
recipientChatId: chatId,
|
|
2769
|
+
resolvedTopic:
|
|
2770
|
+
resolveAgentOutboundTopic({ kind: 'compact-watchdog' })
|
|
2771
|
+
?? chatThreadMap.get(chatId),
|
|
2772
|
+
supergroupChatId: resolveAgentSupergroupChatId(),
|
|
2773
|
+
});
|
|
2664
2774
|
const text =
|
|
2665
2775
|
`🗜️ <b>Context compaction</b>\n` +
|
|
2666
2776
|
`Working context hit ~${occ.toLocaleString()} tokens ` +
|
|
@@ -4546,7 +4656,27 @@ function ensureIssuesCard(chatId: string, threadId: number | undefined): void {
|
|
|
4546
4656
|
// incident fix. In-memory only; a gateway recreate naturally resets it.
|
|
4547
4657
|
let inFlightUpdate: { requestId: string; startedAt: number } | null = null
|
|
4548
4658
|
|
|
4659
|
+
// Fix A — silence-fallback tuning (status-surface darkening, 2026-06-05). A long
|
|
4660
|
+
// quiet tool stretch (foreground sub-agent / big research) crossed the 300s
|
|
4661
|
+
// fallback and nulled currentTurn, darkening the live activity feed mid-work.
|
|
4662
|
+
// SWITCHROOM_SILENCE_FALLBACK_MS — base threshold (default 300000)
|
|
4663
|
+
// SWITCHROOM_SILENCE_FALLBACK_HARD_MS — hard ceiling for the in-flight-tool
|
|
4664
|
+
// defer (default 900000 = 15min)
|
|
4665
|
+
// SWITCHROOM_SILENCE_DEFER_INFLIGHT_TOOLS=1 — enable the defer (default OFF;
|
|
4666
|
+
// canary on marko against #2162 telemetry)
|
|
4667
|
+
function parsePositiveMsEnv(name: string, fallbackMs: number): number {
|
|
4668
|
+
const raw = process.env[name]
|
|
4669
|
+
if (raw == null || raw === '') return fallbackMs
|
|
4670
|
+
const n = Number(raw)
|
|
4671
|
+
return Number.isFinite(n) && n > 0 ? Math.floor(n) : fallbackMs
|
|
4672
|
+
}
|
|
4673
|
+
const SILENCE_FALLBACK_MS = parsePositiveMsEnv('SWITCHROOM_SILENCE_FALLBACK_MS', 300_000)
|
|
4674
|
+
const SILENCE_FALLBACK_HARD_MS = parsePositiveMsEnv('SWITCHROOM_SILENCE_FALLBACK_HARD_MS', 900_000)
|
|
4675
|
+
const SILENCE_DEFER_INFLIGHT_TOOLS = process.env.SWITCHROOM_SILENCE_DEFER_INFLIGHT_TOOLS === '1'
|
|
4676
|
+
|
|
4549
4677
|
silencePoke.startTimer({
|
|
4678
|
+
thresholdsMs: { fallback: SILENCE_FALLBACK_MS, fallbackHardCeiling: SILENCE_FALLBACK_HARD_MS },
|
|
4679
|
+
deferFallbackWhileToolInFlight: SILENCE_DEFER_INFLIGHT_TOOLS,
|
|
4550
4680
|
emitMetric: (event) => {
|
|
4551
4681
|
// Re-emit through the unified runtime-metrics fan-out (PostHog + JSONL).
|
|
4552
4682
|
emitRuntimeMetric(event)
|
|
@@ -6469,12 +6599,13 @@ async function executeReply(args: Record<string, unknown>): Promise<{ content: A
|
|
|
6469
6599
|
if (TURN_ORIGIN_ROUTING_ENABLED) {
|
|
6470
6600
|
const explicit = args.message_thread_id != null ? Number(args.message_thread_id) : undefined
|
|
6471
6601
|
const originTurn = findTurnByOriginId(args.origin_turn_id as string | undefined)
|
|
6472
|
-
threadId =
|
|
6473
|
-
|
|
6474
|
-
|
|
6475
|
-
|
|
6476
|
-
|
|
6477
|
-
|
|
6602
|
+
threadId = resolveAnswerThreadWithLog(
|
|
6603
|
+
chat_id,
|
|
6604
|
+
Number.isFinite(explicit as number) ? (explicit as number) : undefined,
|
|
6605
|
+
originTurn,
|
|
6606
|
+
turn,
|
|
6607
|
+
'reply',
|
|
6608
|
+
)
|
|
6478
6609
|
} else {
|
|
6479
6610
|
threadId = resolveThreadId(
|
|
6480
6611
|
chat_id,
|
|
@@ -7125,12 +7256,13 @@ async function executeStreamReply(args: Record<string, unknown>): Promise<unknow
|
|
|
7125
7256
|
let injected: number | undefined
|
|
7126
7257
|
if (TURN_ORIGIN_ROUTING_ENABLED) {
|
|
7127
7258
|
const originTurn = findTurnByOriginId(args.origin_turn_id as string | undefined)
|
|
7128
|
-
injected =
|
|
7129
|
-
|
|
7130
|
-
|
|
7131
|
-
|
|
7132
|
-
|
|
7133
|
-
|
|
7259
|
+
injected = resolveAnswerThreadWithLog(
|
|
7260
|
+
String(args.chat_id),
|
|
7261
|
+
undefined,
|
|
7262
|
+
originTurn,
|
|
7263
|
+
turn,
|
|
7264
|
+
'stream_reply',
|
|
7265
|
+
)
|
|
7134
7266
|
} else {
|
|
7135
7267
|
injected = turn?.sessionThreadId
|
|
7136
7268
|
}
|
|
@@ -8850,6 +8982,7 @@ async function drainActivitySummary(turn: CurrentTurn): Promise<void> {
|
|
|
8850
8982
|
{ chat_id: chat, ...(thread != null ? { threadId: thread } : {}), verb: 'activity-summary.send' },
|
|
8851
8983
|
)
|
|
8852
8984
|
turn.activityMessageId = sent.message_id
|
|
8985
|
+
turn.activityEverOpened = true
|
|
8853
8986
|
} else {
|
|
8854
8987
|
const id = turn.activityMessageId
|
|
8855
8988
|
await robustApiCall(
|
|
@@ -8861,7 +8994,18 @@ async function drainActivitySummary(turn: CurrentTurn): Promise<void> {
|
|
|
8861
8994
|
} catch (err) {
|
|
8862
8995
|
const msg = err instanceof Error ? err.message : String(err)
|
|
8863
8996
|
if (!msg.includes('message is not modified')) {
|
|
8864
|
-
|
|
8997
|
+
turn.activityDrainFailures += 1
|
|
8998
|
+
// Surface the failing anchor + topic: the resume-400 bug fed a
|
|
8999
|
+
// fabricated 13-digit message_id as the reply anchor here, so every
|
|
9000
|
+
// send 400'd and the feed never opened. Logging the anchor +
|
|
9001
|
+
// everOpened makes a feed-blanking send self-explanatory (and the
|
|
9002
|
+
// turn-end DEGRADED line aggregates it).
|
|
9003
|
+
process.stderr.write(
|
|
9004
|
+
`telegram gateway: activity-summary drain failed: ${msg} ` +
|
|
9005
|
+
`(chat=${chat} thread=${thread ?? '-'} ` +
|
|
9006
|
+
`replyAnchor=${turn.sourceMessageId ?? 'none'} ` +
|
|
9007
|
+
`everOpened=${turn.activityEverOpened} failures=${turn.activityDrainFailures})\n`,
|
|
9008
|
+
)
|
|
8865
9009
|
}
|
|
8866
9010
|
// Mark as sent so we don't infinite-loop on a stuck render.
|
|
8867
9011
|
turn.activityLastSentRender = target
|
|
@@ -9019,9 +9163,13 @@ function handleSessionEvent(ev: SessionEvent): void {
|
|
|
9019
9163
|
const next: CurrentTurn = {
|
|
9020
9164
|
sessionChatId: ev.chatId,
|
|
9021
9165
|
sessionThreadId: enqThreadIdNum,
|
|
9022
|
-
|
|
9023
|
-
|
|
9024
|
-
|
|
9166
|
+
// Accept the inbound id as a reply anchor only when it is a plausible
|
|
9167
|
+
// Telegram message id. Synthetic boot-resume inbounds fabricate a
|
|
9168
|
+
// 13-digit Date.now() message_id (for ack-tracking); if that reached
|
|
9169
|
+
// the activity-feed reply anchor it 400'd every feed send and darkened
|
|
9170
|
+
// the live feed for the whole resume turn (2026-06-05). The ack-queue
|
|
9171
|
+
// still keys on ev.messageId independently — only the anchor is gated.
|
|
9172
|
+
sourceMessageId: parseSourceMessageId(ev.messageId),
|
|
9025
9173
|
startedAt,
|
|
9026
9174
|
gatewayReceiveAt: startedAt,
|
|
9027
9175
|
replyCalled: false,
|
|
@@ -9042,12 +9190,19 @@ function handleSessionEvent(ev: SessionEvent): void {
|
|
|
9042
9190
|
activityInFlight: null,
|
|
9043
9191
|
activityPendingRender: null,
|
|
9044
9192
|
activityLastSentRender: null,
|
|
9193
|
+
activityEverOpened: false,
|
|
9194
|
+
activityDrainFailures: 0,
|
|
9045
9195
|
mirrorLines: [],
|
|
9046
9196
|
foregroundSubAgents: new Map(),
|
|
9047
9197
|
answerStream: null,
|
|
9048
9198
|
isDm: isDmChatId(ev.chatId),
|
|
9049
9199
|
}
|
|
9050
9200
|
currentTurn = next
|
|
9201
|
+
// Status-surface observability: one line at every turn SET so a later
|
|
9202
|
+
// dark card is traceable to which turn/topic key it belonged to.
|
|
9203
|
+
process.stderr.write(
|
|
9204
|
+
`telegram gateway: ${formatTurnLifecycle('set', 'enqueue', next, startedAt)}\n`,
|
|
9205
|
+
)
|
|
9051
9206
|
// Component 3 — retain in the bounded recently-ended registry so a
|
|
9052
9207
|
// LATE reply (landing after currentTurn flips to a successor) can
|
|
9053
9208
|
// still resolve THIS turn's origin thread by its turnId.
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { describe, it, expect } from 'vitest'
|
|
2
|
+
import { parseSourceMessageId, MAX_TELEGRAM_MESSAGE_ID } from './source-message-id.js'
|
|
3
|
+
|
|
4
|
+
describe('parseSourceMessageId', () => {
|
|
5
|
+
it('accepts a plausible Telegram message id (string or number)', () => {
|
|
6
|
+
expect(parseSourceMessageId('903')).toBe(903)
|
|
7
|
+
expect(parseSourceMessageId(905)).toBe(905)
|
|
8
|
+
expect(parseSourceMessageId('1')).toBe(1)
|
|
9
|
+
})
|
|
10
|
+
|
|
11
|
+
it('REJECTS a fabricated 13-digit Date.now() timestamp (the resume-dark-feed bug)', () => {
|
|
12
|
+
// 2026-06-04T23:34:21.578Z — the exact value that 400'd every feed send.
|
|
13
|
+
expect(parseSourceMessageId('1780616061578')).toBeNull()
|
|
14
|
+
expect(parseSourceMessageId(1_780_616_061_578)).toBeNull()
|
|
15
|
+
})
|
|
16
|
+
|
|
17
|
+
it('rejects anything at or above the Telegram message-id ceiling (2^31)', () => {
|
|
18
|
+
expect(parseSourceMessageId(MAX_TELEGRAM_MESSAGE_ID)).toBeNull()
|
|
19
|
+
expect(parseSourceMessageId(MAX_TELEGRAM_MESSAGE_ID - 1)).toBe(MAX_TELEGRAM_MESSAGE_ID - 1)
|
|
20
|
+
})
|
|
21
|
+
|
|
22
|
+
it('rejects null / undefined / empty / non-numeric / non-positive', () => {
|
|
23
|
+
expect(parseSourceMessageId(null)).toBeNull()
|
|
24
|
+
expect(parseSourceMessageId(undefined)).toBeNull()
|
|
25
|
+
expect(parseSourceMessageId('')).toBeNull()
|
|
26
|
+
expect(parseSourceMessageId('12a')).toBeNull()
|
|
27
|
+
expect(parseSourceMessageId('-5')).toBeNull() // leading "-" fails the digit test
|
|
28
|
+
expect(parseSourceMessageId(0)).toBeNull()
|
|
29
|
+
expect(parseSourceMessageId(-5)).toBeNull()
|
|
30
|
+
expect(parseSourceMessageId('3.5')).toBeNull()
|
|
31
|
+
})
|
|
32
|
+
})
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Guard for the per-turn reply anchor (`turn.sourceMessageId`).
|
|
3
|
+
*
|
|
4
|
+
* Telegram Bot API message ids are positive integers that fit within a signed
|
|
5
|
+
* 32-bit int; `reply_parameters.message_id` HARD-rejects anything larger with
|
|
6
|
+
* 400 "field 'message_id' must be a valid Number" (and `allow_sending_without_reply`
|
|
7
|
+
* does NOT bypass that range check).
|
|
8
|
+
*
|
|
9
|
+
* Synthetic boot-resume inbounds (`resume-inbound-builder.ts`) fabricate a
|
|
10
|
+
* `message_id` from `Date.now()` (~1.78e13) so the deliver-until-acked queue can
|
|
11
|
+
* ack the synthetic by its own enqueue id. That round-trip is fine on its own —
|
|
12
|
+
* but the enqueue handler also turns `ev.messageId` into `turn.sourceMessageId`,
|
|
13
|
+
* which `drainActivitySummary` sends as the activity-feed reply anchor. A
|
|
14
|
+
* fabricated 13-digit timestamp there 400s EVERY feed send for the whole turn,
|
|
15
|
+
* so the live status feed is dark for the entire first post-restart turn (the
|
|
16
|
+
* resume-dark-feed incident, 2026-06-05).
|
|
17
|
+
*
|
|
18
|
+
* This guard accepts a value as a real anchor ONLY when it is a plausible
|
|
19
|
+
* Telegram message id; anything non-numeric or out of range yields null, so the
|
|
20
|
+
* feed posts UNANCHORED (still correct — the anchor is a nicety, not required).
|
|
21
|
+
* The synthetic's ack-tracking is unaffected: it keys on the enqueue event's own
|
|
22
|
+
* id, never on this anchor.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
/** Telegram message ids fit within a signed 32-bit int for reply anchoring;
|
|
26
|
+
* anything at/above this is not a real message id (e.g. a wall-clock ms ts). */
|
|
27
|
+
export const MAX_TELEGRAM_MESSAGE_ID = 2 ** 31
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Parse an inbound's `messageId` into a usable reply anchor, or null when it is
|
|
31
|
+
* not a plausible Telegram message id (non-numeric, non-positive, non-integer,
|
|
32
|
+
* or out of the reply-anchor range — e.g. a fabricated `Date.now()` timestamp).
|
|
33
|
+
*/
|
|
34
|
+
export function parseSourceMessageId(raw: string | number | undefined | null): number | null {
|
|
35
|
+
if (raw == null) return null
|
|
36
|
+
const s = String(raw)
|
|
37
|
+
if (!/^\d+$/.test(s)) return null
|
|
38
|
+
const n = Number(s)
|
|
39
|
+
if (!Number.isSafeInteger(n) || n <= 0 || n >= MAX_TELEGRAM_MESSAGE_ID) return null
|
|
40
|
+
return n
|
|
41
|
+
}
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import { describe, it, expect } from 'vitest'
|
|
2
|
+
import {
|
|
3
|
+
formatTurnLifecycle,
|
|
4
|
+
detectStatusSurfaceDegraded,
|
|
5
|
+
type StatusSurfaceTurnView,
|
|
6
|
+
} from './status-surface-log.js'
|
|
7
|
+
|
|
8
|
+
function turn(overrides: Partial<StatusSurfaceTurnView> = {}): StatusSurfaceTurnView {
|
|
9
|
+
return {
|
|
10
|
+
turnId: '-100123:_#1780000000000',
|
|
11
|
+
sessionChatId: '-100123',
|
|
12
|
+
sessionThreadId: undefined,
|
|
13
|
+
startedAt: 1_780_000_000_000,
|
|
14
|
+
toolCallCount: 0,
|
|
15
|
+
activityMessageId: null,
|
|
16
|
+
activityEverOpened: false,
|
|
17
|
+
activityDrainFailures: 0,
|
|
18
|
+
replyCalled: false,
|
|
19
|
+
finalAnswerDelivered: false,
|
|
20
|
+
...overrides,
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
describe('formatTurnLifecycle', () => {
|
|
25
|
+
it('renders a set line with no age and a "-" thread for General', () => {
|
|
26
|
+
const line = formatTurnLifecycle('set', 'enqueue', turn(), 1_780_000_005_000)
|
|
27
|
+
expect(line).toContain('turn-lifecycle set reason=enqueue')
|
|
28
|
+
expect(line).toContain('turnId=-100123:_#1780000000000')
|
|
29
|
+
expect(line).toContain('chat=-100123')
|
|
30
|
+
expect(line).toContain('thread=-')
|
|
31
|
+
expect(line).toContain('age_ms=0') // set never reports age
|
|
32
|
+
})
|
|
33
|
+
|
|
34
|
+
it('renders a clear line with the turn age and live state', () => {
|
|
35
|
+
const line = formatTurnLifecycle(
|
|
36
|
+
'clear',
|
|
37
|
+
'turn_end',
|
|
38
|
+
turn({ sessionThreadId: 3, toolCallCount: 5, activityMessageId: 42, activityEverOpened: true, replyCalled: true, finalAnswerDelivered: true }),
|
|
39
|
+
1_780_000_300_000, // +300s
|
|
40
|
+
)
|
|
41
|
+
expect(line).toContain('turn-lifecycle clear reason=turn_end')
|
|
42
|
+
expect(line).toContain('thread=3')
|
|
43
|
+
expect(line).toContain('tools=5')
|
|
44
|
+
expect(line).toContain('activityMsgId=42')
|
|
45
|
+
expect(line).toContain('feedOpened=true')
|
|
46
|
+
expect(line).toContain('replyCalled=true')
|
|
47
|
+
expect(line).toContain('finalAnswer=true')
|
|
48
|
+
expect(line).toContain('age_ms=300000')
|
|
49
|
+
})
|
|
50
|
+
|
|
51
|
+
it('never emits a negative age even if startedAt is in the future (clock skew)', () => {
|
|
52
|
+
const line = formatTurnLifecycle('clear', 'turn_end', turn({ startedAt: 2_000_000_000_000 }), 1_780_000_000_000)
|
|
53
|
+
expect(line).toContain('age_ms=0')
|
|
54
|
+
})
|
|
55
|
+
|
|
56
|
+
it('carries no prefix or trailing newline — the caller owns transport', () => {
|
|
57
|
+
const line = formatTurnLifecycle('set', 'enqueue', turn(), 0)
|
|
58
|
+
expect(line.startsWith('telegram gateway:')).toBe(false)
|
|
59
|
+
expect(line.endsWith('\n')).toBe(false)
|
|
60
|
+
})
|
|
61
|
+
})
|
|
62
|
+
|
|
63
|
+
describe('detectStatusSurfaceDegraded', () => {
|
|
64
|
+
it('flags a turn that did tool work but never opened the feed due to send failures (the resume-400 signature)', () => {
|
|
65
|
+
const d = detectStatusSurfaceDegraded(
|
|
66
|
+
turn({ toolCallCount: 3, activityEverOpened: false, activityDrainFailures: 10 }),
|
|
67
|
+
)
|
|
68
|
+
expect(d).not.toBeNull()
|
|
69
|
+
expect(d!.reason).toBe('feed-never-opened')
|
|
70
|
+
expect(d!.detail).toContain('drainFailures=10')
|
|
71
|
+
})
|
|
72
|
+
|
|
73
|
+
it('does NOT flag a healthy turn where the feed opened, even if later cleared (activityMessageId nulled)', () => {
|
|
74
|
+
// clearActivitySummary nulls activityMessageId async on the healthy path;
|
|
75
|
+
// the sticky activityEverOpened keeps this from false-positiving.
|
|
76
|
+
expect(
|
|
77
|
+
detectStatusSurfaceDegraded(
|
|
78
|
+
turn({ toolCallCount: 4, activityMessageId: null, activityEverOpened: true, activityDrainFailures: 0 }),
|
|
79
|
+
),
|
|
80
|
+
).toBeNull()
|
|
81
|
+
})
|
|
82
|
+
|
|
83
|
+
it('does NOT flag a turn that never attempted a feed send (e.g. ack-first suppression)', () => {
|
|
84
|
+
expect(
|
|
85
|
+
detectStatusSurfaceDegraded(
|
|
86
|
+
turn({ toolCallCount: 2, activityEverOpened: false, activityDrainFailures: 0 }),
|
|
87
|
+
),
|
|
88
|
+
).toBeNull()
|
|
89
|
+
})
|
|
90
|
+
|
|
91
|
+
it('does NOT flag a turn with no tool work (nothing to surface)', () => {
|
|
92
|
+
expect(
|
|
93
|
+
detectStatusSurfaceDegraded(
|
|
94
|
+
turn({ toolCallCount: 0, activityEverOpened: false, activityDrainFailures: 3 }),
|
|
95
|
+
),
|
|
96
|
+
).toBeNull()
|
|
97
|
+
})
|
|
98
|
+
})
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Status-surface observability — pure formatters for the gateway's live-status
|
|
3
|
+
* lane (progress card / activity feed / typing indicator).
|
|
4
|
+
*
|
|
5
|
+
* Why a dedicated module: when an agent's live status went dark (marko,
|
|
6
|
+
* 2026-06-05), the lane was nearly silent in the logs — `currentTurn` (the
|
|
7
|
+
* variable that drives the card/feed/typing) was nulled with no breadcrumb, and
|
|
8
|
+
* the activity feed failed every send with no turn-level signal. Two latent
|
|
9
|
+
* bugs were invisible for days: a 300s silence-poke teardown that nulled the
|
|
10
|
+
* card mid-work, and a resume-synthetic whose fabricated 13-digit message_id
|
|
11
|
+
* made every feed send 400. Neither left a greppable "the card went dark and
|
|
12
|
+
* here's why" line.
|
|
13
|
+
*
|
|
14
|
+
* These pure functions give the gateway exactly that: ONE structured line per
|
|
15
|
+
* currentTurn lifecycle transition, and a single DEGRADED warning when a turn
|
|
16
|
+
* did tool work but the feed never opened because its sends failed. Pure
|
|
17
|
+
* formatters + injected transport (the caller owns `process.stderr.write`),
|
|
18
|
+
* mirroring `silence-poke.ts` / `worker-activity-feed.ts`, so they're
|
|
19
|
+
* unit-testable without a live gateway.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* The `currentTurn` fields the status-surface logs read. The gateway's
|
|
24
|
+
* `CurrentTurn` atom structurally satisfies this (TS structural typing), so the
|
|
25
|
+
* gateway passes the turn directly — no import cycle back into `gateway.ts`.
|
|
26
|
+
*/
|
|
27
|
+
export interface StatusSurfaceTurnView {
|
|
28
|
+
turnId: string
|
|
29
|
+
sessionChatId: string
|
|
30
|
+
sessionThreadId: number | undefined
|
|
31
|
+
startedAt: number
|
|
32
|
+
toolCallCount: number
|
|
33
|
+
/** Live activity-feed message id; null until the first send captures it. */
|
|
34
|
+
activityMessageId: number | null
|
|
35
|
+
/**
|
|
36
|
+
* Sticky: true once the activity feed ever opened a message this turn. Unlike
|
|
37
|
+
* `activityMessageId` (which `clearActivitySummary` nulls async on the
|
|
38
|
+
* healthy finalize path), this is never reset — so a turn that DID surface
|
|
39
|
+
* the feed can't false-positive as degraded at turn-end.
|
|
40
|
+
*/
|
|
41
|
+
activityEverOpened: boolean
|
|
42
|
+
/** Count of real activity-feed send/edit failures this turn (429s and
|
|
43
|
+
* "message is not modified" excluded). */
|
|
44
|
+
activityDrainFailures: number
|
|
45
|
+
replyCalled: boolean
|
|
46
|
+
finalAnswerDelivered: boolean
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export type TurnLifecycleAction = 'set' | 'clear'
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* One structured line per `currentTurn` set/clear. `currentTurn` drives the
|
|
53
|
+
* progress card / activity feed / typing; logging every transition — with the
|
|
54
|
+
* topic key, how far the turn got, and the reason it ended — makes a dark card
|
|
55
|
+
* explainable after the fact. Returned WITHOUT the `telegram gateway: ` prefix
|
|
56
|
+
* or trailing newline so the caller owns transport (and tests assert the body).
|
|
57
|
+
*
|
|
58
|
+
* `now` is only consulted for the `clear` age; for `set` it is ignored.
|
|
59
|
+
*/
|
|
60
|
+
export function formatTurnLifecycle(
|
|
61
|
+
action: TurnLifecycleAction,
|
|
62
|
+
reason: string,
|
|
63
|
+
t: StatusSurfaceTurnView,
|
|
64
|
+
now: number,
|
|
65
|
+
): string {
|
|
66
|
+
const ageMs = action === 'clear' ? Math.max(0, now - t.startedAt) : 0
|
|
67
|
+
return (
|
|
68
|
+
`turn-lifecycle ${action} reason=${reason} turnId=${t.turnId} ` +
|
|
69
|
+
`chat=${t.sessionChatId} thread=${t.sessionThreadId ?? '-'} ` +
|
|
70
|
+
`tools=${t.toolCallCount} activityMsgId=${t.activityMessageId ?? 'none'} ` +
|
|
71
|
+
`feedOpened=${t.activityEverOpened} drainFailures=${t.activityDrainFailures} ` +
|
|
72
|
+
`replyCalled=${t.replyCalled} finalAnswer=${t.finalAnswerDelivered} age_ms=${ageMs}`
|
|
73
|
+
)
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Turn-end health check: did the turn do tool work but never get a live feed
|
|
78
|
+
* message onto the screen BECAUSE its sends failed? That is the exact signature
|
|
79
|
+
* of the resume-400 bug (every activity-summary send throws, so the feed never
|
|
80
|
+
* opens) — a single greppable line would have caught it in seconds.
|
|
81
|
+
*
|
|
82
|
+
* Returns null when the surface was healthy or legitimately silent:
|
|
83
|
+
* - no tool work this turn (nothing to surface), OR
|
|
84
|
+
* - the feed opened fine (`activityEverOpened`), OR
|
|
85
|
+
* - the feed never even attempted a send (`activityDrainFailures === 0`, e.g.
|
|
86
|
+
* an ack-first turn whose feed was intentionally suppressed) — absence of a
|
|
87
|
+
* send is not a failure.
|
|
88
|
+
*/
|
|
89
|
+
export function detectStatusSurfaceDegraded(
|
|
90
|
+
t: StatusSurfaceTurnView,
|
|
91
|
+
): { reason: string; detail: string } | null {
|
|
92
|
+
if (t.toolCallCount === 0) return null
|
|
93
|
+
if (t.activityEverOpened) return null
|
|
94
|
+
if (t.activityDrainFailures === 0) return null
|
|
95
|
+
return {
|
|
96
|
+
reason: 'feed-never-opened',
|
|
97
|
+
detail:
|
|
98
|
+
`tools=${t.toolCallCount} drainFailures=${t.activityDrainFailures} ` +
|
|
99
|
+
`activityMsgId=none — the live activity feed failed every send this turn ` +
|
|
100
|
+
`(card was dark despite tool work)`,
|
|
101
|
+
}
|
|
102
|
+
}
|
|
@@ -20,6 +20,14 @@
|
|
|
20
20
|
* edits, and tool churn DO NOT reset the silence clock — the model could
|
|
21
21
|
* be ripping through 20 tool calls and still be "silent" to the user.
|
|
22
22
|
*
|
|
23
|
+
* Fix A caveat (opt-in, `deferFallbackWhileToolInFlight`): tool churn still
|
|
24
|
+
* doesn't reset the *clock*, but when the threshold is crossed WITH a parent
|
|
25
|
+
* tool genuinely in flight, the terminal unwedge is DEFERRED (not skipped) up to
|
|
26
|
+
* `fallbackHardCeiling`. Since #2162 the live activity feed renders that tool
|
|
27
|
+
* work, so the "still silent to the user" premise no longer holds while a tool
|
|
28
|
+
* is visibly running; nulling `currentTurn` there would darken the very feed the
|
|
29
|
+
* user is watching. A turn with no in-flight tool is unaffected.
|
|
30
|
+
*
|
|
23
31
|
* Terminal action, once per turn:
|
|
24
32
|
*
|
|
25
33
|
* t=0 startTurn() — silence clock starts at turnStartedAt
|
|
@@ -81,6 +89,16 @@ export interface ThresholdsMs {
|
|
|
81
89
|
/** Silence (since last outbound, or turn start) after which the
|
|
82
90
|
* framework sends the user-visible fallback AND unwedges the turn. */
|
|
83
91
|
fallback: number
|
|
92
|
+
/**
|
|
93
|
+
* Fix A — hard ceiling for the in-flight-tool defer. When
|
|
94
|
+
* `deferFallbackWhileToolInFlight` is on, the fallback is held back while a
|
|
95
|
+
* parent tool is genuinely in flight (the agent is demonstrably working and
|
|
96
|
+
* the live activity feed is showing it). This bounds that defer: once silence
|
|
97
|
+
* crosses the ceiling the fallback fires REGARDLESS of an in-flight tool, so a
|
|
98
|
+
* hung-mid-tool turn can't pin the conversation forever. Ignored unless the
|
|
99
|
+
* defer is on; defaults to no ceiling (Infinity) when omitted.
|
|
100
|
+
*/
|
|
101
|
+
fallbackHardCeiling?: number
|
|
84
102
|
}
|
|
85
103
|
|
|
86
104
|
export const DEFAULT_THRESHOLDS: ThresholdsMs = {
|
|
@@ -122,6 +140,21 @@ export interface SilencePokeDeps {
|
|
|
122
140
|
thresholdsMs?: ThresholdsMs
|
|
123
141
|
/** Poll interval (tests). */
|
|
124
142
|
pollIntervalMs?: number
|
|
143
|
+
/**
|
|
144
|
+
* Fix A — when true, the 300s framework fallback is DEFERRED while a parent
|
|
145
|
+
* tool is genuinely in flight (`inFlightTools` non-empty): the agent is
|
|
146
|
+
* demonstrably working, and since #2162 the live activity feed shows that
|
|
147
|
+
* work, so nulling `currentTurn` (which the fallback does) would darken a feed
|
|
148
|
+
* the user is actively watching. The defer is bounded by
|
|
149
|
+
* `thresholdsMs.fallbackHardCeiling` so a hung-mid-tool turn still unwedges; a
|
|
150
|
+
* turn with NO in-flight tool fires at the base threshold exactly as before.
|
|
151
|
+
* Default false (legacy behaviour) — enable per-agent to canary.
|
|
152
|
+
*
|
|
153
|
+
* A crashed agent is recovered independently by the bridge-disconnect sweep
|
|
154
|
+
* (`onDanglingTurnsSwept`), so deferring here does not reintroduce the #1556
|
|
155
|
+
* dangling-turn wedge for the crash case.
|
|
156
|
+
*/
|
|
157
|
+
deferFallbackWhileToolInFlight?: boolean
|
|
125
158
|
}
|
|
126
159
|
|
|
127
160
|
const state = new Map<string, SilencePokeState>()
|
|
@@ -366,6 +399,20 @@ function tick(now: number): void {
|
|
|
366
399
|
if (silence < 0) continue
|
|
367
400
|
|
|
368
401
|
if (!s.fallbackFired && silence >= thresholds.fallback) {
|
|
402
|
+
// Fix A: defer the unwedge while a parent tool is genuinely in flight —
|
|
403
|
+
// the agent is demonstrably working and the live activity feed is showing
|
|
404
|
+
// it, so firing here (which nulls currentTurn) would darken that feed
|
|
405
|
+
// mid-work. Bounded by the hard ceiling so a hung-mid-tool turn still
|
|
406
|
+
// unwedges. `continue` WITHOUT setting fallbackFired so the next tick
|
|
407
|
+
// re-checks — once the tool ends and the turn stays silent past the base
|
|
408
|
+
// threshold, or the ceiling is crossed, it fires normally.
|
|
409
|
+
if (
|
|
410
|
+
activeDeps.deferFallbackWhileToolInFlight === true &&
|
|
411
|
+
s.inFlightTools.size > 0 &&
|
|
412
|
+
silence < (thresholds.fallbackHardCeiling ?? Number.POSITIVE_INFINITY)
|
|
413
|
+
) {
|
|
414
|
+
continue
|
|
415
|
+
}
|
|
369
416
|
s.fallbackFired = true
|
|
370
417
|
const { chatId, threadId } = parseKey(key)
|
|
371
418
|
const recentThinking = s.lastThinkingAt != null
|
|
@@ -45,13 +45,15 @@ describe('component 3 — turn-origin reply routing', () => {
|
|
|
45
45
|
const fn = gatewaySrc.split('async function executeReply')[1]?.split('\nasync function ')[0] ?? ''
|
|
46
46
|
expect(fn).toMatch(/TURN_ORIGIN_ROUTING_ENABLED/)
|
|
47
47
|
expect(fn).toMatch(/findTurnByOriginId\(args\.origin_turn_id/)
|
|
48
|
-
|
|
48
|
+
// The resolution + reply-route telemetry go through resolveAnswerThreadWithLog,
|
|
49
|
+
// which calls the pure resolveAnswerThreadId internally (incl. tier-4 recovery).
|
|
50
|
+
expect(fn).toMatch(/resolveAnswerThread\w*\(/)
|
|
49
51
|
})
|
|
50
52
|
|
|
51
53
|
it('executeStreamReply resolves the answer thread via the origin turn too', () => {
|
|
52
54
|
const fn = gatewaySrc.split('async function executeStreamReply')[1]?.split('\nasync function ')[0] ?? ''
|
|
53
55
|
expect(fn).toMatch(/findTurnByOriginId\(args\.origin_turn_id/)
|
|
54
|
-
expect(fn).toMatch(/
|
|
56
|
+
expect(fn).toMatch(/resolveAnswerThread\w*\(/)
|
|
55
57
|
})
|
|
56
58
|
|
|
57
59
|
it('the reply + stream_reply tool schemas expose origin_turn_id to the model', () => {
|
|
@@ -26,7 +26,10 @@ interface TestFixtures {
|
|
|
26
26
|
fallbacks: FrameworkFallbackContext[]
|
|
27
27
|
}
|
|
28
28
|
|
|
29
|
-
function setupDeps(opts?: {
|
|
29
|
+
function setupDeps(opts?: {
|
|
30
|
+
thresholds?: Partial<typeof DEFAULT_THRESHOLDS> & { fallbackHardCeiling?: number }
|
|
31
|
+
deferFallbackWhileToolInFlight?: boolean
|
|
32
|
+
}): TestFixtures {
|
|
30
33
|
const fixtures: TestFixtures = { emitted: [], fallbacks: [] }
|
|
31
34
|
__setDepsForTests({
|
|
32
35
|
emitMetric: (e) => fixtures.emitted.push(e),
|
|
@@ -35,6 +38,9 @@ function setupDeps(opts?: { thresholds?: Partial<typeof DEFAULT_THRESHOLDS> }):
|
|
|
35
38
|
...DEFAULT_THRESHOLDS,
|
|
36
39
|
...(opts?.thresholds ?? {}),
|
|
37
40
|
},
|
|
41
|
+
...(opts?.deferFallbackWhileToolInFlight != null
|
|
42
|
+
? { deferFallbackWhileToolInFlight: opts.deferFallbackWhileToolInFlight }
|
|
43
|
+
: {}),
|
|
38
44
|
})
|
|
39
45
|
return fixtures
|
|
40
46
|
}
|
|
@@ -528,3 +534,65 @@ describe('silence-poke — performance', () => {
|
|
|
528
534
|
expect(elapsed).toBeLessThan(50)
|
|
529
535
|
})
|
|
530
536
|
})
|
|
537
|
+
|
|
538
|
+
// ─── Fix A: defer the unwedge while a parent tool is genuinely in flight ──────
|
|
539
|
+
// A long quiet tool stretch (foreground sub-agent / big research) crossed the
|
|
540
|
+
// 300s fallback and nulled currentTurn, darkening the live activity feed
|
|
541
|
+
// mid-work. The opt-in defer keeps the turn alive while a tool is in flight,
|
|
542
|
+
// bounded by a hard ceiling so a hung-mid-tool turn still unwedges.
|
|
543
|
+
describe('silence-poke — Fix A: in-flight-tool defer', () => {
|
|
544
|
+
it('legacy default (defer OFF): fires at 300s even with a tool in flight', () => {
|
|
545
|
+
const f = setupDeps() // deferFallbackWhileToolInFlight unset → off
|
|
546
|
+
startTurn('c:0', 0)
|
|
547
|
+
noteToolStart('c:0', 't1', 'Bash', 'long audit', 10_000)
|
|
548
|
+
__tickForTests(300_000)
|
|
549
|
+
expect(f.fallbacks).toHaveLength(1) // unchanged legacy behaviour
|
|
550
|
+
})
|
|
551
|
+
|
|
552
|
+
it('defer ON: does NOT fire at 300s while a tool is in flight', () => {
|
|
553
|
+
const f = setupDeps({ deferFallbackWhileToolInFlight: true, thresholds: { fallbackHardCeiling: 900_000 } })
|
|
554
|
+
startTurn('c:0', 0)
|
|
555
|
+
noteToolStart('c:0', 't1', 'Bash', 'long audit', 10_000)
|
|
556
|
+
__tickForTests(300_000)
|
|
557
|
+
__tickForTests(450_000) // still working, tool still in flight
|
|
558
|
+
expect(f.fallbacks).toHaveLength(0) // deferred — the live feed stays alive
|
|
559
|
+
})
|
|
560
|
+
|
|
561
|
+
it('defer ON: fires once the tool ends and the turn stays silent past threshold', () => {
|
|
562
|
+
const f = setupDeps({ deferFallbackWhileToolInFlight: true, thresholds: { fallbackHardCeiling: 900_000 } })
|
|
563
|
+
startTurn('c:0', 0)
|
|
564
|
+
noteToolStart('c:0', 't1', 'Bash', null, 10_000)
|
|
565
|
+
__tickForTests(300_000)
|
|
566
|
+
expect(f.fallbacks).toHaveLength(0) // deferred while in flight
|
|
567
|
+
noteToolEnd('c:0', 't1', 400_000) // tool completes, no reply follows
|
|
568
|
+
__tickForTests(400_001) // silence (from turn start) already well past 300s
|
|
569
|
+
expect(f.fallbacks).toHaveLength(1) // now unwedges promptly
|
|
570
|
+
})
|
|
571
|
+
|
|
572
|
+
it('defer ON: fires at the hard ceiling even with a tool still in flight (hung-mid-tool)', () => {
|
|
573
|
+
const f = setupDeps({ deferFallbackWhileToolInFlight: true, thresholds: { fallbackHardCeiling: 900_000 } })
|
|
574
|
+
startTurn('c:0', 0)
|
|
575
|
+
noteToolStart('c:0', 't1', 'Bash', 'wedged tool', 10_000)
|
|
576
|
+
__tickForTests(300_000)
|
|
577
|
+
expect(f.fallbacks).toHaveLength(0) // deferred
|
|
578
|
+
__tickForTests(900_000) // crosses the hard ceiling
|
|
579
|
+
expect(f.fallbacks).toHaveLength(1) // bounded — still unwedges
|
|
580
|
+
})
|
|
581
|
+
|
|
582
|
+
it('defer ON: a turn with NO in-flight tool fires at the base threshold (genuine silence)', () => {
|
|
583
|
+
const f = setupDeps({ deferFallbackWhileToolInFlight: true, thresholds: { fallbackHardCeiling: 900_000 } })
|
|
584
|
+
startTurn('c:0', 0)
|
|
585
|
+
// no tool ever started — genuinely silent/wedged
|
|
586
|
+
__tickForTests(300_000)
|
|
587
|
+
expect(f.fallbacks).toHaveLength(1) // unaffected by the defer
|
|
588
|
+
})
|
|
589
|
+
|
|
590
|
+
it('defer ON without a hard ceiling: defers indefinitely while the tool stays in flight', () => {
|
|
591
|
+
const f = setupDeps({ deferFallbackWhileToolInFlight: true }) // no fallbackHardCeiling → Infinity
|
|
592
|
+
startTurn('c:0', 0)
|
|
593
|
+
noteToolStart('c:0', 't1', 'Bash', null, 10_000)
|
|
594
|
+
__tickForTests(300_000)
|
|
595
|
+
__tickForTests(3_600_000) // an hour in
|
|
596
|
+
expect(f.fallbacks).toHaveLength(0)
|
|
597
|
+
})
|
|
598
|
+
})
|
|
@@ -440,3 +440,64 @@ describe('createWorkerActivityFeed', () => {
|
|
|
440
440
|
expect(bot.sent[0].opts?.message_thread_id).toBe(42)
|
|
441
441
|
})
|
|
442
442
|
})
|
|
443
|
+
|
|
444
|
+
// ─── log sink: success-path observability ────────────────────────────────────
|
|
445
|
+
// Before this, the feed only logged on FAILURE, so a feed that rendered fine
|
|
446
|
+
// was invisible in the gateway log — the exact gap that made the marko
|
|
447
|
+
// status-dark incident hard to triage. Assert paint/edit/finish each emit a
|
|
448
|
+
// structured, greppable line naming the worker, chat, thread, and message id.
|
|
449
|
+
describe('createWorkerActivityFeed — log sink', () => {
|
|
450
|
+
it('logs paint on first send, edit on each in-place update, and finish on terminal', async () => {
|
|
451
|
+
const bot = makeFakeBot()
|
|
452
|
+
const logs: string[] = []
|
|
453
|
+
let clock = 10_000
|
|
454
|
+
const feed = createWorkerActivityFeed({
|
|
455
|
+
bot,
|
|
456
|
+
now: () => clock,
|
|
457
|
+
minEditIntervalMs: 0,
|
|
458
|
+
log: (m) => logs.push(m),
|
|
459
|
+
})
|
|
460
|
+
|
|
461
|
+
await feed.update('w-research', 'chat-9', view({ toolCount: 1, latestSummary: 'first' }), 7)
|
|
462
|
+
clock = 11_000
|
|
463
|
+
await feed.update('w-research', 'chat-9', view({ toolCount: 2, latestSummary: 'second' }), 7)
|
|
464
|
+
clock = 12_000
|
|
465
|
+
await feed.finish('w-research', view({ state: 'done', toolCount: 2 }))
|
|
466
|
+
|
|
467
|
+
const paint = logs.find((l) => l.startsWith('worker-feed: paint'))
|
|
468
|
+
const edit = logs.find((l) => l.startsWith('worker-feed: edit'))
|
|
469
|
+
const finish = logs.find((l) => l.startsWith('worker-feed: finish'))
|
|
470
|
+
|
|
471
|
+
expect(paint).toBeDefined()
|
|
472
|
+
expect(paint).toContain('agent=w-research')
|
|
473
|
+
expect(paint).toContain('chat=chat-9')
|
|
474
|
+
expect(paint).toContain('thread=7')
|
|
475
|
+
expect(paint).toMatch(/msgId=\d+/)
|
|
476
|
+
expect(paint).toMatch(/bytes=\d+/)
|
|
477
|
+
|
|
478
|
+
expect(edit).toBeDefined()
|
|
479
|
+
expect(edit).toContain('agent=w-research')
|
|
480
|
+
|
|
481
|
+
expect(finish).toBeDefined()
|
|
482
|
+
expect(finish).toContain('state=done')
|
|
483
|
+
})
|
|
484
|
+
|
|
485
|
+
it('renders thread=- in the log line when no forum topic is set', async () => {
|
|
486
|
+
const bot = makeFakeBot()
|
|
487
|
+
const logs: string[] = []
|
|
488
|
+
let clock = 10_000
|
|
489
|
+
const feed = createWorkerActivityFeed({ bot, now: () => clock, log: (m) => logs.push(m) })
|
|
490
|
+
await feed.update('w1', 'chat', view()) // no threadId
|
|
491
|
+
expect(logs.find((l) => l.startsWith('worker-feed: paint'))).toContain('thread=-')
|
|
492
|
+
})
|
|
493
|
+
|
|
494
|
+
it('does not log a paint when the worker stays below firstPaintMin (still silent)', async () => {
|
|
495
|
+
const bot = makeFakeBot()
|
|
496
|
+
const logs: string[] = []
|
|
497
|
+
let clock = 0
|
|
498
|
+
const feed = createWorkerActivityFeed({ bot, now: () => clock, firstPaintMinMs: 8000, log: (m) => logs.push(m) })
|
|
499
|
+
clock = 3000
|
|
500
|
+
await feed.update('w1', 'chat', view({ elapsedMs: 3000 }))
|
|
501
|
+
expect(logs.some((l) => l.startsWith('worker-feed: paint'))).toBe(false)
|
|
502
|
+
})
|
|
503
|
+
})
|
|
@@ -208,6 +208,8 @@ export interface WorkerActivityFeedOpts {
|
|
|
208
208
|
}
|
|
209
209
|
|
|
210
210
|
interface WorkerHandle {
|
|
211
|
+
/** jsonl agent id — carried so success/failure log lines can name the worker. */
|
|
212
|
+
agentId: string
|
|
211
213
|
chatId: string
|
|
212
214
|
threadId?: number
|
|
213
215
|
messageId: number | null
|
|
@@ -309,6 +311,10 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
309
311
|
h.messageId = sent.message_id
|
|
310
312
|
h.lastBody = body
|
|
311
313
|
h.lastEditAt = nowFn()
|
|
314
|
+
log(
|
|
315
|
+
`worker-feed: paint agent=${h.agentId} chat=${h.chatId} ` +
|
|
316
|
+
`thread=${h.threadId ?? '-'} msgId=${h.messageId} bytes=${body.length}`,
|
|
317
|
+
)
|
|
312
318
|
} catch (err) {
|
|
313
319
|
noteRateLimited(h, err, 'send')
|
|
314
320
|
log(`worker-feed: send failed: ${(err as Error).message}`)
|
|
@@ -324,6 +330,10 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
324
330
|
await opts.bot.editMessageText(h.chatId, h.messageId, body, sendOptsFor(h))
|
|
325
331
|
h.lastBody = body
|
|
326
332
|
h.lastEditAt = nowFn()
|
|
333
|
+
log(
|
|
334
|
+
`worker-feed: edit agent=${h.agentId} chat=${h.chatId} ` +
|
|
335
|
+
`thread=${h.threadId ?? '-'} msgId=${h.messageId} bytes=${body.length}`,
|
|
336
|
+
)
|
|
327
337
|
} catch (err) {
|
|
328
338
|
noteRateLimited(h, err, 'edit')
|
|
329
339
|
// Stale message_id (manually deleted / edit window gone). Re-post
|
|
@@ -351,6 +361,10 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
351
361
|
await opts.bot.editMessageText(h.chatId, h.messageId, body, sendOptsFor(h))
|
|
352
362
|
h.lastBody = body
|
|
353
363
|
h.lastEditAt = nowFn()
|
|
364
|
+
log(
|
|
365
|
+
`worker-feed: finish agent=${h.agentId} chat=${h.chatId} ` +
|
|
366
|
+
`thread=${h.threadId ?? '-'} msgId=${h.messageId} state=${view.state} bytes=${body.length}`,
|
|
367
|
+
)
|
|
354
368
|
} catch (err) {
|
|
355
369
|
noteRateLimited(h, err, 'finish')
|
|
356
370
|
log(`worker-feed: finish edit failed: ${(err as Error).message}`)
|
|
@@ -371,6 +385,7 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
371
385
|
let h = handles.get(agentId)
|
|
372
386
|
if (h == null) {
|
|
373
387
|
h = {
|
|
388
|
+
agentId,
|
|
374
389
|
chatId,
|
|
375
390
|
threadId,
|
|
376
391
|
messageId: null,
|