switchroom 0.19.19 → 0.19.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auth-broker/index.js +53 -0
- package/dist/cli/switchroom.js +2444 -1264
- package/dist/host-control/main.js +54 -1
- package/dist/vault/approvals/kernel-server.js +53 -0
- package/dist/vault/broker/server.js +53 -0
- package/package.json +4 -2
- package/skills/switchroom-release/SKILL.md +103 -20
- package/telegram-plugin/card-format.ts +92 -3
- package/telegram-plugin/dist/gateway/gateway.js +769 -172
- package/telegram-plugin/edit-flood-fuse.ts +477 -0
- package/telegram-plugin/format.ts +19 -7
- package/telegram-plugin/gateway/boot-sweep-gate.ts +164 -0
- package/telegram-plugin/gateway/callback-query-handlers.ts +454 -81
- package/telegram-plugin/gateway/gateway.ts +66 -56
- package/telegram-plugin/gateway/inbound-interceptors.ts +27 -4
- package/telegram-plugin/gateway/narrative-lane.ts +49 -3
- package/telegram-plugin/gateway/status-pin-api.ts +145 -0
- package/telegram-plugin/hooks/subagent-tracker-posttool.mjs +325 -45
- package/telegram-plugin/retry-api-call.ts +15 -2
- package/telegram-plugin/send-gate.ts +1 -1
- package/telegram-plugin/status-no-truncate.ts +64 -1
- package/telegram-plugin/status-pin-driver.ts +50 -27
- package/telegram-plugin/status-pin.ts +43 -5
- package/telegram-plugin/tests/activity-card-send-gate.test.ts +275 -0
- package/telegram-plugin/tests/activity-card-wiring.test.ts +16 -7
- package/telegram-plugin/tests/boot-pin-sweep-wiring.test.ts +101 -0
- package/telegram-plugin/tests/boot-sweep-gate.test.ts +293 -0
- package/telegram-plugin/tests/boot-version-string.test.ts +0 -0
- package/telegram-plugin/tests/edit-flood-fuse.test.ts +431 -0
- package/telegram-plugin/tests/pinned-card-collapse.test.ts +356 -0
- package/telegram-plugin/tests/status-pin-api.test.ts +178 -0
- package/telegram-plugin/tests/status-pin-boot-recovery.test.ts +94 -11
- package/telegram-plugin/tests/status-pin.test.ts +106 -5
- package/telegram-plugin/tests/subagent-tracker-hooks.test.ts +631 -1
- package/telegram-plugin/tests/tool-activity-summary.test.ts +19 -10
- package/telegram-plugin/tests/vault-approval-posture.test.ts +6 -1
- package/telegram-plugin/tests/vault-passphrase-retry.test.ts +666 -0
- package/telegram-plugin/tests/vault-request-access-unlock-resume.test.ts +42 -21
- package/telegram-plugin/tests/worker-feed-coalesce.test.ts +233 -1
- package/telegram-plugin/tool-activity-summary.ts +85 -13
- package/telegram-plugin/worker-activity-feed.ts +5 -1
- package/vendor/hindsight-memory/scripts/drain_pending.py +193 -25
- package/vendor/hindsight-memory/scripts/lib/pending.py +84 -5
- package/vendor/hindsight-memory/scripts/lib/retain_split.py +21 -10
- package/vendor/hindsight-memory/scripts/recall.py +74 -5
- package/vendor/hindsight-memory/scripts/tests/test_pending_drops.py +158 -4
- package/vendor/hindsight-memory/scripts/tests/test_pending_failure_class.py +105 -0
- package/vendor/hindsight-memory/scripts/tests/test_pending_wedge.py +300 -0
- package/vendor/hindsight-memory/scripts/tests/test_recall_degraded_notice.py +365 -0
- package/vendor/hindsight-memory/scripts/tests/test_recall_envelope_strip_telemetry.py +12 -4
- package/vendor/hindsight-memory/scripts/tests/test_recall_transcript_fallback.py +27 -2
- package/vendor/hindsight-memory/scripts/tests/test_retain_split.py +19 -11
- package/vendor/hindsight-memory/tests/test_drain_pending.py +28 -2
|
@@ -16,11 +16,18 @@
|
|
|
16
16
|
* `🛠 Worker` message). We NEVER send a new message to pin. This is
|
|
17
17
|
* the boundary that keeps us inside the one sanctioned exception on
|
|
18
18
|
* `chat-is-the-single-source-of-truth`.
|
|
19
|
-
* - On UNPIN we DROP THE CLAIM (clear state)
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
19
|
+
* - On UNPIN we DROP THE CLAIM (clear state) when the unpin succeeded OR
|
|
20
|
+
* failed TERMINALLY (`isUnpinTerminalError`: Telegram answered with a
|
|
21
|
+
* stable 4xx — rights revoked, message/chat gone). Pin state must never
|
|
22
|
+
* get stuck: the message may have been unpinned out-of-band (operator,
|
|
23
|
+
* or a crash), and re-claiming it would be more confusing than surfacing
|
|
24
|
+
* it again later.
|
|
25
|
+
* - #3664 Defect B: a NEVER-CONFIRMED failure (`FLOOD_WAIT_ACTIVE` — a
|
|
26
|
+
* local pre-call fail-fast — network, 5xx, retries exhausted) is the one
|
|
27
|
+
* case we do NOT drop: the message is provably still pinned, so the claim
|
|
28
|
+
* is RETAINED for retry. Dropping it erased both the in-memory claim and
|
|
29
|
+
* the durable store row, orphaning a live pin nothing could ever find
|
|
30
|
+
* again. Retention is bounded by the boot sweep's forfeit ladder.
|
|
24
31
|
* - API failures are reported via `onError` but never throw; the caller
|
|
25
32
|
* decides logging cadence.
|
|
26
33
|
*
|
|
@@ -30,7 +37,7 @@
|
|
|
30
37
|
*/
|
|
31
38
|
|
|
32
39
|
import type { PinState, DesiredPin, PinRightsCache } from './status-pin.js'
|
|
33
|
-
import { decidePinAction, isPinRightsError } from './status-pin.js'
|
|
40
|
+
import { decidePinAction, isPinRightsError, isUnpinTerminalError } from './status-pin.js'
|
|
34
41
|
|
|
35
42
|
/** Minimal subset of grammy's `bot.api` the pin driver depends on.
|
|
36
43
|
* Lets tests swap in a fake without dragging in the full Bot type. */
|
|
@@ -75,8 +82,10 @@ export interface ReconcilePinArgs {
|
|
|
75
82
|
* - `pin` : pins the existing message SILENTLY; on failure the claim is
|
|
76
83
|
* NOT taken (returns prevState) so the next reconcile retries
|
|
77
84
|
* rather than tracking a message it never pinned.
|
|
78
|
-
* - `unpin`: unpins best-effort and returns `null` — the claim is dropped
|
|
79
|
-
*
|
|
85
|
+
* - `unpin`: unpins best-effort and returns `null` — the claim is dropped on
|
|
86
|
+
* success and on a TERMINAL failure (never leave state stuck
|
|
87
|
+
* pinned). A never-confirmed failure returns prevState so the
|
|
88
|
+
* still-pinned message keeps a record to retry from (#3664).
|
|
80
89
|
* - `noop` : returns prevState unchanged.
|
|
81
90
|
*/
|
|
82
91
|
export async function reconcilePin(
|
|
@@ -89,27 +98,41 @@ export async function reconcilePin(
|
|
|
89
98
|
if (action.kind === 'unpin') {
|
|
90
99
|
// Skip the unpin API call in a chat the bot can't manage pins in — the
|
|
91
100
|
// call would fail with the same rights 400 and spam the log. The claim is
|
|
92
|
-
// dropped
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
101
|
+
// dropped (a pin we can never unpin is not worth tracking), so skipping
|
|
102
|
+
// is safe.
|
|
103
|
+
if (args.rightsCache?.isBlocked(args.chatId)) return null
|
|
104
|
+
try {
|
|
105
|
+
await args.api.unpinChatMessage(args.chatId, action.messageId)
|
|
106
|
+
} catch (err) {
|
|
107
|
+
// Symmetric with the pin path below: a permanent rights 400 on UNPIN
|
|
108
|
+
// (rights revoked mid-session after we pinned) also enters the
|
|
109
|
+
// negative cache and logs once via onPinRightsDisabled — otherwise
|
|
110
|
+
// every later unpin attempt would burn an API call and spam
|
|
111
|
+
// `status-pin unpin failed` per attempt, the exact class this cache
|
|
112
|
+
// exists to kill (#3073 review finding). That class is terminal, so the
|
|
113
|
+
// claim is dropped below.
|
|
114
|
+
if (args.rightsCache && isPinRightsError(err)) {
|
|
115
|
+
const firstTime = args.rightsCache.block(args.chatId)
|
|
116
|
+
if (firstTime) args.onPinRightsDisabled?.(args.chatId)
|
|
117
|
+
} else {
|
|
118
|
+
args.onError?.('unpin', err)
|
|
109
119
|
}
|
|
120
|
+
// #3664 Defect B: drop the claim ONLY on a TERMINAL failure — one where
|
|
121
|
+
// Telegram answered and the answer can't change (4xx ≠ 429: rights
|
|
122
|
+
// revoked, message/chat gone, bot kicked). On a NEVER-CONFIRMED failure
|
|
123
|
+
// (`FLOOD_WAIT_ACTIVE`, which is a local pre-call fail-fast; network;
|
|
124
|
+
// 5xx; retries exhausted) the message is still pinned, so we RETAIN the
|
|
125
|
+
// claim: returning prevState makes the caller keep both the in-memory
|
|
126
|
+
// claim and the durable status-pins.json row (see the non-null branch of
|
|
127
|
+
// reconcileAndPersistStatusPin), so the next reconcile, the mid-session
|
|
128
|
+
// reaper and the next-boot sweep can all retry. Dropping the last record
|
|
129
|
+
// of a pin that is provably still up is strictly worse than a claim that
|
|
130
|
+
// retries — and retention is bounded by the boot sweep's
|
|
131
|
+
// BOOT_UNPIN_MAX_ATTEMPTS forfeit ladder.
|
|
132
|
+
if (!isUnpinTerminalError(err)) return args.prevState
|
|
110
133
|
}
|
|
111
|
-
//
|
|
112
|
-
// leave a permanent pin on a crash / out-of-band unpin — the exact
|
|
134
|
+
// Unpin confirmed (or terminally rejected) — drop the claim. A stuck claim
|
|
135
|
+
// would leave a permanent pin on a crash / out-of-band unpin — the exact
|
|
113
136
|
// failure this driver exists to prevent (see slot-banner-driver.ts).
|
|
114
137
|
return null
|
|
115
138
|
}
|
|
@@ -21,7 +21,8 @@
|
|
|
21
21
|
* gateway (via `status-pin-driver.ts`) translates a `PinAction` into
|
|
22
22
|
* actual Telegram API calls. The design mirrors `slot-banner.ts` +
|
|
23
23
|
* `slot-banner-driver.ts` — one pure decision, one side-effecting
|
|
24
|
-
* reconcile, the claim dropped on unpin
|
|
24
|
+
* reconcile, the claim dropped on unpin unless the unpin never reached
|
|
25
|
+
* Telegram at all (`isUnpinTerminalError`, #3664).
|
|
25
26
|
*/
|
|
26
27
|
|
|
27
28
|
/** The message the framework is currently claiming as pinned for a key. */
|
|
@@ -43,8 +44,9 @@ export type PinAction =
|
|
|
43
44
|
* back into PinState. No new message is sent — this pins a message the
|
|
44
45
|
* chat already rendered. */
|
|
45
46
|
| { kind: 'pin'; messageId: number }
|
|
46
|
-
/** Unpin + forget. Caller unpins (best-effort) and clears state
|
|
47
|
-
* the
|
|
47
|
+
/** Unpin + forget. Caller unpins (best-effort) and clears state on success
|
|
48
|
+
* or a TERMINAL failure; a never-confirmed failure keeps the claim so the
|
|
49
|
+
* still-pinned message can be retried (`isUnpinTerminalError`, #3664). */
|
|
48
50
|
| { kind: 'unpin'; messageId: number }
|
|
49
51
|
|
|
50
52
|
/**
|
|
@@ -53,8 +55,8 @@ export type PinAction =
|
|
|
53
55
|
*
|
|
54
56
|
* Exactly one action per (prev, desired) — the whole point is that pin
|
|
55
57
|
* state can never get stuck: an in-flight → done transition always yields
|
|
56
|
-
* an `unpin`, and the driver drops the claim
|
|
57
|
-
*
|
|
58
|
+
* an `unpin`, and the driver drops the claim once the unpin is confirmed
|
|
59
|
+
* or terminally rejected (see `isUnpinTerminalError`).
|
|
58
60
|
*/
|
|
59
61
|
export function decidePinAction(
|
|
60
62
|
prev: PinState | null,
|
|
@@ -114,6 +116,42 @@ export function isPinRightsError(err: unknown): boolean {
|
|
|
114
116
|
return errorDescription(err).includes('not enough rights')
|
|
115
117
|
}
|
|
116
118
|
|
|
119
|
+
/**
|
|
120
|
+
* True when an UNPIN failure is TERMINAL — Telegram received the request and
|
|
121
|
+
* answered definitively, so the answer will not change on retry and the claim
|
|
122
|
+
* is safe to drop (#3664 Defect B).
|
|
123
|
+
*
|
|
124
|
+
* The distinction that matters: a 4xx (other than 429) means the call REACHED
|
|
125
|
+
* Telegram and was rejected for a reason that is stable — the bot lacks pin
|
|
126
|
+
* rights, the message/chat is gone, the bot was kicked. Re-issuing the same
|
|
127
|
+
* unpin forever cannot help, and holding the claim would leave the driver
|
|
128
|
+
* stuck (the failure mode the drop-on-unpin contract exists to prevent).
|
|
129
|
+
*
|
|
130
|
+
* Everything else is NOT confirmed and MUST retain the claim:
|
|
131
|
+
*
|
|
132
|
+
* - `FLOOD_WAIT_ACTIVE` (retry-api-call.ts) — a purely LOCAL fail-fast: the
|
|
133
|
+
* send gate refused to make the call, so no request ever left the process
|
|
134
|
+
* and the message is provably still pinned. It carries Telegram's 429
|
|
135
|
+
* duck-type shape, which is why 429 is excluded here.
|
|
136
|
+
* - `retryApiCall: max retries exceeded`, network HttpError, 5xx — the
|
|
137
|
+
* unpin may never have landed.
|
|
138
|
+
*
|
|
139
|
+
* Dropping the claim on those erased the in-memory claim AND (via
|
|
140
|
+
* `reconcileAndPersistStatusPin`, which deletes the durable row on a null
|
|
141
|
+
* return) the persisted record — leaving a message pinned in Telegram that no
|
|
142
|
+
* reaper and no boot sweep could ever see again. Retaining is bounded: the
|
|
143
|
+
* boot sweep retries a retained row and forfeits it after
|
|
144
|
+
* `BOOT_UNPIN_MAX_ATTEMPTS`, so a retained claim can't be permanent either.
|
|
145
|
+
*/
|
|
146
|
+
export function isUnpinTerminalError(err: unknown): boolean {
|
|
147
|
+
if (isPinRightsError(err)) return true
|
|
148
|
+
if (err != null && typeof err === 'object') {
|
|
149
|
+
const code = (err as { error_code?: unknown }).error_code
|
|
150
|
+
if (typeof code === 'number') return code >= 400 && code < 500 && code !== 429
|
|
151
|
+
}
|
|
152
|
+
return false
|
|
153
|
+
}
|
|
154
|
+
|
|
117
155
|
/**
|
|
118
156
|
* Per-process, per-chat negative cache for chats the bot cannot pin in.
|
|
119
157
|
*
|
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Regression: the live activity/progress card must be RATE-BOUND by the send
|
|
3
|
+
* gate (#3620).
|
|
4
|
+
*
|
|
5
|
+
* ── The bug this pins ─────────────────────────────────────────────────────
|
|
6
|
+
* `drainActivitySummary` edited the card with
|
|
7
|
+
* robustApiCall(() => bot.api.editMessageText(...), { chat_id, verb })
|
|
8
|
+
* — no `messageId`, no `editPayload`. `sendGate.gate()` only routes to its
|
|
9
|
+
* edit path when BOTH are present, so every card edit fell through to the
|
|
10
|
+
* plain-send path and got NONE of the per-message protections:
|
|
11
|
+
* - no 1500ms per-message edit floor
|
|
12
|
+
* - no last-write-wins coalescing (the incident logged `coalesced=0`)
|
|
13
|
+
* - no identical-payload no-op skip (`dropped=0`)
|
|
14
|
+
* - no long-horizon per-message edit budget
|
|
15
|
+
* and, being untagged, it was admitted as `critical` (UNTAGGED_SEND_CLASS) —
|
|
16
|
+
* never shed, queued unbounded. One message could therefore be edited at the
|
|
17
|
+
* per-chat bucket rate for as long as a turn ran, which earned agent
|
|
18
|
+
* `overlord` a 3713s Telegram flood ban on 2026-07-25.
|
|
19
|
+
*
|
|
20
|
+
* ── Oracle ────────────────────────────────────────────────────────────────
|
|
21
|
+
* The REAL `createNarrativeLane` driven against a fake bot recorder, with the
|
|
22
|
+
* REAL `createSendGate` wired into `robustApiCall` on a deterministic fake
|
|
23
|
+
* clock — i.e. the exact production seam, no re-implementation.
|
|
24
|
+
*
|
|
25
|
+
* ── The outcome asserted (not the code path) ──────────────────────────────
|
|
26
|
+
* The per-chat/global buckets are configured WIDE OPEN so they cannot be the
|
|
27
|
+
* thing doing the limiting; the only remaining brake is the per-message edit
|
|
28
|
+
* discipline. Then N=40 renders over 10s of fake time must produce a bounded
|
|
29
|
+
* number of API calls AND the last call must carry the NEWEST body. On the
|
|
30
|
+
* pre-fix code this test fails on the call count (40 edits issued).
|
|
31
|
+
*/
|
|
32
|
+
import { describe, it, expect } from 'vitest'
|
|
33
|
+
import { tmpdir } from 'node:os'
|
|
34
|
+
import { createNarrativeLane } from '../gateway/narrative-lane.js'
|
|
35
|
+
import { createSendGate, SEND_GATE_DEFAULTS, type Clock, type SendGateOpts } from '../send-gate.js'
|
|
36
|
+
import type { CurrentTurn, NarrativeLaneDeps } from '../gateway/gateway.js'
|
|
37
|
+
|
|
38
|
+
const CHAT = '1001'
|
|
39
|
+
|
|
40
|
+
// ── deterministic clock (same shape as send-gate.test.ts) ─────────────────
|
|
41
|
+
class FakeClock implements Clock {
|
|
42
|
+
private cur = 0
|
|
43
|
+
private seq = 0
|
|
44
|
+
private timers: { at: number; id: number; resolve: () => void }[] = []
|
|
45
|
+
|
|
46
|
+
now(): number {
|
|
47
|
+
return this.cur
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
sleep(ms: number): Promise<void> {
|
|
51
|
+
return new Promise<void>((resolve) => {
|
|
52
|
+
this.timers.push({ at: this.cur + ms, id: this.seq++, resolve })
|
|
53
|
+
})
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
async advance(ms: number): Promise<void> {
|
|
57
|
+
const target = this.cur + ms
|
|
58
|
+
for (;;) {
|
|
59
|
+
await flush()
|
|
60
|
+
const due = this.timers
|
|
61
|
+
.filter((t) => t.at <= target)
|
|
62
|
+
.sort((a, b) => a.at - b.at || a.id - b.id)
|
|
63
|
+
if (due.length === 0) break
|
|
64
|
+
const t = due[0]!
|
|
65
|
+
this.timers = this.timers.filter((x) => x !== t)
|
|
66
|
+
this.cur = t.at
|
|
67
|
+
t.resolve()
|
|
68
|
+
await flush()
|
|
69
|
+
}
|
|
70
|
+
this.cur = target
|
|
71
|
+
await flush()
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function flush(): Promise<void> {
|
|
76
|
+
return new Promise<void>((r) => setImmediate(r))
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
// ── fake bot recorder ─────────────────────────────────────────────────────
|
|
80
|
+
interface RecordedCall { method: string; text: string; message_id?: number; at: number }
|
|
81
|
+
|
|
82
|
+
function makeFakeBot(clock: FakeClock) {
|
|
83
|
+
const calls: RecordedCall[] = []
|
|
84
|
+
let nextId = 3000
|
|
85
|
+
const api = {
|
|
86
|
+
sendRichMessage: async (_c: string, b: { markdown: string }) => {
|
|
87
|
+
const id = ++nextId
|
|
88
|
+
calls.push({ method: 'sendRichMessage', text: b.markdown, message_id: id, at: clock.now() })
|
|
89
|
+
return { message_id: id }
|
|
90
|
+
},
|
|
91
|
+
sendMessage: async (_c: string, t: string) => {
|
|
92
|
+
const id = ++nextId
|
|
93
|
+
calls.push({ method: 'sendMessage', text: t, message_id: id, at: clock.now() })
|
|
94
|
+
return { message_id: id }
|
|
95
|
+
},
|
|
96
|
+
editMessageText: async (_c: string, m: number, b: unknown) => {
|
|
97
|
+
calls.push({
|
|
98
|
+
method: 'editMessageText',
|
|
99
|
+
text: typeof b === 'string' ? b : (b as { markdown: string }).markdown,
|
|
100
|
+
message_id: m,
|
|
101
|
+
at: clock.now(),
|
|
102
|
+
})
|
|
103
|
+
return {}
|
|
104
|
+
},
|
|
105
|
+
deleteMessage: async () => true,
|
|
106
|
+
}
|
|
107
|
+
return { api, calls }
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
function makeLaneTurn(lane: ReturnType<typeof createNarrativeLane>): CurrentTurn {
|
|
111
|
+
const turn = {
|
|
112
|
+
turnId: 'turn-gate-1',
|
|
113
|
+
sessionChatId: CHAT,
|
|
114
|
+
sessionThreadId: undefined,
|
|
115
|
+
sourceMessageId: null,
|
|
116
|
+
registryKey: null,
|
|
117
|
+
startedAt: 0,
|
|
118
|
+
currentModel: null,
|
|
119
|
+
totalTokens: 0,
|
|
120
|
+
labeledToolCount: 0,
|
|
121
|
+
mirrorLines: [] as string[],
|
|
122
|
+
foregroundSubAgents: new Map<string, string[]>(),
|
|
123
|
+
activityPendingRender: null as string | null,
|
|
124
|
+
activityLastSentRender: null as string | null,
|
|
125
|
+
activityMessageId: null as number | null,
|
|
126
|
+
activityInFlight: null as Promise<void> | null,
|
|
127
|
+
activityEverOpened: false,
|
|
128
|
+
activityDrainFailures: 0,
|
|
129
|
+
finalAnswerEverDelivered: false,
|
|
130
|
+
finalAnswerDelivered: false,
|
|
131
|
+
replyCalled: false,
|
|
132
|
+
capturedText: [] as string[],
|
|
133
|
+
lastReplyText: '',
|
|
134
|
+
answerStream: null,
|
|
135
|
+
liveness: { recentlyStreaming: () => false, onStreamEvent: () => {}, note: () => {} },
|
|
136
|
+
} as unknown as CurrentTurn
|
|
137
|
+
;(turn as { narrativeGate?: unknown }).narrativeGate = lane.makeNarrativeGate(turn)
|
|
138
|
+
return turn
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/**
|
|
142
|
+
* The lane wired to the REAL send gate. `bucketOverrides` lets a case widen the
|
|
143
|
+
* chat/global buckets so the ONLY brake left is the per-message edit discipline
|
|
144
|
+
* (that is what makes the assertion a statement about the fix and not about the
|
|
145
|
+
* unrelated per-chat bucket).
|
|
146
|
+
*/
|
|
147
|
+
function makeGatedLane(bucketOverrides: Record<string, number> = {}) {
|
|
148
|
+
const clock = new FakeClock()
|
|
149
|
+
const { api, calls } = makeFakeBot(clock)
|
|
150
|
+
const gate = createSendGate({
|
|
151
|
+
enabled: true,
|
|
152
|
+
clock,
|
|
153
|
+
globalPerSec: 1000,
|
|
154
|
+
globalBurst: 1000,
|
|
155
|
+
perChatPerSec: 1000,
|
|
156
|
+
perChatBurst: 1000,
|
|
157
|
+
perGroupPerMin: 100000,
|
|
158
|
+
perGroupBurst: 100000,
|
|
159
|
+
...bucketOverrides,
|
|
160
|
+
})
|
|
161
|
+
const noop = () => {}
|
|
162
|
+
const fakeEA = {
|
|
163
|
+
mayDrain: () => true,
|
|
164
|
+
openOrEditCard: (_p: string, fn: () => void) => fn(),
|
|
165
|
+
finalizeCard: (fn: () => void) => fn(),
|
|
166
|
+
markSubstantiveFinalDelivered: (fn: () => void) => fn(),
|
|
167
|
+
}
|
|
168
|
+
const deps = {
|
|
169
|
+
ACTIVITY_CARD_STORE_PATH: `${tmpdir()}/lane-sendgate-activity-cards.json`,
|
|
170
|
+
CLEAR_STATUS_ON_COMPLETION: false,
|
|
171
|
+
FEED_HEARTBEAT_ENABLED: false,
|
|
172
|
+
FEED_HEARTBEAT_MIN_STALE_MS: 6000,
|
|
173
|
+
FEED_LIVENESS_OPEN_ENABLED: false,
|
|
174
|
+
FEED_LIVENESS_OPEN_MS: 5000,
|
|
175
|
+
PIN_STATUS_WHILE_WORKING: false,
|
|
176
|
+
POST_ANSWER_LIVENESS_STALE_MS: 90000,
|
|
177
|
+
STATIC: false,
|
|
178
|
+
activeDraftStreams: new Map(),
|
|
179
|
+
activityCardPersistEnabled: false,
|
|
180
|
+
activityCardStoreFs: {
|
|
181
|
+
readFileSync: () => '', writeFileSync: noop, mkdirSync: noop, renameSync: noop, unlinkSync: noop,
|
|
182
|
+
},
|
|
183
|
+
bot: { api },
|
|
184
|
+
cardDrainGate: (_t: unknown, _ea: unknown, run: () => void) => run(),
|
|
185
|
+
currentTurnMap: { get: () => null, byKey: new Map() },
|
|
186
|
+
earlyLivenessOpenTimers: new Map(),
|
|
187
|
+
emissionAuthorityFor: () => fakeEA,
|
|
188
|
+
feedOpenGateDeps: () => ({ hasOutboundDeliveredSince: () => false, historyEnabled: false, finalAnswerMinChars: 200 }),
|
|
189
|
+
getCurrentTurn: () => null,
|
|
190
|
+
reconcileStatusPin: noop,
|
|
191
|
+
robustApiCall: (fn: () => Promise<unknown>, opts?: SendGateOpts) => gate.gate(fn, opts),
|
|
192
|
+
statusKey: (c: string, t?: number | null) => `${c}:${t ?? 'main'}`,
|
|
193
|
+
} as unknown as NarrativeLaneDeps
|
|
194
|
+
const lane = createNarrativeLane(deps)
|
|
195
|
+
return { lane, calls, clock, gate }
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
describe('activity card ↔ send gate (#3620)', () => {
|
|
199
|
+
it('40 rapid card updates over 10s collapse to a handful of edits, newest last', async () => {
|
|
200
|
+
// Buckets are wide open: whatever bounds the edits here is the PER-MESSAGE
|
|
201
|
+
// edit discipline, which is exactly what the fix engages.
|
|
202
|
+
const { lane, calls, clock } = makeGatedLane()
|
|
203
|
+
const turn = makeLaneTurn(lane)
|
|
204
|
+
|
|
205
|
+
lane.showNarrativeStep(turn, 'Step 0 opening the card')
|
|
206
|
+
await clock.advance(10)
|
|
207
|
+
await turn.activityInFlight
|
|
208
|
+
expect(turn.activityMessageId).not.toBeNull()
|
|
209
|
+
|
|
210
|
+
const N = 40
|
|
211
|
+
const windowMs = 10_000
|
|
212
|
+
for (let i = 1; i <= N; i++) {
|
|
213
|
+
lane.showNarrativeStep(turn, `Step ${i} of the work`)
|
|
214
|
+
await clock.advance(windowMs / N)
|
|
215
|
+
}
|
|
216
|
+
// Let any in-flight floor sleep settle so the newest body is painted.
|
|
217
|
+
await clock.advance(SEND_GATE_DEFAULTS.editFloorMs * 2)
|
|
218
|
+
await turn.activityInFlight
|
|
219
|
+
await lane.drainActivitySummary(turn)
|
|
220
|
+
await clock.advance(SEND_GATE_DEFAULTS.editFloorMs * 2)
|
|
221
|
+
await turn.activityInFlight
|
|
222
|
+
|
|
223
|
+
const edits = calls.filter((c) => c.method === 'editMessageText')
|
|
224
|
+
// (a) BOUNDED: 10s of fake time under a 1500ms floor admits at most
|
|
225
|
+
// ceil(10000/1500)+1 = 8 edits (+2 for the post-window settle drains). The
|
|
226
|
+
// pre-fix code issues one edit per render (40) and fails here.
|
|
227
|
+
const K = Math.ceil(windowMs / SEND_GATE_DEFAULTS.editFloorMs) + 3
|
|
228
|
+
expect(edits.length).toBeLessThanOrEqual(K)
|
|
229
|
+
expect(edits.length).toBeGreaterThan(0)
|
|
230
|
+
// Every edit went to the one card — never a second bubble.
|
|
231
|
+
expect(calls.filter((c) => c.method === 'sendRichMessage')).toHaveLength(1)
|
|
232
|
+
expect(edits.every((e) => e.message_id === turn.activityMessageId)).toBe(true)
|
|
233
|
+
// (b) NEWEST WINS: the last edit carries the latest state, and no
|
|
234
|
+
// intermediate frame is issued after it.
|
|
235
|
+
expect(edits[edits.length - 1]!.text).toContain(`Step ${N} of the work`)
|
|
236
|
+
// (c) SPACING: consecutive edits of the same message respect the floor.
|
|
237
|
+
for (let i = 1; i < edits.length; i++) {
|
|
238
|
+
expect(edits[i]!.at - edits[i - 1]!.at).toBeGreaterThanOrEqual(SEND_GATE_DEFAULTS.editFloorMs)
|
|
239
|
+
}
|
|
240
|
+
})
|
|
241
|
+
|
|
242
|
+
it('a shed card edit is not treated as painted — the newest render survives for the next drain', async () => {
|
|
243
|
+
// A one-token-per-minute chat bucket: the card open consumes it, so the
|
|
244
|
+
// next COSMETIC card edit sheds.
|
|
245
|
+
const { lane, calls, clock } = makeGatedLane({ perChatPerSec: 1 / 60, perChatBurst: 1 })
|
|
246
|
+
const turn = makeLaneTurn(lane)
|
|
247
|
+
|
|
248
|
+
lane.showNarrativeStep(turn, 'Opening the card')
|
|
249
|
+
await clock.advance(10)
|
|
250
|
+
await turn.activityInFlight
|
|
251
|
+
expect(calls.filter((c) => c.method === 'sendRichMessage')).toHaveLength(1)
|
|
252
|
+
|
|
253
|
+
lane.showNarrativeStep(turn, 'A shed update')
|
|
254
|
+
await clock.advance(10)
|
|
255
|
+
await turn.activityInFlight
|
|
256
|
+
// Shed → nothing hit the API, and the render is still PENDING (not
|
|
257
|
+
// mis-recorded as sent), so it is not silently lost.
|
|
258
|
+
expect(calls.filter((c) => c.method === 'editMessageText')).toHaveLength(0)
|
|
259
|
+
expect(turn.activityPendingRender).not.toBeNull()
|
|
260
|
+
expect(turn.activityPendingRender).not.toBe(turn.activityLastSentRender)
|
|
261
|
+
|
|
262
|
+
// Once the bucket refills, the next drain paints the NEWEST body — not the
|
|
263
|
+
// shed intermediate one.
|
|
264
|
+
lane.showNarrativeStep(turn, 'The newest update')
|
|
265
|
+
await clock.advance(120_000)
|
|
266
|
+
await turn.activityInFlight
|
|
267
|
+
await lane.drainActivitySummary(turn)
|
|
268
|
+
await clock.advance(10_000)
|
|
269
|
+
await turn.activityInFlight
|
|
270
|
+
|
|
271
|
+
const edits = calls.filter((c) => c.method === 'editMessageText')
|
|
272
|
+
expect(edits.length).toBeGreaterThan(0)
|
|
273
|
+
expect(edits[edits.length - 1]!.text).toContain('The newest update')
|
|
274
|
+
})
|
|
275
|
+
})
|
|
@@ -119,26 +119,35 @@ describe('activity-card durability wiring', () => {
|
|
|
119
119
|
|
|
120
120
|
it('the boot reaper runs ONLY after the startup mutex is won (never at import time)', () => {
|
|
121
121
|
// #3026: the reaper is now invoked via the mutex-gated orchestrator
|
|
122
|
-
// runBootPinCleanupAndDmSweep() (which
|
|
122
|
+
// runBootPinCleanupAndDmSweep() (which runs it alongside
|
|
123
123
|
// statusPinBootCleanup + queuedCardBootReaper, then runs the DM
|
|
124
124
|
// stale-pin sweep). Invariant unchanged: reachable only post-lock.
|
|
125
|
-
// (1) The orchestrator
|
|
125
|
+
// (1) The orchestrator runs the reaper. Since the #3664 S2 salvage the
|
|
126
|
+
// steps are INJECTED into runBootPinSweepSteps (each individually
|
|
127
|
+
// absorbed, so a throwing reaper cannot strand the DM sweep behind it)
|
|
128
|
+
// instead of being awaited inline — so the marker is the step binding.
|
|
126
129
|
const orchestrator = between(
|
|
127
130
|
gatewaySrc,
|
|
128
|
-
'
|
|
131
|
+
'function runBootPinCleanupAndDmSweep()',
|
|
129
132
|
'dmPinSweepEligible = true',
|
|
130
133
|
)
|
|
131
|
-
expect(orchestrator).toMatch(/
|
|
134
|
+
expect(orchestrator).toMatch(/activityCardReaper: activityCardBootReaper,/)
|
|
132
135
|
// (2) Both orchestrator invocation sites (mutex-won + mutex-fallback)
|
|
133
|
-
// sit AFTER acquireStartupLock.
|
|
136
|
+
// sit AFTER acquireStartupLock. Since #3664 they ARM the boot sweep gate
|
|
137
|
+
// rather than dispatching directly: the sweep now also needs `lockedBot`
|
|
138
|
+
// to exist (gate.botReady() at the end of initGatewayBot), because every
|
|
139
|
+
// boot unpin was dereferencing an unassigned lockedBot. The mutex half of
|
|
140
|
+
// the invariant — reachable only post-lock — is unchanged and asserted
|
|
141
|
+
// here; the bot-ready half lives in boot-pin-sweep-wiring.test.ts.
|
|
134
142
|
const afterLock = between(gatewaySrc, 'await acquireStartupLock({', 'catch (err)')
|
|
135
|
-
expect(afterLock).toMatch(/
|
|
136
|
-
// (3) Neither the reaper nor the
|
|
143
|
+
expect(afterLock).toMatch(/bootPinSweepGate\.arm\(\)/)
|
|
144
|
+
// (3) Neither the reaper nor the sweep is armed/invoked at module import
|
|
137
145
|
// time — the same losing-double-boot hazard the NOTE at
|
|
138
146
|
// statusPinBootCleanup documents.
|
|
139
147
|
const beforeMain = gatewaySrc.split('await acquireStartupLock({')[0] ?? ''
|
|
140
148
|
expect(beforeMain).not.toMatch(/^\s*void activityCardBootReaper\(\)/m)
|
|
141
149
|
expect(beforeMain).not.toMatch(/^\s*void runBootPinCleanupAndDmSweep\(\)/m)
|
|
150
|
+
expect(beforeMain).not.toMatch(/^\s*bootPinSweepGate\.(arm|botReady)\(\)/m)
|
|
142
151
|
})
|
|
143
152
|
|
|
144
153
|
it('the reaper wrapper counts a benign-400 as vanished, not finalized (honest boot log)', () => {
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Boot pin sweep WIRING fence (#3664 Defect A).
|
|
3
|
+
*
|
|
4
|
+
* The behaviour lives in `boot-sweep-gate.test.ts`; this file pins the gateway
|
|
5
|
+
* WIRING, because the regression was a wiring regression: `caa4d7568` (#3310)
|
|
6
|
+
* moved the `lockedBot` assignment into `initGatewayBot()` and left the sweep's
|
|
7
|
+
* kick-off at module-eval time. `tsc` stayed green (the `!` definite-assignment
|
|
8
|
+
* assertion on `lockedBot` suppresses the error) and no test noticed, so every
|
|
9
|
+
* boot-cleanup unpin threw `undefined is not an object` for a month.
|
|
10
|
+
*
|
|
11
|
+
* These are STRUCTURAL assertions (the gateway module can't be instantiated
|
|
12
|
+
* in-process — same pattern as silence-liveness-wiring.test.ts) that fail if a
|
|
13
|
+
* future refactor re-introduces a module-eval-time dispatch of the sweep.
|
|
14
|
+
*/
|
|
15
|
+
import { describe, it, expect } from 'vitest'
|
|
16
|
+
import { readFileSync } from 'node:fs'
|
|
17
|
+
import { resolve } from 'node:path'
|
|
18
|
+
|
|
19
|
+
const gatewaySrc = readFileSync(resolve(__dirname, '..', 'gateway', 'gateway.ts'), 'utf-8')
|
|
20
|
+
|
|
21
|
+
describe('boot pin sweep wiring (#3664)', () => {
|
|
22
|
+
it('never dispatches runBootPinCleanupAndDmSweep() directly — only through the gate', () => {
|
|
23
|
+
// The original bug shape, verbatim: a fire-and-forget call at module-eval
|
|
24
|
+
// time. Any direct invocation (voided or awaited) bypasses the bot-ready
|
|
25
|
+
// half of the precondition.
|
|
26
|
+
const directCalls =
|
|
27
|
+
gatewaySrc.match(/(?<!function\s)(?<![.\w])runBootPinCleanupAndDmSweep\s*\(/g) ?? []
|
|
28
|
+
expect(directCalls).toEqual([])
|
|
29
|
+
})
|
|
30
|
+
|
|
31
|
+
it('passes the sweep to the gate as a reference, and arms it at both mutex sites', () => {
|
|
32
|
+
expect(gatewaySrc).toContain('createBootSweepGate({ run: runBootPinCleanupAndDmSweep')
|
|
33
|
+
// Both startup-lock outcomes (mutex acquired, and the non-atomic
|
|
34
|
+
// writePidFile fallback) arm — neither may dispatch on its own.
|
|
35
|
+
const armSites = gatewaySrc.match(/bootPinSweepGate\.arm\(\)/g) ?? []
|
|
36
|
+
expect(armSites.length).toBe(2)
|
|
37
|
+
})
|
|
38
|
+
|
|
39
|
+
it('signals botReady() only from initGatewayBot, AFTER lockedBot is assigned', () => {
|
|
40
|
+
const readySites = gatewaySrc.match(/bootPinSweepGate\.botReady\(\)/g) ?? []
|
|
41
|
+
expect(readySites.length).toBe(1)
|
|
42
|
+
|
|
43
|
+
const initStart = gatewaySrc.indexOf('async function initGatewayBot()')
|
|
44
|
+
const assignIdx = gatewaySrc.indexOf('lockedBot = chatLock.wrapBot(')
|
|
45
|
+
const readyIdx = gatewaySrc.indexOf('bootPinSweepGate.botReady()')
|
|
46
|
+
expect(initStart).toBeGreaterThan(-1)
|
|
47
|
+
expect(assignIdx).toBeGreaterThan(initStart)
|
|
48
|
+
expect(readyIdx).toBeGreaterThan(assignIdx)
|
|
49
|
+
})
|
|
50
|
+
|
|
51
|
+
it('runs the sweep steps through runBootPinSweepSteps, not a bare await chain', () => {
|
|
52
|
+
// Salvage S2. The BEHAVIOUR (a throwing step must not strand the steps
|
|
53
|
+
// behind it, and must never strand the DM-eligibility flip) is proven in
|
|
54
|
+
// boot-sweep-gate.test.ts. This fence is what makes that proof apply to the
|
|
55
|
+
// gateway: the reapers must reach the runner as injected steps, and nothing
|
|
56
|
+
// may reintroduce the bare sequential chain that skipped the isolation.
|
|
57
|
+
expect(gatewaySrc).toContain('return runBootPinSweepSteps({')
|
|
58
|
+
const stepFns = ['statusPinBootCleanup', 'activityCardBootReaper', 'queuedCardBootReaper']
|
|
59
|
+
for (const stepFn of stepFns) {
|
|
60
|
+
// Passed BY REFERENCE as a step, never invoked inline in the sweep.
|
|
61
|
+
expect(gatewaySrc).toContain(`: ${stepFn},`)
|
|
62
|
+
const inlineAwaits = gatewaySrc.match(new RegExp(`await\\s+${stepFn}\\s*\\(`, 'g')) ?? []
|
|
63
|
+
expect(inlineAwaits).toEqual([])
|
|
64
|
+
}
|
|
65
|
+
// The DM-eligibility flip is what a throw used to strand, so it must live
|
|
66
|
+
// INSIDE the enableDmSweep step callback — not as a statement sequenced
|
|
67
|
+
// after the reapers, where one rejection skips it for the whole session.
|
|
68
|
+
const lines = gatewaySrc.split('\n')
|
|
69
|
+
const flipIdxs = lines
|
|
70
|
+
.map((l, i) => ({ l, i }))
|
|
71
|
+
.filter(({ l }) => /^\s*dmPinSweepEligible = true\s*$/.test(l))
|
|
72
|
+
.map(({ i }) => i)
|
|
73
|
+
expect(flipIdxs.length).toBe(1)
|
|
74
|
+
expect(lines[flipIdxs[0] - 1]).toContain('enableDmSweep: () => {')
|
|
75
|
+
})
|
|
76
|
+
|
|
77
|
+
it('routes the pin API through the asserting seam, never a raw lockedBot.api pin', () => {
|
|
78
|
+
// Salvage S1/S3: `statusPinApi()` must build on `createStatusPinApi`, which
|
|
79
|
+
// asserts the bot exists and converts a send-gate SHED into a throw. A
|
|
80
|
+
// hand-rolled `robustApiCall(() => lockedBot.api.pinChatMessage(...))`
|
|
81
|
+
// anywhere in the gateway would bypass both invariants.
|
|
82
|
+
expect(gatewaySrc).toContain('return createStatusPinApi(')
|
|
83
|
+
// Every `status-pin.*` verb is now issued from status-pin-api.ts. A raw
|
|
84
|
+
// `robustApiCall(() => lockedBot.api.unpinChatMessage(…), { verb:
|
|
85
|
+
// 'status-pin.unpin' })` reappearing here would be a driver call that
|
|
86
|
+
// skips assertBotReady + assertLanded.
|
|
87
|
+
const statusPinVerbs = gatewaySrc.match(/verb:\s*['"`]status-pin\.(?:un)?pin['"`]/g) ?? []
|
|
88
|
+
expect(statusPinVerbs).toEqual([])
|
|
89
|
+
})
|
|
90
|
+
|
|
91
|
+
it('arms only from inside the startup-mutex block, never at bare module scope', () => {
|
|
92
|
+
// Every arm() site must be indented (i.e. nested inside the
|
|
93
|
+
// `if (isGatewayMain) { … }` startup-lock block), never a top-level
|
|
94
|
+
// statement that would run unconditionally on import.
|
|
95
|
+
for (const line of gatewaySrc.split('\n')) {
|
|
96
|
+
if (line.includes('bootPinSweepGate.arm()')) {
|
|
97
|
+
expect(line.startsWith(' ')).toBe(true)
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
})
|
|
101
|
+
})
|