switchroom 0.18.26 → 0.18.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -2
- package/dist/cli/ms-365-write-pretool.mjs +4953 -14
- package/dist/cli/switchroom.js +1 -1
- package/dist/host-control/main.js +1 -1
- package/package.json +1 -1
- package/profiles/_base/start.sh.hbs +16 -0
- package/telegram-plugin/dist/gateway/gateway.js +571 -43
- package/telegram-plugin/flushed-turn-supersede.ts +58 -0
- package/telegram-plugin/gateway/derive-turn-id.ts +32 -0
- package/telegram-plugin/gateway/gateway.ts +358 -53
- package/telegram-plugin/gateway/handback-preturn-signal.ts +442 -0
- package/telegram-plugin/gateway/model-command.ts +68 -0
- package/telegram-plugin/gateway/ms365-write-approval.test.ts +101 -0
- package/telegram-plugin/gateway/ms365-write-approval.ts +65 -3
- package/telegram-plugin/gateway/subagent-handback-inbound-builder.ts +12 -0
- package/telegram-plugin/gateway/turn-active-marker.ts +35 -0
- package/telegram-plugin/gateway/worker-pin-reaper.ts +54 -0
- package/telegram-plugin/send-gate.test.ts +138 -0
- package/telegram-plugin/send-gate.ts +104 -1
- package/telegram-plugin/tests/activity-ever-opened-sticky.test.ts +14 -4
- package/telegram-plugin/tests/effort-command.test.ts +47 -0
- package/telegram-plugin/tests/flushed-turn-supersede.test.ts +60 -0
- package/telegram-plugin/tests/handback-preturn-adoption-roundtrip.test.ts +211 -0
- package/telegram-plugin/tests/handback-preturn-signal.test.ts +346 -0
- package/telegram-plugin/tests/model-command.test.ts +112 -0
- package/telegram-plugin/tests/multitopic-routing-wiring.test.ts +14 -2
- package/telegram-plugin/tests/outbound-send-chunks.test.ts +57 -0
- package/telegram-plugin/tests/permission-no-repeat-wiring.test.ts +18 -11
- package/telegram-plugin/tests/reply-owner-resolve.test.ts +90 -0
- package/telegram-plugin/tests/subagent-handback-inbound-builder.test.ts +5 -0
- package/telegram-plugin/tests/turn-active-marker.test.ts +29 -0
- package/telegram-plugin/tests/worker-activity-feed.test.ts +121 -0
- package/telegram-plugin/tests/worker-feed-migration-eviction.test.ts +140 -0
- package/telegram-plugin/tests/worker-pin-reaper.test.ts +78 -0
- package/telegram-plugin/worker-activity-feed.ts +169 -6
|
@@ -57,10 +57,27 @@ export interface Ms365WritePreview {
|
|
|
57
57
|
/** Byte delta — present only for OneDrive uploads with known sizes. */
|
|
58
58
|
sizeBytesBefore?: number;
|
|
59
59
|
sizeBytesAfter?: number;
|
|
60
|
+
/**
|
|
61
|
+
* "start → end" human string for calendar events, resolved from Graph.
|
|
62
|
+
* Present only when the opaque event id resolved successfully (#3267).
|
|
63
|
+
*/
|
|
64
|
+
eventWhen?: string;
|
|
65
|
+
/**
|
|
66
|
+
* Structural before→after diff for the fields the mutation changes
|
|
67
|
+
* (calendar body/location/time). Present only when resolved (#3267).
|
|
68
|
+
*/
|
|
69
|
+
changes?: Ms365PreviewChange[];
|
|
60
70
|
/** 1-line agent rationale — advisory; operator should not over-trust. */
|
|
61
71
|
agentRationale?: string;
|
|
62
72
|
}
|
|
63
73
|
|
|
74
|
+
/** A single before→after change rendered on the card. */
|
|
75
|
+
export interface Ms365PreviewChange {
|
|
76
|
+
field: string;
|
|
77
|
+
before?: string;
|
|
78
|
+
after?: string;
|
|
79
|
+
}
|
|
80
|
+
|
|
64
81
|
/**
|
|
65
82
|
* Validate a wire payload into a typed Ms365WritePreview. Returns null
|
|
66
83
|
* on malformed input (defense in depth — the hook is trusted but the
|
|
@@ -84,10 +101,32 @@ export function validateMs365Preview(input: unknown): Ms365WritePreview | null {
|
|
|
84
101
|
if (typeof o.deepLink === "string") out.deepLink = o.deepLink;
|
|
85
102
|
if (typeof o.sizeBytesBefore === "number") out.sizeBytesBefore = o.sizeBytesBefore;
|
|
86
103
|
if (typeof o.sizeBytesAfter === "number") out.sizeBytesAfter = o.sizeBytesAfter;
|
|
104
|
+
if (typeof o.eventWhen === "string") out.eventWhen = o.eventWhen;
|
|
105
|
+
const changes = sanitizeChanges(o.changes);
|
|
106
|
+
if (changes) out.changes = changes;
|
|
87
107
|
if (typeof o.agentRationale === "string") out.agentRationale = o.agentRationale;
|
|
88
108
|
return out;
|
|
89
109
|
}
|
|
90
110
|
|
|
111
|
+
/**
|
|
112
|
+
* Validate the wire `changes` array into typed before→after entries. Drops
|
|
113
|
+
* malformed entries defensively; returns undefined when nothing usable.
|
|
114
|
+
*/
|
|
115
|
+
function sanitizeChanges(input: unknown): Ms365PreviewChange[] | undefined {
|
|
116
|
+
if (!Array.isArray(input)) return undefined;
|
|
117
|
+
const out: Ms365PreviewChange[] = [];
|
|
118
|
+
for (const raw of input) {
|
|
119
|
+
if (!raw || typeof raw !== "object") continue;
|
|
120
|
+
const c = raw as Record<string, unknown>;
|
|
121
|
+
if (typeof c.field !== "string" || c.field.length === 0) continue;
|
|
122
|
+
const entry: Ms365PreviewChange = { field: c.field };
|
|
123
|
+
if (typeof c.before === "string") entry.before = c.before;
|
|
124
|
+
if (typeof c.after === "string") entry.after = c.after;
|
|
125
|
+
out.push(entry);
|
|
126
|
+
}
|
|
127
|
+
return out.length > 0 ? out : undefined;
|
|
128
|
+
}
|
|
129
|
+
|
|
91
130
|
// ────────────────────────────────────────────────────────────────────────
|
|
92
131
|
// Handler — DI shape mirrors DriveApprovalHandlerDeps
|
|
93
132
|
// ────────────────────────────────────────────────────────────────────────
|
|
@@ -168,6 +207,9 @@ export function buildMs365CardText(p: Ms365WritePreview): string {
|
|
|
168
207
|
lines.push(`ID: ${truncate(p.itemId, 96)}`);
|
|
169
208
|
}
|
|
170
209
|
lines.push(`Account: ${truncate(p.accountEmail, 96)}`);
|
|
210
|
+
if (p.eventWhen) {
|
|
211
|
+
lines.push(`When: ${truncate(p.eventWhen, 96)}`);
|
|
212
|
+
}
|
|
171
213
|
if (
|
|
172
214
|
typeof p.sizeBytesBefore === "number" ||
|
|
173
215
|
typeof p.sizeBytesAfter === "number"
|
|
@@ -181,13 +223,26 @@ export function buildMs365CardText(p: Ms365WritePreview): string {
|
|
|
181
223
|
if (p.deepLink) {
|
|
182
224
|
lines.push(`Link: ${truncate(p.deepLink, 256)}`);
|
|
183
225
|
}
|
|
226
|
+
if (p.changes && p.changes.length > 0) {
|
|
227
|
+
lines.push("");
|
|
228
|
+
lines.push("Changes:");
|
|
229
|
+
for (const c of p.changes.slice(0, 8)) {
|
|
230
|
+
const before = c.before !== undefined ? truncate(c.before, 96) : "(none)";
|
|
231
|
+
const after = c.after !== undefined ? truncate(c.after, 96) : "(cleared)";
|
|
232
|
+
lines.push(`• ${c.field}: ${before} → ${after}`);
|
|
233
|
+
}
|
|
234
|
+
}
|
|
184
235
|
if (p.agentRationale) {
|
|
185
236
|
lines.push("");
|
|
186
237
|
lines.push(`💬 ${truncate(p.agentRationale, 512)}`);
|
|
187
238
|
}
|
|
188
239
|
lines.push("");
|
|
240
|
+
// With a resolved structural diff present the operator is no longer
|
|
241
|
+
// approving a blind write; soften the attestation warning accordingly.
|
|
189
242
|
lines.push(
|
|
190
|
-
|
|
243
|
+
p.changes && p.changes.length > 0
|
|
244
|
+
? "⚠️ Attestation (RFC §8 v1.5): the diff above is derived from live Graph state + the mutation payload. Verify before approving."
|
|
245
|
+
: "⚠️ Weak attestation (RFC §8 v1): operator should click through to verify the actual change before approving. Structural diff coming v1.5.",
|
|
191
246
|
);
|
|
192
247
|
// hardenCardBreaks: labelled field lines (Agent:/Tool:/Item:/Account:/Size:…)
|
|
193
248
|
// would soft-collapse into one blob under the GFM rich renderer; this card is
|
|
@@ -196,8 +251,15 @@ export function buildMs365CardText(p: Ms365WritePreview): string {
|
|
|
196
251
|
}
|
|
197
252
|
|
|
198
253
|
function truncate(s: string, n: number): string {
|
|
199
|
-
|
|
200
|
-
|
|
254
|
+
// Collapse control whitespace (newline / carriage-return / tab) to a single
|
|
255
|
+
// space FIRST — every field on this card is single-line, and this card is a
|
|
256
|
+
// security decision surface. A Graph-sourced value like an event subject of
|
|
257
|
+
// `Team sync\nAccount: attacker@x` would otherwise inject a fake-looking
|
|
258
|
+
// labelled line onto the card (#3267 review Finding 2). Length-truncation
|
|
259
|
+
// alone does not defend against this.
|
|
260
|
+
const oneLine = s.replace(/[\r\n\t]+/g, " ");
|
|
261
|
+
if (oneLine.length <= n) return oneLine;
|
|
262
|
+
return oneLine.slice(0, n - 1) + "…";
|
|
201
263
|
}
|
|
202
264
|
|
|
203
265
|
function humanBytes(bytes: number): string {
|
|
@@ -115,6 +115,18 @@ export function buildSubagentHandbackInbound(opts: {
|
|
|
115
115
|
meta: {
|
|
116
116
|
source: 'subagent_handback',
|
|
117
117
|
outcome: opts.ctx.outcome,
|
|
118
|
+
// #3268 — round-trip the fabricated `ts` through `meta.message_id` so it
|
|
119
|
+
// survives to enqueue. `ev.messageId` at enqueue is parsed from the
|
|
120
|
+
// channel envelope's `message_id` attribute, which is rendered ONLY from
|
|
121
|
+
// `meta.message_id` — the top-level `messageId` field does NOT survive the
|
|
122
|
+
// bridge. Without this, enqueue's `deriveTurnId` returns null → the
|
|
123
|
+
// dead-air pre-turn card's identity-based adoption never matches (the card
|
|
124
|
+
// is orphaned + a false "handback never started" reap message fires on
|
|
125
|
+
// every SUCCESSFUL handback). Mirrors resume-inbound-builder.ts's
|
|
126
|
+
// `message_id: String(ts)` for the identical enqueue-round-trip reason. It
|
|
127
|
+
// is NEVER used as a Telegram reply anchor: `parseSourceMessageId` gates
|
|
128
|
+
// the 13-digit synthetic ts out of the reply-anchor path at enqueue.
|
|
129
|
+
message_id: String(ts),
|
|
118
130
|
// meta.message_thread_id is the model-visible channel attribute
|
|
119
131
|
// (mirrors the real-inbound shape) so the model's reply targets
|
|
120
132
|
// the dispatching topic. Mirrors gateway.ts:10557.
|
|
@@ -38,6 +38,24 @@ import { join } from "node:path";
|
|
|
38
38
|
|
|
39
39
|
export const TURN_ACTIVE_MARKER_FILE = "turn-active.json";
|
|
40
40
|
|
|
41
|
+
/**
|
|
42
|
+
* Absolute ceiling (ms) beyond which a turn-active signal cannot reflect a
|
|
43
|
+
* real in-flight turn. This is the marker sweep's `hardTtlMs` (`gateway.ts`
|
|
44
|
+
* `sweepStaleTurnActiveMarker` callsite). Exported so the `/model` & `/effort`
|
|
45
|
+
* busy-gate cross-checks the in-memory turn atom and the pending-approval hold
|
|
46
|
+
* against the SAME ceiling it sweeps the marker file at (#3262) — instead of
|
|
47
|
+
* the atom leaking past it and reading as a phantom "active turn" on an idle
|
|
48
|
+
* session.
|
|
49
|
+
*/
|
|
50
|
+
export const TURN_ACTIVE_HARD_TTL_MS = 10 * 60_000;
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Idle-sweep threshold (ms): the marker is swept this soon when the caller
|
|
54
|
+
* asserts no turn is in flight. Exported alongside the hard TTL so both
|
|
55
|
+
* bounds have a single source of truth.
|
|
56
|
+
*/
|
|
57
|
+
export const TURN_ACTIVE_IDLE_SWEEP_MS = 60_000;
|
|
58
|
+
|
|
41
59
|
export interface TurnActiveMarker {
|
|
42
60
|
turnKey: string;
|
|
43
61
|
chatId: string;
|
|
@@ -196,3 +214,20 @@ export function readTurnActiveMarkerAgeMs(stateDir: string, now?: number): numbe
|
|
|
196
214
|
return null; // ENOENT / unstattable → not working
|
|
197
215
|
}
|
|
198
216
|
}
|
|
217
|
+
|
|
218
|
+
/**
|
|
219
|
+
* Effective age (ms) of a live turn for the phantom-turn cross-check (#3262):
|
|
220
|
+
* prefer the turn-active liveness marker's mtime age (touched on every
|
|
221
|
+
* tool_use / sub-agent activity, so a genuinely long turn keeps it small),
|
|
222
|
+
* falling back to `now - turnStartedAt` when the marker is absent (e.g. already
|
|
223
|
+
* swept away). Pure so the fallback branch is unit-testable with an injected
|
|
224
|
+
* clock. `markerAgeMs` is the result of `readTurnActiveMarkerAgeMs` (null when
|
|
225
|
+
* the marker is gone).
|
|
226
|
+
*/
|
|
227
|
+
export function effectiveTurnAgeMs(
|
|
228
|
+
markerAgeMs: number | null,
|
|
229
|
+
turnStartedAt: number,
|
|
230
|
+
now: number,
|
|
231
|
+
): number {
|
|
232
|
+
return markerAgeMs ?? now - turnStartedAt;
|
|
233
|
+
}
|
|
@@ -52,6 +52,10 @@ export interface WorkerPinCandidate {
|
|
|
52
52
|
chatId: string
|
|
53
53
|
/** Wall-clock ms the claim was first taken (gateway's pinnedAt registry). */
|
|
54
54
|
pinnedAt: number
|
|
55
|
+
/** The pinned message id. Present for store-only candidates (whose reap is a
|
|
56
|
+
* raw per-message unpin — no in-memory claim to reconcile through). Omitted
|
|
57
|
+
* for in-memory candidates, which reap via reconcileStatusPin off the claim. */
|
|
58
|
+
messageId?: number
|
|
55
59
|
}
|
|
56
60
|
|
|
57
61
|
export interface WorkerPinReap extends WorkerPinCandidate {
|
|
@@ -86,6 +90,56 @@ export type WorkerRegistryStatus = 'terminal' | 'running' | 'unknown'
|
|
|
86
90
|
* for stalled / missing / lookup-error; a DB hiccup must degrade to 'unknown'
|
|
87
91
|
* — kept until the TTL — never to a spurious 'terminal' unpin).
|
|
88
92
|
*/
|
|
93
|
+
/** A durable-store row (status-pins.json) as seen by the reconciling sweep. */
|
|
94
|
+
export interface StoreOrphanRow {
|
|
95
|
+
pinKey: string
|
|
96
|
+
chatId: string
|
|
97
|
+
messageId: number
|
|
98
|
+
/** Pin API call still in-flight (persist-intent-first). Never reaped. */
|
|
99
|
+
pending?: boolean
|
|
100
|
+
/** Time-scoped `tool:` pin. Never a worker orphan; excluded. */
|
|
101
|
+
expiresAt?: number
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Build the extra `wk:` reap candidates that exist ONLY in the durable store
|
|
106
|
+
* (status-pins.json) and NOT in the in-memory claim map — the divergence window
|
|
107
|
+
* where the in-memory claim was lost/never-held but the Telegram pin AND its
|
|
108
|
+
* store row survive. Without this, such an orphan lingers until the next
|
|
109
|
+
* gateway boot (runStatusPinBootCleanup); this lets the PERIODIC sweep recover
|
|
110
|
+
* it too, group-safely (per-message unpin of a bot-tracked row — never an
|
|
111
|
+
* unpin-all, never a human pin).
|
|
112
|
+
*
|
|
113
|
+
* The store persists NO timestamp, so a store-only candidate carries
|
|
114
|
+
* `pinnedAt = now`: only a TERMINAL registry verdict can reap it. It is never
|
|
115
|
+
* TTL-reaped (no trustworthy age to age it out), never touched while its worker
|
|
116
|
+
* is `running`, and it is always a `wk:` row (the documented leak class).
|
|
117
|
+
* `pending` rows (pin API in-flight) and time-scoped `tool:` rows are excluded,
|
|
118
|
+
* as are rows already tracked in memory (the in-memory reaper owns those, with
|
|
119
|
+
* a real `pinnedAt` that can drive the TTL).
|
|
120
|
+
*/
|
|
121
|
+
export function storeOnlyWorkerPinCandidates(args: {
|
|
122
|
+
rows: Iterable<StoreOrphanRow>
|
|
123
|
+
inMemoryPinKeys: ReadonlySet<string>
|
|
124
|
+
now: number
|
|
125
|
+
}): WorkerPinCandidate[] {
|
|
126
|
+
const out: WorkerPinCandidate[] = []
|
|
127
|
+
for (const r of args.rows) {
|
|
128
|
+
if (workerAgentIdOfPinKey(r.pinKey) == null) continue // only wk: rows
|
|
129
|
+
if (r.pending) continue // pin API in-flight — do not race the write
|
|
130
|
+
if (r.expiresAt != null) continue // time-scoped tool pin — not a worker
|
|
131
|
+
if (r.chatId.length === 0) continue // can't unpin without a chat
|
|
132
|
+
if (args.inMemoryPinKeys.has(r.pinKey)) continue // in-memory reaper owns it
|
|
133
|
+
out.push({
|
|
134
|
+
pinKey: r.pinKey,
|
|
135
|
+
chatId: r.chatId,
|
|
136
|
+
pinnedAt: args.now,
|
|
137
|
+
messageId: r.messageId,
|
|
138
|
+
})
|
|
139
|
+
}
|
|
140
|
+
return out
|
|
141
|
+
}
|
|
142
|
+
|
|
89
143
|
export function decideWorkerPinReaps(args: {
|
|
90
144
|
pins: Iterable<WorkerPinCandidate>
|
|
91
145
|
statusOf: (agentId: string) => WorkerRegistryStatus
|
|
@@ -137,6 +137,8 @@ describe('send-gate: SEND_GATE_DEFAULTS (compile-time default PIN)', () => {
|
|
|
137
137
|
perGroupPerMin: 18,
|
|
138
138
|
perGroupBurst: 2,
|
|
139
139
|
editFloorMs: 1500,
|
|
140
|
+
perMessageEditWindowMs: 300_000,
|
|
141
|
+
perMessageEditMaxPerWindow: 150,
|
|
140
142
|
})
|
|
141
143
|
})
|
|
142
144
|
})
|
|
@@ -157,6 +159,8 @@ describe('send-gate: sendGateConfigFromEnv (yaml → env → createSendGate)', (
|
|
|
157
159
|
SWITCHROOM_TG_SEND_GATE_PER_GROUP_PER_MIN: '30',
|
|
158
160
|
SWITCHROOM_TG_SEND_GATE_PER_GROUP_BURST: '4',
|
|
159
161
|
SWITCHROOM_TG_SEND_GATE_EDIT_FLOOR_MS: '2000',
|
|
162
|
+
SWITCHROOM_TG_SEND_GATE_PER_MSG_EDIT_WINDOW_MS: '30000',
|
|
163
|
+
SWITCHROOM_TG_SEND_GATE_PER_MSG_EDIT_MAX: '8',
|
|
160
164
|
}),
|
|
161
165
|
)
|
|
162
166
|
expect(cfg).toEqual({
|
|
@@ -168,9 +172,23 @@ describe('send-gate: sendGateConfigFromEnv (yaml → env → createSendGate)', (
|
|
|
168
172
|
perGroupPerMin: 30,
|
|
169
173
|
perGroupBurst: 4,
|
|
170
174
|
editFloorMs: 2000,
|
|
175
|
+
perMessageEditWindowMs: 30000,
|
|
176
|
+
perMessageEditMaxPerWindow: 8,
|
|
171
177
|
})
|
|
172
178
|
})
|
|
173
179
|
|
|
180
|
+
it('per-message edit budget: MAX accepts 0 (disables the backstop); WINDOW must be positive', () => {
|
|
181
|
+
expect(
|
|
182
|
+
sendGateConfigFromEnv(E({ SWITCHROOM_TG_SEND_GATE_PER_MSG_EDIT_MAX: '0' }))
|
|
183
|
+
.perMessageEditMaxPerWindow,
|
|
184
|
+
).toBe(0)
|
|
185
|
+
// A zero/negative window is malformed → dropped so createSendGate's default applies.
|
|
186
|
+
expect(
|
|
187
|
+
sendGateConfigFromEnv(E({ SWITCHROOM_TG_SEND_GATE_PER_MSG_EDIT_WINDOW_MS: '0' }))
|
|
188
|
+
.perMessageEditWindowMs,
|
|
189
|
+
).toBeUndefined()
|
|
190
|
+
})
|
|
191
|
+
|
|
174
192
|
it('accepts a fractional per-sec rate (rates are not integers)', () => {
|
|
175
193
|
const cfg = sendGateConfigFromEnv(E({ SWITCHROOM_TG_SEND_GATE_PER_CHAT_PER_SEC: '0.5' }))
|
|
176
194
|
expect(cfg.perChatPerSec).toBe(0.5)
|
|
@@ -857,3 +875,123 @@ describe('send-gate: flood windows + boot ramp (L3 / §7 hook)', () => {
|
|
|
857
875
|
expect(c2.filter((c) => c.at === 11_000).length).toBe(10) // full burst, no ramp
|
|
858
876
|
})
|
|
859
877
|
})
|
|
878
|
+
|
|
879
|
+
describe('send-gate: long-horizon per-message edit budget (backstop)', () => {
|
|
880
|
+
it('paces + coalesces a sustained same-message cosmetic edit stream under the rolling cap', async () => {
|
|
881
|
+
const clock = new FakeClock()
|
|
882
|
+
const { calls, fn } = recorder(clock)
|
|
883
|
+
// Tight, fast budget for a deterministic test: 2 cosmetic edits / 5s window,
|
|
884
|
+
// 1s edit floor. A distinct edit every 1s would sail through the floor
|
|
885
|
+
// forever (this is exactly the sub-1/s flood the backstop exists to stop).
|
|
886
|
+
const gate = createSendGate({
|
|
887
|
+
enabled: true,
|
|
888
|
+
clock,
|
|
889
|
+
editFloorMs: 1000,
|
|
890
|
+
perMessageEditWindowMs: 5000,
|
|
891
|
+
perMessageEditMaxPerWindow: 2,
|
|
892
|
+
})
|
|
893
|
+
const msg = 77
|
|
894
|
+
const attempts = 20
|
|
895
|
+
const promises: Promise<unknown>[] = []
|
|
896
|
+
for (let i = 0; i < attempts; i++) {
|
|
897
|
+
promises.push(
|
|
898
|
+
gate.gate(fn(`v${i}`), { messageId: msg, editPayload: `v${i}`, priorityClass: 'cosmetic' }),
|
|
899
|
+
)
|
|
900
|
+
await flush()
|
|
901
|
+
await clock.advance(1000)
|
|
902
|
+
}
|
|
903
|
+
// Drain any final budget-deferred send (a full window is enough).
|
|
904
|
+
await clock.advance(5000)
|
|
905
|
+
await flush()
|
|
906
|
+
await Promise.allSettled(promises)
|
|
907
|
+
|
|
908
|
+
const stats = gate.stats().global
|
|
909
|
+
// The sustained ~1/s stream is paced FAR below the attempt count — bounded
|
|
910
|
+
// by ~2 sends per 5s over the ~25s simulated span, not 20 one-per-second.
|
|
911
|
+
expect(stats.sent).toBeLessThan(attempts)
|
|
912
|
+
expect(stats.sent).toBeLessThanOrEqual(10)
|
|
913
|
+
// The backstop actively deferred at least one edit (not just the 1s floor).
|
|
914
|
+
expect(stats.budgetDeferred).toBeGreaterThan(0)
|
|
915
|
+
// Coalescing collapsed the deferred edits (last-write-wins), so the FINAL
|
|
916
|
+
// send carries the newest payload — no stale body is shown.
|
|
917
|
+
expect(calls[calls.length - 1].label).toBe(`v${attempts - 1}`)
|
|
918
|
+
})
|
|
919
|
+
|
|
920
|
+
it('does NOT throttle distinct message_ids — each message gets its own budget', async () => {
|
|
921
|
+
const clock = new FakeClock()
|
|
922
|
+
const { calls, fn } = recorder(clock)
|
|
923
|
+
const gate = createSendGate({
|
|
924
|
+
enabled: true,
|
|
925
|
+
clock,
|
|
926
|
+
editFloorMs: 1000,
|
|
927
|
+
perMessageEditWindowMs: 5000,
|
|
928
|
+
perMessageEditMaxPerWindow: 1,
|
|
929
|
+
})
|
|
930
|
+
// One cosmetic edit to each of two distinct messages at t=0. A per-message
|
|
931
|
+
// budget of 1 must NOT make message B's first edit wait on message A's.
|
|
932
|
+
const pA = gate.gate(fn('A'), { messageId: 1, editPayload: 'A', priorityClass: 'cosmetic' })
|
|
933
|
+
const pB = gate.gate(fn('B'), { messageId: 2, editPayload: 'B', priorityClass: 'cosmetic' })
|
|
934
|
+
await flush()
|
|
935
|
+
await Promise.all([pA, pB])
|
|
936
|
+
expect(calls.map((c) => c.label).sort()).toEqual(['A', 'B'])
|
|
937
|
+
expect(calls.every((c) => c.at === 0)).toBe(true)
|
|
938
|
+
expect(gate.stats().global.budgetDeferred).toBe(0)
|
|
939
|
+
})
|
|
940
|
+
|
|
941
|
+
it('does NOT throttle non-cosmetic (useful/untagged) edits of the same message', async () => {
|
|
942
|
+
const clock = new FakeClock()
|
|
943
|
+
const { calls, fn } = recorder(clock)
|
|
944
|
+
// Budget so tight it would allow only ONE cosmetic edit ever in the span —
|
|
945
|
+
// yet untagged edits default to `useful` and must bypass the budget entirely
|
|
946
|
+
// (only the 1s floor applies). Legitimate stream/draft edits are untouched.
|
|
947
|
+
const gate = createSendGate({
|
|
948
|
+
enabled: true,
|
|
949
|
+
clock,
|
|
950
|
+
editFloorMs: 1000,
|
|
951
|
+
perMessageEditWindowMs: 100_000,
|
|
952
|
+
perMessageEditMaxPerWindow: 1,
|
|
953
|
+
})
|
|
954
|
+
const msg = 9
|
|
955
|
+
const promises: Promise<unknown>[] = []
|
|
956
|
+
for (let i = 0; i < 5; i++) {
|
|
957
|
+
promises.push(gate.gate(fn(`u${i}`), { messageId: msg, editPayload: `u${i}` }))
|
|
958
|
+
await flush()
|
|
959
|
+
await clock.advance(1000)
|
|
960
|
+
}
|
|
961
|
+
await clock.advance(1000)
|
|
962
|
+
await flush()
|
|
963
|
+
await Promise.allSettled(promises)
|
|
964
|
+
// All five distinct useful edits landed (floor-spaced), none budget-deferred.
|
|
965
|
+
expect(gate.stats().global.sent).toBe(5)
|
|
966
|
+
expect(gate.stats().global.budgetDeferred).toBe(0)
|
|
967
|
+
expect(calls.map((c) => c.label)).toEqual(['u0', 'u1', 'u2', 'u3', 'u4'])
|
|
968
|
+
})
|
|
969
|
+
|
|
970
|
+
it('disables the backstop when perMessageEditMaxPerWindow is 0', async () => {
|
|
971
|
+
const clock = new FakeClock()
|
|
972
|
+
const { calls, fn } = recorder(clock)
|
|
973
|
+
const gate = createSendGate({
|
|
974
|
+
enabled: true,
|
|
975
|
+
clock,
|
|
976
|
+
editFloorMs: 1000,
|
|
977
|
+
perMessageEditMaxPerWindow: 0,
|
|
978
|
+
})
|
|
979
|
+
const msg = 55
|
|
980
|
+
const promises: Promise<unknown>[] = []
|
|
981
|
+
for (let i = 0; i < 6; i++) {
|
|
982
|
+
promises.push(
|
|
983
|
+
gate.gate(fn(`c${i}`), { messageId: msg, editPayload: `c${i}`, priorityClass: 'cosmetic' }),
|
|
984
|
+
)
|
|
985
|
+
await flush()
|
|
986
|
+
await clock.advance(1000)
|
|
987
|
+
}
|
|
988
|
+
await clock.advance(1000)
|
|
989
|
+
await flush()
|
|
990
|
+
await Promise.allSettled(promises)
|
|
991
|
+
// With the budget off, only the 1s floor gates: every floor-spaced distinct
|
|
992
|
+
// edit lands and nothing is budget-deferred.
|
|
993
|
+
expect(gate.stats().global.sent).toBe(6)
|
|
994
|
+
expect(gate.stats().global.budgetDeferred).toBe(0)
|
|
995
|
+
expect(calls.map((c) => c.label)).toEqual(['c0', 'c1', 'c2', 'c3', 'c4', 'c5'])
|
|
996
|
+
})
|
|
997
|
+
})
|
|
@@ -197,6 +197,14 @@ export interface BucketCounters {
|
|
|
197
197
|
* because the open window exceeded the fail-fast ceiling (part3-design §3).
|
|
198
198
|
*/
|
|
199
199
|
failedFast: number
|
|
200
|
+
/**
|
|
201
|
+
* Cosmetic edits DEFERRED at least once by the long-horizon per-message edit
|
|
202
|
+
* budget (rolling-window cap). Counts driver loop iterations that slept on the
|
|
203
|
+
* budget, not distinct messages — a paced runaway stream increments this each
|
|
204
|
+
* time it waits out the window. A non-zero, climbing value means the backstop
|
|
205
|
+
* is actively pacing a sustained same-message edit stream.
|
|
206
|
+
*/
|
|
207
|
+
budgetDeferred: number
|
|
200
208
|
}
|
|
201
209
|
|
|
202
210
|
export interface SendGateStats {
|
|
@@ -256,6 +264,35 @@ export interface SendGateConfig {
|
|
|
256
264
|
perGroupBurst?: number
|
|
257
265
|
/** Minimum ms between edits of the same message_id. Default 1500. */
|
|
258
266
|
editFloorMs?: number
|
|
267
|
+
/**
|
|
268
|
+
* Long-horizon per-message edit budget (rolling window). DEFENSE-IN-DEPTH
|
|
269
|
+
* backstop for a runaway same-message edit stream (finn incident: sustained
|
|
270
|
+
* worker-card clock-only edits → ~88min 429 ban). It caps a SINGLE message to
|
|
271
|
+
* at most `perMessageEditMaxPerWindow` cosmetic edits per
|
|
272
|
+
* `perMessageEditWindowMs`; beyond that the driver DEFERS the next edit until
|
|
273
|
+
* the window slides, and newer edits coalesce (last-write-wins) onto the
|
|
274
|
+
* pending slot in the meantime — so a runaway is paced + collapsed rather than
|
|
275
|
+
* sustained.
|
|
276
|
+
*
|
|
277
|
+
* IMPORTANT — this is a coarse rate ceiling, NOT the primary fix. The gate
|
|
278
|
+
* cannot tell a SUBSTANTIVE cosmetic edit (a new worker step) from an
|
|
279
|
+
* elapsed-clock cosmetic edit; both carry a changed payload. So the primary
|
|
280
|
+
* cure for the clock-churn flood is at the source (`worker-activity-feed.ts`
|
|
281
|
+
* suppresses elapsed-only edits via a substance signature), and THIS backstop
|
|
282
|
+
* must sit ABOVE legitimate substantive cadence so it never throttles real
|
|
283
|
+
* updates. The worker feed's own min-edit interval is 2500ms (≤24 edits/min);
|
|
284
|
+
* the default here — 150 edits / 300_000ms ⟹ a 30 edits/min sustained ceiling
|
|
285
|
+
* — sits above that, so a normal (even continuously-updating) card never
|
|
286
|
+
* binds, while a stream that sustains faster than the feed's throttle for
|
|
287
|
+
* minutes (e.g. a future regression re-introducing per-tick churn, or a
|
|
288
|
+
* lowered floor) is paced back to 30/min. Scoped strictly to `cosmetic`-class
|
|
289
|
+
* edits of the SAME `${chat_id}:${messageId}` (worker-feed, typing,
|
|
290
|
+
* reactions); `useful` / `critical` edits and all non-edit sends are
|
|
291
|
+
* untouched, and distinct messages each get their own budget. Set
|
|
292
|
+
* `perMessageEditMaxPerWindow: 0` to disable the backstop.
|
|
293
|
+
*/
|
|
294
|
+
perMessageEditWindowMs?: number
|
|
295
|
+
perMessageEditMaxPerWindow?: number
|
|
259
296
|
/**
|
|
260
297
|
* Flood windows to re-open at construction (part3-design §7). PR 2 loads
|
|
261
298
|
* these from `flood-wait.json` BEFORE the first outbound call so a restart
|
|
@@ -433,6 +470,14 @@ interface MessageEditState {
|
|
|
433
470
|
running: boolean
|
|
434
471
|
/** Per-message flood suppression window (part3-design §7). */
|
|
435
472
|
suppressedUntilMs: number
|
|
473
|
+
/**
|
|
474
|
+
* Send-START timestamps of recent COSMETIC edits to this message, kept within
|
|
475
|
+
* the long-horizon rolling window (`perMessageEditWindowMs`). Pruned to the
|
|
476
|
+
* window on each driver loop and appended at each send start; its length is
|
|
477
|
+
* the message's edit count over the trailing window and gates the per-message
|
|
478
|
+
* edit budget. Bounded by the budget cap; empty when the backstop is disabled.
|
|
479
|
+
*/
|
|
480
|
+
editWindowTs: number[]
|
|
436
481
|
}
|
|
437
482
|
|
|
438
483
|
function hashPayload(payload: unknown): string {
|
|
@@ -517,6 +562,15 @@ export const SEND_GATE_DEFAULTS = {
|
|
|
517
562
|
perGroupBurst: 2,
|
|
518
563
|
/** Minimum ms between edits of the same message_id. */
|
|
519
564
|
editFloorMs: 1500,
|
|
565
|
+
/** Long-horizon per-message edit budget: rolling window length (ms). */
|
|
566
|
+
perMessageEditWindowMs: 300_000,
|
|
567
|
+
/**
|
|
568
|
+
* Long-horizon per-message edit budget: max cosmetic edits per window.
|
|
569
|
+
* 150 / 300s ⟹ a 30 edits/min sustained ceiling — above the worker feed's
|
|
570
|
+
* own 24/min cadence so legitimate substantive updates never bind (see the
|
|
571
|
+
* SendGateConfig field doc).
|
|
572
|
+
*/
|
|
573
|
+
perMessageEditMaxPerWindow: 150,
|
|
520
574
|
} as const
|
|
521
575
|
|
|
522
576
|
export function createSendGate(config: SendGateConfig): SendGate {
|
|
@@ -529,6 +583,14 @@ export function createSendGate(config: SendGateConfig): SendGate {
|
|
|
529
583
|
const perGroupPerMin = config.perGroupPerMin ?? SEND_GATE_DEFAULTS.perGroupPerMin
|
|
530
584
|
const perGroupBurst = config.perGroupBurst ?? SEND_GATE_DEFAULTS.perGroupBurst
|
|
531
585
|
const editFloorMs = config.editFloorMs ?? SEND_GATE_DEFAULTS.editFloorMs
|
|
586
|
+
const perMessageEditWindowMs = Math.max(
|
|
587
|
+
1,
|
|
588
|
+
Math.floor(config.perMessageEditWindowMs ?? SEND_GATE_DEFAULTS.perMessageEditWindowMs),
|
|
589
|
+
)
|
|
590
|
+
const perMessageEditMaxPerWindow = Math.max(
|
|
591
|
+
0,
|
|
592
|
+
Math.floor(config.perMessageEditMaxPerWindow ?? SEND_GATE_DEFAULTS.perMessageEditMaxPerWindow),
|
|
593
|
+
)
|
|
532
594
|
const messageStateTtlMs = config.messageStateTtlMs ?? 60_000
|
|
533
595
|
const maxMessageStates = config.maxMessageStates ?? 5_000
|
|
534
596
|
const usefulTtlMs = config.usefulTtlMs ?? 120_000
|
|
@@ -546,6 +608,7 @@ export function createSendGate(config: SendGateConfig): SendGate {
|
|
|
546
608
|
shed: 0,
|
|
547
609
|
expired: 0,
|
|
548
610
|
failedFast: 0,
|
|
611
|
+
budgetDeferred: 0,
|
|
549
612
|
}
|
|
550
613
|
|
|
551
614
|
const bootStart = clock.now()
|
|
@@ -605,6 +668,7 @@ export function createSendGate(config: SendGateConfig): SendGate {
|
|
|
605
668
|
pending: null,
|
|
606
669
|
running: false,
|
|
607
670
|
suppressedUntilMs: 0,
|
|
671
|
+
editWindowTs: [],
|
|
608
672
|
}
|
|
609
673
|
perMessage.set(key, state)
|
|
610
674
|
}
|
|
@@ -902,7 +966,28 @@ export function createSendGate(config: SendGateConfig): SendGate {
|
|
|
902
966
|
try {
|
|
903
967
|
while (state.pending) {
|
|
904
968
|
const now = clock.now()
|
|
905
|
-
|
|
969
|
+
let readyAt = Math.max(state.lastSentMs + editFloorMs, state.suppressedUntilMs)
|
|
970
|
+
// Long-horizon per-message edit budget (backstop). Scoped to cosmetic
|
|
971
|
+
// edits of THIS message: prune the rolling window, and if it is already
|
|
972
|
+
// full, defer until the oldest in-window send ages out. Reading the
|
|
973
|
+
// pending edit's (possibly upgraded) class here means a critical edit
|
|
974
|
+
// that coalesced onto a cosmetic driver is NOT budget-capped — it must
|
|
975
|
+
// never block unbounded (part3-design §3). Newer edits keep coalescing
|
|
976
|
+
// into `state.pending` while we sleep, so pacing preserves last-write-
|
|
977
|
+
// wins rather than sending stale bodies.
|
|
978
|
+
if (perMessageEditMaxPerWindow > 0 && state.pending.priorityClass === 'cosmetic') {
|
|
979
|
+
const windowStart = now - perMessageEditWindowMs
|
|
980
|
+
while (state.editWindowTs.length > 0 && state.editWindowTs[0] <= windowStart) {
|
|
981
|
+
state.editWindowTs.shift()
|
|
982
|
+
}
|
|
983
|
+
if (state.editWindowTs.length >= perMessageEditMaxPerWindow) {
|
|
984
|
+
const budgetReadyAt = state.editWindowTs[0] + perMessageEditWindowMs
|
|
985
|
+
if (budgetReadyAt > readyAt) {
|
|
986
|
+
readyAt = budgetReadyAt
|
|
987
|
+
counters.budgetDeferred++
|
|
988
|
+
}
|
|
989
|
+
}
|
|
990
|
+
}
|
|
906
991
|
const waitMs = readyAt - now
|
|
907
992
|
if (waitMs > 0) {
|
|
908
993
|
// Still inside the floor / an open window — sleep, then re-read
|
|
@@ -958,6 +1043,16 @@ export function createSendGate(config: SendGateConfig): SendGate {
|
|
|
958
1043
|
// Reserve the send-start time BEFORE awaiting the network so the floor
|
|
959
1044
|
// is measured from send start (matches the per-message serialization).
|
|
960
1045
|
state.lastSentMs = clock.now()
|
|
1046
|
+
// Record this send against the long-horizon budget when it is a cosmetic
|
|
1047
|
+
// edit (the only class the budget gates). Recorded at send START so the
|
|
1048
|
+
// rolling window measures dispatch cadence, consistent with the floor.
|
|
1049
|
+
if (perMessageEditMaxPerWindow > 0 && p.priorityClass === 'cosmetic') {
|
|
1050
|
+
state.editWindowTs.push(state.lastSentMs)
|
|
1051
|
+
// Bound the array against pathological inputs (it is naturally ≈ the
|
|
1052
|
+
// budget cap since we prune to the window each loop).
|
|
1053
|
+
const overflow = state.editWindowTs.length - (perMessageEditMaxPerWindow + 1)
|
|
1054
|
+
if (overflow > 0) state.editWindowTs.splice(0, overflow)
|
|
1055
|
+
}
|
|
961
1056
|
try {
|
|
962
1057
|
// N3 (liveness): this awaits `p.fn()` with no watchdog. A `fn` that
|
|
963
1058
|
// NEVER settles would keep `state.running` true forever, making the
|
|
@@ -1222,6 +1317,8 @@ export type SendGateEnvConfig = Pick<SendGateConfig, 'enabled'> &
|
|
|
1222
1317
|
| 'perGroupPerMin'
|
|
1223
1318
|
| 'perGroupBurst'
|
|
1224
1319
|
| 'editFloorMs'
|
|
1320
|
+
| 'perMessageEditWindowMs'
|
|
1321
|
+
| 'perMessageEditMaxPerWindow'
|
|
1225
1322
|
| 'conservativeGlobalFloodScope'
|
|
1226
1323
|
>
|
|
1227
1324
|
>
|
|
@@ -1270,6 +1367,12 @@ export function sendGateConfigFromEnv(
|
|
|
1270
1367
|
if (perGroupBurst !== undefined) out.perGroupBurst = perGroupBurst
|
|
1271
1368
|
const editFloorMs = parseNonNegativeInt(env.SWITCHROOM_TG_SEND_GATE_EDIT_FLOOR_MS)
|
|
1272
1369
|
if (editFloorMs !== undefined) out.editFloorMs = editFloorMs
|
|
1370
|
+
// Long-horizon per-message edit budget (flood-ban backstop). Window ≥1ms is
|
|
1371
|
+
// enforced in createSendGate; MAX_PER_WINDOW=0 disables the backstop.
|
|
1372
|
+
const perMsgWindowMs = parsePositiveInt(env.SWITCHROOM_TG_SEND_GATE_PER_MSG_EDIT_WINDOW_MS)
|
|
1373
|
+
if (perMsgWindowMs !== undefined) out.perMessageEditWindowMs = perMsgWindowMs
|
|
1374
|
+
const perMsgMax = parseNonNegativeInt(env.SWITCHROOM_TG_SEND_GATE_PER_MSG_EDIT_MAX)
|
|
1375
|
+
if (perMsgMax !== undefined) out.perMessageEditMaxPerWindow = perMsgMax
|
|
1273
1376
|
// #3111 break-glass: restore the pre-#3111 always-open-global flood posture.
|
|
1274
1377
|
// Unset ⇒ scope-precise default (global opened only on genuinely global 429s).
|
|
1275
1378
|
const conservativeGlobal = parseBoolFlag(env.SWITCHROOM_TG_SEND_GATE_CONSERVATIVE_GLOBAL)
|
|
@@ -10,8 +10,14 @@
|
|
|
10
10
|
* resume-400 signature) from "feed opened + finalized".
|
|
11
11
|
*
|
|
12
12
|
* Load-bearing constraints:
|
|
13
|
-
* 1. `activityEverOpened = true` is set
|
|
14
|
-
* send-message success site in
|
|
13
|
+
* 1. `activityEverOpened = true` is set only at legitimate feed-OPEN signal
|
|
14
|
+
* sites in gateway.ts — the send-message success site in
|
|
15
|
+
* drainActivitySummary, AND the sub-agent-handback pre-turn ADOPTION site
|
|
16
|
+
* (#3268): an adopted turn inherits an already-open pre-turn card via a
|
|
17
|
+
* seeded `activityMessageId`, so it only ever EDITs the feed (never hits
|
|
18
|
+
* the open branch), and must stamp the flag itself so the turn-end
|
|
19
|
+
* DEGRADED check doesn't false-flag it as "feed never opened". Both are
|
|
20
|
+
* set-TRUE (never a reset), preserving the sticky-true invariant.
|
|
15
21
|
* 2. `turn.activityEverOpened = false` NEVER appears in gateway.ts (it is only
|
|
16
22
|
* initialised to `false` in the turn-initialiser object literal, never reset
|
|
17
23
|
* via a standalone assignment).
|
|
@@ -28,9 +34,13 @@ const gatewaySrc = readFileSync(
|
|
|
28
34
|
)
|
|
29
35
|
|
|
30
36
|
describe('M-2: activityEverOpened sticky-true invariant', () => {
|
|
31
|
-
it('activityEverOpened = true appears
|
|
37
|
+
it('activityEverOpened = true appears only at the two feed-OPEN signal sites', () => {
|
|
38
|
+
// Site 1: drainActivitySummary send-message success. Site 2: the #3268
|
|
39
|
+
// handback pre-turn ADOPTION seed (an adopted turn only edits, so it stamps
|
|
40
|
+
// the flag itself). Both are set-TRUE; the sticky invariant (no reset to
|
|
41
|
+
// false) is enforced by the next test.
|
|
32
42
|
const setTrueMatches = [...gatewaySrc.matchAll(/activityEverOpened\s*=\s*true/g)]
|
|
33
|
-
expect(setTrueMatches).toHaveLength(
|
|
43
|
+
expect(setTrueMatches).toHaveLength(2)
|
|
34
44
|
})
|
|
35
45
|
|
|
36
46
|
it('turn.activityEverOpened = false never appears (no standalone reset)', () => {
|