switchroom 0.18.27 → 0.18.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/handoff-briefing.sh +8 -1
- package/dist/auth-broker/index.js +0 -57
- package/dist/cli/switchroom.js +501 -497
- package/dist/host-control/main.js +1 -58
- package/dist/vault/approvals/kernel-server.js +0 -57
- package/dist/vault/broker/server.js +0 -57
- package/package.json +1 -1
- package/profiles/_base/start.sh.hbs +37 -19
- package/telegram-plugin/dist/gateway/gateway.js +655 -587
- package/telegram-plugin/gateway/backstop-delivery.ts +272 -0
- package/telegram-plugin/gateway/forward-origin.ts +9 -1
- package/telegram-plugin/gateway/gateway.ts +511 -397
- package/telegram-plugin/gateway/model-command.ts +227 -602
- package/telegram-plugin/gateway/turn-record-status.ts +45 -0
- package/telegram-plugin/gateway/worker-pin-reaper.ts +54 -0
- package/telegram-plugin/history.ts +153 -23
- package/telegram-plugin/shared/local-time.ts +56 -0
- package/telegram-plugin/tests/backstop-delivery.test.ts +250 -0
- package/telegram-plugin/tests/forward-origin.test.ts +30 -3
- package/telegram-plugin/tests/gateway-session-model-relaunch.test.ts +86 -59
- package/telegram-plugin/tests/history.test.ts +88 -0
- package/telegram-plugin/tests/local-time.test.ts +68 -0
- package/telegram-plugin/tests/model-command.test.ts +317 -1535
- package/telegram-plugin/tests/turn-flush-safety.test.ts +34 -0
- package/telegram-plugin/tests/worker-feed-migration-eviction.test.ts +140 -0
- package/telegram-plugin/tests/worker-pin-reaper.test.ts +78 -0
- package/telegram-plugin/tier-downgrade.ts +4 -3
- package/telegram-plugin/turn-flush-safety.ts +25 -1
- package/telegram-plugin/worker-activity-feed.ts +78 -5
- package/vendor/hindsight-memory/scripts/lib/content.py +40 -6
- package/vendor/hindsight-memory/tests/test_content.py +28 -7
|
@@ -629,6 +629,40 @@ describe('selectFlushDeliveryText — deliver the terminal answer, strip only na
|
|
|
629
629
|
expect(selectFlushDeliveryText([])).toBe('')
|
|
630
630
|
})
|
|
631
631
|
|
|
632
|
+
// #3276 guard 8 — the NARRATION_OPENER regex alone misses progress narration
|
|
633
|
+
// that does NOT open with "Let me…/I'll…" but trails off into an ellipsis or
|
|
634
|
+
// colon ("Checking now…"). Pre-fix, `isNarrationBlock` returned false for it,
|
|
635
|
+
// so the whole `narration\n\nanswer` blob was delivered. FAILS pre-fix.
|
|
636
|
+
it('strips a non-opener progress narration ("Checking now…") that precedes the answer', () => {
|
|
637
|
+
const out = selectFlushDeliveryText(['Checking now…', 'The build is green.'])
|
|
638
|
+
expect(out).toBe('The build is green.')
|
|
639
|
+
expect(out).not.toContain('Checking now')
|
|
640
|
+
})
|
|
641
|
+
|
|
642
|
+
it('strips a colon-terminated progress narration ("Pulling the numbers:")', () => {
|
|
643
|
+
const out = selectFlushDeliveryText(['Pulling the numbers:', 'Revenue was 4.2M.'])
|
|
644
|
+
expect(out).toBe('Revenue was 4.2M.')
|
|
645
|
+
})
|
|
646
|
+
|
|
647
|
+
it('still delivers a SHORT terminal answer after progress narration (no min-char re-gate)', () => {
|
|
648
|
+
// "yes, done" is well under FLUSH_SUBSTANTIVE_MIN_CHARS and MUST still deliver.
|
|
649
|
+
const out = selectFlushDeliveryText(['Looking into that...', 'yes, done'])
|
|
650
|
+
expect(out).toBe('yes, done')
|
|
651
|
+
})
|
|
652
|
+
|
|
653
|
+
// #3276 finding 7 — the narration heuristic only ever strips PRECEDING blocks;
|
|
654
|
+
// the terminal answer block is always preserved. A colon-terminated line that
|
|
655
|
+
// IS the whole answer (e.g. a lead-in the model never continued) must NOT be
|
|
656
|
+
// dropped, whether it stands alone or is the terminal block after narration.
|
|
657
|
+
it('never drops a colon-terminated line that IS the whole answer (single block)', () => {
|
|
658
|
+
expect(selectFlushDeliveryText(['Here are the results:'])).toBe('Here are the results:')
|
|
659
|
+
})
|
|
660
|
+
|
|
661
|
+
it('keeps a colon-terminated TERMINAL block (it is the answer, not narration)', () => {
|
|
662
|
+
const out = selectFlushDeliveryText(['Checking now…', 'Here are the results:'])
|
|
663
|
+
expect(out).toBe('Here are the results:')
|
|
664
|
+
})
|
|
665
|
+
|
|
632
666
|
it('decideTurnFlush delivers the narrowed answer, not the whole blob', () => {
|
|
633
667
|
const decision = decideTurnFlush({
|
|
634
668
|
chatId: 'chat1',
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Regression guard for the feed-registry churn bug (FIX 2): a worker row whose
|
|
3
|
+
* `agentId` migrated feeds (a fresh `update()` under a different chat/thread)
|
|
4
|
+
* used to leave the OLD group's `workers` map holding an orphan row that the
|
|
5
|
+
* `agentIndex` no longer pointed at. `groupOfAgent(agentId)` could then never
|
|
6
|
+
* resolve it, so the heartbeat TTL sweep's `terminateWorker(agentId)` no-op'd
|
|
7
|
+
* and the row churned every tick forever — the self-contradictory
|
|
8
|
+
* `worker-feed: TTL reap … no update in 0s (>= 3000s); force-terminating
|
|
9
|
+
* leaked row` log, repeating ~every 6s.
|
|
10
|
+
*
|
|
11
|
+
* Two guarantees are asserted as OUTCOMES:
|
|
12
|
+
* 1. Migration eviction (root cause): after the same agentId updates a
|
|
13
|
+
* different feed, the old group's row is gone immediately — total tracked
|
|
14
|
+
* rows never double-count the migrated worker.
|
|
15
|
+
* 2. Sweep force-eviction is idempotent: once every worker has ended, the TTL
|
|
16
|
+
* sweep drives `size` to 0 and it STAYS 0 across repeated heartbeat ticks
|
|
17
|
+
* (no churning row that re-reaps forever).
|
|
18
|
+
*/
|
|
19
|
+
import { describe, it, expect } from 'vitest'
|
|
20
|
+
import {
|
|
21
|
+
createWorkerActivityFeed,
|
|
22
|
+
type BotApiForWorkerFeed,
|
|
23
|
+
type WorkerActivityView,
|
|
24
|
+
} from '../worker-activity-feed.js'
|
|
25
|
+
|
|
26
|
+
function view(desc: string, elapsedMs: number): WorkerActivityView {
|
|
27
|
+
return { description: desc, lastTool: null, toolCount: 1, latestSummary: 'step', elapsedMs, state: 'running' }
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
type IndexControl = { repointAgentIndex: (agentId: string, feedKey: string | null) => void }
|
|
31
|
+
|
|
32
|
+
function makeFeed(nowRef: { t: number }) {
|
|
33
|
+
const pins: { messageId: number | null }[] = []
|
|
34
|
+
let seq = 900
|
|
35
|
+
let control: IndexControl | null = null
|
|
36
|
+
const bot: BotApiForWorkerFeed = {
|
|
37
|
+
sendMessage: async () => ({ message_id: seq++ }),
|
|
38
|
+
editMessageText: async () => true,
|
|
39
|
+
}
|
|
40
|
+
const feed = createWorkerActivityFeed({
|
|
41
|
+
bot,
|
|
42
|
+
now: () => nowRef.t,
|
|
43
|
+
minEditIntervalMs: 0,
|
|
44
|
+
heartbeatTickMs: 1000,
|
|
45
|
+
firstPaintMinMs: 0,
|
|
46
|
+
staleWorkerTtlMs: 3_000_000, // 3000s — matches the observed churn log
|
|
47
|
+
setInterval: () => 0,
|
|
48
|
+
clearInterval: () => {},
|
|
49
|
+
reconcilePin: ({ messageId }) => pins.push({ messageId }),
|
|
50
|
+
exposeTestControls: (c) => {
|
|
51
|
+
control = c
|
|
52
|
+
},
|
|
53
|
+
})
|
|
54
|
+
return { feed, pins, control: () => control as IndexControl }
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
async function drain(): Promise<void> {
|
|
58
|
+
for (let i = 0; i < 12; i++) await new Promise((r) => setImmediate(r))
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
describe('worker-feed migration eviction + churn-free sweep (FIX 2)', () => {
|
|
62
|
+
it('evicts the old-feed orphan when an agentId migrates feeds — no double-count', async () => {
|
|
63
|
+
const nowRef = { t: 1_000_000 }
|
|
64
|
+
const { feed } = makeFeed(nowRef)
|
|
65
|
+
|
|
66
|
+
// Worker registers in feed A (chat -100).
|
|
67
|
+
await feed.update('agent-1', '-100', view('task', 0))
|
|
68
|
+
await drain()
|
|
69
|
+
expect(feed.size).toBe(1)
|
|
70
|
+
|
|
71
|
+
// SAME agentId now updates a DIFFERENT feed (chat -200) — a migration. The
|
|
72
|
+
// old feed-A row must be evicted, not orphaned. Total rows stay 1.
|
|
73
|
+
nowRef.t += 1000
|
|
74
|
+
await feed.update('agent-1', '-200', view('task', 1000))
|
|
75
|
+
await drain()
|
|
76
|
+
expect(feed.size).toBe(1)
|
|
77
|
+
})
|
|
78
|
+
|
|
79
|
+
it('TTL sweep drives size to 0 and stays 0 across repeated ticks (no churn)', async () => {
|
|
80
|
+
const nowRef = { t: 2_000_000 }
|
|
81
|
+
const { feed } = makeFeed(nowRef)
|
|
82
|
+
|
|
83
|
+
await feed.update('agent-1', '-100', view('task', 0))
|
|
84
|
+
await drain()
|
|
85
|
+
// Migrate to a second feed to exercise the exact desync path that leaked.
|
|
86
|
+
nowRef.t += 1000
|
|
87
|
+
await feed.update('agent-1', '-200', view('task', 1000))
|
|
88
|
+
await drain()
|
|
89
|
+
expect(feed.size).toBe(1)
|
|
90
|
+
|
|
91
|
+
// Advance past the silence TTL so the sweep force-terminates the last row.
|
|
92
|
+
nowRef.t += 3_000_001
|
|
93
|
+
feed.heartbeatTick()
|
|
94
|
+
await drain()
|
|
95
|
+
expect(feed.size).toBe(0)
|
|
96
|
+
|
|
97
|
+
// Idempotency: further ticks must not resurrect / re-reap a churning row.
|
|
98
|
+
for (let i = 0; i < 3; i++) {
|
|
99
|
+
nowRef.t += 3_000_001
|
|
100
|
+
feed.heartbeatTick()
|
|
101
|
+
await drain()
|
|
102
|
+
}
|
|
103
|
+
expect(feed.size).toBe(0)
|
|
104
|
+
})
|
|
105
|
+
|
|
106
|
+
it('TTL sweep FORCE-EVICTS a leaked row whose agentIndex has desynced from its group', async () => {
|
|
107
|
+
// Reproduce the exact desync the migration-eviction fix now prevents any
|
|
108
|
+
// public sequence from producing: a live row physically in group A while
|
|
109
|
+
// the agentIndex points somewhere else, so `groupOfAgent(agentId) !== g`.
|
|
110
|
+
// This drives the heartbeat sweep's FORCE-EVICT branch specifically — the
|
|
111
|
+
// one that removes the orphan directly from the group it lives in instead
|
|
112
|
+
// of no-op'ing through `terminateWorker` (which would churn forever).
|
|
113
|
+
const nowRef = { t: 3_000_000 }
|
|
114
|
+
const { feed, control } = makeFeed(nowRef)
|
|
115
|
+
|
|
116
|
+
await feed.update('agent-1', '-100', view('task', 0))
|
|
117
|
+
await drain()
|
|
118
|
+
expect(feed.size).toBe(1)
|
|
119
|
+
|
|
120
|
+
// Corrupt the index so it no longer resolves to the group holding the row.
|
|
121
|
+
// `terminateWorker('agent-1')` would now look up a non-existent group and
|
|
122
|
+
// no-op — the leak the force-evict branch exists to catch.
|
|
123
|
+
control().repointAgentIndex('agent-1', 'bogus-feed-key')
|
|
124
|
+
|
|
125
|
+
// Past the silence TTL: the sweep must force-evict the row from its real
|
|
126
|
+
// group, NOT rely on terminateWorker.
|
|
127
|
+
nowRef.t += 3_000_001
|
|
128
|
+
feed.heartbeatTick()
|
|
129
|
+
await drain()
|
|
130
|
+
expect(feed.size).toBe(0)
|
|
131
|
+
|
|
132
|
+
// And it stays gone — no churning row re-reaped every tick.
|
|
133
|
+
for (let i = 0; i < 3; i++) {
|
|
134
|
+
nowRef.t += 3_000_001
|
|
135
|
+
feed.heartbeatTick()
|
|
136
|
+
await drain()
|
|
137
|
+
}
|
|
138
|
+
expect(feed.size).toBe(0)
|
|
139
|
+
})
|
|
140
|
+
})
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { describe, it, expect } from "vitest";
|
|
2
2
|
import {
|
|
3
3
|
decideWorkerPinReaps,
|
|
4
|
+
storeOnlyWorkerPinCandidates,
|
|
4
5
|
workerAgentIdOfPinKey,
|
|
5
6
|
WORKER_PIN_TTL_MS_DEFAULT,
|
|
6
7
|
type WorkerPinCandidate,
|
|
@@ -130,3 +131,80 @@ describe("decideWorkerPinReaps (#3001 mid-session wk: sweep)", () => {
|
|
|
130
131
|
expect(reaps).toHaveLength(0);
|
|
131
132
|
});
|
|
132
133
|
});
|
|
134
|
+
|
|
135
|
+
describe("storeOnlyWorkerPinCandidates (#3001 durable group net)", () => {
|
|
136
|
+
const inMem = (keys: string[] = []) => new Set(keys);
|
|
137
|
+
|
|
138
|
+
it("promotes a wk: store row that has NO in-memory claim into a reap candidate (group chat)", () => {
|
|
139
|
+
// A GROUP chat id is negative; the reconciling sweep must recover a
|
|
140
|
+
// bot-tracked orphan there via a per-message unpin (never an unpin-all).
|
|
141
|
+
const cands = storeOnlyWorkerPinCandidates({
|
|
142
|
+
rows: [{ pinKey: "wk:group:-100:5", chatId: "-100", messageId: 42 }],
|
|
143
|
+
inMemoryPinKeys: inMem(),
|
|
144
|
+
now: NOW,
|
|
145
|
+
});
|
|
146
|
+
expect(cands).toHaveLength(1);
|
|
147
|
+
expect(cands[0].pinKey).toBe("wk:group:-100:5");
|
|
148
|
+
expect(cands[0].chatId).toBe("-100");
|
|
149
|
+
// messageId is carried so the gateway can per-message unpin (group-safe).
|
|
150
|
+
expect(cands[0].messageId).toBe(42);
|
|
151
|
+
// A terminal registry verdict reaps it now, however "young" (pinnedAt=now).
|
|
152
|
+
const reaps = decideWorkerPinReaps({
|
|
153
|
+
pins: cands,
|
|
154
|
+
statusOf: () => "terminal" as const,
|
|
155
|
+
ttlMs: TTL,
|
|
156
|
+
now: NOW,
|
|
157
|
+
});
|
|
158
|
+
expect(reaps.map((r) => r.pinKey)).toEqual(["wk:group:-100:5"]);
|
|
159
|
+
});
|
|
160
|
+
|
|
161
|
+
it("NEVER promotes an untracked/human pin — only wk: rows qualify (fg:/tool:/banner: excluded)", () => {
|
|
162
|
+
const cands = storeOnlyWorkerPinCandidates({
|
|
163
|
+
rows: [
|
|
164
|
+
{ pinKey: "fg:-100:9", chatId: "-100", messageId: 1 },
|
|
165
|
+
{ pinKey: "tool:-100:9", chatId: "-100", messageId: 2, expiresAt: NOW + 1 },
|
|
166
|
+
{ pinKey: "banner:owner", chatId: "-100", messageId: 3 },
|
|
167
|
+
],
|
|
168
|
+
inMemoryPinKeys: inMem(),
|
|
169
|
+
now: NOW,
|
|
170
|
+
});
|
|
171
|
+
expect(cands).toHaveLength(0);
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
it("skips rows the in-memory reaper already owns (dedup by pinKey)", () => {
|
|
175
|
+
const cands = storeOnlyWorkerPinCandidates({
|
|
176
|
+
rows: [{ pinKey: "wk:a", chatId: "-100", messageId: 7 }],
|
|
177
|
+
inMemoryPinKeys: inMem(["wk:a"]),
|
|
178
|
+
now: NOW,
|
|
179
|
+
});
|
|
180
|
+
expect(cands).toHaveLength(0);
|
|
181
|
+
});
|
|
182
|
+
|
|
183
|
+
it("skips pending (pin API in-flight) and time-scoped tool rows and empty-chat rows", () => {
|
|
184
|
+
const cands = storeOnlyWorkerPinCandidates({
|
|
185
|
+
rows: [
|
|
186
|
+
{ pinKey: "wk:pending", chatId: "-100", messageId: 1, pending: true },
|
|
187
|
+
{ pinKey: "wk:tool", chatId: "-100", messageId: 2, expiresAt: NOW + 1000 },
|
|
188
|
+
{ pinKey: "wk:nochat", chatId: "", messageId: 3 },
|
|
189
|
+
],
|
|
190
|
+
inMemoryPinKeys: inMem(),
|
|
191
|
+
now: NOW,
|
|
192
|
+
});
|
|
193
|
+
expect(cands).toHaveLength(0);
|
|
194
|
+
});
|
|
195
|
+
|
|
196
|
+
it("a store-orphan with a live 'running' worker is kept (never reaped mid-run)", () => {
|
|
197
|
+
const cands = storeOnlyWorkerPinCandidates({
|
|
198
|
+
rows: [{ pinKey: "wk:live", chatId: "-100", messageId: 5 }],
|
|
199
|
+
inMemoryPinKeys: inMem(),
|
|
200
|
+
now: NOW,
|
|
201
|
+
});
|
|
202
|
+
const reaps = decideWorkerPinReaps({
|
|
203
|
+
pins: cands,
|
|
204
|
+
statusOf: () => "running" as const,
|
|
205
|
+
ttlMs: TTL,
|
|
206
|
+
now: NOW,
|
|
207
|
+
});
|
|
208
|
+
expect(reaps).toHaveLength(0);
|
|
209
|
+
});
|
|
210
|
+
});
|
|
@@ -20,9 +20,10 @@
|
|
|
20
20
|
* account-swap, never by a downgrade.
|
|
21
21
|
*
|
|
22
22
|
* There is NO automatic return to the premium model. The `/model` override is
|
|
23
|
-
* SESSION-SCOPED
|
|
24
|
-
*
|
|
25
|
-
* downgrade SIGTERM
|
|
23
|
+
* SESSION-SCOPED: rev 5 (reference/rfcs/session-model-stickiness.md §0.05) makes
|
|
24
|
+
* every switch a consume-once `.session-model` carrier relaunch, so the override
|
|
25
|
+
* dies on the downgrade SIGTERM (the carrier was consumed on its own apply-boot).
|
|
26
|
+
* The downgrade writes a consume-once `.session-model`
|
|
26
27
|
* carrier for the CONFIGURED DEFAULT: start.sh applies+deletes it on the resume
|
|
27
28
|
* boot, and every subsequent restart boots the configured default too. The
|
|
28
29
|
* premium model is never restored on its own — the user must re-issue
|
|
@@ -193,8 +193,32 @@ export function selectFlushDeliveryText(blocks: string[]): string {
|
|
|
193
193
|
const NARRATION_OPENER =
|
|
194
194
|
/^(let me\b|lemme\b|i'?ll\b|i will\b|i am going to\b|i'?m going to\b|i'?m about to\b|going to\b|first,?\s+(?:let me|i'?ll|i will)\b|now,?\s+(?:let me|i'?ll|i will)\b|next,?\s+(?:let me|i'?ll|i will)\b|let'?s\b)/i
|
|
195
195
|
|
|
196
|
+
/**
|
|
197
|
+
* #3276 guard 8 — the `NARRATION_OPENER` regex alone misses common progress
|
|
198
|
+
* narration that opens with a gerund/present-continuous verb and trails off
|
|
199
|
+
* into an ellipsis or colon: "Checking now…", "Pulling the numbers:",
|
|
200
|
+
* "Looking into that…". Relying on the opener regex leaked those blocks into
|
|
201
|
+
* the delivered answer. This recognises a NON-terminal narration line
|
|
202
|
+
* deterministically, WITHOUT re-gating on a min-char floor (a short real
|
|
203
|
+
* answer like "Yes, done." must still deliver): a single short line that ends
|
|
204
|
+
* with an ellipsis or a colon is progress narration, not the terminal answer.
|
|
205
|
+
*
|
|
206
|
+
* Kept conservative on purpose — only a SINGLE-line block (no internal
|
|
207
|
+
* paragraph) under the substantive floor, ending in `…` / `...` / `:`, so a
|
|
208
|
+
* genuine multi-paragraph answer that happens to end a paragraph with a colon
|
|
209
|
+
* is never mistaken for narration.
|
|
210
|
+
*/
|
|
211
|
+
const NARRATION_TRAILER = /(?:\.{3}|…|:)\s*$/
|
|
212
|
+
|
|
213
|
+
function isTrailingNarrationLine(block: string): boolean {
|
|
214
|
+
const t = block.trim()
|
|
215
|
+
if (t.length === 0 || t.length >= FLUSH_SUBSTANTIVE_MIN_CHARS) return false
|
|
216
|
+
if (t.includes('\n')) return false
|
|
217
|
+
return NARRATION_TRAILER.test(t)
|
|
218
|
+
}
|
|
219
|
+
|
|
196
220
|
function isNarrationBlock(block: string): boolean {
|
|
197
|
-
return NARRATION_OPENER.test(block.trimStart())
|
|
221
|
+
return NARRATION_OPENER.test(block.trimStart()) || isTrailingNarrationLine(block)
|
|
198
222
|
}
|
|
199
223
|
|
|
200
224
|
export type FlushDecision =
|
|
@@ -393,6 +393,21 @@ export interface WorkerActivityFeedOpts {
|
|
|
393
393
|
threadId?: number
|
|
394
394
|
messageId: number | null
|
|
395
395
|
}) => void
|
|
396
|
+
/**
|
|
397
|
+
* TEST-ONLY (never set in production). Invoked once at construction with a
|
|
398
|
+
* narrow control that repoints the internal `agentId → feedKey` index WITHOUT
|
|
399
|
+
* moving the worker's row. Its sole purpose is to reproduce the
|
|
400
|
+
* `agentIndex`/`workers` DESYNC that the migration-eviction root-cause fix now
|
|
401
|
+
* prevents any public API sequence from producing — so the heartbeat sweep's
|
|
402
|
+
* force-evict backstop (the branch that removes a leaked row when
|
|
403
|
+
* `groupOfAgent(agentId)` no longer resolves to the group physically holding
|
|
404
|
+
* it) can be covered by a deterministic test. A no-op when unset.
|
|
405
|
+
*/
|
|
406
|
+
exposeTestControls?: (controls: {
|
|
407
|
+
/** Set the internal index entry for `agentId` to `feedKey` (or delete it
|
|
408
|
+
* when `feedKey` is null) without touching any group's `workers` map. */
|
|
409
|
+
repointAgentIndex: (agentId: string, feedKey: string | null) => void
|
|
410
|
+
}) => void
|
|
396
411
|
}
|
|
397
412
|
|
|
398
413
|
/**
|
|
@@ -897,6 +912,20 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
897
912
|
maybeDeleteGroup(g)
|
|
898
913
|
}
|
|
899
914
|
|
|
915
|
+
/**
|
|
916
|
+
* Force-evict a LEAKED row from the specific group it lives in, used when the
|
|
917
|
+
* agentIndex has desynced (points at a different feed, or none) so the normal
|
|
918
|
+
* `removeWorker`/`terminateWorker` path — which resolves the group via the
|
|
919
|
+
* index — can't reach it. Deletes the index entry ONLY if it still points at
|
|
920
|
+
* THIS group, so a live row for the same agentId in another feed keeps its
|
|
921
|
+
* mapping intact.
|
|
922
|
+
*/
|
|
923
|
+
function evictRowFromGroup(g: FeedGroup, agentId: string): void {
|
|
924
|
+
g.workers.delete(agentId)
|
|
925
|
+
if (agentIndex.get(agentId) === g.feedKey) agentIndex.delete(agentId)
|
|
926
|
+
maybeDeleteGroup(g)
|
|
927
|
+
}
|
|
928
|
+
|
|
900
929
|
/**
|
|
901
930
|
* Delete an empty group. A group is gone only when it has NO tracked workers
|
|
902
931
|
* AND no staged terminal repaints pending — the latter keeps the group object
|
|
@@ -1243,7 +1272,7 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
1243
1272
|
// that keeps getting `update()` cues every heartbeat resets
|
|
1244
1273
|
// `lastUpdateAt` forever, so only an age anchor immune to that reset can
|
|
1245
1274
|
// reap it (Carrie 5h zombie pin, re-edited 3000+ times).
|
|
1246
|
-
const staleAgentIds: Array<{ agentId: string; reason: 'silence' | 'absolute' }> = []
|
|
1275
|
+
const staleAgentIds: Array<{ g: FeedGroup; agentId: string; reason: 'silence' | 'absolute' }> = []
|
|
1247
1276
|
const staleFinished: Array<{ g: FeedGroup; agentId: string; reason: 'silence' | 'absolute' }> = []
|
|
1248
1277
|
for (const g of groups.values()) {
|
|
1249
1278
|
for (const row of g.workers.values()) {
|
|
@@ -1253,8 +1282,12 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
1253
1282
|
// Attribute the reap to the absolute cap only when silence alone would
|
|
1254
1283
|
// NOT have fired — so the log names the trigger that actually caught it.
|
|
1255
1284
|
const reason: 'silence' | 'absolute' = silent ? 'silence' : 'absolute'
|
|
1285
|
+
// Carry the group the row was ACTUALLY found in (not `groupOfAgent`,
|
|
1286
|
+
// which resolves via the agentIndex — that index can desync from a
|
|
1287
|
+
// group's `workers` map when an agentId migrated feeds, orphaning the
|
|
1288
|
+
// old row. Using `g` guarantees the reap evicts the row that exists.)
|
|
1256
1289
|
if (row.finished) staleFinished.push({ g, agentId: row.agentId, reason })
|
|
1257
|
-
else staleAgentIds.push({ agentId: row.agentId, reason })
|
|
1290
|
+
else staleAgentIds.push({ g, agentId: row.agentId, reason })
|
|
1258
1291
|
}
|
|
1259
1292
|
}
|
|
1260
1293
|
// GC any FINISHED row still lingering past the TTL (#3207: finished rows
|
|
@@ -1274,15 +1307,32 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
1274
1307
|
removeWorker(g, agentId)
|
|
1275
1308
|
syncPin(g)
|
|
1276
1309
|
}
|
|
1277
|
-
for (const { agentId, reason } of staleAgentIds) {
|
|
1278
|
-
|
|
1310
|
+
for (const { g, agentId, reason } of staleAgentIds) {
|
|
1311
|
+
// Read the row from the group it was FOUND in, so the log reports the
|
|
1312
|
+
// real age even when the agentIndex has desynced (the old bug printed
|
|
1313
|
+
// "no update in 0s" because it read `groupOfAgent(agentId)`, which
|
|
1314
|
+
// resolved to a different/absent group and fell back to `now`).
|
|
1315
|
+
const row = g.workers.get(agentId)
|
|
1279
1316
|
if (reason === 'absolute') {
|
|
1280
1317
|
const age = Math.floor((now - (row?.createdAtMs ?? now)) / 1000)
|
|
1281
1318
|
log(`worker-feed: ABSOLUTE cap reap agent=${agentId} — row age ${age}s (>= ${Math.floor(absoluteRowLifetimeCapMs / 1000)}s); force-terminating immortal row (survives lastUpdateAt reset)`)
|
|
1282
1319
|
} else {
|
|
1283
1320
|
log(`worker-feed: TTL reap agent=${agentId} — no update in ${Math.floor((now - (row?.lastUpdateAt ?? now)) / 1000)}s (>= ${Math.floor(staleWorkerTtlMs / 1000)}s); force-terminating leaked row`)
|
|
1284
1321
|
}
|
|
1285
|
-
|
|
1322
|
+
// Normal honest finalize when the agentIndex still points at THIS group
|
|
1323
|
+
// (renders the terminal recap + releases the pin through the chain). But
|
|
1324
|
+
// when the index has desynced — it resolves to a different group or none
|
|
1325
|
+
// — `terminateWorker` would look up the wrong/no row and no-op, leaving
|
|
1326
|
+
// this row to churn every heartbeat forever. Detect that and force-evict
|
|
1327
|
+
// the leaked row directly from the group it actually lives in.
|
|
1328
|
+
if (groupOfAgent(agentId) === g) {
|
|
1329
|
+
void terminateWorker(agentId)
|
|
1330
|
+
} else {
|
|
1331
|
+
log(`worker-feed: force-evicting leaked row agent=${agentId} feed=${g.feedKey} — agentIndex desynced (points elsewhere); removing directly`)
|
|
1332
|
+
markFinalized(agentId)
|
|
1333
|
+
evictRowFromGroup(g, agentId)
|
|
1334
|
+
syncPin(g)
|
|
1335
|
+
}
|
|
1286
1336
|
}
|
|
1287
1337
|
for (const g of [...groups.values()]) {
|
|
1288
1338
|
// Deferred-finalize re-drive: a terminal edit that hit a cooldown/flood
|
|
@@ -1369,6 +1419,15 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
1369
1419
|
|
|
1370
1420
|
heartbeatTimer = setIntervalFn(heartbeatTick, heartbeatTickMs)
|
|
1371
1421
|
|
|
1422
|
+
// TEST-ONLY: hand out the narrow index-repoint control (see opts doc). Never
|
|
1423
|
+
// provided in production wiring, so this is a no-op there.
|
|
1424
|
+
opts.exposeTestControls?.({
|
|
1425
|
+
repointAgentIndex: (agentId, feedKey) => {
|
|
1426
|
+
if (feedKey == null) agentIndex.delete(agentId)
|
|
1427
|
+
else agentIndex.set(agentId, feedKey)
|
|
1428
|
+
},
|
|
1429
|
+
})
|
|
1430
|
+
|
|
1372
1431
|
return {
|
|
1373
1432
|
has(agentId) {
|
|
1374
1433
|
const g = groupOfAgent(agentId)
|
|
@@ -1401,6 +1460,20 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
1401
1460
|
if (existingRow?.finished === true) return Promise.resolve()
|
|
1402
1461
|
|
|
1403
1462
|
const feedKey = feedKeyOf(chatId, threadId)
|
|
1463
|
+
// Feed-migration cleanup (root-cause of the "no update in 0s" churn): if
|
|
1464
|
+
// this agentId is already indexed to a DIFFERENT feed, its prior row is
|
|
1465
|
+
// about to be orphaned (we re-index below to the new feed, but the old
|
|
1466
|
+
// group's `workers` map still holds the stale row — invisible to
|
|
1467
|
+
// `groupOfAgent` forever after, so the TTL sweep can never evict it and
|
|
1468
|
+
// it churns every heartbeat). Evict the stale row from its old group now.
|
|
1469
|
+
const priorFeedKey = agentIndex.get(agentId)
|
|
1470
|
+
if (priorFeedKey != null && priorFeedKey !== feedKey) {
|
|
1471
|
+
const priorGroup = groups.get(priorFeedKey)
|
|
1472
|
+
if (priorGroup != null) {
|
|
1473
|
+
evictRowFromGroup(priorGroup, agentId)
|
|
1474
|
+
syncPin(priorGroup)
|
|
1475
|
+
}
|
|
1476
|
+
}
|
|
1404
1477
|
let g = groups.get(feedKey)
|
|
1405
1478
|
if (g == null) {
|
|
1406
1479
|
g = {
|
|
@@ -8,8 +8,14 @@ truncateRecallQuery, sliceLastTurnsByUserBoundary, prepareRetentionTranscript,
|
|
|
8
8
|
formatMemories.
|
|
9
9
|
"""
|
|
10
10
|
|
|
11
|
+
import os
|
|
11
12
|
import re
|
|
12
|
-
from datetime import datetime
|
|
13
|
+
from datetime import datetime
|
|
14
|
+
|
|
15
|
+
try: # Python 3.9+; present in every switchroom agent image.
|
|
16
|
+
from zoneinfo import ZoneInfo
|
|
17
|
+
except ImportError: # pragma: no cover - defensive only
|
|
18
|
+
ZoneInfo = None # type: ignore[assignment,misc]
|
|
13
19
|
|
|
14
20
|
# ---------------------------------------------------------------------------
|
|
15
21
|
# Memory tag stripping (anti-feedback-loop)
|
|
@@ -254,16 +260,44 @@ def format_memories(results: list) -> str:
|
|
|
254
260
|
return "\n\n".join(lines)
|
|
255
261
|
|
|
256
262
|
|
|
263
|
+
def _resolve_agent_timezone() -> str:
|
|
264
|
+
"""Resolve the agent's configured IANA timezone.
|
|
265
|
+
|
|
266
|
+
Mirrors the switchroom cascade used by ``bin/timezone-hook.sh`` and
|
|
267
|
+
``src/config/timezone.ts``: ``SWITCHROOM_TIMEZONE`` → ``TZ`` → ``UTC``.
|
|
268
|
+
This recall hook runs as a plugin subprocess INSIDE the agent container,
|
|
269
|
+
so it inherits both env vars (compose.ts bakes them onto the container).
|
|
270
|
+
"""
|
|
271
|
+
return os.environ.get("SWITCHROOM_TIMEZONE") or os.environ.get("TZ") or "UTC"
|
|
272
|
+
|
|
273
|
+
|
|
257
274
|
def format_current_time() -> str:
|
|
258
|
-
"""Format current
|
|
275
|
+
"""Format the current time in the agent's LOCAL timezone for recall context.
|
|
259
276
|
|
|
260
|
-
|
|
261
|
-
|
|
277
|
+
Switchroom #tz-fix: previously this emitted ``"%Y-%m-%d %H:%M UTC"``, the
|
|
278
|
+
STRONGEST of several competing UTC "current time" strings the model saw
|
|
279
|
+
each turn — it fought the single correct local-time hint and made agents
|
|
280
|
+
intermittently report UTC / wrong-offset times. We now render the agent's
|
|
281
|
+
LOCAL wall clock in am/pm form (e.g. ``2026-07-16 04:09 PM AEST``) so the
|
|
282
|
+
recall block never injects a UTC "now". Deterministic — the timezone comes
|
|
283
|
+
from the container env, not model discipline.
|
|
262
284
|
|
|
263
285
|
Port of: formatCurrentTimeForRecall() in index.js
|
|
264
286
|
"""
|
|
265
|
-
|
|
266
|
-
|
|
287
|
+
tz_name = _resolve_agent_timezone()
|
|
288
|
+
now = None
|
|
289
|
+
if ZoneInfo is not None:
|
|
290
|
+
try:
|
|
291
|
+
now = datetime.now(ZoneInfo(tz_name))
|
|
292
|
+
except Exception: # unknown/invalid zone — degrade, don't crash recall.
|
|
293
|
+
now = None
|
|
294
|
+
if now is None:
|
|
295
|
+
# No zoneinfo or bad zone: fall back to the process-local clock, which
|
|
296
|
+
# already honours the container's TZ env via libc. am/pm form, no "UTC".
|
|
297
|
+
now = datetime.now().astimezone()
|
|
298
|
+
# "%Y-%m-%d %I:%M %p %Z" → e.g. "2026-07-16 04:09 PM AEST" (mirrors the
|
|
299
|
+
# %I:%M %p %Z tail of bin/timezone-hook.sh's format).
|
|
300
|
+
return now.strftime("%Y-%m-%d %I:%M %p %Z")
|
|
267
301
|
|
|
268
302
|
|
|
269
303
|
# ---------------------------------------------------------------------------
|
|
@@ -585,10 +585,31 @@ class TestPrepareRetentionTranscript:
|
|
|
585
585
|
|
|
586
586
|
|
|
587
587
|
class TestFormatCurrentTime:
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
588
|
+
# switchroom #tz-fix: recall's "Current time - …" line now renders the
|
|
589
|
+
# agent's LOCAL wall clock in am/pm, NOT UTC — it was the strongest of
|
|
590
|
+
# several competing UTC "now" strings the model saw each turn.
|
|
591
|
+
def test_local_am_pm_shape(self, monkeypatch):
|
|
592
|
+
# A real IANA zone → am/pm form with the zone abbreviation, no "UTC".
|
|
593
|
+
monkeypatch.setenv("SWITCHROOM_TIMEZONE", "Australia/Melbourne")
|
|
594
|
+
monkeypatch.delenv("TZ", raising=False)
|
|
595
|
+
out = format_current_time()
|
|
596
|
+
# e.g. "2026-07-16 04:09 PM AEST" / "… AEDT" (DST) — abbrev is 3-5 alpha.
|
|
597
|
+
assert re.fullmatch(
|
|
598
|
+
r"\d{4}-\d{2}-\d{2} \d{2}:\d{2} (?:AM|PM) [A-Za-z]{2,5}", out
|
|
599
|
+
), out
|
|
600
|
+
|
|
601
|
+
def test_never_emits_utc_current_time(self, monkeypatch):
|
|
602
|
+
# The whole point of the fix: no UTC "current time" for a real zone.
|
|
603
|
+
monkeypatch.setenv("SWITCHROOM_TIMEZONE", "Australia/Melbourne")
|
|
604
|
+
monkeypatch.delenv("TZ", raising=False)
|
|
605
|
+
out = format_current_time()
|
|
606
|
+
assert "UTC" not in out
|
|
607
|
+
assert not out.endswith("Z")
|
|
608
|
+
|
|
609
|
+
def test_am_pm_when_no_zone_configured(self, monkeypatch):
|
|
610
|
+
# No SWITCHROOM_TIMEZONE / TZ → still am/pm shape (process-local clock),
|
|
611
|
+
# never the old "%Y-%m-%d %H:%M UTC" 24h form.
|
|
612
|
+
monkeypatch.delenv("SWITCHROOM_TIMEZONE", raising=False)
|
|
613
|
+
monkeypatch.delenv("TZ", raising=False)
|
|
614
|
+
out = format_current_time()
|
|
615
|
+
assert re.match(r"\d{4}-\d{2}-\d{2} \d{2}:\d{2} (?:AM|PM)", out), out
|