switchroom 0.18.27 → 0.18.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/switchroom.js +1 -1
- package/dist/host-control/main.js +1 -1
- package/package.json +1 -1
- package/telegram-plugin/dist/gateway/gateway.js +81 -11
- package/telegram-plugin/gateway/gateway.ts +53 -12
- package/telegram-plugin/gateway/worker-pin-reaper.ts +54 -0
- package/telegram-plugin/tests/worker-feed-migration-eviction.test.ts +140 -0
- package/telegram-plugin/tests/worker-pin-reaper.test.ts +78 -0
- package/telegram-plugin/worker-activity-feed.ts +78 -5
package/dist/cli/switchroom.js
CHANGED
|
@@ -2120,7 +2120,7 @@ var init_esm = __esm(() => {
|
|
|
2120
2120
|
});
|
|
2121
2121
|
|
|
2122
2122
|
// src/build-info.ts
|
|
2123
|
-
var VERSION = "0.18.
|
|
2123
|
+
var VERSION = "0.18.28", COMMIT_SHA = "ef0be9a2";
|
|
2124
2124
|
|
|
2125
2125
|
// src/cli/resolve-version.ts
|
|
2126
2126
|
import { existsSync, readFileSync } from "node:fs";
|
|
@@ -26605,7 +26605,7 @@ import { existsSync as existsSync9, readFileSync as readFileSync7 } from "node:f
|
|
|
26605
26605
|
import { dirname as dirname4, join as join7 } from "node:path";
|
|
26606
26606
|
|
|
26607
26607
|
// src/build-info.ts
|
|
26608
|
-
var VERSION = "0.18.
|
|
26608
|
+
var VERSION = "0.18.28";
|
|
26609
26609
|
|
|
26610
26610
|
// src/cli/resolve-version.ts
|
|
26611
26611
|
function readPackageVersion() {
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "switchroom",
|
|
3
3
|
"//version": "NOT the release version — source of truth is the git tag, resolved by scripts/build.mjs:resolveVersion() (see CLAUDE.md > Standard release process). This field is stale by design and only the Layer-4 dev/non-tag fallback for build.mjs + src/cli/resolve-version.ts; do NOT bump it expecting a release to pick it up. npm-pack tarball naming needs a real version — do that as an UNCOMMITTED pack-time bump (see release step 6), never a committed one.",
|
|
4
|
-
"version": "0.18.
|
|
4
|
+
"version": "0.18.28",
|
|
5
5
|
"description": "Run Claude Code 24/7 on your Claude Pro/Max subscription over Telegram. Open-source alternative to OpenClaw and NanoClaw — no API keys.",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"bin": {
|
|
@@ -41118,6 +41118,12 @@ function createWorkerActivityFeed(opts) {
|
|
|
41118
41118
|
agentIndex.delete(agentId);
|
|
41119
41119
|
maybeDeleteGroup(g);
|
|
41120
41120
|
}
|
|
41121
|
+
function evictRowFromGroup(g, agentId) {
|
|
41122
|
+
g.workers.delete(agentId);
|
|
41123
|
+
if (agentIndex.get(agentId) === g.feedKey)
|
|
41124
|
+
agentIndex.delete(agentId);
|
|
41125
|
+
maybeDeleteGroup(g);
|
|
41126
|
+
}
|
|
41121
41127
|
function maybeDeleteGroup(g) {
|
|
41122
41128
|
if (g.workers.size === 0 && g.pendingFinalize.size === 0)
|
|
41123
41129
|
groups.delete(g.feedKey);
|
|
@@ -41309,7 +41315,7 @@ function createWorkerActivityFeed(opts) {
|
|
|
41309
41315
|
if (row.finished)
|
|
41310
41316
|
staleFinished.push({ g, agentId: row.agentId, reason });
|
|
41311
41317
|
else
|
|
41312
|
-
staleAgentIds.push({ agentId: row.agentId, reason });
|
|
41318
|
+
staleAgentIds.push({ g, agentId: row.agentId, reason });
|
|
41313
41319
|
}
|
|
41314
41320
|
}
|
|
41315
41321
|
for (const { g, agentId, reason } of staleFinished) {
|
|
@@ -41323,15 +41329,22 @@ function createWorkerActivityFeed(opts) {
|
|
|
41323
41329
|
removeWorker(g, agentId);
|
|
41324
41330
|
syncPin(g);
|
|
41325
41331
|
}
|
|
41326
|
-
for (const { agentId, reason } of staleAgentIds) {
|
|
41327
|
-
const row =
|
|
41332
|
+
for (const { g, agentId, reason } of staleAgentIds) {
|
|
41333
|
+
const row = g.workers.get(agentId);
|
|
41328
41334
|
if (reason === "absolute") {
|
|
41329
41335
|
const age = Math.floor((now - (row?.createdAtMs ?? now)) / 1000);
|
|
41330
41336
|
log(`worker-feed: ABSOLUTE cap reap agent=${agentId} \u2014 row age ${age}s (>= ${Math.floor(absoluteRowLifetimeCapMs / 1000)}s); force-terminating immortal row (survives lastUpdateAt reset)`);
|
|
41331
41337
|
} else {
|
|
41332
41338
|
log(`worker-feed: TTL reap agent=${agentId} \u2014 no update in ${Math.floor((now - (row?.lastUpdateAt ?? now)) / 1000)}s (>= ${Math.floor(staleWorkerTtlMs / 1000)}s); force-terminating leaked row`);
|
|
41333
41339
|
}
|
|
41334
|
-
|
|
41340
|
+
if (groupOfAgent(agentId) === g) {
|
|
41341
|
+
terminateWorker(agentId);
|
|
41342
|
+
} else {
|
|
41343
|
+
log(`worker-feed: force-evicting leaked row agent=${agentId} feed=${g.feedKey} \u2014 agentIndex desynced (points elsewhere); removing directly`);
|
|
41344
|
+
markFinalized(agentId);
|
|
41345
|
+
evictRowFromGroup(g, agentId);
|
|
41346
|
+
syncPin(g);
|
|
41347
|
+
}
|
|
41335
41348
|
}
|
|
41336
41349
|
for (const g of [...groups.values()]) {
|
|
41337
41350
|
if (g.pendingFinalize.size > 0 && now >= g.cooldownUntil) {
|
|
@@ -41378,6 +41391,14 @@ function createWorkerActivityFeed(opts) {
|
|
|
41378
41391
|
}
|
|
41379
41392
|
}
|
|
41380
41393
|
heartbeatTimer = setIntervalFn(heartbeatTick, heartbeatTickMs);
|
|
41394
|
+
opts.exposeTestControls?.({
|
|
41395
|
+
repointAgentIndex: (agentId, feedKey) => {
|
|
41396
|
+
if (feedKey == null)
|
|
41397
|
+
agentIndex.delete(agentId);
|
|
41398
|
+
else
|
|
41399
|
+
agentIndex.set(agentId, feedKey);
|
|
41400
|
+
}
|
|
41401
|
+
});
|
|
41381
41402
|
return {
|
|
41382
41403
|
has(agentId) {
|
|
41383
41404
|
const g = groupOfAgent(agentId);
|
|
@@ -41405,6 +41426,14 @@ function createWorkerActivityFeed(opts) {
|
|
|
41405
41426
|
if (existingRow?.finished === true)
|
|
41406
41427
|
return Promise.resolve();
|
|
41407
41428
|
const feedKey = feedKeyOf(chatId, threadId);
|
|
41429
|
+
const priorFeedKey = agentIndex.get(agentId);
|
|
41430
|
+
if (priorFeedKey != null && priorFeedKey !== feedKey) {
|
|
41431
|
+
const priorGroup = groups.get(priorFeedKey);
|
|
41432
|
+
if (priorGroup != null) {
|
|
41433
|
+
evictRowFromGroup(priorGroup, agentId);
|
|
41434
|
+
syncPin(priorGroup);
|
|
41435
|
+
}
|
|
41436
|
+
}
|
|
41408
41437
|
let g = groups.get(feedKey);
|
|
41409
41438
|
if (g == null) {
|
|
41410
41439
|
g = {
|
|
@@ -76759,6 +76788,28 @@ function workerAgentIdOfPinKey(pinKey) {
|
|
|
76759
76788
|
const agentId = pinKey.slice(WORKER_PIN_KEY_PREFIX.length);
|
|
76760
76789
|
return agentId.length > 0 ? agentId : null;
|
|
76761
76790
|
}
|
|
76791
|
+
function storeOnlyWorkerPinCandidates(args) {
|
|
76792
|
+
const out = [];
|
|
76793
|
+
for (const r of args.rows) {
|
|
76794
|
+
if (workerAgentIdOfPinKey(r.pinKey) == null)
|
|
76795
|
+
continue;
|
|
76796
|
+
if (r.pending)
|
|
76797
|
+
continue;
|
|
76798
|
+
if (r.expiresAt != null)
|
|
76799
|
+
continue;
|
|
76800
|
+
if (r.chatId.length === 0)
|
|
76801
|
+
continue;
|
|
76802
|
+
if (args.inMemoryPinKeys.has(r.pinKey))
|
|
76803
|
+
continue;
|
|
76804
|
+
out.push({
|
|
76805
|
+
pinKey: r.pinKey,
|
|
76806
|
+
chatId: r.chatId,
|
|
76807
|
+
pinnedAt: args.now,
|
|
76808
|
+
messageId: r.messageId
|
|
76809
|
+
});
|
|
76810
|
+
}
|
|
76811
|
+
return out;
|
|
76812
|
+
}
|
|
76762
76813
|
function decideWorkerPinReaps(args) {
|
|
76763
76814
|
const reaps = [];
|
|
76764
76815
|
for (const pin of args.pins) {
|
|
@@ -84131,10 +84182,10 @@ function effectiveTurnAgeMs(markerAgeMs, turnStartedAt, now) {
|
|
|
84131
84182
|
}
|
|
84132
84183
|
|
|
84133
84184
|
// ../src/build-info.ts
|
|
84134
|
-
var VERSION = "0.18.
|
|
84135
|
-
var COMMIT_SHA = "
|
|
84136
|
-
var COMMIT_DATE = "2026-07-
|
|
84137
|
-
var LATEST_PR =
|
|
84185
|
+
var VERSION = "0.18.28";
|
|
84186
|
+
var COMMIT_SHA = "ef0be9a2";
|
|
84187
|
+
var COMMIT_DATE = "2026-07-16T14:42:35+10:00";
|
|
84188
|
+
var LATEST_PR = 3274;
|
|
84138
84189
|
var COMMITS_AHEAD_OF_TAG = 0;
|
|
84139
84190
|
|
|
84140
84191
|
// gateway/boot-version.ts
|
|
@@ -89325,11 +89376,19 @@ async function runMidSessionCardReaper() {
|
|
|
89325
89376
|
}
|
|
89326
89377
|
if (WORKER_PIN_REAPER_ENABLED && PIN_STATUS_WHILE_WORKING) {
|
|
89327
89378
|
try {
|
|
89328
|
-
const
|
|
89379
|
+
const inMemoryKeys = new Set([...statusPinState.keys()].filter((k) => k.startsWith("wk:")));
|
|
89380
|
+
const inMemoryCandidates = [...inMemoryKeys].map((k) => ({
|
|
89329
89381
|
pinKey: k,
|
|
89330
89382
|
chatId: statusPinChatIds.get(k) ?? "",
|
|
89331
89383
|
pinnedAt: statusPinPinnedAt.get(k) ?? now
|
|
89332
89384
|
}));
|
|
89385
|
+
const storeOnlyCandidates = statusPinPersistEnabled ? storeOnlyWorkerPinCandidates({
|
|
89386
|
+
rows: loadStatusPins(STATUS_PIN_STORE_PATH, statusPinStoreFs),
|
|
89387
|
+
inMemoryPinKeys: inMemoryKeys,
|
|
89388
|
+
now
|
|
89389
|
+
}) : [];
|
|
89390
|
+
const storeOnlyKeys = new Set(storeOnlyCandidates.map((c) => c.pinKey));
|
|
89391
|
+
const candidates = [...inMemoryCandidates, ...storeOnlyCandidates];
|
|
89333
89392
|
const reaps = decideWorkerPinReaps({
|
|
89334
89393
|
pins: candidates,
|
|
89335
89394
|
statusOf: (agentId) => {
|
|
@@ -89356,9 +89415,20 @@ async function runMidSessionCardReaper() {
|
|
|
89356
89415
|
now
|
|
89357
89416
|
});
|
|
89358
89417
|
for (const reap of reaps) {
|
|
89359
|
-
|
|
89418
|
+
const storeOnly = storeOnlyKeys.has(reap.pinKey);
|
|
89419
|
+
process.stderr.write(`telegram gateway: worker-pin reaper unpinning ${reap.pinKey} (chat=${reap.chatId} reason=${reap.reason}${storeOnly ? " source=store-orphan" : ""})
|
|
89420
|
+
`);
|
|
89421
|
+
if (storeOnly && reap.messageId != null) {
|
|
89422
|
+
try {
|
|
89423
|
+
await statusPinApi().unpinChatMessage(reap.chatId, reap.messageId);
|
|
89424
|
+
} catch (err) {
|
|
89425
|
+
process.stderr.write(`telegram gateway: worker-pin reaper store-orphan unpin failed (${reap.pinKey} chat=${reap.chatId} msg=${reap.messageId}): ${err.message}
|
|
89360
89426
|
`);
|
|
89361
|
-
|
|
89427
|
+
}
|
|
89428
|
+
await mutateStatusPinRow(STATUS_PIN_STORE_PATH, statusPinStoreFs, reap.pinKey, null);
|
|
89429
|
+
} else {
|
|
89430
|
+
await reconcileStatusPin(reap.pinKey, reap.chatId, { pinned: false });
|
|
89431
|
+
}
|
|
89362
89432
|
}
|
|
89363
89433
|
} catch (err) {
|
|
89364
89434
|
process.stderr.write(`telegram gateway: worker-pin reaper error: ${err.message}
|
|
@@ -604,6 +604,7 @@ import {
|
|
|
604
604
|
} from './queued-card-store.js'
|
|
605
605
|
import {
|
|
606
606
|
decideWorkerPinReaps,
|
|
607
|
+
storeOnlyWorkerPinCandidates,
|
|
607
608
|
WORKER_PIN_TTL_MS_DEFAULT,
|
|
608
609
|
} from './worker-pin-reaper.js'
|
|
609
610
|
import { driveEscalation } from './escalation-drive.js'
|
|
@@ -9023,16 +9024,32 @@ async function runMidSessionCardReaper(): Promise<void> {
|
|
|
9023
9024
|
// the durable store row clear together.
|
|
9024
9025
|
if (WORKER_PIN_REAPER_ENABLED && PIN_STATUS_WHILE_WORKING) {
|
|
9025
9026
|
try {
|
|
9026
|
-
const
|
|
9027
|
-
.filter((k) => k.startsWith('wk:'))
|
|
9028
|
-
|
|
9029
|
-
|
|
9030
|
-
|
|
9031
|
-
|
|
9032
|
-
|
|
9033
|
-
|
|
9034
|
-
|
|
9035
|
-
|
|
9027
|
+
const inMemoryKeys = new Set(
|
|
9028
|
+
[...statusPinState.keys()].filter((k) => k.startsWith('wk:')),
|
|
9029
|
+
)
|
|
9030
|
+
const inMemoryCandidates = [...inMemoryKeys].map((k) => ({
|
|
9031
|
+
pinKey: k,
|
|
9032
|
+
chatId: statusPinChatIds.get(k) ?? '',
|
|
9033
|
+
// A missing timestamp (should not happen — set on every claim)
|
|
9034
|
+
// degrades to "claimed just now": terminality can still reap it,
|
|
9035
|
+
// the TTL gate never can. Conservative, never a spurious unpin.
|
|
9036
|
+
pinnedAt: statusPinPinnedAt.get(k) ?? now,
|
|
9037
|
+
}))
|
|
9038
|
+
// #3001 durable group net: fold in `wk:` rows that live in the DURABLE
|
|
9039
|
+
// store but have NO in-memory claim — the divergence window where the
|
|
9040
|
+
// claim was lost but the Telegram pin + store row survive. These would
|
|
9041
|
+
// otherwise linger until the next boot cleanup; the reconciling sweep
|
|
9042
|
+
// recovers them mid-session too, group-safely (per-message unpin of a
|
|
9043
|
+
// bot-tracked row, NEVER an unpin-all, NEVER an untracked/human pin).
|
|
9044
|
+
const storeOnlyCandidates = statusPinPersistEnabled
|
|
9045
|
+
? storeOnlyWorkerPinCandidates({
|
|
9046
|
+
rows: loadStatusPins(STATUS_PIN_STORE_PATH, statusPinStoreFs),
|
|
9047
|
+
inMemoryPinKeys: inMemoryKeys,
|
|
9048
|
+
now,
|
|
9049
|
+
})
|
|
9050
|
+
: []
|
|
9051
|
+
const storeOnlyKeys = new Set(storeOnlyCandidates.map((c) => c.pinKey))
|
|
9052
|
+
const candidates = [...inMemoryCandidates, ...storeOnlyCandidates]
|
|
9036
9053
|
const reaps = decideWorkerPinReaps({
|
|
9037
9054
|
pins: candidates,
|
|
9038
9055
|
statusOf: (agentId) => {
|
|
@@ -9064,11 +9081,35 @@ async function runMidSessionCardReaper(): Promise<void> {
|
|
|
9064
9081
|
now,
|
|
9065
9082
|
})
|
|
9066
9083
|
for (const reap of reaps) {
|
|
9084
|
+
const storeOnly = storeOnlyKeys.has(reap.pinKey)
|
|
9067
9085
|
process.stderr.write(
|
|
9068
9086
|
`telegram gateway: worker-pin reaper unpinning ${reap.pinKey} ` +
|
|
9069
|
-
`(chat=${reap.chatId} reason=${reap.reason}
|
|
9087
|
+
`(chat=${reap.chatId} reason=${reap.reason}` +
|
|
9088
|
+
`${storeOnly ? ' source=store-orphan' : ''})\n`,
|
|
9070
9089
|
)
|
|
9071
|
-
|
|
9090
|
+
if (storeOnly && reap.messageId != null) {
|
|
9091
|
+
// No in-memory claim to reconcile: unpin the exact tracked message
|
|
9092
|
+
// (per-message — group-safe) and drop the store row directly. Both
|
|
9093
|
+
// are best-effort/idempotent; a failed unpin still drops the row so
|
|
9094
|
+
// the next boot's cleanup is the final backstop.
|
|
9095
|
+
try {
|
|
9096
|
+
await statusPinApi().unpinChatMessage(reap.chatId, reap.messageId)
|
|
9097
|
+
} catch (err) {
|
|
9098
|
+
process.stderr.write(
|
|
9099
|
+
`telegram gateway: worker-pin reaper store-orphan unpin failed ` +
|
|
9100
|
+
`(${reap.pinKey} chat=${reap.chatId} msg=${reap.messageId}): ` +
|
|
9101
|
+
`${(err as Error).message}\n`,
|
|
9102
|
+
)
|
|
9103
|
+
}
|
|
9104
|
+
await mutateStatusPinRow(
|
|
9105
|
+
STATUS_PIN_STORE_PATH,
|
|
9106
|
+
statusPinStoreFs,
|
|
9107
|
+
reap.pinKey,
|
|
9108
|
+
null,
|
|
9109
|
+
)
|
|
9110
|
+
} else {
|
|
9111
|
+
await reconcileStatusPin(reap.pinKey, reap.chatId, { pinned: false })
|
|
9112
|
+
}
|
|
9072
9113
|
}
|
|
9073
9114
|
} catch (err) {
|
|
9074
9115
|
process.stderr.write(
|
|
@@ -52,6 +52,10 @@ export interface WorkerPinCandidate {
|
|
|
52
52
|
chatId: string
|
|
53
53
|
/** Wall-clock ms the claim was first taken (gateway's pinnedAt registry). */
|
|
54
54
|
pinnedAt: number
|
|
55
|
+
/** The pinned message id. Present for store-only candidates (whose reap is a
|
|
56
|
+
* raw per-message unpin — no in-memory claim to reconcile through). Omitted
|
|
57
|
+
* for in-memory candidates, which reap via reconcileStatusPin off the claim. */
|
|
58
|
+
messageId?: number
|
|
55
59
|
}
|
|
56
60
|
|
|
57
61
|
export interface WorkerPinReap extends WorkerPinCandidate {
|
|
@@ -86,6 +90,56 @@ export type WorkerRegistryStatus = 'terminal' | 'running' | 'unknown'
|
|
|
86
90
|
* for stalled / missing / lookup-error; a DB hiccup must degrade to 'unknown'
|
|
87
91
|
* — kept until the TTL — never to a spurious 'terminal' unpin).
|
|
88
92
|
*/
|
|
93
|
+
/** A durable-store row (status-pins.json) as seen by the reconciling sweep. */
|
|
94
|
+
export interface StoreOrphanRow {
|
|
95
|
+
pinKey: string
|
|
96
|
+
chatId: string
|
|
97
|
+
messageId: number
|
|
98
|
+
/** Pin API call still in-flight (persist-intent-first). Never reaped. */
|
|
99
|
+
pending?: boolean
|
|
100
|
+
/** Time-scoped `tool:` pin. Never a worker orphan; excluded. */
|
|
101
|
+
expiresAt?: number
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Build the extra `wk:` reap candidates that exist ONLY in the durable store
|
|
106
|
+
* (status-pins.json) and NOT in the in-memory claim map — the divergence window
|
|
107
|
+
* where the in-memory claim was lost/never-held but the Telegram pin AND its
|
|
108
|
+
* store row survive. Without this, such an orphan lingers until the next
|
|
109
|
+
* gateway boot (runStatusPinBootCleanup); this lets the PERIODIC sweep recover
|
|
110
|
+
* it too, group-safely (per-message unpin of a bot-tracked row — never an
|
|
111
|
+
* unpin-all, never a human pin).
|
|
112
|
+
*
|
|
113
|
+
* The store persists NO timestamp, so a store-only candidate carries
|
|
114
|
+
* `pinnedAt = now`: only a TERMINAL registry verdict can reap it. It is never
|
|
115
|
+
* TTL-reaped (no trustworthy age to age it out), never touched while its worker
|
|
116
|
+
* is `running`, and it is always a `wk:` row (the documented leak class).
|
|
117
|
+
* `pending` rows (pin API in-flight) and time-scoped `tool:` rows are excluded,
|
|
118
|
+
* as are rows already tracked in memory (the in-memory reaper owns those, with
|
|
119
|
+
* a real `pinnedAt` that can drive the TTL).
|
|
120
|
+
*/
|
|
121
|
+
export function storeOnlyWorkerPinCandidates(args: {
|
|
122
|
+
rows: Iterable<StoreOrphanRow>
|
|
123
|
+
inMemoryPinKeys: ReadonlySet<string>
|
|
124
|
+
now: number
|
|
125
|
+
}): WorkerPinCandidate[] {
|
|
126
|
+
const out: WorkerPinCandidate[] = []
|
|
127
|
+
for (const r of args.rows) {
|
|
128
|
+
if (workerAgentIdOfPinKey(r.pinKey) == null) continue // only wk: rows
|
|
129
|
+
if (r.pending) continue // pin API in-flight — do not race the write
|
|
130
|
+
if (r.expiresAt != null) continue // time-scoped tool pin — not a worker
|
|
131
|
+
if (r.chatId.length === 0) continue // can't unpin without a chat
|
|
132
|
+
if (args.inMemoryPinKeys.has(r.pinKey)) continue // in-memory reaper owns it
|
|
133
|
+
out.push({
|
|
134
|
+
pinKey: r.pinKey,
|
|
135
|
+
chatId: r.chatId,
|
|
136
|
+
pinnedAt: args.now,
|
|
137
|
+
messageId: r.messageId,
|
|
138
|
+
})
|
|
139
|
+
}
|
|
140
|
+
return out
|
|
141
|
+
}
|
|
142
|
+
|
|
89
143
|
export function decideWorkerPinReaps(args: {
|
|
90
144
|
pins: Iterable<WorkerPinCandidate>
|
|
91
145
|
statusOf: (agentId: string) => WorkerRegistryStatus
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Regression guard for the feed-registry churn bug (FIX 2): a worker row whose
|
|
3
|
+
* `agentId` migrated feeds (a fresh `update()` under a different chat/thread)
|
|
4
|
+
* used to leave the OLD group's `workers` map holding an orphan row that the
|
|
5
|
+
* `agentIndex` no longer pointed at. `groupOfAgent(agentId)` could then never
|
|
6
|
+
* resolve it, so the heartbeat TTL sweep's `terminateWorker(agentId)` no-op'd
|
|
7
|
+
* and the row churned every tick forever — the self-contradictory
|
|
8
|
+
* `worker-feed: TTL reap … no update in 0s (>= 3000s); force-terminating
|
|
9
|
+
* leaked row` log, repeating ~every 6s.
|
|
10
|
+
*
|
|
11
|
+
* Two guarantees are asserted as OUTCOMES:
|
|
12
|
+
* 1. Migration eviction (root cause): after the same agentId updates a
|
|
13
|
+
* different feed, the old group's row is gone immediately — total tracked
|
|
14
|
+
* rows never double-count the migrated worker.
|
|
15
|
+
* 2. Sweep force-eviction is idempotent: once every worker has ended, the TTL
|
|
16
|
+
* sweep drives `size` to 0 and it STAYS 0 across repeated heartbeat ticks
|
|
17
|
+
* (no churning row that re-reaps forever).
|
|
18
|
+
*/
|
|
19
|
+
import { describe, it, expect } from 'vitest'
|
|
20
|
+
import {
|
|
21
|
+
createWorkerActivityFeed,
|
|
22
|
+
type BotApiForWorkerFeed,
|
|
23
|
+
type WorkerActivityView,
|
|
24
|
+
} from '../worker-activity-feed.js'
|
|
25
|
+
|
|
26
|
+
function view(desc: string, elapsedMs: number): WorkerActivityView {
|
|
27
|
+
return { description: desc, lastTool: null, toolCount: 1, latestSummary: 'step', elapsedMs, state: 'running' }
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
type IndexControl = { repointAgentIndex: (agentId: string, feedKey: string | null) => void }
|
|
31
|
+
|
|
32
|
+
function makeFeed(nowRef: { t: number }) {
|
|
33
|
+
const pins: { messageId: number | null }[] = []
|
|
34
|
+
let seq = 900
|
|
35
|
+
let control: IndexControl | null = null
|
|
36
|
+
const bot: BotApiForWorkerFeed = {
|
|
37
|
+
sendMessage: async () => ({ message_id: seq++ }),
|
|
38
|
+
editMessageText: async () => true,
|
|
39
|
+
}
|
|
40
|
+
const feed = createWorkerActivityFeed({
|
|
41
|
+
bot,
|
|
42
|
+
now: () => nowRef.t,
|
|
43
|
+
minEditIntervalMs: 0,
|
|
44
|
+
heartbeatTickMs: 1000,
|
|
45
|
+
firstPaintMinMs: 0,
|
|
46
|
+
staleWorkerTtlMs: 3_000_000, // 3000s — matches the observed churn log
|
|
47
|
+
setInterval: () => 0,
|
|
48
|
+
clearInterval: () => {},
|
|
49
|
+
reconcilePin: ({ messageId }) => pins.push({ messageId }),
|
|
50
|
+
exposeTestControls: (c) => {
|
|
51
|
+
control = c
|
|
52
|
+
},
|
|
53
|
+
})
|
|
54
|
+
return { feed, pins, control: () => control as IndexControl }
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
async function drain(): Promise<void> {
|
|
58
|
+
for (let i = 0; i < 12; i++) await new Promise((r) => setImmediate(r))
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
describe('worker-feed migration eviction + churn-free sweep (FIX 2)', () => {
|
|
62
|
+
it('evicts the old-feed orphan when an agentId migrates feeds — no double-count', async () => {
|
|
63
|
+
const nowRef = { t: 1_000_000 }
|
|
64
|
+
const { feed } = makeFeed(nowRef)
|
|
65
|
+
|
|
66
|
+
// Worker registers in feed A (chat -100).
|
|
67
|
+
await feed.update('agent-1', '-100', view('task', 0))
|
|
68
|
+
await drain()
|
|
69
|
+
expect(feed.size).toBe(1)
|
|
70
|
+
|
|
71
|
+
// SAME agentId now updates a DIFFERENT feed (chat -200) — a migration. The
|
|
72
|
+
// old feed-A row must be evicted, not orphaned. Total rows stay 1.
|
|
73
|
+
nowRef.t += 1000
|
|
74
|
+
await feed.update('agent-1', '-200', view('task', 1000))
|
|
75
|
+
await drain()
|
|
76
|
+
expect(feed.size).toBe(1)
|
|
77
|
+
})
|
|
78
|
+
|
|
79
|
+
it('TTL sweep drives size to 0 and stays 0 across repeated ticks (no churn)', async () => {
|
|
80
|
+
const nowRef = { t: 2_000_000 }
|
|
81
|
+
const { feed } = makeFeed(nowRef)
|
|
82
|
+
|
|
83
|
+
await feed.update('agent-1', '-100', view('task', 0))
|
|
84
|
+
await drain()
|
|
85
|
+
// Migrate to a second feed to exercise the exact desync path that leaked.
|
|
86
|
+
nowRef.t += 1000
|
|
87
|
+
await feed.update('agent-1', '-200', view('task', 1000))
|
|
88
|
+
await drain()
|
|
89
|
+
expect(feed.size).toBe(1)
|
|
90
|
+
|
|
91
|
+
// Advance past the silence TTL so the sweep force-terminates the last row.
|
|
92
|
+
nowRef.t += 3_000_001
|
|
93
|
+
feed.heartbeatTick()
|
|
94
|
+
await drain()
|
|
95
|
+
expect(feed.size).toBe(0)
|
|
96
|
+
|
|
97
|
+
// Idempotency: further ticks must not resurrect / re-reap a churning row.
|
|
98
|
+
for (let i = 0; i < 3; i++) {
|
|
99
|
+
nowRef.t += 3_000_001
|
|
100
|
+
feed.heartbeatTick()
|
|
101
|
+
await drain()
|
|
102
|
+
}
|
|
103
|
+
expect(feed.size).toBe(0)
|
|
104
|
+
})
|
|
105
|
+
|
|
106
|
+
it('TTL sweep FORCE-EVICTS a leaked row whose agentIndex has desynced from its group', async () => {
|
|
107
|
+
// Reproduce the exact desync the migration-eviction fix now prevents any
|
|
108
|
+
// public sequence from producing: a live row physically in group A while
|
|
109
|
+
// the agentIndex points somewhere else, so `groupOfAgent(agentId) !== g`.
|
|
110
|
+
// This drives the heartbeat sweep's FORCE-EVICT branch specifically — the
|
|
111
|
+
// one that removes the orphan directly from the group it lives in instead
|
|
112
|
+
// of no-op'ing through `terminateWorker` (which would churn forever).
|
|
113
|
+
const nowRef = { t: 3_000_000 }
|
|
114
|
+
const { feed, control } = makeFeed(nowRef)
|
|
115
|
+
|
|
116
|
+
await feed.update('agent-1', '-100', view('task', 0))
|
|
117
|
+
await drain()
|
|
118
|
+
expect(feed.size).toBe(1)
|
|
119
|
+
|
|
120
|
+
// Corrupt the index so it no longer resolves to the group holding the row.
|
|
121
|
+
// `terminateWorker('agent-1')` would now look up a non-existent group and
|
|
122
|
+
// no-op — the leak the force-evict branch exists to catch.
|
|
123
|
+
control().repointAgentIndex('agent-1', 'bogus-feed-key')
|
|
124
|
+
|
|
125
|
+
// Past the silence TTL: the sweep must force-evict the row from its real
|
|
126
|
+
// group, NOT rely on terminateWorker.
|
|
127
|
+
nowRef.t += 3_000_001
|
|
128
|
+
feed.heartbeatTick()
|
|
129
|
+
await drain()
|
|
130
|
+
expect(feed.size).toBe(0)
|
|
131
|
+
|
|
132
|
+
// And it stays gone — no churning row re-reaped every tick.
|
|
133
|
+
for (let i = 0; i < 3; i++) {
|
|
134
|
+
nowRef.t += 3_000_001
|
|
135
|
+
feed.heartbeatTick()
|
|
136
|
+
await drain()
|
|
137
|
+
}
|
|
138
|
+
expect(feed.size).toBe(0)
|
|
139
|
+
})
|
|
140
|
+
})
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { describe, it, expect } from "vitest";
|
|
2
2
|
import {
|
|
3
3
|
decideWorkerPinReaps,
|
|
4
|
+
storeOnlyWorkerPinCandidates,
|
|
4
5
|
workerAgentIdOfPinKey,
|
|
5
6
|
WORKER_PIN_TTL_MS_DEFAULT,
|
|
6
7
|
type WorkerPinCandidate,
|
|
@@ -130,3 +131,80 @@ describe("decideWorkerPinReaps (#3001 mid-session wk: sweep)", () => {
|
|
|
130
131
|
expect(reaps).toHaveLength(0);
|
|
131
132
|
});
|
|
132
133
|
});
|
|
134
|
+
|
|
135
|
+
describe("storeOnlyWorkerPinCandidates (#3001 durable group net)", () => {
|
|
136
|
+
const inMem = (keys: string[] = []) => new Set(keys);
|
|
137
|
+
|
|
138
|
+
it("promotes a wk: store row that has NO in-memory claim into a reap candidate (group chat)", () => {
|
|
139
|
+
// A GROUP chat id is negative; the reconciling sweep must recover a
|
|
140
|
+
// bot-tracked orphan there via a per-message unpin (never an unpin-all).
|
|
141
|
+
const cands = storeOnlyWorkerPinCandidates({
|
|
142
|
+
rows: [{ pinKey: "wk:group:-100:5", chatId: "-100", messageId: 42 }],
|
|
143
|
+
inMemoryPinKeys: inMem(),
|
|
144
|
+
now: NOW,
|
|
145
|
+
});
|
|
146
|
+
expect(cands).toHaveLength(1);
|
|
147
|
+
expect(cands[0].pinKey).toBe("wk:group:-100:5");
|
|
148
|
+
expect(cands[0].chatId).toBe("-100");
|
|
149
|
+
// messageId is carried so the gateway can per-message unpin (group-safe).
|
|
150
|
+
expect(cands[0].messageId).toBe(42);
|
|
151
|
+
// A terminal registry verdict reaps it now, however "young" (pinnedAt=now).
|
|
152
|
+
const reaps = decideWorkerPinReaps({
|
|
153
|
+
pins: cands,
|
|
154
|
+
statusOf: () => "terminal" as const,
|
|
155
|
+
ttlMs: TTL,
|
|
156
|
+
now: NOW,
|
|
157
|
+
});
|
|
158
|
+
expect(reaps.map((r) => r.pinKey)).toEqual(["wk:group:-100:5"]);
|
|
159
|
+
});
|
|
160
|
+
|
|
161
|
+
it("NEVER promotes an untracked/human pin — only wk: rows qualify (fg:/tool:/banner: excluded)", () => {
|
|
162
|
+
const cands = storeOnlyWorkerPinCandidates({
|
|
163
|
+
rows: [
|
|
164
|
+
{ pinKey: "fg:-100:9", chatId: "-100", messageId: 1 },
|
|
165
|
+
{ pinKey: "tool:-100:9", chatId: "-100", messageId: 2, expiresAt: NOW + 1 },
|
|
166
|
+
{ pinKey: "banner:owner", chatId: "-100", messageId: 3 },
|
|
167
|
+
],
|
|
168
|
+
inMemoryPinKeys: inMem(),
|
|
169
|
+
now: NOW,
|
|
170
|
+
});
|
|
171
|
+
expect(cands).toHaveLength(0);
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
it("skips rows the in-memory reaper already owns (dedup by pinKey)", () => {
|
|
175
|
+
const cands = storeOnlyWorkerPinCandidates({
|
|
176
|
+
rows: [{ pinKey: "wk:a", chatId: "-100", messageId: 7 }],
|
|
177
|
+
inMemoryPinKeys: inMem(["wk:a"]),
|
|
178
|
+
now: NOW,
|
|
179
|
+
});
|
|
180
|
+
expect(cands).toHaveLength(0);
|
|
181
|
+
});
|
|
182
|
+
|
|
183
|
+
it("skips pending (pin API in-flight) and time-scoped tool rows and empty-chat rows", () => {
|
|
184
|
+
const cands = storeOnlyWorkerPinCandidates({
|
|
185
|
+
rows: [
|
|
186
|
+
{ pinKey: "wk:pending", chatId: "-100", messageId: 1, pending: true },
|
|
187
|
+
{ pinKey: "wk:tool", chatId: "-100", messageId: 2, expiresAt: NOW + 1000 },
|
|
188
|
+
{ pinKey: "wk:nochat", chatId: "", messageId: 3 },
|
|
189
|
+
],
|
|
190
|
+
inMemoryPinKeys: inMem(),
|
|
191
|
+
now: NOW,
|
|
192
|
+
});
|
|
193
|
+
expect(cands).toHaveLength(0);
|
|
194
|
+
});
|
|
195
|
+
|
|
196
|
+
it("a store-orphan with a live 'running' worker is kept (never reaped mid-run)", () => {
|
|
197
|
+
const cands = storeOnlyWorkerPinCandidates({
|
|
198
|
+
rows: [{ pinKey: "wk:live", chatId: "-100", messageId: 5 }],
|
|
199
|
+
inMemoryPinKeys: inMem(),
|
|
200
|
+
now: NOW,
|
|
201
|
+
});
|
|
202
|
+
const reaps = decideWorkerPinReaps({
|
|
203
|
+
pins: cands,
|
|
204
|
+
statusOf: () => "running" as const,
|
|
205
|
+
ttlMs: TTL,
|
|
206
|
+
now: NOW,
|
|
207
|
+
});
|
|
208
|
+
expect(reaps).toHaveLength(0);
|
|
209
|
+
});
|
|
210
|
+
});
|
|
@@ -393,6 +393,21 @@ export interface WorkerActivityFeedOpts {
|
|
|
393
393
|
threadId?: number
|
|
394
394
|
messageId: number | null
|
|
395
395
|
}) => void
|
|
396
|
+
/**
|
|
397
|
+
* TEST-ONLY (never set in production). Invoked once at construction with a
|
|
398
|
+
* narrow control that repoints the internal `agentId → feedKey` index WITHOUT
|
|
399
|
+
* moving the worker's row. Its sole purpose is to reproduce the
|
|
400
|
+
* `agentIndex`/`workers` DESYNC that the migration-eviction root-cause fix now
|
|
401
|
+
* prevents any public API sequence from producing — so the heartbeat sweep's
|
|
402
|
+
* force-evict backstop (the branch that removes a leaked row when
|
|
403
|
+
* `groupOfAgent(agentId)` no longer resolves to the group physically holding
|
|
404
|
+
* it) can be covered by a deterministic test. A no-op when unset.
|
|
405
|
+
*/
|
|
406
|
+
exposeTestControls?: (controls: {
|
|
407
|
+
/** Set the internal index entry for `agentId` to `feedKey` (or delete it
|
|
408
|
+
* when `feedKey` is null) without touching any group's `workers` map. */
|
|
409
|
+
repointAgentIndex: (agentId: string, feedKey: string | null) => void
|
|
410
|
+
}) => void
|
|
396
411
|
}
|
|
397
412
|
|
|
398
413
|
/**
|
|
@@ -897,6 +912,20 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
897
912
|
maybeDeleteGroup(g)
|
|
898
913
|
}
|
|
899
914
|
|
|
915
|
+
/**
|
|
916
|
+
* Force-evict a LEAKED row from the specific group it lives in, used when the
|
|
917
|
+
* agentIndex has desynced (points at a different feed, or none) so the normal
|
|
918
|
+
* `removeWorker`/`terminateWorker` path — which resolves the group via the
|
|
919
|
+
* index — can't reach it. Deletes the index entry ONLY if it still points at
|
|
920
|
+
* THIS group, so a live row for the same agentId in another feed keeps its
|
|
921
|
+
* mapping intact.
|
|
922
|
+
*/
|
|
923
|
+
function evictRowFromGroup(g: FeedGroup, agentId: string): void {
|
|
924
|
+
g.workers.delete(agentId)
|
|
925
|
+
if (agentIndex.get(agentId) === g.feedKey) agentIndex.delete(agentId)
|
|
926
|
+
maybeDeleteGroup(g)
|
|
927
|
+
}
|
|
928
|
+
|
|
900
929
|
/**
|
|
901
930
|
* Delete an empty group. A group is gone only when it has NO tracked workers
|
|
902
931
|
* AND no staged terminal repaints pending — the latter keeps the group object
|
|
@@ -1243,7 +1272,7 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
1243
1272
|
// that keeps getting `update()` cues every heartbeat resets
|
|
1244
1273
|
// `lastUpdateAt` forever, so only an age anchor immune to that reset can
|
|
1245
1274
|
// reap it (Carrie 5h zombie pin, re-edited 3000+ times).
|
|
1246
|
-
const staleAgentIds: Array<{ agentId: string; reason: 'silence' | 'absolute' }> = []
|
|
1275
|
+
const staleAgentIds: Array<{ g: FeedGroup; agentId: string; reason: 'silence' | 'absolute' }> = []
|
|
1247
1276
|
const staleFinished: Array<{ g: FeedGroup; agentId: string; reason: 'silence' | 'absolute' }> = []
|
|
1248
1277
|
for (const g of groups.values()) {
|
|
1249
1278
|
for (const row of g.workers.values()) {
|
|
@@ -1253,8 +1282,12 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
1253
1282
|
// Attribute the reap to the absolute cap only when silence alone would
|
|
1254
1283
|
// NOT have fired — so the log names the trigger that actually caught it.
|
|
1255
1284
|
const reason: 'silence' | 'absolute' = silent ? 'silence' : 'absolute'
|
|
1285
|
+
// Carry the group the row was ACTUALLY found in (not `groupOfAgent`,
|
|
1286
|
+
// which resolves via the agentIndex — that index can desync from a
|
|
1287
|
+
// group's `workers` map when an agentId migrated feeds, orphaning the
|
|
1288
|
+
// old row. Using `g` guarantees the reap evicts the row that exists.)
|
|
1256
1289
|
if (row.finished) staleFinished.push({ g, agentId: row.agentId, reason })
|
|
1257
|
-
else staleAgentIds.push({ agentId: row.agentId, reason })
|
|
1290
|
+
else staleAgentIds.push({ g, agentId: row.agentId, reason })
|
|
1258
1291
|
}
|
|
1259
1292
|
}
|
|
1260
1293
|
// GC any FINISHED row still lingering past the TTL (#3207: finished rows
|
|
@@ -1274,15 +1307,32 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
1274
1307
|
removeWorker(g, agentId)
|
|
1275
1308
|
syncPin(g)
|
|
1276
1309
|
}
|
|
1277
|
-
for (const { agentId, reason } of staleAgentIds) {
|
|
1278
|
-
|
|
1310
|
+
for (const { g, agentId, reason } of staleAgentIds) {
|
|
1311
|
+
// Read the row from the group it was FOUND in, so the log reports the
|
|
1312
|
+
// real age even when the agentIndex has desynced (the old bug printed
|
|
1313
|
+
// "no update in 0s" because it read `groupOfAgent(agentId)`, which
|
|
1314
|
+
// resolved to a different/absent group and fell back to `now`).
|
|
1315
|
+
const row = g.workers.get(agentId)
|
|
1279
1316
|
if (reason === 'absolute') {
|
|
1280
1317
|
const age = Math.floor((now - (row?.createdAtMs ?? now)) / 1000)
|
|
1281
1318
|
log(`worker-feed: ABSOLUTE cap reap agent=${agentId} — row age ${age}s (>= ${Math.floor(absoluteRowLifetimeCapMs / 1000)}s); force-terminating immortal row (survives lastUpdateAt reset)`)
|
|
1282
1319
|
} else {
|
|
1283
1320
|
log(`worker-feed: TTL reap agent=${agentId} — no update in ${Math.floor((now - (row?.lastUpdateAt ?? now)) / 1000)}s (>= ${Math.floor(staleWorkerTtlMs / 1000)}s); force-terminating leaked row`)
|
|
1284
1321
|
}
|
|
1285
|
-
|
|
1322
|
+
// Normal honest finalize when the agentIndex still points at THIS group
|
|
1323
|
+
// (renders the terminal recap + releases the pin through the chain). But
|
|
1324
|
+
// when the index has desynced — it resolves to a different group or none
|
|
1325
|
+
// — `terminateWorker` would look up the wrong/no row and no-op, leaving
|
|
1326
|
+
// this row to churn every heartbeat forever. Detect that and force-evict
|
|
1327
|
+
// the leaked row directly from the group it actually lives in.
|
|
1328
|
+
if (groupOfAgent(agentId) === g) {
|
|
1329
|
+
void terminateWorker(agentId)
|
|
1330
|
+
} else {
|
|
1331
|
+
log(`worker-feed: force-evicting leaked row agent=${agentId} feed=${g.feedKey} — agentIndex desynced (points elsewhere); removing directly`)
|
|
1332
|
+
markFinalized(agentId)
|
|
1333
|
+
evictRowFromGroup(g, agentId)
|
|
1334
|
+
syncPin(g)
|
|
1335
|
+
}
|
|
1286
1336
|
}
|
|
1287
1337
|
for (const g of [...groups.values()]) {
|
|
1288
1338
|
// Deferred-finalize re-drive: a terminal edit that hit a cooldown/flood
|
|
@@ -1369,6 +1419,15 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
1369
1419
|
|
|
1370
1420
|
heartbeatTimer = setIntervalFn(heartbeatTick, heartbeatTickMs)
|
|
1371
1421
|
|
|
1422
|
+
// TEST-ONLY: hand out the narrow index-repoint control (see opts doc). Never
|
|
1423
|
+
// provided in production wiring, so this is a no-op there.
|
|
1424
|
+
opts.exposeTestControls?.({
|
|
1425
|
+
repointAgentIndex: (agentId, feedKey) => {
|
|
1426
|
+
if (feedKey == null) agentIndex.delete(agentId)
|
|
1427
|
+
else agentIndex.set(agentId, feedKey)
|
|
1428
|
+
},
|
|
1429
|
+
})
|
|
1430
|
+
|
|
1372
1431
|
return {
|
|
1373
1432
|
has(agentId) {
|
|
1374
1433
|
const g = groupOfAgent(agentId)
|
|
@@ -1401,6 +1460,20 @@ export function createWorkerActivityFeed(opts: WorkerActivityFeedOpts): WorkerAc
|
|
|
1401
1460
|
if (existingRow?.finished === true) return Promise.resolve()
|
|
1402
1461
|
|
|
1403
1462
|
const feedKey = feedKeyOf(chatId, threadId)
|
|
1463
|
+
// Feed-migration cleanup (root-cause of the "no update in 0s" churn): if
|
|
1464
|
+
// this agentId is already indexed to a DIFFERENT feed, its prior row is
|
|
1465
|
+
// about to be orphaned (we re-index below to the new feed, but the old
|
|
1466
|
+
// group's `workers` map still holds the stale row — invisible to
|
|
1467
|
+
// `groupOfAgent` forever after, so the TTL sweep can never evict it and
|
|
1468
|
+
// it churns every heartbeat). Evict the stale row from its old group now.
|
|
1469
|
+
const priorFeedKey = agentIndex.get(agentId)
|
|
1470
|
+
if (priorFeedKey != null && priorFeedKey !== feedKey) {
|
|
1471
|
+
const priorGroup = groups.get(priorFeedKey)
|
|
1472
|
+
if (priorGroup != null) {
|
|
1473
|
+
evictRowFromGroup(priorGroup, agentId)
|
|
1474
|
+
syncPin(priorGroup)
|
|
1475
|
+
}
|
|
1476
|
+
}
|
|
1404
1477
|
let g = groups.get(feedKey)
|
|
1405
1478
|
if (g == null) {
|
|
1406
1479
|
g = {
|