switchroom 0.14.59 → 0.14.61
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/switchroom.js +2 -2
- package/package.json +1 -1
- package/telegram-plugin/dist/gateway/gateway.js +120 -15
- package/telegram-plugin/gateway/gateway.ts +102 -11
- package/telegram-plugin/gateway/obligation-ledger.ts +65 -2
- package/telegram-plugin/gateway/obligation-store.ts +107 -0
- package/telegram-plugin/tests/obligation-determinism.test.ts +241 -0
- package/telegram-plugin/tests/obligation-ledger.test.ts +69 -0
- package/telegram-plugin/tests/obligation-store.test.ts +117 -0
package/dist/cli/switchroom.js
CHANGED
|
@@ -49452,8 +49452,8 @@ var {
|
|
|
49452
49452
|
} = import__.default;
|
|
49453
49453
|
|
|
49454
49454
|
// src/build-info.ts
|
|
49455
|
-
var VERSION = "0.14.
|
|
49456
|
-
var COMMIT_SHA = "
|
|
49455
|
+
var VERSION = "0.14.61";
|
|
49456
|
+
var COMMIT_SHA = "6b3330f4";
|
|
49457
49457
|
|
|
49458
49458
|
// src/cli/agent.ts
|
|
49459
49459
|
init_source();
|
package/package.json
CHANGED
|
@@ -47135,20 +47135,37 @@ function createPendingInboundBuffer(opts = {}) {
|
|
|
47135
47135
|
// gateway/obligation-ledger.ts
|
|
47136
47136
|
class ObligationLedger {
|
|
47137
47137
|
maxRepresents;
|
|
47138
|
+
hooks;
|
|
47138
47139
|
open = new Map;
|
|
47139
|
-
constructor(maxRepresents = 2) {
|
|
47140
|
+
constructor(maxRepresents = 2, hooks = {}) {
|
|
47140
47141
|
this.maxRepresents = maxRepresents;
|
|
47142
|
+
this.hooks = hooks;
|
|
47143
|
+
}
|
|
47144
|
+
persist() {
|
|
47145
|
+
this.hooks.onChange?.(this.list());
|
|
47141
47146
|
}
|
|
47142
47147
|
openIfAbsent(input) {
|
|
47143
47148
|
if (this.open.has(input.originTurnId))
|
|
47144
47149
|
return false;
|
|
47145
47150
|
this.open.set(input.originTurnId, { ...input, representCount: 0 });
|
|
47151
|
+
this.persist();
|
|
47146
47152
|
return true;
|
|
47147
47153
|
}
|
|
47148
47154
|
close(originTurnId) {
|
|
47149
47155
|
if (originTurnId == null)
|
|
47150
47156
|
return false;
|
|
47151
|
-
|
|
47157
|
+
const closed = this.open.delete(originTurnId);
|
|
47158
|
+
if (closed)
|
|
47159
|
+
this.persist();
|
|
47160
|
+
return closed;
|
|
47161
|
+
}
|
|
47162
|
+
hydrate(snapshot) {
|
|
47163
|
+
this.open.clear();
|
|
47164
|
+
for (const o of snapshot) {
|
|
47165
|
+
if (o != null && typeof o.originTurnId === "string" && o.originTurnId.length > 0) {
|
|
47166
|
+
this.open.set(o.originTurnId, { ...o });
|
|
47167
|
+
}
|
|
47168
|
+
}
|
|
47152
47169
|
}
|
|
47153
47170
|
isOpen(originTurnId) {
|
|
47154
47171
|
return this.open.has(originTurnId);
|
|
@@ -47190,8 +47207,17 @@ class ObligationLedger {
|
|
|
47190
47207
|
if (o === undefined)
|
|
47191
47208
|
return 0;
|
|
47192
47209
|
o.representCount += 1;
|
|
47210
|
+
this.persist();
|
|
47193
47211
|
return o.representCount;
|
|
47194
47212
|
}
|
|
47213
|
+
markEscalateAttempt(originTurnId) {
|
|
47214
|
+
const o = this.open.get(originTurnId);
|
|
47215
|
+
if (o === undefined)
|
|
47216
|
+
return 0;
|
|
47217
|
+
o.escalateAttempts = (o.escalateAttempts ?? 0) + 1;
|
|
47218
|
+
this.persist();
|
|
47219
|
+
return o.escalateAttempts;
|
|
47220
|
+
}
|
|
47195
47221
|
}
|
|
47196
47222
|
var REPRESENT_PREVIEW_MAX = 200;
|
|
47197
47223
|
function buildObligationRepresentInbound(o, now) {
|
|
@@ -47218,6 +47244,47 @@ function obligationEscalationText(o) {
|
|
|
47218
47244
|
return `\u26a0\ufe0f I may have missed an earlier message and I'm not sure I answered it: ` + `"${preview}". If you still need it, please re-send.`;
|
|
47219
47245
|
}
|
|
47220
47246
|
|
|
47247
|
+
// gateway/obligation-store.ts
|
|
47248
|
+
function isObligationRow(x) {
|
|
47249
|
+
if (x == null || typeof x !== "object")
|
|
47250
|
+
return false;
|
|
47251
|
+
const o = x;
|
|
47252
|
+
return typeof o.originTurnId === "string" && o.originTurnId.length > 0 && typeof o.chatId === "string" && typeof o.messageId === "number" && typeof o.text === "string" && typeof o.openedAt === "number" && typeof o.representCount === "number";
|
|
47253
|
+
}
|
|
47254
|
+
function loadObligations(path, fs2) {
|
|
47255
|
+
if (!fs2.existsSync(path))
|
|
47256
|
+
return [];
|
|
47257
|
+
let raw = "";
|
|
47258
|
+
try {
|
|
47259
|
+
raw = fs2.readFileSync(path);
|
|
47260
|
+
} catch {
|
|
47261
|
+
return [];
|
|
47262
|
+
}
|
|
47263
|
+
let parsed;
|
|
47264
|
+
try {
|
|
47265
|
+
parsed = JSON.parse(raw);
|
|
47266
|
+
} catch {
|
|
47267
|
+
return [];
|
|
47268
|
+
}
|
|
47269
|
+
if (parsed == null || typeof parsed !== "object")
|
|
47270
|
+
return [];
|
|
47271
|
+
const env = parsed;
|
|
47272
|
+
if (env.v !== 1 || !Array.isArray(env.obligations))
|
|
47273
|
+
return [];
|
|
47274
|
+
return env.obligations.filter(isObligationRow);
|
|
47275
|
+
}
|
|
47276
|
+
function persistObligations(path, fs2, snapshot, log = (l) => process.stderr.write(l)) {
|
|
47277
|
+
const env = { v: 1, obligations: [...snapshot] };
|
|
47278
|
+
const tmp = path + ".tmp";
|
|
47279
|
+
try {
|
|
47280
|
+
fs2.writeFileSync(tmp, JSON.stringify(env));
|
|
47281
|
+
fs2.renameSync(tmp, path);
|
|
47282
|
+
} catch (err) {
|
|
47283
|
+
log(`obligation-store: persist FAILED path=${path}: ${err.message} \u2014 ` + `durability degraded to in-memory
|
|
47284
|
+
`);
|
|
47285
|
+
}
|
|
47286
|
+
}
|
|
47287
|
+
|
|
47221
47288
|
// gateway/inbound-spool.ts
|
|
47222
47289
|
function spoolId(msg) {
|
|
47223
47290
|
if (msg.meta?.source === "subagent_handback" && typeof msg.meta?.subagent_jsonl_id === "string" && msg.meta.subagent_jsonl_id.length > 0) {
|
|
@@ -52310,10 +52377,10 @@ function sweepStaleTurnActiveMarker(stateDir, opts) {
|
|
|
52310
52377
|
}
|
|
52311
52378
|
|
|
52312
52379
|
// ../src/build-info.ts
|
|
52313
|
-
var VERSION = "0.14.
|
|
52314
|
-
var COMMIT_SHA = "
|
|
52315
|
-
var COMMIT_DATE = "2026-06-
|
|
52316
|
-
var LATEST_PR =
|
|
52380
|
+
var VERSION = "0.14.61";
|
|
52381
|
+
var COMMIT_SHA = "6b3330f4";
|
|
52382
|
+
var COMMIT_DATE = "2026-06-04T10:11:35Z";
|
|
52383
|
+
var LATEST_PR = 2151;
|
|
52317
52384
|
var COMMITS_AHEAD_OF_TAG = 0;
|
|
52318
52385
|
|
|
52319
52386
|
// gateway/boot-version.ts
|
|
@@ -53514,7 +53581,26 @@ var deliveryQueue = createDeliveryQueue();
|
|
|
53514
53581
|
var OBLIGATION_LEDGER_ENABLED = process.env.SWITCHROOM_OBLIGATION_LEDGER === "1";
|
|
53515
53582
|
var OBLIGATION_REPRESENT_MAX = 2;
|
|
53516
53583
|
var OBLIGATION_SWEEP_MS = 5000;
|
|
53517
|
-
var
|
|
53584
|
+
var OBLIGATION_ESCALATE_MAX = 3;
|
|
53585
|
+
var OBLIGATION_STORE_PATH = join35(STATE_DIR, "obligations.json");
|
|
53586
|
+
var obligationStoreFs = {
|
|
53587
|
+
readFileSync: (p) => readFileSync36(p, "utf8"),
|
|
53588
|
+
writeFileSync: (p, d) => writeFileSync25(p, d),
|
|
53589
|
+
renameSync: (a, b) => renameSync12(a, b),
|
|
53590
|
+
existsSync: (p) => existsSync38(p)
|
|
53591
|
+
};
|
|
53592
|
+
var obligationLedger = new ObligationLedger(OBLIGATION_REPRESENT_MAX, {
|
|
53593
|
+
onChange: STATIC || !OBLIGATION_LEDGER_ENABLED ? undefined : (snapshot) => persistObligations(OBLIGATION_STORE_PATH, obligationStoreFs, snapshot)
|
|
53594
|
+
});
|
|
53595
|
+
if (!STATIC && OBLIGATION_LEDGER_ENABLED) {
|
|
53596
|
+
const restored = loadObligations(OBLIGATION_STORE_PATH, obligationStoreFs);
|
|
53597
|
+
if (restored.length > 0) {
|
|
53598
|
+
obligationLedger.hydrate(restored);
|
|
53599
|
+
process.stderr.write(`telegram gateway: obligation-ledger hydrated ${restored.length} open obligation(s) from ${OBLIGATION_STORE_PATH}
|
|
53600
|
+
`);
|
|
53601
|
+
}
|
|
53602
|
+
}
|
|
53603
|
+
var obligationEscalateInFlight = new Set;
|
|
53518
53604
|
var SERIALIZE_UNTIL_REPLIED_ENABLED = process.env.SWITCHROOM_SERIALIZE_UNTIL_REPLIED !== "0";
|
|
53519
53605
|
var _noReplyDrainRaw = process.env.SWITCHROOM_SERIALIZE_NOREPLY_DRAIN_MS;
|
|
53520
53606
|
var _noReplyDrainParsed = _noReplyDrainRaw != null && _noReplyDrainRaw !== "" ? Number(_noReplyDrainRaw) : 2500;
|
|
@@ -53806,6 +53892,9 @@ function endCurrentTurnAtomic(turn) {
|
|
|
53806
53892
|
if (currentTurn !== turn)
|
|
53807
53893
|
return;
|
|
53808
53894
|
currentTurn = null;
|
|
53895
|
+
if (OBLIGATION_LEDGER_ENABLED && turn.finalAnswerDelivered) {
|
|
53896
|
+
obligationLedger.close(turn.turnId);
|
|
53897
|
+
}
|
|
53809
53898
|
if (turn.noReplyDrainTimer != null) {
|
|
53810
53899
|
clearTimeout(turn.noReplyDrainTimer);
|
|
53811
53900
|
turn.noReplyDrainTimer = null;
|
|
@@ -54895,19 +54984,35 @@ function obligationSweep() {
|
|
|
54895
54984
|
return;
|
|
54896
54985
|
if (decision.action === "represent") {
|
|
54897
54986
|
pendingInboundBuffer.push(agent, buildObligationRepresentInbound(o, Date.now()));
|
|
54898
|
-
const
|
|
54899
|
-
process.stderr.write(`telegram gateway: obligation re-presented origin=${o.originTurnId} attempt=${
|
|
54987
|
+
const attempt2 = obligationLedger.markRepresented(o.originTurnId);
|
|
54988
|
+
process.stderr.write(`telegram gateway: obligation re-presented origin=${o.originTurnId} attempt=${attempt2}/${OBLIGATION_REPRESENT_MAX}
|
|
54900
54989
|
`);
|
|
54901
54990
|
return;
|
|
54902
54991
|
}
|
|
54903
|
-
|
|
54904
|
-
|
|
54992
|
+
if (obligationEscalateInFlight.has(o.originTurnId))
|
|
54993
|
+
return;
|
|
54994
|
+
const escId = o.originTurnId;
|
|
54995
|
+
const attempt = obligationLedger.markEscalateAttempt(escId);
|
|
54996
|
+
obligationEscalateInFlight.add(escId);
|
|
54997
|
+
process.stderr.write(`telegram gateway: obligation escalating (exhausted ${OBLIGATION_REPRESENT_MAX} re-presents) origin=${escId} attempt=${attempt}/${OBLIGATION_ESCALATE_MAX}
|
|
54905
54998
|
`);
|
|
54906
|
-
robustApiCall(
|
|
54907
|
-
...
|
|
54908
|
-
}), {
|
|
54909
|
-
|
|
54999
|
+
retryWithThreadFallback(robustApiCall, (tid) => bot.api.sendMessage(o.chatId, obligationEscalationText(o), {
|
|
55000
|
+
...tid != null ? { message_thread_id: tid } : {}
|
|
55001
|
+
}), { threadId: o.threadId, chat_id: o.chatId, verb: "obligation.escalate" }).then(() => {
|
|
55002
|
+
obligationLedger.close(escId);
|
|
55003
|
+
process.stderr.write(`telegram gateway: obligation escalation delivered + closed origin=${escId}
|
|
54910
55004
|
`);
|
|
55005
|
+
}).catch((err) => {
|
|
55006
|
+
if (attempt >= OBLIGATION_ESCALATE_MAX) {
|
|
55007
|
+
obligationLedger.close(escId);
|
|
55008
|
+
process.stderr.write(`telegram gateway: obligation escalation PERMANENTLY undeliverable after ${attempt} attempts \u2014 closing best-effort origin=${escId}: ${err}
|
|
55009
|
+
`);
|
|
55010
|
+
} else {
|
|
55011
|
+
process.stderr.write(`telegram gateway: obligation escalation send failed (attempt ${attempt}/${OBLIGATION_ESCALATE_MAX}), retrying next sweep origin=${escId}: ${err}
|
|
55012
|
+
`);
|
|
55013
|
+
}
|
|
55014
|
+
}).finally(() => {
|
|
55015
|
+
obligationEscalateInFlight.delete(escId);
|
|
54911
55016
|
});
|
|
54912
55017
|
}
|
|
54913
55018
|
if (!STATIC && OBLIGATION_LEDGER_ENABLED) {
|
|
@@ -283,6 +283,7 @@ import {
|
|
|
283
283
|
buildObligationRepresentInbound,
|
|
284
284
|
obligationEscalationText,
|
|
285
285
|
} from './obligation-ledger.js'
|
|
286
|
+
import { loadObligations, persistObligations } from './obligation-store.js'
|
|
286
287
|
import { createInboundSpool } from './inbound-spool.js'
|
|
287
288
|
import { purgeStaleTurnsForChat } from './turn-state-purge.js'
|
|
288
289
|
import { decideInboundDelivery } from './inbound-delivery-gate.js'
|
|
@@ -1397,7 +1398,49 @@ const deliveryQueue = createDeliveryQueue<InboundMessage>()
|
|
|
1397
1398
|
const OBLIGATION_LEDGER_ENABLED = process.env.SWITCHROOM_OBLIGATION_LEDGER === '1'
|
|
1398
1399
|
const OBLIGATION_REPRESENT_MAX = 2
|
|
1399
1400
|
const OBLIGATION_SWEEP_MS = 5_000
|
|
1400
|
-
|
|
1401
|
+
// Bound on escalation SEND attempts. The escalation now closes only AFTER a
|
|
1402
|
+
// successful send (a transient failure stays OPEN and retries next sweep), so a
|
|
1403
|
+
// PERMANENTLY-undeliverable nudge (dead/renumbered topic → Telegram 400 even
|
|
1404
|
+
// after thread-fallback, blocked bot) would loop forever — and, with the
|
|
1405
|
+
// durable snapshot, re-enter that loop on every boot. After this many failed
|
|
1406
|
+
// attempts the gateway closes best-effort (loud log): the user is genuinely
|
|
1407
|
+
// unreachable, so a bounded give-up beats an infinite/poison loop.
|
|
1408
|
+
const OBLIGATION_ESCALATE_MAX = 3
|
|
1409
|
+
// Durable snapshot of the open obligation set on the persistent per-agent
|
|
1410
|
+
// volume (STATE_DIR = /state/agent/telegram in prod). Closes the restart hole:
|
|
1411
|
+
// the in-memory ledger alone empties on restart and the spool's boot-replay
|
|
1412
|
+
// bypasses handleInbound (where obligations OPEN), so a delivered-but-unanswered
|
|
1413
|
+
// message used to lose its obligation across a restart. onChange persists every
|
|
1414
|
+
// mutation; boot hydrate (below) restores OPEN/ESCALATING with representCount +
|
|
1415
|
+
// escalateAttempts intact. STATIC mode (no runtime) and flag-off both skip disk.
|
|
1416
|
+
const OBLIGATION_STORE_PATH = join(STATE_DIR, 'obligations.json')
|
|
1417
|
+
const obligationStoreFs = {
|
|
1418
|
+
readFileSync: (p: string) => readFileSync(p, 'utf8'),
|
|
1419
|
+
writeFileSync: (p: string, d: string) => writeFileSync(p, d),
|
|
1420
|
+
renameSync: (a: string, b: string) => renameSync(a, b),
|
|
1421
|
+
existsSync: (p: string) => existsSync(p),
|
|
1422
|
+
}
|
|
1423
|
+
const obligationLedger = new ObligationLedger(OBLIGATION_REPRESENT_MAX, {
|
|
1424
|
+
onChange:
|
|
1425
|
+
STATIC || !OBLIGATION_LEDGER_ENABLED
|
|
1426
|
+
? undefined
|
|
1427
|
+
: (snapshot) => persistObligations(OBLIGATION_STORE_PATH, obligationStoreFs, snapshot),
|
|
1428
|
+
})
|
|
1429
|
+
if (!STATIC && OBLIGATION_LEDGER_ENABLED) {
|
|
1430
|
+
// Restart-durability: re-open obligations that were OPEN/ESCALATING when the
|
|
1431
|
+
// gateway last ran. The very next idle sweep re-presents or re-escalates them.
|
|
1432
|
+
const restored = loadObligations(OBLIGATION_STORE_PATH, obligationStoreFs)
|
|
1433
|
+
if (restored.length > 0) {
|
|
1434
|
+
obligationLedger.hydrate(restored)
|
|
1435
|
+
process.stderr.write(
|
|
1436
|
+
`telegram gateway: obligation-ledger hydrated ${restored.length} open obligation(s) from ${OBLIGATION_STORE_PATH}\n`,
|
|
1437
|
+
)
|
|
1438
|
+
}
|
|
1439
|
+
}
|
|
1440
|
+
// Origin ids with an escalation send IN FLIGHT — prevents the 5s sweep from
|
|
1441
|
+
// firing a second concurrent send for the same obligation while the first is
|
|
1442
|
+
// still awaiting (an escalation that takes >5s, or repeated failures).
|
|
1443
|
+
const obligationEscalateInFlight = new Set<string>()
|
|
1401
1444
|
|
|
1402
1445
|
// ─── Serialize-until-replied (multitopic reply-routing) ───────────────────
|
|
1403
1446
|
// Component 1 (deliver-before-drain gate). A buffered cross-topic inbound
|
|
@@ -2354,6 +2397,21 @@ function releaseTurnBufferGate(key: string, endingTurn?: CurrentTurn): void {
|
|
|
2354
2397
|
function endCurrentTurnAtomic(turn: CurrentTurn): void {
|
|
2355
2398
|
if (currentTurn !== turn) return
|
|
2356
2399
|
currentTurn = null
|
|
2400
|
+
// PR2 obligation-ledger CLOSE-at-turn-end. Close the ended turn's obligation
|
|
2401
|
+
// when it delivered a final answer. finalAnswerDelivered is the right signal
|
|
2402
|
+
// HERE (not isSubstantiveFinalReply at reply-time): a SHORT genuine answer
|
|
2403
|
+
// ("4") is final-but-not-substantive, so the reply-time substantive-close
|
|
2404
|
+
// missed it → it looked unanswered → the idle sweep double-asked every short
|
|
2405
|
+
// turn (canary, v0.14.59). At turn_end the #2141 logic has already demoted a
|
|
2406
|
+
// bare interim ack to non-final, so finalAnswerDelivered===true means GENUINELY
|
|
2407
|
+
// answered. This runs before the next idle sweep, so a short answer closes
|
|
2408
|
+
// cleanly (no double-ask); an ack-then-ghost / no-reply turn ends with
|
|
2409
|
+
// finalAnswerDelivered===false → stays open → re-presented (the intended
|
|
2410
|
+
// catch). close() is a no-op for synthetic turns (turnId not in the ledger).
|
|
2411
|
+
// No-op when the flag is off.
|
|
2412
|
+
if (OBLIGATION_LEDGER_ENABLED && turn.finalAnswerDelivered) {
|
|
2413
|
+
obligationLedger.close(turn.turnId)
|
|
2414
|
+
}
|
|
2357
2415
|
// Component 2 — clear any prior no-reply drain timer for this turn; a
|
|
2358
2416
|
// fresh end re-evaluates below. (Idempotent — null when never armed.)
|
|
2359
2417
|
if (turn.noReplyDrainTimer != null) {
|
|
@@ -4833,20 +4891,53 @@ function obligationSweep(): void {
|
|
|
4833
4891
|
)
|
|
4834
4892
|
return
|
|
4835
4893
|
}
|
|
4836
|
-
// escalate —
|
|
4837
|
-
|
|
4894
|
+
// escalate — re-present ladder exhausted. Send ONE operator-visible nudge and
|
|
4895
|
+
// close the obligation ONLY AFTER it actually lands. This inverts the old
|
|
4896
|
+
// close-before-send (which silently dropped the terminal whenever the send
|
|
4897
|
+
// failed): the close is now itself an observable terminal. A transient send
|
|
4898
|
+
// failure leaves the obligation OPEN → retried next sweep; a PERMANENT one
|
|
4899
|
+
// (dead topic even after thread-fallback, blocked bot) is bounded by
|
|
4900
|
+
// OBLIGATION_ESCALATE_MAX → close best-effort (the user is unreachable, so a
|
|
4901
|
+
// bounded give-up beats an infinite loop / a boot-surviving poison record).
|
|
4902
|
+
if (obligationEscalateInFlight.has(o.originTurnId)) return // a send is already awaiting
|
|
4903
|
+
const escId = o.originTurnId
|
|
4904
|
+
const attempt = obligationLedger.markEscalateAttempt(escId)
|
|
4905
|
+
obligationEscalateInFlight.add(escId)
|
|
4838
4906
|
process.stderr.write(
|
|
4839
|
-
`telegram gateway: obligation
|
|
4907
|
+
`telegram gateway: obligation escalating (exhausted ${OBLIGATION_REPRESENT_MAX} re-presents) origin=${escId} attempt=${attempt}/${OBLIGATION_ESCALATE_MAX}\n`,
|
|
4840
4908
|
)
|
|
4841
|
-
|
|
4842
|
-
|
|
4909
|
+
// retryWithThreadFallback: a stale/renumbered topic returns THREAD_NOT_FOUND;
|
|
4910
|
+
// retry WITHOUT the thread so the nudge still lands in the chat (the #2096
|
|
4911
|
+
// pattern) instead of being permanently undeliverable to a dead topic.
|
|
4912
|
+
void retryWithThreadFallback(
|
|
4913
|
+
robustApiCall,
|
|
4914
|
+
(tid) =>
|
|
4843
4915
|
bot.api.sendMessage(o.chatId, obligationEscalationText(o), {
|
|
4844
|
-
...(
|
|
4916
|
+
...(tid != null ? { message_thread_id: tid } : {}),
|
|
4845
4917
|
}),
|
|
4846
|
-
{
|
|
4847
|
-
)
|
|
4848
|
-
|
|
4849
|
-
|
|
4918
|
+
{ threadId: o.threadId, chat_id: o.chatId, verb: 'obligation.escalate' },
|
|
4919
|
+
)
|
|
4920
|
+
.then(() => {
|
|
4921
|
+
obligationLedger.close(escId)
|
|
4922
|
+
process.stderr.write(
|
|
4923
|
+
`telegram gateway: obligation escalation delivered + closed origin=${escId}\n`,
|
|
4924
|
+
)
|
|
4925
|
+
})
|
|
4926
|
+
.catch((err) => {
|
|
4927
|
+
if (attempt >= OBLIGATION_ESCALATE_MAX) {
|
|
4928
|
+
obligationLedger.close(escId)
|
|
4929
|
+
process.stderr.write(
|
|
4930
|
+
`telegram gateway: obligation escalation PERMANENTLY undeliverable after ${attempt} attempts — closing best-effort origin=${escId}: ${err}\n`,
|
|
4931
|
+
)
|
|
4932
|
+
} else {
|
|
4933
|
+
process.stderr.write(
|
|
4934
|
+
`telegram gateway: obligation escalation send failed (attempt ${attempt}/${OBLIGATION_ESCALATE_MAX}), retrying next sweep origin=${escId}: ${err}\n`,
|
|
4935
|
+
)
|
|
4936
|
+
}
|
|
4937
|
+
})
|
|
4938
|
+
.finally(() => {
|
|
4939
|
+
obligationEscalateInFlight.delete(escId)
|
|
4940
|
+
})
|
|
4850
4941
|
}
|
|
4851
4942
|
if (!STATIC && OBLIGATION_LEDGER_ENABLED) {
|
|
4852
4943
|
setInterval(obligationSweep, OBLIGATION_SWEEP_MS).unref()
|
|
@@ -38,6 +38,12 @@ export interface Obligation {
|
|
|
38
38
|
readonly openedAt: number
|
|
39
39
|
/** How many times it has been re-presented (0 on first open). */
|
|
40
40
|
representCount: number
|
|
41
|
+
/** How many times the operator-escalation send has been ATTEMPTED (0 until
|
|
42
|
+
* the represent ladder is exhausted). Bounds escalation so a permanently-
|
|
43
|
+
* undeliverable nudge (dead/renumbered topic → Telegram 400, blocked bot)
|
|
44
|
+
* can't loop forever — and, because it is part of the durable snapshot,
|
|
45
|
+
* can't become a boot-surviving poison record either. */
|
|
46
|
+
escalateAttempts?: number
|
|
41
47
|
}
|
|
42
48
|
|
|
43
49
|
/** What the gateway should do for the oldest open obligation at an idle boundary. */
|
|
@@ -57,14 +63,33 @@ export interface ObligationInput {
|
|
|
57
63
|
openedAt: number
|
|
58
64
|
}
|
|
59
65
|
|
|
66
|
+
/** Side-effect hooks the gateway injects. Kept out of the pure decision logic
|
|
67
|
+
* so the ledger stays unit-testable (a capturing `onChange` in tests). */
|
|
68
|
+
export interface ObligationLedgerHooks {
|
|
69
|
+
/** Called after EVERY state mutation (open/close/represent/escalate-attempt)
|
|
70
|
+
* with the full open snapshot, so the gateway can durably persist it. NOT
|
|
71
|
+
* called by hydrate() — that IS the restoration of persisted state. */
|
|
72
|
+
onChange?: (snapshot: Obligation[]) => void
|
|
73
|
+
}
|
|
74
|
+
|
|
60
75
|
export class ObligationLedger {
|
|
61
76
|
private readonly open = new Map<string, Obligation>()
|
|
62
77
|
|
|
63
78
|
/**
|
|
64
79
|
* @param maxRepresents max re-presentations before escalating to an
|
|
65
80
|
* operator-visible nudge instead of re-asking again. Default 2.
|
|
81
|
+
* @param hooks optional side-effect hooks (durable persistence). Omitted in
|
|
82
|
+
* pure unit tests; the gateway wires `onChange` to the snapshot store.
|
|
66
83
|
*/
|
|
67
|
-
constructor(
|
|
84
|
+
constructor(
|
|
85
|
+
private readonly maxRepresents = 2,
|
|
86
|
+
private readonly hooks: ObligationLedgerHooks = {},
|
|
87
|
+
) {}
|
|
88
|
+
|
|
89
|
+
/** Persist the current open set via the injected hook (no-op if unwired). */
|
|
90
|
+
private persist(): void {
|
|
91
|
+
this.hooks.onChange?.(this.list())
|
|
92
|
+
}
|
|
68
93
|
|
|
69
94
|
/**
|
|
70
95
|
* Open an obligation if not already tracked. Idempotent on originTurnId — a
|
|
@@ -75,13 +100,33 @@ export class ObligationLedger {
|
|
|
75
100
|
openIfAbsent(input: ObligationInput): boolean {
|
|
76
101
|
if (this.open.has(input.originTurnId)) return false
|
|
77
102
|
this.open.set(input.originTurnId, { ...input, representCount: 0 })
|
|
103
|
+
this.persist()
|
|
78
104
|
return true
|
|
79
105
|
}
|
|
80
106
|
|
|
81
107
|
/** Close by origin id. Returns true if an obligation was open and is now closed. */
|
|
82
108
|
close(originTurnId: string | null | undefined): boolean {
|
|
83
109
|
if (originTurnId == null) return false
|
|
84
|
-
|
|
110
|
+
const closed = this.open.delete(originTurnId)
|
|
111
|
+
if (closed) this.persist()
|
|
112
|
+
return closed
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* Re-populate from a persisted snapshot at boot — the restart-durability seam.
|
|
117
|
+
* Open obligations (with their representCount + escalateAttempts intact)
|
|
118
|
+
* survive a gateway/container restart, so the next idle sweep re-presents or
|
|
119
|
+
* re-escalates them rather than silently losing a delivered-but-unanswered
|
|
120
|
+
* message. Replaces current contents; does NOT fire onChange (this state is
|
|
121
|
+
* already what's on disk). Tolerates a malformed snapshot (skips bad rows).
|
|
122
|
+
*/
|
|
123
|
+
hydrate(snapshot: readonly Obligation[]): void {
|
|
124
|
+
this.open.clear()
|
|
125
|
+
for (const o of snapshot) {
|
|
126
|
+
if (o != null && typeof o.originTurnId === 'string' && o.originTurnId.length > 0) {
|
|
127
|
+
this.open.set(o.originTurnId, { ...o })
|
|
128
|
+
}
|
|
129
|
+
}
|
|
85
130
|
}
|
|
86
131
|
|
|
87
132
|
isOpen(originTurnId: string): boolean {
|
|
@@ -156,8 +201,26 @@ export class ObligationLedger {
|
|
|
156
201
|
const o = this.open.get(originTurnId)
|
|
157
202
|
if (o === undefined) return 0
|
|
158
203
|
o.representCount += 1
|
|
204
|
+
this.persist()
|
|
159
205
|
return o.representCount
|
|
160
206
|
}
|
|
207
|
+
|
|
208
|
+
/**
|
|
209
|
+
* Record an operator-escalation SEND attempt (bumps escalateAttempts).
|
|
210
|
+
* Returns the new count. The gateway calls this immediately before each
|
|
211
|
+
* escalation send and uses the count to bound retries: a transient send
|
|
212
|
+
* failure leaves the obligation OPEN and is retried next sweep, but after
|
|
213
|
+
* the bound the gateway closes best-effort so a permanently-undeliverable
|
|
214
|
+
* nudge can neither loop forever nor (now that the count is durable) re-enter
|
|
215
|
+
* the loop on every boot.
|
|
216
|
+
*/
|
|
217
|
+
markEscalateAttempt(originTurnId: string): number {
|
|
218
|
+
const o = this.open.get(originTurnId)
|
|
219
|
+
if (o === undefined) return 0
|
|
220
|
+
o.escalateAttempts = (o.escalateAttempts ?? 0) + 1
|
|
221
|
+
this.persist()
|
|
222
|
+
return o.escalateAttempts
|
|
223
|
+
}
|
|
161
224
|
}
|
|
162
225
|
|
|
163
226
|
/** Original message preview length for re-presentation (mirrors resume builder). */
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* obligation-store.ts — durable snapshot for the obligation ledger.
|
|
3
|
+
*
|
|
4
|
+
* Why this exists: `ObligationLedger` is an in-memory Map. A gateway/container
|
|
5
|
+
* restart (switchroom update, agent restart, self-restart, OOM) empties it, so
|
|
6
|
+
* an inbound that was OPEN-but-unanswered when the process died loses its
|
|
7
|
+
* answer-obligation. The inbound-spool redelivers the MESSAGE on boot, but its
|
|
8
|
+
* replay bypasses `handleInbound` (the only place obligations OPEN), so the
|
|
9
|
+
* obligation is never reborn and a post-restart deferral is silently dropped —
|
|
10
|
+
* the determinism hole the systems analysis flagged.
|
|
11
|
+
*
|
|
12
|
+
* This makes the obligation guarantee SELF-CONTAINED across restart: every
|
|
13
|
+
* ledger mutation persists the full open set here; on boot the gateway hydrates
|
|
14
|
+
* the ledger from it, so OPEN/ESCALATING obligations survive WITH their
|
|
15
|
+
* representCount + escalateAttempts intact (the latter so a permanently-
|
|
16
|
+
* undeliverable escalation can't re-enter its retry loop on every boot).
|
|
17
|
+
*
|
|
18
|
+
* Shape choice — SNAPSHOT, not append-log. The open set is tiny and bounded
|
|
19
|
+
* (the count of currently-unanswered messages — normally 0–3), so rewriting the
|
|
20
|
+
* whole set on each change is trivially cheap and needs NO compaction,
|
|
21
|
+
* tombstones, or torn-tail replay. Crash-safety is a single write-tmp +
|
|
22
|
+
* atomic rename: a crash leaves EITHER the prior complete snapshot OR the new
|
|
23
|
+
* complete one — never a torn file. This is strictly simpler than the spool's
|
|
24
|
+
* append+ack+compact idiom, which that file needs only because its set is
|
|
25
|
+
* unbounded. PURE w.r.t. the injected fs seam ⇒ unit-testable.
|
|
26
|
+
*/
|
|
27
|
+
|
|
28
|
+
import type { Obligation } from './obligation-ledger.js'
|
|
29
|
+
|
|
30
|
+
export interface ObligationStoreFsSeam {
|
|
31
|
+
readFileSync: (path: string) => string
|
|
32
|
+
writeFileSync: (path: string, data: string) => void
|
|
33
|
+
/** Atomic same-dir replace (POSIX rename) so a crash mid-write can't tear
|
|
34
|
+
* the snapshot. */
|
|
35
|
+
renameSync: (from: string, to: string) => void
|
|
36
|
+
existsSync: (path: string) => boolean
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
interface SnapshotEnvelope {
|
|
40
|
+
v: 1
|
|
41
|
+
obligations: Obligation[]
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function isObligationRow(x: unknown): x is Obligation {
|
|
45
|
+
if (x == null || typeof x !== 'object') return false
|
|
46
|
+
const o = x as Record<string, unknown>
|
|
47
|
+
return (
|
|
48
|
+
typeof o.originTurnId === 'string' &&
|
|
49
|
+
o.originTurnId.length > 0 &&
|
|
50
|
+
typeof o.chatId === 'string' &&
|
|
51
|
+
typeof o.messageId === 'number' &&
|
|
52
|
+
typeof o.text === 'string' &&
|
|
53
|
+
typeof o.openedAt === 'number' &&
|
|
54
|
+
typeof o.representCount === 'number'
|
|
55
|
+
)
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Load the persisted open set. Returns [] on a missing, unreadable, or
|
|
60
|
+
* malformed file (fail-open to empty: a corrupt snapshot must never crash boot;
|
|
61
|
+
* worst case we lose the cross-restart obligation guarantee for that boot and
|
|
62
|
+
* fall back to the spool's message redelivery — strictly no worse than today).
|
|
63
|
+
*/
|
|
64
|
+
export function loadObligations(path: string, fs: ObligationStoreFsSeam): Obligation[] {
|
|
65
|
+
if (!fs.existsSync(path)) return []
|
|
66
|
+
let raw = ''
|
|
67
|
+
try {
|
|
68
|
+
raw = fs.readFileSync(path)
|
|
69
|
+
} catch {
|
|
70
|
+
return []
|
|
71
|
+
}
|
|
72
|
+
let parsed: unknown
|
|
73
|
+
try {
|
|
74
|
+
parsed = JSON.parse(raw)
|
|
75
|
+
} catch {
|
|
76
|
+
return []
|
|
77
|
+
}
|
|
78
|
+
if (parsed == null || typeof parsed !== 'object') return []
|
|
79
|
+
const env = parsed as Record<string, unknown>
|
|
80
|
+
if (env.v !== 1 || !Array.isArray(env.obligations)) return []
|
|
81
|
+
return env.obligations.filter(isObligationRow)
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Persist the open set atomically (write sibling tmp → rename over the real
|
|
86
|
+
* path). Best-effort relative to fs availability: a write failure is logged but
|
|
87
|
+
* never thrown — a failing store degrades to in-memory-only (today's behaviour),
|
|
88
|
+
* it must not break live delivery.
|
|
89
|
+
*/
|
|
90
|
+
export function persistObligations(
|
|
91
|
+
path: string,
|
|
92
|
+
fs: ObligationStoreFsSeam,
|
|
93
|
+
snapshot: readonly Obligation[],
|
|
94
|
+
log: (line: string) => void = (l) => process.stderr.write(l),
|
|
95
|
+
): void {
|
|
96
|
+
const env: SnapshotEnvelope = { v: 1, obligations: [...snapshot] }
|
|
97
|
+
const tmp = path + '.tmp'
|
|
98
|
+
try {
|
|
99
|
+
fs.writeFileSync(tmp, JSON.stringify(env))
|
|
100
|
+
fs.renameSync(tmp, path)
|
|
101
|
+
} catch (err) {
|
|
102
|
+
log(
|
|
103
|
+
`obligation-store: persist FAILED path=${path}: ${(err as Error).message} — ` +
|
|
104
|
+
`durability degraded to in-memory\n`,
|
|
105
|
+
)
|
|
106
|
+
}
|
|
107
|
+
}
|
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
import { describe, it, expect } from "vitest";
|
|
2
|
+
import { ObligationLedger, type Obligation } from "../gateway/obligation-ledger.js";
|
|
3
|
+
import {
|
|
4
|
+
loadObligations,
|
|
5
|
+
persistObligations,
|
|
6
|
+
type ObligationStoreFsSeam,
|
|
7
|
+
} from "../gateway/obligation-store.js";
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* DETERMINISM PROOF (by exhaustive simulation, not live observation).
|
|
11
|
+
*
|
|
12
|
+
* The operator's requirement: be confident — by reasoning — that EVERY real
|
|
13
|
+
* inbound reaches answered-or-escalated for ANY model behaviour and ANY timing,
|
|
14
|
+
* including across a restart, with no silent drop and no double-ask of a message
|
|
15
|
+
* that WAS answered.
|
|
16
|
+
*
|
|
17
|
+
* This drives the REAL ObligationLedger + the REAL durable snapshot store
|
|
18
|
+
* (obligation-store, over an in-memory fs whose rename is atomic) through a
|
|
19
|
+
* faithful model of the gateway's obligation lifecycle:
|
|
20
|
+
*
|
|
21
|
+
* OPEN — on receipt (handleInbound, idempotent).
|
|
22
|
+
* close — at turn_end iff the turn delivered a final answer (finalAnswerDelivered).
|
|
23
|
+
* represent— idle sweep, bounded by maxRepresents, drives a fresh must-answer turn.
|
|
24
|
+
* escalate — ladder exhausted: send the operator nudge; close ONLY after it
|
|
25
|
+
* lands; a transient send failure stays OPEN and retries; a permanent
|
|
26
|
+
* one is bounded (OBLIGATION_ESCALATE_MAX) → close best-effort.
|
|
27
|
+
* restart — fresh ledger hydrated from the durable snapshot (counters intact).
|
|
28
|
+
*
|
|
29
|
+
* Over thousands of random schedules it asserts the invariant holds and the
|
|
30
|
+
* engine always terminates (no infinite loop). The COALESCED PARTIAL-ANSWER
|
|
31
|
+
* residual is deliberately NOT modelled — it is the one honest hard limit (a
|
|
32
|
+
* turn-keyed ledger cannot see "answered half" without parsing model prose) and
|
|
33
|
+
* is mitigated by coalescing policy, not the ledger.
|
|
34
|
+
*/
|
|
35
|
+
|
|
36
|
+
// Mirrors the gateway constants under test.
|
|
37
|
+
const MAX_REPRESENTS = 2;
|
|
38
|
+
const ESCALATE_MAX = 3;
|
|
39
|
+
|
|
40
|
+
// Deterministic PRNG (mulberry32) so any failure reproduces from its seed.
|
|
41
|
+
function rng(seed: number): () => number {
|
|
42
|
+
let a = seed >>> 0;
|
|
43
|
+
return () => {
|
|
44
|
+
a |= 0;
|
|
45
|
+
a = (a + 0x6d2b79f5) | 0;
|
|
46
|
+
let t = Math.imul(a ^ (a >>> 15), 1 | a);
|
|
47
|
+
t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
|
|
48
|
+
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function memStore(): { fs: ObligationStoreFsSeam } {
|
|
53
|
+
const files = new Map<string, string>();
|
|
54
|
+
return {
|
|
55
|
+
fs: {
|
|
56
|
+
readFileSync: (p) => {
|
|
57
|
+
if (!files.has(p)) throw new Error(`ENOENT ${p}`);
|
|
58
|
+
return files.get(p)!;
|
|
59
|
+
},
|
|
60
|
+
writeFileSync: (p, d) => files.set(p, d),
|
|
61
|
+
renameSync: (a, b) => {
|
|
62
|
+
if (!files.has(a)) throw new Error(`ENOENT ${a}`);
|
|
63
|
+
files.set(b, files.get(a)!);
|
|
64
|
+
files.delete(a);
|
|
65
|
+
},
|
|
66
|
+
existsSync: (p) => files.has(p),
|
|
67
|
+
},
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
type Terminal = "answered" | "escalation-delivered" | "escalation-give-up";
|
|
72
|
+
|
|
73
|
+
interface Msg {
|
|
74
|
+
id: string;
|
|
75
|
+
/** Real numeric Telegram message id (the gateway only opens an obligation
|
|
76
|
+
* when deriveTurnId is non-null, i.e. messageId > 0 — so the durable row
|
|
77
|
+
* always carries a valid number). */
|
|
78
|
+
msgId: number;
|
|
79
|
+
/** Turn attempt index (0 = original, 1 = 1st re-present, 2 = 2nd) at which
|
|
80
|
+
* the model delivers a final answer. >MAX_REPRESENTS ⇒ never answered ⇒ escalates. */
|
|
81
|
+
answerOnAttempt: number;
|
|
82
|
+
/** How many escalation SEND attempts fail before one succeeds. ≥ESCALATE_MAX
|
|
83
|
+
* ⇒ permanently undeliverable ⇒ bounded give-up. */
|
|
84
|
+
escalateFailsFor: number;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
interface Sim {
|
|
88
|
+
terminals: Map<string, Terminal>;
|
|
89
|
+
steps: number;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function runSchedule(msgs: Msg[], seed: number): Sim {
|
|
93
|
+
const PATH = "/state/agent/telegram/obligations.json";
|
|
94
|
+
const store = memStore();
|
|
95
|
+
let ledger = new ObligationLedger(MAX_REPRESENTS, {
|
|
96
|
+
onChange: (snap) => persistObligations(PATH, store.fs, snap),
|
|
97
|
+
});
|
|
98
|
+
const r = rng(seed);
|
|
99
|
+
|
|
100
|
+
const pending = [...msgs]; // not yet received
|
|
101
|
+
const byId = new Map(msgs.map((m) => [m.id, m]));
|
|
102
|
+
const turnsHad = new Map<string, number>(); // total turns delivered to each obligation
|
|
103
|
+
const terminals = new Map<string, Terminal>();
|
|
104
|
+
const received = new Set<string>();
|
|
105
|
+
|
|
106
|
+
const close = (id: string, why: Terminal) => {
|
|
107
|
+
ledger.close(id);
|
|
108
|
+
terminals.set(id, why);
|
|
109
|
+
};
|
|
110
|
+
|
|
111
|
+
// Run one turn for an obligation; close if the model answers on this attempt.
|
|
112
|
+
const deliverTurn = (id: string) => {
|
|
113
|
+
const had = (turnsHad.get(id) ?? 0);
|
|
114
|
+
const attemptIndex = had; // 0-based
|
|
115
|
+
turnsHad.set(id, had + 1);
|
|
116
|
+
if (byId.get(id)!.answerOnAttempt === attemptIndex) close(id, "answered");
|
|
117
|
+
};
|
|
118
|
+
|
|
119
|
+
const ESC_IN_FLIGHT = new Set<string>(); // mirrors the gateway's concurrency guard (no-op in a sync model)
|
|
120
|
+
|
|
121
|
+
let steps = 0;
|
|
122
|
+
const CAP = 10_000; // generous; a real infinite loop blows past this and fails
|
|
123
|
+
while (steps < CAP) {
|
|
124
|
+
steps++;
|
|
125
|
+
const open = ledger.hasOpen();
|
|
126
|
+
// Receive a fresh inbound (interleave: maybe receive while something is open,
|
|
127
|
+
// exercising multi-open). Always receive if nothing is open and work remains.
|
|
128
|
+
if (pending.length > 0 && (!open || r() < 0.5)) {
|
|
129
|
+
const m = pending.shift()!;
|
|
130
|
+
received.add(m.id);
|
|
131
|
+
// OPEN at receipt — keyed origin id; idempotent.
|
|
132
|
+
ledger.openIfAbsent({
|
|
133
|
+
originTurnId: m.id,
|
|
134
|
+
chatId: "-100123",
|
|
135
|
+
threadId: 3,
|
|
136
|
+
messageId: m.msgId,
|
|
137
|
+
text: `msg ${m.id}`,
|
|
138
|
+
openedAt: 1000 + steps,
|
|
139
|
+
});
|
|
140
|
+
deliverTurn(m.id); // original turn (attempt 0)
|
|
141
|
+
} else if (open) {
|
|
142
|
+
const decision = ledger.decideAtIdle();
|
|
143
|
+
const o = decision.obligation as Obligation;
|
|
144
|
+
// INVARIANT (no double-ask): a terminated obligation must never resurface.
|
|
145
|
+
expect(terminals.has(o.originTurnId)).toBe(false);
|
|
146
|
+
if (decision.action === "represent") {
|
|
147
|
+
ledger.markRepresented(o.originTurnId);
|
|
148
|
+
deliverTurn(o.originTurnId); // the re-present turn
|
|
149
|
+
} else if (decision.action === "escalate") {
|
|
150
|
+
if (ESC_IN_FLIGHT.has(o.originTurnId)) continue;
|
|
151
|
+
const attempt = ledger.markEscalateAttempt(o.originTurnId);
|
|
152
|
+
const willSucceed = byId.get(o.originTurnId)!.escalateFailsFor < attempt;
|
|
153
|
+
if (willSucceed) {
|
|
154
|
+
close(o.originTurnId, "escalation-delivered");
|
|
155
|
+
} else if (attempt >= ESCALATE_MAX) {
|
|
156
|
+
close(o.originTurnId, "escalation-give-up");
|
|
157
|
+
}
|
|
158
|
+
// else: transient failure — stays OPEN, retried next sweep.
|
|
159
|
+
}
|
|
160
|
+
} else {
|
|
161
|
+
break; // idle: nothing pending, nothing open → done
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
// Random restart: the durable snapshot is the only thing that survives.
|
|
165
|
+
// A fresh ledger hydrated from disk must resume exactly where we left off.
|
|
166
|
+
if (r() < 0.15) {
|
|
167
|
+
ledger = new ObligationLedger(MAX_REPRESENTS, {
|
|
168
|
+
onChange: (snap) => persistObligations(PATH, store.fs, snap),
|
|
169
|
+
});
|
|
170
|
+
ledger.hydrate(loadObligations(PATH, store.fs));
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
return { terminals, steps };
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
function pick<T>(arr: T[], r: () => number): T {
|
|
178
|
+
return arr[Math.floor(r() * arr.length)];
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
describe("obligation determinism — every inbound reaches a terminal, no silent loss, no double-ask", () => {
|
|
182
|
+
it("holds across 3000 random {model-behavior × timing × restart} schedules", () => {
|
|
183
|
+
const ANSWER = [0, 1, 2, 3, 99]; // 0..2 = answered via ladder; 3/99 = never → escalate
|
|
184
|
+
const ESCFAIL = [0, 1, 2, 3, 5]; // 0 = first send ok; ≥3 = permanently undeliverable
|
|
185
|
+
for (let seed = 1; seed <= 3000; seed++) {
|
|
186
|
+
const r = rng(seed * 7919);
|
|
187
|
+
const n = 1 + Math.floor(r() * 5); // 1..5 messages
|
|
188
|
+
const msgs: Msg[] = [];
|
|
189
|
+
for (let i = 0; i < n; i++) {
|
|
190
|
+
const msgId = seed * 100 + i; // real positive integer id
|
|
191
|
+
msgs.push({
|
|
192
|
+
id: `c:3#${msgId}`,
|
|
193
|
+
msgId,
|
|
194
|
+
answerOnAttempt: pick(ANSWER, r),
|
|
195
|
+
escalateFailsFor: pick(ESCFAIL, r),
|
|
196
|
+
});
|
|
197
|
+
}
|
|
198
|
+
const { terminals, steps } = runSchedule(msgs, seed * 104729);
|
|
199
|
+
|
|
200
|
+
// 1. TERMINATION: the engine settled well within the cap (no infinite loop).
|
|
201
|
+
expect(steps).toBeLessThan(10_000);
|
|
202
|
+
|
|
203
|
+
// 2. NO SILENT LOSS: every message received reached a terminal.
|
|
204
|
+
for (const m of msgs) {
|
|
205
|
+
const t = terminals.get(m.id);
|
|
206
|
+
expect(t, `seed=${seed} msg=${m.id} answer=${m.answerOnAttempt} escFail=${m.escalateFailsFor}`).toBeDefined();
|
|
207
|
+
|
|
208
|
+
// 3. CORRECT TERMINAL per behaviour:
|
|
209
|
+
if (m.answerOnAttempt <= MAX_REPRESENTS) {
|
|
210
|
+
// answerable within the represent ladder → answered (never escalated early)
|
|
211
|
+
expect(t).toBe("answered");
|
|
212
|
+
} else if (m.escalateFailsFor < ESCALATE_MAX) {
|
|
213
|
+
// never answered, escalation eventually lands
|
|
214
|
+
expect(t).toBe("escalation-delivered");
|
|
215
|
+
} else {
|
|
216
|
+
// never answered, escalation permanently undeliverable → bounded give-up
|
|
217
|
+
expect(t).toBe("escalation-give-up");
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
});
|
|
222
|
+
|
|
223
|
+
it("a delivered-but-unanswered obligation survives a restart and is escalated, not lost", () => {
|
|
224
|
+
// Deterministic single case: model NEVER answers, escalation succeeds first try,
|
|
225
|
+
// with a restart forced mid-life via a seed that triggers the 0.15 branch.
|
|
226
|
+
const { terminals } = runSchedule(
|
|
227
|
+
[{ id: "c:3#715", msgId: 715, answerOnAttempt: 99, escalateFailsFor: 0 }],
|
|
228
|
+
42,
|
|
229
|
+
);
|
|
230
|
+
expect(terminals.get("c:3#715")).toBe("escalation-delivered");
|
|
231
|
+
});
|
|
232
|
+
|
|
233
|
+
it("escalation that is permanently undeliverable is bounded (give-up), never an infinite loop", () => {
|
|
234
|
+
const { terminals, steps } = runSchedule(
|
|
235
|
+
[{ id: "c:3#900", msgId: 900, answerOnAttempt: 99, escalateFailsFor: 99 }],
|
|
236
|
+
7,
|
|
237
|
+
);
|
|
238
|
+
expect(terminals.get("c:3#900")).toBe("escalation-give-up");
|
|
239
|
+
expect(steps).toBeLessThan(10_000);
|
|
240
|
+
});
|
|
241
|
+
});
|
|
@@ -165,3 +165,72 @@ describe("buildObligationRepresentInbound", () => {
|
|
|
165
165
|
expect(obligationEscalationText(ob)).toMatch(/re-?send/i);
|
|
166
166
|
});
|
|
167
167
|
});
|
|
168
|
+
|
|
169
|
+
describe("ObligationLedger — durability hooks + escalate-attempt counter", () => {
|
|
170
|
+
function input(id: string, openedAt: number, text = "do the thing") {
|
|
171
|
+
return { originTurnId: id, chatId: "-100123", threadId: 3, messageId: Number(id.split("#").pop() ?? 0), text, openedAt };
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
it("fires onChange after every mutation with the full open snapshot", () => {
|
|
175
|
+
const snapshots: Obligation[][] = [];
|
|
176
|
+
const L = new ObligationLedger(2, { onChange: (s) => snapshots.push(s) });
|
|
177
|
+
L.openIfAbsent(input("c:3#1", 1000)); // open
|
|
178
|
+
L.openIfAbsent(input("c:3#2", 1001)); // open
|
|
179
|
+
L.markRepresented("c:3#1"); // represent
|
|
180
|
+
L.markEscalateAttempt("c:3#1"); // escalate-attempt
|
|
181
|
+
L.close("c:3#1"); // close
|
|
182
|
+
// open, open, represent, escalate-attempt, close = 5 mutations.
|
|
183
|
+
expect(snapshots.length).toBe(5);
|
|
184
|
+
expect(snapshots[1].map((o) => o.originTurnId).sort()).toEqual(["c:3#1", "c:3#2"]);
|
|
185
|
+
// last snapshot reflects the close.
|
|
186
|
+
expect(snapshots[4].map((o) => o.originTurnId)).toEqual(["c:3#2"]);
|
|
187
|
+
});
|
|
188
|
+
|
|
189
|
+
it("does NOT fire onChange for an idempotent (already-open) openIfAbsent", () => {
|
|
190
|
+
const snapshots: Obligation[][] = [];
|
|
191
|
+
const L = new ObligationLedger(2, { onChange: (s) => snapshots.push(s) });
|
|
192
|
+
expect(L.openIfAbsent(input("c:3#1", 1000))).toBe(true);
|
|
193
|
+
expect(L.openIfAbsent(input("c:3#1", 9999))).toBe(false); // dup
|
|
194
|
+
expect(snapshots.length).toBe(1);
|
|
195
|
+
});
|
|
196
|
+
|
|
197
|
+
it("does NOT fire onChange for a close of an unknown id", () => {
|
|
198
|
+
const snapshots: Obligation[][] = [];
|
|
199
|
+
const L = new ObligationLedger(2, { onChange: (s) => snapshots.push(s) });
|
|
200
|
+
expect(L.close("nope")).toBe(false);
|
|
201
|
+
expect(snapshots.length).toBe(0);
|
|
202
|
+
});
|
|
203
|
+
|
|
204
|
+
it("markEscalateAttempt increments per call and persists", () => {
|
|
205
|
+
const snapshots: Obligation[][] = [];
|
|
206
|
+
const L = new ObligationLedger(2, { onChange: (s) => snapshots.push(s) });
|
|
207
|
+
L.openIfAbsent(input("c:3#1", 1000));
|
|
208
|
+
expect(L.markEscalateAttempt("c:3#1")).toBe(1);
|
|
209
|
+
expect(L.markEscalateAttempt("c:3#1")).toBe(2);
|
|
210
|
+
expect(L.list()[0].escalateAttempts).toBe(2);
|
|
211
|
+
expect(L.markEscalateAttempt("missing")).toBe(0);
|
|
212
|
+
});
|
|
213
|
+
|
|
214
|
+
it("hydrate restores the open set WITH counters and does not fire onChange", () => {
|
|
215
|
+
const snapshots: Obligation[][] = [];
|
|
216
|
+
const L = new ObligationLedger(2, { onChange: (s) => snapshots.push(s) });
|
|
217
|
+
L.hydrate([
|
|
218
|
+
{ originTurnId: "c:3#715", chatId: "-100123", threadId: 3, messageId: 715, text: "x", openedAt: 1000, representCount: 2, escalateAttempts: 1 },
|
|
219
|
+
]);
|
|
220
|
+
expect(snapshots.length).toBe(0); // hydrate is restoration, not a mutation
|
|
221
|
+
expect(L.isOpen("c:3#715")).toBe(true);
|
|
222
|
+
expect(L.list()[0].representCount).toBe(2);
|
|
223
|
+
expect(L.list()[0].escalateAttempts).toBe(1);
|
|
224
|
+
// a represented obligation at/over max decides 'escalate', preserving count across restart
|
|
225
|
+
expect(L.decideAtIdle().action).toBe("escalate");
|
|
226
|
+
});
|
|
227
|
+
|
|
228
|
+
it("hydrate skips malformed rows", () => {
|
|
229
|
+
const L = new ObligationLedger();
|
|
230
|
+
L.hydrate([
|
|
231
|
+
{ originTurnId: "c:3#1", chatId: "-100123", messageId: 1, text: "x", openedAt: 1000, representCount: 0 },
|
|
232
|
+
{ originTurnId: "", chatId: "x", messageId: 0, text: "", openedAt: 0, representCount: 0 } as Obligation,
|
|
233
|
+
]);
|
|
234
|
+
expect(L.size()).toBe(1);
|
|
235
|
+
});
|
|
236
|
+
});
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
import { describe, it, expect } from "vitest";
|
|
2
|
+
import {
|
|
3
|
+
loadObligations,
|
|
4
|
+
persistObligations,
|
|
5
|
+
type ObligationStoreFsSeam,
|
|
6
|
+
} from "../gateway/obligation-store.js";
|
|
7
|
+
import type { Obligation } from "../gateway/obligation-ledger.js";
|
|
8
|
+
|
|
9
|
+
/** In-memory fs seam with an atomic rename, so the store's tmp→rename
|
|
10
|
+
* crash-safety contract is exercised without touching the real disk. */
|
|
11
|
+
function memFs(seed: Record<string, string> = {}) {
|
|
12
|
+
const files = new Map<string, string>(Object.entries(seed));
|
|
13
|
+
const calls: string[] = [];
|
|
14
|
+
const fs: ObligationStoreFsSeam = {
|
|
15
|
+
readFileSync: (p) => {
|
|
16
|
+
if (!files.has(p)) throw new Error(`ENOENT ${p}`);
|
|
17
|
+
return files.get(p)!;
|
|
18
|
+
},
|
|
19
|
+
writeFileSync: (p, d) => {
|
|
20
|
+
calls.push(`write:${p}`);
|
|
21
|
+
files.set(p, d);
|
|
22
|
+
},
|
|
23
|
+
renameSync: (a, b) => {
|
|
24
|
+
calls.push(`rename:${a}->${b}`);
|
|
25
|
+
if (!files.has(a)) throw new Error(`ENOENT ${a}`);
|
|
26
|
+
files.set(b, files.get(a)!);
|
|
27
|
+
files.delete(a);
|
|
28
|
+
},
|
|
29
|
+
existsSync: (p) => files.has(p),
|
|
30
|
+
};
|
|
31
|
+
return { fs, files, calls };
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
const PATH = "/state/agent/telegram/obligations.json";
|
|
35
|
+
|
|
36
|
+
function ob(id: string, over: Partial<Obligation> = {}): Obligation {
|
|
37
|
+
return {
|
|
38
|
+
originTurnId: id,
|
|
39
|
+
chatId: "-100123",
|
|
40
|
+
threadId: 3,
|
|
41
|
+
messageId: Number(id.split("#").pop() ?? 1),
|
|
42
|
+
text: "do the thing",
|
|
43
|
+
openedAt: 1000,
|
|
44
|
+
representCount: 0,
|
|
45
|
+
...over,
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
describe("obligation-store", () => {
|
|
50
|
+
it("round-trips the open set, preserving representCount + escalateAttempts", () => {
|
|
51
|
+
const { fs } = memFs();
|
|
52
|
+
const snap: Obligation[] = [
|
|
53
|
+
ob("c:3#715", { representCount: 2, escalateAttempts: 1 }),
|
|
54
|
+
ob("c:5#900", { representCount: 0, openedAt: 2000 }),
|
|
55
|
+
];
|
|
56
|
+
persistObligations(PATH, fs, snap);
|
|
57
|
+
const loaded = loadObligations(PATH, fs);
|
|
58
|
+
expect(loaded).toEqual(snap);
|
|
59
|
+
expect(loaded[0].escalateAttempts).toBe(1);
|
|
60
|
+
expect(loaded[0].representCount).toBe(2);
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
it("persists atomically: writes a sibling .tmp then renames over the path", () => {
|
|
64
|
+
const { fs, calls, files } = memFs();
|
|
65
|
+
persistObligations(PATH, fs, [ob("c:3#1")]);
|
|
66
|
+
expect(calls).toEqual([`write:${PATH}.tmp`, `rename:${PATH}.tmp->${PATH}`]);
|
|
67
|
+
// The tmp is gone (renamed); only the real path remains.
|
|
68
|
+
expect(files.has(PATH)).toBe(true);
|
|
69
|
+
expect(files.has(`${PATH}.tmp`)).toBe(false);
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
it("returns [] for a missing file", () => {
|
|
73
|
+
const { fs } = memFs();
|
|
74
|
+
expect(loadObligations(PATH, fs)).toEqual([]);
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
it("returns [] for a torn / non-JSON file (crash mid-write tolerance)", () => {
|
|
78
|
+
const { fs } = memFs({ [PATH]: '{"v":1,"obligations":[{"originTurnId":"c:3#7' });
|
|
79
|
+
expect(loadObligations(PATH, fs)).toEqual([]);
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
it("returns [] for a wrong-version or wrong-shape envelope", () => {
|
|
83
|
+
const a = memFs({ [PATH]: JSON.stringify({ v: 2, obligations: [ob("c:3#1")] }) });
|
|
84
|
+
expect(loadObligations(PATH, a.fs)).toEqual([]);
|
|
85
|
+
const b = memFs({ [PATH]: JSON.stringify({ v: 1, obligations: "nope" }) });
|
|
86
|
+
expect(loadObligations(PATH, b.fs)).toEqual([]);
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
it("filters out malformed rows but keeps valid ones", () => {
|
|
90
|
+
const raw = JSON.stringify({
|
|
91
|
+
v: 1,
|
|
92
|
+
obligations: [
|
|
93
|
+
ob("c:3#715"),
|
|
94
|
+
{ originTurnId: "", chatId: "x" }, // empty id → dropped
|
|
95
|
+
{ nope: true }, // missing fields → dropped
|
|
96
|
+
ob("c:5#900", { openedAt: 2000 }),
|
|
97
|
+
],
|
|
98
|
+
});
|
|
99
|
+
const { fs } = memFs({ [PATH]: raw });
|
|
100
|
+
const loaded = loadObligations(PATH, fs);
|
|
101
|
+
expect(loaded.map((o) => o.originTurnId)).toEqual(["c:3#715", "c:5#900"]);
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
it("never throws on a write failure — degrades to in-memory (logs)", () => {
|
|
105
|
+
const logs: string[] = [];
|
|
106
|
+
const fs: ObligationStoreFsSeam = {
|
|
107
|
+
readFileSync: () => "",
|
|
108
|
+
writeFileSync: () => {
|
|
109
|
+
throw new Error("EROFS read-only fs");
|
|
110
|
+
},
|
|
111
|
+
renameSync: () => {},
|
|
112
|
+
existsSync: () => false,
|
|
113
|
+
};
|
|
114
|
+
expect(() => persistObligations(PATH, fs, [ob("c:3#1")], (l) => logs.push(l))).not.toThrow();
|
|
115
|
+
expect(logs.join("")).toContain("persist FAILED");
|
|
116
|
+
});
|
|
117
|
+
});
|