switchroom 0.14.62 → 0.14.64

package/telegram-plugin/tests/escalation-drive.test.ts

@@ -0,0 +1,123 @@
+import { describe, it, expect } from "vitest";
+import { driveEscalation } from "../gateway/escalation-drive.js";
+import { ObligationLedger } from "../gateway/obligation-ledger.js";
+
+// Drives the REAL escalation step (the code obligationSweep calls) with the REAL
+// ObligationLedger and the REAL withDeadline — including a fake hanging send,
+// the exact path the total proof flagged and that mtcute / a synchronous test
+// cannot reach. This is the executable verification of the hang-wedge fix.
+
+function openEscalatable(L: ObligationLedger, id: string) {
+  L.openIfAbsent({ originTurnId: id, chatId: "-100", threadId: 3, messageId: 1, text: "x", openedAt: 0 });
+}
+
+const MAX = 3;
+const DEADLINE = 15; // ms — short so the hang case settles fast and deterministically
+
+describe("driveEscalation — the obligation escalation step is bounded and always reaches a terminal", () => {
+  it("a successful send closes the obligation and clears the in-flight flag", async () => {
+    const L = new ObligationLedger(2);
+    openEscalatable(L, "c#1");
+    const inFlight = new Set<string>();
+    await driveEscalation({
+      escId: "c#1",
+      inFlight,
+      ledger: L,
+      send: () => Promise.resolve("sent"),
+      maxAttempts: MAX,
+      deadlineMs: DEADLINE,
+      log: () => {},
+    });
+    expect(L.isOpen("c#1")).toBe(false); // closed
+    expect(inFlight.has("c#1")).toBe(false); // flag cleared
+  });
+
+  it("a transient failure below the cap stays OPEN and clears the flag (retried next sweep)", async () => {
+    const L = new ObligationLedger(2);
+    openEscalatable(L, "c#1");
+    const inFlight = new Set<string>();
+    await driveEscalation({
+      escId: "c#1",
+      inFlight,
+      ledger: L,
+      send: () => Promise.reject(new Error("network blip")),
+      maxAttempts: MAX,
+      deadlineMs: DEADLINE,
+      log: () => {},
+    });
+    expect(L.isOpen("c#1")).toBe(true); // still open — will retry
+    expect(inFlight.has("c#1")).toBe(false); // flag cleared, so the next sweep can re-enter
+  });
+
+  it("THE FIX: a send that NEVER settles still clears the flag (bounded by the deadline)", async () => {
+    const L = new ObligationLedger(2);
+    openEscalatable(L, "c#1");
+    const inFlight = new Set<string>();
+    let sendInvoked = 0;
+    const start = Date.now();
+    // A promise that never resolves/rejects — the stalled send that, pre-fix,
+    // left the in-flight flag set forever and wedged the obligation OPEN.
+    await driveEscalation({
+      escId: "c#1",
+      inFlight,
+      ledger: L,
+      send: () => {
+        sendInvoked++;
+        return new Promise(() => {});
+      },
+      maxAttempts: MAX,
+      deadlineMs: DEADLINE,
+      log: () => {},
+    });
+    expect(sendInvoked).toBe(1);
+    expect(inFlight.has("c#1")).toBe(false); // cleared despite the hang — the wedge is gone
+    expect(Date.now() - start).toBeLessThan(DEADLINE + 500); // settled at the deadline, not "never"
+  });
+
+  it("repeated hung sends reach a bounded terminal (close best-effort), never an infinite loop", async () => {
+    const L = new ObligationLedger(2);
+    openEscalatable(L, "c#1");
+    const inFlight = new Set<string>();
+    let sends = 0;
+    let drives = 0;
+    // Simulate the 5s sweep firing repeatedly while every send hangs.
+    while (L.isOpen("c#1") && drives < 20) {
+      drives++;
+      const p = driveEscalation({
+        escId: "c#1",
+        inFlight,
+        ledger: L,
+        send: () => {
+          sends++;
+          return new Promise(() => {});
+        },
+        maxAttempts: MAX,
+        deadlineMs: DEADLINE,
+        log: () => {},
+      });
+      if (p) await p; // each attempt settles within the deadline
+    }
+    expect(L.isOpen("c#1")).toBe(false); // reached a terminal (closed best-effort)
+    expect(inFlight.has("c#1")).toBe(false);
+    expect(sends).toBe(MAX); // exactly maxAttempts sends, then close — bounded
+    expect(drives).toBeLessThanOrEqual(MAX + 1);
+  });
+
+  it("the in-flight guard prevents a concurrent second send for the same obligation", async () => {
+    const L = new ObligationLedger(2);
+    openEscalatable(L, "c#1");
+    const inFlight = new Set<string>();
+    let sends = 0;
+    const hang = () => {
+      sends++;
+      return new Promise<void>(() => {});
+    };
+    const p1 = driveEscalation({ escId: "c#1", inFlight, ledger: L, send: hang, maxAttempts: MAX, deadlineMs: 60, log: () => {} });
+    // Second call while the first is still awaiting → must be a no-op.
+    const p2 = driveEscalation({ escId: "c#1", inFlight, ledger: L, send: hang, maxAttempts: MAX, deadlineMs: 60, log: () => {} });
+    expect(p2).toBeUndefined(); // guarded
+    expect(sends).toBe(1); // only one send fired
+    expect(L.list()[0].escalateAttempts).toBe(1); // only one attempt recorded
+    await p1; // let the first settle so we don't leak a pending timer
+  });
+});

package/telegram-plugin/tests/obligation-determinism.test.ts

@@ -89,12 +89,18 @@ interface Sim {
   steps: number;
 }
 
-function runSchedule(msgs: Msg[], seed: number): Sim {
+function runSchedule(msgs: Msg[], seed: number, graceMs = 0): Sim {
   const PATH = "/state/agent/telegram/obligations.json";
   const store = memStore();
   let ledger = new ObligationLedger(MAX_REPRESENTS, {
     onChange: (snap) => persistObligations(PATH, store.fs, snap),
   });
+  // Virtual monotonic clock (only meaningful when graceMs>0). Advances every
+  // step by more than one sweep tick so the grace window deterministically
+  // expires within the step budget — proving grace DELAYS but never PREVENTS a
+  // terminal (no livelock).
+  let clock = 1_000_000;
+  const SWEEP_TICK = 5_000;
   const r = rng(seed);
 
   const pending = [...msgs]; // not yet received
@@ -109,11 +115,18 @@ function runSchedule(msgs: Msg[], seed: number): Sim {
   };
 
   // Run one turn for an obligation; close if the model answers on this attempt.
+  // If it does NOT answer and grace is on, stamp the turn-end clock (mirrors the
+  // gateway's endCurrentTurnAtomic !finalAnswerDelivered branch) so the next
+  // decideAtIdle({now, graceMs}) waits out the grace before re-presenting.
   const deliverTurn = (id: string) => {
     const had = (turnsHad.get(id) ?? 0);
     const attemptIndex = had; // 0-based
     turnsHad.set(id, had + 1);
-    if (byId.get(id)!.answerOnAttempt === attemptIndex) close(id, "answered");
+    if (byId.get(id)!.answerOnAttempt === attemptIndex) {
+      close(id, "answered");
+    } else if (graceMs > 0 && ledger.isOpen(id)) {
+      ledger.noteTurnEnded(id, clock);
+    }
   };
 
   const ESC_IN_FLIGHT = new Set<string>(); // mirrors the gateway's concurrency guard (no-op in a sync model)
@@ -139,7 +152,14 @@ function runSchedule(msgs: Msg[], seed: number): Sim {
       });
       deliverTurn(m.id); // original turn (attempt 0)
     } else if (open) {
-      const decision = ledger.decideAtIdle();
+      const decision =
+        graceMs > 0 ? ledger.decideAtIdle({ now: clock, graceMs }) : ledger.decideAtIdle();
+      if (decision.action === "none") {
+        // Every open obligation is within its grace window — the sweep waits.
+        // Advance the clock so grace deterministically expires; no livelock.
+        clock += SWEEP_TICK;
+        continue;
+      }
       const o = decision.obligation as Obligation;
       // INVARIANT (no double-ask): a terminated obligation must never resurface.
       expect(terminals.has(o.originTurnId)).toBe(false);
@@ -169,6 +189,9 @@ function runSchedule(msgs: Msg[], seed: number): Sim {
       });
       ledger.hydrate(loadObligations(PATH, store.fs));
     }
+    // Advance the virtual clock every step so any stamped grace window
+    // deterministically expires within the step budget.
+    clock += SWEEP_TICK;
   }
 
   return { terminals, steps };
@@ -220,6 +243,43 @@ describe("obligation determinism — every inbound reaches a terminal, no silent
     }
   });
 
+  it("holds across 3000 schedules WITH the escalate-grace window on (grace delays, never prevents a terminal)", () => {
+    const ANSWER = [0, 1, 2, 3, 99];
+    const ESCFAIL = [0, 1, 2, 3, 5];
+    const GRACE_MS = 45_000;
+    for (let seed = 1; seed <= 3000; seed++) {
+      const r = rng(seed * 7919);
+      const n = 1 + Math.floor(r() * 5);
+      const msgs: Msg[] = [];
+      for (let i = 0; i < n; i++) {
+        const msgId = seed * 100 + i;
+        msgs.push({
+          id: `c:3#${msgId}`,
+          msgId,
+          answerOnAttempt: pick(ANSWER, r),
+          escalateFailsFor: pick(ESCFAIL, r),
+        });
+      }
+      // Same enumeration as the no-grace proof, but the ledger now runs the grace
+      // path: every non-answering turn stamps noteTurnEnded and the sweep waits
+      // out the window before acting. The terminal each message reaches must be
+      // IDENTICAL to the no-grace run — grace only delays.
+      const { terminals, steps } = runSchedule(msgs, seed * 104729, GRACE_MS);
+      expect(steps).toBeLessThan(10_000); // still terminates (no grace livelock)
+      for (const m of msgs) {
+        const t = terminals.get(m.id);
+        expect(t, `grace seed=${seed} msg=${m.id} answer=${m.answerOnAttempt} escFail=${m.escalateFailsFor}`).toBeDefined();
+        if (m.answerOnAttempt <= MAX_REPRESENTS) {
+          expect(t).toBe("answered");
+        } else if (m.escalateFailsFor < ESCALATE_MAX) {
+          expect(t).toBe("escalation-delivered");
+        } else {
+          expect(t).toBe("escalation-give-up");
+        }
+      }
+    }
+  });
+
   it("a delivered-but-unanswered obligation survives a restart and is escalated, not lost", () => {
     // Deterministic single case: model NEVER answers, escalation succeeds first try,
     // with a restart forced mid-life via a seed that triggers the 0.15 branch.

package/telegram-plugin/tests/obligation-ledger.test.ts

@@ -225,6 +225,20 @@ describe("ObligationLedger — durability hooks + escalate-attempt counter", ()
     expect(L.decideAtIdle().action).toBe("escalate");
   });
 
+  it("interrupt-cancel semantics: closing the in-flight turn's obligation removes it from the sweep, sibling untouched", () => {
+    // Mirrors cancelInterruptedObligation: an `!` interrupt SIGINT-kills the
+    // in-flight turn and closes its obligation, so the sweep can't later
+    // re-present/escalate the question the user explicitly redirected away from.
+    // A queued SIBLING obligation must survive.
+    const L = new ObligationLedger();
+    L.openIfAbsent(input("c:3#700", 1000)); // the in-flight turn's message
+    L.openIfAbsent(input("c:3#701", 1001)); // a queued sibling
+    expect(L.close("c:3#700")).toBe(true); // interrupt cancels the in-flight one
+    expect(L.isOpen("c:3#700")).toBe(false);
+    expect(L.decideAtIdle().obligation?.originTurnId).toBe("c:3#701"); // sibling still actionable
+    expect(L.close("c:3#700")).toBe(false); // re-close / unknown is a safe no-op
+  });
+
   it("hydrate skips malformed rows", () => {
     const L = new ObligationLedger();
     L.hydrate([
@@ -234,3 +248,81 @@ describe("ObligationLedger — durability hooks + escalate-attempt counter", ()
     expect(L.size()).toBe(1);
   });
 });
+
+describe("ObligationLedger — escalate-grace window (over-escalation fix)", () => {
+  function input(id: string, openedAt: number) {
+    return { originTurnId: id, chatId: "-100123", threadId: 3, messageId: Number(id.split("#").pop() ?? 0), text: "x", openedAt };
+  }
+
+  it("no opts (or graceMs<=0) → pre-grace behaviour: acts immediately", () => {
+    const L = new ObligationLedger();
+    L.openIfAbsent(input("c:3#1", 1000));
+    L.noteTurnEnded("c:3#1", 5000); // a turn just ended
+    expect(L.decideAtIdle().action).toBe("represent"); // no grace → act now
+    expect(L.decideAtIdle({ now: 5001, graceMs: 0 }).action).toBe("represent"); // graceMs 0 → off
+  });
+
+  it("skips an obligation whose turn ended < graceMs ago (the trailing-answer window)", () => {
+    const L = new ObligationLedger();
+    L.openIfAbsent(input("c:3#1", 1000));
+    L.noteTurnEnded("c:3#1", 5000);
+    // 30s after turn-end, grace 45s → still within grace → wait (none)
+    expect(L.decideAtIdle({ now: 35000, graceMs: 45000 }).action).toBe("none");
+    // 46s after turn-end → out of grace → act
+    expect(L.decideAtIdle({ now: 51000, graceMs: 45000 }).action).toBe("represent");
+  });
+
+  it("an obligation that never had a turn end (still-queued) is always eligible — no trailing answer to wait for", () => {
+    const L = new ObligationLedger();
+    L.openIfAbsent(input("c:3#1", 1000)); // no noteTurnEnded
+    expect(L.decideAtIdle({ now: 1001, graceMs: 45000 }).action).toBe("represent");
+  });
+
+  it("picks the oldest ELIGIBLE: a newer in-grace obligation does not block an older out-of-grace one", () => {
+    const L = new ObligationLedger();
+    L.openIfAbsent(input("c:3#1", 1000)); // older
+    L.openIfAbsent(input("c:3#2", 2000)); // newer
+    L.noteTurnEnded("c:3#1", 5000); // older's turn ended at 5000 (out of grace by now)
+    L.noteTurnEnded("c:3#2", 60000); // newer's turn ended at 60000 (in grace)
+    const d = L.decideAtIdle({ now: 70000, graceMs: 45000 });
+    // older (#1) ended 65s ago → eligible; newer (#2) ended 10s ago → in grace.
+    // pick oldest eligible = #1.
+    expect(d.action).toBe("represent");
+    expect(d.obligation?.originTurnId).toBe("c:3#1");
+  });
+
+  it("returns none when EVERY open obligation is within grace", () => {
+    const L = new ObligationLedger();
+    L.openIfAbsent(input("c:3#1", 1000));
+    L.openIfAbsent(input("c:3#2", 2000));
+    L.noteTurnEnded("c:3#1", 60000);
+    L.noteTurnEnded("c:3#2", 61000);
+    expect(L.decideAtIdle({ now: 65000, graceMs: 45000 }).action).toBe("none");
+  });
+
+  it("noteTurnEnded is a no-op for an unknown/closed obligation and persists when it applies", () => {
+    const snapshots: Obligation[][] = [];
+    const L = new ObligationLedger(2, { onChange: (s) => snapshots.push(s) });
+    L.openIfAbsent(input("c:3#1", 1000)); // persist #1
+    L.noteTurnEnded("nope", 5000); // unknown → no persist
+    expect(snapshots.length).toBe(1);
+    L.noteTurnEnded("c:3#1", 5000); // applies → persist
+    expect(snapshots.length).toBe(2);
+    expect(snapshots[1][0].lastTurnEndedAt).toBe(5000);
+  });
+
+  it("grace still terminates the ladder: represent → escalate once each is out of grace", () => {
+    const L = new ObligationLedger(2);
+    L.openIfAbsent(input("c:3#1", 1000));
+    // turn 0 ended at 1000; out of grace by t=50000
+    L.noteTurnEnded("c:3#1", 1000);
+    expect(L.decideAtIdle({ now: 50000, graceMs: 45000 }).action).toBe("represent");
+    L.markRepresented("c:3#1");
+    L.noteTurnEnded("c:3#1", 50000); // re-present turn ended
+    expect(L.decideAtIdle({ now: 96000, graceMs: 45000 }).action).toBe("represent");
+    L.markRepresented("c:3#1");
+    L.noteTurnEnded("c:3#1", 96000);
+    // representCount now 2 == max → escalate (once out of grace)
+    expect(L.decideAtIdle({ now: 142000, graceMs: 45000 }).action).toBe("escalate");
+  });
+});