switchroom 0.19.22 → 0.19.23

package/telegram-plugin/tests/captured-answer-resume.test.ts

@@ -271,6 +271,110 @@ describe('#3282 createCapturedResumeDispatcher — the represent RESUME outcome'
     expect(oblLedger.isOpen('#t1')).toBe(false)
   })
 
+  // #3702 L3 — the close condition is LANDED, not confirmed. A snapshot whose
+  // chunks all landed but were NEVER corroborated (the read-back is 100% shed in
+  // production, #3703) is resolved on HYDRATION alone: the resume re-probes,
+  // resolves nothing again, sends zero messages, and closes.
+  //
+  // This is the case the durable outbound-text oracle CANNOT reach — history
+  // disabled here, so `hasOutboundWithText` never fires and the old
+  // confirmation-gated verdict would have kept the obligation OPEN, re-running
+  // the same shed probe on every sweep with no send available to it, until the
+  // represent cap escalated a false "never answered you" to the operator.
+  it('fully-landed-but-UNCONFIRMED snapshot, no oracle ⇒ ZERO sends and the obligation CLOSES', async () => {
+    const chunks = ['c0', 'c1']
+    const snapshot = {
+      chunks,
+      chunkStates: [
+        { index: 0, messageIds: [5000], confirmed: false },
+        { index: 1, messageIds: [5001], confirmed: false },
+      ],
+    }
+    const resumeLedger = new BackstopDeliveryLedger()
+    const sentIdx: number[] = []
+    const oblLedger = new ObligationLedger(2)
+    oblLedger.openIfAbsent(obligation({ capturedDelivery: snapshot }))
+    oblLedger.noteCapturedDelivery('#t1', snapshot)
+
+    // No durable corroboration is available from EITHER source: the snapshot
+    // flags are false, history is off, and the re-probe sheds (ambiguous).
+    mockOracle = () => false
+
+    const lines: string[] = []
+    const dispatcher = createCapturedResumeDispatcher({
+      deliverAnswer: async (a) => {
+        a.resume.hydrate(resumeLedger, a.turnId)
+        const res = await runBackstopDelivery(
+          resumeLedger, a.turnId, a.resume.snapshot.chunks, a.cardMessageId,
+          {
+            sendChunk: async (i: number) => { sentIdx.push(i); return [7000 + i] },
+            readBack: async (): Promise<ReadBackResult> => 'ambiguous',
+          },
+          3,
+        )
+        expect(res.confirmed).toBe(false) // nothing corroborated it, at any point
+        expect(res.landedUnconfirmedIds).toEqual([5000, 5001]) // ...and it is counted
+        return { delivered: res.delivered, sentIds: res.sentIds }
+      },
+      obligationLedger: oblLedger,
+      backstopDeliveryLedger: resumeLedger,
+      flushedTurnSupersede: { record: () => {} },
+      historyEnabled: false,
+      stderr: (s) => lines.push(s),
+    })
+    dispatcher.dispatch(oblLedger.list()[0])
+    await flush(dispatcher)
+
+    expect(sentIdx).toEqual([]) // nothing re-posted — the user never sees a duplicate
+    expect(oblLedger.isOpen('#t1')).toBe(false) // closed on the landed-id evidence
+    // ...and the log must say what actually happened. These ids are LANDED, not
+    // confirmed: calling them "confirmed" is the overstatement #3702 removed from
+    // the verdict, and a log that reintroduces it re-lies to the next debugger.
+    const closed = lines.find((l) => l.includes('resume delivered'))
+    expect(closed).toBeDefined()
+    expect(closed).toContain('2 message id(s) landed')
+    expect(closed).not.toContain('confirmed')
+  })
+
+  // The other half of that decision: a positive ABSENCE still re-opens the send
+  // path, so "landed" is not a rubber stamp. (Inert in production until #3703
+  // wakes the probe, but the mechanism must remain correct.)
+  it('a landed chunk the re-probe finds ABSENT is demoted and RE-SENT, not closed on the stale id', async () => {
+    const chunks = ['c0']
+    const snapshot = { chunks, chunkStates: [{ index: 0, messageIds: [5000], confirmed: false }] }
+    const resumeLedger = new BackstopDeliveryLedger()
+    const sentIdx: number[] = []
+    const oblLedger = new ObligationLedger(2)
+    oblLedger.openIfAbsent(obligation({ capturedDelivery: snapshot }))
+    oblLedger.noteCapturedDelivery('#t1', snapshot)
+    mockOracle = () => false
+
+    let probes = 0
+    const dispatcher = createCapturedResumeDispatcher({
+      deliverAnswer: async (a) => {
+        a.resume.hydrate(resumeLedger, a.turnId)
+        const res = await runBackstopDelivery(
+          resumeLedger, a.turnId, a.resume.snapshot.chunks, a.cardMessageId,
+          {
+            sendChunk: async (i: number) => { sentIdx.push(i); return [7000 + i] },
+            readBack: async (): Promise<ReadBackResult> => (++probes === 1 ? 'absent' : 'exists'),
+          },
+          3,
+        )
+        return { delivered: res.delivered, sentIds: res.sentIds }
+      },
+      obligationLedger: oblLedger,
+      backstopDeliveryLedger: resumeLedger,
+      flushedTurnSupersede: { record: () => {} },
+      historyEnabled: false,
+    })
+    dispatcher.dispatch(oblLedger.list()[0])
+    await flush(dispatcher)
+
+    expect(sentIdx).toEqual([0]) // the silently-dropped chunk IS re-sent
+    expect(oblLedger.isOpen('#t1')).toBe(false)
+  })
+
   it('a partial RESUME (tail still fails) leaves the obligation OPEN + consumes represent budget', async () => {
     const chunks = ['c0', 'c1']
     const snapshot = {

package/telegram-plugin/tests/turns-jsonl-rotate.test.ts

@@ -1,6 +1,22 @@
+import { readFileSync } from 'node:fs'
+import { resolve } from 'node:path'
+import { fileURLToPath } from 'node:url'
+
 import { describe, expect, it, vi } from 'vitest'
 
-import { maybeRotate, TURNS_JSONL_MAX_BYTES, type RotateFs } from '../gateway/turns-jsonl-rotate.js'
+import {
+  maybeRotate,
+  resolveAgentStateDir,
+  resolveTurnsJsonlPath,
+  DEFAULT_AGENT_STATE_DIR,
+  TURNS_JSONL_MAX_BYTES,
+  type RotateFs,
+} from '../gateway/turns-jsonl-rotate.js'
+
+const GATEWAY_SRC = readFileSync(
+  resolve(fileURLToPath(new URL('.', import.meta.url)), '..', 'gateway', 'gateway.ts'),
+  'utf-8',
+)
 
 describe('maybeRotate — turns.jsonl size cap', () => {
   const mkFs = (size: number | undefined) => {
@@ -37,3 +53,78 @@ describe('maybeRotate — turns.jsonl size cap', () => {
     expect(rename).toHaveBeenNthCalledWith(2, '/a/turns.jsonl', '/a/turns.jsonl.1')
   })
 })
+
+describe('resolveTurnsJsonlPath — the turn record must follow the state dir', () => {
+  // Regression: the path was hard-coded to `/state/agent/turns.jsonl`, so a test
+  // that isolated TELEGRAM_STATE_DIR / SWITCHROOM_AGENT_STATE_DIR into a tmpdir
+  // still appended its synthetic turn rows into the PRODUCTION turn record of
+  // whichever agent container it ran in — which the fleet-health L0 sensor then
+  // scored as that agent's real production failures.
+  it('honours SWITCHROOM_AGENT_STATE_DIR', () => {
+    expect(resolveTurnsJsonlPath({ SWITCHROOM_AGENT_STATE_DIR: '/tmp/iso-123' })).toBe(
+      '/tmp/iso-123/turns.jsonl',
+    )
+  })
+
+  it('strips a trailing slash rather than doubling it', () => {
+    expect(resolveTurnsJsonlPath({ SWITCHROOM_AGENT_STATE_DIR: '/tmp/iso-123/' })).toBe(
+      '/tmp/iso-123/turns.jsonl',
+    )
+  })
+
+  it('falls back to the container default when unset or blank', () => {
+    expect(resolveTurnsJsonlPath({})).toBe(`${DEFAULT_AGENT_STATE_DIR}/turns.jsonl`)
+    expect(resolveTurnsJsonlPath({ SWITCHROOM_AGENT_STATE_DIR: '   ' })).toBe(
+      `${DEFAULT_AGENT_STATE_DIR}/turns.jsonl`,
+    )
+  })
+})
+
+describe('resolveAgentStateDir — ONE normaliser for every writer in the state dir', () => {
+  // The gateway had two writers into this dir 26 lines apart reading the env
+  // var two different ways: the turn record via the normaliser, the
+  // context-occupancy snapshot via a bare `process.env.X ?? '/state/agent'`.
+  // For a value like `/x/ ` those resolve to different directories, so the two
+  // artifacts of the same turn land in two places.
+  it('trims, strips a trailing slash, and treats blank as unset', () => {
+    expect(resolveAgentStateDir({ SWITCHROOM_AGENT_STATE_DIR: '/x' })).toBe('/x')
+    expect(resolveAgentStateDir({ SWITCHROOM_AGENT_STATE_DIR: '  /x/  ' })).toBe('/x')
+    expect(resolveAgentStateDir({ SWITCHROOM_AGENT_STATE_DIR: '/x///' })).toBe('/x')
+    expect(resolveAgentStateDir({ SWITCHROOM_AGENT_STATE_DIR: '  ' })).toBe(DEFAULT_AGENT_STATE_DIR)
+    expect(resolveAgentStateDir({})).toBe(DEFAULT_AGENT_STATE_DIR)
+  })
+
+  it('the turn record and the state dir agree for any messy value', () => {
+    const env = { SWITCHROOM_AGENT_STATE_DIR: ' /tmp/iso-9/ ' }
+    expect(resolveTurnsJsonlPath(env)).toBe(`${resolveAgentStateDir(env)}/turns.jsonl`)
+  })
+})
+
+describe('gateway.ts call sites — the defect site itself, not just the helper', () => {
+  // The bug was a hard-coded literal at the CALLSITE. A unit test of the pure
+  // helper passes with the literal re-inlined, so these read the real source.
+  // Source-text assertions are the repo's cheap deterministic pattern for
+  // pinning a callsite (see per-topic-current-turn.test.ts).
+  it('emitTurnRecord resolves the path — the hard-coded literal is gone', () => {
+    const body = GATEWAY_SRC.split('function emitTurnRecord(')[1]?.split('\n}')[0] ?? ''
+    expect(body.length).toBeGreaterThan(50)
+    expect(body).toMatch(/const turnsPath = resolveTurnsJsonlPath\(\)/)
+    // The append + the rotate both use the RESOLVED path, not a literal.
+    expect(body).toMatch(/appendFileSync\(turnsPath, /)
+    expect(body).toMatch(/maybeRotate\(turnsPath, /)
+    expect(body).not.toMatch(/turns\.jsonl/)
+  })
+
+  it('no hard-coded turns.jsonl path survives anywhere in gateway.ts', () => {
+    const hits = GATEWAY_SRC.match(/['"`][^'"`\n]*\/turns\.jsonl['"`]/g) ?? []
+    expect(hits, `hard-coded turns.jsonl path(s): ${JSON.stringify(hits)}`).toEqual([])
+  })
+
+  it('every state-dir read in gateway.ts goes through resolveAgentStateDir', () => {
+    // Consistency, not style: a second inline `process.env.X ?? '/state/agent'`
+    // silently opts that writer out of trim + trailing-slash normalisation.
+    const inline = GATEWAY_SRC.match(/process\.env\.SWITCHROOM_AGENT_STATE_DIR/g) ?? []
+    expect(inline, `inline state-dir read(s) in gateway.ts: ${inline.length}`).toEqual([])
+    expect(GATEWAY_SRC).toMatch(/resolveAgentStateDir\(\)/)
+  })
+})

package/vendor/hindsight-memory/scripts/drain_pending.py

@@ -5,8 +5,12 @@ SessionStart calls into ``drain()`` to retry any retain payloads that
 ``session_end.py`` queued on failure (#1071). Each entry is retried up
 to ``MAX_ATTEMPTS`` (5) times; after that a **permanently** failing entry
 (a 4xx that a re-POST cannot fix — see ``pending.is_permanent_failure``)
-is renamed to ``.dead`` so the queue no longer drains it but the operator
-can still inspect via ``switchroom doctor``. An entry failing on anything
+is retired into the ``pending-dead/`` archive so the queue no longer
+drains it but the operator can still inspect via ``switchroom doctor``.
+The marker deliberately does NOT stay in the live queue directory: it is
+the only remaining copy of that memory, and leaving it among the live
+entries put it in the path of every janitor that sweeps that
+directory. An entry failing on anything
 else — a 5xx, a timeout, a connection error — stays queued past the
 attempt budget: a transient upstream is never evidence that the memory
 is unsaveable, and retiring it would lose content the user believes was
@@ -53,8 +57,30 @@ of that loop, not of lost memory: a full sweep of 5,751 queued entries on
 this fleet (2026-07-25) found **4,048 (70.4%) already existed as
 documents**, 3,815 of them with facts extracted.
 
-``--backlog`` is therefore a two-phase, out-of-hook replay:
-
+``--backlog`` is therefore a three-phase, out-of-hook replay:
+
+* **Phase 0 — collapse duplicates (free, no network).** Queued entries
+  sharing ``(bank_id, part_position, sha256(content))`` are the same
+  memory; the redundant copies are archived so the phases below never pay
+  for one memory twice. Measured 2026-07-26: 1,060 queued files across 11
+  agents fell into ~368 distinct groups — ~65% of the queue was duplicate,
+  with one group repeated 32 times. At ~168 s per phase-2 extraction that
+  one group alone was 90 minutes of LLM lane time for a single memory.
+* **Phase 0b — relocate legacy ``.dead`` markers (free, no network).**
+  Markers written by an older build into the live queue directory are moved
+  into ``pending-dead/``. ``mark_dead`` no longer produces such a marker, so
+  after this phase has run once the live queue holds only live entries and no
+  janitor glob over it can match a memory. Note the CONDITION: phases 0b and
+  0c run in BACKLOG mode only (``drain_backlog``, and not under
+  ``--dry-run``). The SessionStart ``drain()`` never calls them, so on a host
+  where the backlog drain has not run, legacy markers are still sitting in
+  the queue directory.
+* **Phase 0c — re-split over-bound entries (free, no network).** An entry
+  whose content exceeds ``retain_content_limit()`` needs more sequential
+  extraction calls than fit the client deadline, so it can never be drained
+  as-is; splitting it makes every part drainable. Measured 2026-07-26: 18 of
+  211 queued entries exceeded 100,000 chars, the largest 744,546. Backlog
+  mode only, same as 0b.
 * **Phase 1 — reconcile (free).** GET the document. If it exists, the
   memory is already durable; retire the queue entry without a POST. No
   LLM work, no cost, idempotent, resumable at any point. Only for
@@ -84,7 +110,7 @@ reports the upstream is already slow.
 Standalone usage::
 
     python3 drain_pending.py               # bounded in-hook drain
-    python3 drain_pending.py --backlog     # two-phase backlog replay
+    python3 drain_pending.py --backlog     # three-phase backlog replay
     python3 drain_pending.py --backlog --phase reconcile   # free pass only
     python3 drain_pending.py --backlog --dry-run
 """
@@ -105,13 +131,16 @@ from lib.config import debug_log, load_config
 from lib.pending import (
     MAX_ATTEMPTS,
     archive_reconciled,
+    collapse_duplicates,
     is_content_derived_document_id,
     is_permanent_failure,
     iter_entries,
     mark_dead,
+    resplit_over_bound_entries,
+    sweep_legacy_dead_markers,
     update_attempt,
 )
-from lib.retain_split import retain_client_deadline
+from lib.retain_split import retain_client_deadline, retain_content_limit
 
 
 STALL_THRESHOLD = 3
@@ -531,17 +560,24 @@ def _new_summary() -> dict:
         # so it must not be counted as drained/reconciled, which would
         # report a retire that did not happen.
         "archive_failed": 0,
+        # Redundant copies retired by `pending.collapse_duplicates` before
+        # any network work. Backlog mode only — see `_drain_backlog_impl`.
+        "collapsed": 0,
+        "dead_relocated": 0,
+        "resplit": 0,
+        "resplit_parts": 0,
         "stalled": False,
         "budget_exceeded": False,
     }
 
 
 def drain_backlog(config: dict | None = None, **kw) -> dict:
-    """Two-phase backlog replay, off the SessionStart budget entirely.
+    """Three-phase backlog replay, off the SessionStart budget entirely.
 
     See the module docstring. Summary shape is ``drain()``'s plus
-    ``reconciled`` (already durable — no POST issued) and ``unknown``
-    (presence could not be established; left queued).
+    ``reconciled`` (already durable — no POST issued), ``unknown``
+    (presence could not be established; left queued) and ``collapsed``
+    (redundant duplicate copies archived before any network work).
     """
     return drain(config, backlog=True, **kw)
 
@@ -566,6 +602,10 @@ def drain(
          "unknown": int,   # presence unknown, left queued
          "archive_failed": int,  # durable, but the archive was unwritable
                                  # so the entry is STILL QUEUED
+         "collapsed": int, # duplicate copies archived (backlog mode only)
+         "dead_relocated": int,  # legacy .dead markers moved out of the queue dir
+         "resplit":   int, # over-bound entries split into drainable parts
+         "resplit_parts": int,  # parts those entries became
          "stalled": bool,  # stall guard tripped
          "budget_exceeded": bool}
     """
@@ -784,9 +824,65 @@ def _wait_for_upstream(backoff_ms: int, started: float, budget: float) -> bool:
 def _drain_backlog_impl(
     config: dict, phase: str = "both", dry_run: bool = False
 ) -> dict:
-    """Concurrent-capable, long-budget, two-phase backlog replay."""
+    """Concurrent-capable, long-budget, three-phase backlog replay."""
     summary = _new_summary()
 
+    # PHASE 0 — collapse duplicates (local, free, no network at all).
+    #
+    # Runs FIRST because every later phase is per-entry: a GET in phase 1
+    # and, worse, a ~168 s LLM-backed extraction in phase 2. On the measured
+    # 2026-07-26 fleet backlog ~65% of queued files were byte-identical
+    # copies of another queued file (top group 32x), so skipping this pass
+    # means paying phase 2 up to 32 times over for one memory.
+    #
+    # DELIBERATELY NOT run by the in-hook drain. It reads every queued entry
+    # to recompute identity from content — bounded, but not instant — and
+    # the SessionStart drain's whole contract is a hard wall-clock ceiling
+    # on hook latency. New duplicates cannot accumulate there anyway:
+    # `pending.enqueue`'s filename-keyed guard stops those at the producer.
+    if not dry_run:
+        summary["collapsed"] = collapse_duplicates()
+        if summary["collapsed"]:
+            _blog(
+                f"phase 0: collapsed {summary['collapsed']} duplicate entries "
+                f"(byte-identical content already queued under another entry); "
+                f"archived, not deleted"
+            )
+
+        # PHASE 0b — relocate any legacy `.dead` markers out of the live
+        # queue directory. Free, local, and an UPGRADE step: markers written
+        # by an older build sit in the directory external janitors sweep, and
+        # a marker is the only remaining copy of its memory.
+        moved = sweep_legacy_dead_markers()
+        if moved:
+            summary["dead_relocated"] = moved
+            _blog(
+                f"phase 0b: relocated {moved} legacy .dead marker(s) out of "
+                f"the live queue directory; the queue now holds only live "
+                f"entries, so no janitor glob over it can match a memory"
+            )
+
+        # PHASE 0c — re-split entries the drain provably CANNOT retain.
+        #
+        # An entry over `retain_content_limit()` needs more sequential
+        # extraction calls than fit the deadline, so every POST is guaranteed
+        # waste; if the server rejects the body as a 4xx it is classified
+        # permanent and the memory goes `.dead`. Splitting it makes every
+        # part drainable, which is the difference between a lost memory and a
+        # slow one. Runs after the duplicate collapse so a duplicated
+        # over-bound entry is split ONCE, not once per copy.
+        entries_split, parts_written = resplit_over_bound_entries()
+        if entries_split:
+            summary["resplit"] = entries_split
+            summary["resplit_parts"] = parts_written
+            _blog(
+                f"phase 0c: re-split {entries_split} entr"
+                f"{'y' if entries_split == 1 else 'ies'} over the "
+                f"{retain_content_limit()}-char retain bound into "
+                f"{parts_written} drainable part(s); the originals are "
+                f"archived, not deleted"
+            )
+
     if phase in ("reconcile", "both"):
         _reconcile_phase(config, summary, dry_run)
     if phase == "reconcile":
@@ -935,7 +1031,7 @@ def _parse_args(argv: list[str] | None):
     ap.add_argument(
         "--backlog",
         action="store_true",
-        help="two-phase backlog replay, off the SessionStart budget",
+        help="three-phase backlog replay, off the SessionStart budget",
     )
     ap.add_argument(
         "--phase",
@@ -967,6 +1063,9 @@ def main(argv: list[str] | None = None) -> int:
     if any(
         summary[k]
         for k in (
+            "collapsed",
+            "dead_relocated",
+            "resplit",
             "drained",
             "retried",
             "dead",
@@ -978,6 +1077,9 @@ def main(argv: list[str] | None = None) -> int:
         print(
             f"[Hindsight] drain_pending{'(backlog)' if args.backlog else ''}: "
             f"drained={summary['drained']} reconciled={summary['reconciled']} "
+            f"collapsed={summary['collapsed']} "
+            f"dead_relocated={summary['dead_relocated']} "
+            f"resplit={summary['resplit']}(+{summary['resplit_parts']} parts) "
             f"retried={summary['retried']} dead={summary['dead']} "
             f"unknown={summary['unknown']} "
             f"archive_failed={summary['archive_failed']} "