switchroom 0.19.40 → 0.19.42

package/vendor/hindsight-memory/scripts/subagent_retain.py

@@ -37,9 +37,9 @@ Flow:
   2. Resolve the sidechain transcript (agent_transcript_path → derived
      subagents/ dir → project-dir scan).
   3. Volume gate: skip sub-agents below the floor (< 6 human turns OR
-     < 2,000 chars of non-tool-result text) — SubagentStop fires for every
-     Task including 10-second forks, and each retain is an LLM-backed
-     extraction.
+     < 2,000 chars of RETAINED text — the chars the text-only path keeps,
+     #3994) — SubagentStop fires for every Task including 10-second forks, and
+     each retain is an LLM-backed extraction.
   4. Retain a bounded window (last N=40 human turns), tagged ``sidechain`` +
      ``parent_session:<id>``, with a deterministic content-derived document_id
      so re-fires upsert instead of duplicating.
@@ -62,7 +62,7 @@ from lib.bank import derive_bank_id, ensure_bank_mission
 from lib.client import HindsightClient
 from lib.config import debug_log, load_config
 from lib.content import (
-    _extract_message_blocks,
+    _extract_text_content,
     _is_tool_result_only_user_message,
     slice_last_turns_by_user_boundary,
     transcript_first_line_is_sidechain,
@@ -78,8 +78,9 @@ from retain import build_retain_payload, read_transcript
 # Retain the last N human turns of the sidechain. The window is formatted on the
 # TEXT-ONLY path (``retainToolCalls`` is forced False for the sidechain — see
 # ``run_subagent_retain``), so tool_use inputs and tool_result bodies are dropped
-# entirely rather than passed through / truncated. ``_extract_message_blocks`` is
-# still imported here, but only for the volume gate's char count.
+# entirely rather than passed through / truncated. ``_extract_text_content`` is
+# imported here so the volume gate's char count measures the SAME text the retain
+# path keeps (#3994) — gate and payload never diverge.
 SIDECHAIN_WINDOW_TURNS = 40
 
 # Volume gate floors — SubagentStop fires for every Task, so skip trivial forks.
@@ -262,59 +263,63 @@ def count_human_turns(messages: list) -> int:
     return n
 
 
-def non_tool_result_char_count(messages: list, stop_at: int | None = None) -> int:
-    """Total chars of non-tool-result content across the transcript.
+def retained_text_char_count(messages: list, stop_at: int | None = None) -> int:
+    """Total chars of the content the TEXT-ONLY retain path actually keeps.
 
-    Sums the extracted text + tool_use (command/input) blocks and EXCLUDES
-    tool_result bodies — the same "text vs tool output" split
-    ``_extract_message_blocks`` already draws. This is the signal the volume
-    gate wants: a 10-second fork with almost no narration/commands falls under
-    the floor even if it emitted a large tool_result, while a real worker's
-    commands and decisions count.
+    #3994 — GATE COUNTS WHAT IS RETAINED. The sidechain payload is built with
+    ``retainToolCalls = False`` (``run_subagent_retain``), so what reaches memory
+    is exactly ``_extract_text_content``: assistant ``text`` blocks, channel-
+    message tool_use text, and plain-string user turns — and NOTHING else. This
+    gate counts the same thing, per message, so "cleared the floor" now means
+    "has >= N chars of RETAINABLE prose", not "emitted N chars of tool traffic
+    the payload then drops".
 
-    KNOWN MISMATCH (accepted, tracked as a follow-up): this counts ``tool_use``
-    inputs, but those are no longer RETAINED — the sidechain payload is built on
-    the text-only path. So the gate can clear on tool volume that contributes
-    nothing to the stored memory. It cannot produce an EMPTY retain (the
-    ``MIN_HUMAN_TURNS`` floor guarantees prose-bearing user turns — see the
-    module docstring), so this is a precision issue in the gate, not a
-    correctness bug. Tightening it to count only text is a separate change.
+    The previous revision counted ``tool_use`` name+input serialized size on top
+    of text. Those chars are no longer in the payload, so the old gate could
+    clear on tool volume that contributed zero to the stored memory (a tool-heavy
+    / prose-light fork passed, then retained a near-empty document). Counting via
+    ``_extract_text_content`` closes that mismatch: gate and payload measure the
+    identical char set. The 2,000-char floor was re-measured against this metric
+    — see ``docs/measurements/subagent-volume-gate-3994.md`` and the replay
+    harness ``scripts/tests/data/replay_volume_gate_3994.py``.
 
     ``stop_at`` (review finding 4 — early short-circuit): return as soon as the
     running total reaches this many chars. The gate only needs to know whether
-    the floor is CLEARED, not the exact size — so on a large worker transcript
-    we stop the block walk the moment the floor is met (the returned value is
-    then a floor, ``>= stop_at``, sufficient for the ``>=`` comparison and the
-    skip log's "chars>=N" read).
+    the floor is CLEARED, not the exact size — so on a large worker transcript we
+    stop the walk the moment the floor is met (the returned value is then a
+    floor, ``>= stop_at``, sufficient for the ``>=`` comparison and the skip
+    log's "chars>=N" read).
     """
     total = 0
     for m in messages:
         if not isinstance(m, dict):
             continue
-        blocks = _extract_message_blocks(m.get("content", ""), role=m.get("role", ""))
-        for b in blocks:
-            if not isinstance(b, dict) or b.get("type") == "tool_result":
-                continue
-            if b.get("type") == "text":
-                total += len(b.get("text", ""))
-            elif b.get("type") == "tool_use":
-                # Command / input is a process fact; count its serialized size.
-                total += len(b.get("name", "")) + len(json.dumps(b.get("input", {}), ensure_ascii=False))
+        # Count exactly the chars the text-only retain path keeps for this
+        # message — identical extraction to ``prepare_retention_transcript``'s
+        # ``include_tool_calls=False`` branch, so gate and payload never diverge.
+        total += len(_extract_text_content(m.get("content", ""), role=m.get("role", "")))
         if stop_at is not None and total >= stop_at:
             return total
     return total
 
 
+# Back-compat alias: the previous name measured a superset (text + tool_use).
+# Kept so an out-of-tree caller does not break, but it now returns the
+# recalibrated text-only count (#3994).
+non_tool_result_char_count = retained_text_char_count
+
+
 def passes_volume_gate(messages: list, config: dict) -> tuple:
     """Return ``(passed, human_turns, char_count)`` for the volume gate.
 
     Skip sub-agents below EITHER floor: < ``MIN_HUMAN_TURNS`` human turns OR
-    < ``MIN_NON_TOOL_RESULT_CHARS`` chars of non-tool-result text. The char walk
-    short-circuits at the floor (finding 4) — ``char_count`` is exact when below
-    the floor and a lower bound (``>= floor``) once cleared.
+    < ``MIN_NON_TOOL_RESULT_CHARS`` chars of RETAINED text (the chars the
+    text-only path actually keeps, #3994). The char walk short-circuits at the
+    floor (finding 4) — ``char_count`` is exact when below the floor and a lower
+    bound (``>= floor``) once cleared.
     """
     turns = count_human_turns(messages)
-    chars = non_tool_result_char_count(messages, stop_at=MIN_NON_TOOL_RESULT_CHARS)
+    chars = retained_text_char_count(messages, stop_at=MIN_NON_TOOL_RESULT_CHARS)
     passed = turns >= MIN_HUMAN_TURNS and chars >= MIN_NON_TOOL_RESULT_CHARS
     return passed, turns, chars
 
@@ -521,7 +526,23 @@ def run_subagent_retain(hook_input: dict) -> dict:
         document_id=None,  # content-derived from the slice's first/last uuids
     )
     if built is None:
+        # #4001 — this must NOT be silent. It is the "should never happen" branch
+        # (the volume gate guarantees >= MIN_NON_TOOL_RESULT_CHARS of retainable
+        # text, so a window that cleared the gate should always format to
+        # something), so if it fires it means the gate and the formatter disagree
+        # about what counts as retainable — a real regression that would
+        # otherwise vanish with debug off in the shipped settings. Emit
+        # unconditionally on stderr so it surfaces in the gateway log regardless
+        # of the debug flag; keep the debug_log for the verbose trace.
         debug_log(config, "SubagentStop: empty transcript after formatting, skipping")
+        print(
+            "[Hindsight] SubagentStop: sidechain window formatted to EMPTY on the "
+            f"text-only path despite clearing the volume gate "
+            f"(session={session_id} agent={agent_id} turns={turns} chars>={chars}) "
+            "— retain skipped. This should not happen; if it recurs the volume "
+            "gate and the text-only formatter have diverged (investigate #3994).",
+            file=sys.stderr,
+        )
         return {"status": "skipped", "reason": "empty transcript after formatting"}
 
     payload = built["payload"]

package/vendor/hindsight-memory/scripts/tests/data/replay_volume_gate_3994.py

@@ -0,0 +1,295 @@
+#!/usr/bin/env python3
+"""Replay harness for the #3994 sub-agent volume-gate recalibration.
+
+WHAT THIS ANSWERS
+-----------------
+The sidechain retain now counts, in its volume gate, ONLY the chars the
+text-only retain path keeps (``subagent_retain.retained_text_char_count`` ->
+``lib.content._extract_text_content``). The previous gate summed text PLUS
+``tool_use`` name+input serialized size — chars the payload no longer contains.
+This harness re-measures the ``MIN_NON_TOOL_RESULT_CHARS`` floor against the
+CORRECTED metric and shows, per transcript:
+
+  * old_metric   — pre-#3994 count (text + tool_use name+input)
+  * new_metric   — post-#3994 count (text-only, == what is retained)
+  * payload_chars— the ACTUAL text-only formatted payload size
+                   (``prepare_retention_transcript(..., include_tool_calls=False)``)
+  * human_turns  — genuine human turns (``count_human_turns``)
+  * gate decision at a swept set of candidate floors, old vs new
+
+The headline the recalibration turns on: how many tool-heavy / prose-light
+forks the OLD gate wrongly PASSED (clearing on tool volume) that the NEW gate
+correctly SKIPS, while NO real worker (substantial text-only payload) is newly
+skipped at the chosen floor.
+
+DATA
+----
+Two sources, ``--dir`` preferred:
+
+  * ``--dir <path>``: replay real ``*.jsonl`` sidechain transcripts (one Claude
+    Code sidechain per file). Fleet transcripts are NOT committed to this repo
+    (they are private agent memory), so this is how an operator re-runs the
+    measurement against ground truth.
+  * default (no ``--dir``): a SYNTHETIC corpus parameterized by the empirical
+    distributions already measured and cited in ``subagent_retain.py`` /
+    ``test_subagent_retain_learnings.py`` — real klanker sidechain text-only
+    sizes (p10 5,035 / p50 10,058 / p90 26,040 chars; 0 empty, 0 < 500 over 200
+    transcripts) plus the trivial-fork and tool-heavy/prose-light shapes the
+    gate exists to reject. Deterministic (fixed seed) so the committed report is
+    reproducible.
+
+Run:  python3 scripts/tests/data/replay_volume_gate_3994.py [--dir DIR] [--json]
+
+The committed narrative report lives at
+``docs/measurements/subagent-volume-gate-3994.md`` and is regenerated from this
+harness's output.
+"""
+
+import argparse
+import json
+import os
+import random
+import sys
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+SCRIPTS_DIR = os.path.abspath(os.path.join(HERE, "..", ".."))
+if SCRIPTS_DIR not in sys.path:
+    sys.path.insert(0, SCRIPTS_DIR)
+
+import subagent_retain  # noqa: E402
+from lib.content import (  # noqa: E402
+    _extract_message_blocks,
+    prepare_retention_transcript,
+)
+
+CANDIDATE_FLOORS = [500, 1000, 1500, 2000, 3000, 5000]
+SHIPPED_FLOOR = subagent_retain.MIN_NON_TOOL_RESULT_CHARS  # 2000
+MIN_TURNS = subagent_retain.MIN_HUMAN_TURNS  # 6
+
+# A payload this size or larger is a "real worker" whose learnings we must not
+# lose. Set well under the measured klanker p10 (5,035) so the "newly skipped
+# real worker" check is strict, not self-serving.
+REAL_WORKER_PAYLOAD_FLOOR = 2000
+
+
+def _old_metric(messages):
+    """Pre-#3994 count: text + tool_use name+input serialized size."""
+    total = 0
+    for m in messages:
+        if not isinstance(m, dict):
+            continue
+        for b in _extract_message_blocks(m.get("content", ""), role=m.get("role", "")):
+            if not isinstance(b, dict) or b.get("type") == "tool_result":
+                continue
+            if b.get("type") == "text":
+                total += len(b.get("text", ""))
+            elif b.get("type") == "tool_use":
+                total += len(b.get("name", "")) + len(
+                    json.dumps(b.get("input", {}), ensure_ascii=False)
+                )
+    return total
+
+
+def _payload_chars(messages):
+    """Actual text-only formatted payload size (what is retained)."""
+    text, _ = prepare_retention_transcript(
+        messages, ["user", "assistant"], True, include_tool_calls=False
+    )
+    return len(text or "")
+
+
+def measure(messages):
+    new_metric = subagent_retain.retained_text_char_count(messages)
+    return {
+        "human_turns": subagent_retain.count_human_turns(messages),
+        "old_metric": _old_metric(messages),
+        "new_metric": new_metric,
+        "payload_chars": _payload_chars(messages),
+    }
+
+
+# ---------------------------------------------------------------------------
+# Synthetic corpus (deterministic) — models the documented distributions.
+# ---------------------------------------------------------------------------
+
+def _user(text):
+    return {"role": "user", "content": text}
+
+
+def _assistant_prose(text, *, tool_bulk=0):
+    content = [{"type": "text", "text": text}]
+    if tool_bulk:
+        content.append(
+            {"type": "tool_use", "name": "Bash",
+             "input": {"command": "grep -rn x " + ("q" * tool_bulk)}}
+        )
+    return {"role": "assistant", "content": content}
+
+
+def _tool_result_user(bulk):
+    return {"role": "user",
+            "content": [{"type": "tool_result", "tool_use_id": "t", "content": "Z" * bulk}]}
+
+
+def _real_worker(target_text_chars, rng):
+    """A genuine worker: 6-14 human turns, prose totalling ~target text chars,
+    interleaved with heavy tool traffic (the payload drops the tool traffic)."""
+    turns = rng.randint(6, 14)
+    per = max(80, target_text_chars // turns)
+    msgs = []
+    for i in range(turns):
+        msgs.append(_user(f"step {i}: continue the investigation and report findings"))
+        prose = (f"Finding {i}: confirmed the guard only probes observation rows; "
+                 "world/experience facts never traverse the semantic dedup path. ")
+        prose = (prose * ((per // len(prose)) + 1))[:per]
+        msgs.append(_assistant_prose(prose, tool_bulk=rng.randint(2000, 8000)))
+        msgs.append(_tool_result_user(rng.randint(2000, 6000)))
+    return msgs
+
+
+def _trivial_fork(rng):
+    """A 10-second fork: 2-4 turns, almost no prose. Both gates should SKIP."""
+    turns = rng.randint(2, 4)
+    msgs = []
+    for i in range(turns):
+        msgs.append(_user("go"))
+        msgs.append(_assistant_prose("ok", tool_bulk=rng.randint(0, 500)))
+    return msgs
+
+
+def _tool_heavy_prose_light(rng):
+    """The recalibration target: clears MIN_TURNS human turns and emits a LOT of
+    tool traffic, but almost NO retainable prose. OLD gate passes (tool chars),
+    NEW gate skips (text-only ~0). Retaining it stores a near-empty document."""
+    turns = rng.randint(6, 10)
+    msgs = []
+    for i in range(turns):
+        msgs.append(_user("go"))  # genuine human turn, but tiny
+        # No text block at all — pure tool_use with a large command body.
+        msgs.append({"role": "assistant",
+                     "content": [{"type": "tool_use", "name": "Bash",
+                                  "input": {"command": "true " + ("x" * rng.randint(3000, 9000))}}]})
+        msgs.append(_tool_result_user(rng.randint(3000, 8000)))
+    return msgs
+
+
+def synthetic_corpus(seed=1994):
+    rng = random.Random(seed)
+    corpus = []
+    # Real workers across the measured size range (p10..p90+).
+    for _ in range(120):
+        target = int(rng.triangular(1500, 30000, 10058))  # low, high, mode≈p50
+        corpus.append(("real_worker", _real_worker(target, rng)))
+    for _ in range(60):
+        corpus.append(("trivial_fork", _trivial_fork(rng)))
+    for _ in range(40):
+        corpus.append(("tool_heavy_prose_light", _tool_heavy_prose_light(rng)))
+    return corpus
+
+
+def _load_dir(path):
+    corpus = []
+    for name in sorted(os.listdir(path)):
+        if not name.endswith(".jsonl"):
+            continue
+        msgs = subagent_retain.read_transcript(os.path.join(path, name))
+        corpus.append((name, msgs))
+    return corpus
+
+
+def run(corpus):
+    rows = []
+    for label, msgs in corpus:
+        m = measure(msgs)
+        m["label"] = label
+        rows.append(m)
+
+    def gate(row, floor):
+        return row["human_turns"] >= MIN_TURNS and row["new_metric"] >= floor
+
+    def old_gate(row, floor):
+        return row["human_turns"] >= MIN_TURNS and row["old_metric"] >= floor
+
+    real = [r for r in rows if r["payload_chars"] >= REAL_WORKER_PAYLOAD_FLOOR]
+    empty_ish = [r for r in rows if r["payload_chars"] < 500]
+
+    floor_sweep = {}
+    for f in CANDIDATE_FLOORS:
+        newly_skipped_real = [
+            r for r in real if old_gate(r, SHIPPED_FLOOR) and not gate(r, f)
+        ]
+        # Tool-heavy/prose-light forks the OLD gate passed that NEW gate skips.
+        old_pass_new_skip_emptyish = [
+            r for r in empty_ish if old_gate(r, SHIPPED_FLOOR) and not gate(r, f)
+        ]
+        floor_sweep[f] = {
+            "passes_new": sum(1 for r in rows if gate(r, f)),
+            "real_workers_newly_skipped_vs_old_2000": len(newly_skipped_real),
+            "emptyish_forks_now_correctly_skipped": len(old_pass_new_skip_emptyish),
+        }
+
+    summary = {
+        "corpus_size": len(rows),
+        "shipped_floor": SHIPPED_FLOOR,
+        "min_turns": MIN_TURNS,
+        "counts_by_label": _counts(rows),
+        "payload_percentiles_all": _pcts([r["payload_chars"] for r in rows]),
+        "payload_percentiles_gate_passers_new_2000": _pcts(
+            [r["payload_chars"] for r in rows if gate(r, SHIPPED_FLOOR)]
+        ),
+        "old_gate_passed_but_payload_under_500": sum(
+            1 for r in rows if old_gate(r, SHIPPED_FLOOR) and r["payload_chars"] < 500
+        ),
+        "new_gate_passed_but_payload_under_500": sum(
+            1 for r in rows if gate(r, SHIPPED_FLOOR) and r["payload_chars"] < 500
+        ),
+        "floor_sweep": floor_sweep,
+    }
+    return summary, rows
+
+
+def _counts(rows):
+    out = {}
+    for r in rows:
+        out[r["label"]] = out.get(r["label"], 0) + 1
+    return out
+
+
+def _pcts(values):
+    if not values:
+        return {}
+    s = sorted(values)
+
+    def p(q):
+        return s[min(len(s) - 1, int(q * len(s)))]
+
+    return {"n": len(s), "min": s[0], "p10": p(0.10), "p50": p(0.50),
+            "p90": p(0.90), "max": s[-1]}
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--dir", help="directory of real *.jsonl sidechain transcripts")
+    ap.add_argument("--json", action="store_true", help="emit JSON only")
+    args = ap.parse_args()
+
+    if args.dir:
+        corpus = _load_dir(args.dir)
+        source = f"real transcripts from {args.dir}"
+    else:
+        corpus = synthetic_corpus()
+        source = "synthetic corpus (distribution-parameterized, seed=1994)"
+
+    summary, _ = run(corpus)
+    summary["data_source"] = source
+
+    if args.json:
+        print(json.dumps(summary, indent=2))
+        return
+
+    print(f"# Volume-gate recalibration replay (#3994)\nsource: {source}\n")
+    print(json.dumps(summary, indent=2))
+
+
+if __name__ == "__main__":
+    main()

package/vendor/hindsight-memory/scripts/tests/test_backfill_from_logs.py

@@ -58,11 +58,21 @@ class FakeDaemon:
         self.max_inflight_seen = 0
         self._inflight = 0
         self.membership_error = False
+        # #3291 partial-failure simulation: raise on the retain of the Nth
+        # DISTINCT document_id seen (distinct index, POST order = slice order),
+        # so a slice fails persistently (all bounded retries) without a fixed id.
+        self.fail_distinct_indices = set()
+        self._seen_doc_order = []
 
     def retain(self, bank_id, content, document_id="conversation", context=None,
                metadata=None, tags=None, timeout=15, async_processing=True,
                observation_scopes=None):
         self.observation_scopes_seen.append(observation_scopes)
+        if document_id not in self._seen_doc_order:
+            self._seen_doc_order.append(document_id)
+        distinct_idx = self._seen_doc_order.index(document_id)
+        if distinct_idx in self.fail_distinct_indices:
+            raise RuntimeError(f"simulated POST failure for slice #{distinct_idx}")
         self._inflight += 1
         self.max_inflight_seen = max(self.max_inflight_seen, self._inflight)
         try:
@@ -79,6 +89,12 @@ class FakeDaemon:
         return {did for did, d in self.docs.items()
                 if d.get("bank_id") == bank_id and session_id in did}
 
+    def document_exists(self, bank_id, document_id, timeout=30):
+        """Tri-state presence check (mirrors HindsightClient.document_exists):
+        True when the slice doc is durable, False on a confirmed miss."""
+        d = self.docs.get(document_id)
+        return bool(d is not None and d.get("bank_id") == bank_id)
+
 
 def _write_registry(path, rows, with_session_id=False):
     os.makedirs(os.path.dirname(path), exist_ok=True)
@@ -166,6 +182,7 @@ class FromLogsTestBase(unittest.TestCase):
         self._patches = [
             mock.patch.object(HindsightClient, "retain", self._fake_retain),
             mock.patch.object(HindsightClient, "list_session_document_ids", self._fake_membership),
+            mock.patch.object(HindsightClient, "document_exists", self._fake_doc_exists),
             mock.patch("backfill_transcripts.get_api_url", return_value="http://fake"),
             mock.patch("backfill_transcripts._SLEEP", self._fake_sleep),
         ]
@@ -178,6 +195,9 @@ class FromLogsTestBase(unittest.TestCase):
     def _fake_membership(self, *a, **kw):
         return self.daemon.list_session_document_ids(*a, **kw)
 
+    def _fake_doc_exists(self, *a, **kw):
+        return self.daemon.document_exists(*a, **kw)
+
     def _fake_sleep(self, secs):
         self.sleeps.append(secs)
 
@@ -448,6 +468,95 @@ class TestFromLogs(FromLogsTestBase):
         spaced = [s for s in self.sleeps if abs(s - 1.5) < 1e-9]
         self.assertEqual(len(spaced), 3)
 
+    # --- #3291: partial POST failure then re-run must GAP-COMPLETE, not skip --
+    #     Run 1: slices 0..k-1 commit (async_processing=False ⇒ durable), slice k
+    #     fails ⇒ session recorded ``failed``. Run 2 (the fix): the committed
+    #     prefix must NOT classify the session ``present`` and skip it forever —
+    #     it must recompute the deterministic slice ids and POST ONLY the missing
+    #     slice(s). Pre-fix behaviour: run 2 sees the prefix via session
+    #     membership → ``has_documents`` → skip → the tail is lost silently. -----
+    def test_partial_post_failure_then_rerun_gap_completes(self):
+        self._write_yaml({"clerk": None})
+        self._write_settings("clerk", "clerk")
+        base = 1_700_000_000_000
+        # 3 human turns, slice_turns=1 ⇒ 3 non-overlapping slices posted in order.
+        path, (first, last) = self._transcript("clerk", "sess-partial", 3, base)
+        self._registry("clerk", [{
+            "turn_key": "123:_:t", "chat_id": "123", "started_at": str(first + 100),
+            "ended_at": str(last), "ended_via": "sigterm",
+        }])
+
+        # RUN 1 — the LAST slice (distinct index 2) fails every (bounded) attempt.
+        self.daemon.fail_distinct_indices = {2}
+        report1 = self._run("clerk")
+        ar1 = report1["agents"]["clerk"]
+
+        # Total-loss restore attempted all 3 slices; 2 committed, 1 failed.
+        self.assertEqual(ar1["sessions_total_loss"], 1)
+        self.assertEqual(ar1["sessions_restored"], 0)   # NOT fully restored
+        self.assertEqual(ar1["posts_ok"], 2)
+        self.assertEqual(ar1["posts_failed"], 1)
+        entry1 = next(s for s in ar1["sessions"] if s["class"] == "total_loss")
+        all_ids = entry1["document_ids"]
+        self.assertEqual(len(all_ids), 3)
+        committed_after_1 = set(self.daemon.docs)
+        self.assertEqual(len(committed_after_1), 2)     # durable committed prefix
+        missing = [d for d in all_ids if d not in committed_after_1]
+        self.assertEqual(len(missing), 1)
+        # Progress marked failed (NOT done) — the re-run must revisit it.
+        with open(os.environ["HINDSIGHT_BACKFILL_PROGRESS"]) as f:
+            prog = json.load(f)
+        self.assertEqual(prog["clerk/sess-partial"]["status"], "failed")
+
+        # RUN 2 — no injected failure this time. The fix must gap-complete.
+        self.daemon.fail_distinct_indices = set()
+        posts_before = len(self.daemon.posts)
+        report2 = self._run("clerk")
+        ar2 = report2["agents"]["clerk"]
+
+        # It must NOT be skipped as an already-present session.
+        self.assertEqual(ar2["sessions_has_documents"], 0)
+        # Gap completion: exactly the missing slice posted, the 2 present skipped.
+        self.assertEqual(ar2["sessions_gap_completed"], 1)
+        self.assertEqual(ar2["slices_gap_filled"], 1)
+        self.assertEqual(ar2["slices_gap_present"], 2)
+        run2_posts = [did for did, _ in self.daemon.posts[posts_before:]]
+        self.assertEqual(run2_posts, missing)           # ONLY the missing slice
+        # All three slices are now durable — the session is fully recovered.
+        self.assertEqual(set(self.daemon.docs), set(all_ids))
+        with open(os.environ["HINDSIGHT_BACKFILL_PROGRESS"]) as f:
+            prog2 = json.load(f)
+        self.assertEqual(prog2["clerk/sess-partial"]["status"], "done")
+        # Watermark advanced to the true tail slice on completion.
+        self.assertIsNotNone(watermark.load("sess-partial"))
+
+    # --- #3291 self-heal: a persistently-failing slice must surface as an
+    #     operator-visible INCOMPLETE metric + convergence WARN, never a silent
+    #     trap. -----------------------------------------------------------------
+    def test_incomplete_session_surfaces_convergence_metric(self):
+        self._write_yaml({"clerk": None})
+        self._write_settings("clerk", "clerk")
+        base = 1_700_000_000_000
+        path, (first, last) = self._transcript("clerk", "sess-stuck", 2, base)
+        self._registry("clerk", [{
+            "turn_key": "123:_:t", "chat_id": "123", "started_at": str(first + 100),
+            "ended_at": str(last), "ended_via": "sigterm",
+        }])
+        # The first slice fails every attempt → session left INCOMPLETE.
+        self.daemon.fail_distinct_indices = {0}
+
+        report = self._run("clerk")
+        ar = report["agents"]["clerk"]
+        # Metric: one session did not fully recover.
+        self.assertEqual(ar["sessions_incomplete"], 1)
+        self.assertEqual(ar["sessions_restored"], 0)
+        self.assertEqual(report["rollup"]["sessions_incomplete"], 1)
+        # Operator-visible: the formatted report carries the count + a WARN that
+        # tells them to re-run (so a non-converging session is never silent).
+        text = bf.format_report_from_logs(report)
+        self.assertIn("incomplete=1", text)
+        self.assertIn("CONVERGENCE", text)
+
     # --- direct session_id join (migrated registry) -------------------------
     def test_direct_sessionid_join(self):
         self._write_yaml({"clerk": None})