switchroom 0.19.41 → 0.19.43

package/vendor/hindsight-memory/scripts/backfill_transcripts.py

@@ -193,6 +193,7 @@ def _is_dynamic_bank(config: dict) -> bool:
 
 _STALL_THRESHOLD = 3          # consecutive failures → circuit-breaker cooldown
 _BACKOFF_CAP_S = 60.0         # exponential backoff ceiling
+_POST_MAX_ATTEMPTS = 3        # bounded per-slice retry before giving up (#3291)
 
 
 def agents_dir() -> str:
@@ -453,38 +454,59 @@ class Backfill:
             self._consecutive_failures = 0
 
     def _post_slice(self, bank_id: str, built: dict) -> bool:
-        """Serial, paced, confirmed POST of one slice. Returns True on confirmed
-        persistence. NEVER concurrent — one in flight fleet-wide via the shared
-        inflight lock."""
+        """Serial, paced, confirmed POST of one slice with bounded retry.
+
+        Returns True on confirmed persistence, False after exhausting
+        ``_POST_MAX_ATTEMPTS`` transient failures. NEVER concurrent — one in
+        flight fleet-wide via the shared inflight lock.
+
+        The bounded retry (#3291) matters because every POST is
+        ``async_processing=False`` (commit-before-ack): a slice that lands is
+        durable immediately, so a *single*-attempt failure on slice ``k`` used to
+        strand slices ``0..k-1`` as a committed prefix while the session was
+        marked ``failed`` and — via the session-membership skip — never revisited.
+        Retrying each slice in place lets a transient blip resolve without leaving
+        that partial-commit gap. Persistent failure still returns False so the
+        caller records ``failed`` and the gap-completion path (routed from the
+        progress record, not the membership skip) fills the remainder on re-run."""
         client = self._ensure_client()
         if client is None:
             return False
         payload = built["payload"]
         self._pace_before_post()
-        with inflight_lock(blocking=True) as acquired:
-            if not acquired:  # pragma: no cover - blocking acquire fails open
-                return False
-            try:
-                client.retain(
-                    bank_id=bank_id,
-                    content=payload["content"],
-                    document_id=built["document_id"],
-                    context=payload["context"],
-                    metadata=payload["metadata"],
-                    tags=payload["tags"],
-                    timeout=15,
-                    async_processing=False,  # commit-before-ack (daemon contract 2)
-                    observation_scopes=payload.get("observation_scopes"),
-                )
-            except Exception as e:
-                debug_log(self.config, f"backfill: POST failed for {built['document_id']}: {e}")
-                self._posted_count += 1
-                self._on_failure()
-                return False
-        self._posted_count += 1
-        self._post_times.append(time.monotonic())
-        self._consecutive_failures = 0
-        return True
+        last_exc = None
+        for _attempt in range(_POST_MAX_ATTEMPTS):
+            with inflight_lock(blocking=True) as acquired:
+                if not acquired:  # pragma: no cover - blocking acquire fails open
+                    return False
+                try:
+                    client.retain(
+                        bank_id=bank_id,
+                        content=payload["content"],
+                        document_id=built["document_id"],
+                        context=payload["context"],
+                        metadata=payload["metadata"],
+                        tags=payload["tags"],
+                        timeout=15,
+                        async_processing=False,  # commit-before-ack (daemon contract 2)
+                        observation_scopes=payload.get("observation_scopes"),
+                    )
+                except Exception as e:
+                    last_exc = e
+                    debug_log(self.config, f"backfill: POST failed for {built['document_id']}: {e}")
+                    self._posted_count += 1
+                    self._on_failure()  # backoff before the next attempt
+                    continue
+            self._posted_count += 1
+            self._post_times.append(time.monotonic())
+            self._consecutive_failures = 0
+            return True
+        debug_log(
+            self.config,
+            f"backfill: POST gave up for {built['document_id']} after "
+            f"{_POST_MAX_ATTEMPTS} attempts: {last_exc}",
+        )
+        return False
 
     # -- scanning ----------------------------------------------------------- #
     def _list_agents(self, agent_filter: Optional[set]) -> list:
@@ -678,12 +700,49 @@ class Backfill:
             "sessions_already_done": 0,
             "sessions_restored": 0,
             "slices_restored": 0,
+            # Slice-level gap completion (#3291): a session this backfill is the
+            # sole writer of that was previously left partially committed.
+            "sessions_gap_completed": 0,
+            "sessions_gap_incomplete": 0,
+            "slices_gap_filled": 0,
+            "slices_gap_present": 0,
+            # Convergence visibility (#3291 self-heal): sessions that STILL are
+            # not fully recovered after this run (a restore or gap-completion
+            # POST failed). Non-zero ⇒ another run is needed; surfaced so a
+            # non-converging session can never be a SILENT permanent trap.
+            "sessions_incomplete": 0,
             "turns_recovered": 0,
             "posts_ok": 0,
             "posts_failed": 0,
             "sessions": [],
         })
 
+    def _slice_exists(self, bank_id: str, document_id: str) -> Optional[bool]:
+        """Paced tri-state presence check for ONE deterministic slice document.
+
+        ``True`` — already durable (skip, no re-cost). ``False`` — confirmed 404
+        (missing → must post). ``None`` — unknown (transport/5xx): the caller
+        (re)posts, because the daemon UPSERTs on ``document_id`` so a needless
+        re-post of an already-present slice is a safe idempotent no-op, whereas
+        SKIPPING a genuinely-missing slice would re-open the #3291 silent gap. The
+        GET is paced through the shared inflight lock like every other op."""
+        client = self._ensure_client()
+        if client is None:
+            return None
+        self._pace_before_post()
+        try:
+            with inflight_lock(blocking=True) as acquired:
+                if not acquired:  # pragma: no cover - blocking acquire fails open
+                    return None
+                return client.document_exists(bank_id, document_id)
+        except Exception as e:  # pragma: no cover - defensive
+            debug_log(self.config, f"backfill(gap): document_exists failed "
+                                   f"for {document_id}: {e}")
+            return None
+        finally:
+            self._posted_count += 1
+            self._post_times.append(time.monotonic())
+
     def _session_membership(self, bank_id: str, session_id: str) -> str:
         """SESSION-LEVEL, id-scheme-AGNOSTIC membership (BLOCKER-1 fix).
 
@@ -692,18 +751,22 @@ class Backfill:
         ``"error"`` on any query failure (caller fails CLOSED — treats it as
         present ⇒ skip, never restores on uncertainty).
 
-        We deliberately do NOT compare against the tool's ``-r{uuid}`` slice ids:
-        the deployed production banks are keyed with the LEGACY
-        ``{session_id}-{epoch_ms}`` scheme (``retain.py``'s not-yet-live
-        ``slice_document_id`` at line ~430 is the replacement), so an exact slice
-        id set would never match a legacy doc → every slice looks "missing" →
-        every intact session gets duplicated. Presence is decided by the id
-        PREFIX ``{session_id}`` which BOTH schemes share (``{session_id}-...``
-        legacy/new, or the bare ``{session_id}`` chunk-0 id). Restore fires ONLY
-        for a session with ZERO documents — so partial-within-a-populated-session
-        recovery is OUT OF SCOPE on legacy banks (we cannot tell WHICH turns a
-        legacy epoch-ms id covers) and can only be added once the ``-r{uuid}``
-        scheme is deployed and banks are re-keyed.
+        We deliberately do NOT compare against the tool's ``-r{uuid}`` slice ids
+        here: production banks are a MIX of the deterministic ``-r{uuid}`` scheme
+        (``retain.py``'s ``slice_document_id``, live since #3246) and the older
+        ``{session_id}-{epoch_ms}`` docs written before it, so an exact slice-id
+        set would never match a legacy doc → every slice looks "missing" → every
+        intact session gets duplicated. Presence is decided by the id PREFIX
+        ``{session_id}`` which every scheme shares (``{session_id}-...`` legacy or
+        ``-r{uuid}``, or the bare ``{session_id}`` chunk-0 id). Restore fires ONLY
+        for a session with ZERO documents — so partial-tail recovery *within a
+        legacy-populated session* stays OUT OF SCOPE (we cannot tell WHICH turns a
+        legacy epoch-ms id covers). The one partial case we CAN complete safely is
+        a session this backfill is the SOLE writer of — its docs are all
+        deterministic ``-r{uuid}`` ids, so ``_gap_complete_session`` recomputes
+        them and posts only the missing ones; that path is routed from the
+        progress ``failed`` record (a session this tool already classified
+        total-loss), never from this session-membership check.
 
         The GET is paced through the shared inflight lock so must-fix #1's broad
         candidate set can never create a read storm (should-fix 4b)."""
@@ -792,6 +855,21 @@ class Backfill:
                                        "action": "skipped_recent", "idle_s": round(idle, 1)})
                 continue
 
+            # #3291: a session this backfill previously classified total-loss but
+            # left PARTIALLY committed (a slice POST failed ⇒ recorded ``failed``
+            # after slices 0..k-1 landed durably). The session-membership check
+            # below would now see that committed prefix, classify it ``present``,
+            # and skip it FOREVER — silently under-recovering the exact total-loss
+            # sessions this mode exists to rescue. Route it instead into
+            # slice-level gap completion: this tool is the sole writer, so every
+            # doc is a deterministic ``-r{uuid}`` id we can recompute and fill only
+            # the missing slices. Guarded by ``ensure_slice_turns`` (a resume with
+            # a different slice size aborts before any write), so recomputed ids
+            # are guaranteed to match the committed prefix.
+            if self.progress.status(agent, session) == "failed":
+                self._gap_complete_session(agent, session, bank_id, path, info, ar)
+                continue
+
             # SESSION-LEVEL, scheme-agnostic membership (BLOCKER-1 fix): restore
             # ONLY a session the bank has ZERO documents for (true total loss).
             # A session with ANY doc (legacy epoch-ms OR new -r{uuid} scheme) is
@@ -876,9 +954,122 @@ class Backfill:
                 self.progress.record(agent, session, "done", slices=total, turns=n_turns)
             else:
                 session_entry["action"] = "restore_failed"
+                ar["sessions_incomplete"] += 1
                 self.progress.record(agent, session, "failed")
             ar["sessions"].append(session_entry)
 
+        # Convergence visibility: if any session is still short after this run,
+        # tell the operator plainly (and how to converge it) so a session that
+        # keeps failing can never be a silent permanent trap. The mechanism
+        # guarantees forward progress — each re-run posts only what is still
+        # missing — but a persistently-unpostable session needs a human.
+        if self.commit and ar["sessions_incomplete"] > 0:
+            stuck = [s.get("session") for s in ar["sessions"]
+                     if s.get("action") in ("restore_failed", "gap_incomplete")]
+            print(
+                f"[Hindsight] backfill(from-logs): {agent} — "
+                f"{ar['sessions_incomplete']} session(s) still INCOMPLETE after this "
+                f"run (POST failures): {', '.join(str(s) for s in stuck)}. Re-run "
+                f"`--from-logs --commit --agent {agent}` to fill the remaining "
+                f"slices (gap-completion posts ONLY what is still missing).",
+                file=sys.stderr,
+            )
+
+    def _gap_complete_session(self, agent, session, bank_id, path, info, ar) -> None:
+        """Fill the missing slices of a backfill-sole-writer session left
+        partially committed by a prior run (#3291).
+
+        Precondition (enforced by the caller): the session's progress record is
+        ``failed`` — this tool already classified it total-loss and committed a
+        deterministic ``-r{uuid}`` prefix before a POST failed. Because the
+        backfill is the SOLE writer of such a session, EVERY doc it holds is a
+        deterministic slice id we can recompute (``ensure_slice_turns`` guarantees
+        the same boundaries as the prior run). We check each slice's presence and
+        POST ONLY the ones the bank is missing, so the committed prefix is never
+        duplicated and the session always makes forward progress toward complete
+        recovery rather than being skipped forever by session-membership."""
+        messages = read_transcript(path)
+        if not messages:
+            return
+        slices = chunk_by_human_turns(messages, self.slice_turns)
+        if not slices:
+            return
+        built_slices = []
+        for sl in slices:
+            built = build_retain_payload(
+                self.config, session, sl, messages,
+                bank_id=bank_id, api_url=self._api_url or "http://backfill-client",
+                api_token=self._api_token, retain_full_window=True, document_id=None,
+            )
+            if built is not None:
+                built_slices.append((built, sl))
+        if not built_slices:
+            return
+
+        total = len(built_slices)
+        n_turns = sum(len(_human_turn_indices(sl)) for _, sl in built_slices)
+        session_entry = {
+            "session": session, "class": "gap_completion",
+            "join": info["join"], "bank_id": bank_id,
+            "slices": total, "turns": n_turns,
+            "document_ids": [b["document_id"] for b, _ in built_slices],
+            "action": None,
+        }
+
+        if not self.commit:
+            # Dry-run: report the whole session as the gap it WOULD complete
+            # (per-slice presence probing is itself a network read, elided here).
+            session_entry["action"] = "would_gap_complete"
+            ar["sessions"].append(session_entry)
+            return
+
+        # POST only the slices the bank is actually missing. ``True`` ⇒ already
+        # durable (skip); ``False``/``None`` ⇒ (re)post — the daemon UPSERTs on
+        # document_id so a needless re-post is a safe no-op, while skipping a
+        # genuinely-missing slice would re-open the gap.
+        filled = 0
+        present = 0
+        failed = 0
+        for built, _ in built_slices:
+            if self._slice_exists(bank_id, built["document_id"]) is True:
+                present += 1
+                continue
+            if self._post_slice(bank_id, built):
+                filled += 1
+                ar["posts_ok"] += 1
+            else:
+                failed += 1
+                ar["posts_failed"] += 1
+
+        ar["slices_gap_filled"] += filled
+        ar["slices_gap_present"] += present
+        session_entry["slices_filled"] = filled
+        session_entry["slices_present"] = present
+
+        if failed == 0:
+            # Every slice is now durable → the session is fully recovered.
+            ar["sessions_gap_completed"] += 1
+            ar["turns_recovered"] += n_turns
+            session_entry["action"] = "gap_completed"
+            last_built = built_slices[-1][0]
+            try:
+                watermark.commit(
+                    session, last_built["last_uuid"], last_built["document_id"],
+                    transcript_path=path, ordered_uuids=last_built["ordered_uuids"],
+                )
+            except Exception:  # pragma: no cover - defensive
+                pass
+            self.progress.record(agent, session, "done", slices=total, turns=n_turns)
+        else:
+            # Still short — stay ``failed`` so the NEXT run re-enters this same
+            # gap-completion path (never the membership skip) and fills whatever
+            # remains. Convergent: each run strictly reduces the missing set.
+            ar["sessions_gap_incomplete"] += 1
+            ar["sessions_incomplete"] += 1
+            session_entry["action"] = "gap_incomplete"
+            self.progress.record(agent, session, "failed")
+        ar["sessions"].append(session_entry)
+
     def _finalize_report_from_logs(self) -> None:
         roll = {
             "mode": "from_logs",
@@ -895,6 +1086,11 @@ class Backfill:
             "sessions_already_done": 0,
             "sessions_restored": 0,
             "slices_restored": 0,
+            "sessions_gap_completed": 0,
+            "sessions_gap_incomplete": 0,
+            "slices_gap_filled": 0,
+            "slices_gap_present": 0,
+            "sessions_incomplete": 0,
             "turns_recovered": 0,
             "posts_ok": 0,
             "posts_failed": 0,
@@ -909,6 +1105,8 @@ class Backfill:
                 "unmatched_turns", "sessions_has_documents", "sessions_total_loss",
                 "sessions_membership_error", "sessions_active_skipped",
                 "sessions_already_done", "sessions_restored", "slices_restored",
+                "sessions_gap_completed", "sessions_gap_incomplete",
+                "slices_gap_filled", "slices_gap_present", "sessions_incomplete",
                 "turns_recovered", "posts_ok", "posts_failed",
             ):
                 roll[k] += ar.get(k, 0)
@@ -965,11 +1163,15 @@ def format_report_from_logs(report: dict) -> str:
             f"already_done={ar['sessions_already_done']} "
             f"restored={ar['sessions_restored']} "
             f"slices_restored={ar['slices_restored']} "
+            f"gap_completed={ar.get('sessions_gap_completed', 0)} "
+            f"gap_filled={ar.get('slices_gap_filled', 0)} "
+            f"incomplete={ar.get('sessions_incomplete', 0)} "
             f"turns={ar['turns_recovered']} "
             f"posts_ok={ar['posts_ok']} posts_failed={ar['posts_failed']}"
         )
         for s in ar.get("sessions", []):
-            if s.get("class") in ("total_loss", "membership_error", "ambiguous"):
+            if s.get("class") in ("total_loss", "gap_completion",
+                                  "membership_error", "ambiguous"):
                 lines.append(
                     f"      - {s.get('session', s.get('turn_key'))} [{s.get('class')}] "
                     f"join={s.get('join', '-')} "
@@ -987,9 +1189,21 @@ def format_report_from_logs(report: dict) -> str:
         f"membership_err={roll.get('sessions_membership_error', 0)} "
         f"restored={roll.get('sessions_restored', 0)} "
         f"slices_restored={roll.get('slices_restored', 0)} "
+        f"gap_completed={roll.get('sessions_gap_completed', 0)} "
+        f"gap_filled={roll.get('slices_gap_filled', 0)} "
+        f"incomplete={roll.get('sessions_incomplete', 0)} "
         f"turns_would_recover={roll.get('turns_recovered', 0)} "
         f"posts_ok={roll.get('posts_ok', 0)} posts_failed={roll.get('posts_failed', 0)}"
     )
+    incomplete = roll.get("sessions_incomplete", 0)
+    if incomplete:
+        lines.append(
+            f"  ⚠ CONVERGENCE: {incomplete} session(s) still INCOMPLETE (a POST "
+            f"failed). Re-run `--from-logs --commit` — gap-completion posts ONLY "
+            f"the still-missing slices, so each run strictly reduces the gap. A "
+            f"count that never reaches 0 across runs means a persistently-"
+            f"unpostable session that needs a human."
+        )
     lines.append("  NOTE: recovers only ZERO-document (total-loss) sessions. "
                  "Partial-tail recovery within a populated session is OUT OF SCOPE "
                  "on legacy epoch-ms banks (needs the -r{uuid} id scheme deployed).")

package/vendor/hindsight-memory/scripts/lib/content.py

@@ -466,11 +466,20 @@ def shape_recall_query(
     for index, token in enumerate(tokens):
         first_seen.setdefault(token, index)
 
-    candidates = [t for t in first_seen if len(t) > 1 and t not in stop]
+    # Stopword removal is the PRIMARY filter; the length guard lives only in the
+    # fallback below (#3766). Filtering ``len(t) > 1`` here would silently drop a
+    # single-char SUBJECT — a language name ('C', 'R'), a single-digit version
+    # ('v3' tokenizes fine, but a bare '9' in "python 9077" or "Angular 9" does
+    # not) — before the fallback ever runs, leaving the query with only its
+    # multi-char stopwords. Single-char stopwords ('a', 'i', 's', 't') are still
+    # removed because they are in ``stop``; a single-char content word survives.
+    candidates = [t for t in first_seen if t not in stop]
     if not candidates:
         # Every term was a stopword. Fall back to the unfiltered token set so a
         # short conversational prompt ("what did you say about it?") still
-        # searches for something.
+        # searches for something. The ``len(t) > 1`` guard applies HERE — with
+        # every term a stopword there is no subject to protect, so dropping the
+        # remaining single-char noise is safe.
         candidates = [t for t in first_seen if len(t) > 1] or list(first_seen)
 
     def weight(token):

package/vendor/hindsight-memory/scripts/recall.py

@@ -472,15 +472,30 @@ _OVERLAP_STOPWORDS = frozenset({
 def _overlap_tokens(text) -> set:
     """Tokenize text into a stop-word-stripped, lowercased set of terms.
 
-    Punctuation, digits, and short fragments (<= 1 char) are dropped.
-    Returns an empty set on non-string / empty input.
+    Tokens are maximal runs of ALPHANUMERIC characters (``str.isalnum``);
+    everything else (whitespace, punctuation) is a separator. Short fragments
+    (<= 1 char) and stop-words are dropped. Returns an empty set on non-string /
+    empty input.
+
+    #3578 — DIGITS CARRY SIGNAL. This fallback tokenizer feeds the #3369
+    transcript-grep fallback's keyword match (``_build_transcript_fallback``):
+    a recent transcript turn is kept only if it shares a token with the recall
+    query. This fleet talks in identifiers — "did PR 3993 land", "the :9077
+    port", "v0.19.24" — and the previous ``ch.isalpha()`` accumulation dropped
+    every digit on the floor, so a turn whose ONLY tie to the query was the
+    issue number '3993' shared no token and was filtered out. Accumulating
+    ``isalnum`` runs keeps pure-digit and mixed identifiers ('3993', '9077',
+    'v0') as tokens, so issue numbers, ports and versions match symmetrically on
+    both the query and transcript sides. A lone digit (a bare '9') is still
+    dropped by the ``> 1`` length guard, same as a lone letter — it is noise, not
+    an identifier. See ``tests/test_overlap_tokens.py`` for the characterisation.
     """
     if not isinstance(text, str) or not text:
         return set()
     out = set()
     cur = []
     for ch in text:
-        if ch.isalpha():
+        if ch.isalnum():
             cur.append(ch.lower())
         else:
             if cur:

package/vendor/hindsight-memory/scripts/subagent_retain.py

@@ -37,9 +37,9 @@ Flow:
   2. Resolve the sidechain transcript (agent_transcript_path → derived
      subagents/ dir → project-dir scan).
   3. Volume gate: skip sub-agents below the floor (< 6 human turns OR
-     < 2,000 chars of non-tool-result text) — SubagentStop fires for every
-     Task including 10-second forks, and each retain is an LLM-backed
-     extraction.
+     < 2,000 chars of RETAINED text — the chars the text-only path keeps,
+     #3994) — SubagentStop fires for every Task including 10-second forks, and
+     each retain is an LLM-backed extraction.
   4. Retain a bounded window (last N=40 human turns), tagged ``sidechain`` +
      ``parent_session:<id>``, with a deterministic content-derived document_id
      so re-fires upsert instead of duplicating.
@@ -62,7 +62,7 @@ from lib.bank import derive_bank_id, ensure_bank_mission
 from lib.client import HindsightClient
 from lib.config import debug_log, load_config
 from lib.content import (
-    _extract_message_blocks,
+    _extract_text_content,
     _is_tool_result_only_user_message,
     slice_last_turns_by_user_boundary,
     transcript_first_line_is_sidechain,
@@ -78,8 +78,9 @@ from retain import build_retain_payload, read_transcript
 # Retain the last N human turns of the sidechain. The window is formatted on the
 # TEXT-ONLY path (``retainToolCalls`` is forced False for the sidechain — see
 # ``run_subagent_retain``), so tool_use inputs and tool_result bodies are dropped
-# entirely rather than passed through / truncated. ``_extract_message_blocks`` is
-# still imported here, but only for the volume gate's char count.
+# entirely rather than passed through / truncated. ``_extract_text_content`` is
+# imported here so the volume gate's char count measures the SAME text the retain
+# path keeps (#3994) — gate and payload never diverge.
 SIDECHAIN_WINDOW_TURNS = 40
 
 # Volume gate floors — SubagentStop fires for every Task, so skip trivial forks.
@@ -262,59 +263,63 @@ def count_human_turns(messages: list) -> int:
     return n
 
 
-def non_tool_result_char_count(messages: list, stop_at: int | None = None) -> int:
-    """Total chars of non-tool-result content across the transcript.
+def retained_text_char_count(messages: list, stop_at: int | None = None) -> int:
+    """Total chars of the content the TEXT-ONLY retain path actually keeps.
 
-    Sums the extracted text + tool_use (command/input) blocks and EXCLUDES
-    tool_result bodies — the same "text vs tool output" split
-    ``_extract_message_blocks`` already draws. This is the signal the volume
-    gate wants: a 10-second fork with almost no narration/commands falls under
-    the floor even if it emitted a large tool_result, while a real worker's
-    commands and decisions count.
+    #3994 — GATE COUNTS WHAT IS RETAINED. The sidechain payload is built with
+    ``retainToolCalls = False`` (``run_subagent_retain``), so what reaches memory
+    is exactly ``_extract_text_content``: assistant ``text`` blocks, channel-
+    message tool_use text, and plain-string user turns — and NOTHING else. This
+    gate counts the same thing, per message, so "cleared the floor" now means
+    "has >= N chars of RETAINABLE prose", not "emitted N chars of tool traffic
+    the payload then drops".
 
-    KNOWN MISMATCH (accepted, tracked as a follow-up): this counts ``tool_use``
-    inputs, but those are no longer RETAINED — the sidechain payload is built on
-    the text-only path. So the gate can clear on tool volume that contributes
-    nothing to the stored memory. It cannot produce an EMPTY retain (the
-    ``MIN_HUMAN_TURNS`` floor guarantees prose-bearing user turns — see the
-    module docstring), so this is a precision issue in the gate, not a
-    correctness bug. Tightening it to count only text is a separate change.
+    The previous revision counted ``tool_use`` name+input serialized size on top
+    of text. Those chars are no longer in the payload, so the old gate could
+    clear on tool volume that contributed zero to the stored memory (a tool-heavy
+    / prose-light fork passed, then retained a near-empty document). Counting via
+    ``_extract_text_content`` closes that mismatch: gate and payload measure the
+    identical char set. The 2,000-char floor was re-measured against this metric
+    — see ``docs/measurements/subagent-volume-gate-3994.md`` and the replay
+    harness ``scripts/tests/data/replay_volume_gate_3994.py``.
 
     ``stop_at`` (review finding 4 — early short-circuit): return as soon as the
     running total reaches this many chars. The gate only needs to know whether
-    the floor is CLEARED, not the exact size — so on a large worker transcript
-    we stop the block walk the moment the floor is met (the returned value is
-    then a floor, ``>= stop_at``, sufficient for the ``>=`` comparison and the
-    skip log's "chars>=N" read).
+    the floor is CLEARED, not the exact size — so on a large worker transcript we
+    stop the walk the moment the floor is met (the returned value is then a
+    floor, ``>= stop_at``, sufficient for the ``>=`` comparison and the skip
+    log's "chars>=N" read).
     """
     total = 0
     for m in messages:
         if not isinstance(m, dict):
             continue
-        blocks = _extract_message_blocks(m.get("content", ""), role=m.get("role", ""))
-        for b in blocks:
-            if not isinstance(b, dict) or b.get("type") == "tool_result":
-                continue
-            if b.get("type") == "text":
-                total += len(b.get("text", ""))
-            elif b.get("type") == "tool_use":
-                # Command / input is a process fact; count its serialized size.
-                total += len(b.get("name", "")) + len(json.dumps(b.get("input", {}), ensure_ascii=False))
+        # Count exactly the chars the text-only retain path keeps for this
+        # message — identical extraction to ``prepare_retention_transcript``'s
+        # ``include_tool_calls=False`` branch, so gate and payload never diverge.
+        total += len(_extract_text_content(m.get("content", ""), role=m.get("role", "")))
         if stop_at is not None and total >= stop_at:
             return total
     return total
 
 
+# Back-compat alias: the previous name measured a superset (text + tool_use).
+# Kept so an out-of-tree caller does not break, but it now returns the
+# recalibrated text-only count (#3994).
+non_tool_result_char_count = retained_text_char_count
+
+
 def passes_volume_gate(messages: list, config: dict) -> tuple:
     """Return ``(passed, human_turns, char_count)`` for the volume gate.
 
     Skip sub-agents below EITHER floor: < ``MIN_HUMAN_TURNS`` human turns OR
-    < ``MIN_NON_TOOL_RESULT_CHARS`` chars of non-tool-result text. The char walk
-    short-circuits at the floor (finding 4) — ``char_count`` is exact when below
-    the floor and a lower bound (``>= floor``) once cleared.
+    < ``MIN_NON_TOOL_RESULT_CHARS`` chars of RETAINED text (the chars the
+    text-only path actually keeps, #3994). The char walk short-circuits at the
+    floor (finding 4) — ``char_count`` is exact when below the floor and a lower
+    bound (``>= floor``) once cleared.
     """
     turns = count_human_turns(messages)
-    chars = non_tool_result_char_count(messages, stop_at=MIN_NON_TOOL_RESULT_CHARS)
+    chars = retained_text_char_count(messages, stop_at=MIN_NON_TOOL_RESULT_CHARS)
     passed = turns >= MIN_HUMAN_TURNS and chars >= MIN_NON_TOOL_RESULT_CHARS
     return passed, turns, chars
 
@@ -521,7 +526,23 @@ def run_subagent_retain(hook_input: dict) -> dict:
         document_id=None,  # content-derived from the slice's first/last uuids
     )
     if built is None:
+        # #4001 — this must NOT be silent. It is the "should never happen" branch
+        # (the volume gate guarantees >= MIN_NON_TOOL_RESULT_CHARS of retainable
+        # text, so a window that cleared the gate should always format to
+        # something), so if it fires it means the gate and the formatter disagree
+        # about what counts as retainable — a real regression that would
+        # otherwise vanish with debug off in the shipped settings. Emit
+        # unconditionally on stderr so it surfaces in the gateway log regardless
+        # of the debug flag; keep the debug_log for the verbose trace.
         debug_log(config, "SubagentStop: empty transcript after formatting, skipping")
+        print(
+            "[Hindsight] SubagentStop: sidechain window formatted to EMPTY on the "
+            f"text-only path despite clearing the volume gate "
+            f"(session={session_id} agent={agent_id} turns={turns} chars>={chars}) "
+            "— retain skipped. This should not happen; if it recurs the volume "
+            "gate and the text-only formatter have diverged (investigate #3994).",
+            file=sys.stderr,
+        )
         return {"status": "skipped", "reason": "empty transcript after formatting"}
 
     payload = built["payload"]