switchroom 0.19.27 → 0.19.28

package/vendor/hindsight-memory/scripts/lib/pending.py

@@ -556,6 +556,112 @@ def _dir_bytes(d: str, names) -> int:
     return total
 
 
+def _append_ledger(line: str) -> None:
+    """Append one line to ``pending-evictions.log``, rotating at its cap.
+
+    Split out of ``_log_eviction`` (#3688 review R1-M2) so the OTHER way a
+    payload leaves this module for good — ``_trim_dir`` shedding an archived
+    copy — can be ledgered on exactly the same terms instead of leaving no
+    durable trace at all. Best-effort: a ledger that cannot be written *or
+    read back* must never take the caller down with it — hence the guard
+    catches ``ValueError`` as well as ``OSError`` (see there).
+
+    ROTATION PRIORITISES ``evicted=`` INSIDE THE SAME BUDGET, and that is a
+    direct consequence of sharing the file. Before #3688 every line here
+    was an eviction, so "keep the newest ``EVICTIONS_LOG_KEEP_LINES``
+    lines" and "keep the newest 2,000 evictions" were the same sentence.
+    They are not any more: a single ``collapse_duplicates`` run over the
+    measured 1,060-file backlog writes ~192 ``trimmed=`` lines, and a plain
+    tail-rotate would let that noise push real ``evicted=`` lines out of
+    the 7-day window ``switchroom doctor`` reads — i.e. adding
+    observability for the BENIGN channel would have removed it for the one
+    channel that means memory is actually gone.
+
+    So the keep window is filled evictions-first and only then topped up
+    with the newest remaining lines. The budget is unchanged at
+    ``EVICTIONS_LOG_KEEP_LINES`` lines TOTAL — a ledger of nothing but
+    evictions rotates exactly as it did before #3688 — and chronological
+    order is preserved because the selection is by index over the original
+    lines, which ``switchroom doctor``'s ``$1 >= cutoff`` awk depends on.
+    """
+    log = evictions_log_path()
+    try:
+        with open(log, "a", encoding="utf-8") as f:
+            print(line, file=f)
+        # Bounded, not append-forever. `switchroom doctor` windows this by
+        # timestamp so a single legitimate eviction can't turn the row red
+        # permanently, but the FILE still needs a ceiling of its own.
+        if os.path.getsize(log) > EVICTIONS_LOG_MAX_BYTES:
+            with open(log, encoding="utf-8") as f:
+                lines = f.readlines()
+            evictions = [i for i, ln in enumerate(lines) if "evicted=" in ln]
+            others = [i for i, ln in enumerate(lines) if "evicted=" not in ln]
+            keep_idx = set(evictions[-EVICTIONS_LOG_KEEP_LINES:])
+            room = EVICTIONS_LOG_KEEP_LINES - len(keep_idx)
+            if room > 0:
+                keep_idx.update(others[-room:])
+            kept = [ln for i, ln in enumerate(lines) if i in keep_idx]
+            tmp = log + ".tmp"
+            with open(tmp, "w", encoding="utf-8") as f:
+                f.writelines(kept)
+            os.chmod(tmp, 0o600)
+            os.replace(tmp, log)
+    except (OSError, ValueError):
+        # ValueError, not just OSError, because BOTH codec errors are
+        # ValueError subclasses and neither is an OSError (#3688 re-review
+        # B1). The rotate READS the ledger back (``readlines()`` above) —
+        # something no revision before #3688 did — so a ledger holding one
+        # non-UTF-8 byte raises ``UnicodeDecodeError`` here, and a name
+        # carrying a surrogate raises ``UnicodeEncodeError`` at the
+        # ``print`` above. Under a bare ``except OSError`` both escape this
+        # "best-effort" guard: ``_trim_dir`` → ``_log_archive_trim`` → here
+        # runs inside ``archive_reconciled``/``enqueue``, so one corrupt
+        # ledger byte would take down every drain — on the sidecar,
+        # identically, every 900s forever. A ledger that cannot be written
+        # must never take the caller down with it, and that has to include
+        # the codec.
+        pass
+
+
+def _log_archive_trim(
+    name: str, size: int, reason: str, archive: str, depth: int, nbytes: int
+) -> None:
+    """Ledger ONE archived copy shed by ``_trim_dir``.
+
+    Until #3688 review R1-M2 a trim was stderr-only: an entry could enter
+    ``pending-duplicate/`` (or ``pending-evicted/``, or
+    ``pending-reconciled/``) and then be deleted from it with no durable
+    record anywhere, while an *eviction* — the other end of the same
+    payload's life — has had ``pending-evictions.log`` since #3599. stderr
+    is gone with the process; the ledger is what an operator can still read
+    a week later. Same file, because it answers one question ("what left
+    this agent's queue, and when").
+
+    THE FIRST TOKEN IS DELIBERATELY ``trimmed=``, NOT ``evicted=``.
+    ``switchroom doctor``'s probe counts eviction ledger lines with
+    ``awk '$1 >= cutoff && /evicted=/'`` (src/cli/doctor.ts, see
+    ``buildPendingRetainsProbeScript``), and that count FAILS the
+    pending-retains row. A trim is not an eviction — nothing was shed from
+    the live queue and, for ``pending-duplicate/``, a byte-identical copy is
+    still queued — so a trim must not be able to turn that row red. No field
+    on this line may ever be formatted such that ``evicted=`` appears in it;
+    ``ArchiveTrimLedgerTest`` pins that.
+    """
+    _append_ledger(
+        "%s trimmed=%s bytes=%d archive=%s reason=archive-%s "
+        "archive_depth=%d archive_bytes=%d"
+        % (
+            time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+            name,
+            size,
+            archive,
+            reason,
+            depth,
+            nbytes,
+        )
+    )
+
+
 def _trim_dir(a: str, max_entries: int, max_bytes: int) -> int:
     """Keep ``a`` under its count/byte caps, deleting OLDEST first.
 
@@ -610,6 +716,21 @@ def _trim_dir(a: str, max_entries: int, max_bytes: int) -> int:
     ``pending-evicted/`` is a different story again — see ``_evict_to_fit``:
     an entry only reaches it through the ledgered eviction path, and under
     sustained ENOSPC it may not reach it at all.
+
+    EVERY DROP IS LEDGERED, AND ONLY A DROP (#3688 review R1-M2, tightened
+    by re-review B2). One ``trimmed=… archive=…`` line per shed copy goes
+    to ``pending-evictions.log`` via ``_log_archive_trim``, so the trim
+    horizon above is observable after the fact and not only in a stderr
+    stream nobody kept. That line is deliberately NOT counted as an
+    eviction by doctor — see ``_log_archive_trim``.
+
+    A victim whose ``os.remove`` FAILS (EACCES, read-only mount) is neither
+    ledgered nor counted nor named on stderr: it is still on disk, so a
+    ``trimmed=`` line for it would be a false claim of deletion in exactly
+    the record an operator consults after suspected loss, and the returned
+    count is what callers report as "copies shed". ``TrimDirBoundaryTest``
+    pins that a failed remove yields ``0``, no ledger line, and every file
+    still present.
     """
     # `.json` ONLY, and that is load-bearing beyond "skip stray files": a
     # `.dead` marker is named `<entry>.json.dead`, so it can never be
@@ -622,14 +743,35 @@ def _trim_dir(a: str, max_entries: int, max_bytes: int) -> int:
     except OSError:
         return 0
     dropped = []
-    while names and (
-        len(names) > max_entries or _dir_bytes(a, names) > max_bytes
-    ):
+    label = os.path.basename(a.rstrip("/"))
+    nbytes = _dir_bytes(a, names)
+    while names and (len(names) > max_entries or nbytes > max_bytes):
+        reason = "count" if len(names) > max_entries else "bytes"
+        victim = names.pop(0)
+        vpath = os.path.join(a, victim)
         try:
-            os.remove(os.path.join(a, names[0]))
+            vsize = os.path.getsize(vpath)
         except OSError:
-            pass
-        dropped.append(names.pop(0))
+            vsize = 0
+        try:
+            os.remove(vpath)
+        except OSError:
+            # The copy is still on disk, so the running total would drift.
+            # Re-measure rather than assume — this preserves the original
+            # recompute-every-iteration semantics on the one path where a
+            # decrement would be wrong (EACCES / read-only mount).
+            nbytes = _dir_bytes(a, names)
+            # NOT counted and NOT ledgered (#3688 re-review B2). The ledger
+            # is the forensic record an operator reads AFTER suspected data
+            # loss; a `trimmed=` line for a copy still sitting on disk sends
+            # them hunting a payload that never left, and the returned count
+            # (and the stderr line below) would name files that were not
+            # dropped. A ledger that claims deletions which never happened
+            # is worse than no ledger.
+            continue
+        nbytes -= vsize
+        dropped.append(victim)
+        _log_archive_trim(victim, vsize, reason, label, len(names), nbytes)
     if dropped:
         shown = ", ".join(dropped[:10])
         if len(dropped) > 10:
@@ -736,6 +878,47 @@ def archive_duplicate(path: str) -> Optional[str]:
     return dest
 
 
+def _attempt_count(entry: dict) -> int:
+    """How many retries this copy has BURNED, for survivor selection.
+
+    Not ``int(entry.get("attempt_count", 1) or 1)`` (#3688 review R1-L3).
+    That expression had two defects, both latent rather than reachable
+    today — ``_build_entry`` seeds ``1`` and ``update_attempt`` only ever
+    increments — which is exactly why they need pinning rather than
+    ignoring: nothing would have caught them turning real.
+
+    * ``0 or 1`` is ``1``, so a NEVER-ATTEMPTED copy scored the same as one
+      that had already burned an attempt, and lost the tie-break to it if
+      it happened to be the newer file. That is backwards from the rule
+      ``collapse_duplicates`` documents ("the copy with the most retries
+      left before ``MAX_ATTEMPTS`` wins") — a 0 must beat a 1.
+    * ``int("many")`` RAISES. A single semantically-malformed entry — valid
+      JSON, so ``iter_entries`` hands it over rather than quarantining it —
+      took the whole collapse pass down with it, and with it (before the
+      guard in ``drain_pending._drain_backlog_impl``) the whole drain.
+      Unparseable now scores ``MAX_ATTEMPTS``: we cannot tell how many
+      attempts it has left, so it is never PREFERRED as the survivor, but
+      it is never the reason a collapse fails either. Losing it costs
+      nothing anyway — every copy in a group is byte-identical, and the
+      loser is archived, not deleted.
+
+    A missing or ``None`` count reads as ``1``, matching what
+    ``_build_entry`` writes, so an entry from an older build that predates
+    the field is treated as freshly queued rather than exhausted.
+    """
+    raw = entry.get("attempt_count", 1)
+    if raw is None:
+        return 1
+    # bool is an int subclass; True would silently score 1. Neither True nor
+    # False is an attempt count, so both take the unparseable branch.
+    if isinstance(raw, bool):
+        return MAX_ATTEMPTS
+    try:
+        return int(raw)
+    except (TypeError, ValueError):
+        return MAX_ATTEMPTS
+
+
 def collapse_duplicates() -> int:
     """Collapse entries sharing ``(bank_id, part_position, sha256(content))``.
 
@@ -754,7 +937,8 @@ def collapse_duplicates() -> int:
     memory; the one with the most attempts left is the one most likely to
     get there before ``MAX_ATTEMPTS`` promotes it to ``.dead``. Picking the
     oldest outright would systematically keep the most-attempted copy —
-    exactly backwards.
+    exactly backwards. ``_attempt_count`` defines "lowest", including what
+    a ``0`` and what a malformed count are worth.
 
     An entry whose key is ``None`` (no ``content``) is never grouped: its
     identity cannot be established, so it is always kept.
@@ -773,10 +957,7 @@ def collapse_duplicates() -> int:
     for key, members in groups.items():
         if len(members) < 2:
             continue
-        survivor = min(
-            members,
-            key=lambda m: (int(m[2].get("attempt_count", 1) or 1), m[0]),
-        )
+        survivor = min(members, key=lambda m: (_attempt_count(m[2]), m[0]))
         for member in members:
             if member is survivor:
                 continue
@@ -1047,23 +1228,7 @@ def _log_eviction(name: str, size: int, reason: str, depth: int, nbytes: int) ->
         depth,
         nbytes,
     )
-    log = evictions_log_path()
-    try:
-        with open(log, "a", encoding="utf-8") as f:
-            print(line, file=f)
-        # Bounded, not append-forever. `switchroom doctor` windows this by
-        # timestamp so a single legitimate eviction can't turn the row red
-        # permanently, but the FILE still needs a ceiling of its own.
-        if os.path.getsize(log) > EVICTIONS_LOG_MAX_BYTES:
-            with open(log, encoding="utf-8") as f:
-                kept = f.readlines()[-(EVICTIONS_LOG_KEEP_LINES):]
-            tmp = log + ".tmp"
-            with open(tmp, "w", encoding="utf-8") as f:
-                f.writelines(kept)
-            os.chmod(tmp, 0o600)
-            os.replace(tmp, log)
-    except OSError:
-        pass
+    _append_ledger(line)
     print(
         "[Hindsight] pending-retains FULL - evicted OLDEST entry to keep the "
         "newest memory: %s (%d bytes, %s; queue now %d entries / %d bytes). "

package/vendor/hindsight-memory/scripts/tests/test_drain_circuit_breaker.py

@@ -0,0 +1,401 @@
+"""A permanently-failing entry must stop costing LLM lane time.
+
+Lives under ``scripts/tests/`` because that is the only python test directory
+CI discovers (``ci-tests-python.yml`` runs ``python3 -m unittest discover
+tests/`` with ``working-directory: vendor/hindsight-memory/scripts``).
+
+BACKGROUND. The permanence gate (``pending.is_permanent_failure``) keeps an
+entry queued past ``MAX_ATTEMPTS`` on anything but a 4xx, and ``_over_budget``
+/ ``_drain_order`` then DEMOTE it so it cannot starve healthy entries behind
+it. Both are right, and together they bound head-of-line blocking — but not
+TOTAL work: the entry is still retried in full on every single run, forever.
+
+That was affordable when every drain was supervised: a 4-second in-hook pass,
+or an operator watching a manual sweep. The ``hindsight-drain`` sidecar makes
+it unaffordable. Its backlog budget is 3600s (``_backlog_budget_seconds``)
+against a 900s cooldown, so a wedged agent spends ~80% of wall-clock draining,
+unattended, against 4 lanes shared with live retains, reflect and
+consolidation — indefinitely. The interim host stopgap recorded exactly that
+shape while it ran: ``finn 280s drained=0 retried=3 STALLED``, ``carrie 564s
+drained=0 retried=2 (lane busy)``.
+
+So an entry past ``_attempt_ceiling()`` is PARKED. Parking is not retirement:
+the entry stays on disk, keeps being swept for free by the reconcile pass, and
+comes back in full under ``--force``. What it stops is paying ~168s of a
+shared lane, again, for a POST that has already failed 20 times.
+
+Every test here fails without the ``_park_broken`` call in the drain paths:
+the parked entry is retained again, which is the unbounded-retry behaviour.
+"""
+
+import io
+import json
+import os
+import re
+import sys
+import unittest
+import unittest.mock
+from contextlib import redirect_stderr
+
+SCRIPTS_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+if SCRIPTS_DIR not in sys.path:
+    sys.path.insert(0, SCRIPTS_DIR)
+
+import drain_pending  # noqa: E402
+import lib.pending as pending  # noqa: E402
+
+from tests.test_pending_drops import (  # noqa: E402
+    CONFIG,
+    _QueueTempDirMixin,
+    _cd_doc,
+    _payload,
+)
+
+
+class CircuitBreakerTest(_QueueTempDirMixin, unittest.TestCase):
+    ENV_KEYS = _QueueTempDirMixin.ENV_KEYS + ("HINDSIGHT_DRAIN_ATTEMPT_CEILING",)
+
+    def setUp(self):
+        super().setUp()
+        os.environ.pop("HINDSIGHT_DRAIN_ATTEMPT_CEILING", None)
+        self.posted = []
+
+    def _seed(self, attempts_by_content):
+        """Queue one entry per ``{content: attempt_count}`` pair."""
+        clock = iter(1000.0 + i for i in range(50))
+        with unittest.mock.patch.object(pending.time, "time", lambda: next(clock)):
+            for i, content in enumerate(attempts_by_content):
+                pending.enqueue(
+                    _payload(content=content, doc=_cd_doc(i)), RuntimeError("x")
+                )
+        for path, entry in pending.iter_entries():
+            entry["attempt_count"] = attempts_by_content[entry["content"]]
+            with open(path, "w", encoding="utf-8") as f:
+                json.dump(entry, f)
+
+    def _record_post(self, entry, timeout):
+        self.posted.append(entry["content"])
+
+    def _drain(self, present=False, **kw):
+        with unittest.mock.patch.object(
+            drain_pending, "_retry_one", self._record_post
+        ):
+            with unittest.mock.patch.object(
+                drain_pending, "_document_state", lambda e, timeout=30: present
+            ):
+                with redirect_stderr(io.StringIO()) as err:
+                    summary = drain_pending.drain(CONFIG, **kw)
+        return summary, err.getvalue()
+
+    # ── the breaker itself ────────────────────────────────────────────
+
+    def test_the_unattended_backlog_drain_stops_retrying_a_wedged_entry(self):
+        ceiling = drain_pending._attempt_ceiling()
+        self._seed({"wedged": ceiling, "healthy": 1})
+
+        summary, err = self._drain(backlog=True, phase="drain")
+
+        self.assertEqual(
+            self.posted,
+            ["healthy"],
+            "an entry past the attempt ceiling must not be retained again",
+        )
+        self.assertEqual(summary["parked"], 1)
+        self.assertIn("parked 1 entries past the attempt ceiling", err)
+
+    def test_parking_is_not_retirement_the_entry_is_still_on_disk(self):
+        """The whole permanence gate rests on never destroying a memory. A
+        breaker that deleted, or promoted to ``.dead``, would trade the
+        wedge for the data loss the gate exists to prevent."""
+        ceiling = drain_pending._attempt_ceiling()
+        self._seed({"wedged": ceiling})
+
+        self._drain(backlog=True, phase="drain")
+
+        self.assertEqual(pending.count(), 1)
+        self.assertEqual(
+            [e["content"] for _p, e in pending.iter_entries()], ["wedged"]
+        )
+        self.assertEqual(
+            [n for n in os.listdir(self._dir) if n.endswith(".dead")],
+            [],
+            "parking must not retire the entry",
+        )
+
+    def test_a_parked_entry_is_still_reconciled_for_free(self):
+        """Parking withholds the expensive retry, not the cheap proof: if the
+        document DID land, the free presence pass still retires the entry."""
+        ceiling = drain_pending._attempt_ceiling()
+        self._seed({"wedged": ceiling})
+
+        summary, _ = self._drain(backlog=True, present=True, phase="reconcile")
+
+        self.assertEqual(summary["reconciled"], 1)
+        self.assertEqual(self.posted, [], "no POST was needed")
+        self.assertEqual(pending.count(), 0)
+
+    def test_force_retries_the_parked_entries_anyway(self):
+        """The escape hatch. Without it the breaker would be a one-way door,
+        and an operator who fixed the upstream could never replay."""
+        ceiling = drain_pending._attempt_ceiling()
+        self._seed({"wedged": ceiling, "healthy": 1})
+
+        summary, _ = self._drain(backlog=True, phase="drain", force=True)
+
+        self.assertEqual(sorted(self.posted), ["healthy", "wedged"])
+        self.assertEqual(summary["parked"], 0)
+
+    def test_an_entry_one_attempt_below_the_ceiling_still_drains(self):
+        """The boundary. A breaker that fired early would park entries the
+        demotion logic is still successfully draining."""
+        ceiling = drain_pending._attempt_ceiling()
+        self._seed({"nearly": ceiling - 1})
+
+        summary, _ = self._drain(backlog=True, phase="drain")
+
+        self.assertEqual(self.posted, ["nearly"])
+        self.assertEqual(summary["parked"], 0)
+
+    def test_the_in_hook_drain_parks_too(self):
+        """Same policy on the SessionStart path: a 4s budget spent on an
+        entry that has failed 20 times is 4s not spent on the rest."""
+        ceiling = drain_pending._attempt_ceiling()
+        self._seed({"wedged": ceiling, "healthy": 1})
+
+        summary, _ = self._drain()
+
+        self.assertEqual(self.posted, ["healthy"])
+        self.assertEqual(summary["parked"], 1)
+
+    def test_repeated_unattended_runs_never_touch_the_wedged_entry_again(self):
+        """The actual failure mode: not one wasted run, but every run from
+        here to the heat death of the container."""
+        ceiling = drain_pending._attempt_ceiling()
+        self._seed({"wedged": ceiling})
+
+        for _ in range(5):
+            self._drain(backlog=True, phase="drain")
+
+        self.assertEqual(self.posted, [], "5 unattended ticks, zero lane time")
+        self.assertEqual(pending.count(), 1, "and the memory is still there")
+
+    # ── the ceiling ───────────────────────────────────────────────────
+
+    def test_the_default_ceiling_is_a_multiple_of_the_attempt_budget(self):
+        self.assertEqual(
+            drain_pending._attempt_ceiling(),
+            pending.MAX_ATTEMPTS * drain_pending.ATTEMPT_CEILING_MULTIPLE,
+        )
+        self.assertGreater(
+            drain_pending._attempt_ceiling(),
+            pending.MAX_ATTEMPTS,
+            "parking at the attempt budget would collide with demotion",
+        )
+
+    def test_the_ceiling_is_operator_tunable(self):
+        os.environ["HINDSIGHT_DRAIN_ATTEMPT_CEILING"] = "7"
+        self.assertEqual(drain_pending._attempt_ceiling(), 7)
+        self._seed({"wedged": 7, "healthy": 6})
+
+        summary, _ = self._drain(backlog=True, phase="drain")
+
+        self.assertEqual(self.posted, ["healthy"])
+        self.assertEqual(summary["parked"], 1)
+
+    def test_a_ceiling_below_the_attempt_budget_is_clamped_up(self):
+        """Below ``MAX_ATTEMPTS`` the breaker would park entries before the
+        ordinary retry policy has finished with them."""
+        os.environ["HINDSIGHT_DRAIN_ATTEMPT_CEILING"] = "1"
+        self.assertEqual(drain_pending._attempt_ceiling(), pending.MAX_ATTEMPTS)
+
+    def test_a_garbage_ceiling_falls_back_to_the_default(self):
+        os.environ["HINDSIGHT_DRAIN_ATTEMPT_CEILING"] = "not-a-number"
+        self.assertEqual(
+            drain_pending._attempt_ceiling(),
+            pending.MAX_ATTEMPTS * drain_pending.ATTEMPT_CEILING_MULTIPLE,
+        )
+
+    def test_a_corrupt_attempt_counter_never_parks_and_never_raises(self):
+        """Same rule as ``_over_budget``: a hand-edited counter must not
+        decide policy, and must not blow up the drain path."""
+        self.assertFalse(drain_pending._circuit_broken({"attempt_count": "many"}))
+        self.assertFalse(drain_pending._circuit_broken({"attempt_count": None}))
+        self.assertFalse(drain_pending._circuit_broken({}))
+
+    # ── reporting: parking must be VISIBLE, on every path ─────────────
+
+    def _main(self, argv, present=False):
+        """Run the CLI entrypoint, capturing what an operator would see."""
+        with unittest.mock.patch.object(drain_pending, "load_config", lambda: CONFIG):
+            with unittest.mock.patch.object(
+                drain_pending, "_retry_one", self._record_post
+            ):
+                with unittest.mock.patch.object(
+                    drain_pending, "_document_state", lambda e, timeout=30: present
+                ):
+                    with redirect_stderr(io.StringIO()) as err:
+                        rc = drain_pending.main(argv)
+        return rc, err.getvalue()
+
+    def test_an_in_hook_run_that_only_parks_still_says_so(self):
+        """THE OBSERVABILITY HOLE THE CIRCUIT BREAKER OPENS.
+
+        A run whose entire queue is past the ceiling parks everything and
+        does nothing else, so every other summary counter is 0. With
+        ``parked`` missing from ``main()``'s gate that run printed NOTHING —
+        rc=0, empty stdout, empty stderr — which is byte-identical to "the
+        queue was empty". The operator watching a queue that will not shrink
+        then has no way to tell "parked by the breaker, needs ``--force``"
+        from "ordinary backlog, drains on its own", because ``switchroom
+        doctor`` reads only the queue DIRECTORY and both look like a file.
+
+        The ``--backlog`` path narrates its own parking. This is the IN-HOOK
+        path, which is the one that runs on every session boot.
+        """
+        ceiling = drain_pending._attempt_ceiling()
+        self._seed({"a": ceiling, "b": ceiling, "c": ceiling})
+
+        rc, err = self._main([])
+
+        self.assertEqual(rc, 0)
+        self.assertEqual(self.posted, [], "nothing was retried — all parked")
+        self.assertNotEqual(
+            err.strip(), "", "a run that parked the whole queue must not be silent"
+        )
+        self.assertIn("parked=3", err)
+
+    def test_the_summary_line_reports_parked_alongside_other_work(self):
+        """The gate is not the only half. With ``parked`` in the gate but
+        absent from the printed line, a mixed run opens the gate on its other
+        counters and still never names the parked entries."""
+        ceiling = drain_pending._attempt_ceiling()
+        self._seed({"wedged": ceiling, "healthy": 1})
+
+        rc, err = self._main([], present=True)
+
+        self.assertEqual(rc, 0)
+        self.assertIn("reconciled=1", err)
+        self.assertIn("parked=1", err)
+
+
+class CeilingArithmeticTest(_QueueTempDirMixin, unittest.TestCase):
+    """How long 20 attempts actually buys — measured, not asserted in prose.
+
+    ``ATTEMPT_CEILING_MULTIPLE``'s comment used to claim 20 attempts was
+    "most of a day … far past any transient upstream outage". It is not, and
+    the reason is the non-obvious part: attempts convert to wall-clock only
+    via the QUEUE SIZE. A drain run attempts each live entry at most once and
+    ``enqueue`` already records attempt 1, so the floor is 19 further runs —
+    under 5 hours at the sidecar's 900s cooldown. ``STALL_THRESHOLD`` only
+    rations attempts while entries are inside ``MAX_ATTEMPTS``; past it they
+    abstain from the stall guard, so a uniformly over-budget queue is
+    attempted in full on every run.
+
+    This measures the real figures and pins the comment's table to them, so
+    the prose cannot drift away from the code again: change the mechanism and
+    the measured column moves; delete the table and the parse fails.
+    """
+
+    ENV_KEYS = _QueueTempDirMixin.ENV_KEYS + ("HINDSIGHT_DRAIN_ATTEMPT_CEILING",)
+
+    #: Rows the module comment must carry: queue size → drain runs until the
+    #: WHOLE queue is parked, under a total upstream outage.
+    SIZES = (1, 4, 13, 30)
+
+    #: The sidecar's default cooldown (``profiles/_base/start.sh.hbs``); the
+    #: comment's hours column is runs x this.
+    COOLDOWN_S = 900
+
+    def setUp(self):
+        super().setUp()
+        os.environ.pop("HINDSIGHT_DRAIN_ATTEMPT_CEILING", None)
+        # No p95 backoff, no pacing: this test counts RUNS, not seconds.
+        os.environ.pop("HINDSIGHT_DRAIN_P95_CMD", None)
+
+    def _runs_until_wholly_parked(self, n: int) -> int:
+        # Own queue dir per call, so several sizes can be measured in one
+        # test without one size's entries contaminating the next. The mixin
+        # owns HINDSIGHT_PENDING_DIR and restores it in tearDown; the
+        # directory itself lives under the mixin's tmp root and goes with it.
+        os.environ["HINDSIGHT_PENDING_DIR"] = os.path.join(
+            self._tmp, f"pending-retains-{n}"
+        )
+        clock = iter(1000.0 + i for i in range(n + 10))
+        with unittest.mock.patch.object(pending.time, "time", lambda: next(clock)):
+            for i in range(n):
+                pending.enqueue(
+                    _payload(content=f"e{i}", doc=_cd_doc(i)),
+                    ConnectionError("upstream down"),
+                )
+
+        def always_fails(entry, timeout=30):
+            raise ConnectionError("upstream down")
+
+        runs = 0
+        # Generous ceiling on the loop itself: a mechanism change that made
+        # parking unreachable must fail loudly rather than hang CI.
+        while runs < 500:
+            runs += 1
+            with unittest.mock.patch.object(
+                drain_pending, "_retry_one", always_fails
+            ):
+                with unittest.mock.patch.object(
+                    drain_pending, "_document_state", lambda e, timeout=30: False
+                ):
+                    with redirect_stderr(io.StringIO()):
+                        summary = drain_pending.drain(
+                            CONFIG, backlog=True, phase="drain"
+                        )
+            if summary["parked"] == n:
+                return runs
+        self.fail(f"a {n}-entry queue never parked wholly within 500 runs")
+
+    def _documented_table(self) -> dict:
+        """Parse the table out of ``ATTEMPT_CEILING_MULTIPLE``'s comment."""
+        with open(drain_pending.__file__, encoding="utf-8") as f:
+            src = f.read()
+        rows = re.findall(r"^#: (\d+)\s+(\d+)\s+([\d.]+) h$", src, re.M)
+        return {int(q): (int(r), float(h)) for q, r, h in rows}
+
+    def test_the_documented_table_is_what_the_code_actually_does(self):
+        documented = self._documented_table()
+        self.assertEqual(
+            sorted(documented),
+            sorted(self.SIZES),
+            "ATTEMPT_CEILING_MULTIPLE's comment must document exactly these "
+            "queue sizes — the prose is the deliverable here",
+        )
+        for n in self.SIZES:
+            with self.subTest(queue=n):
+                measured = self._runs_until_wholly_parked(n)
+                runs, hours = documented[n]
+                self.assertEqual(
+                    measured,
+                    runs,
+                    f"a {n}-entry queue parks wholly in {measured} runs, but "
+                    f"the comment says {runs}",
+                )
+                self.assertAlmostEqual(
+                    hours, runs * self.COOLDOWN_S / 3600.0, places=1
+                )
+
+    def test_a_small_queue_parks_well_inside_one_overnight_outage(self):
+        """The claim the old comment made, stated as the outcome it is not.
+
+        4 entries — a healthy agent — park entirely in 6 hours. So a single
+        overnight upstream outage CAN park a small queue wholesale, which is
+        exactly what "far past any transient upstream outage" denied. Pinned
+        as an outcome so nobody re-asserts the denial.
+        """
+        measured = self._runs_until_wholly_parked(4)
+        hours = measured * self.COOLDOWN_S / 3600.0
+        self.assertLess(
+            hours,
+            12.0,
+            "a 4-entry queue parking in under 12h is the accepted trade — if "
+            "this ever became untrue the comment must be re-measured too",
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()