switchroom 0.19.19 → 0.19.22

package/vendor/hindsight-memory/scripts/tests/test_pending_drops.py

@@ -26,6 +26,8 @@ import tempfile
 import time
 import unittest
 import unittest.mock
+import urllib.error
+import urllib.request
 from contextlib import redirect_stderr
 
 SCRIPTS_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
@@ -1064,24 +1066,78 @@ class BacklogDrainTest(_QueueTempDirMixin, unittest.TestCase):
         )
         self.assertEqual(pending.count(), 12, "stalled entries stay queued")
 
-    def test_backlog_ages_entries_toward_dead_like_the_sequential_drain(self):
+    def _exhaust_attempts(self):
+        """Queue one entry already sitting on the MAX_ATTEMPTS boundary."""
         self._queue(1)
         path, entry = pending.iter_entries()[0]
         entry["attempt_count"] = pending.MAX_ATTEMPTS
         with open(path, "w", encoding="utf-8") as f:
             json.dump(entry, f)
+        return path
 
+    def _drain_failing_with(self, error):
         def always_fail(entry, timeout):
-            raise ConnectionError("upstream down")
+            raise error
 
         with unittest.mock.patch.object(drain_pending, "_retry_one", always_fail):
             with redirect_stderr(io.StringIO()):
-                summary = drain_pending.drain_backlog(CONFIG, phase="drain")
+                return drain_pending.drain_backlog(CONFIG, phase="drain")
+
+    def test_backlog_ages_entries_toward_dead_like_the_sequential_drain(self):
+        """A PERMANENT failure past MAX_ATTEMPTS still retires the entry."""
+        path = self._exhaust_attempts()
+        summary = self._drain_failing_with(
+            RuntimeError('HTTP 400 from http://h/v1/x: {"detail":"bad payload"}')
+        )
 
         self.assertEqual(summary["dead"], 1)
         self.assertTrue(os.path.exists(path + ".dead"))
         self.assertFalse(os.path.exists(path))
 
+    def test_extraction_500_past_max_attempts_never_kills_the_memory(self):
+        """The regression this gate exists for.
+
+        An HTTP 500 "Fact extraction failed … JSONDecodeError" means the
+        extraction model returned an empty or non-JSON completion on THAT
+        sampling run. The identical content persists fine on a later attempt,
+        so exhausting the attempt budget on it must NOT retire the memory.
+        Before the ``is_permanent_failure`` gate this drain produced
+        ``dead=1`` and removed the queue entry.
+        """
+        path = self._exhaust_attempts()
+        summary = self._drain_failing_with(
+            RuntimeError(
+                "HTTP 500 from http://h/v1/x: "
+                '{"detail":"Fact extraction failed: 1/1 chunks failed. '
+                "First failures: chunk 0: JSONDecodeError: Expecting value: "
+                'line 1 column 1 (char 0)"}'
+            )
+        )
+
+        self.assertEqual(summary["dead"], 0, "a 5xx must never retire a memory")
+        self.assertEqual(summary["retried"], 1)
+        self.assertFalse(os.path.exists(path + ".dead"))
+        self.assertTrue(os.path.exists(path), "the queued memory survives")
+        self.assertEqual(pending.count(), 1)
+
+    def test_transient_transport_failure_past_max_attempts_stays_queued(self):
+        """Timeouts / connection errors are transient too — never ``.dead``."""
+        for error in (ConnectionError("upstream down"), TimeoutError("timed out")):
+            with self.subTest(error=type(error).__name__):
+                path = self._exhaust_attempts()
+                summary = self._drain_failing_with(error)
+                self.assertEqual(summary["dead"], 0)
+                self.assertTrue(os.path.exists(path))
+                os.unlink(path)
+
+    def test_attempt_count_keeps_climbing_past_the_budget_on_transients(self):
+        """The counter still records reality; only ``.dead`` is gated."""
+        path = self._exhaust_attempts()
+        self._drain_failing_with(ConnectionError("upstream down"))
+        with open(path, encoding="utf-8") as f:
+            entry = json.load(f)
+        self.assertEqual(entry["attempt_count"], pending.MAX_ATTEMPTS + 1)
+
     def test_dry_run_issues_no_writes(self):
         self._queue(3)
         with unittest.mock.patch.object(
@@ -1095,6 +1151,96 @@ class BacklogDrainTest(_QueueTempDirMixin, unittest.TestCase):
         self.assertEqual(pending.count(), 3, "a dry run must not delete anything")
 
 
+class TransportFailureEndToEndTest(_QueueTempDirMixin, unittest.TestCase):
+    """The permanence gate over the REAL client wrapping (#3669 follow-up).
+
+    Every other permanence test injects a synthetic exception at
+    ``drain_pending._retry_one`` — i.e. ABOVE ``lib/client.py``. That skips
+    the layer which actually decides what a fault LOOKS like to
+    ``pending.is_permanent_failure``: ``client._request`` catches only
+    ``HTTPError`` and re-raises it as ``RuntimeError("HTTP {code} from …")``
+    (lib/client.py:91-97), while a bare ``URLError`` propagates untouched.
+    So the gate's input shape is produced there and asserted nowhere.
+
+    These tests patch ``urllib.request.urlopen`` instead and drive the real
+    drain, keeping that wrapping in the loop. #3669 added an equivalent
+    end-to-end case, but to ``vendor/hindsight-memory/tests/`` — which no
+    workflow discovers (ci-tests-python.yml runs ``unittest discover
+    tests/`` from ``vendor/hindsight-memory/scripts``), so it never ran.
+    """
+
+    def _queue_one(self):
+        pending.enqueue(_payload(content="turn-0", doc=_cd_doc(0)), RuntimeError("boom"))
+        self.assertEqual(pending.count(), 1)
+
+    def _exhaust_attempts(self):
+        """One entry sitting on the MAX_ATTEMPTS boundary."""
+        self._queue_one()
+        path, entry = pending.iter_entries()[0]
+        entry["attempt_count"] = pending.MAX_ATTEMPTS
+        with open(path, "w", encoding="utf-8") as f:
+            json.dump(entry, f)
+        return path
+
+    @staticmethod
+    def _http_error(code):
+        def raise_it(*_a, **_kw):
+            raise urllib.error.HTTPError(
+                "http://127.0.0.1:9/none/v1/retain",
+                code,
+                "rejected",
+                {},
+                io.BytesIO(b'{"detail":"rejected"}'),
+            )
+
+        return raise_it
+
+    def _drain_with_urlopen(self, side_effect):
+        with unittest.mock.patch("urllib.request.urlopen", side_effect=side_effect):
+            with redirect_stderr(io.StringIO()):
+                return drain_pending.drain_backlog(CONFIG, phase="drain")
+
+    def test_outage_past_max_attempts_keeps_the_memory(self):
+        """A dead daemon raises bare ``URLError`` — no ``.code``, so transient.
+
+        Before the gate this retired the entry at MAX_ATTEMPTS: an outage
+        destroyed memory that was perfectly saveable.
+        """
+        path = self._exhaust_attempts()
+
+        def down(*_a, **_kw):
+            raise urllib.error.URLError("still down")
+
+        summary = self._drain_with_urlopen(down)
+        self.assertEqual(summary["dead"], 0, "an outage must never retire a memory")
+        self.assertTrue(os.path.exists(path), "the queued memory survives")
+        self.assertFalse(os.path.exists(path + ".dead"))
+
+    def test_rate_limit_past_max_attempts_keeps_the_memory(self):
+        """429 wears a 4xx code but is the most transient fault there is.
+
+        This is the case the raw ``400 <= code < 500`` rule would get
+        wrong, asserted through the real ``HTTP {code} from …`` wrapping.
+        """
+        path = self._exhaust_attempts()
+        summary = self._drain_with_urlopen(self._http_error(429))
+        self.assertEqual(summary["dead"], 0, "a rate limit must never retire a memory")
+        self.assertTrue(os.path.exists(path))
+        self.assertFalse(os.path.exists(path + ".dead"))
+
+    def test_server_rejection_past_max_attempts_still_retires(self):
+        """The other half of the gate: a fix that never retires is a leak.
+
+        A positively-rejected payload must still age to ``.dead``, or the
+        queue grows without bound.
+        """
+        path = self._exhaust_attempts()
+        summary = self._drain_with_urlopen(self._http_error(400))
+        self.assertEqual(summary["dead"], 1, "a 4xx rejection must still retire")
+        self.assertTrue(os.path.exists(path + ".dead"))
+        self.assertFalse(os.path.exists(path))
+
+
 class ArchiveFailureNeverDeletesTest(_QueueTempDirMixin, unittest.TestCase):
     """A failure to ARCHIVE is not a licence to DELETE (#3599 review R3-M1).
 
@@ -1696,7 +1842,15 @@ class ClampAndEnvKnobBoundaryTest(_QueueTempDirMixin, unittest.TestCase):
         """
         os.environ.pop("HINDSIGHT_DRAIN_BACKLOG_TIMEOUT", None)
         os.environ.pop("HINDSIGHT_RETAIN_CLIENT_DEADLINE_S", None)
-        self.assertEqual(drain_pending._backlog_timeout(), 280)
+        # Assert the DERIVATION, per this test's own docstring: the drain
+        # deadline is retain_client_deadline(), whatever that currently is.
+        # It was pinned to the literal 280 here, which meant the test went
+        # stale the moment the deadline became derived from the litellm
+        # routing chain (280 -> 310) instead of being hand-set.
+        self.assertEqual(
+            drain_pending._backlog_timeout(),
+            int(retain_split.retain_client_deadline()),
+        )
 
         os.environ["HINDSIGHT_RETAIN_CLIENT_DEADLINE_S"] = "600"
         try:

package/vendor/hindsight-memory/scripts/tests/test_pending_failure_class.py

@@ -0,0 +1,105 @@
+"""``pending.is_permanent_failure`` — the gate on retiring a memory.
+
+Lives under ``scripts/tests/`` because that is the only python test
+directory CI discovers (``ci-tests-python.yml`` runs ``python3 -m unittest
+discover tests/`` with ``working-directory: vendor/hindsight-memory/scripts``).
+
+The rule under test is asymmetric on purpose. Misclassifying a permanent
+failure as retryable costs a re-POST (an upsert — time). Misclassifying a
+transient failure as permanent costs the user's memory. So only a positively
+identified client-side 4xx is permanent; everything else, including anything
+unrecognised, is retryable.
+"""
+
+import os
+import sys
+import unittest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from lib.pending import is_permanent_failure  # noqa: E402
+
+
+def _http(code, body=""):
+    """The shape ``client.HindsightClient._request`` raises."""
+    return RuntimeError(f"HTTP {code} from http://hindsight/v1/x: {body}")
+
+
+class TestIsPermanentFailure(unittest.TestCase):
+    def test_client_errors_are_permanent(self):
+        for code in (400, 401, 403, 404, 409, 413, 422):
+            with self.subTest(code=code):
+                self.assertTrue(is_permanent_failure(_http(code)))
+
+    def test_transient_4xx_are_not_permanent(self):
+        """408/425/429 wear a 4xx status but describe a transient state."""
+        for code in (408, 425, 429):
+            with self.subTest(code=code):
+                self.assertFalse(is_permanent_failure(_http(code)))
+
+    def test_server_errors_are_retryable(self):
+        for code in (500, 502, 503, 504):
+            with self.subTest(code=code):
+                self.assertFalse(is_permanent_failure(_http(code)))
+
+    def test_extraction_failure_500_is_retryable(self):
+        """The measured dominant failure (2026-07-26 fleet backlog).
+
+        Both observed variants are one bad sampling run, not bad content:
+        an empty completion, and a numbered prose list where JSON was asked
+        for. Neither is evidence the memory can never be persisted.
+        """
+        empty = _http(
+            500,
+            '{"detail":"Fact extraction failed: 1/1 chunks failed. First '
+            'failures: chunk 0: JSONDecodeError: Expecting value: line 1 '
+            'column 1 (char 0)"}',
+        )
+        prose = _http(
+            500,
+            '{"detail":"Fact extraction failed: 1/1 chunks failed. First '
+            'failures: chunk 0: JSONDecodeError: Extra data: line 1 column 2 '
+            '(char 1)"}',
+        )
+        self.assertFalse(is_permanent_failure(empty))
+        self.assertFalse(is_permanent_failure(prose))
+
+    def test_transport_failures_are_retryable(self):
+        for err in (
+            TimeoutError("timed out"),
+            ConnectionError("connection reset"),
+            OSError("network unreachable"),
+        ):
+            with self.subTest(err=type(err).__name__):
+                self.assertFalse(is_permanent_failure(err))
+
+    def test_unclassifiable_error_is_retryable(self):
+        """The fail-safe direction: unknown means keep the memory."""
+        self.assertFalse(is_permanent_failure(RuntimeError("something odd")))
+        self.assertFalse(is_permanent_failure(RuntimeError("HTTP  from x")))
+        self.assertFalse(is_permanent_failure(RuntimeError("HTTP notanumber")))
+
+    def test_status_read_from_a_chained_cause(self):
+        """A live ``urllib.error.HTTPError`` chained on ``__cause__``."""
+
+        class _FakeHTTPError(Exception):
+            code = 404
+
+        wrapped = RuntimeError("upstream rejected the retain")
+        wrapped.__cause__ = _FakeHTTPError()
+        self.assertTrue(is_permanent_failure(wrapped))
+
+        wrapped5xx = RuntimeError("upstream blew up")
+        cause = _FakeHTTPError()
+        cause.code = 503
+        wrapped5xx.__cause__ = cause
+        self.assertFalse(is_permanent_failure(wrapped5xx))
+
+    def test_direct_code_attribute_wins(self):
+        err = OSError("boom")
+        err.code = 400
+        self.assertTrue(is_permanent_failure(err))
+
+
+if __name__ == "__main__":
+    unittest.main()

package/vendor/hindsight-memory/scripts/tests/test_pending_wedge.py

@@ -0,0 +1,300 @@
+"""The drain must never wedge on entries the permanence gate keeps forever.
+
+Lives under ``scripts/tests/`` because that is the only python test directory
+CI discovers (``ci-tests-python.yml`` runs ``python3 -m unittest discover
+tests/`` with ``working-directory: vendor/hindsight-memory/scripts``).
+
+BACKGROUND. ``pending.is_permanent_failure`` stopped the drain retiring a
+memory to ``.dead`` on a transient failure — correct, because a flaky
+extraction model was destroying memories that were never unsaveable. But
+``.dead`` was doing double duty: it was the honesty policy AND it was the
+queue's only un-wedging mechanism. An entry that fails transiently on every
+drain (an oversized payload enqueued before #3610 split them, say, which
+``lib/pending``'s own docstring notes "still never drains until it is split")
+is now immortal. ``iter_entries()`` is oldest-first and backlog concurrency
+defaults to 1, so those immortal entries sit at the head of every run, and
+three of them trip ``STALL_THRESHOLD`` before anything behind them is tried.
+
+Without the ``_drain_order`` / ``_over_budget`` demotions this file pins,
+that produced ``stalled=True, drained=0`` on EVERY run, forever, with healthy
+entries queued behind and never attempted.
+"""
+
+import io
+import json
+import os
+import sys
+import time
+import unittest
+import unittest.mock
+from contextlib import redirect_stderr
+
+SCRIPTS_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+if SCRIPTS_DIR not in sys.path:
+    sys.path.insert(0, SCRIPTS_DIR)
+
+import drain_pending  # noqa: E402
+import lib.pending as pending  # noqa: E402
+
+from tests.test_pending_drops import (  # noqa: E402
+    CONFIG,
+    _QueueTempDirMixin,
+    _cd_doc,
+    _payload,
+)
+
+#: The immortal entries: they fail with a TRANSIENT error on every drain, so
+#: the permanence gate correctly refuses to retire them — forever.
+POISON = {"c0", "c1", "c2"}
+
+
+class DrainWedgeTest(_QueueTempDirMixin, unittest.TestCase):
+    def _seed(self, n, attempts_for):
+        # Entry filenames are `<unix-ms>-<uuid>.json` and FIFO order is the
+        # lexicographic sort of that name, so pin the clock to make "oldest"
+        # deterministic rather than a uuid tie-break.
+        clock = iter(1000.0 + i for i in range(50))
+        with unittest.mock.patch.object(pending.time, "time", lambda: next(clock)):
+            for i in range(n):
+                pending.enqueue(
+                    _payload(content=f"c{i}", doc=_cd_doc(i)), RuntimeError("x")
+                )
+        for path, entry in pending.iter_entries():
+            entry["attempt_count"] = attempts_for(entry["content"])
+            with open(path, "w", encoding="utf-8") as f:
+                json.dump(entry, f)
+
+    @staticmethod
+    def _retry(entry, timeout):
+        if entry["content"] in POISON:
+            raise TimeoutError("client deadline exceeded")
+        return None
+
+    def _drain_once(self):
+        # `_document_state` mirrors `_retry`: a healthy entry's POST lands and
+        # the confirming GET sees it, a poison entry never persists. Anything
+        # else would let the free reconcile pass retire the poison entries and
+        # hide the very wedge under test.
+        with unittest.mock.patch.object(drain_pending, "_retry_one", self._retry):
+            with unittest.mock.patch.object(
+                drain_pending,
+                "_document_state",
+                lambda e, timeout=30: e["content"] not in POISON,
+            ):
+                with redirect_stderr(io.StringIO()):
+                    return drain_pending.drain_backlog(CONFIG, phase="drain")
+
+    def test_poison_heads_do_not_starve_healthy_entries_behind_them(self):
+        """The common shape: a few immortal entries in front of good ones.
+
+        They are the OLDEST by construction (they have been failing longest),
+        so oldest-first ordering puts them exactly where they do most damage.
+        """
+        self._seed(6, lambda c: pending.MAX_ATTEMPTS if c in POISON else 0)
+
+        summary = self._drain_once()
+
+        self.assertEqual(summary["drained"], 3, "healthy entries must drain")
+        self.assertFalse(summary["stalled"], "3 immortal entries must not stall the run")
+        self.assertEqual(pending.count(), 3, "only the immortal entries remain queued")
+
+    def test_whole_queue_over_budget_still_drains_after_recovery(self):
+        """The shape that demotion-by-ORDERING alone does not fix.
+
+        An upstream down for a week takes every entry past ``MAX_ATTEMPTS``.
+        When it recovers, the over-budget partition holds the entire queue, so
+        ordering is a no-op and the poison entries are back at the head. Only
+        the stall-guard abstention (``_over_budget``) gets past them.
+        """
+        self._seed(6, lambda c: pending.MAX_ATTEMPTS)
+
+        summary = self._drain_once()
+
+        self.assertEqual(summary["drained"], 3, "recovered entries must drain")
+        self.assertFalse(summary["stalled"])
+        self.assertEqual(pending.count(), 3)
+
+    def test_repeated_runs_converge_instead_of_repeating_a_zero_drain(self):
+        """The wedge was defined by NON-convergence: identical useless runs."""
+        self._seed(6, lambda c: pending.MAX_ATTEMPTS if c in POISON else 0)
+
+        depths = []
+        for _ in range(3):
+            self._drain_once()
+            depths.append(pending.count())
+
+        self.assertEqual(depths, [3, 3, 3], "converges after run 0 and stays there")
+
+    def test_a_genuine_upstream_outage_still_stalls(self):
+        """The guard rail on the fix: abstention must not disarm the guard.
+
+        Entries INSIDE their attempt budget still vote, so a real outage is
+        caught after ``STALL_THRESHOLD`` and the drain stops hammering it.
+        """
+        self._seed(6, lambda c: 0)
+
+        def all_fail(entry, timeout):
+            raise TimeoutError("upstream down")
+
+        with unittest.mock.patch.object(drain_pending, "_retry_one", all_fail):
+            with unittest.mock.patch.object(
+                drain_pending, "_document_state", lambda e, timeout=30: None
+            ):
+                with redirect_stderr(io.StringIO()):
+                    summary = drain_pending.drain_backlog(CONFIG, phase="drain")
+
+        self.assertTrue(summary["stalled"], "a real outage must still stall")
+        self.assertEqual(
+            summary["retried"], 3, "and stop at STALL_THRESHOLD, not run the queue"
+        )
+
+    def test_the_entry_crossing_the_budget_on_this_failure_still_votes(self):
+        """``update_attempt`` mutates ``attempt_count`` in place.
+
+        So the abstention check must read the counter BEFORE
+        ``_record_failure``. Reading it after would let an entry at
+        ``MAX_ATTEMPTS - 1`` abstain on the failure that takes it to the
+        budget, quietly weakening the guard for ordinary entries. Three such
+        entries are a genuine outage signal and must still stall.
+        """
+        self._seed(6, lambda c: pending.MAX_ATTEMPTS - 1)
+
+        def all_fail(entry, timeout):
+            raise TimeoutError("upstream down")
+
+        with unittest.mock.patch.object(drain_pending, "_retry_one", all_fail):
+            with unittest.mock.patch.object(
+                drain_pending, "_document_state", lambda e, timeout=30: None
+            ):
+                with redirect_stderr(io.StringIO()):
+                    summary = drain_pending.drain_backlog(CONFIG, phase="drain")
+
+        self.assertTrue(summary["stalled"])
+        self.assertEqual(summary["retried"], 3)
+
+
+    def test_the_in_hook_sequential_drain_is_protected_too(self):
+        """``drain()`` is the path that runs on EVERY boot, not ``--backlog``.
+
+        It carries its own copy of the stall guard, so it needs its own
+        demotion or the wedge just moves to the path that matters most: the
+        in-hook budget is ~4s, so a single always-timing-out entry at the head
+        consumes the whole run before anything else is reached.
+        """
+        self._seed(6, lambda c: pending.MAX_ATTEMPTS if c in POISON else 0)
+
+        with unittest.mock.patch.object(drain_pending, "_retry_one", self._retry):
+            with unittest.mock.patch.object(
+                drain_pending,
+                "_document_state",
+                lambda e, timeout=30: False,
+            ):
+                with redirect_stderr(io.StringIO()):
+                    summary = drain_pending.drain(CONFIG)
+
+        self.assertEqual(summary["drained"], 3, "healthy entries drain in-hook too")
+        self.assertFalse(summary["stalled"])
+        self.assertEqual(pending.count(), 3)
+
+    def test_poison_heads_do_not_eat_the_in_hook_budget(self):
+        """Pins the ORDERING specifically, which the abstention alone does not.
+
+        Abstaining from the stall guard stops a poison entry ENDING the run,
+        but not its consuming the run. The in-hook budget is ~4s of wall clock
+        and a chronically failing entry fails by TIMING OUT, so three of them
+        at the head burn the whole budget and the loop exits on
+        `budget_exceeded` with the healthy entries behind them untouched —
+        no stall guard involved.
+
+        Mutation-checked: reverting only the ``_drain_order`` call in
+        ``drain()`` (leaving the abstention intact) fails this test and no
+        other in the suite.
+        """
+        os.environ["HINDSIGHT_DRAIN_BUDGET_S"] = "0.5"
+        self._seed(6, lambda c: pending.MAX_ATTEMPTS if c in POISON else 0)
+
+        def slow_poison(entry, timeout):
+            if entry["content"] in POISON:
+                time.sleep(0.25)  # a client deadline being burned
+                raise TimeoutError("client deadline exceeded")
+            return None
+
+        with unittest.mock.patch.object(drain_pending, "_retry_one", slow_poison):
+            with unittest.mock.patch.object(
+                drain_pending, "_document_state", lambda e, timeout=30: False
+            ):
+                with redirect_stderr(io.StringIO()):
+                    summary = drain_pending.drain(CONFIG)
+
+        self.assertEqual(
+            summary["drained"], 3, "healthy entries must be reached before the budget"
+        )
+
+    def test_poison_heads_do_not_eat_the_backlog_budget(self):
+        """The same budget-consumption wedge on the ``--backlog`` path.
+
+        The tests above reach the backlog drain through the STALL guard, which
+        the abstention alone already defeats — so none of them notice if
+        ``_drain_backlog_impl`` stops ordering. But its ordering is load-
+        bearing for the same reason the in-hook one is, only slower: waves are
+        width 1 (``_backlog_concurrency``) and a chronically failing entry
+        burns ``HINDSIGHT_DRAIN_BACKLOG_TIMEOUT`` (280s) before failing, so
+        ~13 of them at the head exhaust the 1h budget and the run ends on
+        `budget_exceeded` having retained nothing — again with no stall guard
+        involved, and again identically on the next run.
+
+        Scaled down here: 3 poison entries at 0.6s against a 1.0s budget (the
+        floor ``_backlog_budget_seconds`` clamps to). Ordered correctly the
+        healthy entries cost ~0s and all drain; ordered oldest-first the
+        budget is gone after the second poison entry and none do.
+
+        Mutation-checked: reverting only the ``_drain_order`` call in
+        ``_drain_backlog_impl`` fails this test and no other in the suite.
+        """
+        os.environ["HINDSIGHT_DRAIN_BACKLOG_BUDGET_S"] = "1.0"
+        self._seed(6, lambda c: pending.MAX_ATTEMPTS if c in POISON else 0)
+
+        def slow_poison(entry, timeout):
+            if entry["content"] in POISON:
+                time.sleep(0.6)  # a backlog-mode deadline being burned
+                raise TimeoutError("client deadline exceeded")
+            return None
+
+        with unittest.mock.patch.object(drain_pending, "_retry_one", slow_poison):
+            with unittest.mock.patch.object(
+                drain_pending,
+                "_document_state",
+                lambda e, timeout=30: e["content"] not in POISON,
+            ):
+                with redirect_stderr(io.StringIO()):
+                    summary = drain_pending.drain_backlog(CONFIG, phase="drain")
+
+        self.assertEqual(
+            summary["drained"], 3, "healthy entries must be reached before the budget"
+        )
+        self.assertFalse(summary["stalled"], "this wedge is budget, not stall")
+
+
+class DrainOrderTest(unittest.TestCase):
+    """``_drain_order`` is a STABLE partition — age order survives inside it."""
+
+    def test_over_budget_entries_go_last_and_keep_relative_age(self):
+        entries = [
+            ("p0", {"attempt_count": pending.MAX_ATTEMPTS}),
+            ("p1", {"attempt_count": 0}),
+            ("p2", {"attempt_count": pending.MAX_ATTEMPTS + 4}),
+            ("p3", {"attempt_count": 2}),
+        ]
+        self.assertEqual(
+            [p for p, _ in drain_pending._drain_order(entries)],
+            ["p1", "p3", "p0", "p2"],
+        )
+
+    def test_a_missing_or_corrupt_counter_is_treated_as_fresh(self):
+        for value in ({}, {"attempt_count": None}, {"attempt_count": "many"}):
+            with self.subTest(value=value):
+                self.assertFalse(drain_pending._over_budget(value))
+
+
+if __name__ == "__main__":
+    unittest.main()