switchroom 0.19.18 → 0.19.22

package/vendor/hindsight-memory/tests/test_drain_pending.py

@@ -1,5 +1,6 @@
 """Tests for drain_pending.drain() (#1071)."""
 
+import io
 import json
 import os
 import sys
@@ -32,10 +33,23 @@ class FakeOk:
         return False
 
 
-def _seed_entry(pending_dir: str, document_id: str = "doc-x", attempt: int = 1) -> str:
+#: A POST-#3244 content-derived document_id (``retain.slice_document_id``):
+#: ``{session}-r{start_uuid}-{end_uuid}``. The presence-GET reconcile is gated
+#: on this shape — a pre-#3244 bare session id is answered 200 by ANY retain in
+#: that session — so the id shape is load-bearing in every drain fixture here.
+CONTENT_DERIVED_DOC_ID = (
+    "11111111-2222-4333-8444-555555555555"
+    "-r00000000-0000-4000-8000-000000000001"
+    "-00000000-0000-4000-8000-000000000002"
+)
+
+
+def _seed_entry(
+    pending_dir: str, document_id: str = CONTENT_DERIVED_DOC_ID, attempt: int = 1
+) -> str:
     os.makedirs(pending_dir, mode=0o700, exist_ok=True)
     ts_ms = int(time.time() * 1000)
-    name = f"{ts_ms}-{document_id}.json"
+    name = f"{ts_ms}-{document_id[:24]}.json"
     path = os.path.join(pending_dir, name)
     payload = {
         "schema": 1,
@@ -84,6 +98,15 @@ class DrainPendingTest(unittest.TestCase):
         for n in ("drain_pending", "lib.pending"):
             sys.modules.pop(n, None)
 
+    def _reconciled(self):
+        """Basenames archived under ``pending-reconciled/`` (sibling dir)."""
+        import lib.pending as pending
+
+        try:
+            return sorted(os.listdir(pending.reconciled_dir()))
+        except OSError:
+            return []
+
     def test_drain_empty_queue_is_noop(self):
         import drain_pending
 
@@ -94,15 +117,82 @@ class DrainPendingTest(unittest.TestCase):
         self.assertFalse(summary["stalled"])
         self.assertFalse(summary["budget_exceeded"])
 
-    def test_drain_success_deletes_entry(self):
+    def test_drain_reconciles_an_already_durable_entry_without_posting(self):
+        """switchroom #3596: GET before POST, in the SessionStart path too.
+
+        This test used to assert `drained == 1` against a mock that answered
+        200 to EVERYTHING, including the presence GET — i.e. it asserted
+        that an already-durable document gets re-POSTed anyway. That is the
+        re-post loop: the hook's clamped timeout guarantees the client gives
+        up, so the entry survives and is re-posted on every boot forever
+        while the memory was never actually lost.
+        """
         path = _seed_entry(self._pending)
         import drain_pending
 
-        with patch("urllib.request.urlopen", return_value=FakeOk()):
+        posts = []
+
+        def record(req, *a, **kw):
+            posts.append(getattr(req, "method", None) or req.get_method())
+            return FakeOk()
+
+        with patch("urllib.request.urlopen", side_effect=record):
+            summary = drain_pending.drain({})
+        self.assertEqual(summary["reconciled"], 1)
+        self.assertEqual(summary["drained"], 0)
+        self.assertEqual(summary["retried"], 0)
+        self.assertFalse(os.path.exists(path))
+        self.assertEqual(self._reconciled(), [os.path.basename(path)])
+        self.assertEqual(posts, ["GET"], "no retain POST for a durable document")
+
+    def test_drain_refuses_to_reconcile_a_pre_3244_bare_session_id(self):
+        """A bare session id's 200 is not evidence about THIS entry.
+
+        The bank answers 200 for a bare session id after any successful
+        retain in that session, so reconciling on presence would retire an
+        entry whose own content was never committed.
+        """
+        path = _seed_entry(
+            self._pending, document_id="d52ae253-2d26-42e5-a86b-9a354cc0ace5"
+        )
+        import drain_pending
+
+        methods = []
+
+        def record(req, *a, **kw):
+            methods.append(getattr(req, "method", None) or req.get_method())
+            raise urllib.error.URLError("upstream down")
+
+        with patch("urllib.request.urlopen", side_effect=record):
             summary = drain_pending.drain({})
+
+        self.assertEqual(summary["reconciled"], 0)
+        self.assertEqual(methods, ["POST"], "no free-pass GET for a bare session id")
+        self.assertTrue(os.path.exists(path), "entry stays queued")
+        self.assertEqual(self._reconciled(), [])
+
+    def test_drain_success_archives_the_entry(self):
+        """A genuinely absent document is POSTed and the entry RETIRED.
+
+        Retired means moved into ``pending-reconciled/``, not ``os.remove``d:
+        this in-hook path retires on the POST's own commit-before-ack 200
+        (``async_processing=False``, no confirming GET) — the daemon's word
+        about itself, not an independent read (#3244) — so the removal must
+        stay recoverable.
+        """
+        path = _seed_entry(self._pending)
+        import drain_pending
+
+        with patch.object(
+            drain_pending, "_document_state", lambda e, timeout=30: False
+        ):
+            with patch("urllib.request.urlopen", return_value=FakeOk()):
+                summary = drain_pending.drain({})
         self.assertEqual(summary["drained"], 1)
+        self.assertEqual(summary["reconciled"], 0)
         self.assertEqual(summary["retried"], 0)
         self.assertFalse(os.path.exists(path))
+        self.assertEqual(self._reconciled(), [os.path.basename(path)])
 
     def test_drain_failure_bumps_attempt_count(self):
         path = _seed_entry(self._pending, attempt=1)
@@ -123,14 +213,17 @@ class DrainPendingTest(unittest.TestCase):
         self.assertEqual(entry["attempt_count"], 2)
         self.assertIn("last_attempt_at", entry)
 
-    def test_drain_max_attempts_marks_dead(self):
+    def test_drain_max_attempts_marks_dead_on_a_permanent_failure(self):
+        """Exhausting the budget on a 4xx still retires the entry."""
         from lib.pending import MAX_ATTEMPTS
 
         path = _seed_entry(self._pending, attempt=MAX_ATTEMPTS)
         import drain_pending
 
         def boom(*a, **kw):
-            raise urllib.error.URLError("still down")
+            raise urllib.error.HTTPError(
+                "http://h/v1/x", 400, "Bad Request", {}, io.BytesIO(b"bad payload")
+            )
 
         with patch("urllib.request.urlopen", side_effect=boom):
             summary = drain_pending.drain({})
@@ -138,6 +231,28 @@ class DrainPendingTest(unittest.TestCase):
         self.assertFalse(os.path.exists(path))
         self.assertTrue(os.path.exists(path + ".dead"))
 
+    def test_drain_max_attempts_keeps_the_memory_on_a_transient_failure(self):
+        """A dead upstream must never retire a memory, however many attempts.
+
+        ``URLError`` is what an unreachable daemon raises. Before the
+        permanence gate this retired the entry at MAX_ATTEMPTS — i.e. an
+        outage destroyed memory that was perfectly saveable.
+        """
+        from lib.pending import MAX_ATTEMPTS
+
+        path = _seed_entry(self._pending, attempt=MAX_ATTEMPTS)
+        import drain_pending
+
+        def boom(*a, **kw):
+            raise urllib.error.URLError("still down")
+
+        with patch("urllib.request.urlopen", side_effect=boom):
+            summary = drain_pending.drain({})
+        self.assertEqual(summary["dead"], 0)
+        self.assertEqual(summary["retried"], 1)
+        self.assertTrue(os.path.exists(path))
+        self.assertFalse(os.path.exists(path + ".dead"))
+
     def test_drain_stall_guard_stops_after_threshold(self):
         # Seed 10 entries; with a same-error-class stream, the stall
         # guard should trip at STALL_THRESHOLD (3) and leave the rest.
@@ -249,11 +364,18 @@ class DrainPendingTest(unittest.TestCase):
 
         import drain_pending
 
-        with patch("urllib.request.urlopen", side_effect=maybe_ok):
-            summary = drain_pending.drain({})
+        # All three documents are genuinely absent, so every entry takes the
+        # POST path — otherwise the presence GET would consume call slots
+        # and this would be testing the reconcile phase by accident.
+        with patch.object(
+            drain_pending, "_document_state", lambda e, timeout=30: False
+        ):
+            with patch("urllib.request.urlopen", side_effect=maybe_ok):
+                summary = drain_pending.drain({})
 
         self.assertEqual(summary["drained"], 2)
         self.assertEqual(summary["retried"], 1)
+        self.assertEqual(summary["reconciled"], 0)
 
 
 if __name__ == "__main__":

package/vendor/hindsight-memory/tests/test_pending.py

@@ -1,5 +1,7 @@
 """Tests for the pending-retains persistent queue (#1071)."""
 
+import contextlib
+import io
 import json
 import os
 import sys
@@ -65,17 +67,26 @@ class PendingQueueTest(unittest.TestCase):
         self.assertIn("failed_at", entry)
         self.assertEqual(entry["schema"], pending_mod.SCHEMA)
 
-    def test_enqueue_filename_is_unix_ms_uuid(self):
+    def test_enqueue_filename_is_unix_ms_key_uuid(self):
+        """``<unix-ms>-<dupe-key>-<uuid>.json`` (switchroom #3596).
+
+        The dedupe key moved INTO the name so a duplicate lookup is a
+        listing prefix match with zero file reads. The leading millisecond
+        timestamp is unchanged, so the lexicographic sort in
+        ``_list_entries`` is still oldest-first.
+        """
         path = pending_mod.enqueue(self._sample_payload(), RuntimeError("boom"))
         name = os.path.basename(path)
         self.assertTrue(name.endswith(".json"))
         head = name[: -len(".json")]
-        ts_part, uuid_part = head.split("-", 1)
+        ts_part, key_part, uuid_part = head.split("-")
         self.assertTrue(ts_part.isdigit())
         # Filename ts should be within 10 s of now
         now_ms = int(time.time() * 1000)
         self.assertLess(abs(now_ms - int(ts_part)), 10_000)
         self.assertEqual(len(uuid_part), 12)
+        self.assertEqual(len(key_part), 16)
+        self.assertRegex(key_part, r"^[0-9a-f]{16}$")
 
     def test_enqueue_atomic_no_tmp_left_behind(self):
         pending_mod.enqueue(self._sample_payload(), RuntimeError("boom"))
@@ -83,16 +94,30 @@ class PendingQueueTest(unittest.TestCase):
         self.assertEqual(len(names), 1)
         self.assertFalse(any(n.endswith(".tmp") for n in names))
 
-    def test_enqueue_returns_none_when_full(self):
-        # Pre-populate with MAX_ENTRIES dummy files.
+    def test_enqueue_evicts_oldest_when_full_instead_of_refusing(self):
+        """switchroom #3596: a full queue sheds the OLDEST entry.
+
+        This test previously asserted the opposite -- that ``enqueue()``
+        returns ``None`` at ``MAX_ENTRIES`` -- which meant throwing away the
+        turn that had just happened, the one most likely to still matter,
+        while keeping a queue full of stale ones.
+        """
         os.makedirs(self._dir, mode=0o700)
+        oldest = os.path.join(self._dir, f"{0:013d}-aaaaaaaaaaaa.json")
         for i in range(pending_mod.MAX_ENTRIES):
             with open(os.path.join(self._dir, f"{i:013d}-aaaaaaaaaaaa.json"), "w") as f:
                 json.dump({"placeholder": True}, f)
-        result = pending_mod.enqueue(self._sample_payload(), RuntimeError("boom"))
-        self.assertIsNone(result)
-        # Count unchanged
+
+        stderr = io.StringIO()
+        with contextlib.redirect_stderr(stderr):
+            result = pending_mod.enqueue(self._sample_payload(), RuntimeError("boom"))
+
+        self.assertIsNotNone(result, "the incoming entry must never be refused")
+        self.assertTrue(os.path.exists(result))
+        self.assertFalse(os.path.exists(oldest), "the oldest entry was evicted")
+        # Cap still honoured: one in, one out.
         self.assertEqual(pending_mod.count(), pending_mod.MAX_ENTRIES)
+        self.assertIn("evicted OLDEST", stderr.getvalue())
 
     def test_iter_entries_ordered_oldest_first(self):
         p1 = pending_mod.enqueue(self._sample_payload("doc-1"), RuntimeError("e1"))