memwal 0.1.11.dev3__tar.gz → 0.1.11.dev4__tar.gz

{memwal-0.1.11.dev3 → memwal-0.1.11.dev4}/CHANGELOG.md

@@ -1,5 +1,11 @@
 # memwal
 
+## Unreleased
+
+### Fixed
+
+- `wait_for_remember_jobs` / `remember_bulk_and_wait` no longer return a batch they could not read as if it were still uploading. When every status poll is rate-limited, the wait makes one confirming read and raises `MemWalRateLimited` (`status` 429, `job_ids`, `retry_after`) if that is refused too. An item still unsettled at the deadline names its last known status, or says no read got through. (WALM-671, #967)
+
 ## 0.1.12
 
 ### Fixed

{memwal-0.1.11.dev3 → memwal-0.1.11.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: memwal
-Version: 0.1.11.dev3
+Version: 0.1.11.dev4
 Summary: Python SDK for Walrus Memory — Privacy-first AI memory with Ed25519 signing
 Project-URL: Homepage, https://memory.walrus.xyz
 Project-URL: Documentation, https://memory.walrus.xyz

{memwal-0.1.11.dev3 → memwal-0.1.11.dev4}/memwal/__init__.py

@@ -27,6 +27,7 @@ from .client import (
     MemWalClockDriftError,
     MemWalCompatibilityError,
     MemWalError,
+    MemWalRateLimited,
     MemWalRememberJobFailed,
     MemWalRememberJobNotFound,
     MemWalRememberJobTimeout,
@@ -82,6 +83,7 @@ __all__ = [
     "MemWalError",
     "MemWalCompatibilityError",
     "MemWalClockDriftError",
+    "MemWalRateLimited",
     "MemWalRememberJobFailed",
     "MemWalRememberJobNotFound",
     "MemWalRememberJobTimeout",
@@ -126,4 +128,4 @@ __all__ = [
     "RecallManualResult",
 ]
 
-__version__ = "0.1.11.dev3"
+__version__ = "0.1.11.dev4"

{memwal-0.1.11.dev3 → memwal-0.1.11.dev4}/memwal/client.py

@@ -662,6 +662,31 @@ class MemWal:
         attempt = 0
         saw_rate_limit = False
         retry_after_ms = 0
+        last_seen: Dict[str, RememberBulkStatusItem] = {}
+        last_refusal: Optional[int] = None
+        # Whether any status read got through. An item missing from an answer
+        # was not refused, so it must not be reported as if it had been.
+        answered = False
+
+        def settle(item: RememberBulkStatusItem) -> bool:
+            last_seen[item.job_id] = item
+            if item.status == "done":
+                results[item.job_id] = RememberBulkItemResult(
+                    id=item.job_id,
+                    blob_id=item.blob_id or "",
+                    status="done",
+                    error=None,
+                )
+                return True
+            if item.status in ("failed", "not_found"):
+                results[item.job_id] = RememberBulkItemResult(
+                    id=item.job_id,
+                    blob_id=item.blob_id or "",
+                    status="failed",
+                    error=item.error,
+                )
+                return True
+            return False
 
         while pending and _now_ms() < deadline_ms:
             if retry_after_ms > 0:
@@ -675,40 +700,66 @@ class MemWal:
                 batch = await self.get_remember_bulk_status(pending)
             except _HttpStatusError as err:
                 if _is_transient_polling_status(err.status):
+                    last_refusal = err.status
                     if err.status == 429:
                         saw_rate_limit = True
                         retry_after_ms = _clamped_retry_after_ms(err, deadline_ms)
                     continue
                 raise
+            answered = True
 
-            terminal_ids: Set[str] = set()
-            for item in batch.results:
-                if item.status == "done":
-                    results[item.job_id] = RememberBulkItemResult(
-                        id=item.job_id,
-                        blob_id=item.blob_id or "",
-                        status="done",
-                        error=None,
-                    )
-                    terminal_ids.add(item.job_id)
-                elif item.status in ("failed", "not_found"):
-                    results[item.job_id] = RememberBulkItemResult(
-                        id=item.job_id,
-                        blob_id=item.blob_id or "",
-                        status="failed",
-                        error=item.error,
-                    )
-                    terminal_ids.add(item.job_id)
+            terminal_ids: Set[str] = {
+                item.job_id
+                for item in batch.results
+                if item.job_id in pending and settle(item)
+            }
             pending = [jid for jid in pending if jid not in terminal_ids]
 
+        if opts.timeout_ms > 0 and len(pending) == len(job_ids) and not last_seen:
+            # Nothing settled and nothing was seen: the polls may all have been
+            # refused. Confirm with one direct read before reporting anything.
+            try:
+                probed = await self.get_remember_bulk_status(list(job_ids))
+            except _HttpStatusError as err:
+                if err.status == 429:
+                    # Header first, else the body's retry_after_seconds, so a
+                    # proxy that strips Retry-After does not lose the hint.
+                    wait_ms = _retry_after_ms(err)
+                    retry_after = f"{wait_ms / 1000:g}" if wait_ms else None
+                    raise MemWalRateLimited(job_ids, retry_after) from err
+                # Any other probe failure is not evidence about the jobs.
+            else:
+                answered = True
+                settled = {
+                    item.job_id
+                    for item in probed.results
+                    if item.job_id in pending and settle(item)
+                }
+                pending = [jid for jid in pending if jid not in settled]
+
+        for job_id in pending:
+            seen = last_seen.get(job_id)
+            if seen is not None:
+                suffix = f": {_redact_internal_urls(seen.error)}" if seen.error else ""
+                error = f"still {seen.status} after {opts.timeout_ms}ms{suffix}"
+            elif answered:
+                error = "not in the relayer's status answer; this item may not be stored"
+            elif last_refusal is not None:
+                error = (
+                    f"no status read got through (last poll: HTTP {last_refusal}); "
+                    "this item may not be stored"
+                )
+            else:
+                error = f"polling timed out after {opts.timeout_ms}ms"
+            # Only the "no status read got through" message already names the 429.
+            names_rate_limit = seen is None and not answered and last_refusal == 429
+            if saw_rate_limit and not names_rate_limit:
+                error += "; wait hit a rate limit (429)"
+            results[job_id] = RememberBulkItemResult(
+                id=job_id, blob_id="", status="timeout", error=error
+            )
+
         ordered = [results[job_id] for job_id in job_ids]
-        if saw_rate_limit:
-            for item in ordered:
-                if item.status == "timeout":
-                    item.error = (
-                        f"polling timed out after {opts.timeout_ms}ms; "
-                        "wait hit a rate limit (429)"
-                    )
         succeeded = sum(1 for r in ordered if r.status == "done")
         failed = sum(1 for r in ordered if r.status == "failed")
         timed_out = sum(1 for r in ordered if r.status == "timeout")
@@ -1601,6 +1652,24 @@ class MemWalRememberJobFailed(MemWalError):
         self.error = error
 
 
+class MemWalRateLimited(MemWalError):
+    """Every status read for a bulk wait was rate-limited, the confirming read
+    included, so none of its writes could be confirmed (GH #967). Status reads
+    count against the same delegate-key budget as writes."""
+
+    def __init__(self, job_ids: Sequence[str], retry_after: Optional[str] = None) -> None:
+        wait = f" after ~{retry_after}s" if retry_after else ""
+        super().__init__(
+            f"Walrus Memory rate-limited every status read for this batch, so none of its "
+            f"{len(job_ids)} writes could be confirmed. Do not treat them as stored: check them "
+            f"with get_remember_bulk_status(job_ids){wait}. Status reads count against the same "
+            "delegate-key budget as writes."
+        )
+        self.status = 429
+        self.job_ids = list(job_ids)
+        self.retry_after = retry_after
+
+
 class MemWalRememberJobTimeout(MemWalError):
     """Polling loop exceeded the configured timeout."""
 

{memwal-0.1.11.dev3 → memwal-0.1.11.dev4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "memwal"
-version = "0.1.11.dev3"
+version = "0.1.11.dev4"
 description = "Python SDK for Walrus Memory — Privacy-first AI memory with Ed25519 signing"
 readme = "README.md"
 license = "MIT"

{memwal-0.1.11.dev3 → memwal-0.1.11.dev4}/tests/test_client.py

@@ -9,6 +9,7 @@ from __future__ import annotations
 
 import base64
 import json
+import time
 from datetime import datetime, timedelta, timezone
 from typing import Any, Sequence
 
@@ -22,7 +23,9 @@ from memwal.client import (
     MemWalClockDriftError,
     MemWalCompatibilityError,
     MemWalError,
+    MemWalRateLimited,
     MemWalSync,
+    _HttpStatusError,
 )
 from memwal.types import (
     RecallManualOptions,
@@ -462,6 +465,99 @@ class TestRememberBulkAsync:
         assert [item.status for item in result.results] == ["done", "done"]
 
 
+class TestBulkWaitUnderRateLimit:
+    async def test_every_read_refused_raises_rate_limited(
+        self, memwal_client: MemWal
+    ) -> None:
+        ids = [f"job-{i}" for i in range(12)]
+
+        async def refused(job_ids: Sequence[str]) -> RememberBulkStatusResult:
+            raise _HttpStatusError(429, '{"error":"rate limited"}', retry_after="30")
+
+        memwal_client.get_remember_bulk_status = refused  # type: ignore[method-assign]
+        with pytest.raises(MemWalRateLimited) as exc:
+            await memwal_client.wait_for_remember_jobs(
+                ids, RememberBulkOptions(poll_interval_ms=1, timeout_ms=300)
+            )
+        assert exc.value.status == 429
+        assert exc.value.job_ids == ids
+        assert exc.value.retry_after == "30"
+        assert "none of its 12 writes could be confirmed" in str(exc.value)
+
+    async def test_rate_limited_reads_the_body_when_the_header_is_stripped(
+        self, memwal_client: MemWal
+    ) -> None:
+        async def refused(job_ids: Sequence[str]) -> RememberBulkStatusResult:
+            raise _HttpStatusError(429, '{"retry_after_seconds":45}')
+
+        memwal_client.get_remember_bulk_status = refused  # type: ignore[method-assign]
+        with pytest.raises(MemWalRateLimited) as exc:
+            await memwal_client.wait_for_remember_jobs(
+                ["job-1"], RememberBulkOptions(poll_interval_ms=1, timeout_ms=300)
+            )
+        assert exc.value.retry_after == "45"
+        assert "after ~45s" in str(exc.value)
+
+    async def test_confirming_read_settles_a_refused_wait(
+        self, memwal_client: MemWal
+    ) -> None:
+        start = time.monotonic()
+
+        async def probe_answers(job_ids: Sequence[str]) -> RememberBulkStatusResult:
+            # Refused for the whole 250ms wait; the read after the deadline is
+            # the probe. Time-based because jitter varies the poll count.
+            if time.monotonic() - start < 0.25:
+                raise _HttpStatusError(429, "{}")
+            return RememberBulkStatusResult(
+                results=[
+                    RememberBulkStatusItem(job_id="job-a", status="done", blob_id="blob-a"),
+                    RememberBulkStatusItem(
+                        job_id="job-b", status="failed", error="walrus upload failed"
+                    ),
+                ]
+            )
+
+        memwal_client.get_remember_bulk_status = probe_answers  # type: ignore[method-assign]
+        out = await memwal_client.wait_for_remember_jobs(
+            ["job-a", "job-b", "job-c"],
+            RememberBulkOptions(poll_interval_ms=1, timeout_ms=250),
+        )
+        assert [r.status for r in out.results] == ["done", "failed", "timeout"]
+        # A read did get through; it just left job-c out, so saying no read
+        # got through would be false.
+        assert "not in the relayer's status answer" in (out.results[2].error or "")
+
+    async def test_partly_settled_batch_is_returned(self, memwal_client: MemWal) -> None:
+        reads = 0
+
+        async def then_refused(job_ids: Sequence[str]) -> RememberBulkStatusResult:
+            nonlocal reads
+            reads += 1
+            if reads == 1:
+                return RememberBulkStatusResult(
+                    results=[
+                        RememberBulkStatusItem(
+                            job_id="job-a", status="done", blob_id="blob-a"
+                        ),
+                        RememberBulkStatusItem(
+                            job_id="job-b",
+                            status="running",
+                            error="still retrying this upload",
+                        ),
+                    ]
+                )
+            raise _HttpStatusError(429, "{}")
+
+        memwal_client.get_remember_bulk_status = then_refused  # type: ignore[method-assign]
+        out = await memwal_client.wait_for_remember_jobs(
+            ["job-a", "job-b"], RememberBulkOptions(poll_interval_ms=1, timeout_ms=300)
+        )
+        assert [r.status for r in out.results] == ["done", "timeout"]
+        assert "still running after 300ms: still retrying this upload" in (
+            out.results[1].error or ""
+        )
+
+
 # ============================================================
 # recall() tests
 # ============================================================

{memwal-0.1.11.dev3 → memwal-0.1.11.dev4}/tests/test_polling_delay.py

@@ -139,5 +139,6 @@ async def test_bulk_timeout_names_429_and_keeps_unreturned_jobs_pending(
     assert settled.results[0].status == "done"
     assert settled.results[1].status == "timeout"
     assert settled.results[1].error == (
-        "polling timed out after 800ms; wait hit a rate limit (429)"
+        "not in the relayer's status answer; this item may not be stored; "
+        "wait hit a rate limit (429)"
     )