memwal 0.1.11.dev5__tar.gz → 0.1.11.dev6__tar.gz

{memwal-0.1.11.dev5 → memwal-0.1.11.dev6}/CHANGELOG.md

@@ -7,6 +7,7 @@
 - `remember`, `remember_bulk`, `analyze`, and `embed` reject whitespace-only text before the request. A blank body used to be accepted and then sit until the caller timed out. (#1129)
 - A 401 `AUTH_REJECTED` now says a wallet address is not a MemWal account ID. (#1132)
 - `wait_for_remember_jobs` / `remember_bulk_and_wait` no longer return a batch they could not read as if it were still uploading. When every status poll is rate-limited, the wait makes one confirming read and raises `MemWalRateLimited` (`status` 429, `job_ids`, `retry_after`) if that is refused too. An item still unsettled at the deadline names its last known status, or says no read got through. (WALM-671, #967)
+- `MemWalRememberJobTimeout` carries `last_status`, `server_error` and `last_refusal`, and its message names the relayer's last reported state. Under upload congestion a job stays `running` for up to ~25 minutes while the relayer retries, and the timeout used to hide that, so a job that was still retrying looked dead. (WALM-670, #966)
 
 ## 0.1.12
 

{memwal-0.1.11.dev5 → memwal-0.1.11.dev6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: memwal
-Version: 0.1.11.dev5
+Version: 0.1.11.dev6
 Summary: Python SDK for Walrus Memory — Privacy-first AI memory with Ed25519 signing
 Project-URL: Homepage, https://memory.walrus.xyz
 Project-URL: Documentation, https://memory.walrus.xyz

{memwal-0.1.11.dev5 → memwal-0.1.11.dev6}/memwal/__init__.py

@@ -128,4 +128,4 @@ __all__ = [
     "RecallManualResult",
 ]
 
-__version__ = "0.1.11.dev5"
+__version__ = "0.1.11.dev6"

{memwal-0.1.11.dev5 → memwal-0.1.11.dev6}/memwal/client.py

@@ -485,6 +485,9 @@ class MemWal:
         attempt = 0
         saw_rate_limit = False
         retry_after_ms = 0
+        last_status: Optional[str] = None
+        server_error: Optional[str] = None
+        last_refusal: Optional[int] = None
 
         while _now_ms() < deadline_ms:
             if retry_after_ms > 0:
@@ -503,6 +506,7 @@ class MemWal:
                 )
             except _HttpStatusError as err:
                 if _is_transient_polling_status(err.status):
+                    last_refusal = err.status
                     if err.status == 429:
                         saw_rate_limit = True
                         retry_after_ms = _clamped_retry_after_ms(err, deadline_ms)
@@ -512,6 +516,8 @@ class MemWal:
             status_str = data.get("status")
             if status_str is None or status_str == "not_found":
                 raise MemWalRememberJobNotFound(job_id)
+            last_status = status_str
+            server_error = data.get("error")
 
             if status_str == "done":
                 return RememberResult(
@@ -528,7 +534,14 @@ class MemWal:
 
             # pending / running / uploaded — keep polling.
 
-        raise _remember_job_timeout(job_id, timeout_ms, saw_rate_limit)
+        raise MemWalRememberJobTimeout(
+            job_id=job_id,
+            timeout_ms=timeout_ms,
+            last_status=last_status,
+            server_error=server_error,
+            last_refusal=last_refusal,
+            saw_rate_limit=saw_rate_limit,
+        )
 
     async def remember_and_wait(
         self,
@@ -1689,24 +1702,52 @@ class MemWalRateLimited(MemWalError):
 
 
 class MemWalRememberJobTimeout(MemWalError):
-    """Polling loop exceeded the configured timeout."""
+    """Polling loop exceeded the configured timeout.
 
-    def __init__(self, job_id: str, timeout_ms: int) -> None:
+    Carries the last status the relayer reported (GH #966): under upload
+    congestion a job stays ``running`` for tens of minutes, and without it a
+    job that is still retrying looks dead.
+    """
+
+    def __init__(
+        self,
+        job_id: str,
+        timeout_ms: int,
+        last_status: Optional[str] = None,
+        server_error: Optional[str] = None,
+        last_refusal: Optional[int] = None,
+        saw_rate_limit: bool = False,
+    ) -> None:
+        # "no status read got through (last poll: HTTP 429)" already names the rate limit.
+        named = last_status is None and last_refusal == 429
+        rate_limited = "; wait hit a rate limit (429)" if saw_rate_limit and not named else ""
+        # Redacted once, for both the message and the attribute.
+        server_error = _redact_internal_urls(server_error) if server_error else None
+        if last_status is not None:
+            # The wait only times out on a job still pending/running/uploaded
+            # (a failed job raises), so the advice is to keep waiting on it.
+            suffix = f" ({server_error})" if server_error else ""
+            detail = (
+                f"last status: {last_status}{suffix}. The job is still in progress: "
+                f"call wait_for_remember_job({job_id!r}) to keep waiting on it. Sending "
+                "the text again under a new idempotency key stores it twice; a key you "
+                "passed as idempotency_key is not remembered by the client, so pass it "
+                "again on any retry."
+            )
+        else:
+            detail = (
+                f"no status read got through (last poll: HTTP {last_refusal or 0}), "
+                "so the job's state is unknown."
+            )
         super().__init__(
-            f"remember job timed out after {timeout_ms}ms (job_id={job_id})"
+            f"remember job timed out after {timeout_ms}ms (job_id={job_id}){rate_limited}; {detail}"
         )
         self.status = 504
         self.job_id = job_id
         self.timeout_ms = timeout_ms
-
-
-def _remember_job_timeout(
-    job_id: str, timeout_ms: int, saw_rate_limit: bool
-) -> MemWalRememberJobTimeout:
-    exc = MemWalRememberJobTimeout(job_id, timeout_ms)
-    if saw_rate_limit:
-        exc.args = (f"{exc.args[0]}; wait hit a rate limit (429)",)
-    return exc
+        self.last_status = last_status
+        self.server_error = server_error
+        self.last_refusal = last_refusal
 
 
 async def _discard_http_client(memwal: MemWal) -> None:

{memwal-0.1.11.dev5 → memwal-0.1.11.dev6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "memwal"
-version = "0.1.11.dev5"
+version = "0.1.11.dev6"
 description = "Python SDK for Walrus Memory — Privacy-first AI memory with Ed25519 signing"
 readme = "README.md"
 license = "MIT"

{memwal-0.1.11.dev5 → memwal-0.1.11.dev6}/tests/test_client.py

@@ -24,6 +24,7 @@ from memwal.client import (
     MemWalCompatibilityError,
     MemWalError,
     MemWalRateLimited,
+    MemWalRememberJobTimeout,
     MemWalSync,
     _HttpStatusError,
 )
@@ -558,6 +559,89 @@ class TestBulkWaitUnderRateLimit:
         )
 
 
+class TestRememberJobTimeoutDetail:
+    async def test_reports_last_status_and_server_message(
+        self, memwal_client: MemWal
+    ) -> None:
+        async def running(*_args: Any, **_kwargs: Any) -> dict:
+            return {
+                "job_id": "slow-job",
+                "status": "running",
+                "error": "the hosted relayer is still retrying this upload",
+            }
+
+        memwal_client._signed_request = running  # type: ignore[method-assign]
+        with pytest.raises(MemWalRememberJobTimeout) as exc:
+            await memwal_client.wait_for_remember_job(
+                "slow-job", poll_interval_ms=0, timeout_ms=300
+            )
+        assert exc.value.status == 504
+        assert exc.value.last_status == "running"
+        assert "still retrying" in (exc.value.server_error or "")
+        assert str(exc.value).startswith("remember job timed out after 300ms")
+        assert "last status: running" in str(exc.value)
+        # The job is still live, so the advice is to wait on its id, not to
+        # POST again: `remember` drops its key once accepted, so a re-send
+        # would mint a new key and a second paid write.
+        assert "call wait_for_remember_job('slow-job')" in str(exc.value)
+        assert "idempotency_key is not remembered" in str(exc.value)
+
+    async def test_timeout_attribute_carries_the_redacted_server_error(
+        self, memwal_client: MemWal
+    ) -> None:
+        async def running(*_args: Any, **_kwargs: Any) -> dict:
+            return {
+                "job_id": "slow-job",
+                "status": "running",
+                "error": "sidecar http://localhost:9000/upload unreachable",
+            }
+
+        memwal_client._signed_request = running  # type: ignore[method-assign]
+        with pytest.raises(MemWalRememberJobTimeout) as exc:
+            await memwal_client.wait_for_remember_job(
+                "slow-job", poll_interval_ms=0, timeout_ms=300
+            )
+        assert exc.value.server_error == "sidecar [internal] unreachable"
+        assert "localhost" not in str(exc.value)
+
+    async def test_every_poll_refused_reports_unknown_state(
+        self, memwal_client: MemWal
+    ) -> None:
+        async def refused(*_args: Any, **_kwargs: Any) -> dict:
+            raise _HttpStatusError(429, '{"error":"rate limited"}')
+
+        memwal_client._signed_request = refused  # type: ignore[method-assign]
+        with pytest.raises(MemWalRememberJobTimeout) as exc:
+            await memwal_client.wait_for_remember_job(
+                "hidden-job", poll_interval_ms=0, timeout_ms=300
+            )
+        assert exc.value.last_status is None
+        assert exc.value.last_refusal == 429
+        assert "no status read got through (last poll: HTTP 429)" in str(exc.value)
+
+    async def test_seen_then_rate_limited_keeps_both(
+        self, memwal_client: MemWal
+    ) -> None:
+        calls = 0
+
+        async def running_then_refused(*_args: Any, **_kwargs: Any) -> dict:
+            nonlocal calls
+            calls += 1
+            if calls == 1:
+                return {"job_id": "slow-job", "status": "running"}
+            raise _HttpStatusError(429, '{"error":"rate limited"}')
+
+        memwal_client._signed_request = running_then_refused  # type: ignore[method-assign]
+        with pytest.raises(MemWalRememberJobTimeout) as exc:
+            await memwal_client.wait_for_remember_job(
+                "slow-job", poll_interval_ms=0, timeout_ms=300
+            )
+        assert exc.value.last_status == "running"
+        assert "(job_id=slow-job); wait hit a rate limit (429); last status: running." in str(
+            exc.value
+        )
+
+
 # ============================================================
 # recall() tests
 # ============================================================