argus-code-review 0.3.2__tar.gz → 0.3.3__tar.gz

{argus_code_review-0.3.2 → argus_code_review-0.3.3}/CHANGELOG.md

@@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.3.3] - 2026-10-22
+
+### Changed
+
+- Gemini reviewer sessions now retry ONCE on a transient failure (HTTP 429/500/502/503/504, `RESOURCE_EXHAUSTED`/`UNAVAILABLE`/`DEADLINE_EXCEEDED`/`INTERNAL`, or an httpx timeout/connect/read/remote-protocol error) that happens before the model returned any response. The cooldown depends on the error: 45-60 s jitter for 429/`RESOURCE_EXHAUSTED` (per-minute quota windows), 5-10 s for 5xx and network errors; a server `Retry-After`/`retryDelay` >= 5 s is honored; never more than 60 s. A daily-quota 429 (`PerDay` quota or a retry delay over 120 s) and auth, permission, bad-request and not-found errors are never retried. Overridable via `ARGUS_GEMINI_RETRY_RATE_LIMIT_{MIN,MAX}_S`, `ARGUS_GEMINI_RETRY_COOLDOWN_{MIN,MAX}_S` and `ARGUS_GEMINI_RETRY_MAX_WAIT_S`.
+- A failed Gemini session now logs a WARNING with the exception type, HTTP code/status and a scrubbed, 300-character-truncated message, and `SessionResult.failure_detail` carries a short label such as `429 RESOURCE_EXHAUSTED`. `failure_reason` stays `worker_crashed`; degraded-coverage note text is unchanged. The existing ERROR log for a failed `generate_content` call is also scrubbed of key-shaped strings.
+
 ## [0.3.2] - 2026-10-21
 
 ### Added
@@ -441,7 +448,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   packaged set.
 - `argus --version`, `argus prompts list`, and `argus prompts export`.
 
-[Unreleased]: https://github.com/redesignhealth/argus-review/compare/v0.3.1...HEAD
+[Unreleased]: https://github.com/redesignhealth/argus-review/compare/v0.3.3...HEAD
+[0.3.3]: https://github.com/redesignhealth/argus-review/compare/v0.3.2...v0.3.3
 [0.3.2]: https://github.com/redesignhealth/argus-review/compare/v0.3.1...v0.3.2
 [0.3.1]: https://github.com/redesignhealth/argus-review/compare/v0.3.0...v0.3.1
 [0.3.0]: https://github.com/redesignhealth/argus-review/compare/v0.2.11...v0.3.0

{argus_code_review-0.3.2 → argus_code_review-0.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: argus-code-review
-Version: 0.3.2
+Version: 0.3.3
 Summary: Self-orchestrated PR review agent using LangGraph + Claude Agent SDK
 Project-URL: Repository, https://github.com/redesignhealth/argus-review
 Project-URL: Issues, https://github.com/redesignhealth/argus-review/issues

{argus_code_review-0.3.2 → argus_code_review-0.3.3}/argus/gemini_runner.py

@@ -144,7 +144,10 @@ from __future__ import annotations
 import asyncio
 import json
 import logging
+import os
+import random
 import re
+from dataclasses import dataclass
 from datetime import datetime, timezone
 from typing import Any
 
@@ -208,6 +211,171 @@ _FORCE_TOOL_CONFIG = types.ToolConfig(
 # uncaught.
 _HTTP_TIMEOUT_FRACTION = 0.8
 
+# ---------------------------------------------------------------------------
+# One-shot retry of transient Gemini failures
+# ---------------------------------------------------------------------------
+# A session is retried AT MOST ONCE, and only when it failed with a transient
+# error (HTTP 429/500/502/503/504, RESOURCE_EXHAUSTED/UNAVAILABLE/
+# DEADLINE_EXCEEDED/INTERNAL, or an httpx timeout/connect/read/remote-protocol
+# error) before the model returned ANY response (i.e. on turn 0), so the retry
+# restarts the session cleanly from the beginning. Later failures are not
+# retried. The cooldown depends on the error kind (see `_retry_decision`):
+# 45-60 s jitter for 429/RESOURCE_EXHAUSTED (per-minute quota windows), 5-10 s
+# for 5xx/network errors; a server Retry-After/retryDelay >= 5 s is honored;
+# never more than 60 s. Daily-quota 429s are not retried. Overridable via
+# ARGUS_GEMINI_RETRY_RATE_LIMIT_{MIN,MAX}_S, ARGUS_GEMINI_RETRY_COOLDOWN_{MIN,MAX}_S
+# and ARGUS_GEMINI_RETRY_MAX_WAIT_S (read per call, so tests can set them to 0).
+# The 60 s worst case is far inside ARGUS_SESSION_TIMEOUT (default 1500 s), and
+# the retry is skipped if <25% of that timeout remains.
+_RETRY_RATE_LIMIT_MIN_S = 45.0  # 429 / RESOURCE_EXHAUSTED: per-minute windows
+_RETRY_RATE_LIMIT_MAX_S = 60.0
+_RETRY_SHORT_MIN_S = 5.0  # 5xx / UNAVAILABLE / DEADLINE_EXCEEDED / network
+_RETRY_SHORT_MAX_S = 10.0
+_RETRY_MAX_WAIT_S = 60.0  # hard cap on any single wait, incl. a server Retry-After
+_RETRY_AFTER_MIN_S = 5.0  # a server delay below this is ignored (jitter used)
+# A server-suggested delay above this means the quota window is not a
+# per-minute one (minutes/hours): fail fast rather than retry.
+_RETRY_GIVEUP_DELAY_S = 120.0
+# Do not retry when less than this fraction of the session timeout remains
+# (e.g. the first attempt already burned its HTTP timeout).
+_RETRY_MIN_REMAINING_FRACTION = 0.25
+_TRANSIENT_HTTP_CODES = frozenset({429, 500, 502, 503, 504})
+_TRANSIENT_STATUSES = frozenset(
+    {"RESOURCE_EXHAUSTED", "UNAVAILABLE", "DEADLINE_EXCEEDED", "INTERNAL"}
+)
+_TRANSIENT_HTTPX = (
+    httpx.TimeoutException,
+    httpx.ConnectError,
+    httpx.ReadError,
+    httpx.RemoteProtocolError,
+)
+_DETAIL_MAX_CHARS = 300
+_SECRET_RE = re.compile(
+    r"(AIza[0-9A-Za-z_\-]{20,}|(?i:(?:api[_-]?key|key|token)=)[^&\s\"']+|"
+    r"(?i:bearer\s+)[A-Za-z0-9._\-]+|sk-[A-Za-z0-9_\-]{16,})"
+)
+
+
+def _env_float(name: str, default: float) -> float:
+    raw = os.environ.get(name)
+    if raw is None or raw == "":
+        return default
+    try:
+        return max(0.0, float(raw))
+    except ValueError:
+        return default
+
+
+@dataclass
+class _Progress:
+    """Mutable per-session state shared between attempts and the failure handler."""
+
+    responses: int = 0  # successful generate_content responses received
+    retried: bool = False
+    retry_wait_s: float = 0.0
+
+
+def _scrub(text: str) -> str:
+    """Redact anything key-shaped and truncate for logging."""
+    text = _SECRET_RE.sub("***", text).replace("\n", " ")
+    return text[:_DETAIL_MAX_CHARS]
+
+
+def _error_code_status(exc: BaseException) -> tuple[int | None, str | None]:
+    """Best-effort (HTTP code, status string) for an SDK/transport exception."""
+    if isinstance(exc, genai_errors.APIError):
+        code = getattr(exc, "code", None)
+        status = getattr(exc, "status", None)
+        return (
+            code if isinstance(code, int) else None,
+            status if isinstance(status, str) else None,
+        )
+    if isinstance(exc, httpx.HTTPStatusError):
+        return exc.response.status_code, None
+    return None, None
+
+
+def _is_transient_error(exc: BaseException) -> bool:
+    code, status = _error_code_status(exc)
+    if code in _TRANSIENT_HTTP_CODES or (status or "").upper() in _TRANSIENT_STATUSES:
+        return True
+    if isinstance(exc, (genai_errors.APIError, httpx.HTTPStatusError)):
+        return False  # a definite API answer that is not transient (400/401/403/404...)
+    return isinstance(exc, _TRANSIENT_HTTPX)
+
+
+def _retry_after_seconds(exc: BaseException) -> float | None:
+    """Server-suggested delay from a Retry-After header or RetryInfo.retryDelay."""
+    response = getattr(exc, "response", None)
+    headers = getattr(response, "headers", None)
+    try:
+        raw = headers.get("retry-after") if headers is not None else None
+        if raw is not None:
+            return max(0.0, float(raw))
+    except (TypeError, ValueError, AttributeError):
+        pass
+    details = getattr(exc, "details", None)
+    err = details.get("error", details) if isinstance(details, dict) else None
+    items = err.get("details") if isinstance(err, dict) else None
+    for item in items if isinstance(items, list) else []:
+        delay = item.get("retryDelay") if isinstance(item, dict) else None
+        if isinstance(delay, str) and delay.endswith("s"):
+            try:
+                return max(0.0, float(delay[:-1]))
+            except ValueError:
+                pass
+    return None
+
+
+def _is_daily_quota(exc: BaseException, server_delay: float | None) -> bool:
+    """True when a 429 reports a per-day quota or a many-minutes retry delay."""
+    if server_delay is not None and server_delay > _RETRY_GIVEUP_DELAY_S:
+        return True
+    return bool(re.search(r"per\s*day|perday", str(exc), re.IGNORECASE))
+
+
+def _retry_decision(exc: BaseException) -> tuple[str, float] | None:
+    """``(kind, wait_seconds)`` if ``exc`` should be retried once, else ``None``.
+
+    Kinds: ``rate_limit`` (429/RESOURCE_EXHAUSTED, 45-60 s jitter),
+    ``server`` (5xx/UNAVAILABLE/DEADLINE_EXCEEDED/INTERNAL) and ``network``
+    (5-10 s jitter). A server Retry-After/retryDelay >= 5 s replaces the
+    jittered value; the wait never exceeds 60 s. A daily-quota 429 is never
+    retried.
+    """
+    if not _is_transient_error(exc):
+        return None
+    code, status = _error_code_status(exc)
+    server_delay = _retry_after_seconds(exc)
+    if code == 429 or (status or "").upper() == "RESOURCE_EXHAUSTED":
+        if _is_daily_quota(exc, server_delay):
+            return None
+        kind = "rate_limit"
+        lo = _env_float("ARGUS_GEMINI_RETRY_RATE_LIMIT_MIN_S", _RETRY_RATE_LIMIT_MIN_S)
+        hi = _env_float("ARGUS_GEMINI_RETRY_RATE_LIMIT_MAX_S", _RETRY_RATE_LIMIT_MAX_S)
+    else:
+        kind = "server" if code is not None or status else "network"
+        lo = _env_float("ARGUS_GEMINI_RETRY_COOLDOWN_MIN_S", _RETRY_SHORT_MIN_S)
+        hi = _env_float("ARGUS_GEMINI_RETRY_COOLDOWN_MAX_S", _RETRY_SHORT_MAX_S)
+    cap = _env_float("ARGUS_GEMINI_RETRY_MAX_WAIT_S", _RETRY_MAX_WAIT_S)
+    wait = random.uniform(lo, max(lo, hi))  # noqa: S311 - jitter, not security
+    if server_delay is not None and server_delay >= _RETRY_AFTER_MIN_S:
+        wait = server_delay
+    return kind, min(wait, cap)
+
+
+def _failure_detail(exc: BaseException) -> str:
+    """Short, key-free label such as ``429 RESOURCE_EXHAUSTED`` or ``ConnectError``."""
+    code, status = _error_code_status(exc)
+    detail = " ".join(p for p in (str(code) if code is not None else "", status or "") if p)
+    detail = detail or type(exc).__name__
+    if (code == 429 or (status or "").upper() == "RESOURCE_EXHAUSTED") and _is_daily_quota(
+        exc, _retry_after_seconds(exc)
+    ):
+        detail += " (daily quota)"
+    return detail
+
+
 # Matches the fenced-json extraction argus.helpers.parse_review_result
 # already applies for the Claude path.
 _FENCED_JSON_RE = re.compile(r"```(?:json)?\s*(\{.*?\})\s*```", re.DOTALL)
@@ -599,6 +767,7 @@ async def _run_turns(
     repo_root: str,
     timeout_s: float,
     started_at: datetime,
+    progress: _Progress | None = None,
 ) -> SessionResult:
     """The actual tool-calling loop. Returns ``failure_reason="turn_budget_exhausted"``
     when the loop ran out of turns without a ``finish_review`` call, else
@@ -736,8 +905,7 @@ async def _run_turns(
                             entry.role if entry else "unknown",
                             model,
                             _turn,
-                            exc,
-                            exc_info=True,
+                            _scrub(str(exc)),
                         )
                         raise
                     logger.warning(
@@ -760,6 +928,8 @@ async def _run_turns(
                         model=model, contents=contents, config=config
                     )
 
+                if progress is not None:
+                    progress.responses += 1
                 usage = response.usage_metadata
                 if usage is not None:
                     usage_prompt_total += usage.prompt_token_count or 0
@@ -922,6 +1092,35 @@ async def _run_turns(
         )
 
 
+async def _run_turns_with_retry(
+    *, progress: _Progress, timeout_s: float, **kwargs: Any
+) -> SessionResult:
+    """``_run_turns`` with at most one retry of a transient turn-0 failure."""
+    label = kwargs.get("label") or "unlabeled"
+    t0 = asyncio.get_running_loop().time()
+    try:
+        return await _run_turns(timeout_s=timeout_s, progress=progress, **kwargs)
+    except (genai_errors.APIError, httpx.HTTPError) as exc:
+        remaining = timeout_s - (asyncio.get_running_loop().time() - t0)
+        decision = _retry_decision(exc) if progress.responses == 0 else None
+        if decision is None or remaining < timeout_s * _RETRY_MIN_REMAINING_FRACTION:
+            raise
+        kind, wait = decision
+        if wait >= remaining:
+            raise
+        progress.retried = True
+        progress.retry_wait_s = wait
+        logger.info(
+            "gemini session [%s] retrying after %s (kind=%s) in %.1fs",
+            label,
+            _failure_detail(exc),
+            kind,
+            wait,
+        )
+        await asyncio.sleep(wait)
+    return await _run_turns(timeout_s=timeout_s, progress=progress, **kwargs)
+
+
 def _redact_gemini_inputs(inputs: dict[str, Any], **_: Any) -> dict[str, Any]:
     """LangSmith ``process_inputs`` hook: redact credentials from Settings."""
     if not isinstance(inputs, dict):
@@ -988,9 +1187,11 @@ async def run_session_gemini(
     )
 
     started_at = datetime.now(timezone.utc)
+    progress = _Progress()
     try:
         return await asyncio.wait_for(
-            _run_turns(
+            _run_turns_with_retry(
+                progress=progress,
                 entry=entry,
                 model=model,
                 system_prompt=system_prompt,
@@ -1057,12 +1258,15 @@ async def run_session_gemini(
         # identically here, even though this runner has no literal
         # subprocess "worker" the way the Claude Agent SDK path does.
         finished_at = datetime.now(timezone.utc)
+        detail = _failure_detail(exc)
         logger.warning(
-            "Gemini session [%s] failed with a non-timeout error after %.1fs (%s: %s)",
+            "Gemini session [%s] failed after %.1fs (%s, %s%s): %s",
             label or "unlabeled",
             (finished_at - started_at).total_seconds(),
             type(exc).__name__,
-            exc,
+            detail,
+            ", after one retry" if progress.retried else "",
+            _scrub(str(exc)),
         )
         return SessionResult(
             result_text="",
@@ -1075,4 +1279,5 @@ async def run_session_gemini(
             context7_call_count=0,
             model=model,
             failure_reason="worker_crashed",
+            failure_detail=detail,
         )

{argus_code_review-0.3.2 → argus_code_review-0.3.3}/argus/runners.py

@@ -572,6 +572,9 @@ class SessionResult:
     model: str | None = None
     failure_reason: Literal["timeout", "worker_crashed", "turn_budget_exhausted"] | None = None
     timed_out: bool = False
+    # Short, key-free reason detail (e.g. "429 RESOURCE_EXHAUSTED") for logs and
+    # diagnostics; not persisted and does not affect failure_reason semantics.
+    failure_detail: str | None = None
 
     def __post_init__(self) -> None:
         self.timed_out, self.failure_reason = _sync_timed_out_and_failure_reason(

{argus_code_review-0.3.2 → argus_code_review-0.3.3}/tests/test_gemini_runner.py

@@ -64,6 +64,10 @@ def _isolate_gemini_cache_dir(tmp_path: Any, monkeypatch: pytest.MonkeyPatch) ->
     from argus.config import clear_cache as clear_settings_cache
 
     monkeypatch.setenv("ARGUS_GEMINI_CACHE_DIR", str(tmp_path / "gemini-cache"))
+    # Existing tests must not sleep through the real retry cooldown.
+    for name in ("RATE_LIMIT", "COOLDOWN"):
+        monkeypatch.setenv(f"ARGUS_GEMINI_RETRY_{name}_MIN_S", "0")
+        monkeypatch.setenv(f"ARGUS_GEMINI_RETRY_{name}_MAX_S", "0")
     clear_settings_cache()
     yield
     clear_settings_cache()
@@ -149,7 +153,7 @@ def _make_response(
     return response
 
 
-def _make_fake_client(responses: list[MagicMock], cache_name: str | None = None) -> MagicMock:
+def _make_fake_client(responses: list[Any], cache_name: str | None = None) -> MagicMock:
     """Build a fake genai.Client with .aio.models.generate_content and
     .caches.create wired up for a test.
 
@@ -1712,3 +1716,238 @@ class TestGoogleBaseUrlOverride:
         assert all(
             ho.base_url == "https://proxy.example.com/v1beta" for ho in constructed_http_options
         )
+
+
+# ---------------------------------------------------------------------------
+# One-shot retry of transient failures
+# ---------------------------------------------------------------------------
+
+
+def _api_error(code: int, status: str, cls: type = genai_errors.ClientError) -> Any:
+    return cls(code, {"message": f"boom {status}", "status": status})
+
+
+async def _run(fake_client: MagicMock, label: str = "retry-test") -> Any:
+    with patch("argus.gemini_runner.genai.Client", return_value=fake_client):
+        return await run_session_gemini(
+            entry=_make_entry(caching="off"),
+            system_prompt="sys",
+            user_message="msg",
+            settings=_make_settings(),
+            label=label,
+            repo_root="/tmp/does-not-need-to-exist",
+        )
+
+
+class TestTransientRetry:
+    async def test_429_then_success_retries_once(self, caplog: pytest.LogCaptureFixture) -> None:
+        ok = _make_response(calls=[], usage=_make_usage(1, 1), text="fine")
+        client = _make_fake_client([_api_error(429, "RESOURCE_EXHAUSTED"), ok])
+        with caplog.at_level("INFO", logger="argus.gemini_runner"):
+            result = await _run(client)
+        assert result.failure_reason is None
+        assert client.aio.models.generate_content.await_count == 2
+        assert any(
+            "retrying after 429 RESOURCE_EXHAUSTED (kind=rate_limit) in" in r.message
+            for r in caplog.records
+        )
+
+    async def test_503_then_503_only_one_retry_with_detail(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        err = _api_error(503, "UNAVAILABLE", genai_errors.ServerError)
+        client = _make_fake_client([err, err, err])
+        with caplog.at_level("INFO", logger="argus.gemini_runner"):
+            result = await _run(client)
+        assert client.aio.models.generate_content.await_count == 2
+        assert result.failure_reason == "worker_crashed"
+        assert result.failure_detail == "503 UNAVAILABLE"
+        warnings = [r for r in caplog.records if r.levelname == "WARNING"]
+        assert any("after one retry" in r.getMessage() for r in warnings)
+
+    @pytest.mark.parametrize(
+        ("code", "status"),
+        [
+            (401, "UNAUTHENTICATED"),
+            (403, "PERMISSION_DENIED"),
+            (400, "INVALID_ARGUMENT"),
+            (404, "NOT_FOUND"),
+        ],
+    )
+    async def test_non_transient_not_retried(self, code: int, status: str) -> None:
+        client = _make_fake_client([_api_error(code, status)])
+        result = await _run(client)
+        assert client.aio.models.generate_content.await_count == 1
+        assert result.failure_reason == "worker_crashed"
+        assert result.failure_detail == f"{code} {status}"
+
+    async def test_httpx_connect_error_is_retried(self) -> None:
+        ok = _make_response(calls=[], usage=_make_usage(1, 1), text="fine")
+        client = _make_fake_client([httpx.ConnectError("reset"), ok])
+        result = await _run(client)
+        assert result.failure_reason is None
+        assert client.aio.models.generate_content.await_count == 2
+
+    async def test_no_retry_after_model_output_already_produced(self) -> None:
+        first = _make_response(calls=[("record_finding", {})], usage=_make_usage(1, 1))
+        client = _make_fake_client(
+            [first, _api_error(503, "UNAVAILABLE", genai_errors.ServerError)]
+        )
+        result = await _run(client)
+        assert client.aio.models.generate_content.await_count == 2  # no third call
+        assert result.failure_reason == "worker_crashed"
+
+    async def test_429_cooldown_jitter_45_to_60(self, monkeypatch: pytest.MonkeyPatch) -> None:
+        from argus.gemini_runner import _retry_decision
+
+        for n in ("MIN", "MAX"):
+            monkeypatch.delenv(f"ARGUS_GEMINI_RETRY_RATE_LIMIT_{n}_S")
+        waits = []
+        for _ in range(200):
+            decision = _retry_decision(_api_error(429, "RESOURCE_EXHAUSTED"))
+            assert decision is not None
+            assert decision[0] == "rate_limit"
+            waits.append(decision[1])
+        assert all(45.0 <= w <= 60.0 for w in waits)
+        assert len(set(waits)) > 1
+
+    async def test_503_and_network_short_cooldown(self, monkeypatch: pytest.MonkeyPatch) -> None:
+        from argus.gemini_runner import _retry_decision
+
+        for n in ("MIN", "MAX"):
+            monkeypatch.delenv(f"ARGUS_GEMINI_RETRY_COOLDOWN_{n}_S")
+        for exc, kind in [
+            (_api_error(503, "UNAVAILABLE", genai_errors.ServerError), "server"),
+            (_api_error(500, "INTERNAL", genai_errors.ServerError), "server"),
+            (httpx.ConnectError("reset"), "network"),
+        ]:
+            for _ in range(50):
+                decision = _retry_decision(exc)
+                assert decision is not None
+                assert decision[0] == kind
+                assert 5.0 <= decision[1] <= 10.0
+
+    async def test_retry_after_honored_and_capped_at_60(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        for n in ("MIN", "MAX"):
+            monkeypatch.delenv(f"ARGUS_GEMINI_RETRY_RATE_LIMIT_{n}_S")
+        sleeps: list[float] = []
+
+        async def fake_sleep(d: float) -> None:
+            sleeps.append(d)
+
+        monkeypatch.setattr("argus.gemini_runner.asyncio.sleep", fake_sleep)
+        ok = _make_response(calls=[], usage=_make_usage(1, 1), text="fine")
+
+        def err_with_retry_after(value: str) -> Any:
+            e = _api_error(429, "RESOURCE_EXHAUSTED")
+            e.response = MagicMock(headers={"retry-after": value})
+            return e
+
+        await _run(_make_fake_client([err_with_retry_after("10"), ok]))
+        await _run(_make_fake_client([err_with_retry_after("100"), ok]))
+        # < 5 s is ignored in favour of the 45-60 s jitter
+        await _run(_make_fake_client([err_with_retry_after("2"), ok]))
+        assert sleeps[0] == 10.0
+        assert sleeps[1] == 60.0
+        assert 45.0 <= sleeps[2] <= 60.0
+
+    async def test_retry_delay_in_error_details_is_used(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        sleeps: list[float] = []
+
+        async def fake_sleep(d: float) -> None:
+            sleeps.append(d)
+
+        monkeypatch.setattr("argus.gemini_runner.asyncio.sleep", fake_sleep)
+        err = genai_errors.ClientError(
+            429,
+            {
+                "error": {
+                    "message": "slow down",
+                    "status": "RESOURCE_EXHAUSTED",
+                    "details": [
+                        {"@type": "type.googleapis.com/google.rpc.RetryInfo", "retryDelay": "17s"}
+                    ],
+                }
+            },
+        )
+        ok = _make_response(calls=[], usage=_make_usage(1, 1), text="fine")
+        await _run(_make_fake_client([err, ok]))
+        assert sleeps == [17.0]
+
+    @pytest.mark.parametrize(
+        "payload",
+        [
+            {
+                "message": "Quota exceeded for metric ... PerDayPerProject",
+                "status": "RESOURCE_EXHAUSTED",
+            },
+            {
+                "message": "exceeded your daily quota (requests per day)",
+                "status": "RESOURCE_EXHAUSTED",
+            },
+        ],
+    )
+    async def test_daily_quota_429_not_retried(
+        self, payload: dict[str, str], monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        slept: list[float] = []
+
+        async def fake_sleep(d: float) -> None:
+            slept.append(d)
+
+        monkeypatch.setattr("argus.gemini_runner.asyncio.sleep", fake_sleep)
+        client = _make_fake_client([genai_errors.ClientError(429, payload)])
+        result = await _run(client)
+        assert client.aio.models.generate_content.await_count == 1
+        assert slept == []
+        assert result.failure_reason == "worker_crashed"
+        assert result.failure_detail == "429 RESOURCE_EXHAUSTED (daily quota)"
+
+    async def test_429_with_long_retry_delay_not_retried(self) -> None:
+        err = _api_error(429, "RESOURCE_EXHAUSTED")
+        err.response = MagicMock(headers={"retry-after": "3600"})
+        client = _make_fake_client([err])
+        result = await _run(client)
+        assert client.aio.models.generate_content.await_count == 1
+        assert "daily quota" in (result.failure_detail or "")
+
+    async def test_only_one_retry_sleeps_once(self, monkeypatch: pytest.MonkeyPatch) -> None:
+        slept: list[float] = []
+
+        async def fake_sleep(d: float) -> None:
+            slept.append(d)
+
+        monkeypatch.setattr("argus.gemini_runner.asyncio.sleep", fake_sleep)
+        err = _api_error(429, "RESOURCE_EXHAUSTED")
+        client = _make_fake_client([err, err, err])
+        result = await _run(client)
+        assert client.aio.models.generate_content.await_count == 2
+        assert len(slept) == 1
+        assert result.failure_detail == "429 RESOURCE_EXHAUSTED"
+
+    async def test_retry_info_log_has_kind_and_cooldown(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        ok = _make_response(calls=[], usage=_make_usage(1, 1), text="fine")
+        client = _make_fake_client([_api_error(429, "RESOURCE_EXHAUSTED"), ok])
+        with caplog.at_level("INFO", logger="argus.gemini_runner"):
+            await _run(client)
+        assert any("(kind=rate_limit) in" in r.getMessage() for r in caplog.records)
+
+    async def test_failure_log_redacts_keys_and_truncates(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        secret = "AIza" + "x" * 35
+        err = genai_errors.ClientError(
+            401, {"message": f"bad key={secret} " + "y" * 1000, "status": "UNAUTHENTICATED"}
+        )
+        client = _make_fake_client([err])
+        with caplog.at_level("WARNING", logger="argus.gemini_runner"):
+            await _run(client)
+        text = " ".join(r.getMessage() for r in caplog.records)
+        assert secret not in text
+        assert len(text) < 1200