gooddata-eval 1.68.1.dev2__py3-none-any.whl → 1.68.1.dev3__py3-none-any.whl

gooddata_eval/core/chat/sse_client.py

@@ -14,15 +14,88 @@ protocol, not on this class.
 """
 
 import json
+import logging
+import os
+import time
 from dataclasses import dataclass, field
-from typing import Any, Iterable
+from typing import Any, Callable, Iterable, TypeVar
 
 import httpx
 
 from gooddata_eval.core.models import ChatResult, DatasetItem
 
+_log = logging.getLogger(__name__)
+
 SSE_DATA_PREFIX = "data: "
 
+_RETRYABLE_STATUS_CODES: frozenset[int] = frozenset({429, 502, 503, 504})
+_METADATA_SYNC_MARKER = "METADATA_SYNC_IN_PROGRESS"
+
+
+class ChatError(RuntimeError):
+    """Non-retryable error reported by the chat SSE stream."""
+
+    def __init__(self, message: str, *, status_code: int | None = None, detail: str | None = None) -> None:
+        super().__init__(message)
+        self.status_code = status_code
+        self.detail = detail
+
+
+class TransientChatError(ChatError):
+    """Retryable transient error: gen-ai temporarily unavailable or still syncing metadata."""
+
+
+def _int_env(name: str, default: int) -> int:
+    """Read an int from the environment, falling back to ``default`` when unset or blank."""
+    raw = os.getenv(name)
+    return int(raw) if raw else default
+
+
+def _float_env(name: str, default: float) -> float:
+    """Read a float from the environment, falling back to ``default`` when unset or blank."""
+    raw = os.getenv(name)
+    return float(raw) if raw else default
+
+
+# Retry budget. Defaults give a ~2 min worst-case cap per send (5/10/20/40/60s);
+# overridable via env so CI can retune without cutting a new gooddata-eval release.
+_MAX_RETRIES = _int_env("GOODDATA_EVAL_CHAT_MAX_RETRIES", 5)
+_INITIAL_BACKOFF_S = _float_env("GOODDATA_EVAL_CHAT_INITIAL_BACKOFF_S", 5.0)
+_BACKOFF_FACTOR = _float_env("GOODDATA_EVAL_CHAT_BACKOFF_FACTOR", 2.0)
+_MAX_BACKOFF_S = _float_env("GOODDATA_EVAL_CHAT_MAX_BACKOFF_S", 60.0)
+
+T = TypeVar("T")
+
+
+def _is_retryable_exc(exc: Exception) -> bool:
+    if isinstance(exc, TransientChatError):
+        return True
+    if isinstance(exc, httpx.HTTPStatusError):
+        return exc.response.status_code in _RETRYABLE_STATUS_CODES
+    return False
+
+
+def _retry_transient(operation: Callable[[], T], *, is_retryable: Callable[[Exception], bool]) -> T:
+    """Run ``operation``; retry retryable failures with bounded exponential backoff."""
+    delay = _INITIAL_BACKOFF_S
+    for attempt in range(_MAX_RETRIES + 1):  # 0..N => N retries + 1 initial attempt
+        try:
+            return operation()
+        except Exception as exc:  # noqa: PERF203 — retry loop: per-attempt try/except is intentional
+            if attempt == _MAX_RETRIES or not is_retryable(exc):
+                raise
+            sleep_s = min(delay, _MAX_BACKOFF_S)
+            _log.warning(
+                "Transient gen-ai error (attempt %d/%d): %s; retrying in %.0fs",
+                attempt + 1,
+                _MAX_RETRIES + 1,
+                exc,
+                sleep_s,
+            )
+            time.sleep(sleep_s)
+            delay *= _BACKOFF_FACTOR
+    raise AssertionError("unreachable")  # loop either returns or raises
+
 
 @dataclass
 class _SseAccumulator:
@@ -114,12 +187,23 @@ def parse_sse_lines(lines: Iterable[str]) -> ChatResult:
         if not line or line.startswith("event: ") or not line.startswith(SSE_DATA_PREFIX):
             continue
         data_str = line[len(SSE_DATA_PREFIX) :]
+        if _METADATA_SYNC_MARKER in data_str:
+            raise TransientChatError(
+                f"SSE transient error: {_METADATA_SYNC_MARKER}",
+                status_code=None,
+                detail=None,
+            )
         try:
             event_data = json.loads(data_str)
         except json.JSONDecodeError:
             continue
         if "statusCode" in event_data:
-            raise RuntimeError(f"SSE error {event_data.get('statusCode')}: {event_data.get('detail')}")
+            code = event_data.get("statusCode")
+            detail = event_data.get("detail")
+            message = f"SSE error {code}: {detail}"
+            if code in _RETRYABLE_STATUS_CODES:
+                raise TransientChatError(message, status_code=code, detail=detail)
+            raise ChatError(message, status_code=code, detail=detail)
         item = event_data.get("item")
         if not item:
             continue
@@ -149,12 +233,17 @@ class ChatClient:
         self._client = httpx.Client(timeout=timeout)
 
     def create_conversation(self) -> str:
-        resp = self._client.post(self._base, headers={**self._auth, "Content-Type": "application/json"})
-        resp.raise_for_status()
-        body = resp.json()
-        if "conversationId" not in body:
-            raise ValueError(f"GoodData /chat/conversations response missing 'conversationId': {body}")
-        return body["conversationId"]
+        def _do() -> str:
+            resp = self._client.post(self._base, headers={**self._auth, "Content-Type": "application/json"})
+            resp.raise_for_status()
+            body = resp.json()
+            if "conversationId" not in body:
+                raise ValueError(f"GoodData /chat/conversations response missing 'conversationId': {body}")
+            return body["conversationId"]
+
+        # NOTE: retrying create is not idempotent — a created-then-503 can leak an
+        # orphaned (ephemeral) conversation. Acceptable for eval; do not reuse blindly.
+        return _retry_transient(_do, is_retryable=_is_retryable_exc)
 
     def delete_conversation(self, conversation_id: str) -> None:
         try:
@@ -166,9 +255,13 @@ class ChatClient:
         url = f"{self._base}/{conversation_id}/messages"
         headers = {**self._auth, "Accept": "text/event-stream", "Content-Type": "application/json"}
         body = {"item": {"role": "user", "content": {"type": "text", "text": question}}}
-        with self._client.stream("POST", url, json=body, headers=headers) as resp:
-            resp.raise_for_status()
-            return parse_sse_lines(resp.iter_lines())
+
+        def _do() -> ChatResult:
+            with self._client.stream("POST", url, json=body, headers=headers) as resp:
+                resp.raise_for_status()
+                return parse_sse_lines(resp.iter_lines())
+
+        return _retry_transient(_do, is_retryable=_is_retryable_exc)
 
     def ask(self, item: DatasetItem) -> ChatResult:
         """Run one single-turn conversation: create, send, parse, clean up."""

{gooddata_eval-1.68.1.dev2.dist-info → gooddata_eval-1.68.1.dev3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gooddata-eval
-Version: 1.68.1.dev2
+Version: 1.68.1.dev3
 Summary: Evaluate the GoodData AI agent against your own questions and models.
 Project-URL: Source, https://github.com/gooddata/gooddata-python-sdk
 Author-email: GoodData <support@gooddata.com>
@@ -17,7 +17,7 @@ Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Software Development
 Classifier: Typing :: Typed
 Requires-Python: >=3.10
-Requires-Dist: gooddata-sdk~=1.68.1.dev2
+Requires-Dist: gooddata-sdk~=1.68.1.dev3
 Requires-Dist: httpx<1.0,>=0.27
 Requires-Dist: orjson<4.0.0,>=3.9.15
 Requires-Dist: pydantic<3.0,>=2.6

{gooddata_eval-1.68.1.dev2.dist-info → gooddata_eval-1.68.1.dev3.dist-info}/RECORD

@@ -21,7 +21,7 @@ gooddata_eval/core/agentic/metric_skill.py,sha256=jFqVsxNEx-7IHa4v8aNLkDzD9OogDs
 gooddata_eval/core/agentic/search_tool.py,sha256=EiWuzwdWUTY8H6wLST1rBB-fvc4meMazLKhiy0nl9K0,7634
 gooddata_eval/core/agentic/visualization.py,sha256=ScSfx4-4lwHswqF_0ugOrIFu5d4tlAQf6CxaFVV9_mE,16743
 gooddata_eval/core/chat/__init__.py,sha256=U54lANKm34yjqm7dZL9KkGouPbwAaQWC9wiAmb5B5g4,32
-gooddata_eval/core/chat/sse_client.py,sha256=LVjThACrLQFXoeRo_HiUMAQPx1ixEQUXbw1q0C6_oWk,7573
+gooddata_eval/core/chat/sse_client.py,sha256=1FVi2HWf2-uHMro7uWSiTiQTMEbh6WhVvQW8flnxymY,11205
 gooddata_eval/core/dataset/__init__.py,sha256=U54lANKm34yjqm7dZL9KkGouPbwAaQWC9wiAmb5B5g4,32
 gooddata_eval/core/dataset/langfuse_source.py,sha256=R2KzNx6LxAUtDZwHL5_UW5CJ6q-QnQGW7AhLZKBTDKU,5589
 gooddata_eval/core/dataset/local.py,sha256=qoFFtRx_bNwlT1U37o8CKkSPwCad3JpmHPcNRuGSbjE,1233
@@ -44,8 +44,8 @@ gooddata_eval/core/reporting/console.py,sha256=kCHt4sCSaSRh0ZWqaSEJOl6_Jgb6uiouS
 gooddata_eval/core/reporting/json_report.py,sha256=NIA_PvnZ0PbJnXz_zZauxc1gGnzqlLD_ouVqWeNmjtw,2837
 gooddata_eval/core/summary/__init__.py,sha256=U54lANKm34yjqm7dZL9KkGouPbwAaQWC9wiAmb5B5g4,32
 gooddata_eval/core/summary/http_client.py,sha256=dPArJ6zoCu1W9xmYXFA1WMDNsSOkhn3gaCpYgCUp3gk,2174
-gooddata_eval-1.68.1.dev2.dist-info/METADATA,sha256=UJ6kD_VVKrTS38_cgQFLH_7Dc5YCjBClr0KbQ13jePU,9513
-gooddata_eval-1.68.1.dev2.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
-gooddata_eval-1.68.1.dev2.dist-info/entry_points.txt,sha256=28nFp5Viknx4haPYgzK9QHlwPFfC6rPF6RmjC2ht8EI,56
-gooddata_eval-1.68.1.dev2.dist-info/licenses/LICENSE.txt,sha256=LVfVlC9maU3K9lgMwdZnClQQsOIbOekdcdVKr9qzJtk,256836
-gooddata_eval-1.68.1.dev2.dist-info/RECORD,,
+gooddata_eval-1.68.1.dev3.dist-info/METADATA,sha256=ZGte7olPAbOw-BM69NnUhCskaAP4lSQRwwFTnapqJJM,9513
+gooddata_eval-1.68.1.dev3.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+gooddata_eval-1.68.1.dev3.dist-info/entry_points.txt,sha256=28nFp5Viknx4haPYgzK9QHlwPFfC6rPF6RmjC2ht8EI,56
+gooddata_eval-1.68.1.dev3.dist-info/licenses/LICENSE.txt,sha256=LVfVlC9maU3K9lgMwdZnClQQsOIbOekdcdVKr9qzJtk,256836
+gooddata_eval-1.68.1.dev3.dist-info/RECORD,,