google-cloud-agentplatform 2.3.0__tar.gz → 2.5.0__tar.gz

{google_cloud_agentplatform-2.3.0/google_cloud_agentplatform.egg-info → google_cloud_agentplatform-2.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: google-cloud-agentplatform
-Version: 2.3.0
+Version: 2.5.0
 Summary: Google Cloud Agent Platform Client Library
 Author-email: Google LLC <googleapis-packages@google.com>
 License: Apache-2.0
@@ -31,7 +31,7 @@ Requires-Dist: google-cloud-storage<4.0.0,>=1.32.0; extra == "agent-engines"
 Requires-Dist: opentelemetry-sdk<2; extra == "agent-engines"
 Requires-Dist: opentelemetry-exporter-gcp-logging<2.0.0,>=1.11.0a0; extra == "agent-engines"
 Requires-Dist: opentelemetry-exporter-otlp-proto-http<2; extra == "agent-engines"
-Requires-Dist: opentelemetry-instrumentation-google-genai<1.0.0,>=0.3b0; extra == "agent-engines"
+Requires-Dist: opentelemetry-instrumentation-google-genai<2.0.0,>=0.3b0; extra == "agent-engines"
 Requires-Dist: pydantic<3,>=2.11.1; extra == "agent-engines"
 Provides-Extra: a2a
 Requires-Dist: google-cloud-agentplatform[agent_engines]; extra == "a2a"

{google_cloud_agentplatform-2.3.0 → google_cloud_agentplatform-2.5.0}/agentplatform/_genai/_evals_common.py

@@ -22,6 +22,7 @@ import datetime
 import json
 import logging
 import os
+import random
 import threading
 import time
 from typing import Any, Callable, Literal, Optional, Union, cast
@@ -122,6 +123,11 @@ def _temp_logger_level(logger_name: str, level: int) -> None:  # type: ignore[mi
         logger_instance.setLevel(original_level)
 
 
+def _is_missing(value: Any) -> bool:
+    """Returns True for None or a scalar NaN, such as a missing DataFrame cell."""
+    return bool(pd.api.types.is_scalar(value) and pd.isna(value))
+
+
 def _get_api_client_with_location(
     api_client: BaseApiClient, location: Optional[str]
 ) -> BaseApiClient:
@@ -153,6 +159,8 @@ def _get_runtime_instance(
         client = agentplatform.Client(
             project=api_client.project,
             location=api_client.location,
+            credentials=api_client._credentials,
+            http_options=api_client._http_options,
         )
         _thread_local_data.runtime_instances[agent_name] = client.runtimes.get(
             name=agent_name
@@ -160,6 +168,45 @@ def _get_runtime_instance(
     return _thread_local_data.runtime_instances[agent_name]
 
 
+def _is_retryable_error(e: Exception) -> bool:
+    """Returns True for transient errors that are safe to retry with backoff."""
+    if isinstance(e, genai_errors.APIError):
+        return e.code in _evals_constant.RETRYABLE_STATUS_CODES
+    return isinstance(
+        e, (api_exceptions.ResourceExhausted, api_exceptions.ServiceUnavailable)
+    )
+
+
+def _get_retry_backoff(
+    e: Exception, attempt: int, max_retries: int, operation: str
+) -> Optional[float]:
+    """Returns the seconds to wait before the next attempt, or None to stop.
+
+    Retryable errors back off exponentially with jitter. Other errors fail
+    fast, and the last attempt never waits.
+    """
+    if not _is_retryable_error(e) or attempt == max_retries - 1:
+        logger.error(
+            "Error during %s on attempt %d/%d: %s",
+            operation,
+            attempt + 1,
+            max_retries,
+            e,
+        )
+        return None
+    backoff = 2**attempt + random.uniform(0, 1)
+    logger.warning(
+        "Retryable error during %s on attempt %d/%d: %s. Retrying in %.1f"
+        " seconds...",
+        operation,
+        attempt + 1,
+        max_retries,
+        e,
+        backoff,
+    )
+    return backoff
+
+
 def _generate_content_with_retry(
     api_client: BaseApiClient,
     model: str,
@@ -222,29 +269,11 @@ def _generate_content_with_retry(
                         }
                 else:
                     return response
-        except api_exceptions.ResourceExhausted as e:
-            logger.warning(
-                "Resource Exhausted error on attempt %d/%d: %s. Retrying in %s"
-                " seconds...",
-                attempt + 1,
-                max_retries,
-                e,
-                2**attempt,
-            )
-            if attempt == max_retries - 1:
-                return {"error": f"Resource exhausted after retries: {e}"}
-            time.sleep(2**attempt)
         except Exception as e:  # pylint: disable=broad-exception-caught
-            logger.error(
-                "Unexpected error during generate_content on attempt %d/%d: %s",
-                attempt + 1,
-                max_retries,
-                e,
-            )
-
-            if attempt == max_retries - 1:
-                return {"error": f"Failed after retries: {e}"}
-            time.sleep(1)
+            backoff = _get_retry_backoff(e, attempt, max_retries, "generate_content")
+            if backoff is None:
+                return {"error": f"Failed on attempt {attempt + 1}/{max_retries}: {e}"}
+            time.sleep(backoff)
     return {"error": f"Failed to generate content after {max_retries} retries"}
 
 
@@ -1561,7 +1590,7 @@ def _build_interaction_id_dataset(
     eval_cases = []
     for i, row in enumerate(loaded_data):
         interaction = row.get(_evals_constant.INTERACTION_ID)
-        if not interaction:
+        if _is_missing(interaction) or not interaction:
             raise ValueError(f"Missing `interaction_id` value for row {i}.")
         eval_cases.append(
             types.EvalCase(
@@ -1871,12 +1900,12 @@ def _execute_inference_concurrently(
     max_workers = AGENT_MAX_WORKERS if runtime or agent else MAX_WORKERS
     with tqdm(total=len(prompt_dataset), desc=progress_desc) as pbar:
         with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
-            for index, row in prompt_dataset.iterrows():
+            for index, (_, row) in enumerate(prompt_dataset.iterrows()):
                 try:
                     if (
                         has_agent_data
                         and AGENT_DATA in row.index
-                        and row.get(AGENT_DATA) is not None
+                        and not _is_missing(row.get(AGENT_DATA))
                     ):
                         agent_data_obj = row[AGENT_DATA]
                         if isinstance(agent_data_obj, dict):
@@ -2101,7 +2130,7 @@ def _run_litellm_inference(
 
     with tqdm(total=len(prompt_dataset), desc=f"LiteLLM Inference ({model})") as pbar:
         with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
-            for index, row in prompt_dataset.iterrows():
+            for index, (_, row) in enumerate(prompt_dataset.iterrows()):
                 messages = _convert_prompt_row_to_litellm_messages(row)
                 future = executor.submit(
                     _call_litellm_completion, model=model, messages=messages
@@ -2654,7 +2683,7 @@ def _execute_inference(
             "Gemini Agent inference completed in %.2f seconds.",
             end_time - start_time,
         )
-        return types.EvaluationDataset(
+        evaluation_dataset = types.EvaluationDataset(
             eval_dataset_df=results_df,
             candidate_name=gemini_agent.split("/")[-1],
         )
@@ -2759,7 +2788,12 @@ def _execute_inference(
                 results_df.to_json(full_dest_path, orient="records", lines=True)
                 logger.info("Results saved locally to: %s", full_dest_path)
         except Exception as e:  # pylint: disable=broad-exception-caught
-            logger.error("Failed to save results to %s. Error: %s", full_dest_path, e)
+            logger.error(
+                "Failed to save results to %s. Error: %s",
+                full_dest_path,
+                e,
+                exc_info=True,
+            )
 
     return evaluation_dataset
 
@@ -2886,6 +2920,21 @@ def _resolve_dataset_inputs(
     return processed_eval_dataset, num_response_candidates
 
 
+def _prebuilt_evaluation_run_metric(
+    resolved_metric: types.Metric,
+) -> types.EvaluationRunMetric:
+    """Builds the evaluation run metric for a resolved RubricMetric."""
+    if resolved_metric.name in _evals_constant.SUPPORTED_PREDEFINED_METRICS:
+        metric_config = t.t_metrics([resolved_metric])[0]
+    else:
+        metric_config = {
+            "predefined_metric_spec": {"metric_spec_name": resolved_metric.name}
+        }
+    return types.EvaluationRunMetric(
+        metric=resolved_metric.name, metric_config=metric_config
+    )
+
+
 def _resolve_evaluation_run_metrics(
     metrics: Union[list[types.EvaluationRunMetric], list[types.Metric]], api_client: Any
 ) -> list[types.EvaluationRunMetric]:
@@ -2903,14 +2952,7 @@ def _resolve_evaluation_run_metrics(
                 resolved_metric = metric_instance.resolve(api_client=api_client)
                 if resolved_metric.name:
                     resolved_metrics_list.append(
-                        types.EvaluationRunMetric(
-                            metric=resolved_metric.name,
-                            metric_config=types.UnifiedMetric(
-                                predefined_metric_spec=genai_types.PredefinedMetricSpec(
-                                    metric_spec_name=resolved_metric.name,
-                                )
-                            ),
-                        )
+                        _prebuilt_evaluation_run_metric(resolved_metric)
                     )
             except Exception as e:
                 logger.error(
@@ -2944,14 +2986,7 @@ def _resolve_evaluation_run_metrics(
                     )
                     if resolved_metric.name:
                         resolved_metrics_list.append(
-                            types.EvaluationRunMetric(
-                                metric=resolved_metric.name,
-                                metric_config=types.UnifiedMetric(
-                                    predefined_metric_spec=genai_types.PredefinedMetricSpec(
-                                        metric_spec_name=resolved_metric.name,
-                                    )
-                                ),
-                            )
+                            _prebuilt_evaluation_run_metric(resolved_metric)
                         )
                 else:
                     raise TypeError(
@@ -3020,6 +3055,7 @@ def _execute_evaluation(  # type: ignore[no-untyped-def]
     dest: Optional[str] = None,
     location: Optional[str] = None,
     evaluation_service_qps: Optional[float] = None,
+    allow_cross_region_model: Optional[bool] = None,
     **kwargs,
 ) -> types.EvaluationResult:
     """Evaluates a dataset using the provided metrics.
@@ -3035,6 +3071,8 @@ def _execute_evaluation(  # type: ignore[no-untyped-def]
         evaluation_service_qps: The rate limit (queries per second) for calls
           to the evaluation service. Defaults to 10. Increase this value if
           your project has a higher EvaluateInstances API quota.
+        allow_cross_region_model: Opt-in flag to authorize cross-region
+          routing for judge models.
         **kwargs: Extra arguments to pass to evaluation, such as `agent_info`.
 
     Returns:
@@ -3117,6 +3155,7 @@ def _execute_evaluation(  # type: ignore[no-untyped-def]
     evaluation_result = _evals_metric_handlers.compute_metrics_and_aggregate(
         evaluation_run_config,
         evaluation_service_qps=evaluation_service_qps,
+        allow_cross_region_model=allow_cross_region_model,
     )
     t2 = time.perf_counter()
     logger.info("Evaluation took: %f seconds", t2 - t1)
@@ -3471,7 +3510,9 @@ def _create_runtime_session(
             raise RuntimeError(
                 "Failed to create session: runtime.api_client is None."
             ) from exc
-        operation = runtime.api_client.sessions.create(
+        from . import sessions
+
+        operation = sessions.Sessions(runtime.api_client._api_client).create(
             name=runtime.api_resource.name,
             user_id=user_id,
             config=types.CreateRuntimeSessionConfig(
@@ -3535,10 +3576,12 @@ def _execute_agent_run_with_retry(
             return {"error": "runtime.api_resource is None."}
         if runtime.api_client is None:
             return {"error": "runtime.api_client is None."}
+        from . import sessions
+
         session_name = f"{runtime.api_resource.name}/sessions/{session_id}"
         base_ts = datetime.datetime(2000, 1, 1, tzinfo=datetime.timezone.utc)
         for i, ag_event in enumerate(history_events):
-            runtime.api_client.sessions.events.append(
+            sessions.Sessions(runtime.api_client._api_client).events.append(
                 name=session_name,
                 author=ag_event.author or "user",
                 invocation_id="history",
@@ -3560,28 +3603,11 @@ def _execute_agent_run_with_retry(
                 if event and CONTENT in event and PARTS in event[CONTENT]:
                     responses.append(event)
             return responses
-        except api_exceptions.ResourceExhausted as e:
-            logger.warning(
-                "Resource Exhausted error on attempt %d/%d: %s. Retrying in %s"
-                " seconds...",
-                attempt + 1,
-                max_retries,
-                e,
-                2**attempt,
-            )
-            if attempt == max_retries - 1:
-                return {"error": f"Resource exhausted after retries: {e}"}
-            time.sleep(2**attempt)
         except Exception as e:  # pylint: disable=broad-exception-caught
-            logger.error(
-                "Unexpected error during agent engine run on attempt %d/%d: %s",
-                attempt + 1,
-                max_retries,
-                e,
-            )
-            if attempt == max_retries - 1:
-                return {"error": f"Failed after retries: {e}"}
-            time.sleep(1)
+            backoff = _get_retry_backoff(e, attempt, max_retries, "agent engine run")
+            if backoff is None:
+                return {"error": f"Failed on attempt {attempt + 1}/{max_retries}: {e}"}
+            time.sleep(backoff)
     return {"error": f"Failed to get agent run results after {max_retries} retries"}
 
 
@@ -3613,7 +3639,7 @@ async def _execute_local_agent_run_with_retry_async(
     # Lazy-import ADK dependencies to avoid top-level import failures when
     # google-adk is not installed.
     from google.adk.runners import Runner
-    from google.adk.sessions import InMemorySessionService
+    from google.adk.sessions.in_memory_session_service import InMemorySessionService
 
     # Multi-turn agent scraping with user simulation.
     if user_simulator_config or "conversation_plan" in row:
@@ -3687,28 +3713,13 @@ async def _execute_local_agent_run_with_retry_async(
                     if event and CONTENT in event and PARTS in event[CONTENT]:
                         events.append(event)
                 return events
-            except api_exceptions.ResourceExhausted as e:
-                logger.warning(
-                    "Resource Exhausted error on attempt %d/%d: %s. Retrying"
-                    " in %s seconds...",
-                    attempt + 1,
-                    max_retries,
-                    e,
-                    2**attempt,
-                )
-                if attempt == max_retries - 1:
-                    return {"error": f"Resource exhausted after retries: {e}"}
-                await asyncio.sleep(2**attempt)
             except Exception as e:  # pylint: disable=broad-exception-caught
-                logger.error(
-                    "Unexpected error during agent run on attempt %d/%d: %s",
-                    attempt + 1,
-                    max_retries,
-                    e,
-                )
-                if attempt == max_retries - 1:
-                    return {"error": f"Failed after retries: {e}"}
-                await asyncio.sleep(1)
+                backoff = _get_retry_backoff(e, attempt, max_retries, "agent run")
+                if backoff is None:
+                    return {
+                        "error": f"Failed on attempt {attempt + 1}/{max_retries}: {e}"
+                    }
+                await asyncio.sleep(backoff)
         return {"error": f"Failed to get agent run results after {max_retries} retries"}
 
 
@@ -3864,7 +3875,7 @@ def _drop_empty_columns(df: "pd.DataFrame") -> "pd.DataFrame":
     def is_empty(x: Any) -> bool:
         if isinstance(x, (list, dict)):
             return not x
-        return pd.isna(x)  # type: ignore[no-any-return]
+        return _is_missing(x)
 
     cols_to_drop = [col for col in df.columns if df[col].apply(is_empty).all()]
     return df.drop(columns=cols_to_drop)
@@ -4163,6 +4174,8 @@ def _create_evaluation_set_from_dataframe(
         if _evals_constant.RESPONSE in row or agent_data_obj or intermediate_events:
             # Resolve the oneof conflict: prioritize agent_data over flat text
             response_text = row.get(_evals_constant.RESPONSE) or None
+            if _is_missing(response_text):
+                response_text = None
 
             if agent_data_obj and response_text:
                 logger.info(
@@ -4217,7 +4230,7 @@ def _create_evaluation_set_from_dataframe(
             prompt = types.EvaluationPrompt(
                 prompt_template_data=types.PromptTemplateData(values=values)
             )
-        elif _evals_constant.PROMPT in row:
+        elif not _is_missing(row.get(_evals_constant.PROMPT)):
             prompt = types.EvaluationPrompt(text=row[_evals_constant.PROMPT])
 
         eval_item_requests.append(
@@ -4225,7 +4238,7 @@ def _create_evaluation_set_from_dataframe(
                 prompt=prompt or None,
                 golden_response=(
                     types.CandidateResponse(text=row[_evals_constant.REFERENCE])
-                    if _evals_constant.REFERENCE in row
+                    if not _is_missing(row.get(_evals_constant.REFERENCE))
                     else None
                 ),
                 candidate_responses=(

{google_cloud_agentplatform-2.3.0 → google_cloud_agentplatform-2.5.0}/agentplatform/_genai/_evals_constant.py

@@ -120,3 +120,17 @@ COMMON_DATASET_COLUMNS = frozenset(
         INTERACTION_ID,
     }
 )
+
+# HTTP status codes that are safe to retry with backoff.
+RETRYABLE_STATUS_CODES = frozenset(
+    {
+        408,  # RequestTimeout (DEADLINE_EXCEEDED)
+        409,  # Conflict / Aborted (ABORTED)
+        429,  # TooManyRequests / ResourceExhausted (RESOURCE_EXHAUSTED)
+        499,  # Client Closed Request (CANCELLED)
+        500,  # InternalServerError (INTERNAL)
+        502,  # BadGateway
+        503,  # ServiceUnavailable (UNAVAILABLE)
+        504,  # GatewayTimeout (DEADLINE_EXCEEDED)
+    }
+)