google-cloud-agentplatform 2.1.3__tar.gz → 2.4.0__tar.gz

{google_cloud_agentplatform-2.1.3/google_cloud_agentplatform.egg-info → google_cloud_agentplatform-2.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: google-cloud-agentplatform
-Version: 2.1.3
+Version: 2.4.0
 Summary: Google Cloud Agent Platform Client Library
 Author-email: Google LLC <googleapis-packages@google.com>
 License: Apache-2.0

{google_cloud_agentplatform-2.1.3 → google_cloud_agentplatform-2.4.0}/agentplatform/__init__.py

@@ -54,8 +54,17 @@ def __getattr__(name):  # type: ignore[no-untyped-def]
         global _genai_types
         if _genai_types is None:
             _genai_types = importlib.import_module("._genai.types", __name__)
-        if "vertexai.types" not in sys.modules:
-            sys.modules["vertexai.types"] = _genai_types
+        # `types` is an alias for `._genai.types` rather than a real submodule,
+        # so register it to keep
+        # `from agentplatform.types import TypeName`
+        # working without a prior attribute access. Spell it the google3 way:
+        # Copybara rewrites that prefix in both directions, and the external
+        # spelling fails its reversibility check. This key was `vertexai.types`,
+        # which both misnamed this package's own alias and shadowed the real
+        # `vertexai.types` module.
+        types_module_name = f"{__name__}.types"
+        if types_module_name not in sys.modules:
+            sys.modules[types_module_name] = _genai_types
         return _genai_types
 
     raise AttributeError(f"module '{__name__}' has no attribute '{name}'")

{google_cloud_agentplatform-2.1.3 → google_cloud_agentplatform-2.4.0}/agentplatform/_genai/_evals_common.py

@@ -37,6 +37,11 @@ from google.genai._gaos.types.interactions import functioncallstep
 from google.genai._gaos.types.interactions import functionresultstep
 from google.genai._gaos.types.interactions import modeloutputstep
 from google.genai._gaos.types.interactions import userinputstep
+
+try:
+    from google.genai._gaos.utils.serializers import ALLOW_UNKNOWN_UNION_VARIANTS
+except (ImportError, AttributeError):
+    ALLOW_UNKNOWN_UNION_VARIANTS = "speakeasy_allow_unknown_union_variants"
 from google.genai.models import Models
 import pandas as pd
 from tqdm import tqdm
@@ -775,7 +780,16 @@ def _interaction_dict_to_agent_data(
     Returns:
         An AgentData object with one or more ConversationTurns.
     """
-    typed_interaction = interaction_types.Interaction.model_validate(interaction)
+    # A server response, so unknown step types must degrade rather than raise.
+    # `Step` is an open discriminated union: parse_open_union only falls back to
+    # the Unknown variant when the validation context carries
+    # ALLOW_UNKNOWN_UNION_VARIANTS, and raises without it so that a
+    # user-constructed request payload surfaces its mistakes locally. This
+    # payload comes off the Interactions API, so it is on the tolerant side of
+    # that line. _interaction_steps_to_events already drops steps it cannot map.
+    typed_interaction = interaction_types.Interaction.model_validate(
+        interaction, context={ALLOW_UNKNOWN_UNION_VARIANTS: True}
+    )
     all_events = _interaction_steps_to_events(typed_interaction.steps or [])
 
     # Group events into turns. Each UserInputStep starts a new turn.
@@ -1689,8 +1703,12 @@ def _resolve_interactions_to_eval_cases(
                 break
             interaction_dict = json.loads(response.body)
             try:
+                # Also a server response -- see _interaction_dict_to_agent_data.
+                # Without the context a single step type this client does not
+                # know about raised, and the except below turned that into a
+                # `break`, silently truncating the interaction history.
                 typed_interaction = interaction_types.Interaction.model_validate(
-                    interaction_dict
+                    interaction_dict, context={ALLOW_UNKNOWN_UNION_VARIANTS: True}
                 )
             except Exception as e:
                 logger.warning("Failed to validate interaction model: %s", e)
@@ -2636,7 +2654,7 @@ def _execute_inference(
             "Gemini Agent inference completed in %.2f seconds.",
             end_time - start_time,
         )
-        return types.EvaluationDataset(
+        evaluation_dataset = types.EvaluationDataset(
             eval_dataset_df=results_df,
             candidate_name=gemini_agent.split("/")[-1],
         )
@@ -2741,7 +2759,12 @@ def _execute_inference(
                 results_df.to_json(full_dest_path, orient="records", lines=True)
                 logger.info("Results saved locally to: %s", full_dest_path)
         except Exception as e:  # pylint: disable=broad-exception-caught
-            logger.error("Failed to save results to %s. Error: %s", full_dest_path, e)
+            logger.error(
+                "Failed to save results to %s. Error: %s",
+                full_dest_path,
+                e,
+                exc_info=True,
+            )
 
     return evaluation_dataset
 
@@ -2868,6 +2891,21 @@ def _resolve_dataset_inputs(
     return processed_eval_dataset, num_response_candidates
 
 
+def _prebuilt_evaluation_run_metric(
+    resolved_metric: types.Metric,
+) -> types.EvaluationRunMetric:
+    """Builds the evaluation run metric for a resolved RubricMetric."""
+    if resolved_metric.name in _evals_constant.SUPPORTED_PREDEFINED_METRICS:
+        metric_config = t.t_metrics([resolved_metric])[0]
+    else:
+        metric_config = {
+            "predefined_metric_spec": {"metric_spec_name": resolved_metric.name}
+        }
+    return types.EvaluationRunMetric(
+        metric=resolved_metric.name, metric_config=metric_config
+    )
+
+
 def _resolve_evaluation_run_metrics(
     metrics: Union[list[types.EvaluationRunMetric], list[types.Metric]], api_client: Any
 ) -> list[types.EvaluationRunMetric]:
@@ -2885,14 +2923,7 @@ def _resolve_evaluation_run_metrics(
                 resolved_metric = metric_instance.resolve(api_client=api_client)
                 if resolved_metric.name:
                     resolved_metrics_list.append(
-                        types.EvaluationRunMetric(
-                            metric=resolved_metric.name,
-                            metric_config=types.UnifiedMetric(
-                                predefined_metric_spec=genai_types.PredefinedMetricSpec(
-                                    metric_spec_name=resolved_metric.name,
-                                )
-                            ),
-                        )
+                        _prebuilt_evaluation_run_metric(resolved_metric)
                     )
             except Exception as e:
                 logger.error(
@@ -2926,14 +2957,7 @@ def _resolve_evaluation_run_metrics(
                     )
                     if resolved_metric.name:
                         resolved_metrics_list.append(
-                            types.EvaluationRunMetric(
-                                metric=resolved_metric.name,
-                                metric_config=types.UnifiedMetric(
-                                    predefined_metric_spec=genai_types.PredefinedMetricSpec(
-                                        metric_spec_name=resolved_metric.name,
-                                    )
-                                ),
-                            )
+                            _prebuilt_evaluation_run_metric(resolved_metric)
                         )
                 else:
                     raise TypeError(
@@ -3002,6 +3026,7 @@ def _execute_evaluation(  # type: ignore[no-untyped-def]
     dest: Optional[str] = None,
     location: Optional[str] = None,
     evaluation_service_qps: Optional[float] = None,
+    allow_cross_region_model: Optional[bool] = None,
     **kwargs,
 ) -> types.EvaluationResult:
     """Evaluates a dataset using the provided metrics.
@@ -3017,6 +3042,8 @@ def _execute_evaluation(  # type: ignore[no-untyped-def]
         evaluation_service_qps: The rate limit (queries per second) for calls
           to the evaluation service. Defaults to 10. Increase this value if
           your project has a higher EvaluateInstances API quota.
+        allow_cross_region_model: Opt-in flag to authorize cross-region
+          routing for judge models.
         **kwargs: Extra arguments to pass to evaluation, such as `agent_info`.
 
     Returns:
@@ -3099,6 +3126,7 @@ def _execute_evaluation(  # type: ignore[no-untyped-def]
     evaluation_result = _evals_metric_handlers.compute_metrics_and_aggregate(
         evaluation_run_config,
         evaluation_service_qps=evaluation_service_qps,
+        allow_cross_region_model=allow_cross_region_model,
     )
     t2 = time.perf_counter()
     logger.info("Evaluation took: %f seconds", t2 - t1)

{google_cloud_agentplatform-2.1.3 → google_cloud_agentplatform-2.4.0}/agentplatform/_genai/_evals_data_converters.py

@@ -15,15 +15,18 @@
 """Dataset converters for evals."""
 
 import copy
+import datetime
 import json
 import logging
-from typing import Any, Optional, Union
+from typing import Any, Optional, TypeVar, Union
 
 from google.genai import _common
 from google.genai import types as genai_types
+from pydantic import alias_generators
 from pydantic import ValidationError
 from typing_extensions import override
 
+from . import _evals_constant
 from . import _evals_utils
 from . import _observability_data_converter
 from . import types
@@ -605,6 +608,308 @@ class _OpenAIDataConverter(_evals_utils.EvalDataConverter):
         return types.EvaluationDataset(eval_cases=eval_cases)
 
 
+_ADK_EVAL_CASE_FIELDS = frozenset(
+    {
+        "eval_id",
+        "conversation",
+        "conversation_scenario",
+        "session_input",
+        "creation_timestamp",
+        "rubrics",
+        "final_session_state",
+    }
+)
+_ADK_EVAL_CASE_KEYS = _ADK_EVAL_CASE_FIELDS | {
+    alias_generators.to_camel(field) for field in _ADK_EVAL_CASE_FIELDS
+}
+_ADK_SESSION_INPUT_FIELDS = ("app_name", "user_id", "state")
+_ADK_DEFAULT_RUBRIC_GROUP = "adk_rubrics"
+_REFERENCE_TRAJECTORY = "reference_trajectory"
+
+_GenaiModel = TypeVar("_GenaiModel", bound=_common.BaseModel)
+
+
+def _get_adk_field(data: dict[str, Any], name: str) -> Any:
+    """Returns an ADK eval set field, which ADK accepts in snake or camel case."""
+    if not isinstance(data, dict):
+        raise TypeError(f"Expected a JSON object for '{name}', got {type(data)}.")
+    if name in data:
+        return data[name]
+    return data.get(alias_generators.to_camel(name))
+
+
+def _validate_adk_genai(model: type[_GenaiModel], data: Any) -> _GenaiModel:
+    """Validates ADK eval set JSON as a genai type.
+
+    Validating in JSON mode decodes base64 bytes fields, such as inline data and
+    thought signatures, the same way ADK does when it reads an eval set file.
+
+    Args:
+        model: The genai type to validate as.
+        data: The parsed JSON value from the eval set.
+
+    Returns:
+        The validated genai object.
+    """
+    return model.model_validate_json(json.dumps(data))
+
+
+def _adk_content(data: dict[str, Any], default_role: str) -> genai_types.Content:
+    """Validates an ADK Content, filling in the role when the file omits it."""
+    content = _validate_adk_genai(genai_types.Content, data)
+    if not content.role:
+        content.role = default_role
+    return content
+
+
+def _adk_user_content(invocation: dict[str, Any]) -> genai_types.Content:
+    """Returns the user content of an ADK invocation."""
+    user_content = _get_adk_field(invocation, "user_content")
+    if user_content is None:
+        raise ValueError("Every invocation must have user_content.")
+    return _adk_content(user_content, "user")
+
+
+def _adk_intermediate_events(
+    intermediate_data: Optional[dict[str, Any]],
+) -> list[tuple[Optional[str], genai_types.Content]]:
+    """Returns (author, content) pairs from ADK IntermediateData or InvocationEvents."""
+    if not intermediate_data:
+        return []
+    invocation_events = _get_adk_field(intermediate_data, "invocation_events")
+    if invocation_events is not None:
+        return [
+            (
+                _get_adk_field(event, "author"),
+                _validate_adk_genai(
+                    genai_types.Content, _get_adk_field(event, "content")
+                ),
+            )
+            for event in invocation_events
+            if _get_adk_field(event, "content")
+        ]
+
+    events: list[tuple[Optional[str], genai_types.Content]] = []
+    tool_uses = _get_adk_field(intermediate_data, "tool_uses")
+    if tool_uses:
+        events.append(
+            (
+                None,
+                genai_types.Content(
+                    role="model",
+                    parts=[
+                        genai_types.Part(
+                            function_call=_validate_adk_genai(
+                                genai_types.FunctionCall, tool_use
+                            )
+                        )
+                        for tool_use in tool_uses
+                    ],
+                ),
+            )
+        )
+    tool_responses = _get_adk_field(intermediate_data, "tool_responses")
+    if tool_responses:
+        events.append(
+            (
+                None,
+                genai_types.Content(
+                    role="user",
+                    parts=[
+                        genai_types.Part(
+                            function_response=_validate_adk_genai(
+                                genai_types.FunctionResponse, tool_response
+                            )
+                        )
+                        for tool_response in tool_responses
+                    ],
+                ),
+            )
+        )
+    for author, parts in (
+        _get_adk_field(intermediate_data, "intermediate_responses") or []
+    ):
+        events.append(
+            (
+                author,
+                genai_types.Content(
+                    role="model",
+                    parts=[
+                        _validate_adk_genai(genai_types.Part, part) for part in parts
+                    ],
+                ),
+            )
+        )
+    return events
+
+
+class AdkEvalSetConverter(_evals_utils.EvalDataConverter):
+    """Converter for ADK eval sets (the `.evalset.json` format).
+
+    Reads the eval set JSON directly, so google-adk does not need to be
+    installed.
+    """
+
+    def _invocation_to_messages(
+        self, invocation: dict[str, Any], turn_index: int
+    ) -> list[types.evals.Message]:
+        """Converts an earlier ADK invocation into conversation history messages."""
+        turn_id = _get_adk_field(invocation, "invocation_id") or str(turn_index)
+        timestamp = _get_adk_field(invocation, "creation_timestamp")
+        messages = [
+            types.evals.Message(
+                turn_id=turn_id,
+                author="user",
+                content=_adk_user_content(invocation),
+                creation_timestamp=(
+                    datetime.datetime.fromtimestamp(timestamp, tz=datetime.timezone.utc)
+                    if timestamp
+                    else None
+                ),
+            )
+        ]
+        messages.extend(
+            types.evals.Message(turn_id=turn_id, author=author, content=content)
+            for author, content in _adk_intermediate_events(
+                _get_adk_field(invocation, "intermediate_data")
+            )
+        )
+        final_response = _get_adk_field(invocation, "final_response")
+        if final_response:
+            messages.append(
+                types.evals.Message(
+                    turn_id=turn_id, content=_adk_content(final_response, "model")
+                )
+            )
+        return messages
+
+    def _convert_rubrics(
+        self, rubrics: list[dict[str, Any]]
+    ) -> dict[str, types.RubricGroup]:
+        """Groups ADK rubrics by their type, which ADK uses to select rubrics."""
+        groups: dict[str, list[types.evals.Rubric]] = {}
+        for rubric in rubrics:
+            rubric_type = _get_adk_field(rubric, "type")
+            rubric_content = _get_adk_field(rubric, "rubric_content") or {}
+            groups.setdefault(rubric_type or _ADK_DEFAULT_RUBRIC_GROUP, []).append(
+                types.evals.Rubric(
+                    rubric_id=_get_adk_field(rubric, "rubric_id"),
+                    type=rubric_type,
+                    content=types.evals.RubricContent(
+                        property=types.evals.RubricContentProperty(
+                            description=_get_adk_field(rubric_content, "text_property")
+                        )
+                    ),
+                )
+            )
+        return {
+            name: types.RubricGroup(rubrics=group) for name, group in groups.items()
+        }
+
+    def _convert_eval_case(self, case: dict[str, Any]) -> types.EvalCase:
+        """Converts one ADK EvalCase into an EvalCase."""
+        if not isinstance(case, dict):
+            raise TypeError(f"Expected an ADK EvalCase object, got {type(case)}.")
+        conversation = _get_adk_field(case, "conversation")
+        scenario = _get_adk_field(case, "conversation_scenario")
+        if (conversation is None) == (scenario is None):
+            raise ValueError(
+                "Exactly one of conversation and conversation_scenario must be set."
+            )
+
+        fields: dict[str, Any] = {"eval_case_id": _get_adk_field(case, "eval_id")}
+        rubrics = list(_get_adk_field(case, "rubrics") or [])
+        if scenario is not None:
+            fields["user_scenario"] = types.evals.UserScenario(
+                starting_prompt=_get_adk_field(scenario, "starting_prompt"),
+                conversation_plan=_get_adk_field(scenario, "conversation_plan"),
+            )
+        else:
+            if not conversation:
+                raise ValueError("conversation must have at least one invocation.")
+            *earlier_invocations, last_invocation = conversation
+            fields["prompt"] = _adk_user_content(last_invocation)
+            history = []
+            for turn_index, invocation in enumerate(earlier_invocations):
+                history.extend(self._invocation_to_messages(invocation, turn_index))
+            if history:
+                fields["conversation_history"] = history
+            final_response = _get_adk_field(last_invocation, "final_response")
+            if final_response:
+                fields["reference"] = types.ResponseCandidate(
+                    response=_adk_content(final_response, "model")
+                )
+            reference_trajectory: list[genai_types.Content] = []
+            for _, content in _adk_intermediate_events(
+                _get_adk_field(last_invocation, "intermediate_data")
+            ):
+                reference_trajectory.extend(
+                    genai_types.Content(
+                        role="model",
+                        parts=[genai_types.Part(function_call=part.function_call)],
+                    )
+                    for part in content.parts or []
+                    if part.function_call
+                )
+            if reference_trajectory:
+                fields[_REFERENCE_TRAJECTORY] = reference_trajectory
+            rubrics.extend(_get_adk_field(last_invocation, "rubrics") or [])
+
+        if rubrics:
+            fields["rubric_groups"] = self._convert_rubrics(rubrics)
+        session_input = _get_adk_field(case, "session_input")
+        if session_input:
+            fields[_evals_constant.SESSION_INPUT] = {
+                name: value
+                for name in _ADK_SESSION_INPUT_FIELDS
+                if (value := _get_adk_field(session_input, name)) is not None
+            }
+
+        reserved_keys = set(fields)
+        for field_name in types.EvalCase.model_fields:
+            reserved_keys.update((field_name, alias_generators.to_camel(field_name)))
+        for key, value in case.items():
+            if key in _ADK_EVAL_CASE_KEYS:
+                continue
+            if key in reserved_keys:
+                logger.warning(
+                    "Skipping ADK eval case field '%s' because it conflicts with"
+                    " an EvalCase field.",
+                    key,
+                )
+                continue
+            fields[key] = value
+        return types.EvalCase(**fields)
+
+    @override
+    def convert(self, raw_data: dict[str, Any]) -> types.EvaluationDataset:
+        """Converts a parsed ADK EvalSet into an EvaluationDataset."""
+        if not isinstance(raw_data, dict):
+            raise ValueError(
+                "Expected an ADK EvalSet JSON object, got"
+                f" {type(raw_data)}. For the legacy list format, load the"
+                " file with google.adk.evaluation.local_eval_sets_manager"
+                ".load_eval_set_from_file() and pass"
+                " eval_set.model_dump(mode='json') instead."
+            )
+        adk_eval_cases = _get_adk_field(raw_data, "eval_cases")
+        if not isinstance(adk_eval_cases, list):
+            raise ValueError("Expected an ADK EvalSet with an 'eval_cases' list.")
+
+        eval_cases = []
+        for i, case in enumerate(adk_eval_cases):
+            try:
+                eval_cases.append(self._convert_eval_case(case))
+            except (TypeError, ValueError) as e:
+                case_id = (
+                    _get_adk_field(case, "eval_id") if isinstance(case, dict) else None
+                )
+                raise ValueError(
+                    f"Failed to convert ADK eval case '{case_id or i}': {e}"
+                ) from e
+        return types.EvaluationDataset(eval_cases=eval_cases)
+
+
 def auto_detect_dataset_schema(
     raw_dataset: list[dict[str, Any]],
 ) -> Union[EvalDatasetSchema, str]:

{google_cloud_agentplatform-2.1.3 → google_cloud_agentplatform-2.4.0}/agentplatform/_genai/_evals_metric_handlers.py

@@ -288,6 +288,7 @@ class MetricHandler(abc.ABC, Generic[T]):
     def __init__(self, module: "evals.Evals", metric: T):
         self.module = module
         self.metric: T = metric
+        self.allow_cross_region_model: Optional[bool] = None
 
     @property
     @abc.abstractmethod
@@ -761,6 +762,7 @@ class LLMMetricHandler(MetricHandler[types.LLMMetric]):
                 lambda: self.module._evaluate_instances(
                     metrics=[self.metric],
                     instance=instance,
+                    allow_cross_region_model=self.allow_cross_region_model,
                 ),
                 self.metric_name,
             )
@@ -981,7 +983,7 @@ class PredefinedMetricHandler(MetricHandler[types.Metric]):
             raise ValueError(
                 f"Metric '{self.metric.name}' is not a supported predefined metric."
             )
-        if (
+        if self.metric.name.startswith("multi_turn") and (
             self.metric.judge_model
             or self.metric.judge_model_generation_config
             or self.metric.judge_model_sampling_count
@@ -989,7 +991,8 @@ class PredefinedMetricHandler(MetricHandler[types.Metric]):
             logger.warning(
                 "Autorater config settings (judge_model, "
                 "judge_model_generation_config, judge_model_sampling_count) "
-                "are ignored for predefined metric '%s'.",
+                "are ignored for multi-turn metric '%s'. Use "
+                "judge_model_step_configs to set its judges.",
                 self.metric.name,
             )
 
@@ -1071,6 +1074,7 @@ class PredefinedMetricHandler(MetricHandler[types.Metric]):
                     metrics=[self.metric],
                     instance=payload.get("instance"),
                     autorater_config=payload.get("autorater_config"),
+                    allow_cross_region_model=self.allow_cross_region_model,
                 ),
                 metric_name,
             )
@@ -1318,6 +1322,7 @@ class RegisteredMetricHandler(MetricHandler[types.Metric]):
                     metric_sources=[metric_source],
                     instance=payload.get("instance"),
                     autorater_config=payload.get("autorater_config"),
+                    allow_cross_region_model=self.allow_cross_region_model,
                 ),
                 metric_name,
             )
@@ -1404,12 +1409,16 @@ MetricHandlerType = TypeVar(
 
 
 def get_handler_for_metric(
-    module: "evals.Evals", metric: types.Metric
+    module: "evals.Evals",
+    metric: types.Metric,
+    allow_cross_region_model: Optional[bool] = None,
 ) -> Union[MetricHandlerType, Any]:
     """Returns a metric handler for the given metric."""
     for condition, handler_class in _METRIC_HANDLER_MAPPING:
         if condition(metric):  # type: ignore[no-untyped-call]
-            return handler_class(module=module, metric=metric)
+            handler = handler_class(module=module, metric=metric)
+            handler.allow_cross_region_model = allow_cross_region_model
+            return handler
     raise ValueError(f"Unsupported metric: {metric.name}")
 
 
@@ -1548,6 +1557,7 @@ def _rate_limited_get_metric_result(
 def compute_metrics_and_aggregate(
     evaluation_run_config: EvaluationRunConfig,
     evaluation_service_qps: Optional[float] = None,
+    allow_cross_region_model: Optional[bool] = None,
 ) -> types.EvaluationResult:
     """Computes metrics and aggregates them for a given evaluation run config.
 
@@ -1556,6 +1566,8 @@ def compute_metrics_and_aggregate(
         evaluation_service_qps: Optional QPS limit for the evaluation service.
             Defaults to _DEFAULT_EVAL_SERVICE_QPS (10). Users with higher
             quotas can increase this value.
+        allow_cross_region_model: Opt-in flag to authorize cross-region
+            routing for judge models.
     """
     metric_handlers = []
     all_futures = []
@@ -1574,7 +1586,11 @@ def compute_metrics_and_aggregate(
 
     for eval_metric in evaluation_run_config.metrics:
         metric_handlers.append(
-            get_handler_for_metric(evaluation_run_config.evals_module, eval_metric)
+            get_handler_for_metric(
+                evaluation_run_config.evals_module,
+                eval_metric,
+                allow_cross_region_model=allow_cross_region_model,
+            )
         )
 
     eval_case_count = len(evaluation_run_config.dataset.eval_cases)

{google_cloud_agentplatform-2.1.3 → google_cloud_agentplatform-2.4.0}/agentplatform/_genai/_evals_metric_loaders.py

@@ -215,8 +215,11 @@ class LazyLoadedPrebuiltMetric:
         if self._resolved_metric:
             return self._resolved_metric
 
+        # The shared cache is keyed by name and version only, so a metric with
+        # overrides (such as judge_model_step_configs) must bypass it.
+        use_cache = not self.metric_kwargs
         cache_key = f"{self.name}@{self.version or 'default'}"
-        if cache_key in LazyLoadedPrebuiltMetric._cache:
+        if use_cache and cache_key in LazyLoadedPrebuiltMetric._cache:
             self._resolved_metric = LazyLoadedPrebuiltMetric._cache[cache_key]
             logger.debug("Metric '%s' found in cache.", cache_key)
             return self._resolved_metric
@@ -225,7 +228,8 @@ class LazyLoadedPrebuiltMetric:
         api_metric = self._resolve_api_predefined()
         if api_metric:
             self._resolved_metric = api_metric
-            LazyLoadedPrebuiltMetric._cache[cache_key] = self._resolved_metric
+            if use_cache:
+                LazyLoadedPrebuiltMetric._cache[cache_key] = self._resolved_metric
             return self._resolved_metric
 
         # Fallback to GCS loading for custom LLM-based Prebuilt Metrics
@@ -234,8 +238,9 @@ class LazyLoadedPrebuiltMetric:
         )
         try:
             gcs_metric = self._fetch_and_parse(api_client)
-            final_cache_key = f"{self.name}@{self.version}"
-            LazyLoadedPrebuiltMetric._cache[final_cache_key] = gcs_metric
+            if use_cache:
+                final_cache_key = f"{self.name}@{self.version}"
+                LazyLoadedPrebuiltMetric._cache[final_cache_key] = gcs_metric
             self._resolved_metric = gcs_metric
             return self._resolved_metric
         except Exception as e:

{google_cloud_agentplatform-2.1.3 → google_cloud_agentplatform-2.4.0}/agentplatform/_genai/_evals_visualization.py

@@ -232,6 +232,9 @@ def _extract_dataset_rows(dataset: types.EvaluationDataset) -> list[dict[str, An
     return processed_rows
 
 
+extract_dataset_rows = _extract_dataset_rows
+
+
 def get_evaluation_html(eval_result_json: str) -> str:
     """Returns a self-contained HTML for single evaluation visualization."""
     payload_b64 = _encode_to_base64(eval_result_json)

{google_cloud_agentplatform-2.1.3 → google_cloud_agentplatform-2.4.0}/agentplatform/_genai/_transformers.py

@@ -64,10 +64,14 @@ def t_metrics(
         elif (
             metric_name and metric_name in _evals_constant.SUPPORTED_PREDEFINED_METRICS
         ):
-            metric_payload_item["predefined_metric_spec"] = {
+            predefined_spec: dict[str, Any] = {
                 "metric_spec_name": metric_name,
                 "metric_spec_parameters": metric.metric_spec_parameters,
             }
+            step_configs = getv(metric, ["judge_model_step_configs"])
+            if step_configs:
+                predefined_spec["step_autorater_configs"] = step_configs
+            metric_payload_item["predefined_metric_spec"] = predefined_spec
         # Custom Code Execution Metric
         elif (
             hasattr(metric, "remote_custom_function") and metric.remote_custom_function