google-adk 0.5.0__py3-none-any.whl → 1.1.0__py3-none-any.whl

google/adk/evaluation/agent_evaluator.py

@@ -13,16 +13,30 @@
 # limitations under the License.
 
 import json
+import logging
 import os
 from os import path
+from typing import Any
 from typing import Dict
 from typing import List
+from typing import Optional
 from typing import Union
+import uuid
 
+from pydantic import ValidationError
+
+from .eval_set import EvalSet
 from .evaluation_generator import EvaluationGenerator
+from .evaluator import EvalStatus
+from .evaluator import EvaluationResult
+from .evaluator import Evaluator
+from .local_eval_sets_manager import convert_eval_set_to_pydanctic_schema
 from .response_evaluator import ResponseEvaluator
 from .trajectory_evaluator import TrajectoryEvaluator
 
+logger = logging.getLogger("google_adk." + __name__)
+
+
 # Constants for default runs and evaluation criteria
 NUM_RUNS = 2
 TOOL_TRAJECTORY_SCORE_KEY = "tool_trajectory_avg_score"
@@ -76,12 +90,67 @@ class AgentEvaluator:
     return DEFAULT_CRITERIA
 
   @staticmethod
-  async def evaluate(
-      agent_module,
-      eval_dataset_file_path_or_dir,
+  async def evaluate_eval_set(
+      agent_module: str,
+      eval_set: EvalSet,
+      criteria: dict[str, float],
       num_runs=NUM_RUNS,
       agent_name=None,
-      initial_session_file=None,
+  ):
+    """Evaluates an agent using the given EvalSet.
+
+    Args:
+      agent_module: The path to python module that contains the definition of
+        the agent. There is convention in place here, where the code is going to
+        look for 'root_agent' in the loaded module.
+      eval_set: The eval set.
+      criteria: Evauation criterias, a dictionary of metric names to their
+        respective thresholds.
+      num_runs: Number of times all entries in the eval dataset should be
+        assessed.
+      agent_name: The name of the agent.
+    """
+    eval_case_responses_list = await EvaluationGenerator.generate_responses(
+        eval_set=eval_set,
+        agent_module_path=agent_module,
+        repeat_num=num_runs,
+        agent_name=agent_name,
+    )
+
+    for eval_case_responses in eval_case_responses_list:
+      actual_invocations = [
+          invocation
+          for invocations in eval_case_responses.responses
+          for invocation in invocations
+      ]
+      expected_invocations = (
+          eval_case_responses.eval_case.conversation * num_runs
+      )
+
+      for metric_name, threshold in criteria.items():
+        metric_evaluator = AgentEvaluator._get_metric_evaluator(
+            metric_name=metric_name, threshold=threshold
+        )
+
+        evaluation_result: EvaluationResult = (
+            metric_evaluator.evaluate_invocations(
+                actual_invocations=actual_invocations,
+                expected_invocations=expected_invocations,
+            )
+        )
+
+        assert evaluation_result.overall_eval_status == EvalStatus.PASSED, (
+            f"{metric_name} for {agent_module} Failed. Expected {threshold},"
+            f" but got {evaluation_result.overall_score}."
+        )
+
+  @staticmethod
+  async def evaluate(
+      agent_module: str,
+      eval_dataset_file_path_or_dir: str,
+      num_runs: int = NUM_RUNS,
+      agent_name: Optional[str] = None,
+      initial_session_file: Optional[str] = None,
   ):
     """Evaluates an Agent given eval data.
 
@@ -109,35 +178,102 @@ class AgentEvaluator:
     else:
       test_files = [eval_dataset_file_path_or_dir]
 
-    initial_session_state = {}
-    if initial_session_file:
-      with open(initial_session_file, "r") as f:
-        initial_session_state = json.loads(f.read())["state"]
+    initial_session = AgentEvaluator._get_initial_session(initial_session_file)
 
     for test_file in test_files:
-      dataset = AgentEvaluator._load_dataset(test_file)[0]
       criteria = AgentEvaluator.find_config_for_test_file(test_file)
+      eval_set = AgentEvaluator._load_eval_set_from_file(
+          test_file, criteria, initial_session
+      )
 
-      AgentEvaluator._validate_input([dataset], criteria)
-
-      evaluation_response = await AgentEvaluator._generate_responses(
-          agent_module,
-          [dataset],
-          num_runs,
+      await AgentEvaluator.evaluate_eval_set(
+          agent_module=agent_module,
+          eval_set=eval_set,
+          criteria=criteria,
+          num_runs=num_runs,
           agent_name=agent_name,
-          initial_session={"state": initial_session_state},
       )
 
-      if AgentEvaluator._response_evaluation_required(criteria, [dataset]):
-        AgentEvaluator._evaluate_response_scores(
-            agent_module, evaluation_response, criteria
-        )
+  @staticmethod
+  def migrate_eval_data_to_new_schema(
+      old_eval_data_file: str,
+      new_eval_data_file: str,
+      initial_session_file: Optional[str] = None,
+  ):
+    """A utility for migrating eval data to new schema backed by EvalSet."""
+    if not old_eval_data_file or not new_eval_data_file:
+      raise ValueError(
+          "One of old_eval_data_file or new_eval_data_file is empty."
+      )
+
+    criteria = AgentEvaluator.find_config_for_test_file(old_eval_data_file)
+    initial_session = AgentEvaluator._get_initial_session(initial_session_file)
+
+    eval_set = AgentEvaluator._get_eval_set_from_old_format(
+        old_eval_data_file, criteria, initial_session
+    )
 
-      if AgentEvaluator._trajectory_evaluation_required(criteria, [dataset]):
-        AgentEvaluator._evaluate_tool_trajectory(
-            agent_module, evaluation_response, criteria
+    with open(new_eval_data_file, "w") as f:
+      f.write(eval_set.model_dump_json(indent=2))
+
+  @staticmethod
+  def _load_eval_set_from_file(
+      eval_set_file: str,
+      criteria: dict[str, float],
+      initial_session: dict[str, Any],
+  ) -> EvalSet:
+    """Loads an EvalSet from the given file."""
+    if os.path.isfile(eval_set_file):
+      with open(eval_set_file, "r", encoding="utf-8") as f:
+        content = f.read()
+
+      try:
+        eval_set = EvalSet.model_validate_json(content)
+        assert len(initial_session) == 0, (
+            "Intial session should be specified as a part of EvalSet file."
+            " Explicit initial session is only needed, when specifying data in"
+            " the older schema."
+        )
+        return eval_set
+      except ValidationError:
+        # We assume that the eval data was specified in the old format
+        logger.warning(
+            f"Contents of {eval_set_file} appear to be in older format.To avoid"
+            " this warning, please update your test files to contain data in"
+            " EvalSet schema. You can use `migrate_eval_data_to_new_schema`"
+            " for migrating your old test files."
         )
 
+    # If we are here, the data must be specified in the older format.
+    return AgentEvaluator._get_eval_set_from_old_format(
+        eval_set_file, criteria, initial_session
+    )
+
+  @staticmethod
+  def _get_eval_set_from_old_format(
+      eval_set_file: str,
+      criteria: dict[str, float],
+      initial_session: dict[str, Any],
+  ) -> EvalSet:
+    data = AgentEvaluator._load_dataset(eval_set_file)[0]
+    AgentEvaluator._validate_input([data], criteria)
+    eval_data = {
+        "name": eval_set_file,
+        "data": data,
+        "initial_session": initial_session,
+    }
+    return convert_eval_set_to_pydanctic_schema(
+        eval_set_id=str(uuid.uuid4()), eval_set_in_json_format=[eval_data]
+    )
+
+  @staticmethod
+  def _get_initial_session(initial_session_file: Optional[str] = None):
+    initial_session = {}
+    if initial_session_file:
+      with open(initial_session_file, "r") as f:
+        initial_session = json.loads(f.read())
+    return initial_session
+
   @staticmethod
   def _load_dataset(
       input_data: Union[str, List[str], List[Dict], List[List[Dict]]],
@@ -221,109 +357,13 @@ class AgentEvaluator:
         )
 
   @staticmethod
-  def _get_infer_criteria(eval_dataset):
-    """Infers evaluation criteria based on the provided dataset.
-
-    Args:
-        eval_dataset (list): A list of evaluation samples.
-
-    Returns:
-        dict: Inferred evaluation criteria based on dataset fields.
-    """
-    inferred_criteria = {}
-    sample = eval_dataset[0][0]
-
-    if QUERY_COLUMN in sample and EXPECTED_TOOL_USE_COLUMN in sample:
-      inferred_criteria[TOOL_TRAJECTORY_SCORE_KEY] = DEFAULT_CRITERIA[
-          TOOL_TRAJECTORY_SCORE_KEY
-      ]
-
-    if QUERY_COLUMN in sample and REFERENCE_COLUMN in sample:
-      inferred_criteria[RESPONSE_MATCH_SCORE_KEY] = DEFAULT_CRITERIA[
-          RESPONSE_MATCH_SCORE_KEY
-      ]
-
-    return inferred_criteria
-
-  @staticmethod
-  async def _generate_responses(
-      agent_module, eval_dataset, num_runs, agent_name=None, initial_session={}
-  ):
-    """Generates evaluation responses by running the agent module multiple times."""
-    return EvaluationGenerator.generate_responses(
-        eval_dataset,
-        agent_module,
-        repeat_num=num_runs,
-        agent_name=agent_name,
-        initial_session=initial_session,
-    )
-
-  @staticmethod
-  def _generate_responses_from_session(eval_dataset, session_path):
-    """Generates evaluation responses by running the agent module multiple times."""
-    return EvaluationGenerator.generate_responses_from_session(
-        session_path, eval_dataset
-    )
-
-  @staticmethod
-  def _response_evaluation_required(criteria, eval_dataset):
-    """Checks if response evaluation are needed."""
-    return REFERENCE_COLUMN in eval_dataset[0][0] and any(
-        key in criteria
-        for key in [RESPONSE_EVALUATION_SCORE_KEY, RESPONSE_MATCH_SCORE_KEY]
-    )
-
-  @staticmethod
-  def _trajectory_evaluation_required(evaluation_criteria, eval_dataset):
-    """Checks if response evaluation are needed."""
-    return (
-        EXPECTED_TOOL_USE_COLUMN in eval_dataset[0][0]
-        and TOOL_TRAJECTORY_SCORE_KEY in evaluation_criteria
-    )
-
-  @staticmethod
-  def _evaluate_response_scores(agent_module, evaluation_response, criteria):
-    """Evaluates response scores and raises an assertion error if they don't meet the criteria."""
-    metrics = ResponseEvaluator.evaluate(
-        evaluation_response, criteria, print_detailed_results=True
-    )
-
-    AgentEvaluator._assert_score(
-        metrics,
-        "coherence/mean",
-        criteria.get(RESPONSE_EVALUATION_SCORE_KEY),
-        "Average response evaluation score",
-        agent_module,
-    )
-
-    AgentEvaluator._assert_score(
-        metrics,
-        "rouge_1/mean",
-        criteria.get(RESPONSE_MATCH_SCORE_KEY),
-        "Average response match score",
-        agent_module,
-    )
-
-  @staticmethod
-  def _evaluate_tool_trajectory(agent_module, evaluation_response, criteria):
-    """Evaluates tool trajectory scores and raises an assertion error if they don't meet the criteria."""
-    score = TrajectoryEvaluator.evaluate(
-        evaluation_response, print_detailed_results=True
-    )
-    AgentEvaluator._assert_score(
-        {TOOL_TRAJECTORY_SCORE_KEY: score},
-        TOOL_TRAJECTORY_SCORE_KEY,
-        criteria[TOOL_TRAJECTORY_SCORE_KEY],
-        "Average tool trajectory evaluation score",
-        agent_module,
-    )
+  def _get_metric_evaluator(metric_name: str, threshold: float) -> Evaluator:
+    if metric_name == TOOL_TRAJECTORY_SCORE_KEY:
+      return TrajectoryEvaluator(threshold=threshold)
+    elif (
+        metric_name == RESPONSE_MATCH_SCORE_KEY
+        or metric_name == RESPONSE_EVALUATION_SCORE_KEY
+    ):
+      return ResponseEvaluator(threshold=threshold, metric_name=metric_name)
 
-  @staticmethod
-  def _assert_score(metrics, metric_key, threshold, description, agent_module):
-    """Asserts that a metric meets the specified threshold."""
-    if metric_key in metrics:
-      actual_score = metrics[metric_key]
-      assert actual_score >= threshold, (
-          f"{description} for {agent_module} is lower than expected. "
-          f"Expected >= {threshold}, but got {actual_score}."
-      )
+    raise ValueError(f"Unsupported eval metric: {metric_name}")

google/adk/evaluation/eval_case.py

@@ -0,0 +1,104 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from typing import Any
+from typing import Optional
+from typing import Tuple
+
+from google.genai import types as genai_types
+from pydantic import alias_generators
+from pydantic import BaseModel
+from pydantic import ConfigDict
+from pydantic import Field
+
+
+class EvalBaseModel(BaseModel):
+  model_config = ConfigDict(
+      alias_generator=alias_generators.to_camel,
+      populate_by_name=True,
+  )
+
+
+class IntermediateData(EvalBaseModel):
+  """Container for intermediate data that an agent would generate as it responds with a final answer."""
+
+  tool_uses: list[genai_types.FunctionCall] = []
+  """Tool use trajectory in chronological order."""
+
+  intermediate_responses: list[Tuple[str, list[genai_types.Part]]] = []
+  """Intermediate responses generated by sub-agents to convey progress or status
+  in a multi-agent system, distinct from the final response.
+
+  This is expressed as a Tuple of:
+    - Author: Usually the sub-agent name that generated the intermediate
+      response.
+
+    - A list of Parts that comprise of the response.
+  """
+
+
+class Invocation(EvalBaseModel):
+  """Represents a single invocation."""
+
+  invocation_id: str = ''
+  """Unique identifier for the invocation."""
+
+  user_content: genai_types.Content
+  """Content provided by the user in this invocation."""
+
+  final_response: Optional[genai_types.Content] = None
+  """Final response from the agent."""
+
+  intermediate_data: Optional[IntermediateData] = None
+  """Intermediate steps generated as a part of Agent execution.
+
+  For a multi-agent system, it is also helpful to inspect the route that
+  the agent took to generate final response.
+  """
+
+  creation_timestamp: float = 0.0
+  """Timestamp for the current invocation, primarily intended for debugging purposes."""
+
+
+class SessionInput(EvalBaseModel):
+  """Values that help initialize a Session."""
+
+  app_name: str
+  """The name of the app."""
+
+  user_id: str
+  """The user id."""
+
+  state: dict[str, Any] = Field(default_factory=dict)
+  """The state of the session."""
+
+
+class EvalCase(EvalBaseModel):
+  """An eval case."""
+
+  eval_id: str
+  """Unique identifier for the evaluation case."""
+
+  conversation: list[Invocation]
+  """A conversation between the user and the Agent. The conversation can have any number of invocations."""
+
+  session_input: Optional[SessionInput] = None
+  """Session input that will be passed on to the Agent during eval.
+     It is common for Agents state to be initialized to some initial/default value,
+     for example, your agent may need to know today's date.
+  """
+
+  creation_timestamp: float = 0.0
+  """The time at which this eval case was created."""

google/adk/evaluation/eval_metrics.py

@@ -0,0 +1,74 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from typing import Optional
+
+from pydantic import alias_generators
+from pydantic import BaseModel
+from pydantic import ConfigDict
+
+from .eval_case import Invocation
+from .evaluator import EvalStatus
+
+
+class EvalMetric(BaseModel):
+  """A metric used to evaluate a particular aspect of an eval case."""
+
+  model_config = ConfigDict(
+      alias_generator=alias_generators.to_camel,
+      populate_by_name=True,
+  )
+
+  model_config = ConfigDict(
+      alias_generator=alias_generators.to_camel,
+      populate_by_name=True,
+  )
+
+  metric_name: str
+  """The name of the metric."""
+
+  threshold: float
+  """A threshold value. Each metric decides how to interpret this threshold."""
+
+
+class EvalMetricResult(EvalMetric):
+  """The actual computed score/value of a particular EvalMetric."""
+
+  model_config = ConfigDict(
+      alias_generator=alias_generators.to_camel,
+      populate_by_name=True,
+  )
+
+  score: Optional[float] = None
+  eval_status: EvalStatus
+
+
+class EvalMetricResultPerInvocation(BaseModel):
+  """Eval metric results per invocation."""
+
+  model_config = ConfigDict(
+      alias_generator=alias_generators.to_camel,
+      populate_by_name=True,
+  )
+
+  actual_invocation: Invocation
+  """The actual invocation, usually obtained by inferencing the agent."""
+
+  expected_invocation: Invocation
+  """The expected invocation, usually the reference or golden invocation."""
+
+  eval_metric_results: list[EvalMetricResult] = []
+  """Eval resutls for each applicable metric."""

google/adk/evaluation/eval_result.py

@@ -0,0 +1,86 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from typing import Optional
+
+from pydantic import alias_generators
+from pydantic import BaseModel
+from pydantic import ConfigDict
+from pydantic import Field
+
+from ..sessions.session import Session
+from .eval_metrics import EvalMetric
+from .eval_metrics import EvalMetricResult
+from .eval_metrics import EvalMetricResultPerInvocation
+from .evaluator import EvalStatus
+
+
+class EvalCaseResult(BaseModel):
+  """Case level evaluation results."""
+
+  model_config = ConfigDict(
+      alias_generator=alias_generators.to_camel,
+      populate_by_name=True,
+  )
+
+  eval_set_file: str = Field(
+      deprecated=True,
+      description="This field is deprecated, use eval_set_id instead.",
+  )
+  eval_set_id: str = ""
+  """The eval set id."""
+
+  eval_id: str = ""
+  """The eval case id."""
+
+  final_eval_status: EvalStatus
+  """Final eval status for this eval case."""
+
+  eval_metric_results: list[tuple[EvalMetric, EvalMetricResult]] = Field(
+      deprecated=True,
+      description=(
+          "This field is deprecated, use overall_eval_metric_results instead."
+      ),
+  )
+
+  overall_eval_metric_results: list[EvalMetricResult]
+  """Overall result for each metric for the entire eval case."""
+
+  eval_metric_result_per_invocation: list[EvalMetricResultPerInvocation]
+  """Result for each metric on a per invocation basis."""
+
+  session_id: str
+  """Session id of the session generated as result of inferencing/scraping stage of the eval."""
+
+  session_details: Optional[Session] = None
+  """Session generated as result of inferencing/scraping stage of the eval."""
+
+  user_id: Optional[str] = None
+  """User id used during inferencing/scraping stage of the eval."""
+
+
+class EvalSetResult(BaseModel):
+  """Eval set level evaluation results."""
+
+  model_config = ConfigDict(
+      alias_generator=alias_generators.to_camel,
+      populate_by_name=True,
+  )
+  eval_set_result_id: str
+  eval_set_result_name: str
+  eval_set_id: str
+  eval_case_results: list[EvalCaseResult] = Field(default_factory=list)
+  creation_timestamp: float = 0.0

google/adk/evaluation/eval_set.py

@@ -0,0 +1,39 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Optional
+
+from pydantic import BaseModel
+
+from .eval_case import EvalCase
+
+
+class EvalSet(BaseModel):
+  """A set of eval cases."""
+
+  eval_set_id: str
+  """Unique identifier for the eval set."""
+
+  name: Optional[str] = None
+  """Name of the dataset."""
+
+  description: Optional[str] = None
+  """Description of the dataset."""
+
+  eval_cases: list[EvalCase]
+  """List of eval cases in the dataset. Each case represents a single
+  interaction to be evaluated."""
+
+  creation_timestamp: float = 0.0
+  """The time at which this eval set was created."""

google/adk/evaluation/eval_set_results_manager.py

@@ -0,0 +1,47 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from abc import ABC
+from abc import abstractmethod
+
+from .eval_result import EvalCaseResult
+from .eval_result import EvalSetResult
+
+
+class EvalSetResultsManager(ABC):
+  """An interface to manage Eval Set Results."""
+
+  @abstractmethod
+  def save_eval_set_result(
+      self,
+      app_name: str,
+      eval_set_id: str,
+      eval_case_results: list[EvalCaseResult],
+  ) -> None:
+    """Creates and saves a new EvalSetResult given eval_case_results."""
+    raise NotImplementedError()
+
+  @abstractmethod
+  def get_eval_set_result(
+      self, app_name: str, eval_set_result_id: str
+  ) -> EvalSetResult:
+    """Returns an EvalSetResult identified by app_name and eval_set_result_id."""
+    raise NotImplementedError()
+
+  @abstractmethod
+  def list_eval_set_results(self, app_name: str) -> list[str]:
+    """Returns the eval result ids that belong to the given app_name."""
+    raise NotImplementedError()