google-adk 0.5.0__py3-none-any.whl → 1.0.0__py3-none-any.whl

google/adk/code_executors/built_in_code_executor.py

@@ -0,0 +1,52 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from google.genai import types
+from typing_extensions import override
+
+from ..agents.invocation_context import InvocationContext
+from ..models import LlmRequest
+from .base_code_executor import BaseCodeExecutor
+from .code_execution_utils import CodeExecutionInput
+from .code_execution_utils import CodeExecutionResult
+
+
+class BuiltInCodeExecutor(BaseCodeExecutor):
+  """A code executor that uses the Model's built-in code executor.
+
+  Currently only supports Gemini 2.0+ models, but will be expanded to
+  other models.
+  """
+
+  @override
+  def execute_code(
+      self,
+      invocation_context: InvocationContext,
+      code_execution_input: CodeExecutionInput,
+  ) -> CodeExecutionResult:
+    pass
+
+  def process_llm_request(self, llm_request: LlmRequest) -> None:
+    """Pre-process the LLM request for Gemini 2.0+ models to use the code execution tool."""
+    if llm_request.model and llm_request.model.startswith("gemini-2"):
+      llm_request.config = llm_request.config or types.GenerateContentConfig()
+      llm_request.config.tools = llm_request.config.tools or []
+      llm_request.config.tools.append(
+          types.Tool(code_execution=types.ToolCodeExecution())
+      )
+      return
+    raise ValueError(
+        "Gemini code execution tool is not supported for model"
+        f" {llm_request.model}"
+    )

google/adk/evaluation/__init__.py

@@ -14,7 +14,7 @@
 
 import logging
 
-logger = logging.getLogger(__name__)
+logger = logging.getLogger('google_adk.' + __name__)
 
 __all__ = []
 

google/adk/evaluation/agent_evaluator.py

@@ -13,16 +13,30 @@
 # limitations under the License.
 
 import json
+import logging
 import os
 from os import path
+from typing import Any
 from typing import Dict
 from typing import List
+from typing import Optional
 from typing import Union
+import uuid
 
+from pydantic import ValidationError
+
+from .eval_set import EvalSet
 from .evaluation_generator import EvaluationGenerator
+from .evaluator import EvalStatus
+from .evaluator import EvaluationResult
+from .evaluator import Evaluator
+from .local_eval_sets_manager import convert_eval_set_to_pydanctic_schema
 from .response_evaluator import ResponseEvaluator
 from .trajectory_evaluator import TrajectoryEvaluator
 
+logger = logging.getLogger("google_adk." + __name__)
+
+
 # Constants for default runs and evaluation criteria
 NUM_RUNS = 2
 TOOL_TRAJECTORY_SCORE_KEY = "tool_trajectory_avg_score"
@@ -76,12 +90,67 @@ class AgentEvaluator:
     return DEFAULT_CRITERIA
 
   @staticmethod
-  async def evaluate(
-      agent_module,
-      eval_dataset_file_path_or_dir,
+  async def evaluate_eval_set(
+      agent_module: str,
+      eval_set: EvalSet,
+      criteria: dict[str, float],
       num_runs=NUM_RUNS,
       agent_name=None,
-      initial_session_file=None,
+  ):
+    """Evaluates an agent using the given EvalSet.
+
+    Args:
+      agent_module: The path to python module that contains the definition of
+        the agent. There is convention in place here, where the code is going to
+        look for 'root_agent' in the loaded module.
+      eval_set: The eval set.
+      criteria: Evauation criterias, a dictionary of metric names to their
+        respective thresholds.
+      num_runs: Number of times all entries in the eval dataset should be
+        assessed.
+      agent_name: The name of the agent.
+    """
+    eval_case_responses_list = await EvaluationGenerator.generate_responses(
+        eval_set=eval_set,
+        agent_module_path=agent_module,
+        repeat_num=num_runs,
+        agent_name=agent_name,
+    )
+
+    for eval_case_responses in eval_case_responses_list:
+      actual_invocations = [
+          invocation
+          for invocations in eval_case_responses.responses
+          for invocation in invocations
+      ]
+      expected_invocations = (
+          eval_case_responses.eval_case.conversation * num_runs
+      )
+
+      for metric_name, threshold in criteria.items():
+        metric_evaluator = AgentEvaluator._get_metric_evaluator(
+            metric_name=metric_name, threshold=threshold
+        )
+
+        evaluation_result: EvaluationResult = (
+            metric_evaluator.evaluate_invocations(
+                actual_invocations=actual_invocations,
+                expected_invocations=expected_invocations,
+            )
+        )
+
+        assert evaluation_result.overall_eval_status == EvalStatus.PASSED, (
+            f"{metric_name} for {agent_module} Failed. Expected {threshold},"
+            f" but got {evaluation_result.overall_score}."
+        )
+
+  @staticmethod
+  async def evaluate(
+      agent_module: str,
+      eval_dataset_file_path_or_dir: str,
+      num_runs: int = NUM_RUNS,
+      agent_name: Optional[str] = None,
+      initial_session_file: Optional[str] = None,
   ):
     """Evaluates an Agent given eval data.
 
@@ -109,35 +178,102 @@ class AgentEvaluator:
     else:
       test_files = [eval_dataset_file_path_or_dir]
 
-    initial_session_state = {}
-    if initial_session_file:
-      with open(initial_session_file, "r") as f:
-        initial_session_state = json.loads(f.read())["state"]
+    initial_session = AgentEvaluator._get_initial_session(initial_session_file)
 
     for test_file in test_files:
-      dataset = AgentEvaluator._load_dataset(test_file)[0]
       criteria = AgentEvaluator.find_config_for_test_file(test_file)
+      eval_set = AgentEvaluator._load_eval_set_from_file(
+          test_file, criteria, initial_session
+      )
 
-      AgentEvaluator._validate_input([dataset], criteria)
-
-      evaluation_response = await AgentEvaluator._generate_responses(
-          agent_module,
-          [dataset],
-          num_runs,
+      await AgentEvaluator.evaluate_eval_set(
+          agent_module=agent_module,
+          eval_set=eval_set,
+          criteria=criteria,
+          num_runs=num_runs,
           agent_name=agent_name,
-          initial_session={"state": initial_session_state},
       )
 
-      if AgentEvaluator._response_evaluation_required(criteria, [dataset]):
-        AgentEvaluator._evaluate_response_scores(
-            agent_module, evaluation_response, criteria
-        )
+  @staticmethod
+  def migrate_eval_data_to_new_schema(
+      old_eval_data_file: str,
+      new_eval_data_file: str,
+      initial_session_file: Optional[str] = None,
+  ):
+    """A utility for migrating eval data to new schema backed by EvalSet."""
+    if not old_eval_data_file or not new_eval_data_file:
+      raise ValueError(
+          "One of old_eval_data_file or new_eval_data_file is empty."
+      )
+
+    criteria = AgentEvaluator.find_config_for_test_file(old_eval_data_file)
+    initial_session = AgentEvaluator._get_initial_session(initial_session_file)
+
+    eval_set = AgentEvaluator._get_eval_set_from_old_format(
+        old_eval_data_file, criteria, initial_session
+    )
 
-      if AgentEvaluator._trajectory_evaluation_required(criteria, [dataset]):
-        AgentEvaluator._evaluate_tool_trajectory(
-            agent_module, evaluation_response, criteria
+    with open(new_eval_data_file, "w") as f:
+      f.write(eval_set.model_dump_json(indent=2))
+
+  @staticmethod
+  def _load_eval_set_from_file(
+      eval_set_file: str,
+      criteria: dict[str, float],
+      initial_session: dict[str, Any],
+  ) -> EvalSet:
+    """Loads an EvalSet from the given file."""
+    if os.path.isfile(eval_set_file):
+      with open(eval_set_file, "r", encoding="utf-8") as f:
+        content = f.read()
+
+      try:
+        eval_set = EvalSet.model_validate_json(content)
+        assert len(initial_session) == 0, (
+            "Intial session should be specified as a part of EvalSet file."
+            " Explicit initial session is only needed, when specifying data in"
+            " the older schema."
+        )
+        return eval_set
+      except ValidationError:
+        # We assume that the eval data was specified in the old format
+        logger.warning(
+            f"Contents of {eval_set_file} appear to be in older format.To avoid"
+            " this warning, please update your test files to contain data in"
+            " EvalSet schema. You can use `migrate_eval_data_to_new_schema`"
+            " for migrating your old test files."
         )
 
+    # If we are here, the data must be specified in the older format.
+    return AgentEvaluator._get_eval_set_from_old_format(
+        eval_set_file, criteria, initial_session
+    )
+
+  @staticmethod
+  def _get_eval_set_from_old_format(
+      eval_set_file: str,
+      criteria: dict[str, float],
+      initial_session: dict[str, Any],
+  ) -> EvalSet:
+    data = AgentEvaluator._load_dataset(eval_set_file)[0]
+    AgentEvaluator._validate_input([data], criteria)
+    eval_data = {
+        "name": eval_set_file,
+        "data": data,
+        "initial_session": initial_session,
+    }
+    return convert_eval_set_to_pydanctic_schema(
+        eval_set_id=str(uuid.uuid4()), eval_set_in_json_format=[eval_data]
+    )
+
+  @staticmethod
+  def _get_initial_session(initial_session_file: Optional[str] = None):
+    initial_session = {}
+    if initial_session_file:
+      with open(initial_session_file, "r") as f:
+        initial_session = json.loads(f.read())
+    return initial_session
+
   @staticmethod
   def _load_dataset(
       input_data: Union[str, List[str], List[Dict], List[List[Dict]]],
@@ -221,109 +357,13 @@ class AgentEvaluator:
         )
 
   @staticmethod
-  def _get_infer_criteria(eval_dataset):
-    """Infers evaluation criteria based on the provided dataset.
-
-    Args:
-        eval_dataset (list): A list of evaluation samples.
-
-    Returns:
-        dict: Inferred evaluation criteria based on dataset fields.
-    """
-    inferred_criteria = {}
-    sample = eval_dataset[0][0]
-
-    if QUERY_COLUMN in sample and EXPECTED_TOOL_USE_COLUMN in sample:
-      inferred_criteria[TOOL_TRAJECTORY_SCORE_KEY] = DEFAULT_CRITERIA[
-          TOOL_TRAJECTORY_SCORE_KEY
-      ]
-
-    if QUERY_COLUMN in sample and REFERENCE_COLUMN in sample:
-      inferred_criteria[RESPONSE_MATCH_SCORE_KEY] = DEFAULT_CRITERIA[
-          RESPONSE_MATCH_SCORE_KEY
-      ]
-
-    return inferred_criteria
-
-  @staticmethod
-  async def _generate_responses(
-      agent_module, eval_dataset, num_runs, agent_name=None, initial_session={}
-  ):
-    """Generates evaluation responses by running the agent module multiple times."""
-    return EvaluationGenerator.generate_responses(
-        eval_dataset,
-        agent_module,
-        repeat_num=num_runs,
-        agent_name=agent_name,
-        initial_session=initial_session,
-    )
-
-  @staticmethod
-  def _generate_responses_from_session(eval_dataset, session_path):
-    """Generates evaluation responses by running the agent module multiple times."""
-    return EvaluationGenerator.generate_responses_from_session(
-        session_path, eval_dataset
-    )
-
-  @staticmethod
-  def _response_evaluation_required(criteria, eval_dataset):
-    """Checks if response evaluation are needed."""
-    return REFERENCE_COLUMN in eval_dataset[0][0] and any(
-        key in criteria
-        for key in [RESPONSE_EVALUATION_SCORE_KEY, RESPONSE_MATCH_SCORE_KEY]
-    )
-
-  @staticmethod
-  def _trajectory_evaluation_required(evaluation_criteria, eval_dataset):
-    """Checks if response evaluation are needed."""
-    return (
-        EXPECTED_TOOL_USE_COLUMN in eval_dataset[0][0]
-        and TOOL_TRAJECTORY_SCORE_KEY in evaluation_criteria
-    )
-
-  @staticmethod
-  def _evaluate_response_scores(agent_module, evaluation_response, criteria):
-    """Evaluates response scores and raises an assertion error if they don't meet the criteria."""
-    metrics = ResponseEvaluator.evaluate(
-        evaluation_response, criteria, print_detailed_results=True
-    )
-
-    AgentEvaluator._assert_score(
-        metrics,
-        "coherence/mean",
-        criteria.get(RESPONSE_EVALUATION_SCORE_KEY),
-        "Average response evaluation score",
-        agent_module,
-    )
-
-    AgentEvaluator._assert_score(
-        metrics,
-        "rouge_1/mean",
-        criteria.get(RESPONSE_MATCH_SCORE_KEY),
-        "Average response match score",
-        agent_module,
-    )
-
-  @staticmethod
-  def _evaluate_tool_trajectory(agent_module, evaluation_response, criteria):
-    """Evaluates tool trajectory scores and raises an assertion error if they don't meet the criteria."""
-    score = TrajectoryEvaluator.evaluate(
-        evaluation_response, print_detailed_results=True
-    )
-    AgentEvaluator._assert_score(
-        {TOOL_TRAJECTORY_SCORE_KEY: score},
-        TOOL_TRAJECTORY_SCORE_KEY,
-        criteria[TOOL_TRAJECTORY_SCORE_KEY],
-        "Average tool trajectory evaluation score",
-        agent_module,
-    )
+  def _get_metric_evaluator(metric_name: str, threshold: float) -> Evaluator:
+    if metric_name == TOOL_TRAJECTORY_SCORE_KEY:
+      return TrajectoryEvaluator(threshold=threshold)
+    elif (
+        metric_name == RESPONSE_MATCH_SCORE_KEY
+        or metric_name == RESPONSE_EVALUATION_SCORE_KEY
+    ):
+      return ResponseEvaluator(threshold=threshold, metric_name=metric_name)
 
-  @staticmethod
-  def _assert_score(metrics, metric_key, threshold, description, agent_module):
-    """Asserts that a metric meets the specified threshold."""
-    if metric_key in metrics:
-      actual_score = metrics[metric_key]
-      assert actual_score >= threshold, (
-          f"{description} for {agent_module} is lower than expected. "
-          f"Expected >= {threshold}, but got {actual_score}."
-      )
+    raise ValueError(f"Unsupported eval metric: {metric_name}")

google/adk/evaluation/eval_case.py

@@ -0,0 +1,102 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from typing import Any, Optional, Tuple
+
+from google.genai import types as genai_types
+from pydantic import alias_generators
+from pydantic import BaseModel
+from pydantic import ConfigDict
+from pydantic import Field
+
+
+class EvalBaseModel(BaseModel):
+  model_config = ConfigDict(
+      alias_generator=alias_generators.to_camel,
+      populate_by_name=True,
+  )
+
+
+class IntermediateData(EvalBaseModel):
+  """Container for intermediate data that an agent would generate as it responds with a final answer."""
+
+  tool_uses: list[genai_types.FunctionCall] = []
+  """Tool use trajectory in chronological order."""
+
+  intermediate_responses: list[Tuple[str, list[genai_types.Part]]] = []
+  """Intermediate responses generated by sub-agents to convey progress or status
+  in a multi-agent system, distinct from the final response.
+
+  This is expressed as a Tuple of:
+    - Author: Usually the sub-agent name that generated the intermediate
+      response.
+
+    - A list of Parts that comprise of the response.
+  """
+
+
+class Invocation(EvalBaseModel):
+  """Represents a single invocation."""
+
+  invocation_id: str = ''
+  """Unique identifier for the invocation."""
+
+  user_content: genai_types.Content
+  """Content provided by the user in this invocation."""
+
+  final_response: Optional[genai_types.Content] = None
+  """Final response from the agent."""
+
+  intermediate_data: Optional[IntermediateData] = None
+  """Intermediate steps generated as a part of Agent execution.
+
+  For a multi-agent system, it is also helpful to inspect the route that
+  the agent took to generate final response.
+  """
+
+  creation_timestamp: float = 0.0
+  """Timestamp for the current invocation, primarily intended for debugging purposes."""
+
+
+class SessionInput(EvalBaseModel):
+  """Values that help initialize a Session."""
+
+  app_name: str
+  """The name of the app."""
+
+  user_id: str
+  """The user id."""
+
+  state: dict[str, Any] = Field(default_factory=dict)
+  """The state of the session."""
+
+
+class EvalCase(EvalBaseModel):
+  """An eval case."""
+
+  eval_id: str
+  """Unique identifier for the evaluation case."""
+
+  conversation: list[Invocation]
+  """A conversation between the user and the Agent. The conversation can have any number of invocations."""
+
+  session_input: Optional[SessionInput] = None
+  """Session input that will be passed on to the Agent during eval.
+     It is common for Agents state to be initialized to some initial/default value,
+     for example, your agent may need to know today's date.
+  """
+
+  creation_timestamp: float = 0.0
+  """The time at which this eval case was created."""

google/adk/evaluation/eval_set.py

@@ -0,0 +1,37 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Optional
+from pydantic import BaseModel
+from .eval_case import EvalCase
+
+
+class EvalSet(BaseModel):
+  """A set of eval cases."""
+
+  eval_set_id: str
+  """Unique identifier for the eval set."""
+
+  name: Optional[str] = None
+  """Name of the dataset."""
+
+  description: Optional[str] = None
+  """Description of the dataset."""
+
+  eval_cases: list[EvalCase]
+  """List of eval cases in the dataset. Each case represents a single
+  interaction to be evaluated."""
+
+  creation_timestamp: float = 0.0
+  """The time at which this eval set was created."""

google/adk/evaluation/eval_sets_manager.py

@@ -0,0 +1,42 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from abc import ABC, abstractmethod
+
+from .eval_case import EvalCase
+from .eval_set import EvalSet
+
+
+class EvalSetsManager(ABC):
+  """An interface to manage an Eval Sets."""
+
+  @abstractmethod
+  def get_eval_set(self, app_name: str, eval_set_id: str) -> EvalSet:
+    """Returns an EvalSet identified by an app_name and eval_set_id."""
+    raise NotImplementedError()
+
+  @abstractmethod
+  def create_eval_set(self, app_name: str, eval_set_id: str):
+    """Creates an empty EvalSet given the app_name and eval_set_id."""
+    raise NotImplementedError()
+
+  @abstractmethod
+  def list_eval_sets(self, app_name: str) -> list[str]:
+    """Returns a list of EvalSets that belong to the given app_name."""
+    raise NotImplementedError()
+
+  @abstractmethod
+  def add_eval_case(self, app_name: str, eval_set_id: str, eval_case: EvalCase):
+    """Adds the given EvalCase to an existing EvalSet identified by app_name and eval_set_id."""
+    raise NotImplementedError()