google-adk 0.5.0__py3-none-any.whl → 1.1.0__py3-none-any.whl

google/adk/evaluation/local_eval_sets_manager.py

@@ -0,0 +1,264 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import logging
+import os
+import re
+import time
+from typing import Any
+import uuid
+
+from google.genai import types as genai_types
+from pydantic import ValidationError
+from typing_extensions import override
+
+from .eval_case import EvalCase
+from .eval_case import IntermediateData
+from .eval_case import Invocation
+from .eval_case import SessionInput
+from .eval_set import EvalSet
+from .eval_sets_manager import EvalSetsManager
+
+logger = logging.getLogger("google_adk." + __name__)
+
+_EVAL_SET_FILE_EXTENSION = ".evalset.json"
+
+
+def _convert_invocation_to_pydantic_schema(
+    invocation_in_json_format: dict[str, Any],
+) -> Invocation:
+  """Converts an invocation from old json format to new Pydantic Schema"""
+  query = invocation_in_json_format["query"]
+  reference = invocation_in_json_format["reference"]
+  expected_tool_use = []
+  expected_intermediate_agent_responses = []
+
+  for old_tool_use in invocation_in_json_format.get("expected_tool_use", []):
+    expected_tool_use.append(
+        genai_types.FunctionCall(
+            name=old_tool_use["tool_name"], args=old_tool_use["tool_input"]
+        )
+    )
+
+  for old_intermediate_response in invocation_in_json_format.get(
+      "expected_intermediate_agent_responses", []
+  ):
+    expected_intermediate_agent_responses.append((
+        old_intermediate_response["author"],
+        [genai_types.Part.from_text(text=old_intermediate_response["text"])],
+    ))
+
+  return Invocation(
+      invocation_id=str(uuid.uuid4()),
+      user_content=genai_types.Content(
+          parts=[genai_types.Part.from_text(text=query)], role="user"
+      ),
+      final_response=genai_types.Content(
+          parts=[genai_types.Part.from_text(text=reference)], role="model"
+      ),
+      intermediate_data=IntermediateData(
+          tool_uses=expected_tool_use,
+          intermediate_responses=expected_intermediate_agent_responses,
+      ),
+      creation_timestamp=time.time(),
+  )
+
+
+def convert_eval_set_to_pydanctic_schema(
+    eval_set_id: str,
+    eval_set_in_json_format: list[dict[str, Any]],
+) -> EvalSet:
+  r"""Returns an pydantic EvalSet generated from the json representation.
+
+    Args:
+      eval_set_id: Eval set id.
+      eval_set_in_json_format: Eval set specified in JSON format.
+
+    Here is a sample eval set in JSON format:
+  [
+    {
+      "name": "roll_17_sided_dice_twice",
+      "data": [
+        {
+          "query": "What can you do?",
+          "expected_tool_use": [],
+          "expected_intermediate_agent_responses": [],
+          "reference": "I can roll dice of different sizes and check if a number
+            is prime. I can also use multiple tools in parallel.\n"
+        },
+        {
+          "query": "Roll a 17 sided dice twice for me",
+          "expected_tool_use": [
+            {
+              "tool_name": "roll_die",
+              "tool_input": {
+                "sides": 17
+              }
+            },
+            {
+              "tool_name": "roll_die",
+              "tool_input": {
+                "sides": 17
+              }
+            }
+          ],
+          "expected_intermediate_agent_responses": [],
+          "reference": "I have rolled a 17 sided die twice. The first roll was
+            13 and the second roll was 4.\n"
+        }
+      ],
+      "initial_session": {
+        "state": {},
+        "app_name": "hello_world",
+        "user_id": "user"
+      }
+    }
+  ]
+  """
+  eval_cases = []
+  for old_eval_case in eval_set_in_json_format:
+    new_invocations = []
+
+    for old_invocation in old_eval_case["data"]:
+      new_invocations.append(
+          _convert_invocation_to_pydantic_schema(old_invocation)
+      )
+
+    session_input = None
+    if (
+        "initial_session" in old_eval_case
+        and len(old_eval_case["initial_session"]) > 0
+    ):
+      session_input = SessionInput(
+          app_name=old_eval_case["initial_session"].get("app_name", ""),
+          user_id=old_eval_case["initial_session"].get("user_id", ""),
+          state=old_eval_case["initial_session"].get("state", {}),
+      )
+
+    new_eval_case = EvalCase(
+        eval_id=old_eval_case["name"],
+        conversation=new_invocations,
+        session_input=session_input,
+        creation_timestamp=time.time(),
+    )
+    eval_cases.append(new_eval_case)
+
+  return EvalSet(
+      eval_set_id=eval_set_id,
+      name=eval_set_id,
+      creation_timestamp=time.time(),
+      eval_cases=eval_cases,
+  )
+
+
+def load_eval_set_from_file(
+    eval_set_file_path: str, eval_set_id: str
+) -> EvalSet:
+  """Returns an EvalSet that is read from the given file."""
+  with open(eval_set_file_path, "r", encoding="utf-8") as f:
+    content = f.read()
+    try:
+      return EvalSet.model_validate_json(content)
+    except ValidationError:
+      # We assume that the eval data was specified in the old format and try
+      # to convert it to the new format.
+      return convert_eval_set_to_pydanctic_schema(
+          eval_set_id, json.loads(content)
+      )
+
+
+class LocalEvalSetsManager(EvalSetsManager):
+  """An EvalSets manager that stores eval sets locally on disk."""
+
+  def __init__(self, agents_dir: str):
+    self._agents_dir = agents_dir
+
+  @override
+  def get_eval_set(self, app_name: str, eval_set_id: str) -> EvalSet:
+    """Returns an EvalSet identified by an app_name and eval_set_id."""
+    # Load the eval set file data
+    eval_set_file_path = self._get_eval_set_file_path(app_name, eval_set_id)
+    return load_eval_set_from_file(eval_set_file_path, eval_set_id)
+
+  @override
+  def create_eval_set(self, app_name: str, eval_set_id: str):
+    """Creates an empty EvalSet given the app_name and eval_set_id."""
+    self._validate_id(id_name="Eval Set Id", id_value=eval_set_id)
+
+    # Define the file path
+    new_eval_set_path = self._get_eval_set_file_path(app_name, eval_set_id)
+
+    logger.info("Creating eval set file `%s`", new_eval_set_path)
+
+    if not os.path.exists(new_eval_set_path):
+      # Write the JSON string to the file
+      logger.info("Eval set file doesn't exist, we will create a new one.")
+      new_eval_set = EvalSet(
+          eval_set_id=eval_set_id,
+          name=eval_set_id,
+          eval_cases=[],
+          creation_timestamp=time.time(),
+      )
+      self._write_eval_set(new_eval_set_path, new_eval_set)
+
+  @override
+  def list_eval_sets(self, app_name: str) -> list[str]:
+    """Returns a list of EvalSets that belong to the given app_name."""
+    eval_set_file_path = os.path.join(self._agents_dir, app_name)
+    eval_sets = []
+    for file in os.listdir(eval_set_file_path):
+      if file.endswith(_EVAL_SET_FILE_EXTENSION):
+        eval_sets.append(
+            os.path.basename(file).removesuffix(_EVAL_SET_FILE_EXTENSION)
+        )
+
+    return sorted(eval_sets)
+
+  @override
+  def add_eval_case(self, app_name: str, eval_set_id: str, eval_case: EvalCase):
+    """Adds the given EvalCase to an existing EvalSet identified by app_name and eval_set_id."""
+    eval_case_id = eval_case.eval_id
+    self._validate_id(id_name="Eval Case Id", id_value=eval_case_id)
+
+    eval_set = self.get_eval_set(app_name, eval_set_id)
+
+    if [x for x in eval_set.eval_cases if x.eval_id == eval_case_id]:
+      raise ValueError(
+          f"Eval id `{eval_case_id}` already exists in `{eval_set_id}`"
+          " eval set.",
+      )
+
+    eval_set.eval_cases.append(eval_case)
+
+    eval_set_file_path = self._get_eval_set_file_path(app_name, eval_set_id)
+    self._write_eval_set(eval_set_file_path, eval_set)
+
+  def _get_eval_set_file_path(self, app_name: str, eval_set_id: str) -> str:
+    return os.path.join(
+        self._agents_dir,
+        app_name,
+        eval_set_id + _EVAL_SET_FILE_EXTENSION,
+    )
+
+  def _validate_id(self, id_name: str, id_value: str):
+    pattern = r"^[a-zA-Z0-9_]+$"
+    if not bool(re.fullmatch(pattern, id_value)):
+      raise ValueError(
+          f"Invalid {id_name}. {id_name} should have the `{pattern}` format",
+      )
+
+  def _write_eval_set(self, eval_set_path: str, eval_set: EvalSet):
+    with open(eval_set_path, "w") as f:
+      f.write(eval_set.model_dump_json(indent=2))

google/adk/evaluation/response_evaluator.py

@@ -13,17 +13,122 @@
 # limitations under the License.
 
 from typing import Any
+from typing import Optional
 
+from deprecated import deprecated
+from google.genai import types as genai_types
 import pandas as pd
 from tabulate import tabulate
+from typing_extensions import override
 from vertexai.preview.evaluation import EvalTask
 from vertexai.preview.evaluation import MetricPromptTemplateExamples
 
+from .eval_case import IntermediateData
+from .eval_case import Invocation
+from .evaluator import EvalStatus
+from .evaluator import EvaluationResult
+from .evaluator import Evaluator
+from .evaluator import PerInvocationResult
 
-class ResponseEvaluator:
+
+class ResponseEvaluator(Evaluator):
   """Runs response evaluation for agents."""
 
+  def __init__(self, threshold: float, metric_name: str):
+    if "response_evaluation_score" == metric_name:
+      self._metric_name = MetricPromptTemplateExamples.Pointwise.COHERENCE
+    elif "response_match_score" == metric_name:
+      self._metric_name = "rouge_1"
+    else:
+      raise ValueError(f"`{metric_name}` is not supported.")
+
+    self._threshold = threshold
+
+  @override
+  def evaluate_invocations(
+      self,
+      actual_invocations: list[Invocation],
+      expected_invocations: list[Invocation],
+  ) -> EvaluationResult:
+    total_score = 0.0
+    num_invocations = 0
+    per_invocation_results = []
+    for actual, expected in zip(actual_invocations, expected_invocations):
+      prompt = self._get_text(expected.user_content)
+      reference = self._get_text(expected.final_response)
+      response = self._get_text(actual.final_response)
+      actual_tool_use = self._get_tool_use_trajectory(actual.intermediate_data)
+      reference_trajectory = self._get_tool_use_trajectory(
+          expected.intermediate_data
+      )
+
+      eval_case = {
+          "prompt": prompt,
+          "reference": reference,
+          "response": response,
+          "actual_tool_user": actual_tool_use,
+          "reference_trajectory": reference_trajectory,
+      }
+
+      eval_case_result = ResponseEvaluator._perform_eval(
+          pd.DataFrame([eval_case]), [self._metric_name]
+      )
+      score = self._get_score(eval_case_result)
+      per_invocation_results.append(
+          PerInvocationResult(
+              actual_invocation=actual,
+              expected_invocation=expected,
+              score=score,
+              eval_status=self._get_eval_status(score),
+          )
+      )
+      total_score += score
+      num_invocations += 1
+
+    if per_invocation_results:
+      overall_score = total_score / num_invocations
+      return EvaluationResult(
+          overall_score=overall_score,
+          overall_eval_status=self._get_eval_status(overall_score),
+          per_invocation_results=per_invocation_results,
+      )
+
+    return EvaluationResult()
+
+  def _get_text(self, content: Optional[genai_types.Content]) -> str:
+    if content and content.parts:
+      return "\n".join([p.text for p in content.parts if p.text])
+
+    return ""
+
+  def _get_tool_use_trajectory(
+      self, intermediate_data: Optional[IntermediateData]
+  ) -> list[dict[str, Any]]:
+    tool_use_trajectory = []
+    if not intermediate_data:
+      return tool_use_trajectory
+
+    for function_call in intermediate_data.tool_uses:
+      tool_use_trajectory.append({
+          "tool_name": function_call.name,
+          "tool_input": function_call.args or {},
+      })
+
+    return tool_use_trajectory
+
+  def _get_score(self, eval_result) -> float:
+    return eval_result.summary_metrics[f"{self._metric_name}/mean"].item()
+
+  def _get_eval_status(self, score: float):
+    return EvalStatus.PASSED if score >= self._threshold else EvalStatus.FAILED
+
   @staticmethod
+  @deprecated(
+      reason=(
+          "This method has been deprecated and will be removed soon. Please use"
+          " evaluate_invocations instead."
+      )
+  )
   def evaluate(
       raw_eval_dataset: list[list[dict[str, Any]]],
       evaluation_criteria: list[str],

google/adk/evaluation/trajectory_evaluator.py

@@ -13,17 +13,98 @@
 # limitations under the License.
 
 from typing import Any
+from typing import cast
 
+from deprecated import deprecated
+from google.genai import types as genai_types
 import pandas as pd
 from tabulate import tabulate
+from typing_extensions import override
 
+from .eval_case import Invocation
 from .evaluation_constants import EvalConstants
+from .evaluator import EvalStatus
+from .evaluator import EvaluationResult
+from .evaluator import Evaluator
+from .evaluator import PerInvocationResult
 
 
-class TrajectoryEvaluator:
+class TrajectoryEvaluator(Evaluator):
   """Evaluates tool use trajectories for accuracy."""
 
+  def __init__(self, threshold: float):
+    self._threshold = threshold
+
+  @override
+  def evaluate_invocations(
+      self,
+      actual_invocations: list[Invocation],
+      expected_invocations: list[Invocation],
+  ) -> EvaluationResult:
+    """Returns EvaluationResult after performing evaluations using actual and expected invocations."""
+    total_tool_use_accuracy = 0.0
+    num_invocations = 0
+    per_invocation_results = []
+
+    for actual, expected in zip(actual_invocations, expected_invocations):
+      actual_tool_uses = (
+          actual.intermediate_data.tool_uses if actual.intermediate_data else []
+      )
+      expected_tool_uses = (
+          expected.intermediate_data.tool_uses
+          if expected.intermediate_data
+          else []
+      )
+      tool_use_accuracy = (
+          1.0
+          if self._are_tool_calls_equal(actual_tool_uses, expected_tool_uses)
+          else 0.0
+      )
+      per_invocation_results.append(
+          PerInvocationResult(
+              actual_invocation=actual,
+              expected_invocation=expected,
+              score=tool_use_accuracy,
+              eval_status=self._get_eval_status(tool_use_accuracy),
+          )
+      )
+      total_tool_use_accuracy += tool_use_accuracy
+      num_invocations += 1
+
+    if per_invocation_results:
+      overall_score = total_tool_use_accuracy / num_invocations
+      return EvaluationResult(
+          overall_score=overall_score,
+          overall_eval_status=self._get_eval_status(overall_score),
+          per_invocation_results=per_invocation_results,
+      )
+
+    return EvaluationResult()
+
+  def _are_tool_calls_equal(
+      self,
+      actual_tool_calls: list[genai_types.FunctionCall],
+      expected_tool_calls: list[genai_types.FunctionCall],
+  ) -> bool:
+    if len(actual_tool_calls) != len(expected_tool_calls):
+      return False
+
+    for actual, expected in zip(actual_tool_calls, expected_tool_calls):
+      if actual.name != expected.name or actual.args != expected.args:
+        return False
+
+    return True
+
+  def _get_eval_status(self, score: float):
+    return EvalStatus.PASSED if score >= self._threshold else EvalStatus.FAILED
+
   @staticmethod
+  @deprecated(
+      reason=(
+          "This method has been deprecated and will be removed soon. Please use"
+          " evaluate_invocations instead."
+      )
+  )
   def evaluate(
       eval_dataset: list[list[dict[str, Any]]],
       *,
@@ -35,7 +116,7 @@ class TrajectoryEvaluator:
     tool use trajectories. An exact match scores a 1, 0 otherwise. The final
     number is an average of these individual scores.
 
-    Value range: [0, 1], where 0 is means none of the too use entries aligned,
+    Value range: [0, 1], where 0 means none of the tool use entries aligned,
     and 1 would mean all of them aligned. Higher value is good.
 
     Args:
@@ -137,6 +218,7 @@ class TrajectoryEvaluator:
     return new_row, failure
 
   @staticmethod
+  @deprecated()
   def are_tools_equal(list_a_original, list_b_original):
     # Remove other entries that we don't want to evaluate
     list_a = [

google/adk/events/event.py

@@ -19,6 +19,7 @@ import string
 from typing import Optional
 
 from google.genai import types
+from pydantic import alias_generators
 from pydantic import ConfigDict
 from pydantic import Field
 
@@ -46,7 +47,11 @@ class Event(LlmResponse):
   """
 
   model_config = ConfigDict(
-      extra='forbid', ser_json_bytes='base64', val_json_bytes='base64'
+      extra='forbid',
+      ser_json_bytes='base64',
+      val_json_bytes='base64',
+      alias_generator=alias_generators.to_camel,
+      populate_by_name=True,
   )
   """The pydantic model config."""
 

google/adk/events/event_actions.py

@@ -16,6 +16,7 @@ from __future__ import annotations
 
 from typing import Optional
 
+from pydantic import alias_generators
 from pydantic import BaseModel
 from pydantic import ConfigDict
 from pydantic import Field
@@ -26,7 +27,11 @@ from ..auth.auth_tool import AuthConfig
 class EventActions(BaseModel):
   """Represents the actions attached to an event."""
 
-  model_config = ConfigDict(extra='forbid')
+  model_config = ConfigDict(
+      extra='forbid',
+      alias_generator=alias_generators.to_camel,
+      populate_by_name=True,
+  )
   """The pydantic model config."""
 
   skip_summarization: Optional[bool] = None

google/adk/examples/base_example_provider.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import abc
+
 from .example import Example
 
 

google/adk/examples/example_util.py

@@ -15,8 +15,9 @@
 """Utility functions for converting examples to a string that can be used in system instructions in the prompt."""
 
 import logging
-from typing import Optional, Union
+from typing import Optional
 from typing import TYPE_CHECKING
+from typing import Union
 
 from .base_example_provider import BaseExampleProvider
 from .example import Example
@@ -24,7 +25,7 @@ from .example import Example
 if TYPE_CHECKING:
   from ..sessions.session import Session
 
-logger = logging.getLogger(__name__)
+logger = logging.getLogger("google_adk." + __name__)
 
 # Constant parts of the example string
 _EXAMPLES_INTRO = (

google/adk/flows/llm_flows/_code_execution.py

@@ -22,7 +22,6 @@ import dataclasses
 import os
 import re
 from typing import AsyncGenerator
-from typing import Generator
 from typing import Optional
 from typing import TYPE_CHECKING
 
@@ -31,6 +30,7 @@ from typing_extensions import override
 
 from ...agents.invocation_context import InvocationContext
 from ...code_executors.base_code_executor import BaseCodeExecutor
+from ...code_executors.built_in_code_executor import BuiltInCodeExecutor
 from ...code_executors.code_execution_utils import CodeExecutionInput
 from ...code_executors.code_execution_utils import CodeExecutionResult
 from ...code_executors.code_execution_utils import CodeExecutionUtils
@@ -174,6 +174,11 @@ async def _run_pre_processor(
 
   if not code_executor or not isinstance(code_executor, BaseCodeExecutor):
     return
+
+  if isinstance(code_executor, BuiltInCodeExecutor):
+    code_executor.process_llm_request(llm_request)
+    return
+
   if not code_executor.optimize_data_file:
     return
 
@@ -262,6 +267,9 @@ async def _run_post_processor(
   if not llm_response or not llm_response.content:
     return
 
+  if isinstance(code_executor, BuiltInCodeExecutor):
+    return
+
   code_executor_context = CodeExecutorContext(invocation_context.session.state)
   # Skip if the error count exceeds the max retry attempts.
   if (

google/adk/flows/llm_flows/audio_transcriber.py

@@ -25,8 +25,9 @@ if TYPE_CHECKING:
 class AudioTranscriber:
   """Transcribes audio using Google Cloud Speech-to-Text."""
 
-  def __init__(self):
-    self.client = speech.SpeechClient()
+  def __init__(self, init_client=False):
+    if init_client:
+      self.client = speech.SpeechClient()
 
   def transcribe_file(
       self, invocation_context: InvocationContext
@@ -84,7 +85,7 @@ class AudioTranscriber:
 
     # Step2: transcription
     for speaker, data in bundled_audio:
-      if speaker == 'user':
+      if isinstance(data, genai_types.Blob):
         audio = speech.RecognitionAudio(content=data)
 
         config = speech.RecognitionConfig(