google-adk 1.5.0__py3-none-any.whl → 1.6.1__py3-none-any.whl

google/adk/evaluation/response_evaluator.py

@@ -14,33 +14,55 @@
 
 from __future__ import annotations
 
-from typing import Any
 from typing import Optional
 
-from google.genai import types as genai_types
-import pandas as pd
-from tabulate import tabulate
-from typing_extensions import deprecated
 from typing_extensions import override
-from vertexai.preview.evaluation import EvalTask
-from vertexai.preview.evaluation import MetricPromptTemplateExamples
+from vertexai import types as vertexai_types
 
-from .eval_case import IntermediateData
 from .eval_case import Invocation
 from .eval_metrics import EvalMetric
-from .evaluator import EvalStatus
 from .evaluator import EvaluationResult
 from .evaluator import Evaluator
-from .evaluator import PerInvocationResult
 from .final_response_match_v1 import RougeEvaluator
+from .vertex_ai_eval_facade import _VertexAiEvalFacade
 
 
 class ResponseEvaluator(Evaluator):
-  """Runs response evaluation for agents."""
+  """Evaluates Agent's responses.
+
+  This class supports two metrics:
+  1) response_evaluation_score
+  This metric evaluates how coherent agent's resposne was.
+
+  Value range of this metric is [1,5], with values closer to 5 more desirable.
+
+  2) response_match_score:
+  This metric evaluates if agent's final response matches a golden/expected
+  final response.
+
+  Value range for this metric is [0,1], with values closer to 1 more desirable.
+  """
+
+  def __init__(
+      self,
+      threshold: Optional[float] = None,
+      metric_name: Optional[str] = None,
+      eval_metric: Optional[EvalMetric] = None,
+  ):
+    if (threshold is not None and eval_metric) or (
+        metric_name is not None and eval_metric
+    ):
+      raise ValueError(
+          "Either eval_metric should be specified or both threshold and"
+          " metric_name should be specified."
+      )
+
+    if eval_metric:
+      threshold = eval_metric.threshold
+      metric_name = eval_metric.metric_name
 
-  def __init__(self, threshold: float, metric_name: str):
     if "response_evaluation_score" == metric_name:
-      self._metric_name = MetricPromptTemplateExamples.Pointwise.COHERENCE
+      self._metric_name = vertexai_types.PrebuiltMetric.COHERENCE
     elif "response_match_score" == metric_name:
       self._metric_name = "response_match_score"
     else:
@@ -63,201 +85,6 @@ class ResponseEvaluator(Evaluator):
           actual_invocations, expected_invocations
       )
 
-    total_score = 0.0
-    num_invocations = 0
-    per_invocation_results = []
-    for actual, expected in zip(actual_invocations, expected_invocations):
-      prompt = self._get_text(expected.user_content)
-      reference = self._get_text(expected.final_response)
-      response = self._get_text(actual.final_response)
-      actual_tool_use = self._get_tool_use_trajectory(actual.intermediate_data)
-      reference_trajectory = self._get_tool_use_trajectory(
-          expected.intermediate_data
-      )
-
-      eval_case = {
-          "prompt": prompt,
-          "reference": reference,
-          "response": response,
-          "actual_tool_user": actual_tool_use,
-          "reference_trajectory": reference_trajectory,
-      }
-
-      eval_case_result = ResponseEvaluator._perform_eval(
-          pd.DataFrame([eval_case]), [self._metric_name]
-      )
-      score = self._get_score(eval_case_result)
-      per_invocation_results.append(
-          PerInvocationResult(
-              actual_invocation=actual,
-              expected_invocation=expected,
-              score=score,
-              eval_status=self._get_eval_status(score),
-          )
-      )
-      total_score += score
-      num_invocations += 1
-
-    if per_invocation_results:
-      overall_score = total_score / num_invocations
-      return EvaluationResult(
-          overall_score=overall_score,
-          overall_eval_status=self._get_eval_status(overall_score),
-          per_invocation_results=per_invocation_results,
-      )
-
-    return EvaluationResult()
-
-  def _get_text(self, content: Optional[genai_types.Content]) -> str:
-    if content and content.parts:
-      return "\n".join([p.text for p in content.parts if p.text])
-
-    return ""
-
-  def _get_tool_use_trajectory(
-      self, intermediate_data: Optional[IntermediateData]
-  ) -> list[dict[str, Any]]:
-    tool_use_trajectory = []
-    if not intermediate_data:
-      return tool_use_trajectory
-
-    for function_call in intermediate_data.tool_uses:
-      tool_use_trajectory.append({
-          "tool_name": function_call.name,
-          "tool_input": function_call.args or {},
-      })
-
-    return tool_use_trajectory
-
-  def _get_score(self, eval_result) -> float:
-    return eval_result.summary_metrics[f"{self._metric_name}/mean"].item()
-
-  def _get_eval_status(self, score: float):
-    return EvalStatus.PASSED if score >= self._threshold else EvalStatus.FAILED
-
-  @staticmethod
-  @deprecated(
-      "This method has been deprecated and will be removed soon. Please use"
-      " evaluate_invocations instead."
-  )
-  def evaluate(
-      raw_eval_dataset: list[list[dict[str, Any]]],
-      evaluation_criteria: list[str],
-      *,
-      print_detailed_results: bool = False,
-  ):
-    r"""Returns the value of requested evaluation metrics.
-
-    Args:
-      raw_eval_dataset: The dataset that will be evaluated.
-      evaluation_criteria: The evaluation criteria to be used. This method
-        support two criteria, `response_evaluation_score` and
-        `response_match_score`.
-      print_detailed_results: Prints detailed results on the console. This is
-        usually helpful during debugging.
-
-    A note on evaluation_criteria:
-      `response_match_score`: This metric compares the agents final natural
-        language response with the expected final response, stored in the
-        "reference" field in test/eval files. We use Rouge metric to compare the
-        two responses.
-
-        Value Range: [0, 1]. A score closer to 0 means poor similarity between
-          response and reference. A score closer to 1 means strong similarity
-          between response and reference.
-
-      `response_evaluation_score`: Uses LLM to evalaute coherence of the
-        response, including tool use. This is pointwise metric.
-
-        Value range: [0, 5], where 0 means that the agent's response is not
-        coherent, while 5 means it is . High values are good.
-    A note on raw_eval_dataset:
-      The dataset should be a list session, where each session is represented
-      as a list of interaction that need evaluation. Each evaluation is
-      represented as a dictionary that is expected to have values for the
-      following keys:
-
-        1) query
-        2) response
-        3) acutal_tool_use
-        4) expected_tool_use
-        5) reference
-
-      Here is a sample eval_dataset value with one entry:
-      [
-        [
-          {
-            "query": "roll a die for me",
-            "response": "I rolled a 16 sided die and got 13.\n",
-            "expected_tool_use": [
-              {
-                "tool_name": "roll_die",
-                "tool_input": {
-                  "sides": 16
-                }
-              }
-            ],
-            "acutal_tool_use": [
-              {
-                "tool_name": "roll_die",
-                "tool_input": {
-                  "sides": 16
-                }
-              }
-            ],
-            "reference": "I rolled a 16 sided die and got 13.\n"
-          }
-        ]
-      ]
-    """
-    if not raw_eval_dataset:
-      raise ValueError("The evaluation dataset is empty.")
-
-    metrics = ResponseEvaluator._get_metrics(
-        raw_eval_dataset, evaluation_criteria
-    )
-    flattened_queries = [
-        item for sublist in raw_eval_dataset for item in sublist
-    ]
-    eval_dataset = pd.DataFrame(flattened_queries).rename(
-        columns={"query": "prompt", "expected_tool_use": "reference_trajectory"}
-    )
-
-    eval_result = ResponseEvaluator._perform_eval(
-        dataset=eval_dataset, metrics=metrics
-    )
-
-    if print_detailed_results:
-      ResponseEvaluator._print_results(eval_result)
-    return eval_result.summary_metrics
-
-  @staticmethod
-  def _get_metrics(raw_eval_dataset, criteria):
-    metrics = []
-    if (
-        "response_evaluation_score" in criteria
-        and "query" in raw_eval_dataset[0][0]
-        and "expected_tool_use" in raw_eval_dataset[0][0]
-    ):
-      metrics.append(MetricPromptTemplateExamples.Pointwise.COHERENCE)
-    if (
-        "response_match_score" in criteria
-        and "reference" in raw_eval_dataset[0][0]
-    ):
-      metrics.append("rouge_1")
-    return metrics
-
-  @staticmethod
-  def _perform_eval(dataset, metrics):
-    """This method hides away the call to external service.
-
-    Primarily helps with unit testing.
-    """
-    eval_task = EvalTask(dataset=dataset, metrics=metrics)
-
-    return eval_task.evaluate()
-
-  @staticmethod
-  def _print_results(eval_result):
-    print("Evaluation Summary Metrics:", eval_result.summary_metrics)
-    print(tabulate(eval_result.metrics_table, headers="keys", tablefmt="grid"))
+    return _VertexAiEvalFacade(
+        threshold=self._threshold, metric_name=self._metric_name
+    ).evaluate_invocations(actual_invocations, expected_invocations)

google/adk/evaluation/safety_evaluator.py

@@ -0,0 +1,54 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from typing_extensions import override
+from vertexai import types as vertexai_types
+
+from .eval_case import Invocation
+from .eval_metrics import EvalMetric
+from .evaluator import EvaluationResult
+from .evaluator import Evaluator
+from .vertex_ai_eval_facade import _VertexAiEvalFacade
+
+
+class SafetyEvaluatorV1(Evaluator):
+  """Evaluates safety (harmlessness) of an Agent's Response.
+
+  The class delegates the responsibility to Vertex Gen AI Eval SDK. The V1
+  suffix in the class name is added to convey that there could be other versions
+  of the safety metric as well, and those metrics could use a different strategy
+  to evaluate safety.
+
+  Using this class requires a GCP project. Please set GOOGLE_CLOUD_PROJECT and
+  GOOGLE_CLOUD_LOCATION in your .env file.
+
+  Value range of the metric is [0, 1], with values closer to 1 to be more
+  desirable (safe).
+  """
+
+  def __init__(self, eval_metric: EvalMetric):
+    self._eval_metric = eval_metric
+
+  @override
+  def evaluate_invocations(
+      self,
+      actual_invocations: list[Invocation],
+      expected_invocations: list[Invocation],
+  ) -> EvaluationResult:
+    return _VertexAiEvalFacade(
+        threshold=self._eval_metric.threshold,
+        metric_name=vertexai_types.PrebuiltMetric.SAFETY,
+    ).evaluate_invocations(actual_invocations, expected_invocations)

google/adk/evaluation/trajectory_evaluator.py

@@ -15,7 +15,7 @@
 from __future__ import annotations
 
 from typing import Any
-from typing import cast
+from typing import Optional
 
 from google.genai import types as genai_types
 import pandas as pd
@@ -24,6 +24,7 @@ from typing_extensions import deprecated
 from typing_extensions import override
 
 from .eval_case import Invocation
+from .eval_metrics import EvalMetric
 from .evaluation_constants import EvalConstants
 from .evaluator import EvalStatus
 from .evaluator import EvaluationResult
@@ -34,7 +35,20 @@ from .evaluator import PerInvocationResult
 class TrajectoryEvaluator(Evaluator):
   """Evaluates tool use trajectories for accuracy."""
 
-  def __init__(self, threshold: float):
+  def __init__(
+      self,
+      threshold: Optional[float] = None,
+      eval_metric: Optional[EvalMetric] = None,
+  ):
+    if threshold is not None and eval_metric:
+      raise ValueError(
+          "Either eval_metric should be specified or threshold should be"
+          " specified."
+      )
+
+    if eval_metric:
+      threshold = eval_metric.threshold
+
     self._threshold = threshold
 
   @override

google/adk/evaluation/vertex_ai_eval_facade.py

@@ -0,0 +1,147 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import os
+from typing import Optional
+
+from google.genai import types as genai_types
+import pandas as pd
+from typing_extensions import override
+from vertexai import Client as VertexAiClient
+from vertexai import types as vertexai_types
+
+from .eval_case import Invocation
+from .evaluator import EvalStatus
+from .evaluator import EvaluationResult
+from .evaluator import Evaluator
+from .evaluator import PerInvocationResult
+
+_ERROR_MESSAGE_SUFFIX = """
+You should specify both project id and location. This metric uses Vertex Gen AI
+Eval SDK, and it requires google cloud credentials.
+
+If using an .env file add the values there, or explicitly set in the code using
+the template below:
+
+os.environ['GOOGLE_CLOUD_LOCATION'] = <LOCATION>
+os.environ['GOOGLE_CLOUD_PROJECT'] = <PROJECT ID>
+"""
+
+
+class _VertexAiEvalFacade(Evaluator):
+  """Simple facade for Vertex Gen AI Eval SDK.
+
+  Vertex Gen AI Eval SDK exposes quite a few metrics that are valuable for
+  agentic evals. This class helps us to access those metrics.
+
+  Using this class requires a GCP project. Please set GOOGLE_CLOUD_PROJECT and
+  GOOGLE_CLOUD_LOCATION in your .env file.
+  """
+
+  def __init__(
+      self, threshold: float, metric_name: vertexai_types.PrebuiltMetric
+  ):
+    self._threshold = threshold
+    self._metric_name = metric_name
+
+  @override
+  def evaluate_invocations(
+      self,
+      actual_invocations: list[Invocation],
+      expected_invocations: list[Invocation],
+  ) -> EvaluationResult:
+    total_score = 0.0
+    num_invocations = 0
+    per_invocation_results = []
+    for actual, expected in zip(actual_invocations, expected_invocations):
+      prompt = self._get_text(expected.user_content)
+      reference = self._get_text(expected.final_response)
+      response = self._get_text(actual.final_response)
+      eval_case = {
+          "prompt": prompt,
+          "reference": reference,
+          "response": response,
+      }
+
+      eval_case_result = _VertexAiEvalFacade._perform_eval(
+          dataset=pd.DataFrame([eval_case]), metrics=[self._metric_name]
+      )
+      score = self._get_score(eval_case_result)
+      per_invocation_results.append(
+          PerInvocationResult(
+              actual_invocation=actual,
+              expected_invocation=expected,
+              score=score,
+              eval_status=self._get_eval_status(score),
+          )
+      )
+
+      if score:
+        total_score += score
+        num_invocations += 1
+
+    if per_invocation_results:
+      overall_score = (
+          total_score / num_invocations if num_invocations > 0 else None
+      )
+      return EvaluationResult(
+          overall_score=overall_score,
+          overall_eval_status=self._get_eval_status(overall_score),
+          per_invocation_results=per_invocation_results,
+      )
+
+    return EvaluationResult()
+
+  def _get_text(self, content: Optional[genai_types.Content]) -> str:
+    if content and content.parts:
+      return "\n".join([p.text for p in content.parts if p.text])
+
+    return ""
+
+  def _get_score(self, eval_result) -> Optional[float]:
+    if eval_result and eval_result.summary_metrics:
+      return eval_result.summary_metrics[0].mean_score
+
+    return None
+
+  def _get_eval_status(self, score: Optional[float]):
+    if score:
+      return (
+          EvalStatus.PASSED if score >= self._threshold else EvalStatus.FAILED
+      )
+
+    return EvalStatus.NOT_EVALUATED
+
+  @staticmethod
+  def _perform_eval(dataset, metrics):
+    """This method hides away the call to external service.
+
+    Primarily helps with unit testing.
+    """
+    project_id = os.environ.get("GOOGLE_CLOUD_PROJECT", None)
+    location = os.environ.get("GOOGLE_CLOUD_LOCATION", None)
+
+    if not project_id:
+      raise ValueError("Missing project id." + _ERROR_MESSAGE_SUFFIX)
+    if not location:
+      raise ValueError("Missing location." + _ERROR_MESSAGE_SUFFIX)
+
+    client = VertexAiClient(project=project_id, location=location)
+
+    return client.evals.evaluate(
+        dataset=vertexai_types.EvaluationDataset(eval_dataset_df=dataset),
+        metrics=metrics,
+    )

google/adk/events/event.py

@@ -14,9 +14,8 @@
 from __future__ import annotations
 
 from datetime import datetime
-import random
-import string
 from typing import Optional
+import uuid
 
 from google.genai import types
 from pydantic import alias_generators
@@ -132,5 +131,4 @@ class Event(LlmResponse):
 
   @staticmethod
   def new_id():
-    characters = string.ascii_letters + string.digits
-    return ''.join(random.choice(characters) for _ in range(8))
+    return str(uuid.uuid4())

google/adk/flows/llm_flows/base_llm_flow.py

@@ -16,6 +16,7 @@ from __future__ import annotations
 
 from abc import ABC
 import asyncio
+import datetime
 import inspect
 import logging
 from typing import AsyncGenerator
@@ -320,6 +321,7 @@ class BaseLlmFlow(ABC):
       ):
         # Update the mutable event id to avoid conflict
         model_response_event.id = Event.new_id()
+        model_response_event.timestamp = datetime.datetime.now().timestamp()
         yield event
 
   async def _preprocess_async(

google/adk/memory/in_memory_memory_service.py

@@ -11,8 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-
 from __future__ import annotations
 
 import re
@@ -43,6 +41,9 @@ class InMemoryMemoryService(BaseMemoryService):
   """An in-memory memory service for prototyping purpose only.
 
   Uses keyword matching instead of semantic search.
+
+  It is not suitable for multi-threaded production environments. Use it for
+  testing and development only.
   """
 
   def __init__(self):

google/adk/models/lite_llm.py

@@ -23,6 +23,7 @@ from typing import cast
 from typing import Dict
 from typing import Generator
 from typing import Iterable
+from typing import List
 from typing import Literal
 from typing import Optional
 from typing import Tuple
@@ -485,16 +486,22 @@ def _message_to_generate_content_response(
 
 def _get_completion_inputs(
     llm_request: LlmRequest,
-) -> tuple[Iterable[Message], Iterable[dict]]:
-  """Converts an LlmRequest to litellm inputs.
+) -> Tuple[
+    List[Message],
+    Optional[List[Dict]],
+    Optional[types.SchemaUnion],
+    Optional[Dict],
+]:
+  """Converts an LlmRequest to litellm inputs and extracts generation params.
 
   Args:
     llm_request: The LlmRequest to convert.
 
   Returns:
-    The litellm inputs (message list, tool dictionary and response format).
+    The litellm inputs (message list, tool dictionary, response format and generation params).
   """
-  messages = []
+  # 1. Construct messages
+  messages: List[Message] = []
   for content in llm_request.contents or []:
     message_param_or_list = _content_to_message_param(content)
     if isinstance(message_param_or_list, list):
@@ -511,7 +518,8 @@ def _get_completion_inputs(
         ),
     )
 
-  tools = None
+  # 2. Convert tool declarations
+  tools: Optional[List[Dict]] = None
   if (
       llm_request.config
       and llm_request.config.tools
@@ -522,12 +530,39 @@ def _get_completion_inputs(
         for tool in llm_request.config.tools[0].function_declarations
     ]
 
-  response_format = None
-
-  if llm_request.config.response_schema:
+  # 3. Handle response format
+  response_format: Optional[types.SchemaUnion] = None
+  if llm_request.config and llm_request.config.response_schema:
     response_format = llm_request.config.response_schema
 
-  return messages, tools, response_format
+  # 4. Extract generation parameters
+  generation_params: Optional[Dict] = None
+  if llm_request.config:
+    config_dict = llm_request.config.model_dump(exclude_none=True)
+    # Generate LiteLlm parameters here,
+    # Following https://docs.litellm.ai/docs/completion/input.
+    generation_params = {}
+    param_mapping = {
+        "max_output_tokens": "max_completion_tokens",
+        "stop_sequences": "stop",
+    }
+    for key in (
+        "temperature",
+        "max_output_tokens",
+        "top_p",
+        "top_k",
+        "stop_sequences",
+        "presence_penalty",
+        "frequency_penalty",
+    ):
+      if key in config_dict:
+        mapped_key = param_mapping.get(key, key)
+        generation_params[mapped_key] = config_dict[key]
+
+      if not generation_params:
+        generation_params = None
+
+  return messages, tools, response_format, generation_params
 
 
 def _build_function_declaration_log(
@@ -664,7 +699,9 @@ class LiteLlm(BaseLlm):
     self._maybe_append_user_content(llm_request)
     logger.debug(_build_request_log(llm_request))
 
-    messages, tools, response_format = _get_completion_inputs(llm_request)
+    messages, tools, response_format, generation_params = (
+        _get_completion_inputs(llm_request)
+    )
 
     if "functions" in self._additional_args:
       # LiteLLM does not support both tools and functions together.
@@ -678,6 +715,9 @@ class LiteLlm(BaseLlm):
     }
     completion_args.update(self._additional_args)
 
+    if generation_params:
+      completion_args.update(generation_params)
+
     if stream:
       text = ""
       # Track function calls by index