google-adk 0.0.1__py3-none-any.whl → 0.0.3__py3-none-any.whl

google/adk/evaluation/evaluation_generator.py

@@ -0,0 +1,270 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import importlib
+import uuid
+
+from google.genai import types
+
+from ..agents.base_agent import BaseAgent
+from ..agents.llm_agent import Agent
+from ..agents.llm_agent import BeforeToolCallback
+from ..agents.llm_agent import LlmAgent
+from ..artifacts.in_memory_artifact_service import InMemoryArtifactService
+from ..runners import Runner
+from ..sessions.in_memory_session_service import InMemorySessionService
+from ..sessions.session import Session
+from .evaluation_constants import EvalConstants
+
+
+class EvaluationGenerator:
+  """Generates evaluation responses for agents."""
+
+  @staticmethod
+  def generate_responses(
+      eval_dataset,
+      agent_module_path,
+      repeat_num=3,
+      agent_name=None,
+      initial_session={},
+  ):
+    """Returns evaluation responses for the given dataset and agent.
+
+    Args:
+      eval_dataset: The dataset that needs to be scraped for resposnes.
+      agent_module_path: Path to the module that contains the root agent.
+      repeat_num: Number of time the eval dataset should be repeated. This is
+        usually done to remove uncertainity that a single run may bring.
+      agent_name: The name of the agent that should be evaluated. This is
+        usually the sub-agent.
+      initial_session: Initial session for the eval data.
+    """
+    results = []
+
+    for _ in range(repeat_num):
+      for data in eval_dataset:
+        results.append(
+            EvaluationGenerator._process_query(
+                data, agent_module_path, agent_name, initial_session
+            )
+        )
+
+    return results
+
+  @staticmethod
+  def generate_responses_from_session(session_path, eval_dataset):
+    """Returns evaluation responses by combining session data with eval data.
+
+    Args:
+      session_path: Path to a json file that contains session data.
+      eval_dataset: The eval data set that should be combined with the session
+        data.
+    """
+    results = []
+
+    with open(session_path, "r") as f:
+      session_data = Session.model_validate_json(f.read())
+      print("loaded session", session_path)
+
+    for data in eval_dataset:
+      # load session data from session_path
+      results.append(
+          EvaluationGenerator._process_query_with_session(
+              session_data,
+              data,
+          )
+      )
+
+    return results
+
+  @staticmethod
+  def _process_query(data, module_name, agent_name=None, initial_session={}):
+    """Process a query using the agent and evaluation dataset."""
+    module_path = f"{module_name}"
+    agent_module = importlib.import_module(module_path)
+    root_agent = agent_module.agent.root_agent
+
+    reset_func = getattr(agent_module.agent, "reset_data", None)
+
+    agent_to_evaluate = root_agent
+    if agent_name:
+      agent_to_evaluate = root_agent.find_agent(agent_name)
+      assert agent_to_evaluate, f"Sub-Agent `{agent_name}` not found."
+
+    return EvaluationGenerator._process_query_with_root_agent(
+        data, agent_to_evaluate, reset_func, initial_session
+    )
+
+  @staticmethod
+  def _process_query_with_root_agent(
+      data,
+      root_agent,
+      reset_func,
+      initial_session={},
+      session_id=None,
+      session_service=None,
+      artifact_service=None,
+  ):
+    """Process a query using the agent and evaluation dataset."""
+
+    # we don't know which tools belong to which agent
+    # so we just apply to any agents that has certain tool outputs
+    all_mock_tools = set()
+    for eval_entry in data:
+      expected_tool_use = eval_entry.get(EvalConstants.EXPECTED_TOOL_USE, [])
+      for expected in expected_tool_use:
+        if EvalConstants.MOCK_TOOL_OUTPUT in expected:
+          all_mock_tools.add(expected[EvalConstants.TOOL_NAME])
+
+    eval_data_copy = data.copy()
+    EvaluationGenerator.apply_before_tool_callback(
+        root_agent,
+        lambda *args: EvaluationGenerator.before_tool_callback(
+            *args, eval_dataset=eval_data_copy
+        ),
+        all_mock_tools,
+    )
+
+    if not session_service:
+      session_service = InMemorySessionService()
+
+    app_name = initial_session.get("app_name", "EvaluationGenerator")
+    user_id = initial_session.get("user_id", "test_user_id")
+    session_id = session_id if session_id else str(uuid.uuid4())
+
+    _ = session_service.create_session(
+        app_name=app_name,
+        user_id=user_id,
+        state=initial_session.get("state", {}),
+        session_id=session_id,
+    )
+
+    if not artifact_service:
+      artifact_service = InMemoryArtifactService()
+    runner = Runner(
+        app_name=app_name,
+        agent=root_agent,
+        artifact_service=artifact_service,
+        session_service=session_service,
+    )
+
+    # Reset agent state for each query
+    if callable(reset_func):
+      reset_func()
+
+    responses = data.copy()
+
+    for index, eval_entry in enumerate(responses):
+      response = None
+      query = eval_entry["query"]
+      content = types.Content(role="user", parts=[types.Part(text=query)])
+      turn_actual_tool_uses = []
+
+      for event in runner.run(
+          user_id=user_id, session_id=session_id, new_message=content
+      ):
+        if event.is_final_response() and event.content and event.content.parts:
+          response = event.content.parts[0].text
+        elif event.get_function_calls():
+          for call in event.get_function_calls():
+            turn_actual_tool_uses.append({
+                EvalConstants.TOOL_NAME: call.name,
+                EvalConstants.TOOL_INPUT: call.args,
+            })
+
+      responses[index]["actual_tool_use"] = turn_actual_tool_uses
+      responses[index]["response"] = response
+
+    return responses
+
+  @staticmethod
+  def _process_query_with_session(session_data, data):
+    """Process the queries using the existing session data without invoking the runner."""
+    responses = data.copy()
+
+    # Iterate through the provided queries and align them with the session events
+    for index, eval_entry in enumerate(responses):
+      query = eval_entry["query"]
+      actual_tool_uses = []
+      response = None
+
+      # Search for the corresponding session events
+      for event in session_data.events:
+        # Match the query to a user event
+        if (
+            event.author == "user"
+            and event.content
+            and event.content.parts
+            and event.content.parts[0].text == query
+        ):
+          # Look for subsequent tool usage or model responses
+          for subsequent_event in session_data.events:
+            if subsequent_event.invocation_id == event.invocation_id:
+              # Extract tool usage
+              if subsequent_event.content.parts[0].function_call:
+                call = subsequent_event.content.parts[0].function_call
+                actual_tool_uses.append(
+                    {"tool_name": call.name, "tool_input": call.args}
+                )
+              # Extract final response
+              elif subsequent_event.author != "user":
+                response = subsequent_event.content.parts[0].text
+
+      # Update the results for the current query
+      responses[index]["actual_tool_use"] = actual_tool_uses
+      responses[index]["response"] = response
+    return responses
+
+  @staticmethod
+  def before_tool_callback(tool, args, tool_context, eval_dataset):
+    """Intercept specific tool calls and return predefined outputs
+
+    from eval_dataset.
+    """
+    for index, eval_entry in enumerate(eval_dataset):
+      expected_tool_use = eval_entry.get("expected_tool_use", [])
+      for expected in expected_tool_use:
+        if (
+            EvalConstants.MOCK_TOOL_OUTPUT in expected
+            and tool.name == expected[EvalConstants.TOOL_NAME]
+            and args == expected.get(EvalConstants.TOOL_INPUT, {})
+        ):
+          # pop the matched entry so we don't rematch again
+          eval_dataset.pop(index)
+          return {"result": expected[EvalConstants.MOCK_TOOL_OUTPUT]}
+
+    return None
+
+  @staticmethod
+  def apply_before_tool_callback(
+      agent: BaseAgent,
+      callback: BeforeToolCallback,
+      all_mock_tools: set[str],
+  ):
+    """Recursively apply the before_tool_callback to the root agent and all its subagents."""
+    # check if the agent has tools that defined by evalset
+    # We use function name to check if tools match
+    if not isinstance(agent, Agent) and not isinstance(agent, LlmAgent):
+      return
+
+    for tool in agent.canonical_tools:
+      tool_name = tool.name
+      if tool_name in all_mock_tools:
+        agent.before_tool_callback = callback
+
+    # Apply recursively to subagents if they exist
+    for sub_agent in agent.sub_agents:
+      EvaluationGenerator.apply_before_tool_callback(
+          sub_agent, callback, all_mock_tools
+      )

google/adk/evaluation/response_evaluator.py

@@ -0,0 +1,135 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any
+
+import pandas as pd
+from tabulate import tabulate
+from vertexai.preview.evaluation import EvalTask
+from vertexai.preview.evaluation import MetricPromptTemplateExamples
+
+
+class ResponseEvaluator:
+  """Runs response evaluation for agents."""
+
+  @staticmethod
+  def evaluate(
+      raw_eval_dataset: list[list[dict[str, Any]]],
+      evaluation_criteria: list[str],
+      *,
+      print_detailed_results: bool = False
+  ):
+    r"""Returns the value of requested evaluation metrics.
+
+    Args:
+      raw_eval_dataset: The dataset that will be evaluated.
+      evaluation_criteria: The evaluation criteria to be used. This method
+        support two criterias, `response_evaluation_score` and
+        `response_match_score`.
+      print_detailed_results: Prints detailed results on the console. This is
+        usually helpful during debugging.
+
+    A note on evaluation_criteria:
+      `response_match_score`: This metric compares the agents final natural
+        language reponse with the expected final response, stored in the
+        "reference" field in test/eval files. We use Rouge metric to compare the
+        two responses.
+
+        Value Range: [0, 1]. A score closer to 0 means poor similarity between
+          response and reference. A score closer to 1 means strong similarity
+          between response and reference.
+
+      `response_evaluation_score`: Uses LLM to evalaute coherence of the
+        response, including tool use. This is pointwise metric.
+
+        Value range: [0, 5], where 0 means that the agent's response is not
+        coherent, while 5 means it is . High values are good.
+    A note on raw_eval_dataset:
+      The dataset should be a list session, where each sesssion is represented
+      as a list of interaction that need evaluation. Each evaluation is
+      represented as a dictionary that is expected to have values for the
+      following keys:
+
+        1) query
+        2) response
+        3) acutal_tool_use
+        4) expected_tool_use
+        5) reference
+
+      Here is a sample eval_dataset value with one entry:
+      [
+        [
+          {
+            "query": "roll a die for me",
+            "response": "I rolled a 16 sided die and got 13.\n",
+            "expected_tool_use": [
+              {
+                "tool_name": "roll_die",
+                "tool_input": {
+                  "sides": 16
+                }
+              }
+            ],
+            "acutal_tool_use": [
+              {
+                "tool_name": "roll_die",
+                "tool_input": {
+                  "sides": 16
+                }
+              }
+            ],
+            "reference": "I rolled a 16 sided die and got 13.\n"
+          }
+        ]
+      ]
+    """
+    if not raw_eval_dataset:
+      raise ValueError("The evaluation dataset is empty.")
+
+    metrics = ResponseEvaluator._get_metrics(
+        raw_eval_dataset, evaluation_criteria
+    )
+    flattened_queries = [
+        item for sublist in raw_eval_dataset for item in sublist
+    ]
+    eval_dataset = pd.DataFrame(flattened_queries).rename(
+        columns={"query": "prompt", "expected_tool_use": "reference_trajectory"}
+    )
+    eval_task = EvalTask(dataset=eval_dataset, metrics=metrics)
+
+    eval_result = eval_task.evaluate()
+    if print_detailed_results:
+      ResponseEvaluator._print_results(eval_result)
+    return eval_result.summary_metrics
+
+  @staticmethod
+  def _get_metrics(raw_eval_dataset, criteria):
+    metrics = []
+    if (
+        "response_evaluation_score" in criteria
+        and "query" in raw_eval_dataset[0][0]
+        and "expected_tool_use" in raw_eval_dataset[0][0]
+    ):
+      metrics.append(MetricPromptTemplateExamples.Pointwise.COHERENCE)
+    if (
+        "response_match_score" in criteria
+        and "reference" in raw_eval_dataset[0][0]
+    ):
+      metrics.append("rouge_1")
+    return metrics
+
+  @staticmethod
+  def _print_results(eval_result):
+    print("Evaluation Summary Metrics:", eval_result.summary_metrics)
+    print(tabulate(eval_result.metrics_table, headers="keys", tablefmt="grid"))

google/adk/evaluation/trajectory_evaluator.py

@@ -0,0 +1,184 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any
+
+import pandas as pd
+from tabulate import tabulate
+
+from .evaluation_constants import EvalConstants
+
+
+class TrajectoryEvaluator:
+  """Evaluates tool use trajectories for accuracy."""
+
+  @staticmethod
+  def evaluate(
+      eval_dataset: list[list[dict[str, Any]]],
+      *,
+      print_detailed_results: bool = False,
+  ):
+    r"""Returns the mean tool use accuracy of the eval dataset.
+
+    Tool use accuracy is calculated by comparing the expected and actuall tool
+    use trajectories. An exact match scores a 1, 0 otherwise. The final number
+    is an
+    average of these individual scores.
+
+    Value range: [0, 1], where 0 is means none of the too use entries aligned,
+    and 1 would mean all of them aligned. Higher value is good.
+
+    Args:
+      eval_dataset: The dataset that will be evaluated.
+      print_detailed_results: Prints detailed results on the console. This is
+        usually helpful during debugging.
+
+    A note on eval_dataset:
+      The dataset should be a list session, where each sesssion is represented
+      as a list of interaction that need evaluation. Each evaluation is
+      represented as a dictionary that is expected to have values for the
+      following keys:
+        1) query
+        2) response
+        3) acutal_tool_use
+        4) expected_tool_use
+
+      Here is a sample eval_dataset value with one entry:
+
+      [
+        [
+          {
+            "query": "Roll a 16 sided dice for me",
+            "response": "I rolled a 16 sided die and got 13.\n",
+            "expected_tool_use": [
+              {
+                "tool_name": "roll_die",
+                "tool_input": {
+                  "sides": 16
+                }
+              }
+            ],
+            "acutal_tool_use": [
+              {
+                "tool_name": "roll_die",
+                "tool_input": {
+                  "sides": 16
+                }
+              }
+            ]
+          }
+        ]
+      ]
+    """
+    if not eval_dataset:
+      raise ValueError("The evaluation dataset is empty.")
+
+    results_df = pd.DataFrame(
+        columns=[
+            "query",
+            "response",
+            "actual_tool_use",
+            "expected_tool_use",
+            "tool_use_accuracy",
+        ]
+    )
+    failures = []
+
+    for conversation in eval_dataset:
+      for index, row in enumerate(conversation):
+        new_row, failure = TrajectoryEvaluator._evaluate_row(row)
+        results_df = pd.concat(
+            [results_df, pd.DataFrame([new_row])], ignore_index=True
+        )
+        if failure:
+          failure["turn"] = index + 1
+          failures.append(failure)
+
+    TrajectoryEvaluator._report_failures(failures)
+
+    if print_detailed_results:
+      TrajectoryEvaluator._print_results(results_df)
+
+    return results_df["tool_use_accuracy"].mean()
+
+  @staticmethod
+  def _evaluate_row(row):
+    # We don't evaluate the mock tool outputs.
+    expected = TrajectoryEvaluator._remove_tool_outputs(
+        row["expected_tool_use"]
+    )
+    actual = row["actual_tool_use"]
+    tool_use_accuracy = (
+        1.0 if TrajectoryEvaluator.are_tools_equal(actual, expected) else 0.0
+    )
+
+    new_row = {
+        "query": row["query"],
+        "response": row["response"],
+        "actual_tool_use": actual,
+        "expected_tool_use": expected,
+        "tool_use_accuracy": tool_use_accuracy,
+    }
+    failure = (
+        None
+        if tool_use_accuracy == 1.0
+        else {"query": row["query"], "actual": actual, "expected": expected}
+    )
+    return new_row, failure
+
+  @staticmethod
+  def are_tools_equal(list_a_original, list_b_original):
+    # Remove other entries that we don't want to evaluate
+    list_a = [
+        {"tool_name": tool["tool_name"], "tool_input": tool["tool_input"]}
+        for tool in list_a_original
+    ]
+
+    list_b = [
+        {"tool_name": tool["tool_name"], "tool_input": tool["tool_input"]}
+        for tool in list_b_original
+    ]
+
+    return list_a == list_b
+
+  @staticmethod
+  def _remove_tool_outputs(tool_use_list):
+    """Removes 'mock_tool_output' from each dictionary in the list."""
+    result = []
+    for tool_use in tool_use_list:
+      new_tool_use = (
+          tool_use.copy()
+      )  # Create a copy to avoid modifying the original
+      new_tool_use.pop(
+          EvalConstants.MOCK_TOOL_OUTPUT, None
+      )  # Remove 'tool_output' if it exists
+      result.append(new_tool_use)
+    return result
+
+  @staticmethod
+  def _report_failures(failures):
+    if failures:
+      print("Failures:")
+      for failure in failures:
+        print(f"""{{
+  "turn": {failure["turn"]},
+  "query": '{failure["query"]}',
+  "actual": {failure["actual"]},
+  "expected_tool_use": {failure["expected"]},
+}}
+""")
+
+  @staticmethod
+  def _print_results(results_df):
+    print(tabulate(results_df, headers="keys", tablefmt="grid"))

google/adk/events/__init__.py

@@ -0,0 +1,21 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .event import Event
+from .event_actions import EventActions
+
+__all__ = [
+    'Event',
+    'EventActions',
+]