google-adk 0.0.1__py3-none-any.whl → 0.0.2__py3-none-any.whl

google/adk/code_executors/unsafe_local_code_executor.py

@@ -0,0 +1,71 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from contextlib import redirect_stdout
+import io
+
+from pydantic import Field
+from typing_extensions import override
+
+from ..agents.invocation_context import InvocationContext
+from .base_code_executor import BaseCodeExecutor
+from .code_execution_utils import CodeExecutionInput
+from .code_execution_utils import CodeExecutionResult
+
+
+class UnsafeLocalCodeExecutor(BaseCodeExecutor):
+  """A code executor that unsafely execute code in the current local context."""
+
+  # Overrides the BaseCodeExecutor attribute: this executor cannot be stateful.
+  stateful: bool = Field(default=False, frozen=True, exclude=True)
+
+  # Overrides the BaseCodeExecutor attribute: this executor cannot
+  # optimize_data_file.
+  optimize_data_file: bool = Field(default=False, frozen=True, exclude=True)
+
+  def __init__(self, **data):
+    """Initializes the UnsafeLocalCodeExecutor."""
+    if 'stateful' in data and data['stateful']:
+      raise ValueError('Cannot set `stateful=True` in UnsafeLocalCodeExecutor.')
+    if 'optimize_data_file' in data and data['optimize_data_file']:
+      raise ValueError(
+          'Cannot set `optimize_data_file=True` in UnsafeLocalCodeExecutor.'
+      )
+    super().__init__(**data)
+
+  @override
+  def execute_code(
+      self,
+      invocation_context: InvocationContext,
+      code_execution_input: CodeExecutionInput,
+  ) -> CodeExecutionResult:
+    # Execute the code.
+    output = ''
+    error = ''
+    try:
+      globals_ = {}
+      locals_ = {}
+      stdout = io.StringIO()
+      with redirect_stdout(stdout):
+        exec(code_execution_input.code, globals_, locals_)
+      output = stdout.getvalue()
+    except Exception as e:
+      error = str(e)
+
+    # Collect the final result.
+    return CodeExecutionResult(
+        stdout=output,
+        stderr=error,
+        output_files=[],
+    )

google/adk/code_executors/vertex_ai_code_executor.py

@@ -0,0 +1,234 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import datetime
+import mimetypes
+import os
+from typing import Any, Optional
+
+from typing_extensions import override
+from vertexai.preview.extensions import Extension
+
+from ..agents.invocation_context import InvocationContext
+from .base_code_executor import BaseCodeExecutor
+from .code_execution_utils import CodeExecutionInput
+from .code_execution_utils import CodeExecutionResult
+from .code_execution_utils import File
+
+_SUPPORTED_IMAGE_TYPES = ['png', 'jpg', 'jpeg']
+_SUPPORTED_DATA_FILE_TYPES = ['csv']
+
+_IMPORTED_LIBRARIES = '''
+import io
+import math
+import re
+
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+import scipy
+
+def crop(s: str, max_chars: int = 64) -> str:
+  """Crops a string to max_chars characters."""
+  return s[: max_chars - 3] + '...' if len(s) > max_chars else s
+
+
+def explore_df(df: pd.DataFrame) -> None:
+  """Prints some information about a pandas DataFrame."""
+
+  with pd.option_context(
+      'display.max_columns', None, 'display.expand_frame_repr', False
+  ):
+    # Print the column names to never encounter KeyError when selecting one.
+    df_dtypes = df.dtypes
+
+    # Obtain information about data types and missing values.
+    df_nulls = (len(df) - df.isnull().sum()).apply(
+        lambda x: f'{x} / {df.shape[0]} non-null'
+    )
+
+    # Explore unique total values in columns using `.unique()`.
+    df_unique_count = df.apply(lambda x: len(x.unique()))
+
+    # Explore unique values in columns using `.unique()`.
+    df_unique = df.apply(lambda x: crop(str(list(x.unique()))))
+
+    df_info = pd.concat(
+        (
+            df_dtypes.rename('Dtype'),
+            df_nulls.rename('Non-Null Count'),
+            df_unique_count.rename('Unique Values Count'),
+            df_unique.rename('Unique Values'),
+        ),
+        axis=1,
+    )
+    df_info.index.name = 'Columns'
+    print(f"""Total rows: {df.shape[0]}
+Total columns: {df.shape[1]}
+
+{df_info}""")
+'''
+
+
+def _get_code_interpreter_extension(resource_name: str = None):
+  """Returns: Load or create the code interpreter extension."""
+  if not resource_name:
+    resource_name = os.environ.get('CODE_INTERPRETER_EXTENSION_NAME')
+  if resource_name:
+    new_code_interpreter = Extension(resource_name)
+  else:
+    print('No CODE_INTERPRETER_ID found in the environment. Create a new one.')
+    new_code_interpreter = Extension.from_hub('code_interpreter')
+    os.environ['CODE_INTERPRETER_EXTENSION_NAME'] = (
+        new_code_interpreter.gca_resource.name
+    )
+  return new_code_interpreter
+
+
+class VertexAiCodeExecutor(BaseCodeExecutor):
+  """A code executor that uses Vertex Code Interpreter Extension to execute code.
+
+  Attributes:
+    resource_name: If set, load the existing resource name of the code
+      interpreter extension instead of creating a new one. Format:
+      projects/123/locations/us-central1/extensions/456
+  """
+
+  resource_name: str = None
+  """
+  If set, load the existing resource name of the code interpreter extension
+  instead of creating a new one.
+  Format: projects/123/locations/us-central1/extensions/456
+  """
+
+  _code_interpreter_extension: Extension
+
+  def __init__(
+      self,
+      resource_name: str = None,
+      **data,
+  ):
+    """Initializes the VertexAiCodeExecutor.
+
+    Args:
+      resource_name: If set, load the existing resource name of the code
+        interpreter extension instead of creating a new one. Format:
+        projects/123/locations/us-central1/extensions/456
+      **data: Additional keyword arguments to be passed to the base class.
+    """
+    super().__init__(**data)
+    self.resource_name = resource_name
+    self._code_interpreter_extension = _get_code_interpreter_extension(
+        self.resource_name
+    )
+
+  @override
+  def execute_code(
+      self,
+      invocation_context: InvocationContext,
+      code_execution_input: CodeExecutionInput,
+  ) -> CodeExecutionResult:
+    # Execute the code.
+    code_execution_result = self._execute_code_interpreter(
+        self._get_code_with_imports(code_execution_input.code),
+        code_execution_input.input_files,
+        code_execution_input.execution_id,
+    )
+
+    # Save output file as artifacts.
+    current_timestamp = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
+    file_name_prefix = '%s_' % str(current_timestamp)
+    saved_files = []
+    file_count = 0
+    for output_file in code_execution_result['output_files']:
+      file_type = output_file['name'].split('.')[-1]
+      file_name = file_name_prefix + '%d.%s' % (file_count, file_type)
+      if file_type in _SUPPORTED_IMAGE_TYPES:
+        file_count += 1
+        saved_files.append(
+            File(
+                name='plot_' + file_name,
+                content=output_file['contents'],
+                mime_type=f'image/{file_type}',
+            )
+        )
+      elif file_type in _SUPPORTED_DATA_FILE_TYPES:
+        file_count += 1
+        saved_files.append(
+            File(
+                name='data_' + file_name,
+                content=output_file['contents'],
+                mime_type=f'text/{file_type}',
+            )
+        )
+      else:
+        mime_type, _ = mimetypes.guess_type(file_name)
+        saved_files.append(
+            File(
+                name=file_name,
+                content=output_file['contents'],
+                mime_type=mime_type,
+            )
+        )
+
+    # Collect the final result.
+    return CodeExecutionResult(
+        stdout=code_execution_result.get('execution_result', ''),
+        stderr=code_execution_result.get('execution_error', ''),
+        output_files=saved_files,
+    )
+
+  def _execute_code_interpreter(
+      self,
+      code: str,
+      input_files: Optional[list[File]] = None,
+      session_id: Optional[str] = None,
+  ) -> dict[str, Any]:
+    """Executes the code interpreter extension.
+
+    Args:
+      code: The code to execute.
+      input_files: The input files to execute the code with.
+      session_id: The session ID to execute the code with.
+
+    Returns:
+      The response from the code interpreter extension.
+    """
+    operation_params = {'code': code}
+    if input_files:
+      operation_params['files'] = [
+          {'name': f.name, 'contents': f.content} for f in input_files
+      ]
+    if session_id:
+      operation_params['session_id'] = session_id
+    response = self._code_interpreter_extension.execute(
+        operation_id='execute',
+        operation_params=operation_params,
+    )
+    return response
+
+  def _get_code_with_imports(self, code: str) -> str:
+    """Builds the code string with built-in imports.
+
+    Args:
+      code: The code to execute.
+
+    Returns:
+      The code string with built-in imports.
+    """
+    return f"""
+{_IMPORTED_LIBRARIES}
+
+{code}
+"""

google/adk/evaluation/__init__.py

@@ -0,0 +1,31 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+__all__ = []
+
+try:
+  from .agent_evaluator import AgentEvaluator
+
+  __all__.append('AgentEvaluator')
+except ImportError:
+  logger.debug(
+      'The Vertex[eval] sdk is not installed. If you want to use the Vertex'
+      ' Evaluation with agents, please install it(pip install'
+      ' "google-cloud-aiplatform[evaluation]). If not, you can ignore this'
+      ' warning.'
+  )

google/adk/evaluation/agent_evaluator.py

@@ -0,0 +1,329 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+from os import path
+from typing import Dict
+from typing import List
+from typing import Union
+
+from .evaluation_generator import EvaluationGenerator
+from .response_evaluator import ResponseEvaluator
+from .trajectory_evaluator import TrajectoryEvaluator
+
+# Constants for default runs and evaluation criteria
+NUM_RUNS = 2
+TOOL_TRAJECTORY_SCORE_KEY = "tool_trajectory_avg_score"
+# This evaluation is not very stable.
+# This is always optional unless explicitly specified.
+RESPONSE_EVALUATION_SCORE_KEY = "response_evaluation_score"
+RESPONSE_MATCH_SCORE_KEY = "response_match_score"
+
+ALLOWED_CRITERIA = [
+    TOOL_TRAJECTORY_SCORE_KEY,
+    RESPONSE_EVALUATION_SCORE_KEY,
+    RESPONSE_MATCH_SCORE_KEY,
+]
+
+
+QUERY_COLUMN = "query"
+REFERENCE_COLUMN = "reference"
+EXPECTED_TOOL_USE_COLUMN = "expected_tool_use"
+
+
+DEFAULT_CRITERIA = {
+    TOOL_TRAJECTORY_SCORE_KEY: 1.0,  # 1-point scale; 1.0 is perfect.
+    RESPONSE_MATCH_SCORE_KEY: 0.8,  # Rouge-1 text match; 0.8 is default.
+}
+
+
+def load_json(file_path: str) -> Union[Dict, List]:
+  with open(file_path, "r") as f:
+    return json.load(f)
+
+
+class AgentEvaluator:
+  """An evaluator for Agents, mainly intented for helping with test cases."""
+
+  @staticmethod
+  def find_config_for_test_file(test_file: str):
+    """Find the test_config.json file in the same folder as the test file."""
+    test_folder = os.path.dirname(test_file)
+    config_path = os.path.join(test_folder, "test_config.json")
+    if os.path.exists(config_path):
+      config_data = load_json(config_path)
+      if "criteria" in config_data and isinstance(
+          config_data["criteria"], dict
+      ):
+        return config_data["criteria"]
+      else:
+        raise ValueError(
+            f"Invalid format for test_config.json at {config_path}. Expected a"
+            " 'criteria' dictionary."
+        )
+    return DEFAULT_CRITERIA
+
+  @staticmethod
+  def evaluate(
+      agent_module,
+      eval_dataset_file_path_or_dir,
+      num_runs=NUM_RUNS,
+      agent_name=None,
+      initial_session_file=None,
+  ):
+    """Evaluates an Agent given eval data.
+
+    Args:
+      agent_module: The path to python module that contains the definition of
+        the agent. There is convention in place here, where the code is going to
+        look for 'root_agent' in the loaded module.
+      eval_dataset: The eval data set. This can be either a string representing
+        full path to the file containing eval dataset, or a directory that is
+        recusively explored for all files that have a `.test.json` suffix.
+      num_runs: Number of times all entries in the eval dataset should be
+        assessed.
+      agent_name: The name of the agent.
+      initial_session_file: File that contains initial session state that is
+        needed by all the evals in the eval dataset.
+    """
+    test_files = []
+    if isinstance(eval_dataset_file_path_or_dir, str) and os.path.isdir(
+        eval_dataset_file_path_or_dir
+    ):
+      for root, _, files in os.walk(eval_dataset_file_path_or_dir):
+        for file in files:
+          if file.endswith(".test.json"):
+            test_files.append(path.join(root, file))
+    else:
+      test_files = [eval_dataset_file_path_or_dir]
+
+    initial_session_state = {}
+    if initial_session_file:
+      with open(initial_session_file, "r") as f:
+        initial_session_state = json.loads(f.read())["state"]
+
+    for test_file in test_files:
+      dataset = AgentEvaluator._load_dataset(test_file)[0]
+      criteria = AgentEvaluator.find_config_for_test_file(test_file)
+
+      AgentEvaluator._validate_input([dataset], criteria)
+
+      evaluation_response = AgentEvaluator._generate_responses(
+          agent_module,
+          [dataset],
+          num_runs,
+          agent_name=agent_name,
+          initial_session={"state": initial_session_state},
+      )
+
+      if AgentEvaluator._response_evaluation_required(criteria, [dataset]):
+        AgentEvaluator._evaluate_response_scores(
+            agent_module, evaluation_response, criteria
+        )
+
+      if AgentEvaluator._trajectory_evaluation_required(criteria, [dataset]):
+        AgentEvaluator._evaluate_tool_trajectory(
+            agent_module, evaluation_response, criteria
+        )
+
+  @staticmethod
+  def _load_dataset(
+      input_data: Union[str, List[str], List[Dict], List[List[Dict]]],
+  ) -> List[List[Dict]]:
+    def load_json_file(file_path: str) -> List[Dict]:
+      data = load_json(file_path)
+      if not isinstance(data, list) or not all(
+          isinstance(d, dict) for d in data
+      ):
+        raise ValueError(f"{file_path} must contain a list of dictionaries.")
+      return data
+
+    if isinstance(input_data, str):
+      if os.path.isdir(input_data):
+        test_files = []
+        for root, _, files in os.walk(input_data):
+          for file in files:
+            if file.endswith(".test.json"):
+              test_files.append(os.path.join(root, file))
+        return [load_json_file(f) for f in test_files]
+      elif os.path.isfile(input_data):
+        return [load_json_file(input_data)]
+      else:
+        raise ValueError(f"Input path {input_data} is invalid.")
+    elif isinstance(input_data, list):
+      if all(isinstance(i, str) and os.path.isfile(i) for i in input_data):
+        return [load_json_file(i) for i in input_data]
+      raise TypeError("Input list must contain valid file paths.")
+    raise TypeError("Invalid input type for dataset loading.")
+
+  @staticmethod
+  def _validate_input(eval_dataset, criteria):
+    """Validates that the evaluation criteria align with the provided dataset.
+
+    For efficiency, we only use first row to validate input.
+    """
+    if not eval_dataset:
+      raise ValueError("The evaluation dataset is None or empty.")
+
+    for key in criteria:
+      if key not in ALLOWED_CRITERIA:
+        raise ValueError(
+            f"Invalid criteria key: {key}. Expected one of {ALLOWED_CRITERIA}."
+        )
+
+    if not eval_dataset:
+      raise ValueError("The evaluation dataset is empty.")
+    sample = eval_dataset[0]
+    first_query = sample[0]
+
+    if not isinstance(sample, list) and not isinstance(first_query, dict):
+      raise ValueError(
+          "Each evaluation dataset sample must be list of dictionary. But it's"
+          f" {eval_dataset}"
+      )
+
+    if TOOL_TRAJECTORY_SCORE_KEY in criteria:
+      if (
+          QUERY_COLUMN not in first_query
+          or EXPECTED_TOOL_USE_COLUMN not in first_query
+      ):
+        raise ValueError(
+            f"Samples for {TOOL_TRAJECTORY_SCORE_KEY} must include"
+            f" '{QUERY_COLUMN}' and '{EXPECTED_TOOL_USE_COLUMN}' keys. The"
+            f" sample is {sample}."
+        )
+
+    if RESPONSE_EVALUATION_SCORE_KEY in criteria:
+      if QUERY_COLUMN not in first_query:
+        raise ValueError(
+            f"Samples for {RESPONSE_EVALUATION_SCORE_KEY} must include"
+            f" '{QUERY_COLUMN}' key. The sample is {sample}."
+        )
+
+    if RESPONSE_MATCH_SCORE_KEY in criteria:
+      if QUERY_COLUMN not in first_query or REFERENCE_COLUMN not in first_query:
+        raise ValueError(
+            f"Samples for {RESPONSE_MATCH_SCORE_KEY} must include"
+            f" '{QUERY_COLUMN}' and '{REFERENCE_COLUMN}' keys. The sample is"
+            f" {sample}."
+        )
+
+  @staticmethod
+  def _get_infer_criteria(eval_dataset):
+    """Infers evaluation criteria based on the provided dataset.
+
+    Args:
+        eval_dataset (list): A list of evaluation samples.
+
+    Returns:
+        dict: Inferred evaluation criteria based on dataset fields.
+    """
+    inferred_criteria = {}
+    sample = eval_dataset[0][0]
+
+    if QUERY_COLUMN in sample and EXPECTED_TOOL_USE_COLUMN in sample:
+      inferred_criteria[TOOL_TRAJECTORY_SCORE_KEY] = DEFAULT_CRITERIA[
+          TOOL_TRAJECTORY_SCORE_KEY
+      ]
+
+    if QUERY_COLUMN in sample and REFERENCE_COLUMN in sample:
+      inferred_criteria[RESPONSE_MATCH_SCORE_KEY] = DEFAULT_CRITERIA[
+          RESPONSE_MATCH_SCORE_KEY
+      ]
+
+    return inferred_criteria
+
+  @staticmethod
+  def _generate_responses(
+      agent_module, eval_dataset, num_runs, agent_name=None, initial_session={}
+  ):
+    """Generates evaluation responses by running the agent module multiple times."""
+    return EvaluationGenerator.generate_responses(
+        eval_dataset,
+        agent_module,
+        repeat_num=num_runs,
+        agent_name=agent_name,
+        initial_session=initial_session,
+    )
+
+  @staticmethod
+  def _generate_responses_from_session(eval_dataset, session_path):
+    """Generates evaluation responses by running the agent module multiple times."""
+    return EvaluationGenerator.generate_responses_from_session(
+        session_path, eval_dataset
+    )
+
+  @staticmethod
+  def _response_evaluation_required(criteria, eval_dataset):
+    """Checks if response evaluation are needed."""
+    return REFERENCE_COLUMN in eval_dataset[0][0] and any(
+        key in criteria
+        for key in [RESPONSE_EVALUATION_SCORE_KEY, RESPONSE_MATCH_SCORE_KEY]
+    )
+
+  @staticmethod
+  def _trajectory_evaluation_required(evaluation_criteria, eval_dataset):
+    """Checks if response evaluation are needed."""
+    return (
+        EXPECTED_TOOL_USE_COLUMN in eval_dataset[0][0]
+        and TOOL_TRAJECTORY_SCORE_KEY in evaluation_criteria
+    )
+
+  @staticmethod
+  def _evaluate_response_scores(agent_module, evaluation_response, criteria):
+    """Evaluates response scores and raises an assertion error if they don't meet the criteria."""
+    metrics = ResponseEvaluator.evaluate(
+        evaluation_response, criteria, print_detailed_results=True
+    )
+
+    AgentEvaluator._assert_score(
+        metrics,
+        "coherence/mean",
+        criteria.get(RESPONSE_EVALUATION_SCORE_KEY),
+        "Average response evaluation score",
+        agent_module,
+    )
+
+    AgentEvaluator._assert_score(
+        metrics,
+        "rouge_1/mean",
+        criteria.get(RESPONSE_MATCH_SCORE_KEY),
+        "Average response match score",
+        agent_module,
+    )
+
+  @staticmethod
+  def _evaluate_tool_trajectory(agent_module, evaluation_response, criteria):
+    """Evaluates tool trajectory scores and raises an assertion error if they don't meet the criteria."""
+    score = TrajectoryEvaluator.evaluate(
+        evaluation_response, print_detailed_results=True
+    )
+    AgentEvaluator._assert_score(
+        {TOOL_TRAJECTORY_SCORE_KEY: score},
+        TOOL_TRAJECTORY_SCORE_KEY,
+        criteria[TOOL_TRAJECTORY_SCORE_KEY],
+        "Average tool trajectory evaluation score",
+        agent_module,
+    )
+
+  @staticmethod
+  def _assert_score(metrics, metric_key, threshold, description, agent_module):
+    """Asserts that a metric meets the specified threshold."""
+    if metric_key in metrics:
+      actual_score = metrics[metric_key]
+      assert actual_score >= threshold, (
+          f"{description} for {agent_module} is lower than expected. "
+          f"Expected >= {threshold}, but got {actual_score}."
+      )

google/adk/evaluation/evaluation_constants.py

@@ -0,0 +1,24 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class EvalConstants:
+  """Holds constants for evaluation file constants."""
+
+  QUERY = "query"
+  EXPECTED_TOOL_USE = "expected_tool_use"
+  RESPONSE = "response"
+  REFERENCE = "reference"
+  TOOL_NAME = "tool_name"
+  TOOL_INPUT = "tool_input"
+  MOCK_TOOL_OUTPUT = "mock_tool_output"