google-adk 0.5.0__py3-none-any.whl → 1.0.0__py3-none-any.whl

google/adk/cli/cli.py

@@ -55,7 +55,7 @@ async def run_input_file(
     input_file = InputFile.model_validate_json(f.read())
   input_file.state['_time'] = datetime.now()
 
-  session = session_service.create_session(
+  session = await session_service.create_session(
       app_name=app_name, user_id=user_id, state=input_file.state
   )
   for query in input_file.queries:
@@ -105,6 +105,7 @@ async def run_cli(
     input_file: Optional[str] = None,
     saved_session_file: Optional[str] = None,
     save_session: bool,
+    session_id: Optional[str] = None,
 ) -> None:
   """Runs an interactive CLI for a certain agent.
 
@@ -118,6 +119,7 @@ async def run_cli(
     saved_session_file: Optional[str], the absolute path to the json file that
       contains a previously saved session, exclusive with input_file.
     save_session: bool, whether to save the session on exit.
+    session_id: Optional[str], the session ID to save the session to on exit.
   """
   if agent_parent_dir not in sys.path:
     sys.path.append(agent_parent_dir)
@@ -128,7 +130,7 @@ async def run_cli(
   agent_module_path = os.path.join(agent_parent_dir, agent_folder_name)
   agent_module = importlib.import_module(agent_folder_name)
   user_id = 'test_user'
-  session = session_service.create_session(
+  session = await session_service.create_session(
       app_name=agent_folder_name, user_id=user_id
   )
   root_agent = agent_module.agent.root_agent
@@ -143,14 +145,12 @@ async def run_cli(
         input_path=input_file,
     )
   elif saved_session_file:
-
-    loaded_session = None
     with open(saved_session_file, 'r') as f:
       loaded_session = Session.model_validate_json(f.read())
 
     if loaded_session:
       for event in loaded_session.events:
-        session_service.append_event(session, event)
+        await session_service.append_event(session, event)
         content = event.content
         if not content or not content.parts or not content.parts[0].text:
           continue
@@ -175,11 +175,11 @@ async def run_cli(
     )
 
   if save_session:
-    session_id = input('Session ID to save: ')
+    session_id = session_id or input('Session ID to save: ')
     session_path = f'{agent_module_path}/{session_id}.session.json'
 
     # Fetch the session again to get all the details.
-    session = session_service.get_session(
+    session = await session_service.get_session(
         app_name=session.app_name,
         user_id=session.user_id,
         session_id=session.id,

google/adk/cli/cli_deploy.py

@@ -42,7 +42,7 @@ ENV GOOGLE_CLOUD_LOCATION={gcp_region}
 # Set up environment variables - End
 
 # Install ADK - Start
-RUN pip install google-adk
+RUN pip install google-adk=={adk_version}
 # Install ADK - End
 
 # Copy agent - Start
@@ -54,7 +54,7 @@ COPY "agents/{app_name}/" "/app/agents/{app_name}/"
 
 EXPOSE {port}
 
-CMD adk {command} --port={port} {session_db_option} {trace_to_cloud_option} "/app/agents"
+CMD adk {command} --port={port} {host_option} {session_db_option} {trace_to_cloud_option} "/app/agents"
 """
 
 
@@ -86,6 +86,7 @@ def to_cloud_run(
     with_ui: bool,
     verbosity: str,
     session_db_url: str,
+    adk_version: str,
 ):
   """Deploys an agent to Google Cloud Run.
 
@@ -114,6 +115,7 @@ def to_cloud_run(
     with_ui: Whether to deploy with UI.
     verbosity: The verbosity level of the CLI.
     session_db_url: The database URL to connect the session.
+    adk_version: The ADK version to use in Cloud Run.
   """
   app_name = app_name or os.path.basename(agent_folder)
 
@@ -139,6 +141,7 @@ def to_cloud_run(
 
     # create Dockerfile
     click.echo('Creating Dockerfile...')
+    host_option = '--host=0.0.0.0' if adk_version > '0.5.0' else ''
     dockerfile_content = _DOCKERFILE_TEMPLATE.format(
         gcp_project_id=project,
         gcp_region=region,
@@ -150,6 +153,8 @@ def to_cloud_run(
         if session_db_url
         else '',
         trace_to_cloud_option='--trace_to_cloud' if trace_to_cloud else '',
+        adk_version=adk_version,
+        host_option=host_option,
     )
     dockerfile_path = os.path.join(temp_folder, 'Dockerfile')
     os.makedirs(temp_folder, exist_ok=True)

google/adk/cli/cli_eval.py

@@ -12,47 +12,107 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from enum import Enum
 import importlib.util
 import json
 import logging
 import os
 import sys
-import traceback
 from typing import Any
 from typing import AsyncGenerator
 from typing import Optional
 import uuid
 
 from pydantic import BaseModel
+from pydantic import Field
 
 from ..agents import Agent
+from ..artifacts.base_artifact_service import BaseArtifactService
+from ..evaluation.eval_case import EvalCase
+from ..evaluation.eval_case import Invocation
+from ..evaluation.evaluator import EvalStatus
+from ..evaluation.evaluator import Evaluator
+from ..sessions.base_session_service import BaseSessionService
+from ..sessions.session import Session
+from .utils import common
 
-logger = logging.getLogger(__name__)
+logger = logging.getLogger("google_adk." + __name__)
 
 
-class EvalStatus(Enum):
-  PASSED = 1
-  FAILED = 2
-  NOT_EVALUATED = 3
+class EvalMetric(common.BaseModel):
+  """A metric used to evaluate a particular aspect of an eval case."""
 
-
-class EvalMetric(BaseModel):
   metric_name: str
+  """The name of the metric."""
+
   threshold: float
+  """A threshold value. Each metric decides how to interpret this threshold."""
+
 
+class EvalMetricResult(EvalMetric):
+  """The actual computed score/value of a particular EvalMetric."""
 
-class EvalMetricResult(BaseModel):
-  score: Optional[float]
+  score: Optional[float] = None
   eval_status: EvalStatus
 
 
-class EvalResult(BaseModel):
-  eval_set_file: str
-  eval_id: str
+class EvalMetricResultPerInvocation(common.BaseModel):
+  """Eval metric results per invocation."""
+
+  actual_invocation: Invocation
+  """The actual invocation, usually obtained by inferencing the agent."""
+
+  expected_invocation: Invocation
+  """The expected invocation, usually the reference or golden invocation."""
+
+  eval_metric_results: list[EvalMetricResult] = []
+  """Eval resutls for each applicable metric."""
+
+
+class EvalCaseResult(common.BaseModel):
+  """Case-level evaluation results."""
+
+  eval_set_file: str = Field(
+      deprecated=True,
+      description="This field is deprecated, use eval_set_id instead.",
+  )
+  eval_set_id: str = ""
+  """The eval set id."""
+
+  eval_id: str = ""
+  """The eval case id."""
+
   final_eval_status: EvalStatus
-  eval_metric_results: list[tuple[EvalMetric, EvalMetricResult]]
+  """Final evalu status for this eval case."""
+
+  eval_metric_results: list[tuple[EvalMetric, EvalMetricResult]] = Field(
+      deprecated=True,
+      description=(
+          "This field is deprecated, use overall_eval_metric_results instead."
+      ),
+  )
+
+  overall_eval_metric_results: list[EvalMetricResult]
+  """Overall result for each metric for the entire eval case."""
+
+  eval_metric_result_per_invocation: list[EvalMetricResultPerInvocation]
+  """Result for each metric on a per invocation basis."""
+
   session_id: str
+  """Session id of the session generated as result of inferencing/scraping stage of the eval."""
+
+  session_details: Optional[Session] = None
+  """Session generated as result of inferencing/scraping stage of the eval."""
+
+  user_id: Optional[str] = None
+  """User id used during inferencing/scraping stage of the eval."""
+
+
+class EvalSetResult(common.BaseModel):
+  eval_set_result_id: str
+  eval_set_result_name: str
+  eval_set_id: str
+  eval_case_results: list[EvalCaseResult] = Field(default_factory=list)
+  creation_timestamp: float = 0.0
 
 
 MISSING_EVAL_DEPENDENCIES_MESSAGE = (
@@ -147,14 +207,25 @@ def parse_and_get_evals_to_run(
 
 
 async def run_evals(
-    eval_set_to_evals: dict[str, list[str]],
+    eval_cases_by_eval_set_id: dict[str, list[EvalCase]],
     root_agent: Agent,
     reset_func: Optional[Any],
     eval_metrics: list[EvalMetric],
-    session_service=None,
-    artifact_service=None,
-    print_detailed_results=False,
-) -> AsyncGenerator[EvalResult, None]:
+    session_service: Optional[BaseSessionService] = None,
+    artifact_service: Optional[BaseArtifactService] = None,
+) -> AsyncGenerator[EvalCaseResult, None]:
+  """Returns a stream of EvalCaseResult for each eval case that was evaluated.
+
+  Args:
+    eval_cases_by_eval_set_id: Eval cases categorized by eval set id to which
+      they belong.
+    root_agent: Agent to use for inferencing.
+    reset_func: If present, this will be called before invoking the agent before
+      every inferencing step.
+    eval_metrics: A list of metrics that should be used during evaluation.
+    session_service: The session service to use during inferencing.
+    artifact_service: The artifact service to use during inferencing.
+  """
   try:
     from ..evaluation.agent_evaluator import EvaluationGenerator
     from ..evaluation.response_evaluator import ResponseEvaluator
@@ -162,28 +233,19 @@ async def run_evals(
   except ModuleNotFoundError as e:
     raise ModuleNotFoundError(MISSING_EVAL_DEPENDENCIES_MESSAGE) from e
 
-  """Returns a summary of eval runs."""
-  for eval_set_file, evals_to_run in eval_set_to_evals.items():
-    with open(eval_set_file, "r", encoding="utf-8") as file:
-      eval_items = json.load(file)  # Load JSON into a list
-
-    assert eval_items, f"No eval data found in eval set file: {eval_set_file}"
-
-    for eval_item in eval_items:
-      eval_name = eval_item["name"]
-      eval_data = eval_item["data"]
-      initial_session = eval_item.get("initial_session", {})
-
-      if evals_to_run and eval_name not in evals_to_run:
-        continue
+  for eval_set_id, eval_cases in eval_cases_by_eval_set_id.items():
+    for eval_case in eval_cases:
+      eval_name = eval_case.eval_id
+      initial_session = eval_case.session_input
+      user_id = initial_session.user_id if initial_session else "test_user_id"
 
       try:
-        print(f"Running Eval: {eval_set_file}:{eval_name}")
+        print(f"Running Eval: {eval_set_id}:{eval_name}")
         session_id = f"{EVAL_SESSION_ID_PREFIX}{str(uuid.uuid4())}"
 
-        scrape_result = (
-            await EvaluationGenerator._process_query_with_root_agent(
-                data=eval_data,
+        inference_result = (
+            await EvaluationGenerator._generate_inferences_from_root_agent(
+                invocations=eval_case.conversation,
                 root_agent=root_agent,
                 reset_func=reset_func,
                 initial_session=initial_session,
@@ -193,68 +255,74 @@ async def run_evals(
             )
         )
 
-        eval_metric_results = []
+        # Initialize the per-invocation metric results to an empty list.
+        # We will fill this as we evaluate each metric.
+        eval_metric_result_per_invocation = []
+        for actual, expected in zip(inference_result, eval_case.conversation):
+          eval_metric_result_per_invocation.append(
+              EvalMetricResultPerInvocation(
+                  actual_invocation=actual,
+                  expected_invocation=expected,
+                  eval_metric_results=[],
+              )
+          )
+
+        overall_eval_metric_results = []
+
         for eval_metric in eval_metrics:
-          eval_metric_result = None
-          if eval_metric.metric_name == TOOL_TRAJECTORY_SCORE_KEY:
-            score = TrajectoryEvaluator.evaluate(
-                [scrape_result], print_detailed_results=print_detailed_results
+          metric_evaluator = _get_evaluator(eval_metric)
+
+          evaluation_result = metric_evaluator.evaluate_invocations(
+              actual_invocations=inference_result,
+              expected_invocations=eval_case.conversation,
+          )
+
+          overall_eval_metric_results.append(
+              EvalMetricResult(
+                  metric_name=eval_metric.metric_name,
+                  threshold=eval_metric.threshold,
+                  score=evaluation_result.overall_score,
+                  eval_status=evaluation_result.overall_eval_status,
+              )
+          )
+          for index, per_invocation_result in enumerate(
+              evaluation_result.per_invocation_results
+          ):
+            eval_metric_result_per_invocation[index].eval_metric_results.append(
+                EvalMetricResult(
+                    metric_name=eval_metric.metric_name,
+                    threshold=eval_metric.threshold,
+                    score=per_invocation_result.score,
+                    eval_status=per_invocation_result.eval_status,
+                )
             )
-            eval_metric_result = _get_eval_metric_result(eval_metric, score)
-          elif eval_metric.metric_name == RESPONSE_MATCH_SCORE_KEY:
-            score = ResponseEvaluator.evaluate(
-                [scrape_result],
-                [RESPONSE_MATCH_SCORE_KEY],
-                print_detailed_results=print_detailed_results,
-            )
-            eval_metric_result = _get_eval_metric_result(
-                eval_metric, score["rouge_1/mean"].item()
-            )
-          elif eval_metric.metric_name == RESPONSE_EVALUATION_SCORE_KEY:
-            score = ResponseEvaluator.evaluate(
-                [scrape_result],
-                [RESPONSE_EVALUATION_SCORE_KEY],
-                print_detailed_results=print_detailed_results,
-            )
-            eval_metric_result = _get_eval_metric_result(
-                eval_metric, score["coherence/mean"].item()
-            )
-          else:
-            logger.warning("`%s` is not supported.", eval_metric.metric_name)
-            eval_metric_results.append((
-                eval_metric,
-                EvalMetricResult(eval_status=EvalStatus.NOT_EVALUATED),
-            ))
-
-          eval_metric_results.append((
-              eval_metric,
-              eval_metric_result,
-          ))
-          _print_eval_metric_result(eval_metric, eval_metric_result)
 
         final_eval_status = EvalStatus.NOT_EVALUATED
-
         # Go over the all the eval statuses and mark the final eval status as
         # passed if all of them pass, otherwise mark the final eval status to
         # failed.
-        for eval_metric_result in eval_metric_results:
-          eval_status = eval_metric_result[1].eval_status
-          if eval_status == EvalStatus.PASSED:
+        for overall_eval_metric_result in overall_eval_metric_results:
+          overall_eval_status = overall_eval_metric_result.eval_status
+          if overall_eval_status == EvalStatus.PASSED:
             final_eval_status = EvalStatus.PASSED
-          elif eval_status == EvalStatus.NOT_EVALUATED:
+          elif overall_eval_status == EvalStatus.NOT_EVALUATED:
             continue
-          elif eval_status == EvalStatus.FAILED:
+          elif overall_eval_status == EvalStatus.FAILED:
             final_eval_status = EvalStatus.FAILED
             break
           else:
             raise ValueError("Unknown eval status.")
 
-        yield EvalResult(
-            eval_set_file=eval_set_file,
+        yield EvalCaseResult(
+            eval_set_file=eval_set_id,
+            eval_set_id=eval_set_id,
             eval_id=eval_name,
             final_eval_status=final_eval_status,
-            eval_metric_results=eval_metric_results,
+            eval_metric_results=[],
+            overall_eval_metric_results=overall_eval_metric_results,
+            eval_metric_result_per_invocation=eval_metric_result_per_invocation,
             session_id=session_id,
+            user_id=user_id,
         )
 
         if final_eval_status == EvalStatus.PASSED:
@@ -264,21 +332,26 @@ async def run_evals(
 
         print(f"Result: {result}\n")
 
-      except Exception as e:
-        print(f"Error: {e}")
-        logger.info("Error: %s", str(traceback.format_exc()))
-
+      except Exception:
+        # Catching the general exception, so that we don't block other eval
+        # cases.
+        logger.exception(f"Eval failed for `{eval_set_id}:{eval_name}`")
 
-def _get_eval_metric_result(eval_metric, score):
-  eval_status = (
-      EvalStatus.PASSED if score >= eval_metric.threshold else EvalStatus.FAILED
-  )
-  return EvalMetricResult(score=score, eval_status=eval_status)
 
-
-def _print_eval_metric_result(eval_metric, eval_metric_result):
-  print(
-      f"Metric: {eval_metric.metric_name}\tStatus:"
-      f" {eval_metric_result.eval_status}\tScore:"
-      f" {eval_metric_result.score}\tThreshold: {eval_metric.threshold}"
-  )
+def _get_evaluator(eval_metric: EvalMetric) -> Evaluator:
+  try:
+    from ..evaluation.response_evaluator import ResponseEvaluator
+    from ..evaluation.trajectory_evaluator import TrajectoryEvaluator
+  except ModuleNotFoundError as e:
+    raise ModuleNotFoundError(MISSING_EVAL_DEPENDENCIES_MESSAGE) from e
+  if eval_metric.metric_name == TOOL_TRAJECTORY_SCORE_KEY:
+    return TrajectoryEvaluator(threshold=eval_metric.threshold)
+  elif (
+      eval_metric.metric_name == RESPONSE_MATCH_SCORE_KEY
+      or eval_metric.metric_name == RESPONSE_EVALUATION_SCORE_KEY
+  ):
+    return ResponseEvaluator(
+        threshold=eval_metric.threshold, metric_name=eval_metric.metric_name
+    )
+
+  raise ValueError(f"Unsupported eval metric: {eval_metric}")