google-adk 0.5.0__py3-none-any.whl → 1.1.0__py3-none-any.whl

google/adk/cli/cli_tools_click.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import asyncio
+import collections
 from contextlib import asynccontextmanager
 from datetime import datetime
 import logging
@@ -26,13 +27,60 @@ import uvicorn
 
 from . import cli_create
 from . import cli_deploy
+from .. import version
+from ..evaluation.local_eval_set_results_manager import LocalEvalSetResultsManager
+from ..sessions.in_memory_session_service import InMemorySessionService
 from .cli import run_cli
 from .cli_eval import MISSING_EVAL_DEPENDENCIES_MESSAGE
 from .fast_api import get_fast_api_app
 from .utils import envs
 from .utils import logs
 
-logger = logging.getLogger(__name__)
+
+class HelpfulCommand(click.Command):
+  """Command that shows full help on error instead of just the error message.
+
+  A custom Click Command class that overrides the default error handling
+  behavior to display the full help text when a required argument is missing,
+  followed by the error message. This provides users with better context
+  about command usage without needing to run a separate --help command.
+
+  Args:
+    *args: Variable length argument list to pass to the parent class.
+    **kwargs: Arbitrary keyword arguments to pass to the parent class.
+
+  Returns:
+    None. Inherits behavior from the parent Click Command class.
+
+  Returns:
+  """
+
+  def __init__(self, *args, **kwargs):
+    super().__init__(*args, **kwargs)
+
+  def parse_args(self, ctx, args):
+    """Override the parse_args method to show help text on error.
+
+    Args:
+      ctx: Click context object for the current command.
+      args: List of command-line arguments to parse.
+
+    Returns:
+      The parsed arguments as returned by the parent class's parse_args method.
+
+    Raises:
+      click.MissingParameter: When a required parameter is missing, but this
+        is caught and handled by displaying the help text before exiting.
+    """
+    try:
+      return super().parse_args(ctx, args)
+    except click.MissingParameter as exc:
+      click.echo(ctx.get_help())
+      click.secho(f"\nError: {str(exc)}", fg="red", err=True)
+      ctx.exit(2)
+
+
+logger = logging.getLogger("google_adk." + __name__)
 
 
 @click.group(context_settings={"max_content_width": 240})
@@ -47,7 +95,7 @@ def deploy():
   pass
 
 
-@main.command("create")
+@main.command("create", cls=HelpfulCommand)
 @click.option(
     "--model",
     type=str,
@@ -113,7 +161,7 @@ def validate_exclusive(ctx, param, value):
   return value
 
 
-@main.command("run")
+@main.command("run", cls=HelpfulCommand)
 @click.option(
     "--save_session",
     type=bool,
@@ -122,6 +170,15 @@ def validate_exclusive(ctx, param, value):
     default=False,
     help="Optional. Whether to save the session to a json file on exit.",
 )
+@click.option(
+    "--session_id",
+    type=str,
+    help=(
+        "Optional. The session ID to save the session to on exit when"
+        " --save_session is set to true. User will be prompted to enter a"
+        " session ID if not set."
+    ),
+)
 @click.option(
     "--replay",
     type=click.Path(
@@ -156,6 +213,7 @@ def validate_exclusive(ctx, param, value):
 def cli_run(
     agent: str,
     save_session: bool,
+    session_id: Optional[str],
     replay: Optional[str],
     resume: Optional[str],
 ):
@@ -179,11 +237,12 @@ def cli_run(
           input_file=replay,
           saved_session_file=resume,
           save_session=save_session,
+          session_id=session_id,
       )
   )
 
 
-@main.command("eval")
+@main.command("eval", cls=HelpfulCommand)
 @click.argument(
     "agent_module_file_path",
     type=click.Path(
@@ -231,8 +290,9 @@ def cli_eval(
   envs.load_dotenv_for_agent(agent_module_file_path, ".")
 
   try:
+    from ..evaluation.local_eval_sets_manager import load_eval_set_from_file
+    from .cli_eval import EvalCaseResult
     from .cli_eval import EvalMetric
-    from .cli_eval import EvalResult
     from .cli_eval import EvalStatus
     from .cli_eval import get_evaluation_criteria_or_default
     from .cli_eval import get_root_agent
@@ -249,48 +309,94 @@ def cli_eval(
         EvalMetric(metric_name=metric_name, threshold=threshold)
     )
 
-  print(f"Using evaluation creiteria: {evaluation_criteria}")
+  print(f"Using evaluation criteria: {evaluation_criteria}")
 
   root_agent = get_root_agent(agent_module_file_path)
   reset_func = try_get_reset_func(agent_module_file_path)
 
-  eval_set_to_evals = parse_and_get_evals_to_run(eval_set_file_path)
+  eval_set_file_path_to_evals = parse_and_get_evals_to_run(eval_set_file_path)
+  eval_set_id_to_eval_cases = {}
+
+  # Read the eval_set files and get the cases.
+  for eval_set_file_path, eval_case_ids in eval_set_file_path_to_evals.items():
+    eval_set = load_eval_set_from_file(eval_set_file_path, eval_set_file_path)
+    eval_cases = eval_set.eval_cases
+
+    if eval_case_ids:
+      # There are eval_ids that we should select.
+      eval_cases = [
+          e for e in eval_set.eval_cases if e.eval_id in eval_case_ids
+      ]
+
+    eval_set_id_to_eval_cases[eval_set.eval_set_id] = eval_cases
+
+  async def _collect_eval_results() -> list[EvalCaseResult]:
+    session_service = InMemorySessionService()
+    eval_case_results = []
+    async for eval_case_result in run_evals(
+        eval_set_id_to_eval_cases,
+        root_agent,
+        reset_func,
+        eval_metrics,
+        session_service=session_service,
+    ):
+      eval_case_result.session_details = await session_service.get_session(
+          app_name=os.path.basename(agent_module_file_path),
+          user_id=eval_case_result.user_id,
+          session_id=eval_case_result.session_id,
+      )
+      eval_case_results.append(eval_case_result)
+    return eval_case_results
 
   try:
-    eval_results = list(
-        asyncio.run(
-            run_evals(
-                eval_set_to_evals,
-                root_agent,
-                reset_func,
-                eval_metrics,
-                print_detailed_results=print_detailed_results,
-            )
-        )
-    )
+    eval_results = asyncio.run(_collect_eval_results())
   except ModuleNotFoundError:
     raise click.ClickException(MISSING_EVAL_DEPENDENCIES_MESSAGE)
 
+  # Write eval set results.
+  local_eval_set_results_manager = LocalEvalSetResultsManager(
+      agents_dir=os.path.dirname(agent_module_file_path)
+  )
+  eval_set_id_to_eval_results = collections.defaultdict(list)
+  for eval_case_result in eval_results:
+    eval_set_id = eval_case_result.eval_set_id
+    eval_set_id_to_eval_results[eval_set_id].append(eval_case_result)
+
+  for eval_set_id, eval_case_results in eval_set_id_to_eval_results.items():
+    local_eval_set_results_manager.save_eval_set_result(
+        app_name=os.path.basename(agent_module_file_path),
+        eval_set_id=eval_set_id,
+        eval_case_results=eval_case_results,
+    )
+
   print("*********************************************************************")
   eval_run_summary = {}
 
   for eval_result in eval_results:
-    eval_result: EvalResult
+    eval_result: EvalCaseResult
 
-    if eval_result.eval_set_file not in eval_run_summary:
-      eval_run_summary[eval_result.eval_set_file] = [0, 0]
+    if eval_result.eval_set_id not in eval_run_summary:
+      eval_run_summary[eval_result.eval_set_id] = [0, 0]
 
     if eval_result.final_eval_status == EvalStatus.PASSED:
-      eval_run_summary[eval_result.eval_set_file][0] += 1
+      eval_run_summary[eval_result.eval_set_id][0] += 1
     else:
-      eval_run_summary[eval_result.eval_set_file][1] += 1
+      eval_run_summary[eval_result.eval_set_id][1] += 1
   print("Eval Run Summary")
-  for eval_set_file, pass_fail_count in eval_run_summary.items():
+  for eval_set_id, pass_fail_count in eval_run_summary.items():
     print(
-        f"{eval_set_file}:\n  Tests passed: {pass_fail_count[0]}\n  Tests"
+        f"{eval_set_id}:\n  Tests passed: {pass_fail_count[0]}\n  Tests"
         f" failed: {pass_fail_count[1]}"
     )
 
+  if print_detailed_results:
+    for eval_result in eval_results:
+      eval_result: EvalCaseResult
+      print(
+          "*********************************************************************"
+      )
+      print(eval_result.model_dump_json(indent=2))
+
 
 @main.command("web")
 @click.option(
@@ -305,6 +411,13 @@ def cli_eval(
   - See https://docs.sqlalchemy.org/en/20/core/engines.html#backend-specific-urls for more details on supported DB URLs."""
     ),
 )
+@click.option(
+    "--host",
+    type=str,
+    help="Optional. The binding host of the server",
+    default="127.0.0.1",
+    show_default=True,
+)
 @click.option(
     "--port",
     type=int,
@@ -324,16 +437,6 @@ def cli_eval(
     default="INFO",
     help="Optional. Set the logging level",
 )
-@click.option(
-    "--log_to_tmp",
-    is_flag=True,
-    show_default=True,
-    default=False,
-    help=(
-        "Optional. Whether to log to system temp folder instead of console."
-        " This is useful for local debugging."
-    ),
-)
 @click.option(
     "--trace_to_cloud",
     is_flag=True,
@@ -341,6 +444,11 @@ def cli_eval(
     default=False,
     help="Optional. Whether to enable cloud trace for telemetry.",
 )
+@click.option(
+    "--reload/--no-reload",
+    default=True,
+    help="Optional. Whether to enable auto reload for server.",
+)
 @click.argument(
     "agents_dir",
     type=click.Path(
@@ -350,12 +458,13 @@ def cli_eval(
 )
 def cli_web(
     agents_dir: str,
-    log_to_tmp: bool,
     session_db_url: str = "",
     log_level: str = "INFO",
     allow_origins: Optional[list[str]] = None,
+    host: str = "127.0.0.1",
     port: int = 8000,
     trace_to_cloud: bool = False,
+    reload: bool = True,
 ):
   """Starts a FastAPI server with Web UI for agents.
 
@@ -366,12 +475,7 @@ def cli_web(
 
     adk web --session_db_url=[db_url] --port=[port] path/to/agents_dir
   """
-  if log_to_tmp:
-    logs.log_to_tmp_folder()
-  else:
-    logs.log_to_stderr()
-
-  logging.getLogger().setLevel(log_level)
+  logs.setup_adk_logger(getattr(logging, log_level.upper()))
 
   @asynccontextmanager
   async def _lifespan(app: FastAPI):
@@ -396,7 +500,7 @@ def cli_web(
     )
 
   app = get_fast_api_app(
-      agent_dir=agents_dir,
+      agents_dir=agents_dir,
       session_db_url=session_db_url,
       allow_origins=allow_origins,
       web=True,
@@ -405,9 +509,9 @@ def cli_web(
   )
   config = uvicorn.Config(
       app,
-      host="0.0.0.0",
+      host=host,
       port=port,
-      reload=True,
+      reload=reload,
   )
 
   server = uvicorn.Server(config)
@@ -427,6 +531,13 @@ def cli_web(
   - See https://docs.sqlalchemy.org/en/20/core/engines.html#backend-specific-urls for more details on supported DB URLs."""
     ),
 )
+@click.option(
+    "--host",
+    type=str,
+    help="Optional. The binding host of the server",
+    default="127.0.0.1",
+    show_default=True,
+)
 @click.option(
     "--port",
     type=int,
@@ -446,16 +557,6 @@ def cli_web(
     default="INFO",
     help="Optional. Set the logging level",
 )
-@click.option(
-    "--log_to_tmp",
-    is_flag=True,
-    show_default=True,
-    default=False,
-    help=(
-        "Optional. Whether to log to system temp folder instead of console."
-        " This is useful for local debugging."
-    ),
-)
 @click.option(
     "--trace_to_cloud",
     is_flag=True,
@@ -463,6 +564,11 @@ def cli_web(
     default=False,
     help="Optional. Whether to enable cloud trace for telemetry.",
 )
+@click.option(
+    "--reload/--no-reload",
+    default=True,
+    help="Optional. Whether to enable auto reload for server.",
+)
 # The directory of agents, where each sub-directory is a single agent.
 # By default, it is the current working directory
 @click.argument(
@@ -474,12 +580,13 @@ def cli_web(
 )
 def cli_api_server(
     agents_dir: str,
-    log_to_tmp: bool,
     session_db_url: str = "",
     log_level: str = "INFO",
     allow_origins: Optional[list[str]] = None,
+    host: str = "127.0.0.1",
     port: int = 8000,
     trace_to_cloud: bool = False,
+    reload: bool = True,
 ):
   """Starts a FastAPI server for agents.
 
@@ -490,24 +597,19 @@ def cli_api_server(
 
     adk api_server --session_db_url=[db_url] --port=[port] path/to/agents_dir
   """
-  if log_to_tmp:
-    logs.log_to_tmp_folder()
-  else:
-    logs.log_to_stderr()
-
-  logging.getLogger().setLevel(log_level)
+  logs.setup_adk_logger(getattr(logging, log_level.upper()))
 
   config = uvicorn.Config(
       get_fast_api_app(
-          agent_dir=agents_dir,
+          agents_dir=agents_dir,
           session_db_url=session_db_url,
           allow_origins=allow_origins,
           web=False,
           trace_to_cloud=trace_to_cloud,
       ),
-      host="0.0.0.0",
+      host=host,
       port=port,
-      reload=True,
+      reload=reload,
   )
   server = uvicorn.Server(config)
   server.run()
@@ -556,7 +658,6 @@ def cli_api_server(
 )
 @click.option(
     "--trace_to_cloud",
-    type=bool,
     is_flag=True,
     show_default=True,
     default=False,
@@ -564,7 +665,6 @@ def cli_api_server(
 )
 @click.option(
     "--with_ui",
-    type=bool,
     is_flag=True,
     show_default=True,
     default=False,
@@ -612,6 +712,16 @@ def cli_api_server(
         exists=True, dir_okay=True, file_okay=False, resolve_path=True
     ),
 )
+@click.option(
+    "--adk_version",
+    type=str,
+    default=version.__version__,
+    show_default=True,
+    help=(
+        "Optional. The ADK version used in Cloud Run deployment. (default: the"
+        " version in the dev environment)"
+    ),
+)
 def cli_deploy_cloud_run(
     agent: str,
     project: Optional[str],
@@ -624,6 +734,7 @@ def cli_deploy_cloud_run(
     with_ui: bool,
     verbosity: str,
     session_db_url: str,
+    adk_version: str,
 ):
   """Deploys an agent to Cloud Run.
 
@@ -646,6 +757,7 @@ def cli_deploy_cloud_run(
         with_ui=with_ui,
         verbosity=verbosity,
         session_db_url=session_db_url,
+        adk_version=adk_version,
     )
   except Exception as e:
     click.secho(f"Deploy failed: {e}", fg="red", err=True)