agent-starter-pack 0.18.2__py3-none-any.whl → 0.21.0__py3-none-any.whl

agent_starter_pack/agents/{crewai_coding_crew/notebooks/evaluating_crewai_agent.ipynb → adk_a2a_base/notebooks/evaluating_adk_agent.ipynb}

@@ -8,7 +8,7 @@
       },
       "outputs": [],
       "source": [
-        "# Copyright 2024 Google LLC\n",
+        "# Copyright 2025 Google LLC\n",
         "#\n",
         "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -29,26 +29,26 @@
         "id": "JAPoU8Sm5E6e"
       },
       "source": [
-        "# Evaluating Agents - Evaluate a CrewAI agent with Vertex AI Gen AI Evaluation\n",
+        "# Evaluate your ADK agent using Vertex AI Gen AI Evaluation service\n",
         "\n",
         "<table align=\"left\">\n",
         "  <td style=\"text-align: center\">\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/generative-ai/blob/main//generative-ai/gemini/evaluation/evaluating_crewai_agent.ipynb\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/generative-ai/blob/main/gemini/evaluation/evaluating_adk_agent.ipynb\">\n",
         "      <img width=\"32px\" src=\"https://www.gstatic.com/pantheon/images/bigquery/welcome_page/colab-logo.svg\" alt=\"Google Colaboratory logo\"><br> Open in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td style=\"text-align: center\">\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/colab/import/https:%2F%2Fraw.githubusercontent.com%2FGoogleCloudPlatform%2Fgenerative-ai%2Fmain%2Fgemini%2Fevaluation%2Fevaluating_crewai_agent.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/colab/import/https:%2F%2Fraw.githubusercontent.com%2FGoogleCloudPlatform%2Fgenerative-ai%2Fmain%2Fgemini%2Fevaluation%2Fevaluating_adk_agent.ipynb\">\n",
         "      <img width=\"32px\" src=\"https://lh3.googleusercontent.com/JmcxdQi-qOpctIvWKgPtrzZdJJK-J3sWE1RsfjZNwshCFgE_9fULcNpuXYTilIR2hjwN\" alt=\"Google Cloud Colab Enterprise logo\"><br> Open in Colab Enterprise\n",
         "    </a>\n",
         "  </td>\n",
         "  <td style=\"text-align: center\">\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/generative-ai/main/gemini/evaluation/evaluating_crewai_agent.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/generative-ai/main/gemini/evaluation/evaluating_adk_agent.ipynb\">\n",
         "      <img src=\"https://www.gstatic.com/images/branding/gcpiconscolors/vertexai/v1/32px.svg\" alt=\"Vertex AI logo\"><br> Open in Vertex AI Workbench\n",
         "    </a>\n",
         "  </td>\n",
         "  <td style=\"text-align: center\">\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/generative-ai/blob/main//generative-ai/gemini/evaluation/evaluating_crewai_agent.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/evaluation/evaluating_adk_agent.ipynb\">\n",
         "      <img width=\"32px\" src=\"https://www.svgrepo.com/download/217753/github.svg\" alt=\"GitHub logo\"><br> View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
@@ -58,23 +58,23 @@
         "\n",
         "<b>Share to:</b>\n",
         "\n",
-        "<a href=\"https://www.linkedin.com/sharing/share-offsite/?url=https%3A//github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/evaluation/evaluating_crewai_agent.ipynb\" target=\"_blank\">\n",
+        "<a href=\"https://www.linkedin.com/sharing/share-offsite/?url=https%3A//github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/evaluation/evaluating_adk_agent.ipynb\" target=\"_blank\">\n",
         "  <img width=\"20px\" src=\"https://upload.wikimedia.org/wikipedia/commons/8/81/LinkedIn_icon.svg\" alt=\"LinkedIn logo\">\n",
         "</a>\n",
         "\n",
-        "<a href=\"https://bsky.app/intent/compose?text=https%3A//github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/evaluation/evaluating_crewai_agent.ipynb\" target=\"_blank\">\n",
+        "<a href=\"https://bsky.app/intent/compose?text=https%3A//github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/evaluation/evaluating_adk_agent.ipynb\" target=\"_blank\">\n",
         "  <img width=\"20px\" src=\"https://upload.wikimedia.org/wikipedia/commons/7/7a/Bluesky_Logo.svg\" alt=\"Bluesky logo\">\n",
         "</a>\n",
         "\n",
-        "<a href=\"https://twitter.com/intent/tweet?url=https%3A//github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/evaluation/evaluating_crewai_agent.ipynb\" target=\"_blank\">\n",
+        "<a href=\"https://twitter.com/intent/tweet?url=https%3A//github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/evaluation/evaluating_adk_agent.ipynb\" target=\"_blank\">\n",
         "  <img width=\"20px\" src=\"https://upload.wikimedia.org/wikipedia/commons/5/5a/X_icon_2.svg\" alt=\"X logo\">\n",
         "</a>\n",
         "\n",
-        "<a href=\"https://reddit.com/submit?url=https%3A//github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/evaluation/evaluating_crewai_agent.ipynb\" target=\"_blank\">\n",
+        "<a href=\"https://reddit.com/submit?url=https%3A//github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/evaluation/evaluating_adk_agent.ipynb\" target=\"_blank\">\n",
         "  <img width=\"20px\" src=\"https://redditinc.com/hubfs/Reddit%20Inc/Brand/Reddit_Logo.png\" alt=\"Reddit logo\">\n",
         "</a>\n",
         "\n",
-        "<a href=\"https://www.facebook.com/sharer/sharer.php?u=https%3A//github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/evaluation/evaluating_crewai_agent.ipynb\" target=\"_blank\">\n",
+        "<a href=\"https://www.facebook.com/sharer/sharer.php?u=https%3A//github.com/GoogleCloudPlatform/generative-ai/blob/main/gemini/evaluation/evaluating_adk_agent.ipynb\" target=\"_blank\">\n",
         "  <img width=\"20px\" src=\"https://upload.wikimedia.org/wikipedia/commons/5/51/Facebook_f_logo_%282019%29.svg\" alt=\"Facebook logo\">\n",
         "</a>"
       ]
@@ -85,9 +85,9 @@
         "id": "84f0f73a0f76"
       },
       "source": [
-        "| | |\n",
-        "|-|-|\n",
-        "| Authors |  [Ivan Nardini](https://github.com/inardini), [Naveksha Sood](https://github.com/navekshasood)  |"
+        "| Author(s) |\n",
+        "| --- |\n",
+        "| [Ivan Nardini](https://github.com/inardini) |"
       ]
     },
     {
@@ -98,23 +98,17 @@
       "source": [
         "## Overview\n",
         "\n",
-        "Just like any Generative AI application, AI agents require thorough evaluation to ensure they perform reliably and effectively. This evaluation should happen both in real-time (online) and on large datasets of test cases (offline). Developers building agent applications face a significant challenge in evaluating their performance. Both subjective (human feedback) and objective (measurable metrics) evaluations are essential for building trust in agent behavior.\n",
+        "Agent Development Kit (ADK in short) is a flexible and modular open source framework for developing and deploying AI agents. While ADK has its own evaluation module, using Vertex AI Gen AI Evaluation provides a toolkit of quality controlled and explainable methods and metrics to evaluate any generative model or application, including agents, and benchmark the evaluation results against your own judgment, using your own evaluation criteria.\n",
         "\n",
-        "Vertex AI Model Evaluation provides a toolkit of quality controlled and explainable methods and metrics to evaluate any generative model or application, including agents, and benchmark the evaluation results against your own judgment, using your own evaluation criteria.\n",
-        "\n",
-        "This tutorial shows how to evaluate a Crew AI agent using Vertex AI Gen AI Evaluation for agent evaluation.\n",
-        "\n",
-        "The tutorial uses the following Google Cloud services and resources:\n",
-        "\n",
-        "*  Vertex AI Gen AI Evaluation\n",
+        "This tutorial shows how to evaluate an ADK agent using Vertex AI Gen AI Evaluation for agent evaluation.\n",
         "\n",
         "The steps performed include:\n",
         "\n",
-        "* Build local agent using Crew AI\n",
+        "* Build local agent using ADK\n",
         "* Prepare Agent Evaluation dataset\n",
         "* Single tool usage evaluation\n",
         "* Trajectory evaluation\n",
-        "* Response evaluation\n"
+        "* Response evaluation"
       ]
     },
     {
@@ -132,7 +126,7 @@
         "id": "No17Cw5hgx12"
       },
       "source": [
-        "### Install Vertex AI SDK and other required packages\n"
+        "### Install Google Gen AI SDK and other required packages\n"
       ]
     },
     {
@@ -143,49 +137,8 @@
       },
       "outputs": [],
       "source": [
-        "%pip install --upgrade --user --quiet \"google-cloud-aiplatform[evaluation]\" \\\n",
-        "    \"crewai\" \"crewai-tools\" \\\n",
-        "    \"cloudpickle==3.0.0\" \\\n",
-        "    \"pydantic==2.7.4\" \\\n",
-        "    \"requests\""
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "R5Xep4W9lq-Z"
-      },
-      "source": [
-        "### Restart runtime\n",
-        "\n",
-        "To use the newly installed packages in this Jupyter runtime, you must restart the runtime. You can do this by running the cell below, which restarts the current kernel.\n",
-        "\n",
-        "The restart might take a minute or longer. After it's restarted, continue to the next step."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "XRvKdaPDTznN"
-      },
-      "outputs": [],
-      "source": [
-        "import IPython\n",
-        "\n",
-        "app = IPython.Application.instance()\n",
-        "app.kernel.do_shutdown(True)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "SbmM4z7FOBpM"
-      },
-      "source": [
-        "<div class=\"alert alert-block alert-warning\">\n",
-        "<b>⚠️ The kernel is going to restart. In Colab or Colab Enterprise, you might see an error message that says \"Your session crashed for an unknown reason.\" This is expected. Wait until it's finished before continuing to the next step. ⚠️</b>\n",
-        "</div>\n"
+        "%pip install --upgrade --quiet 'google-adk'\n",
+        "%pip install --upgrade --quiet 'google-cloud-aiplatform[evaluation]'"
       ]
     },
     {
@@ -221,7 +174,7 @@
         "id": "DF4l8DTdWgPY"
       },
       "source": [
-        "### Set Google Cloud project information and initialize Vertex AI SDK\n",
+        "### Set Google Cloud project information\n",
         "\n",
         "To get started using Vertex AI, you must have an existing Google Cloud project and [enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com).\n",
         "\n",
@@ -242,14 +195,21 @@
         "import vertexai\n",
         "\n",
         "PROJECT_ID = \"[your-project-id]\"  # @param {type: \"string\", placeholder: \"[your-project-id]\", isTemplate: true}\n",
-        "\n",
         "if not PROJECT_ID or PROJECT_ID == \"[your-project-id]\":\n",
         "    PROJECT_ID = str(os.environ.get(\"GOOGLE_CLOUD_PROJECT\"))\n",
-        "os.environ[\"GOOGLE_CLOUD_PROJECT\"] = PROJECT_ID\n",
         "\n",
         "LOCATION = os.environ.get(\"GOOGLE_CLOUD_REGION\", \"us-central1\")\n",
         "\n",
-        "EXPERIMENT_NAME = \"evaluate-crewai-agent\"  # @param {type:\"string\"}\n",
+        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type: \"string\", placeholder: \"[your-bucket-name]\", isTemplate: true}\n",
+        "BUCKET_URI = f\"gs://{BUCKET_NAME}\"\n",
+        "\n",
+        "!gsutil mb -l {LOCATION} {BUCKET_URI}\n",
+        "\n",
+        "os.environ[\"GOOGLE_CLOUD_PROJECT\"] = PROJECT_ID\n",
+        "os.environ[\"GOOGLE_CLOUD_LOCATION\"] = LOCATION\n",
+        "os.environ[\"GOOGLE_GENAI_USE_VERTEXAI\"] = \"True\"\n",
+        "\n",
+        "EXPERIMENT_NAME = \"evaluate-adk-agent\"  # @param {type:\"string\"}\n",
         "\n",
         "vertexai.init(project=PROJECT_ID, location=LOCATION, experiment=EXPERIMENT_NAME)"
       ]
@@ -273,24 +233,27 @@
       },
       "outputs": [],
       "source": [
+        "import json\n",
+        "import asyncio\n",
+        "\n",
         "# General\n",
         "import random\n",
         "import string\n",
-        "import warnings\n",
+        "from typing import Any\n",
         "\n",
         "from IPython.display import HTML, Markdown, display\n",
-        "import pandas as pd\n",
-        "import plotly.graph_objects as go\n",
-        "\n",
-        "warnings.filterwarnings(\"ignore\", category=Warning, module=\"opentelemetry.trace\")\n",
+        "from google.adk.agents import Agent\n",
         "\n",
-        "# Build agent\n",
-        "from crewai import Agent, Crew, Process, Task\n",
-        "from crewai.flow.flow import Flow, listen, start\n",
-        "from crewai.tools import tool\n",
+        "# Build agent with adk\n",
+        "from google.adk.events import Event\n",
+        "from google.adk.runners import Runner\n",
+        "from google.adk.sessions import InMemorySessionService\n",
         "\n",
         "# Evaluate agent\n",
         "from google.cloud import aiplatform\n",
+        "from google.genai import types\n",
+        "import pandas as pd\n",
+        "import plotly.graph_objects as go\n",
         "from vertexai.preview.evaluation import EvalTask\n",
         "from vertexai.preview.evaluation.metrics import (\n",
         "    PointwiseMetric,\n",
@@ -323,51 +286,62 @@
         "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
         "\n",
         "\n",
-        "def parse_crewai_output_to_dictionary(crew, crew_output):\n",
+        "def parse_adk_output_to_dictionary(events: list[Event], *, as_json: bool = False):\n",
         "    \"\"\"\n",
-        "    Parse CrewAI output into a structured dictionary format.\n",
+        "    Parse ADK event output into a structured dictionary format,\n",
+        "    with the predicted trajectory dumped as a JSON string.\n",
+        "\n",
         "    \"\"\"\n",
-        "    final_output = {\"response\": str(crew_output), \"predicted_trajectory\": []}\n",
-        "\n",
-        "    for agent in crew.agents:\n",
-        "        try:\n",
-        "            for tool_result in agent.tools_results:\n",
-        "                tool_info = {\n",
-        "                    \"tool_name\": tool_result.get(\"tool_name\", \"\"),\n",
-        "                    \"tool_input\": tool_result.get(\"tool_args\", {}),\n",
+        "\n",
+        "    final_response = \"\"\n",
+        "    trajectory = []\n",
+        "\n",
+        "    for event in events:\n",
+        "        if not getattr(event, \"content\", None) or not getattr(event.content, \"parts\", None):\n",
+        "            continue\n",
+        "        for part in event.content.parts:\n",
+        "            if getattr(part, \"function_call\", None):\n",
+        "                info = {\n",
+        "                    \"tool_name\": part.function_call.name,\n",
+        "                    \"tool_input\": dict(part.function_call.args),\n",
         "                }\n",
-        "                final_output[\"predicted_trajectory\"].append(tool_info)\n",
-        "        except AttributeError as e:\n",
-        "            final_output[\"error\"] = f\"Agent does not have tools_results: {str(e)}\"\n",
-        "            print(f\"Error: {e}\")\n",
+        "                if info not in trajectory:\n",
+        "                    trajectory.append(info)\n",
+        "            if event.content.role == \"model\" and getattr(part, \"text\", None):\n",
+        "                final_response = part.text.strip()\n",
         "\n",
-        "    return final_output\n",
+        "    if as_json:\n",
+        "        trajectory_out = json.dumps(trajectory)\n",
+        "    else:\n",
+        "        trajectory_out = trajectory\n",
+        "\n",
+        "    return {\"response\": final_response, \"predicted_trajectory\": trajectory_out}\n",
         "\n",
         "\n",
         "def format_output_as_markdown(output: dict) -> str:\n",
         "    \"\"\"Convert the output dictionary to a formatted markdown string.\"\"\"\n",
-        "    markdown = \"### AI Response\\n\"\n",
-        "    markdown += f\"{output['response']}\\n\\n\"\n",
-        "\n",
+        "    markdown = \"### AI Response\\n\" + output[\"response\"] + \"\\n\\n\"\n",
         "    if output[\"predicted_trajectory\"]:\n",
         "        markdown += \"### Function Calls\\n\"\n",
         "        for call in output[\"predicted_trajectory\"]:\n",
         "            markdown += f\"- **Function**: `{call['tool_name']}`\\n\"\n",
-        "            markdown += \"  - **Arguments**:\\n\"\n",
+        "            markdown += \"  - **Arguments**\\n\"\n",
         "            for key, value in call[\"tool_input\"].items():\n",
         "                markdown += f\"    - `{key}`: `{value}`\\n\"\n",
-        "\n",
         "    return markdown\n",
         "\n",
         "\n",
         "def display_eval_report(eval_result: pd.DataFrame) -> None:\n",
         "    \"\"\"Display the evaluation results.\"\"\"\n",
-        "    metrics_df = pd.DataFrame.from_dict(eval_result.summary_metrics, orient=\"index\").T\n",
         "    display(Markdown(\"### Summary Metrics\"))\n",
-        "    display(metrics_df)\n",
-        "\n",
-        "    display(Markdown(f\"### Row-wise Metrics\"))\n",
-        "    display(eval_result.metrics_table)\n",
+        "    display(\n",
+        "        pd.DataFrame(\n",
+        "            eval_result.summary_metrics.items(), columns=[\"metric\", \"value\"]\n",
+        "        )\n",
+        "    )\n",
+        "    if getattr(eval_result, \"metrics_table\", None) is not None:\n",
+        "        display(Markdown(\"### Row‑wise Metrics\"))\n",
+        "        display(eval_result.metrics_table.head())\n",
         "\n",
         "\n",
         "def display_drilldown(row: pd.Series) -> None:\n",
@@ -513,9 +487,9 @@
         "id": "bDaa2Mtsifmq"
       },
       "source": [
-        "## Build CrewAI agent\n",
+        "## Build ADK agent\n",
         "\n",
-        "Build your application using CrewAI, including the Gemini model and custom tools that you define."
+        "Build your application using ADK, including the Gemini model and custom tools that you define."
       ]
     },
     {
@@ -537,7 +511,6 @@
       },
       "outputs": [],
       "source": [
-        "@tool\n",
         "def get_product_details(product_name: str):\n",
         "    \"\"\"Gathers basic details about a product.\"\"\"\n",
         "    details = {\n",
@@ -550,7 +523,6 @@
         "    return details.get(product_name, \"Product details not found.\")\n",
         "\n",
         "\n",
-        "@tool\n",
         "def get_product_price(product_name: str):\n",
         "    \"\"\"Gathers price about a product.\"\"\"\n",
         "    details = {\n",
@@ -563,47 +535,6 @@
         "    return details.get(product_name, \"Product price not found.\")"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "be70714d9fae"
-      },
-      "source": [
-        "### Define router using Flow\n",
-        "\n",
-        "Set up a router to direct conversation flow by selecting the appropriate tool based on user input or interaction state.\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "516b5108d327"
-      },
-      "outputs": [],
-      "source": [
-        "class ProductFlow(Flow):\n",
-        "    @start\n",
-        "    def begin_flow(self):\n",
-        "        \"\"\"Starts the product information flow\"\"\"\n",
-        "        return \"check_request\"\n",
-        "\n",
-        "    @listen(\"check_request\")\n",
-        "    def router(self, state: dict) -> str:\n",
-        "        \"\"\"Routes the product request to appropriate handler\"\"\"\n",
-        "        # Get the last message from the state\n",
-        "        last_message = state.get(\"last_message\", {})\n",
-        "        tool_calls = last_message.get(\"tool_calls\", [])\n",
-        "\n",
-        "        if tool_calls:\n",
-        "            function_name = tool_calls[0].get(\"name\")\n",
-        "            if function_name == \"get_product_price\":\n",
-        "                return \"get_product_price\"\n",
-        "            else:\n",
-        "                return \"get_product_details\"\n",
-        "        return \"end\""
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -623,7 +554,7 @@
       },
       "outputs": [],
       "source": [
-        "model = \"vertex_ai/gemini-1.5-pro-002\""
+        "model = \"gemini-2.0-flash\""
       ]
     },
     {
@@ -643,38 +574,53 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "dAFdi7SujGP8"
+        "id": "gD5OB44g4sc3"
       },
       "outputs": [],
       "source": [
-        "def agent_parsed_outcome(input):\n",
-        "    product_researcher = Agent(\n",
-        "        role=\"Product Researcher\",\n",
-        "        goal=\"Research product details and prices accurately\",\n",
-        "        backstory=\"Expert at gathering and analyzing product information\",\n",
-        "        llm=model,\n",
-        "        tools=[get_product_details, get_product_price],\n",
-        "        allow_delegation=False,\n",
-        "    )\n",
-        "\n",
-        "    # Create task based on the input\n",
-        "    research_task = Task(\n",
-        "        description=f\"Analyze this user request: '{input}'. \"\n",
-        "        f\"If the request is about price, use get_product_price tool. \"\n",
-        "        f\"Otherwise, use get_product_details tool to get product information.\",\n",
-        "        expected_output=\"Product information including details and/or price based on the user request.\",\n",
-        "        agent=product_researcher,\n",
-        "    )\n",
-        "\n",
-        "    # Create crew with sequential process\n",
-        "    crew = Crew(\n",
-        "        agents=[product_researcher],\n",
-        "        tasks=[research_task],\n",
-        "        process=Process.sequential,\n",
-        "    )\n",
-        "\n",
-        "    result = crew.kickoff()\n",
-        "    return parse_crewai_output_to_dictionary(crew, result)"
+        "async def agent_parsed_outcome(query):\n",
+        "   app_name = \"product_research_app\"\n",
+        "   user_id = \"user_one\"\n",
+        "   session_id = \"session_one\"\n",
+        "   \n",
+        "   product_research_agent = Agent(\n",
+        "       name=\"ProductResearchAgent\",\n",
+        "       model=model,\n",
+        "       description=\"An agent that performs product research.\",\n",
+        "       instruction=f\"\"\"\n",
+        "       Analyze this user request: '{query}'.\n",
+        "       If the request is about price, use get_product_price tool.\n",
+        "       Otherwise, use get_product_details tool to get product information.\n",
+        "       \"\"\",\n",
+        "       tools=[get_product_details, get_product_price],\n",
+        "   )\n",
+        "\n",
+        "   session_service = InMemorySessionService()\n",
+        "   await session_service.create_session(\n",
+        "       app_name=app_name, user_id=user_id, session_id=session_id\n",
+        "   )\n",
+        "\n",
+        "   runner = Runner(\n",
+        "       agent=product_research_agent, app_name=app_name, session_service=session_service\n",
+        "   )\n",
+        "\n",
+        "   content = types.Content(role=\"user\", parts=[types.Part(text=query)])\n",
+        "   events = [event async for event in runner.run_async(user_id=user_id, session_id=session_id, new_message=content)]\n",
+        "   \n",
+        "   return parse_adk_output_to_dictionary(events)\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# --- Sync wrapper for Vertex‑AI evaluation\n",
+        "def agent_parsed_outcome_sync(prompt: str):\n",
+        "    result = asyncio.run(agent_parsed_outcome(prompt))\n",
+        "    result[\"predicted_trajectory\"] = json.dumps(result[\"predicted_trajectory\"])\n",
+        "    return result"
       ]
     },
     {
@@ -696,7 +642,7 @@
       },
       "outputs": [],
       "source": [
-        "response = agent_parsed_outcome(input=\"Get product details for shoes\")\n",
+        "response = await agent_parsed_outcome(query=\"Get product details for shoes\")\n",
         "display(Markdown(format_output_as_markdown(response)))"
       ]
     },
@@ -708,7 +654,7 @@
       },
       "outputs": [],
       "source": [
-        "response = agent_parsed_outcome(input=\"Get product price for shoes\")\n",
+        "response = await agent_parsed_outcome(query=\"Get product price for shoes\")\n",
         "display(Markdown(format_output_as_markdown(response)))"
       ]
     },
@@ -718,7 +664,7 @@
         "id": "aOGPePsorpUl"
       },
       "source": [
-        "## Evaluating a CrewAI agent with Vertex AI Gen AI Evaluation\n",
+        "## Evaluating a ADK agent with Vertex AI Gen AI Evaluation\n",
         "\n",
         "When working with AI agents, it's important to keep track of their performance and how well they're working. You can look at this in two main ways: **monitoring** and **observability**.\n",
         "\n",
@@ -772,7 +718,7 @@
         "        \"Get product details and price for shoes\",\n",
         "        \"Get product details for speaker?\",\n",
         "    ],\n",
-        "    \"reference_trajectory\": [\n",
+        "    \"predicted_trajectory\": [\n",
         "        [\n",
         "            {\n",
         "                \"tool_name\": \"get_product_price\",\n",
@@ -894,10 +840,11 @@
         "    dataset=eval_sample_dataset,\n",
         "    metrics=single_tool_usage_metrics,\n",
         "    experiment=EXPERIMENT_NAME,\n",
+        "    output_uri_prefix=BUCKET_URI + \"/single-metric-eval\",\n",
         ")\n",
         "\n",
         "single_tool_call_eval_result = single_tool_call_eval_task.evaluate(\n",
-        "    runnable=agent_parsed_outcome, experiment_run_name=EXPERIMENT_RUN\n",
+        "    runnable=agent_parsed_outcome_sync, experiment_run_name=EXPERIMENT_RUN\n",
         ")\n",
         "\n",
         "display_eval_report(single_tool_call_eval_result)"
@@ -998,11 +945,14 @@
         "EXPERIMENT_RUN = f\"trajectory-{get_id()}\"\n",
         "\n",
         "trajectory_eval_task = EvalTask(\n",
-        "    dataset=eval_sample_dataset, metrics=trajectory_metrics, experiment=EXPERIMENT_NAME\n",
+        "    dataset=eval_sample_dataset,\n",
+        "    metrics=trajectory_metrics,\n",
+        "    experiment=EXPERIMENT_NAME,\n",
+        "    output_uri_prefix=BUCKET_URI + \"/multiple-metric-eval\",\n",
         ")\n",
         "\n",
         "trajectory_eval_result = trajectory_eval_task.evaluate(\n",
-        "    runnable=agent_parsed_outcome, experiment_run_name=EXPERIMENT_RUN\n",
+        "    runnable=agent_parsed_outcome_sync, experiment_run_name=EXPERIMENT_RUN\n",
         ")\n",
         "\n",
         "display_eval_report(trajectory_eval_result)"
@@ -1102,11 +1052,14 @@
         "EXPERIMENT_RUN = f\"response-{get_id()}\"\n",
         "\n",
         "response_eval_task = EvalTask(\n",
-        "    dataset=eval_sample_dataset, metrics=response_metrics, experiment=EXPERIMENT_NAME\n",
+        "    dataset=eval_sample_dataset,\n",
+        "    metrics=response_metrics,\n",
+        "    experiment=EXPERIMENT_NAME,\n",
+        "    output_uri_prefix=BUCKET_URI + \"/response-metric-eval\",\n",
         ")\n",
         "\n",
         "response_eval_result = response_eval_task.evaluate(\n",
-        "    runnable=agent_parsed_outcome, experiment_run_name=EXPERIMENT_RUN\n",
+        "    runnable=agent_parsed_outcome_sync, experiment_run_name=EXPERIMENT_RUN\n",
         ")\n",
         "\n",
         "display_eval_report(response_eval_result)"
@@ -1287,10 +1240,13 @@
         "    dataset=eval_sample_dataset,\n",
         "    metrics=response_tool_metrics,\n",
         "    experiment=EXPERIMENT_NAME,\n",
+        "    output_uri_prefix=BUCKET_URI + \"/reasoning-metric-eval\",\n",
         ")\n",
         "\n",
         "response_eval_tool_result = response_eval_tool_task.evaluate(\n",
-        "    runnable=agent_parsed_outcome, experiment_run_name=EXPERIMENT_RUN\n",
+        "    # Uncomment the line below if you are providing the agent with an unparsed dataset\n",
+        "    #runnable=agent_parsed_outcome_sync, \n",
+        "    experiment_run_name=EXPERIMENT_RUN\n",
         ")\n",
         "\n",
         "display_eval_report(response_eval_tool_result)"
@@ -1446,15 +1402,22 @@
         "        ],\n",
         "    ],\n",
         "    \"response\": [\n",
-        "        500,\n",
-        "        50,\n",
+        "        \"500\",\n",
+        "        \"50\",\n",
         "        \"A super fast and light usb charger\",\n",
-        "        100,\n",
+        "        \"100\",\n",
         "        \"A voice-controlled smart speaker that plays music, sets alarms, and controls smart home devices.\",\n",
         "    ],\n",
         "}\n",
         "\n",
-        "byod_eval_sample_dataset = pd.DataFrame(byod_eval_data)"
+        "byod_eval_sample_dataset = pd.DataFrame(byod_eval_data)\n",
+        "byod_eval_sample_dataset[\"predicted_trajectory\"] = byod_eval_sample_dataset[\n",
+        "    \"predicted_trajectory\"\n",
+        "].apply(json.dumps)\n",
+        "byod_eval_sample_dataset[\"reference_trajectory\"] = byod_eval_sample_dataset[\n",
+        "    \"reference_trajectory\"\n",
+        "].apply(json.dumps)\n",
+        "byod_eval_sample_dataset[\"response\"] = byod_eval_sample_dataset[\"response\"].apply(json.dumps)"
       ]
     },
     {
@@ -1482,6 +1445,7 @@
         "    dataset=byod_eval_sample_dataset,\n",
         "    metrics=response_tool_metrics,\n",
         "    experiment=EXPERIMENT_NAME,\n",
+        "    output_uri_prefix=BUCKET_URI + \"/byod-eval\",\n",
         ")\n",
         "\n",
         "byod_response_eval_tool_result = byod_response_eval_tool_task.evaluate(\n",
@@ -1497,7 +1461,7 @@
         "id": "9eU3LG6r7q-3"
       },
       "source": [
-        "#### Visualize evaluation results\n",
+        "### Visualize evaluation results\n",
         "\n",
         "Visualize evaluation result sample."
       ]
@@ -1523,7 +1487,7 @@
       "source": [
         "display_radar_plot(\n",
         "    byod_response_eval_tool_result,\n",
-        "    title=\"Agent evaluation metrics\",\n",
+        "    title=\"ADK agent evaluation\",\n",
         "    metrics=[f\"{metric}/mean\" for metric in response_tool_metrics],\n",
         ")"
       ]
@@ -1531,7 +1495,7 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "2a4e033321ad"
+        "id": "fIppkS2jq_Dn"
       },
       "source": [
         "## Cleaning up\n"
@@ -1558,7 +1522,7 @@
   ],
   "metadata": {
     "colab": {
-      "name": "evaluating_crewai_agent.ipynb",
+      "name": "evaluating_adk_agent.ipynb",
       "toc_visible": true
     },
     "kernelspec": {