agent-starter-pack 0.13.1__tar.gz → 0.14.1__tar.gz

{agent_starter_pack-0.13.1 → agent_starter_pack-0.14.1}/.claude/settings.local.json

@@ -8,7 +8,9 @@
       "Bash(gcloud compute networks vpc-access connectors list --project=agent-starter-pack-e2e-dev --region=europe-west1)",
       "Bash(gcloud alpha network-services address-reservations list --project=agent-starter-pack-e2e-dev --location=europe-west1)",
       "Bash(gcloud beta serverless address-reservations list --project=agent-starter-pack-e2e-dev --location=europe-west1)",
-      "Bash(gcloud run services delete test-797fa619-1755528509 --region=europe-west1 --project=agent-starter-pack-e2e-dev --quiet)"
+      "Bash(gcloud run services delete test-797fa619-1755528509 --region=europe-west1 --project=agent-starter-pack-e2e-dev --quiet)",
+      "Bash(make test)",
+      "WebFetch(domain:github.com)"
     ],
     "deny": [],
     "ask": []

{agent_starter_pack-0.13.1 → agent_starter_pack-0.14.1}/GEMINI.md

@@ -151,6 +151,7 @@ Test changes across multiple dimensions:
 - Agent types (live_api, adk_base, etc.)
 - Deployment targets (cloud_run, agent_engine)
 - Feature combinations (data_ingestion, frontend_type)
+- Example command for testing the starter pack creation - from the root of the repo run: `uv run agent-starter-pack create myagent-$(date +%s) --output-dir target`
 
 ### Common Pitfalls
 - **Hardcoded URLs**: Use relative paths for frontend connections

{agent_starter_pack-0.13.1 → agent_starter_pack-0.14.1}/Makefile

@@ -11,6 +11,7 @@ generate-lock:
 	uv run src/utils/generate_locks.py
 
 lint:
+	uv sync --dev --extra lint
 	uv run ruff check . --config pyproject.toml --diff
 	uv run ruff format . --check  --config pyproject.toml --diff
 	uv run mypy --config-file pyproject.toml ./src/cli ./tests ./src/frontends/streamlit

{agent_starter_pack-0.13.1 → agent_starter_pack-0.14.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agent-starter-pack
-Version: 0.13.1
+Version: 0.14.1
 Summary: CLI to bootstrap production-ready Google Cloud GenAI agent projects from templates.
 Author-email: Google LLC <agent-starter-pack@google.com>
 License: Apache-2.0
@@ -69,7 +69,7 @@ agent-starter-pack create my-awesome-agent
 
 If you have [`uv`](https://github.com/astral-sh/uv) installed, you can create and set up your project with a single command:
 ```bash
-uvx agent-starter-pack create my-fullstack-agent
+uvx agent-starter-pack create my-awesome-agent
 ```
 This command handles creating the project without needing to pre-install the package into a virtual environment.
 </details>
@@ -78,10 +78,10 @@ This command handles creating the project without needing to pre-install the pac
 
 ### 🔧 Enhance Existing Agents
 
-Already have an agent? Add production-ready deployment and infrastructure:
+Already have an agent? Add production-ready deployment and infrastructure by running this command in your project's root folder:
 
 ```bash
-agent-starter-pack enhance my-existing-agent
+uvx agent-starter-pack enhance
 ```
 
 See [Installation Guide](https://googlecloudplatform.github.io/agent-starter-pack/guide/installation) for more options, or try with zero setup in [Firebase Studio](https://studio.firebase.google.com/new?template=https%3A%2F%2Fgithub.com%2FGoogleCloudPlatform%2Fagent-starter-pack%2Ftree%2Fmain%2Fsrc%2Fresources%2Fidx) or [Cloud Shell](https://shell.cloud.google.com/cloudshell/editor?cloudshell_git_repo=https%3A%2F%2Fgithub.com%2Feliasecchig%2Fasp-open-in-cloud-shell&cloudshell_print=open-in-cs).
@@ -104,6 +104,14 @@ See [Installation Guide](https://googlecloudplatform.github.io/agent-starter-pac
 
 Looking to explore more ADK examples? Check out the [ADK Samples Repository](https://github.com/google/adk-samples) for additional examples and use cases demonstrating ADK's capabilities.
 
+---
+
+## 🌟 Community Showcase
+
+Explore amazing projects built with the Agent Starter Pack! 
+
+**[View Community Showcase →](https://googlecloudplatform.github.io/agent-starter-pack/guide/community-showcase)**
+
 #### Extra Features
 
 The `agent-starter-pack` offers two key features to accelerate and simplify the development of your agent:
@@ -125,6 +133,7 @@ This starter pack covers all aspects of Agent development, from prototyping and
 - Python 3.10+
 - [Google Cloud SDK](https://cloud.google.com/sdk/docs/install)
 - [Terraform](https://developer.hashicorp.com/terraform/downloads) (for deployment)
+- [Make](https://www.gnu.org/software/make/) (for development tasks)
 
 
 ## 📚 Documentation

{agent_starter_pack-0.13.1 → agent_starter_pack-0.14.1}/README.md

@@ -45,7 +45,7 @@ agent-starter-pack create my-awesome-agent
 
 If you have [`uv`](https://github.com/astral-sh/uv) installed, you can create and set up your project with a single command:
 ```bash
-uvx agent-starter-pack create my-fullstack-agent
+uvx agent-starter-pack create my-awesome-agent
 ```
 This command handles creating the project without needing to pre-install the package into a virtual environment.
 </details>
@@ -54,10 +54,10 @@ This command handles creating the project without needing to pre-install the pac
 
 ### 🔧 Enhance Existing Agents
 
-Already have an agent? Add production-ready deployment and infrastructure:
+Already have an agent? Add production-ready deployment and infrastructure by running this command in your project's root folder:
 
 ```bash
-agent-starter-pack enhance my-existing-agent
+uvx agent-starter-pack enhance
 ```
 
 See [Installation Guide](https://googlecloudplatform.github.io/agent-starter-pack/guide/installation) for more options, or try with zero setup in [Firebase Studio](https://studio.firebase.google.com/new?template=https%3A%2F%2Fgithub.com%2FGoogleCloudPlatform%2Fagent-starter-pack%2Ftree%2Fmain%2Fsrc%2Fresources%2Fidx) or [Cloud Shell](https://shell.cloud.google.com/cloudshell/editor?cloudshell_git_repo=https%3A%2F%2Fgithub.com%2Feliasecchig%2Fasp-open-in-cloud-shell&cloudshell_print=open-in-cs).
@@ -80,6 +80,14 @@ See [Installation Guide](https://googlecloudplatform.github.io/agent-starter-pac
 
 Looking to explore more ADK examples? Check out the [ADK Samples Repository](https://github.com/google/adk-samples) for additional examples and use cases demonstrating ADK's capabilities.
 
+---
+
+## 🌟 Community Showcase
+
+Explore amazing projects built with the Agent Starter Pack! 
+
+**[View Community Showcase →](https://googlecloudplatform.github.io/agent-starter-pack/guide/community-showcase)**
+
 #### Extra Features
 
 The `agent-starter-pack` offers two key features to accelerate and simplify the development of your agent:
@@ -101,6 +109,7 @@ This starter pack covers all aspects of Agent development, from prototyping and
 - Python 3.10+
 - [Google Cloud SDK](https://cloud.google.com/sdk/docs/install)
 - [Terraform](https://developer.hashicorp.com/terraform/downloads) (for deployment)
+- [Make](https://www.gnu.org/software/make/) (for development tasks)
 
 
 ## 📚 Documentation

{agent_starter_pack-0.13.1 → agent_starter_pack-0.14.1}/agents/adk_base/notebooks/evaluating_adk_agent.ipynb

@@ -234,6 +234,7 @@
       "outputs": [],
       "source": [
         "import json\n",
+        "import asyncio\n",
         "\n",
         "# General\n",
         "import random\n",
@@ -285,7 +286,7 @@
         "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
         "\n",
         "\n",
-        "def parse_adk_output_to_dictionary(events: list[Event]) -> dict[str, Any]:\n",
+        "def parse_adk_output_to_dictionary(events: list[Event], *, as_json: bool = False):\n",
         "    \"\"\"\n",
         "    Parse ADK event output into a structured dictionary format,\n",
         "    with the predicted trajectory dumped as a JSON string.\n",
@@ -293,63 +294,54 @@
         "    \"\"\"\n",
         "\n",
         "    final_response = \"\"\n",
-        "    predicted_trajectory_list = []\n",
+        "    trajectory = []\n",
         "\n",
         "    for event in events:\n",
-        "        # Ensure content and parts exist before accessing them\n",
-        "        if not event.content or not event.content.parts:\n",
+        "        if not getattr(event, \"content\", None) or not getattr(event.content, \"parts\", None):\n",
         "            continue\n",
-        "\n",
-        "        # Iterate through ALL parts in the event's content\n",
         "        for part in event.content.parts:\n",
-        "            if part.function_call:\n",
-        "                tool_info = {\n",
+        "            if getattr(part, \"function_call\", None):\n",
+        "                info = {\n",
         "                    \"tool_name\": part.function_call.name,\n",
         "                    \"tool_input\": dict(part.function_call.args),\n",
         "                }\n",
-        "                # Ensure we don't add duplicates if the same call appears somehow\n",
-        "                if tool_info not in predicted_trajectory_list:\n",
-        "                    predicted_trajectory_list.append(tool_info)\n",
-        "\n",
-        "            # The final text response is usually in the last event from the model\n",
-        "            if event.content.role == \"model\" and part.text:\n",
-        "                # Overwrite response; the last text response found is likely the final one\n",
+        "                if info not in trajectory:\n",
+        "                    trajectory.append(info)\n",
+        "            if event.content.role == \"model\" and getattr(part, \"text\", None):\n",
         "                final_response = part.text.strip()\n",
         "\n",
-        "    # Dump the collected trajectory list into a JSON string\n",
-        "    final_output = {\n",
-        "        \"response\": str(final_response),\n",
-        "        \"predicted_trajectory\": json.dumps(predicted_trajectory_list),\n",
-        "    }\n",
+        "    if as_json:\n",
+        "        trajectory_out = json.dumps(trajectory)\n",
+        "    else:\n",
+        "        trajectory_out = trajectory\n",
         "\n",
-        "    return final_output\n",
+        "    return {\"response\": final_response, \"predicted_trajectory\": trajectory_out}\n",
         "\n",
         "\n",
         "def format_output_as_markdown(output: dict) -> str:\n",
         "    \"\"\"Convert the output dictionary to a formatted markdown string.\"\"\"\n",
-        "    markdown = \"### AI Response\\n\"\n",
-        "    markdown += f\"{output['response']}\\n\\n\"\n",
-        "\n",
+        "    markdown = \"### AI Response\\n\" + output[\"response\"] + \"\\n\\n\"\n",
         "    if output[\"predicted_trajectory\"]:\n",
-        "        output[\"predicted_trajectory\"] = json.loads(output[\"predicted_trajectory\"])\n",
         "        markdown += \"### Function Calls\\n\"\n",
         "        for call in output[\"predicted_trajectory\"]:\n",
         "            markdown += f\"- **Function**: `{call['tool_name']}`\\n\"\n",
-        "            markdown += \"  - **Arguments**:\\n\"\n",
+        "            markdown += \"  - **Arguments**\\n\"\n",
         "            for key, value in call[\"tool_input\"].items():\n",
         "                markdown += f\"    - `{key}`: `{value}`\\n\"\n",
-        "\n",
         "    return markdown\n",
         "\n",
         "\n",
         "def display_eval_report(eval_result: pd.DataFrame) -> None:\n",
         "    \"\"\"Display the evaluation results.\"\"\"\n",
-        "    metrics_df = pd.DataFrame.from_dict(eval_result.summary_metrics, orient=\"index\").T\n",
         "    display(Markdown(\"### Summary Metrics\"))\n",
-        "    display(metrics_df)\n",
-        "\n",
-        "    display(Markdown(\"### Row-wise Metrics\"))\n",
-        "    display(eval_result.metrics_table)\n",
+        "    display(\n",
+        "        pd.DataFrame(\n",
+        "            eval_result.summary_metrics.items(), columns=[\"metric\", \"value\"]\n",
+        "        )\n",
+        "    )\n",
+        "    if getattr(eval_result, \"metrics_table\", None) is not None:\n",
+        "        display(Markdown(\"### Row‑wise Metrics\"))\n",
+        "        display(eval_result.metrics_table.head())\n",
         "\n",
         "\n",
         "def display_drilldown(row: pd.Series) -> None:\n",
@@ -586,37 +578,49 @@
       },
       "outputs": [],
       "source": [
-        "def agent_parsed_outcome(query):\n",
-        "\n",
-        "    app_name = \"product_research_app\"\n",
-        "    user_id = \"user_one\"\n",
-        "    session_id = \"session_one\"\n",
-        "\n",
-        "    product_research_agent = Agent(\n",
-        "        name=\"ProductResearchAgent\",\n",
-        "        model=model,\n",
-        "        description=\"An agent that performs product research.\",\n",
-        "        instruction=f\"\"\"\n",
-        "        Analyze this user request: '{query}'.\n",
-        "        If the request is about price, use get_product_price tool.\n",
-        "        Otherwise, use get_product_details tool to get product information.\n",
-        "        \"\"\",\n",
-        "        tools=[get_product_details, get_product_price],\n",
-        "    )\n",
-        "\n",
-        "    session_service = InMemorySessionService()\n",
-        "    _ = session_service.create_session(\n",
-        "        app_name=app_name, user_id=user_id, session_id=session_id\n",
-        "    )\n",
-        "    runner = Runner(\n",
-        "        agent=product_research_agent, app_name=app_name, session_service=session_service\n",
-        "    )\n",
-        "\n",
-        "    content = types.Content(role=\"user\", parts=[types.Part(text=query)])\n",
-        "    events = list(\n",
-        "        runner.run(user_id=user_id, session_id=session_id, new_message=content)\n",
-        "    )\n",
-        "    return parse_adk_output_to_dictionary(events)"
+        "async def agent_parsed_outcome(query):\n",
+        "   app_name = \"product_research_app\"\n",
+        "   user_id = \"user_one\"\n",
+        "   session_id = \"session_one\"\n",
+        "   \n",
+        "   product_research_agent = Agent(\n",
+        "       name=\"ProductResearchAgent\",\n",
+        "       model=model,\n",
+        "       description=\"An agent that performs product research.\",\n",
+        "       instruction=f\"\"\"\n",
+        "       Analyze this user request: '{query}'.\n",
+        "       If the request is about price, use get_product_price tool.\n",
+        "       Otherwise, use get_product_details tool to get product information.\n",
+        "       \"\"\",\n",
+        "       tools=[get_product_details, get_product_price],\n",
+        "   )\n",
+        "\n",
+        "   session_service = InMemorySessionService()\n",
+        "   await session_service.create_session(\n",
+        "       app_name=app_name, user_id=user_id, session_id=session_id\n",
+        "   )\n",
+        "\n",
+        "   runner = Runner(\n",
+        "       agent=product_research_agent, app_name=app_name, session_service=session_service\n",
+        "   )\n",
+        "\n",
+        "   content = types.Content(role=\"user\", parts=[types.Part(text=query)])\n",
+        "   events = [event async for event in runner.run_async(user_id=user_id, session_id=session_id, new_message=content)]\n",
+        "   \n",
+        "   return parse_adk_output_to_dictionary(events)\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# --- Sync wrapper for Vertex‑AI evaluation\n",
+        "def agent_parsed_outcome_sync(prompt: str):\n",
+        "    result = asyncio.run(agent_parsed_outcome(prompt))\n",
+        "    result[\"predicted_trajectory\"] = json.dumps(result[\"predicted_trajectory\"])\n",
+        "    return result"
       ]
     },
     {
@@ -638,7 +642,7 @@
       },
       "outputs": [],
       "source": [
-        "response = agent_parsed_outcome(query=\"Get product details for shoes\")\n",
+        "response = await agent_parsed_outcome(query=\"Get product details for shoes\")\n",
         "display(Markdown(format_output_as_markdown(response)))"
       ]
     },
@@ -650,7 +654,7 @@
       },
       "outputs": [],
       "source": [
-        "response = agent_parsed_outcome(query=\"Get product price for shoes\")\n",
+        "response = await agent_parsed_outcome(query=\"Get product price for shoes\")\n",
         "display(Markdown(format_output_as_markdown(response)))"
       ]
     },
@@ -714,7 +718,7 @@
         "        \"Get product details and price for shoes\",\n",
         "        \"Get product details for speaker?\",\n",
         "    ],\n",
-        "    \"reference_trajectory\": [\n",
+        "    \"predicted_trajectory\": [\n",
         "        [\n",
         "            {\n",
         "                \"tool_name\": \"get_product_price\",\n",
@@ -840,7 +844,7 @@
         ")\n",
         "\n",
         "single_tool_call_eval_result = single_tool_call_eval_task.evaluate(\n",
-        "    runnable=agent_parsed_outcome, experiment_run_name=EXPERIMENT_RUN\n",
+        "    runnable=agent_parsed_outcome_sync, experiment_run_name=EXPERIMENT_RUN\n",
         ")\n",
         "\n",
         "display_eval_report(single_tool_call_eval_result)"
@@ -948,7 +952,7 @@
         ")\n",
         "\n",
         "trajectory_eval_result = trajectory_eval_task.evaluate(\n",
-        "    runnable=agent_parsed_outcome, experiment_run_name=EXPERIMENT_RUN\n",
+        "    runnable=agent_parsed_outcome_sync, experiment_run_name=EXPERIMENT_RUN\n",
         ")\n",
         "\n",
         "display_eval_report(trajectory_eval_result)"
@@ -1055,7 +1059,7 @@
         ")\n",
         "\n",
         "response_eval_result = response_eval_task.evaluate(\n",
-        "    runnable=agent_parsed_outcome, experiment_run_name=EXPERIMENT_RUN\n",
+        "    runnable=agent_parsed_outcome_sync, experiment_run_name=EXPERIMENT_RUN\n",
         ")\n",
         "\n",
         "display_eval_report(response_eval_result)"
@@ -1240,7 +1244,9 @@
         ")\n",
         "\n",
         "response_eval_tool_result = response_eval_tool_task.evaluate(\n",
-        "    runnable=agent_parsed_outcome, experiment_run_name=EXPERIMENT_RUN\n",
+        "    # Uncomment the line below if you are providing the agent with an unparsed dataset\n",
+        "    #runnable=agent_parsed_outcome_sync, \n",
+        "    experiment_run_name=EXPERIMENT_RUN\n",
         ")\n",
         "\n",
         "display_eval_report(response_eval_tool_result)"
@@ -1410,7 +1416,8 @@
         "].apply(json.dumps)\n",
         "byod_eval_sample_dataset[\"reference_trajectory\"] = byod_eval_sample_dataset[\n",
         "    \"reference_trajectory\"\n",
-        "].apply(json.dumps)"
+        "].apply(json.dumps)\n",
+        "byod_eval_sample_dataset[\"response\"] = byod_eval_sample_dataset[\"response\"].apply(json.dumps)"
       ]
     },
     {

{agent_starter_pack-0.13.1 → agent_starter_pack-0.14.1}/agents/agentic_rag/notebooks/evaluating_adk_agent.ipynb

@@ -234,6 +234,7 @@
       "outputs": [],
       "source": [
         "import json\n",
+        "import asyncio\n",
         "\n",
         "# General\n",
         "import random\n",
@@ -285,7 +286,7 @@
         "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
         "\n",
         "\n",
-        "def parse_adk_output_to_dictionary(events: list[Event]) -> dict[str, Any]:\n",
+        "def parse_adk_output_to_dictionary(events: list[Event], *, as_json: bool = False):\n",
         "    \"\"\"\n",
         "    Parse ADK event output into a structured dictionary format,\n",
         "    with the predicted trajectory dumped as a JSON string.\n",
@@ -293,63 +294,54 @@
         "    \"\"\"\n",
         "\n",
         "    final_response = \"\"\n",
-        "    predicted_trajectory_list = []\n",
+        "    trajectory = []\n",
         "\n",
         "    for event in events:\n",
-        "        # Ensure content and parts exist before accessing them\n",
-        "        if not event.content or not event.content.parts:\n",
+        "        if not getattr(event, \"content\", None) or not getattr(event.content, \"parts\", None):\n",
         "            continue\n",
-        "\n",
-        "        # Iterate through ALL parts in the event's content\n",
         "        for part in event.content.parts:\n",
-        "            if part.function_call:\n",
-        "                tool_info = {\n",
+        "            if getattr(part, \"function_call\", None):\n",
+        "                info = {\n",
         "                    \"tool_name\": part.function_call.name,\n",
         "                    \"tool_input\": dict(part.function_call.args),\n",
         "                }\n",
-        "                # Ensure we don't add duplicates if the same call appears somehow\n",
-        "                if tool_info not in predicted_trajectory_list:\n",
-        "                    predicted_trajectory_list.append(tool_info)\n",
-        "\n",
-        "            # The final text response is usually in the last event from the model\n",
-        "            if event.content.role == \"model\" and part.text:\n",
-        "                # Overwrite response; the last text response found is likely the final one\n",
+        "                if info not in trajectory:\n",
+        "                    trajectory.append(info)\n",
+        "            if event.content.role == \"model\" and getattr(part, \"text\", None):\n",
         "                final_response = part.text.strip()\n",
         "\n",
-        "    # Dump the collected trajectory list into a JSON string\n",
-        "    final_output = {\n",
-        "        \"response\": str(final_response),\n",
-        "        \"predicted_trajectory\": json.dumps(predicted_trajectory_list),\n",
-        "    }\n",
+        "    if as_json:\n",
+        "        trajectory_out = json.dumps(trajectory)\n",
+        "    else:\n",
+        "        trajectory_out = trajectory\n",
         "\n",
-        "    return final_output\n",
+        "    return {\"response\": final_response, \"predicted_trajectory\": trajectory_out}\n",
         "\n",
         "\n",
         "def format_output_as_markdown(output: dict) -> str:\n",
         "    \"\"\"Convert the output dictionary to a formatted markdown string.\"\"\"\n",
-        "    markdown = \"### AI Response\\n\"\n",
-        "    markdown += f\"{output['response']}\\n\\n\"\n",
-        "\n",
+        "    markdown = \"### AI Response\\n\" + output[\"response\"] + \"\\n\\n\"\n",
         "    if output[\"predicted_trajectory\"]:\n",
-        "        output[\"predicted_trajectory\"] = json.loads(output[\"predicted_trajectory\"])\n",
         "        markdown += \"### Function Calls\\n\"\n",
         "        for call in output[\"predicted_trajectory\"]:\n",
         "            markdown += f\"- **Function**: `{call['tool_name']}`\\n\"\n",
-        "            markdown += \"  - **Arguments**:\\n\"\n",
+        "            markdown += \"  - **Arguments**\\n\"\n",
         "            for key, value in call[\"tool_input\"].items():\n",
         "                markdown += f\"    - `{key}`: `{value}`\\n\"\n",
-        "\n",
         "    return markdown\n",
         "\n",
         "\n",
         "def display_eval_report(eval_result: pd.DataFrame) -> None:\n",
         "    \"\"\"Display the evaluation results.\"\"\"\n",
-        "    metrics_df = pd.DataFrame.from_dict(eval_result.summary_metrics, orient=\"index\").T\n",
         "    display(Markdown(\"### Summary Metrics\"))\n",
-        "    display(metrics_df)\n",
-        "\n",
-        "    display(Markdown(\"### Row-wise Metrics\"))\n",
-        "    display(eval_result.metrics_table)\n",
+        "    display(\n",
+        "        pd.DataFrame(\n",
+        "            eval_result.summary_metrics.items(), columns=[\"metric\", \"value\"]\n",
+        "        )\n",
+        "    )\n",
+        "    if getattr(eval_result, \"metrics_table\", None) is not None:\n",
+        "        display(Markdown(\"### Row‑wise Metrics\"))\n",
+        "        display(eval_result.metrics_table.head())\n",
         "\n",
         "\n",
         "def display_drilldown(row: pd.Series) -> None:\n",
@@ -586,37 +578,49 @@
       },
       "outputs": [],
       "source": [
-        "def agent_parsed_outcome(query):\n",
-        "\n",
-        "    app_name = \"product_research_app\"\n",
-        "    user_id = \"user_one\"\n",
-        "    session_id = \"session_one\"\n",
-        "\n",
-        "    product_research_agent = Agent(\n",
-        "        name=\"ProductResearchAgent\",\n",
-        "        model=model,\n",
-        "        description=\"An agent that performs product research.\",\n",
-        "        instruction=f\"\"\"\n",
-        "        Analyze this user request: '{query}'.\n",
-        "        If the request is about price, use get_product_price tool.\n",
-        "        Otherwise, use get_product_details tool to get product information.\n",
-        "        \"\"\",\n",
-        "        tools=[get_product_details, get_product_price],\n",
-        "    )\n",
-        "\n",
-        "    session_service = InMemorySessionService()\n",
-        "    _ = session_service.create_session(\n",
-        "        app_name=app_name, user_id=user_id, session_id=session_id\n",
-        "    )\n",
-        "    runner = Runner(\n",
-        "        agent=product_research_agent, app_name=app_name, session_service=session_service\n",
-        "    )\n",
-        "\n",
-        "    content = types.Content(role=\"user\", parts=[types.Part(text=query)])\n",
-        "    events = list(\n",
-        "        runner.run(user_id=user_id, session_id=session_id, new_message=content)\n",
-        "    )\n",
-        "    return parse_adk_output_to_dictionary(events)"
+        "async def agent_parsed_outcome(query):\n",
+        "   app_name = \"product_research_app\"\n",
+        "   user_id = \"user_one\"\n",
+        "   session_id = \"session_one\"\n",
+        "   \n",
+        "   product_research_agent = Agent(\n",
+        "       name=\"ProductResearchAgent\",\n",
+        "       model=model,\n",
+        "       description=\"An agent that performs product research.\",\n",
+        "       instruction=f\"\"\"\n",
+        "       Analyze this user request: '{query}'.\n",
+        "       If the request is about price, use get_product_price tool.\n",
+        "       Otherwise, use get_product_details tool to get product information.\n",
+        "       \"\"\",\n",
+        "       tools=[get_product_details, get_product_price],\n",
+        "   )\n",
+        "\n",
+        "   session_service = InMemorySessionService()\n",
+        "   await session_service.create_session(\n",
+        "       app_name=app_name, user_id=user_id, session_id=session_id\n",
+        "   )\n",
+        "\n",
+        "   runner = Runner(\n",
+        "       agent=product_research_agent, app_name=app_name, session_service=session_service\n",
+        "   )\n",
+        "\n",
+        "   content = types.Content(role=\"user\", parts=[types.Part(text=query)])\n",
+        "   events = [event async for event in runner.run_async(user_id=user_id, session_id=session_id, new_message=content)]\n",
+        "   \n",
+        "   return parse_adk_output_to_dictionary(events)\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "# --- Sync wrapper for Vertex‑AI evaluation\n",
+        "def agent_parsed_outcome_sync(prompt: str):\n",
+        "    result = asyncio.run(agent_parsed_outcome(prompt))\n",
+        "    result[\"predicted_trajectory\"] = json.dumps(result[\"predicted_trajectory\"])\n",
+        "    return result"
       ]
     },
     {
@@ -638,7 +642,7 @@
       },
       "outputs": [],
       "source": [
-        "response = agent_parsed_outcome(query=\"Get product details for shoes\")\n",
+        "response = await agent_parsed_outcome(query=\"Get product details for shoes\")\n",
         "display(Markdown(format_output_as_markdown(response)))"
       ]
     },
@@ -650,7 +654,7 @@
       },
       "outputs": [],
       "source": [
-        "response = agent_parsed_outcome(query=\"Get product price for shoes\")\n",
+        "response = await agent_parsed_outcome(query=\"Get product price for shoes\")\n",
         "display(Markdown(format_output_as_markdown(response)))"
       ]
     },
@@ -714,7 +718,7 @@
         "        \"Get product details and price for shoes\",\n",
         "        \"Get product details for speaker?\",\n",
         "    ],\n",
-        "    \"reference_trajectory\": [\n",
+        "    \"predicted_trajectory\": [\n",
         "        [\n",
         "            {\n",
         "                \"tool_name\": \"get_product_price\",\n",
@@ -840,7 +844,7 @@
         ")\n",
         "\n",
         "single_tool_call_eval_result = single_tool_call_eval_task.evaluate(\n",
-        "    runnable=agent_parsed_outcome, experiment_run_name=EXPERIMENT_RUN\n",
+        "    runnable=agent_parsed_outcome_sync, experiment_run_name=EXPERIMENT_RUN\n",
         ")\n",
         "\n",
         "display_eval_report(single_tool_call_eval_result)"
@@ -948,7 +952,7 @@
         ")\n",
         "\n",
         "trajectory_eval_result = trajectory_eval_task.evaluate(\n",
-        "    runnable=agent_parsed_outcome, experiment_run_name=EXPERIMENT_RUN\n",
+        "    runnable=agent_parsed_outcome_sync, experiment_run_name=EXPERIMENT_RUN\n",
         ")\n",
         "\n",
         "display_eval_report(trajectory_eval_result)"
@@ -1055,7 +1059,7 @@
         ")\n",
         "\n",
         "response_eval_result = response_eval_task.evaluate(\n",
-        "    runnable=agent_parsed_outcome, experiment_run_name=EXPERIMENT_RUN\n",
+        "    runnable=agent_parsed_outcome_sync, experiment_run_name=EXPERIMENT_RUN\n",
         ")\n",
         "\n",
         "display_eval_report(response_eval_result)"
@@ -1240,7 +1244,9 @@
         ")\n",
         "\n",
         "response_eval_tool_result = response_eval_tool_task.evaluate(\n",
-        "    runnable=agent_parsed_outcome, experiment_run_name=EXPERIMENT_RUN\n",
+        "    # Uncomment the line below if you are providing the agent with an unparsed dataset\n",
+        "    #runnable=agent_parsed_outcome_sync, \n",
+        "    experiment_run_name=EXPERIMENT_RUN\n",
         ")\n",
         "\n",
         "display_eval_report(response_eval_tool_result)"
@@ -1410,7 +1416,8 @@
         "].apply(json.dumps)\n",
         "byod_eval_sample_dataset[\"reference_trajectory\"] = byod_eval_sample_dataset[\n",
         "    \"reference_trajectory\"\n",
-        "].apply(json.dumps)"
+        "].apply(json.dumps)\n",
+        "byod_eval_sample_dataset[\"response\"] = byod_eval_sample_dataset[\"response\"].apply(json.dumps)"
       ]
     },
     {