agent-starter-pack 0.3.2__py3-none-any.whl → 0.3.4__py3-none-any.whl

agents/crewai_coding_crew/notebooks/evaluating_langgraph_agent.ipynb

@@ -143,12 +143,8 @@
       },
       "outputs": [],
       "source": [
-        "%pip install --upgrade --user --quiet \"google-cloud-aiplatform[evaluation]\" \\\n",
-        "    \"langchain_google_vertexai\" \\\n",
-        "    \"langgraph\" \\\n",
-        "    \"cloudpickle==3.0.0\" \\\n",
-        "    \"pydantic==2.7.4\" \\\n",
-        "    \"requests\""
+        "%pip install \"langchain_google_vertexai\" \"langgraph\"\n",
+        "%pip install --upgrade --user --quiet \"google-cloud-aiplatform[evaluation]\""
       ]
     },
     {
@@ -273,6 +269,8 @@
       },
       "outputs": [],
       "source": [
+        "import json\n",
+        "\n",
         "# General\n",
         "import random\n",
         "import string\n",
@@ -355,7 +353,7 @@
         "                        }\n",
         "                    )\n",
         "\n",
-        "    final_output[\"predicted_trajectory\"] = function_calls\n",
+        "    final_output[\"predicted_trajectory\"] = json.dumps(function_calls)\n",
         "    return final_output\n",
         "\n",
         "\n",
@@ -365,6 +363,7 @@
         "    markdown += f\"{output['response']}\\n\\n\"\n",
         "\n",
         "    if output[\"predicted_trajectory\"]:\n",
+        "        output[\"predicted_trajectory\"] = json.loads(output[\"predicted_trajectory\"])\n",
         "        markdown += \"### Function Calls\\n\"\n",
         "        for call in output[\"predicted_trajectory\"]:\n",
         "            markdown += f\"- **Function**: `{call['tool_name']}`\\n\"\n",
@@ -381,7 +380,7 @@
         "    display(Markdown(\"### Summary Metrics\"))\n",
         "    display(metrics_df)\n",
         "\n",
-        "    display(Markdown(f\"### Row-wise Metrics\"))\n",
+        "    display(Markdown(\"### Row-wise Metrics\"))\n",
         "    display(eval_result.metrics_table)\n",
         "\n",
         "\n",
@@ -599,7 +598,7 @@
       "source": [
         "def router(\n",
         "    state: list[BaseMessage],\n",
-        ") -> Literal[\"get_product_details\", \"get_product_price\", \"__end__\"]:\n",
+        ") -> Literal[\"get_product_details\", \"get_product_price\", END]:\n",
         "    \"\"\"Initiates product details or price retrieval if the user asks for a product.\"\"\"\n",
         "    # Get the tool_calls from the last message in the conversation history.\n",
         "    tool_calls = state[-1].tool_calls\n",
@@ -614,7 +613,7 @@
         "            return \"get_product_details\"\n",
         "    else:\n",
         "        # End the conversation flow.\n",
-        "        return \"__end__\""
+        "        return END"
       ]
     },
     {
@@ -636,7 +635,7 @@
       },
       "outputs": [],
       "source": [
-        "llm = \"gemini-1.5-pro\""
+        "llm = \"gemini-2.0-flash\""
       ]
     },
     {
@@ -899,6 +898,7 @@
         "    dataset=eval_sample_dataset,\n",
         "    metrics=single_tool_usage_metrics,\n",
         "    experiment=EXPERIMENT_NAME,\n",
+        "    output_uri_prefix=BUCKET_URI + \"/single-metric-eval\",\n",
         ")\n",
         "\n",
         "single_tool_call_eval_result = single_tool_call_eval_task.evaluate(\n",
@@ -1003,7 +1003,10 @@
         "EXPERIMENT_RUN = f\"trajectory-{get_id()}\"\n",
         "\n",
         "trajectory_eval_task = EvalTask(\n",
-        "    dataset=eval_sample_dataset, metrics=trajectory_metrics, experiment=EXPERIMENT_NAME\n",
+        "    dataset=eval_sample_dataset,\n",
+        "    metrics=trajectory_metrics,\n",
+        "    experiment=EXPERIMENT_NAME,\n",
+        "    output_uri_prefix=BUCKET_URI + \"/multiple-metric-eval\",\n",
         ")\n",
         "\n",
         "trajectory_eval_result = trajectory_eval_task.evaluate(\n",
@@ -1107,7 +1110,10 @@
         "EXPERIMENT_RUN = f\"response-{get_id()}\"\n",
         "\n",
         "response_eval_task = EvalTask(\n",
-        "    dataset=eval_sample_dataset, metrics=response_metrics, experiment=EXPERIMENT_NAME\n",
+        "    dataset=eval_sample_dataset,\n",
+        "    metrics=response_metrics,\n",
+        "    experiment=EXPERIMENT_NAME,\n",
+        "    output_uri_prefix=BUCKET_URI + \"/response-metric-eval\",\n",
         ")\n",
         "\n",
         "response_eval_result = response_eval_task.evaluate(\n",
@@ -1292,6 +1298,7 @@
         "    dataset=eval_sample_dataset,\n",
         "    metrics=response_tool_metrics,\n",
         "    experiment=EXPERIMENT_NAME,\n",
+        "    output_uri_prefix=BUCKET_URI + \"/reasoning-metric-eval\",\n",
         ")\n",
         "\n",
         "response_eval_tool_result = response_eval_tool_task.evaluate(\n",
@@ -1436,15 +1443,21 @@
         "        ],\n",
         "    ],\n",
         "    \"response\": [\n",
-        "        500,\n",
-        "        50,\n",
+        "        \"500\",\n",
+        "        \"50\",\n",
         "        \"A super fast and light usb charger\",\n",
-        "        100,\n",
+        "        \"100\",\n",
         "        \"A voice-controlled smart speaker that plays music, sets alarms, and controls smart home devices.\",\n",
         "    ],\n",
         "}\n",
         "\n",
-        "byod_eval_sample_dataset = pd.DataFrame(byod_eval_data)"
+        "byod_eval_sample_dataset = pd.DataFrame(byod_eval_data)\n",
+        "byod_eval_sample_dataset[\"predicted_trajectory\"] = byod_eval_sample_dataset[\n",
+        "    \"predicted_trajectory\"\n",
+        "].apply(json.dumps)\n",
+        "byod_eval_sample_dataset[\"reference_trajectory\"] = byod_eval_sample_dataset[\n",
+        "    \"reference_trajectory\"\n",
+        "].apply(json.dumps)"
       ]
     },
     {

agents/langgraph_base_react/notebooks/evaluating_langgraph_agent.ipynb

@@ -143,12 +143,8 @@
       },
       "outputs": [],
       "source": [
-        "%pip install --upgrade --user --quiet \"google-cloud-aiplatform[evaluation]\" \\\n",
-        "    \"langchain_google_vertexai\" \\\n",
-        "    \"langgraph\" \\\n",
-        "    \"cloudpickle==3.0.0\" \\\n",
-        "    \"pydantic==2.7.4\" \\\n",
-        "    \"requests\""
+        "%pip install \"langchain_google_vertexai\" \"langgraph\"\n",
+        "%pip install --upgrade --user --quiet \"google-cloud-aiplatform[evaluation]\""
       ]
     },
     {
@@ -273,6 +269,8 @@
       },
       "outputs": [],
       "source": [
+        "import json\n",
+        "\n",
         "# General\n",
         "import random\n",
         "import string\n",
@@ -355,7 +353,7 @@
         "                        }\n",
         "                    )\n",
         "\n",
-        "    final_output[\"predicted_trajectory\"] = function_calls\n",
+        "    final_output[\"predicted_trajectory\"] = json.dumps(function_calls)\n",
         "    return final_output\n",
         "\n",
         "\n",
@@ -365,6 +363,7 @@
         "    markdown += f\"{output['response']}\\n\\n\"\n",
         "\n",
         "    if output[\"predicted_trajectory\"]:\n",
+        "        output[\"predicted_trajectory\"] = json.loads(output[\"predicted_trajectory\"])\n",
         "        markdown += \"### Function Calls\\n\"\n",
         "        for call in output[\"predicted_trajectory\"]:\n",
         "            markdown += f\"- **Function**: `{call['tool_name']}`\\n\"\n",
@@ -381,7 +380,7 @@
         "    display(Markdown(\"### Summary Metrics\"))\n",
         "    display(metrics_df)\n",
         "\n",
-        "    display(Markdown(f\"### Row-wise Metrics\"))\n",
+        "    display(Markdown(\"### Row-wise Metrics\"))\n",
         "    display(eval_result.metrics_table)\n",
         "\n",
         "\n",
@@ -599,7 +598,7 @@
       "source": [
         "def router(\n",
         "    state: list[BaseMessage],\n",
-        ") -> Literal[\"get_product_details\", \"get_product_price\", \"__end__\"]:\n",
+        ") -> Literal[\"get_product_details\", \"get_product_price\", END]:\n",
         "    \"\"\"Initiates product details or price retrieval if the user asks for a product.\"\"\"\n",
         "    # Get the tool_calls from the last message in the conversation history.\n",
         "    tool_calls = state[-1].tool_calls\n",
@@ -614,7 +613,7 @@
         "            return \"get_product_details\"\n",
         "    else:\n",
         "        # End the conversation flow.\n",
-        "        return \"__end__\""
+        "        return END"
       ]
     },
     {
@@ -636,7 +635,7 @@
       },
       "outputs": [],
       "source": [
-        "llm = \"gemini-1.5-pro\""
+        "llm = \"gemini-2.0-flash\""
       ]
     },
     {
@@ -899,6 +898,7 @@
         "    dataset=eval_sample_dataset,\n",
         "    metrics=single_tool_usage_metrics,\n",
         "    experiment=EXPERIMENT_NAME,\n",
+        "    output_uri_prefix=BUCKET_URI + \"/single-metric-eval\",\n",
         ")\n",
         "\n",
         "single_tool_call_eval_result = single_tool_call_eval_task.evaluate(\n",
@@ -1003,7 +1003,10 @@
         "EXPERIMENT_RUN = f\"trajectory-{get_id()}\"\n",
         "\n",
         "trajectory_eval_task = EvalTask(\n",
-        "    dataset=eval_sample_dataset, metrics=trajectory_metrics, experiment=EXPERIMENT_NAME\n",
+        "    dataset=eval_sample_dataset,\n",
+        "    metrics=trajectory_metrics,\n",
+        "    experiment=EXPERIMENT_NAME,\n",
+        "    output_uri_prefix=BUCKET_URI + \"/multiple-metric-eval\",\n",
         ")\n",
         "\n",
         "trajectory_eval_result = trajectory_eval_task.evaluate(\n",
@@ -1107,7 +1110,10 @@
         "EXPERIMENT_RUN = f\"response-{get_id()}\"\n",
         "\n",
         "response_eval_task = EvalTask(\n",
-        "    dataset=eval_sample_dataset, metrics=response_metrics, experiment=EXPERIMENT_NAME\n",
+        "    dataset=eval_sample_dataset,\n",
+        "    metrics=response_metrics,\n",
+        "    experiment=EXPERIMENT_NAME,\n",
+        "    output_uri_prefix=BUCKET_URI + \"/response-metric-eval\",\n",
         ")\n",
         "\n",
         "response_eval_result = response_eval_task.evaluate(\n",
@@ -1292,6 +1298,7 @@
         "    dataset=eval_sample_dataset,\n",
         "    metrics=response_tool_metrics,\n",
         "    experiment=EXPERIMENT_NAME,\n",
+        "    output_uri_prefix=BUCKET_URI + \"/reasoning-metric-eval\",\n",
         ")\n",
         "\n",
         "response_eval_tool_result = response_eval_tool_task.evaluate(\n",
@@ -1436,15 +1443,21 @@
         "        ],\n",
         "    ],\n",
         "    \"response\": [\n",
-        "        500,\n",
-        "        50,\n",
+        "        \"500\",\n",
+        "        \"50\",\n",
         "        \"A super fast and light usb charger\",\n",
-        "        100,\n",
+        "        \"100\",\n",
         "        \"A voice-controlled smart speaker that plays music, sets alarms, and controls smart home devices.\",\n",
         "    ],\n",
         "}\n",
         "\n",
-        "byod_eval_sample_dataset = pd.DataFrame(byod_eval_data)"
+        "byod_eval_sample_dataset = pd.DataFrame(byod_eval_data)\n",
+        "byod_eval_sample_dataset[\"predicted_trajectory\"] = byod_eval_sample_dataset[\n",
+        "    \"predicted_trajectory\"\n",
+        "].apply(json.dumps)\n",
+        "byod_eval_sample_dataset[\"reference_trajectory\"] = byod_eval_sample_dataset[\n",
+        "    \"reference_trajectory\"\n",
+        "].apply(json.dumps)"
       ]
     },
     {

src/base_template/Makefile

@@ -1,6 +1,6 @@
 install:
 	@command -v uv >/dev/null 2>&1 || { echo "uv is not installed. Installing uv..."; curl -LsSf https://astral.sh/uv/0.6.12/install.sh | sh; source ~/.bashrc; }
-	uv sync --dev {% if cookiecutter.agent_name != 'live_api' %}--extra streamlit{%- endif %} --extra jupyter --frozen{% if cookiecutter.agent_name == 'live_api' %} && npm --prefix frontend install{%- endif %}
+	uv sync --dev{% if cookiecutter.agent_name != 'live_api' and "adk" not in cookiecutter.tags %} --extra streamlit{%- endif %} --extra jupyter --frozen{% if cookiecutter.agent_name == 'live_api' %} && npm --prefix frontend install{%- endif %}
 
 test:
 	uv run pytest tests/unit && uv run pytest tests/integration

src/base_template/deployment/cd/deploy-to-prod.yaml

@@ -66,6 +66,7 @@ steps:
       - "40"
       - "--service-account"
       - "${_CLOUD_RUN_APP_SA_EMAIL}"
+      - "--session-affinity"
       - "--set-env-vars"
       - "COMMIT_SHA=${COMMIT_SHA}{%- if cookiecutter.data_ingestion %}{%- if cookiecutter.datastore_type == "vertex_ai_search" %},DATA_STORE_ID=${_DATA_STORE_ID},DATA_STORE_REGION=${_DATA_STORE_REGION}{%- elif cookiecutter.datastore_type == "vertex_ai_vector_search" %},VECTOR_SEARCH_INDEX=${_VECTOR_SEARCH_INDEX},VECTOR_SEARCH_INDEX_ENDPOINT=${_VECTOR_SEARCH_INDEX_ENDPOINT},VECTOR_SEARCH_BUCKET=${_VECTOR_SEARCH_BUCKET}{%- endif %}{%- endif %}"
 {%- elif cookiecutter.deployment_target == 'agent_engine' %}

src/base_template/deployment/cd/staging.yaml

@@ -80,6 +80,7 @@ steps:
       - "40"
       - "--service-account"
       - "${_CLOUD_RUN_APP_SA_EMAIL}"
+      - "--session-affinity"
       - "--set-env-vars"
       - "COMMIT_SHA=${COMMIT_SHA}{%- if cookiecutter.data_ingestion %}{%- if cookiecutter.datastore_type == "vertex_ai_search" %},DATA_STORE_ID=${_DATA_STORE_ID},DATA_STORE_REGION=${_DATA_STORE_REGION}{%- elif cookiecutter.datastore_type == "vertex_ai_vector_search" %},VECTOR_SEARCH_INDEX=${_VECTOR_SEARCH_INDEX},VECTOR_SEARCH_INDEX_ENDPOINT=${_VECTOR_SEARCH_INDEX_ENDPOINT},VECTOR_SEARCH_BUCKET=${_VECTOR_SEARCH_BUCKET}{%- endif %}{%- endif %}"
 

src/base_template/pyproject.toml

@@ -37,12 +37,14 @@ dev = [
 ]
 
 [project.optional-dependencies]
+{% if cookiecutter.agent_name != 'live_api' and "adk" not in cookiecutter.tags %}
 streamlit = [
     "streamlit~=1.42.0",
     "streamlit-extras~=0.4.3",
     "extra-streamlit-components~=0.1.71",
     "streamlit-feedback~=0.1.3",
 ]
+{% endif %}
 jupyter = [
     "jupyter~=1.0.0",
 ]

src/resources/idx/.idx/dev.nix

@@ -0,0 +1,57 @@
+
+# To learn more about how to use Nix to configure your environment
+# see: https://firebase.google.com/docs/studio/customize-workspace
+{ pkgs, ... }: {
+  # Which nixpkgs channel to use.
+  channel = "stable-24.11"; # or "unstable"
+
+  # Use https://search.nixos.org/packages to find packages
+  packages = [
+    pkgs.python311
+    pkgs.python311Packages.pip
+    pkgs.python311Packages.uv
+    pkgs.gnumake
+    pkgs.terraform
+    pkgs.gh
+  ];
+  # Sets environment variables in the workspace
+  env = {};
+  idx = {
+    # Search for the extensions you want on https://open-vsx.org/ and use "publisher.id"
+    extensions = [
+      "ms-toolsai.jupyter"
+      "ms-python.python"
+      "krish-r.vscode-toggle-terminal"
+    ];
+    workspace = {
+      # Runs when a workspace is first created with this `dev.nix` file
+      onCreate = {
+        create-venv = ''
+        # Load environment variables from .env file if it exists
+        source .env
+        echo "Logging into gcloud..."
+        echo "Please authenticate with Google Cloud by following the prompts."
+        gcloud auth login --update-adc --brief --quiet
+
+        echo "Setting gcloud project..."
+        gcloud config set project $GOOGLE_CLOUD_PROJECT
+
+        echo "Creating Python virtual environment and installing packages..."
+        uv venv && uv pip install agent-starter-pack
+        echo "alias agent-starter-pack=\"~/$WS_NAME/.venv/bin/agent-starter-pack\"" >> ~/.bashrc
+        source ~/.bashrc
+
+        echo "Running agent starter pack creation..."
+        uv run agent-starter-pack create $AGENT_NAME
+        code ~/$WS_NAME/$AGENT_NAME/README.md
+        exec bash
+        '';
+        # Open editors for the following files by default, if they exist:
+        default.openFiles = [];
+      };
+      # To run something each time the workspace is (re)started, use the `onStart` hook
+    };
+    # Enable previews and customize configuration
+    previews = {};
+  };
+}

src/resources/idx/idx-template.json

@@ -0,0 +1,21 @@
+{
+  "name": "Agent Starter Pack",
+  "description": "Production-ready Gen AI Agent templates for Google Cloud. Addressing common challenges (Deployment & Operations, Evaluation, Customization, Observability) in building and deploying GenAI agents.",
+  "icon": "https://github.com/GoogleCloudPlatform/agent-starter-pack/blob/main/docs/images/icon.png?raw=true",
+  "params": [
+    {
+      "id": "agent_name",
+      "name": "Agent Name",
+      "description": "The name of the agent being created.",
+      "type": "string",
+      "required": true
+    },
+    {
+      "id": "google_cloud_project_id",
+      "name": "Your Google Cloud Project ID",
+      "description": "The Google Cloud Platform Project ID where resources will be managed.",
+      "type": "string",
+      "required": true
+    }
+  ]
+}

src/resources/idx/idx-template.nix

@@ -0,0 +1,26 @@
+# No user-configurable parameters
+# Accept additional arguments to this template corresponding to template
+# parameter IDs
+{ pkgs, agent_name ? "", google_cloud_project_id ? "", ... }: {
+  # Shell script that produces the final environment
+  bootstrap = ''
+    # Copy the folder containing the `idx-template` files to the final
+    # project folder for the new workspace. ${./.} inserts the directory
+    # of the checked-out Git folder containing this template.
+    cp -rf ${./.} "$out"
+
+    # Set some permissions
+    chmod -R +w "$out"
+
+    # Create .env file with the parameter values
+    cat > "$out/.env" << EOF
+    AGENT_NAME=${agent_name}
+    GOOGLE_CLOUD_PROJECT=${google_cloud_project_id}
+    WS_NAME=$WS_NAME
+    EOF
+
+    # Remove the template files themselves and any connection to the template's
+    # Git repository
+    rm -rf "$out/.git" "$out/idx-template".{nix,json}
+  '';
+}