agent-starter-pack 0.18.2__py3-none-any.whl → 0.21.0__py3-none-any.whl

agent_starter_pack/base_template/README.md

@@ -12,11 +12,15 @@ This project is organized as follows:
 ├── {{cookiecutter.agent_directory}}/                 # Core application code
 │   ├── agent.py         # Main agent logic
 {%- if cookiecutter.deployment_target == 'cloud_run' %}
-│   ├── server.py        # FastAPI Backend server
+│   ├── fast_api_app.py  # FastAPI Backend server
 {%- elif cookiecutter.deployment_target == 'agent_engine' %}
 │   ├── agent_engine_app.py # Agent Engine application logic
 {%- endif %}
-│   └── utils/           # Utility functions and helpers
+│   └── app_utils/       # App utilities and helpers
+{%- if cookiecutter.is_a2a and cookiecutter.agent_name == 'langgraph_base' %}
+│       ├── executor/    # A2A protocol executor implementation
+│       └── converters/  # Message converters for A2A protocol
+{%- endif %}
 {%- if cookiecutter.cicd_runner == 'google_cloud_build' %}
 ├── .cloudbuild/         # CI/CD pipeline configurations for Google Cloud Build
 {%- elif cookiecutter.cicd_runner == 'github_actions' %}
@@ -62,7 +66,7 @@ make install && make playground
 | `make deploy`        | Deploy agent to Cloud Run (use `IAP=true` to enable Identity-Aware Proxy, `PORT=8080` to specify container port) |
 | `make local-backend` | Launch local development server with hot-reload |
 {%- elif cookiecutter.deployment_target == 'agent_engine' %}
-| `make playground`    | Launch Streamlit interface for testing agent locally and remotely |
+| `make playground`    | Launch local development environment for testing agent |
 | `make deploy`        | Deploy agent to Agent Engine |
 {%- if cookiecutter.is_adk_live %}
 | `make local-backend` | Launch local development server with hot-reload |
@@ -71,9 +75,12 @@ make install && make playground
 | `make playground-remote` | Connect to remote deployed agent with local frontend |
 | `make build-frontend` | Build the frontend for production |
 {%- endif %}
-{%- if cookiecutter.is_adk %}
-| `make register-gemini-enterprise` | Register deployed agent to Gemini Enterprise (see Makefile for parameters) |
-{%- endif %}
+{%- if cookiecutter.is_adk or cookiecutter.is_a2a %}
+| `make register-gemini-enterprise` | Register deployed agent to Gemini Enterprise ([docs](https://googlecloudplatform.github.io/agent-starter-pack/cli/register_gemini_enterprise.html)) |
+{%- endif -%}
+{%- endif -%}
+{%- if cookiecutter.is_a2a %}
+| `make inspector`     | Launch A2A Protocol Inspector to test your agent implementation                             |
 {%- endif %}
 | `make test`          | Run unit and integration tests                                                              |
 | `make lint`          | Run code quality checks (codespell, ruff, mypy)                                             |
@@ -84,6 +91,72 @@ make install && make playground
 
 For full command options and usage, refer to the [Makefile](Makefile).
 
+{%- if cookiecutter.is_a2a %}
+
+## Using the A2A Inspector
+
+This agent implements the [Agent2Agent (A2A) Protocol](https://a2a-protocol.org/), enabling interoperability with agents across different frameworks and languages.
+
+The [A2A Inspector](https://github.com/a2aproject/a2a-inspector) provides the following core features:
+- 🔍 View agent card and capabilities
+- ✅ Validate A2A specification compliance
+- 💬 Test communication with live chat interface
+- 🐛 Debug with the raw message console
+
+### Local Testing
+{%- if cookiecutter.deployment_target == 'cloud_run' %}
+
+1. Start your agent:
+   ```bash
+   make local-backend
+   ```
+
+2. In a separate terminal, launch the A2A Protocol Inspector:
+   ```bash
+   make inspector
+   ```
+
+3. Open http://localhost:5001 and connect to `http://localhost:8000`
+{%- else %}
+
+> **Note:** For Agent Engine deployments, local testing with A2A endpoints requires deployment first, as `make playground` uses the ADK web interface. For local development, use `make playground`. To test A2A protocol compliance, follow the Remote Testing instructions below.
+{%- endif %}
+
+### Remote Testing
+
+1. Deploy your agent:
+   ```bash
+   make deploy
+   ```
+
+2. Launch the inspector:
+   ```bash
+   make inspector
+   ```
+
+3. Get an authentication token:
+   ```bash
+{%- if cookiecutter.deployment_target == 'cloud_run' %}
+   gcloud auth print-identity-token
+{%- else %}
+   gcloud auth print-access-token
+{%- endif %}
+   ```
+
+4. In the inspector UI at http://localhost:5001:
+   - Add an HTTP header with name: `Authorization`
+   - Set the value to: `Bearer <your-token-from-step-3>`
+{%- if cookiecutter.deployment_target == 'cloud_run' %}
+   - Connect to your deployed Cloud Run URL
+{%- else %}
+   - Connect to your Agent Engine URL using this format:
+     ```
+     https://us-central1-aiplatform.googleapis.com/v1beta1/projects/{PROJECT_ID}/locations/{REGION}/reasoningEngines/{ENGINE_ID}/a2a/v1/card
+     ```
+     Find your `PROJECT_ID`, `REGION`, and `ENGINE_ID` in the `latest_deployment_metadata.json` file created after deployment.
+{%- endif %}
+{%- endif %}
+
 {% if cookiecutter.is_adk_live %}
 ## Usage
 
@@ -124,7 +197,7 @@ Here’s the recommended workflow for local development:
     ```bash
     make ui
     ```
-    This launches the Streamlit application, which connects to the backend server at `http://localhost:8000`.
+    This launches the frontend application, which connects to the backend server at `http://localhost:8000`.
     </details>
     <br>
 
@@ -143,9 +216,9 @@ This template follows a "bring your own agent" approach - you focus on your busi
 
 1. **Prototype:** Build your Generative AI Agent using the intro notebooks in `notebooks/` for guidance. Use Vertex AI Evaluation to assess performance.
 2. **Integrate:** Import your agent into the app by editing `{{cookiecutter.agent_directory}}/agent.py`.
-3. **Test:** Explore your agent functionality using the Streamlit playground with `make playground`. The playground offers features like chat history, user feedback, and various input types, and automatically reloads your agent on code changes.
+3. **Test:** Explore your agent functionality using the local playground with `make playground`. The playground automatically reloads your agent on code changes.
 4. **Deploy:** Set up and initiate the CI/CD pipelines, customizing tests as necessary. Refer to the [deployment section](#deployment) for comprehensive instructions. For streamlined infrastructure deployment, simply run `uvx agent-starter-pack setup-cicd`. Check out the [`agent-starter-pack setup-cicd` CLI command](https://googlecloudplatform.github.io/agent-starter-pack/cli/setup_cicd.html). Currently supports GitHub with both Google Cloud Build and GitHub Actions as CI/CD runners.
-5. **Monitor:** Track performance and gather insights using Cloud Logging, Tracing, and the Looker Studio dashboard to iterate on your application.
+5. **Monitor:** Track performance and gather insights using BigQuery telemetry data, Cloud Logging, and Cloud Trace to iterate on your application.
 
 The project includes a `GEMINI.md` file that provides context for AI tools like Gemini CLI when asking questions about your template.
 {% endif %}
@@ -175,8 +248,20 @@ The repository includes a Terraform configuration for the setup of a production
 
 {% if not cookiecutter.is_adk_live %}
 ## Monitoring and Observability
-> You can use [this Looker Studio dashboard]({%- if cookiecutter.is_adk %}https://lookerstudio.google.com/reporting/46b35167-b38b-4e44-bd37-701ef4307418/page/tEnnC{%- else %}https://lookerstudio.google.com/c/reporting/fa742264-4b4b-4c56-81e6-a667dd0f853f/page/tEnnC{%- endif %}
-) template for visualizing events being logged in BigQuery. See the "Setup Instructions" tab to getting started.
 
-The application uses OpenTelemetry for comprehensive observability with all events being sent to Google Cloud Trace and Logging for monitoring and to BigQuery for long term storage. 
+The application uses [OpenTelemetry GenAI instrumentation](https://opentelemetry.io/docs/specs/semconv/gen-ai/) for comprehensive observability. Telemetry data is automatically captured and exported to:
+
+- **Google Cloud Storage**: GenAI telemetry in JSONL format for efficient querying
+- **BigQuery**: External tables and linked datasets provide immediate access to telemetry data via SQL queries
+- **Cloud Logging**: Dedicated logging bucket with 10-year retention for GenAI operation logs
+
+**Query your telemetry data:**
+
+```bash
+# Example: Query recent completions
+bq query --use_legacy_sql=false \
+  "SELECT * FROM \`{{cookiecutter.project_name}}_telemetry.completions\` LIMIT 10"
+```
+
+For detailed setup instructions, example queries, testing in dev, and optional dashboard visualization, see the [starter pack observability guide](https://googlecloudplatform.github.io/agent-starter-pack/guide/observability.html).
 {%- endif %}

agent_starter_pack/base_template/deployment/terraform/dev/apis.tf

@@ -25,12 +25,10 @@ locals {
     "serviceusage.googleapis.com",
     "logging.googleapis.com",
     "cloudtrace.googleapis.com",
-{%- if cookiecutter.is_adk and cookiecutter.session_type == "alloydb" %}
-    "compute.googleapis.com",
-    "servicenetworking.googleapis.com",
-    "alloydb.googleapis.com",
-    "secretmanager.googleapis.com",
-    "dns.googleapis.com"
+    "telemetry.googleapis.com",
+{%- if cookiecutter.is_adk and cookiecutter.session_type == "cloud_sql" %}
+    "sqladmin.googleapis.com",
+    "secretmanager.googleapis.com"
 {%- endif %}
   ]
 }

agent_starter_pack/base_template/deployment/terraform/dev/providers.tf

@@ -17,7 +17,11 @@ terraform {
   required_providers {
     google = {
       source  = "hashicorp/google"
-      version = "> 7.0.0"
+      version = "~> 7.10.0"
+    }
+    random = {
+      source  = "hashicorp/random"
+      version = "~> 3.7.0"
     }
   }
 }

agent_starter_pack/base_template/deployment/terraform/dev/variables.tf

@@ -49,15 +49,17 @@ variable "app_sa_roles" {
   description = "List of roles to assign to the application service account"
   type        = list(string)
   default = [
-{%- if cookiecutter.session_type == "alloydb" %}
-    "roles/secretmanager.secretAccessor",
-{%- endif %}
+
     "roles/aiplatform.user",
     "roles/discoveryengine.editor",
     "roles/logging.logWriter",
     "roles/cloudtrace.agent",
     "roles/storage.admin",
     "roles/serviceusage.serviceUsageConsumer",
+{%- if cookiecutter.session_type == "cloud_sql" %}
+    "roles/cloudsql.client",
+    "roles/secretmanager.secretAccessor",
+{%- endif %}
   ]
 }
 {% if cookiecutter.data_ingestion %}

agent_starter_pack/base_template/deployment/terraform/dev/{% if cookiecutter.is_adk %}telemetry.tf{% else %}unused_telemetry.tf{% endif %}

@@ -0,0 +1,193 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# BigQuery dataset for telemetry external tables
+resource "google_bigquery_dataset" "telemetry_dataset" {
+  project       = var.dev_project_id
+  dataset_id    = replace("${var.project_name}_telemetry", "-", "_")
+  friendly_name = "${var.project_name} Telemetry"
+  location      = var.region
+  description   = "Dataset for GenAI telemetry data stored in GCS"
+  depends_on    = [google_project_service.services]
+}
+
+# BigQuery connection for accessing GCS telemetry data
+resource "google_bigquery_connection" "genai_telemetry_connection" {
+  project       = var.dev_project_id
+  location      = var.region
+  connection_id = "${var.project_name}-genai-telemetry"
+  friendly_name = "${var.project_name} GenAI Telemetry Connection"
+
+  cloud_resource {}
+
+  depends_on = [google_project_service.services]
+}
+
+# Wait for the BigQuery connection service account to propagate in IAM
+resource "time_sleep" "wait_for_bq_connection_sa" {
+  create_duration = "10s"
+
+  depends_on = [google_bigquery_connection.genai_telemetry_connection]
+}
+
+# Grant the BigQuery connection service account access to read from the logs bucket
+resource "google_storage_bucket_iam_member" "telemetry_connection_access" {
+  bucket = google_storage_bucket.logs_data_bucket.name
+  role   = "roles/storage.objectViewer"
+  member = "serviceAccount:${google_bigquery_connection.genai_telemetry_connection.cloud_resource[0].service_account_id}"
+
+  depends_on = [time_sleep.wait_for_bq_connection_sa]
+}
+
+# ====================================================================
+# Dedicated Cloud Logging Bucket for GenAI Telemetry
+# ====================================================================
+
+# Create a custom Cloud Logging bucket for GenAI telemetry logs with long-term retention
+resource "google_logging_project_bucket_config" "genai_telemetry_bucket" {
+  project          = var.dev_project_id
+  location         = var.region
+  bucket_id        = "${var.project_name}-genai-telemetry"
+  retention_days   = 3650  # 10 years retention (maximum allowed)
+  enable_analytics = true  # Required for linked datasets
+  description      = "Dedicated Cloud Logging bucket for ${var.project_name} GenAI telemetry with 10 year retention"
+
+  depends_on = [google_project_service.services]
+}
+
+# Log sink to route only GenAI telemetry logs to the dedicated bucket
+# Filter by bucket name in the GCS path (which includes project_name) to isolate this agent's logs
+resource "google_logging_project_sink" "genai_logs_to_bucket" {
+  name        = "${var.project_name}-genai-logs"
+  project     = var.dev_project_id
+  destination = "logging.googleapis.com/projects/${var.dev_project_id}/locations/${var.region}/buckets/${google_logging_project_bucket_config.genai_telemetry_bucket.bucket_id}"
+  filter      = "log_name=\"projects/${var.dev_project_id}/logs/gen_ai.client.inference.operation.details\" AND (labels.\"gen_ai.input.messages_ref\" =~ \".*${var.project_name}.*\" OR labels.\"gen_ai.output.messages_ref\" =~ \".*${var.project_name}.*\")"
+
+  unique_writer_identity = true
+  depends_on             = [google_logging_project_bucket_config.genai_telemetry_bucket]
+}
+
+# Create a linked dataset to the GenAI telemetry logs bucket for querying via BigQuery
+resource "google_logging_linked_dataset" "genai_logs_linked_dataset" {
+  link_id     = replace("${var.project_name}_genai_telemetry_logs", "-", "_")
+  bucket      = google_logging_project_bucket_config.genai_telemetry_bucket.bucket_id
+  description = "Linked dataset for ${var.project_name} GenAI telemetry Cloud Logging bucket"
+  location    = var.region
+  parent      = "projects/${var.dev_project_id}"
+
+  depends_on = [
+    google_logging_project_bucket_config.genai_telemetry_bucket,
+    google_logging_project_sink.genai_logs_to_bucket
+  ]
+}
+
+# Wait for linked dataset to fully propagate
+resource "time_sleep" "wait_for_linked_dataset" {
+  create_duration = "10s"
+
+  depends_on = [google_logging_linked_dataset.genai_logs_linked_dataset]
+}
+
+# ====================================================================
+# Feedback Logs to Cloud Logging Bucket
+# ====================================================================
+
+# Log sink for user feedback logs - routes to the same Cloud Logging bucket
+resource "google_logging_project_sink" "feedback_logs_to_bucket" {
+  name        = "${var.project_name}-feedback"
+  project     = var.dev_project_id
+  destination = "logging.googleapis.com/projects/${var.dev_project_id}/locations/${var.region}/buckets/${google_logging_project_bucket_config.genai_telemetry_bucket.bucket_id}"
+  filter      = var.feedback_logs_filter
+
+  unique_writer_identity = true
+  depends_on             = [google_logging_project_bucket_config.genai_telemetry_bucket]
+}
+
+# ====================================================================
+# Completions External Table (GCS-based)
+# ====================================================================
+
+# External table for completions data (messages/parts) stored in GCS
+resource "google_bigquery_table" "completions_external_table" {
+  project             = var.dev_project_id
+  dataset_id          = google_bigquery_dataset.telemetry_dataset.dataset_id
+  table_id            = "completions"
+  deletion_protection = false
+
+  external_data_configuration {
+    autodetect            = false
+    source_format         = "NEWLINE_DELIMITED_JSON"
+    source_uris           = ["gs://${google_storage_bucket.logs_data_bucket.name}/completions/*"]
+    connection_id         = google_bigquery_connection.genai_telemetry_connection.name
+    ignore_unknown_values = true
+    max_bad_records       = 1000
+  }
+
+  # Schema matching the ADK completions format
+  schema = jsonencode([
+    {
+      name = "parts"
+      type = "RECORD"
+      mode = "REPEATED"
+      fields = [
+        { name = "type", type = "STRING", mode = "NULLABLE" },
+        { name = "content", type = "STRING", mode = "NULLABLE" },
+        { name = "mime_type", type = "STRING", mode = "NULLABLE" },
+        { name = "uri", type = "STRING", mode = "NULLABLE" },
+        { name = "data", type = "BYTES", mode = "NULLABLE" },
+        { name = "id", type = "STRING", mode = "NULLABLE" },
+        { name = "name", type = "STRING", mode = "NULLABLE" },
+        { name = "arguments", type = "JSON", mode = "NULLABLE" },
+        { name = "response", type = "JSON", mode = "NULLABLE" }
+      ]
+    },
+    { name = "role", type = "STRING", mode = "NULLABLE" },
+    { name = "index", type = "INTEGER", mode = "NULLABLE" }
+  ])
+
+  depends_on = [
+    google_storage_bucket.logs_data_bucket,
+    google_bigquery_connection.genai_telemetry_connection,
+    google_storage_bucket_iam_member.telemetry_connection_access
+  ]
+}
+
+# ====================================================================
+# Completions View (Joins Logs with GCS Data)
+# ====================================================================
+
+# View that joins Cloud Logging data with GCS-stored completions data
+resource "google_bigquery_table" "completions_view" {
+  project             = var.dev_project_id
+  dataset_id          = google_bigquery_dataset.telemetry_dataset.dataset_id
+  table_id            = "completions_view"
+  description         = "View of GenAI completion logs joined with the GCS prompt/response external table"
+  deletion_protection = false
+
+  view {
+    query = templatefile("${path.module}/../sql/completions.sql", {
+      project_id                 = var.dev_project_id
+      dataset_id                 = google_bigquery_dataset.telemetry_dataset.dataset_id
+      completions_external_table = google_bigquery_table.completions_external_table.table_id
+      logs_link_id               = google_logging_linked_dataset.genai_logs_linked_dataset.link_id
+    })
+    use_legacy_sql = false
+  }
+
+  depends_on = [
+    google_logging_linked_dataset.genai_logs_linked_dataset,
+    google_bigquery_table.completions_external_table,
+    time_sleep.wait_for_linked_dataset
+  ]
+}

agent_starter_pack/base_template/deployment/terraform/github.tf

@@ -98,13 +98,6 @@ resource "github_actions_variable" "cicd_project_id" {
   depends_on    = [github_repository.repo]
 }
 
-resource "github_actions_variable" "bucket_name_load_test_results" {
-  repository    = var.repository_name
-  variable_name = "BUCKET_NAME_LOAD_TEST_RESULTS"
-  value         = google_storage_bucket.bucket_load_test_results.name
-  depends_on    = [github_repository.repo]
-}
-
 resource "github_actions_variable" "app_sa_email_staging" {
   repository    = var.repository_name
   variable_name = "APP_SA_EMAIL_STAGING"
@@ -119,17 +112,31 @@ resource "github_actions_variable" "app_sa_email_prod" {
   depends_on    = [github_repository.repo]
 }
 
+resource "github_actions_variable" "app_service_account_staging" {
+  repository    = var.repository_name
+  variable_name = "APP_SERVICE_ACCOUNT_STAGING"
+  value         = google_service_account.app_sa["staging"].email
+  depends_on    = [github_repository.repo]
+}
+
+resource "github_actions_variable" "app_service_account_prod" {
+  repository    = var.repository_name
+  variable_name = "APP_SERVICE_ACCOUNT_PROD"
+  value         = google_service_account.app_sa["prod"].email
+  depends_on    = [github_repository.repo]
+}
+
 resource "github_actions_variable" "logs_bucket_name_staging" {
   repository    = var.repository_name
   variable_name = "LOGS_BUCKET_NAME_STAGING"
-  value         = google_storage_bucket.logs_data_bucket[var.staging_project_id].url
+  value         = google_storage_bucket.logs_data_bucket[var.staging_project_id].name
   depends_on    = [github_repository.repo]
 }
 
 resource "github_actions_variable" "logs_bucket_name_prod" {
   repository    = var.repository_name
   variable_name = "LOGS_BUCKET_NAME_PROD"
-  value         = google_storage_bucket.logs_data_bucket[var.prod_project_id].url
+  value         = google_storage_bucket.logs_data_bucket[var.prod_project_id].name
   depends_on    = [github_repository.repo]
 }
 

agent_starter_pack/base_template/deployment/terraform/locals.tf

@@ -20,7 +20,10 @@ locals {
     "serviceusage.googleapis.com",
     "bigquery.googleapis.com",
     "cloudresourcemanager.googleapis.com",
-    "cloudtrace.googleapis.com"
+    "cloudtrace.googleapis.com",
+{%- if cookiecutter.is_adk and cookiecutter.session_type == "cloud_sql" %}
+    "sqladmin.googleapis.com",
+{%- endif %}
   ]
 
   deploy_project_services = [
@@ -33,12 +36,9 @@ locals {
     "serviceusage.googleapis.com",
     "logging.googleapis.com",
     "cloudtrace.googleapis.com",
-{%- if cookiecutter.is_adk and cookiecutter.session_type == "alloydb" %}
-    "compute.googleapis.com",
-    "servicenetworking.googleapis.com",
-    "alloydb.googleapis.com",
-    "secretmanager.googleapis.com",
-    "dns.googleapis.com"
+{%- if cookiecutter.is_adk and cookiecutter.session_type == "cloud_sql" %}
+    "sqladmin.googleapis.com",
+    "secretmanager.googleapis.com"
 {%- endif %}
   ]
 

agent_starter_pack/base_template/deployment/terraform/providers.tf

@@ -18,12 +18,16 @@ terraform {
   required_providers {
     google = {
       source  = "hashicorp/google"
-      version = "> 7.0.0"
+      version = "~> 7.10.0"
     }
     github = {
       source  = "integrations/github"
       version = "~> 6.5.0"
     }
+    random = {
+      source  = "hashicorp/random"
+      version = "~> 3.7.0"
+    }
   }
 }
 

agent_starter_pack/base_template/deployment/terraform/sql/completions.sql

@@ -0,0 +1,138 @@
+-- Optimized join of Cloud Logging data with GCS-stored prompt/response data.
+-- This query extracts both input and output messages referenced in logs.
+-- Note: Input files contain full conversation history, so messages may appear multiple times.
+
+-- Extract message references from Cloud Logging (scan once, extract both input/output)
+WITH log_refs AS (
+  SELECT
+    insert_id,
+    timestamp,
+    labels,
+    trace,
+    span_id,
+    JSON_VALUE(labels, '$.\"gen_ai.input.messages_ref\"') AS input_ref,
+    JSON_VALUE(labels, '$.\"gen_ai.output.messages_ref\"') AS output_ref
+  FROM `${project_id}.${logs_link_id}._AllLogs`
+  WHERE JSON_VALUE(labels, '$.\"gen_ai.input.messages_ref\"') IS NOT NULL
+     OR JSON_VALUE(labels, '$.\"gen_ai.output.messages_ref\"') IS NOT NULL
+),
+
+-- Unpivot to get one row per message reference
+unpivoted_refs AS (
+  SELECT
+    insert_id,
+    timestamp,
+    labels,
+    trace,
+    span_id,
+    input_ref AS messages_ref_uri,
+    'input' AS message_type
+  FROM log_refs
+  WHERE input_ref IS NOT NULL
+
+  UNION ALL
+
+  SELECT
+    insert_id,
+    timestamp,
+    labels,
+    trace,
+    span_id,
+    output_ref AS messages_ref_uri,
+    'output' AS message_type
+  FROM log_refs
+  WHERE output_ref IS NOT NULL
+),
+
+-- Join with completions external table and extract api_call_id once
+joined_data AS (
+  SELECT
+    lr.insert_id,
+    lr.timestamp,
+    lr.labels,
+    lr.trace,
+    lr.span_id,
+    lr.messages_ref_uri,
+    lr.message_type,
+    SPLIT(REGEXP_EXTRACT(lr.messages_ref_uri, r'/([^/]+)\.jsonl'), '_')[OFFSET(0)] AS api_call_id,
+    c.role,
+    c.parts,
+    c.index AS message_idx
+  FROM unpivoted_refs lr
+  JOIN `${project_id}.${dataset_id}.${completions_external_table}` c
+    ON lr.messages_ref_uri = c._FILE_NAME
+),
+
+-- Flatten the parts array
+flattened AS (
+  SELECT
+    insert_id,
+    timestamp,
+    labels,
+    trace,
+    span_id,
+    messages_ref_uri,
+    message_type,
+    api_call_id,
+    role,
+    message_idx,
+    part_idx,
+    part.type AS part_type,
+    part.content,
+    part.uri,
+    part.mime_type,
+    TO_HEX(MD5(part.data)) AS data_md5_hex,
+    part.id AS tool_id,
+    part.name AS tool_name,
+    part.arguments AS tool_args,
+    part.response AS tool_response
+  FROM joined_data
+  CROSS JOIN UNNEST(parts) AS part WITH OFFSET AS part_idx
+),
+
+-- Deduplicate by trace: keep only the latest log entry per trace
+-- (Tool calls create multiple log entries with same trace but different timestamps)
+deduplicated AS (
+  SELECT
+    *,
+    ROW_NUMBER() OVER (
+      PARTITION BY trace, message_type, role, message_idx, part_idx
+      ORDER BY timestamp DESC
+    ) AS row_num
+  FROM flattened
+)
+
+SELECT
+  -- Core identifiers and timestamps
+  timestamp,
+  insert_id,
+  trace,
+  span_id,
+  api_call_id,
+
+  -- Message metadata
+  message_type,
+  role,
+  message_idx,
+  part_idx,
+
+  -- Message content
+  content,
+
+  -- Tool/function calling
+  part_type,
+  tool_name,
+  tool_args,
+  tool_response,
+
+  -- Additional metadata
+  uri,
+  mime_type,
+  data_md5_hex,
+
+  -- Raw fields
+  labels,
+  messages_ref_uri
+FROM deduplicated
+WHERE row_num = 1  -- Keep only the latest entry per trace/message/part
+ORDER BY trace ASC, message_type ASC, message_idx ASC, part_idx ASC

agent_starter_pack/base_template/deployment/terraform/storage.tf

@@ -17,15 +17,6 @@ provider "google" {
   user_project_override = true
 }
 
-resource "google_storage_bucket" "bucket_load_test_results" {
-  name                        = "${var.cicd_runner_project_id}-${var.project_name}-load-test"
-  location                    = var.region
-  project                     = var.cicd_runner_project_id
-  uniform_bucket_level_access = true
-  force_destroy               = true
-  depends_on                  = [resource.google_project_service.cicd_services, resource.google_project_service.deploy_project_services]
-}
-
 resource "google_storage_bucket" "logs_data_bucket" {
   for_each                    = toset(local.all_project_ids)
   name                        = "${each.value}-${var.project_name}-logs"