agent-starter-pack 0.3.3__py3-none-any.whl → 0.21.0__py3-none-any.whl

agent_starter_pack/base_template/deployment/terraform/github.tf

@@ -0,0 +1,337 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+provider "github" {
+  owner = var.repository_owner
+}
+
+# Try to get existing repo
+data "github_repository" "existing_repo" {
+  count = var.create_repository ? 0 : 1
+  full_name = "${var.repository_owner}/${var.repository_name}"
+}
+
+# Only create GitHub repo if create_repository is true
+resource "github_repository" "repo" {
+  count       = var.create_repository ? 1 : 0
+  name        = var.repository_name
+  description = "Repository created with goo.gle/agent-starter-pack"
+  visibility  = "private"
+
+  has_issues      = true
+  has_wiki        = false
+  has_projects    = false
+  has_downloads   = false
+
+  allow_merge_commit = true
+  allow_squash_merge = true
+  allow_rebase_merge = true
+  
+  auto_init = false
+}
+
+{% if cookiecutter.cicd_runner == 'github_actions' %}
+resource "github_actions_variable" "gcp_project_number" {
+  repository    = var.repository_name
+  variable_name = "GCP_PROJECT_NUMBER"
+  value         = data.google_project.cicd_project.number
+  depends_on    = [github_repository.repo]
+}
+
+resource "github_actions_secret" "wif_pool_id" {
+  repository      = var.repository_name
+  secret_name     = "WIF_POOL_ID"
+  plaintext_value = google_iam_workload_identity_pool.github_pool.workload_identity_pool_id
+  depends_on      = [github_repository.repo, data.github_repository.existing_repo]
+}
+
+resource "github_actions_secret" "wif_provider_id" {
+  repository      = var.repository_name
+  secret_name     = "WIF_PROVIDER_ID"
+  plaintext_value = google_iam_workload_identity_pool_provider.github_provider.workload_identity_pool_provider_id
+  depends_on      = [github_repository.repo, data.github_repository.existing_repo]
+}
+
+resource "github_actions_secret" "gcp_service_account" {
+  repository      = var.repository_name
+  secret_name     = "GCP_SERVICE_ACCOUNT"
+  plaintext_value = google_service_account.cicd_runner_sa.email
+  depends_on      = [github_repository.repo, data.github_repository.existing_repo]
+}
+
+resource "github_actions_variable" "staging_project_id" {
+  repository    = var.repository_name
+  variable_name = "STAGING_PROJECT_ID"
+  value         = var.staging_project_id
+  depends_on    = [github_repository.repo]
+}
+
+resource "github_actions_variable" "prod_project_id" {
+  repository    = var.repository_name
+  variable_name = "PROD_PROJECT_ID"
+  value         = var.prod_project_id
+  depends_on    = [github_repository.repo]
+}
+
+resource "github_actions_variable" "region" {
+  repository    = var.repository_name
+  variable_name = "REGION"
+  value         = var.region
+  depends_on    = [github_repository.repo]
+}
+
+resource "github_actions_variable" "cicd_project_id" {
+  repository    = var.repository_name
+  variable_name = "CICD_PROJECT_ID"
+  value         = var.cicd_runner_project_id
+  depends_on    = [github_repository.repo]
+}
+
+resource "github_actions_variable" "app_sa_email_staging" {
+  repository    = var.repository_name
+  variable_name = "APP_SA_EMAIL_STAGING"
+  value         = google_service_account.app_sa["staging"].email
+  depends_on    = [github_repository.repo]
+}
+
+resource "github_actions_variable" "app_sa_email_prod" {
+  repository    = var.repository_name
+  variable_name = "APP_SA_EMAIL_PROD"
+  value         = google_service_account.app_sa["prod"].email
+  depends_on    = [github_repository.repo]
+}
+
+resource "github_actions_variable" "app_service_account_staging" {
+  repository    = var.repository_name
+  variable_name = "APP_SERVICE_ACCOUNT_STAGING"
+  value         = google_service_account.app_sa["staging"].email
+  depends_on    = [github_repository.repo]
+}
+
+resource "github_actions_variable" "app_service_account_prod" {
+  repository    = var.repository_name
+  variable_name = "APP_SERVICE_ACCOUNT_PROD"
+  value         = google_service_account.app_sa["prod"].email
+  depends_on    = [github_repository.repo]
+}
+
+resource "github_actions_variable" "logs_bucket_name_staging" {
+  repository    = var.repository_name
+  variable_name = "LOGS_BUCKET_NAME_STAGING"
+  value         = google_storage_bucket.logs_data_bucket[var.staging_project_id].name
+  depends_on    = [github_repository.repo]
+}
+
+resource "github_actions_variable" "logs_bucket_name_prod" {
+  repository    = var.repository_name
+  variable_name = "LOGS_BUCKET_NAME_PROD"
+  value         = google_storage_bucket.logs_data_bucket[var.prod_project_id].name
+  depends_on    = [github_repository.repo]
+}
+
+{% if cookiecutter.deployment_target == 'cloud_run' %}
+resource "github_actions_variable" "container_name" {
+  repository    = var.repository_name
+  variable_name = "CONTAINER_NAME"
+  value         = var.project_name
+  depends_on    = [github_repository.repo]
+}
+
+resource "github_actions_variable" "artifact_registry_repo_name" {
+  repository    = var.repository_name
+  variable_name = "ARTIFACT_REGISTRY_REPO_NAME"
+  value         = google_artifact_registry_repository.repo-artifacts-genai.repository_id
+  depends_on    = [github_repository.repo]
+}
+{% endif %}
+
+{% if cookiecutter.data_ingestion %}
+resource "github_actions_variable" "pipeline_gcs_root_staging" {
+  repository    = var.repository_name
+  variable_name = "PIPELINE_GCS_ROOT_STAGING"
+  value         = "gs://${google_storage_bucket.data_ingestion_pipeline_gcs_root["staging"].name}"
+  depends_on    = [github_repository.repo]
+}
+
+resource "github_actions_variable" "pipeline_gcs_root_prod" {
+  repository    = var.repository_name
+  variable_name = "PIPELINE_GCS_ROOT_PROD"
+  value         = "gs://${google_storage_bucket.data_ingestion_pipeline_gcs_root["prod"].name}"
+  depends_on    = [github_repository.repo]
+}
+
+resource "github_actions_variable" "pipeline_sa_email_staging" {
+  repository    = var.repository_name
+  variable_name = "PIPELINE_SA_EMAIL_STAGING"
+  value         = google_service_account.vertexai_pipeline_app_sa["staging"].email
+  depends_on    = [github_repository.repo]
+}
+
+resource "github_actions_variable" "pipeline_sa_email_prod" {
+  repository    = var.repository_name
+  variable_name = "PIPELINE_SA_EMAIL_PROD"
+  value         = google_service_account.vertexai_pipeline_app_sa["prod"].email
+  depends_on    = [github_repository.repo]
+}
+
+resource "github_actions_variable" "pipeline_name" {
+  repository    = var.repository_name
+  variable_name = "PIPELINE_NAME"
+  value         = var.project_name
+  depends_on    = [github_repository.repo]
+}
+
+resource "github_actions_variable" "pipeline_cron_schedule" {
+  repository    = var.repository_name
+  variable_name = "PIPELINE_CRON_SCHEDULE"
+  value         = var.pipeline_cron_schedule
+  depends_on    = [github_repository.repo]
+}
+
+{% if cookiecutter.datastore_type == "vertex_ai_search" %}
+resource "github_actions_variable" "data_store_id_staging" {
+  repository    = var.repository_name
+  variable_name = "DATA_STORE_ID_STAGING"
+  value         = google_discovery_engine_data_store.data_store_staging.data_store_id
+  depends_on    = [github_repository.repo]
+}
+
+resource "github_actions_variable" "data_store_id_prod" {
+  repository    = var.repository_name
+  variable_name = "DATA_STORE_ID_PROD"
+  value         = google_discovery_engine_data_store.data_store_prod.data_store_id
+  depends_on    = [github_repository.repo]
+}
+
+resource "github_actions_variable" "data_store_region" {
+  repository    = var.repository_name
+  variable_name = "DATA_STORE_REGION"
+  value         = var.data_store_region
+  depends_on    = [github_repository.repo]
+}
+{% elif cookiecutter.datastore_type == "vertex_ai_vector_search" %}
+resource "github_actions_variable" "vector_search_index_staging" {
+  repository    = var.repository_name
+  variable_name = "VECTOR_SEARCH_INDEX_STAGING"
+  value         = google_vertex_ai_index.vector_search_index_staging.id
+  depends_on    = [github_repository.repo]
+}
+
+resource "github_actions_variable" "vector_search_index_prod" {
+  repository    = var.repository_name
+  variable_name = "VECTOR_SEARCH_INDEX_PROD"
+  value         = google_vertex_ai_index.vector_search_index_prod.id
+  depends_on    = [github_repository.repo]
+}
+
+resource "github_actions_variable" "vector_search_index_endpoint_staging" {
+  repository    = var.repository_name
+  variable_name = "VECTOR_SEARCH_INDEX_ENDPOINT_STAGING"
+  value         = google_vertex_ai_index_endpoint.vector_search_index_endpoint_staging.id
+  depends_on    = [github_repository.repo]
+}
+
+resource "github_actions_variable" "vector_search_index_endpoint_prod" {
+  repository    = var.repository_name
+  variable_name = "VECTOR_SEARCH_INDEX_ENDPOINT_PROD"
+  value         = google_vertex_ai_index_endpoint.vector_search_index_endpoint_prod.id
+  depends_on    = [github_repository.repo]
+}
+
+resource "github_actions_variable" "vector_search_bucket_staging" {
+  repository    = var.repository_name
+  variable_name = "VECTOR_SEARCH_BUCKET_STAGING"
+  value         = google_storage_bucket.vector_search_data_bucket["staging"].url
+  depends_on    = [github_repository.repo]
+}
+
+resource "github_actions_variable" "vector_search_bucket_prod" {
+  repository    = var.repository_name
+  variable_name = "VECTOR_SEARCH_BUCKET_PROD"
+  value         = google_storage_bucket.vector_search_data_bucket["prod"].url
+  depends_on    = [github_repository.repo]
+}
+{% endif %}
+{% endif %}
+
+resource "github_repository_environment" "production_environment" {
+  repository  = var.repository_name
+  environment = "production"
+  depends_on  = [github_repository.repo, data.github_repository.existing_repo]
+
+  deployment_branch_policy {
+    protected_branches     = false
+    custom_branch_policies = true
+  }
+}
+{% else %}
+
+# Reference existing GitHub PAT secret created by gcloud CLI
+data "google_secret_manager_secret" "github_pat" {
+  project   = var.cicd_runner_project_id
+  secret_id = var.github_pat_secret_id
+}
+
+# Get CICD project data for Cloud Build service account
+data "google_project" "cicd_project" {
+  project_id = var.cicd_runner_project_id
+}
+
+# Grant Cloud Build service account access to GitHub PAT secret
+resource "google_secret_manager_secret_iam_member" "cloudbuild_secret_accessor" {
+  project   = var.cicd_runner_project_id
+  secret_id = data.google_secret_manager_secret.github_pat.secret_id
+  role      = "roles/secretmanager.secretAccessor"
+  member    = "serviceAccount:service-${data.google_project.cicd_project.number}@gcp-sa-cloudbuild.iam.gserviceaccount.com"
+  depends_on = [resource.google_project_service.cicd_services]
+}
+
+# Create the GitHub connection (fallback for manual Terraform usage)
+resource "google_cloudbuildv2_connection" "github_connection" {
+  count      = var.create_cb_connection ? 0 : 1
+  project    = var.cicd_runner_project_id
+  location   = var.region
+  name       = var.host_connection_name
+
+  github_config {
+    app_installation_id = var.github_app_installation_id
+    authorizer_credential {
+      oauth_token_secret_version = "${data.google_secret_manager_secret.github_pat.id}/versions/latest"
+    }
+  }
+  depends_on = [
+    resource.google_project_service.cicd_services,
+    resource.google_project_service.deploy_project_services,
+    resource.google_secret_manager_secret_iam_member.cloudbuild_secret_accessor
+  ]
+}
+
+
+resource "google_cloudbuildv2_repository" "repo" {
+  project  = var.cicd_runner_project_id
+  location = var.region
+  name     = var.repository_name
+  
+  # Use existing connection ID when it exists, otherwise use the created connection
+  parent_connection = var.create_cb_connection ? "projects/${var.cicd_runner_project_id}/locations/${var.region}/connections/${var.host_connection_name}" : google_cloudbuildv2_connection.github_connection[0].id
+  remote_uri       = "https://github.com/${var.repository_owner}/${var.repository_name}.git"
+  depends_on = [
+    resource.google_project_service.cicd_services,
+    resource.google_project_service.deploy_project_services,
+    data.github_repository.existing_repo,
+    github_repository.repo,
+    google_cloudbuildv2_connection.github_connection,
+  ]
+}
+{% endif %}

{src → agent_starter_pack}/base_template/deployment/terraform/iam.tf

@@ -25,7 +25,7 @@ resource "google_project_iam_member" "cicd_project_roles" {
   project    = var.cicd_runner_project_id
   role       = each.value
   member     = "serviceAccount:${resource.google_service_account.cicd_runner_sa.email}"
-  depends_on = [resource.google_project_service.cicd_services, resource.google_project_service.shared_services]
+  depends_on = [resource.google_project_service.cicd_services, resource.google_project_service.deploy_project_services]
 
 }
 
@@ -42,24 +42,12 @@ resource "google_project_iam_member" "other_projects_roles" {
   project    = each.value.project_id
   role       = each.value.role
   member     = "serviceAccount:${resource.google_service_account.cicd_runner_sa.email}"
-  depends_on = [resource.google_project_service.cicd_services, resource.google_project_service.shared_services]
+  depends_on = [resource.google_project_service.cicd_services, resource.google_project_service.deploy_project_services]
 }
-{% if cookiecutter.deployment_target == 'cloud_run' %}
-# 3. Allow Cloud Run service SA to pull containers stored in the CICD project
-resource "google_project_iam_member" "cicd_run_invoker_artifact_registry_reader" {
-  for_each = local.deploy_project_ids
-  project  = var.cicd_runner_project_id
-
-  role       = "roles/artifactregistry.reader"
-  member     = "serviceAccount:service-${data.google_project.projects[each.key].number}@serverless-robot-prod.iam.gserviceaccount.com"
-  depends_on = [resource.google_project_service.cicd_services, resource.google_project_service.shared_services]
-
-}
-
-# 4. Grant Cloud Run SA the required permissions to run the application
-resource "google_project_iam_member" "cloud_run_app_sa_roles" {
+# 3. Grant application SA the required permissions to run the application
+resource "google_project_iam_member" "app_sa_roles" {
   for_each = {
-    for pair in setproduct(keys(local.deploy_project_ids), var.cloud_run_app_roles) :
+    for pair in setproduct(keys(local.deploy_project_ids), var.app_sa_roles) :
     join(",", pair) => {
       project = local.deploy_project_ids[pair[0]]
       role    = pair[1]
@@ -68,47 +56,38 @@ resource "google_project_iam_member" "cloud_run_app_sa_roles" {
 
   project    = each.value.project
   role       = each.value.role
-  member     = "serviceAccount:${google_service_account.cloud_run_app_sa[split(",", each.key)[0]].email}"
-  depends_on = [resource.google_project_service.cicd_services, resource.google_project_service.shared_services]
+  member     = "serviceAccount:${google_service_account.app_sa[split(",", each.key)[0]].email}"
+  depends_on = [resource.google_project_service.cicd_services, resource.google_project_service.deploy_project_services]
 }
-{% elif cookiecutter.deployment_target == 'agent_engine' %}
-resource "google_project_service_identity" "vertex_sa" {
+
+{% if cookiecutter.deployment_target == 'cloud_run' %}
+# 4. Allow Cloud Run service SA to pull containers stored in the CICD project
+resource "google_project_iam_member" "cicd_run_invoker_artifact_registry_reader" {
   for_each = local.deploy_project_ids
-  provider = google-beta
-  project  = each.value
-  service  = "aiplatform.googleapis.com"
-}
+  project  = var.cicd_runner_project_id
 
-# 3. Grant required permissions to Vertex AI Service Agent SA
-resource "google_project_iam_member" "vertex_ai_sa_permissions" {
-  for_each = {
-    for pair in setproduct(keys(local.deploy_project_ids), var.agentengine_sa_roles) :
-    "${pair[0]}_${pair[1]}" => {
-      project = local.deploy_project_ids[pair[0]]
-      role = pair[1]
-    }
-  }
+  role       = "roles/artifactregistry.reader"
+  member     = "serviceAccount:service-${data.google_project.projects[each.key].number}@serverless-robot-prod.iam.gserviceaccount.com"
+  depends_on = [resource.google_project_service.cicd_services, resource.google_project_service.deploy_project_services]
 
-  project     = each.value.project
-  role        = each.value.role
-  member      = "serviceAccount:service-${data.google_project.projects[split("_", each.key)[0]].number}@gcp-sa-aiplatform-re.iam.gserviceaccount.com"
-  depends_on  = [resource.google_project_service.shared_services, resource.google_project_service_identity.vertex_sa]
 }
+
 {% endif %}
 
+
 # Special assignment: Allow the CICD SA to create tokens
 resource "google_service_account_iam_member" "cicd_run_invoker_token_creator" {
   service_account_id = google_service_account.cicd_runner_sa.name
   role               = "roles/iam.serviceAccountTokenCreator"
   member             = "serviceAccount:${resource.google_service_account.cicd_runner_sa.email}"
-  depends_on         = [resource.google_project_service.cicd_services, resource.google_project_service.shared_services]
+  depends_on         = [resource.google_project_service.cicd_services, resource.google_project_service.deploy_project_services]
 }
 # Special assignment: Allow the CICD SA to impersonate himself for trigger creation
 resource "google_service_account_iam_member" "cicd_run_invoker_account_user" {
   service_account_id = google_service_account.cicd_runner_sa.name
   role               = "roles/iam.serviceAccountUser"
   member             = "serviceAccount:${resource.google_service_account.cicd_runner_sa.email}"
-  depends_on         = [resource.google_project_service.cicd_services, resource.google_project_service.shared_services]
+  depends_on         = [resource.google_project_service.cicd_services, resource.google_project_service.deploy_project_services]
 }
 
 {%- if cookiecutter.data_ingestion %}
@@ -125,6 +104,6 @@ resource "google_project_iam_member" "vertexai_pipeline_sa_roles" {
   project    = each.value.project
   role       = each.value.role
   member     = "serviceAccount:${google_service_account.vertexai_pipeline_app_sa[split(",", each.key)[0]].email}"
-  depends_on = [resource.google_project_service.cicd_services, resource.google_project_service.shared_services]
+  depends_on = [resource.google_project_service.cicd_services, resource.google_project_service.deploy_project_services]
 }
 {%- endif %}

{src → agent_starter_pack}/base_template/deployment/terraform/locals.tf

@@ -20,10 +20,13 @@ locals {
     "serviceusage.googleapis.com",
     "bigquery.googleapis.com",
     "cloudresourcemanager.googleapis.com",
-    "cloudtrace.googleapis.com"
+    "cloudtrace.googleapis.com",
+{%- if cookiecutter.is_adk and cookiecutter.session_type == "cloud_sql" %}
+    "sqladmin.googleapis.com",
+{%- endif %}
   ]
 
-  shared_services = [
+  deploy_project_services = [
     "aiplatform.googleapis.com",
     "run.googleapis.com",
     "discoveryengine.googleapis.com",
@@ -32,7 +35,11 @@ locals {
     "bigquery.googleapis.com",
     "serviceusage.googleapis.com",
     "logging.googleapis.com",
-    "cloudtrace.googleapis.com"
+    "cloudtrace.googleapis.com",
+{%- if cookiecutter.is_adk and cookiecutter.session_type == "cloud_sql" %}
+    "sqladmin.googleapis.com",
+    "secretmanager.googleapis.com"
+{%- endif %}
   ]
 
   deploy_project_ids = {

{src/resources/setup_cicd → agent_starter_pack/base_template/deployment/terraform}/providers.tf

@@ -12,28 +12,35 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
 terraform {
   required_version = ">= 1.0.0"
   required_providers {
     google = {
       source  = "hashicorp/google"
-      version = "< 7.0.0"
+      version = "~> 7.10.0"
     }
     github = {
       source  = "integrations/github"
       version = "~> 6.5.0"
     }
+    random = {
+      source  = "hashicorp/random"
+      version = "~> 3.7.0"
+    }
   }
 }
 
 provider "google" {
   alias                 = "staging_billing_override"
   billing_project       = var.staging_project_id
+  region = var.region
   user_project_override = true
 }
 
 provider "google" {
   alias                 = "prod_billing_override"
   billing_project       = var.prod_project_id
+  region = var.region
   user_project_override = true
 }

{src → agent_starter_pack}/base_template/deployment/terraform/service_accounts.tf

@@ -16,18 +16,17 @@ resource "google_service_account" "cicd_runner_sa" {
   account_id   = "${var.project_name}-cb"
   display_name = "CICD Runner SA"
   project      = var.cicd_runner_project_id
-  depends_on   = [resource.google_project_service.cicd_services, resource.google_project_service.shared_services]
+  depends_on   = [resource.google_project_service.cicd_services, resource.google_project_service.deploy_project_services]
 }
-{% if cookiecutter.deployment_target == 'cloud_run' %}
-resource "google_service_account" "cloud_run_app_sa" {
+# Agent service account
+resource "google_service_account" "app_sa" {
   for_each = local.deploy_project_ids
 
-  account_id   = "${var.project_name}-cr"
-  display_name = "Cloud Run Generative AI app SA"
+  account_id   = "${var.project_name}-app"
+  display_name = "${var.project_name} Agent Service Account"
   project      = each.value
-  depends_on   = [resource.google_project_service.cicd_services, resource.google_project_service.shared_services]
+  depends_on   = [resource.google_project_service.cicd_services, resource.google_project_service.deploy_project_services]
 }
-{% endif %}
 
 {% if cookiecutter.data_ingestion %}
 # Service account to run Vertex AI pipeline
@@ -37,6 +36,6 @@ resource "google_service_account" "vertexai_pipeline_app_sa" {
   account_id   = "${var.project_name}-rag"
   display_name = "Vertex AI Pipeline app SA"
   project      = each.value
-  depends_on   = [resource.google_project_service.cicd_services, resource.google_project_service.shared_services]
+  depends_on   = [resource.google_project_service.cicd_services, resource.google_project_service.deploy_project_services]
 }
 {% endif %}

agent_starter_pack/base_template/deployment/terraform/sql/completions.sql

@@ -0,0 +1,138 @@
+-- Optimized join of Cloud Logging data with GCS-stored prompt/response data.
+-- This query extracts both input and output messages referenced in logs.
+-- Note: Input files contain full conversation history, so messages may appear multiple times.
+
+-- Extract message references from Cloud Logging (scan once, extract both input/output)
+WITH log_refs AS (
+  SELECT
+    insert_id,
+    timestamp,
+    labels,
+    trace,
+    span_id,
+    JSON_VALUE(labels, '$.\"gen_ai.input.messages_ref\"') AS input_ref,
+    JSON_VALUE(labels, '$.\"gen_ai.output.messages_ref\"') AS output_ref
+  FROM `${project_id}.${logs_link_id}._AllLogs`
+  WHERE JSON_VALUE(labels, '$.\"gen_ai.input.messages_ref\"') IS NOT NULL
+     OR JSON_VALUE(labels, '$.\"gen_ai.output.messages_ref\"') IS NOT NULL
+),
+
+-- Unpivot to get one row per message reference
+unpivoted_refs AS (
+  SELECT
+    insert_id,
+    timestamp,
+    labels,
+    trace,
+    span_id,
+    input_ref AS messages_ref_uri,
+    'input' AS message_type
+  FROM log_refs
+  WHERE input_ref IS NOT NULL
+
+  UNION ALL
+
+  SELECT
+    insert_id,
+    timestamp,
+    labels,
+    trace,
+    span_id,
+    output_ref AS messages_ref_uri,
+    'output' AS message_type
+  FROM log_refs
+  WHERE output_ref IS NOT NULL
+),
+
+-- Join with completions external table and extract api_call_id once
+joined_data AS (
+  SELECT
+    lr.insert_id,
+    lr.timestamp,
+    lr.labels,
+    lr.trace,
+    lr.span_id,
+    lr.messages_ref_uri,
+    lr.message_type,
+    SPLIT(REGEXP_EXTRACT(lr.messages_ref_uri, r'/([^/]+)\.jsonl'), '_')[OFFSET(0)] AS api_call_id,
+    c.role,
+    c.parts,
+    c.index AS message_idx
+  FROM unpivoted_refs lr
+  JOIN `${project_id}.${dataset_id}.${completions_external_table}` c
+    ON lr.messages_ref_uri = c._FILE_NAME
+),
+
+-- Flatten the parts array
+flattened AS (
+  SELECT
+    insert_id,
+    timestamp,
+    labels,
+    trace,
+    span_id,
+    messages_ref_uri,
+    message_type,
+    api_call_id,
+    role,
+    message_idx,
+    part_idx,
+    part.type AS part_type,
+    part.content,
+    part.uri,
+    part.mime_type,
+    TO_HEX(MD5(part.data)) AS data_md5_hex,
+    part.id AS tool_id,
+    part.name AS tool_name,
+    part.arguments AS tool_args,
+    part.response AS tool_response
+  FROM joined_data
+  CROSS JOIN UNNEST(parts) AS part WITH OFFSET AS part_idx
+),
+
+-- Deduplicate by trace: keep only the latest log entry per trace
+-- (Tool calls create multiple log entries with same trace but different timestamps)
+deduplicated AS (
+  SELECT
+    *,
+    ROW_NUMBER() OVER (
+      PARTITION BY trace, message_type, role, message_idx, part_idx
+      ORDER BY timestamp DESC
+    ) AS row_num
+  FROM flattened
+)
+
+SELECT
+  -- Core identifiers and timestamps
+  timestamp,
+  insert_id,
+  trace,
+  span_id,
+  api_call_id,
+
+  -- Message metadata
+  message_type,
+  role,
+  message_idx,
+  part_idx,
+
+  -- Message content
+  content,
+
+  -- Tool/function calling
+  part_type,
+  tool_name,
+  tool_args,
+  tool_response,
+
+  -- Additional metadata
+  uri,
+  mime_type,
+  data_md5_hex,
+
+  -- Raw fields
+  labels,
+  messages_ref_uri
+FROM deduplicated
+WHERE row_num = 1  -- Keep only the latest entry per trace/message/part
+ORDER BY trace ASC, message_type ASC, message_idx ASC, part_idx ASC