bigquery-agent-analytics 0.3.0__tar.gz → 0.3.2__tar.gz

bigquery_agent_analytics-0.3.2/.github/workflows/producers-ci.yml

@@ -0,0 +1,100 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Producers CI
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'producers/**'
+      - '.github/workflows/producers-ci.yml'
+  pull_request:
+    branches: [main]
+    paths:
+      - 'producers/**'
+      - '.github/workflows/producers-ci.yml'
+
+concurrency:
+  group: producers-ci-${{ github.ref }}
+  cancel-in-progress: true
+
+defaults:
+  run:
+    working-directory: producers
+
+jobs:
+  format:
+    name: Producers format check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install formatting tools
+        run: pip install "pyink>=24.3.0" "isort>=5.0"
+
+      - name: isort --check
+        run: isort --check-only src/ tests/
+
+      - name: pyink --check
+        run: pyink --check src/ tests/
+
+  test:
+    name: Producers test (Python ${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install package with dev dependencies
+        run: pip install -e ".[dev]"
+
+      - name: Run tests
+        run: pytest --tb=short -q
+
+  build:
+    name: Producers build
+    runs-on: ubuntu-latest
+    needs: [format, test]
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install build tools
+        run: pip install build
+
+      - name: Build sdist and wheel
+        run: python -m build
+
+      - name: List artifacts
+        run: ls -la dist/
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: producers-dist
+          path: producers/dist/

{bigquery_agent_analytics-0.3.0 → bigquery_agent_analytics-0.3.2}/.gitignore

@@ -11,11 +11,20 @@ build/
 .venv/
 venv/
 env/
+.adk/
 uv.lock
+.env
 
 # Script outputs
 scripts/reports/
 
+# Example run artifacts
+examples/*/reports/
+examples/*/reports_*/
+examples/*/trials_*/
+scripts/**/*.log
+examples/**/*.log
+
 # Local workspace metadata
 .code*/
 deploy/streaming_evaluation/.streaming_evaluation_state.json

{bigquery_agent_analytics-0.3.0 → bigquery_agent_analytics-0.3.2}/CHANGELOG.md

@@ -7,6 +7,226 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.3.2] - 2026-05-22
+
+### Release highlights
+
+The migration-v5 production track lands the design-partner asks from
+[#187](https://github.com/GoogleCloudPlatform/BigQuery-Agent-Analytics-SDK/issues/187)
+end-to-end: backfill mode, compiled-only extraction (with zero-LLM
+guarantee), explicit FK→PK mapping that re-enables MAKO self-edges,
+an opt-in orphan-session watchdog, the Beat 5 feedback / reward
+loop in both the demo and the live notebook, hardened deploy
+defaults (split SAs + tunable retries), and a Terraform module
+mirroring the bash deploy. The migration-v5 cron path is now
+complete for audit-critical production deployments.
+
+### Added
+
+- **Backfill mode for `materialize_window`** ([#188](https://github.com/GoogleCloudPlatform/BigQuery-Agent-Analytics-SDK/pull/188),
+  closes [#177](https://github.com/GoogleCloudPlatform/BigQuery-Agent-Analytics-SDK/issues/177))
+  — `bqaa-materialize-window --backfill --from / --to --state-key-suffix`
+  runs a one-shot historical window with isolated state-table
+  rows (the suffix folds into the `state_key` hash, so backfill
+  rows can't advance the steady-state cron checkpoint).
+- **Compiled-only extraction mode + deploy path** ([#192](https://github.com/GoogleCloudPlatform/BigQuery-Agent-Analytics-SDK/pull/192),
+  [#193](https://github.com/GoogleCloudPlatform/BigQuery-Agent-Analytics-SDK/pull/193),
+  closes [#178](https://github.com/GoogleCloudPlatform/BigQuery-Agent-Analytics-SDK/issues/178))
+  — `--extraction-mode=compiled-only` routes the orchestrator
+  through structured extractors only, never calls `AI.GENERATE`,
+  surfaces uncovered spans as typed `empty_extraction` failures
+  with sample diagnostics. The deploy script (`#193`) stages
+  `reference_extractor.py` into the container, wires
+  `BQAA_REFERENCE_EXTRACTORS_MODULE`, and makes
+  `roles/aiplatform.user` conditional (idempotently removes the
+  grant when transitioning an existing ai-fallback deploy to
+  compiled-only). `TestCompiledOnlyMakesZeroLLMCalls` proves the
+  zero-LLM contract.
+- **Explicit FK→PK mapping for binding columns** ([#191](https://github.com/GoogleCloudPlatform/BigQuery-Agent-Analytics-SDK/pull/191),
+  [#222](https://github.com/GoogleCloudPlatform/BigQuery-Agent-Analytics-SDK/pull/222),
+  closes [#179](https://github.com/GoogleCloudPlatform/BigQuery-Agent-Analytics-SDK/issues/179))
+  — `from_columns` / `to_columns` accept `list[dict[str, str]]`
+  shape (`[{src_decision_execution_id: id}]`) in addition to the
+  legacy `list[str]`. Materializer, validators, and the PG DDL
+  compiler consume the canonical
+  `ResolvedRelationship.from_column_mapping`. MAKO's
+  `evolvedFrom` / `supersededBy` self-edges are re-added to the
+  migration v5 binding.
+- **Orphan-session watchdog** ([#224](https://github.com/GoogleCloudPlatform/BigQuery-Agent-Analytics-SDK/pull/224),
+  closes [#180](https://github.com/GoogleCloudPlatform/BigQuery-Agent-Analytics-SDK/issues/180))
+  — opt-in via `--max-session-age-hours N`. Each cron pass scans
+  for sessions whose first event is older than the cutoff but
+  which never emitted `AGENT_COMPLETED`; flags them as
+  `session_orphaned` failures and records the running set in the
+  state table (`mode='orphan_scan'` + `mode='orphan_ledger'`
+  rows). Strict `>` watermark, explicit timestamp-bound partition
+  pruning on the resolved-orphan probe.
+- **Migration v5 Beat 5 — feedback / reward loop** ([#227](https://github.com/GoogleCloudPlatform/BigQuery-Agent-Analytics-SDK/pull/227),
+  [#228](https://github.com/GoogleCloudPlatform/BigQuery-Agent-Analytics-SDK/pull/228),
+  closes [#181](https://github.com/GoogleCloudPlatform/BigQuery-Agent-Analytics-SDK/issues/181),
+  [#184](https://github.com/GoogleCloudPlatform/BigQuery-Agent-Analytics-SDK/issues/184),
+  [#185](https://github.com/GoogleCloudPlatform/BigQuery-Agent-Analytics-SDK/issues/185))
+  — demo scope grows from 6 → 11 entities and 9 → 14
+  relationships. Adds `BusinessConstraint`,
+  `ConstraintApplication`, `RejectionReason`, `OutcomeSignal`,
+  `RewardComputation` + their edges. The demo agent emits four
+  new tools (`apply_constraint`, `record_rejection`,
+  `record_outcome_signal`, `compute_reward`); the reference
+  extractor covers all four; the notebook's new Beat 5 cells run
+  the two payoff GQL traversals end-to-end against a live
+  BigQuery scratch dataset (29 / 29 cells executed, counts 1–29,
+  6 unique reward rows + 8 unique rejection rows verified live).
+- **Split runtime + scheduler-caller SAs by default** ([#230](https://github.com/GoogleCloudPlatform/BigQuery-Agent-Analytics-SDK/pull/230),
+  closes [#182](https://github.com/GoogleCloudPlatform/BigQuery-Agent-Analytics-SDK/issues/182))
+  — `deploy_cloud_run_job.sh` creates two SAs by default:
+  `bqaa-periodic-runtime-sa` (BigQuery + Vertex AI roles) and
+  `bqaa-periodic-scheduler-sa` (only `roles/run.invoker` on the
+  job). `--single-sa` is the escape hatch for the pre-#182
+  combined identity. Least-privilege: scheduler-caller never
+  needs BigQuery permissions.
+- **Tunable `--max-retries` on Cloud Run Job** ([#230](https://github.com/GoogleCloudPlatform/BigQuery-Agent-Analytics-SDK/pull/230),
+  closes [#183](https://github.com/GoogleCloudPlatform/BigQuery-Agent-Analytics-SDK/issues/183))
+  — default 2 (was hard-coded 1). The orchestrator's session-
+  level idempotency + append-only state table make additional
+  retries safe. The retry count flows into `BQAA_MAX_RETRIES` on
+  the job's env so the runtime's startup log surfaces it in
+  `jsonPayload.max_retries` — operators correlating Cloud
+  Monitoring alert noise with retry behaviour see the policy
+  without `gcloud run jobs describe`.
+- **Terraform module for periodic materialization** ([#231](https://github.com/GoogleCloudPlatform/BigQuery-Agent-Analytics-SDK/pull/231),
+  closes [#186](https://github.com/GoogleCloudPlatform/BigQuery-Agent-Analytics-SDK/issues/186))
+  — `examples/migration_v5/periodic_materialization/terraform/`
+  mirrors the bash deploy's resources (graph dataset, both SAs,
+  IAM bindings, Cloud Run v2 Job, Scheduler trigger) with
+  defaults matching `deploy_cloud_run_job.sh`'s post-#230
+  surface. New `build_image.sh` helper produces the staging
+  layout the runtime needs. Image build/publish are intentionally
+  outside the module — `var.image_uri` takes the published
+  container image. `var.manage_apis` enables the required APIs
+  on a clean project; `var.deletion_protection` matches the bash
+  deploy's lifecycle. Live-verified end-to-end against
+  `test-project-0728-467323`.
+
+### Fixed
+
+- **Orphan-watchdog empty-array streaming-insert crash hotfix** ([#225](https://github.com/GoogleCloudPlatform/BigQuery-Agent-Analytics-SDK/pull/225))
+  — BigQuery's streaming-insert API rejects empty
+  `ARRAY<STRING>` values with `Field value of
+  flagged_session_ids cannot be empty.`. Pre-fix, every cron
+  pass after [#224](https://github.com/GoogleCloudPlatform/BigQuery-Agent-Analytics-SDK/pull/224)
+  crashed at insert time. Fix: omit nullable columns from the
+  `insert_rows_json` payload when their value is `None` or
+  empty.
+
+## [0.3.1] - 2026-05-18
+
+### Release highlights
+
+Focused follow-up to 0.3.0 that publishes the periodic-materialization
+production path. The `bqaa-materialize-window` CLI merged after the
+0.3.0 cut, so customers `pip install`-ing the SDK couldn't run the
+cron path the migration v5 playbook documents. 0.3.1 closes that gap
+and ships the surrounding deployment artifacts and a behavior fix:
+
+- **`bqaa-materialize-window` CLI on PyPI** ([#162](https://github.com/GoogleCloudPlatform/BigQuery-Agent-Analytics-SDK/pull/162))
+  — cron-friendly scheduled-graph-refresh command, available as a
+  standalone console script and as a `bq-agent-sdk materialize-window`
+  subcommand.
+- **Empty-extraction silent-failure mode closed** ([#167](https://github.com/GoogleCloudPlatform/BigQuery-Agent-Analytics-SDK/pull/167))
+  — `ok=true` is now an honest signal; per-session extraction
+  failures classify as `empty_extraction` vs `materialization_failed`
+  and flip `ok=false`. **Operator-visible behavior change** (see
+  Fixed).
+- **Cloud Run Job + Cloud Scheduler example** ([#165](https://github.com/GoogleCloudPlatform/BigQuery-Agent-Analytics-SDK/pull/165),
+  [#166](https://github.com/GoogleCloudPlatform/BigQuery-Agent-Analytics-SDK/pull/166))
+  — packaged deployment template under
+  `examples/migration_v5/periodic_materialization/` with
+  one-command deploy + `--smoke` end-to-end verification.
+- **Customer playbook** ([#168](https://github.com/GoogleCloudPlatform/BigQuery-Agent-Analytics-SDK/pull/168))
+  — the production cron path documented end-to-end: required
+  APIs/IAM, recommended schedules, JSON log shape, Cloud Monitoring
+  alerts, state-table inspection, teardown, troubleshooting.
+
+### Added
+
+- **`bqaa-materialize-window` console script** (PR
+  [#162](https://github.com/GoogleCloudPlatform/BigQuery-Agent-Analytics-SDK/pull/162)).
+  New cron-friendly entry point for keeping the materialized graph
+  fresh on a schedule. Shipped as a standalone console script
+  (`bqaa-materialize-window`) and as a `bq-agent-sdk
+  materialize-window` subcommand; both call paths share
+  `src/bigquery_agent_analytics/materialize_window.py`. Terminal-
+  event-driven discovery (`event_type = @completion_event_type`,
+  partition-pruned), pinned `run_started_at` snapshot, append-only
+  state table keyed on a content-derived `state_key` (project +
+  dataset + graph + events_table + ontology fingerprint + binding
+  fingerprint + discovery mode), overlap-windowed re-scan for
+  late-arriving events, per-session loop with idempotent retries
+  and checkpoint advance only on success, worst-status-wins
+  per-table aggregation, structured JSON report with C2 compiled-
+  extractor outcome counters (`compiled_unchanged` /
+  `compiled_filtered` / `fallback_for_event`), binding-validate
+  pre-flight, checkpoint that never regresses across overlap
+  re-scans, numeric/identifier guardrails at the boundary. Exit
+  codes: `0` clean, `1` expected failure (session error or
+  binding drift), `2` unexpected internal error.
+
+- **Migration v5 Cloud Run Job + Cloud Scheduler example** in
+  [`examples/migration_v5/periodic_materialization/`](examples/migration_v5/periodic_materialization/)
+  (PRs
+  [#165](https://github.com/GoogleCloudPlatform/BigQuery-Agent-Analytics-SDK/pull/165),
+  [#166](https://github.com/GoogleCloudPlatform/BigQuery-Agent-Analytics-SDK/pull/166)).
+  Packaged Cloud Run Job + Cloud Scheduler deployment that wraps
+  `bqaa-materialize-window` for the migration v5 binding. One-
+  command `deploy_cloud_run_job.sh` creates a runtime service
+  account with narrow IAM (`dataViewer` on the events dataset,
+  `dataEditor` on the graph dataset, `bigquery.jobUser` +
+  `aiplatform.user` at the project, `run.invoker` on the job for
+  the scheduler SA), deploys the job, wires the Cloud Scheduler
+  trigger, and optionally runs `--smoke` to verify end-to-end in
+  one shot. IAM matrix, dataset-role contract (events read-only,
+  graph read/write), and live-deploy evidence captured against the
+  canonical test project.
+
+- **Migration v5 periodic materialization playbook** in
+  [`examples/migration_v5/periodic_materialization/README.md`](examples/migration_v5/periodic_materialization/README.md)
+  (PR [#168](https://github.com/GoogleCloudPlatform/BigQuery-Agent-Analytics-SDK/pull/168)).
+  Documents the customer path for keeping the MAKO graph fresh on
+  a schedule: local dry-run via `run_job.py`, Cloud Run Job +
+  Cloud Scheduler deployment with `--smoke`, required APIs and
+  IAM, recommended schedules, Cloud Logging JSON report shape,
+  Cloud Monitoring alerts, state-table inspection, teardown, and
+  troubleshooting. Complements the migration v5 four-guarantee
+  notebook by covering the production cron path.
+
+### Fixed
+
+- **`materialize-window` no longer reports `ok=true` on silent
+  extraction failures** (PR
+  [#167](https://github.com/GoogleCloudPlatform/BigQuery-Agent-Analytics-SDK/pull/167)).
+  Previously, when per-event extraction returned an empty graph
+  (e.g., runtime SA missing `roles/aiplatform.user` so every
+  `AI.GENERATE` call failed and was swallowed), the orchestrator
+  reported `sessions_materialized == sessions_discovered`,
+  `ok=true`, and an empty `rows_materialized` dict. Operators
+  alerting on `jsonPayload.ok` saw "all good" while the entity
+  tables stayed empty. Now, after `materialize_with_status`
+  succeeds, the orchestrator inspects the materialized rows and
+  per-table statuses; sessions producing zero materialized rows
+  break the loop and classify the failure as
+  `empty_extraction` (extraction returned empty — check AI/IAM)
+  or `materialization_failed` (extraction produced rows but every
+  insert failed — check write perms / schema). `ok=false` is the
+  unmistakable red signal, and `failures[].error_code`
+  distinguishes the failure mode without log digging. Per-table
+  statuses now also surface in `result.table_statuses` for
+  failed sessions (previously only `ok` sessions contributed).
+  **Operator-visible behavior change**: alerts on
+  `jsonPayload.ok=false` are sufficient; no second-line
+  `rows_materialized == {}` check needed. The empty-window
+  heartbeat path (`sessions_discovered == 0`) is unchanged — an
+  idle cron firing still reports `ok=true`.
+
 ## [0.3.0] - 2026-05-15
 
 ### Release highlights

{bigquery_agent_analytics-0.3.0 → bigquery_agent_analytics-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bigquery-agent-analytics
-Version: 0.3.0
+Version: 0.3.2
 Summary: SDK for analyzing and evaluating agent traces stored in BigQuery.
 Author: Google LLC
 License-Expression: Apache-2.0

{bigquery_agent_analytics-0.3.0 → bigquery_agent_analytics-0.3.2}/autoformat.sh

@@ -21,9 +21,9 @@ set -euo pipefail
 REPO_ROOT="$(cd "$(dirname "$0")" && pwd)"
 
 echo "==> Running isort (import sorting)..."
-isort "${REPO_ROOT}/src/" "${REPO_ROOT}/tests/" "${REPO_ROOT}/examples/"
+isort "${REPO_ROOT}/src/" "${REPO_ROOT}/tests/" "${REPO_ROOT}/examples/" "${REPO_ROOT}/scripts/"
 
 echo "==> Running pyink (code formatting)..."
-pyink --config "${REPO_ROOT}/pyproject.toml" "${REPO_ROOT}/src/" "${REPO_ROOT}/tests/" "${REPO_ROOT}/examples/"
+pyink --config "${REPO_ROOT}/pyproject.toml" "${REPO_ROOT}/src/" "${REPO_ROOT}/tests/" "${REPO_ROOT}/examples/" "${REPO_ROOT}/scripts/"
 
 echo "==> Done. All Python files formatted."

{bigquery_agent_analytics-0.3.0 → bigquery_agent_analytics-0.3.2}/docs/README.md

@@ -61,6 +61,7 @@ architecture, rationale, and implementation plans behind key SDK features.
 
 | Document | Description |
 |----------|-------------|
+| [Migration v5 periodic materialization playbook](../examples/migration_v5/periodic_materialization/README.md) | Customer deployment path for keeping the MAKO migration v5 graph fresh on a schedule: local dry-run, Cloud Run Job + Cloud Scheduler deploy with `--smoke`, IAM matrix, schedule guidance, JSON log shape, Cloud Monitoring alert filters, state-table inspection, cleanup, and troubleshooting. |
 | [proposal_bigquery_agent_cli.md](proposal_bigquery_agent_cli.md) | CLI proposal and command design |
 | [python_udf_support_design.md](python_udf_support_design.md) | BigQuery Python UDF architecture |
 | [remote_function_rationale.md](remote_function_rationale.md) | Cloud Run remote function rationale |