dbt-dqm 0.2.0a1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dbt_dqm-0.2.0a1/.github/ISSUE_TEMPLATE/bug_report.yml +28 -0
- dbt_dqm-0.2.0a1/.github/ISSUE_TEMPLATE/feature_request.yml +19 -0
- dbt_dqm-0.2.0a1/.github/dependabot.yml +32 -0
- dbt_dqm-0.2.0a1/.github/pull_request_template.md +13 -0
- dbt_dqm-0.2.0a1/.github/workflows/bigquery-scheduled.yml +113 -0
- dbt_dqm-0.2.0a1/.github/workflows/ci.yml +224 -0
- dbt_dqm-0.2.0a1/.github/workflows/release.yml +149 -0
- dbt_dqm-0.2.0a1/.gitignore +16 -0
- dbt_dqm-0.2.0a1/CHANGELOG.md +184 -0
- dbt_dqm-0.2.0a1/CODE_OF_CONDUCT.md +58 -0
- dbt_dqm-0.2.0a1/CONTRIBUTING.md +12 -0
- dbt_dqm-0.2.0a1/LICENSE.md +190 -0
- dbt_dqm-0.2.0a1/PKG-INFO +304 -0
- dbt_dqm-0.2.0a1/README.md +270 -0
- dbt_dqm-0.2.0a1/SECURITY.md +30 -0
- dbt_dqm-0.2.0a1/dbt_project.yml +35 -0
- dbt_dqm-0.2.0a1/docs/design/warehouse-browsing.md +163 -0
- dbt_dqm-0.2.0a1/docs/functional-requirements.md +99 -0
- dbt_dqm-0.2.0a1/docs/implementation-0.2.md +152 -0
- dbt_dqm-0.2.0a1/docs/project-review.md +245 -0
- dbt_dqm-0.2.0a1/docs/qa-release-safeguards.md +118 -0
- dbt_dqm-0.2.0a1/docs/release-0.2.0a1.md +109 -0
- dbt_dqm-0.2.0a1/docs/release-checklist.md +84 -0
- dbt_dqm-0.2.0a1/docs/roadmap.md +59 -0
- dbt_dqm-0.2.0a1/docs/scale-results/allocator-ab-2026-10-10.json +202 -0
- dbt_dqm-0.2.0a1/docs/scale-results/bigquery-active-scope.json +67 -0
- dbt_dqm-0.2.0a1/docs/scale-results/bigquery-baseline.json +67 -0
- dbt_dqm-0.2.0a1/docs/scale-results/bigquery-observation-floor.json +67 -0
- dbt_dqm-0.2.0a1/docs/scale-results/browse-2026-10-11/bigquery-1m-chunks-0fd69f7.json +606 -0
- dbt_dqm-0.2.0a1/docs/scale-results/browse-2026-10-11/bigquery-1m-ef2e406.json +584 -0
- dbt_dqm-0.2.0a1/docs/scale-results/browse-2026-10-11/postgres-1m-ef2e406.json +387 -0
- dbt_dqm-0.2.0a1/docs/scale-results/browse-2026-10-11/postgres-plan-indexed-count_all.txt +23 -0
- dbt_dqm-0.2.0a1/docs/scale-results/browse-2026-10-11/postgres-plan-indexed-first_page.txt +16 -0
- dbt_dqm-0.2.0a1/docs/scale-results/browse-2026-10-11/postgres-plan-indexed-metrics.txt +25 -0
- dbt_dqm-0.2.0a1/docs/scale-results/browse-2026-10-11/postgres-plan-indexed-status_blocked.txt +27 -0
- dbt_dqm-0.2.0a1/docs/scale-results/browse-2026-10-11/postgres-plan-no-index-count_all.txt +23 -0
- dbt_dqm-0.2.0a1/docs/scale-results/browse-2026-10-11/postgres-plan-no-index-first_page.txt +27 -0
- dbt_dqm-0.2.0a1/docs/scale-results/browse-2026-10-11/postgres-plan-no-index-metrics.txt +25 -0
- dbt_dqm-0.2.0a1/docs/scale-results/browse-2026-10-11/postgres-plan-no-index-status_blocked.txt +27 -0
- dbt_dqm-0.2.0a1/docs/scale-results/cache-boundary-safeguards.json +10 -0
- dbt_dqm-0.2.0a1/docs/scale-results/candidate-613e089-cache-boundary.json +38 -0
- dbt_dqm-0.2.0a1/docs/scale-results/candidate-69703f7-cache-boundary.json +37 -0
- dbt_dqm-0.2.0a1/docs/scale-results/memory-interval-2026-10-10.json +463 -0
- dbt_dqm-0.2.0a1/docs/scale-results/pandas-3-2026-10-11.json +260 -0
- dbt_dqm-0.2.0a1/docs/scale-results/postgres-memory-1000000-payload8.json +29 -0
- dbt_dqm-0.2.0a1/docs/scale-results/postgres-memory-250000-payload8.json +29 -0
- dbt_dqm-0.2.0a1/docs/scale-results/postgres-memory-500000-payload8.json +29 -0
- dbt_dqm-0.2.0a1/docs/scale-results/postgres-planner/before-active-scope-postgres-1000000-payload8.json +41 -0
- dbt_dqm-0.2.0a1/docs/scale-results/postgres-planner/postgres-1000000-payload8.json +41 -0
- dbt_dqm-0.2.0a1/docs/scale-results/postgres-planner/postgres-20000-payload8.json +42 -0
- dbt_dqm-0.2.0a1/docs/scale-results/postgres-planner/postgres-200000-payload8.json +41 -0
- dbt_dqm-0.2.0a1/docs/scale-results/postgres-planner/postgres-plan-mass_pass-nestloop-off-change_set.txt +472 -0
- dbt_dqm-0.2.0a1/docs/scale-results/postgres-planner/postgres-plan-mass_pass-nestloop-on-change_set.txt +269 -0
- dbt_dqm-0.2.0a1/docs/scale-results/postgres-planner/postgres-plan-steady-nestloop-off-change_set.txt +449 -0
- dbt_dqm-0.2.0a1/docs/scale-results/postgres-planner/postgres-plan-steady-nestloop-on-change_set.txt +445 -0
- dbt_dqm-0.2.0a1/docs/scale-results/pyarrow-23-cache-boundary.json +43 -0
- dbt_dqm-0.2.0a1/docs/scale-results/pyarrow-25-2026-10-11.json +81 -0
- dbt_dqm-0.2.0a1/docs/scale.md +375 -0
- dbt_dqm-0.2.0a1/docs/scheduled-bigquery.md +69 -0
- dbt_dqm-0.2.0a1/docs/warehouse-interfaces.md +227 -0
- dbt_dqm-0.2.0a1/integration_tests/acceptance/bigquery_datasets.py +70 -0
- dbt_dqm-0.2.0a1/integration_tests/acceptance/browse_checks.py +91 -0
- dbt_dqm-0.2.0a1/integration_tests/acceptance/conftest.py +61 -0
- dbt_dqm-0.2.0a1/integration_tests/acceptance/test_bigquery.py +1202 -0
- dbt_dqm-0.2.0a1/integration_tests/acceptance/test_bigquery_permissions.py +185 -0
- dbt_dqm-0.2.0a1/integration_tests/acceptance/test_postgres.py +1475 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/dbt_project.yml +15 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/macros/acceptance_protocol.sql +90 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/macros/generate_schema_name.sql +12 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/macros/reset_demo_dqm.sql +42 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/macros/schema.yml +19 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/models/demo_records.sql +17 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/models/schema.yml +66 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/package-lock.yml +4 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/packages.yml +3 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/profiles.yml.example +11 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/seeds/demo_cases.csv +13 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/tests/assert_collection_guardrail_fixtures.sql +34 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/tests/assert_scenario_outcomes.sql +85 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/tests/demo_customer_email_invalid.sql +25 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/tests/demo_invalid_duplicate_grain.sql +10 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/tests/demo_invalid_empty_grain.sql +10 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/tests/demo_invalid_fail_calc.sql +2 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/tests/demo_invalid_limit.sql +2 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/tests/demo_invalid_missing_column.sql +10 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/tests/demo_invalid_non_string_grain.sql +10 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/tests/demo_invalid_owner.sql +2 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/tests/demo_invalid_priority.sql +2 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/tests/demo_invalid_view_storage.sql +2 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/tests/demo_order_amount_invalid.sql +24 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/tests/demo_order_line_invalid.sql +30 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/tests/schema.yml +17 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_postgres/dbt_project.yml +14 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_postgres/macros/assert_postgres_query_plans.sql +21 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_postgres/macros/reset_demo_dqm.sql +46 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_postgres/macros/schema.yml +11 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_postgres/models/demo_records.sql +17 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_postgres/models/schema.yml +66 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_postgres/package-lock.yml +4 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_postgres/packages.yml +2 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_postgres/profiles.yml.example +17 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_postgres/seeds/demo_cases.csv +13 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_postgres/tests/assert_collection_guardrail_fixtures.sql +34 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_postgres/tests/assert_scenario_outcomes.sql +79 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_postgres/tests/demo_customer_email_invalid.sql +25 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_postgres/tests/demo_invalid_duplicate_grain.sql +10 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_postgres/tests/demo_invalid_empty_grain.sql +10 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_postgres/tests/demo_invalid_fail_calc.sql +2 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_postgres/tests/demo_invalid_limit.sql +2 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_postgres/tests/demo_invalid_missing_column.sql +10 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_postgres/tests/demo_invalid_non_string_grain.sql +10 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_postgres/tests/demo_invalid_owner.sql +2 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_postgres/tests/demo_invalid_priority.sql +2 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_postgres/tests/demo_invalid_view_storage.sql +2 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_postgres/tests/demo_order_amount_invalid.sql +24 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_postgres/tests/demo_order_line_invalid.sql +30 -0
- dbt_dqm-0.2.0a1/integration_tests/demo_postgres/tests/schema.yml +17 -0
- dbt_dqm-0.2.0a1/integration_tests/scale/app_memory.py +77 -0
- dbt_dqm-0.2.0a1/integration_tests/scale/bigquery_scale.py +316 -0
- dbt_dqm-0.2.0a1/integration_tests/scale/browse_probe.py +199 -0
- dbt_dqm-0.2.0a1/integration_tests/scale/cache_boundary.py +54 -0
- dbt_dqm-0.2.0a1/integration_tests/scale/postgres_scale.py +448 -0
- dbt_dqm-0.2.0a1/macros/adapters.sql +249 -0
- dbt_dqm-0.2.0a1/macros/capture_test_results.sql +421 -0
- dbt_dqm-0.2.0a1/macros/health.sql +48 -0
- dbt_dqm-0.2.0a1/macros/helpers.sql +70 -0
- dbt_dqm-0.2.0a1/macros/history.sql +15 -0
- dbt_dqm-0.2.0a1/macros/maintenance.sql +301 -0
- dbt_dqm-0.2.0a1/macros/materializations.sql +58 -0
- dbt_dqm-0.2.0a1/macros/reconciliation.sql +204 -0
- dbt_dqm-0.2.0a1/macros/recovery.sql +147 -0
- dbt_dqm-0.2.0a1/macros/replay.sql +409 -0
- dbt_dqm-0.2.0a1/macros/schema.yml +182 -0
- dbt_dqm-0.2.0a1/macros/schema_management.sql +404 -0
- dbt_dqm-0.2.0a1/macros/table_schemas.sql +156 -0
- dbt_dqm-0.2.0a1/models/dqm_all_issues.sql +3 -0
- dbt_dqm-0.2.0a1/models/dqm_annotation_changes.sql +25 -0
- dbt_dqm-0.2.0a1/models/dqm_area_health.sql +39 -0
- dbt_dqm-0.2.0a1/models/dqm_current_issues.sql +3 -0
- dbt_dqm-0.2.0a1/models/dqm_issue_timeline.sql +6 -0
- dbt_dqm-0.2.0a1/models/dqm_maintenance_health.sql +136 -0
- dbt_dqm-0.2.0a1/models/dqm_missed_issues.sql +6 -0
- dbt_dqm-0.2.0a1/models/dqm_reconcile.sql +4 -0
- dbt_dqm-0.2.0a1/models/dqm_test_health.sql +103 -0
- dbt_dqm-0.2.0a1/models/schema.yml +163 -0
- dbt_dqm-0.2.0a1/packages.yml +2 -0
- dbt_dqm-0.2.0a1/pyproject.toml +64 -0
- dbt_dqm-0.2.0a1/scripts/compile_bigquery.py +40 -0
- dbt_dqm-0.2.0a1/selectors.yml +7 -0
- dbt_dqm-0.2.0a1/src/dbt_dqm_app/__init__.py +3 -0
- dbt_dqm-0.2.0a1/src/dbt_dqm_app/browse.py +293 -0
- dbt_dqm-0.2.0a1/src/dbt_dqm_app/cli.py +127 -0
- dbt_dqm-0.2.0a1/src/dbt_dqm_app/config.py +146 -0
- dbt_dqm-0.2.0a1/src/dbt_dqm_app/display.py +241 -0
- dbt_dqm-0.2.0a1/src/dbt_dqm_app/errors.py +107 -0
- dbt_dqm-0.2.0a1/src/dbt_dqm_app/frames.py +19 -0
- dbt_dqm-0.2.0a1/src/dbt_dqm_app/limits.py +34 -0
- dbt_dqm-0.2.0a1/src/dbt_dqm_app/store.py +476 -0
- dbt_dqm-0.2.0a1/src/dbt_dqm_app/streamlit_app.py +1051 -0
- dbt_dqm-0.2.0a1/src/dbt_dqm_app/warehouse.py +852 -0
- dbt_dqm-0.2.0a1/tests/assert_at_most_one_active_occurrence.sql +10 -0
- dbt_dqm-0.2.0a1/tests/assert_collection_evidence_guardrails.sql +13 -0
- dbt_dqm-0.2.0a1/tests/assert_event_id_unique.sql +4 -0
- dbt_dqm-0.2.0a1/tests/assert_identity_hash_vectors.sql +49 -0
- dbt_dqm-0.2.0a1/tests/assert_occurrence_id_unique.sql +4 -0
- dbt_dqm-0.2.0a1/tests/assert_receipt_unique.sql +4 -0
- dbt_dqm-0.2.0a1/tests/test_acceptance_sweep.py +58 -0
- dbt_dqm-0.2.0a1/tests/test_annotation_staging.py +81 -0
- dbt_dqm-0.2.0a1/tests/test_browse.py +314 -0
- dbt_dqm-0.2.0a1/tests/test_browse_warehouse.py +180 -0
- dbt_dqm-0.2.0a1/tests/test_cache_limits.py +190 -0
- dbt_dqm-0.2.0a1/tests/test_cli.py +56 -0
- dbt_dqm-0.2.0a1/tests/test_config.py +98 -0
- dbt_dqm-0.2.0a1/tests/test_credentials.py +114 -0
- dbt_dqm-0.2.0a1/tests/test_dbt_hints.py +32 -0
- dbt_dqm-0.2.0a1/tests/test_dependency_floors.py +57 -0
- dbt_dqm-0.2.0a1/tests/test_docs_consistency.py +66 -0
- dbt_dqm-0.2.0a1/tests/test_features.py +188 -0
- dbt_dqm-0.2.0a1/tests/test_frames.py +18 -0
- dbt_dqm-0.2.0a1/tests/test_maintenance_health.py +93 -0
- dbt_dqm-0.2.0a1/tests/test_missing_values.py +47 -0
- dbt_dqm-0.2.0a1/tests/test_oauthlib_compat.py +89 -0
- dbt_dqm-0.2.0a1/tests/test_protocol_render.py +292 -0
- dbt_dqm-0.2.0a1/tests/test_release_workflow.py +255 -0
- dbt_dqm-0.2.0a1/tests/test_scheduled_workflow.py +32 -0
- dbt_dqm-0.2.0a1/tests/test_store.py +310 -0
- dbt_dqm-0.2.0a1/tests/test_sync_and_errors.py +153 -0
- dbt_dqm-0.2.0a1/tests/test_ui.py +135 -0
- dbt_dqm-0.2.0a1/tests/test_ui_browse.py +269 -0
- dbt_dqm-0.2.0a1/tests/test_wif.py +145 -0
- dbt_dqm-0.2.0a1/uv.lock +1900 -0
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
name: Bug report
|
|
2
|
+
description: Report incorrect collection, lifecycle, adapter, or app behavior
|
|
3
|
+
body:
|
|
4
|
+
- type: textarea
|
|
5
|
+
id: behavior
|
|
6
|
+
attributes:
|
|
7
|
+
label: Observed behavior
|
|
8
|
+
validations:
|
|
9
|
+
required: true
|
|
10
|
+
- type: textarea
|
|
11
|
+
id: reproduction
|
|
12
|
+
attributes:
|
|
13
|
+
label: Minimal reproduction
|
|
14
|
+
validations:
|
|
15
|
+
required: true
|
|
16
|
+
- type: input
|
|
17
|
+
id: versions
|
|
18
|
+
attributes:
|
|
19
|
+
label: dbt-dqm, dbt, adapter, warehouse, and Python versions
|
|
20
|
+
validations:
|
|
21
|
+
required: true
|
|
22
|
+
- type: checkboxes
|
|
23
|
+
id: safety
|
|
24
|
+
attributes:
|
|
25
|
+
label: Data safety
|
|
26
|
+
options:
|
|
27
|
+
- label: I removed credentials, customer data, and complete failed-row contents.
|
|
28
|
+
required: true
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
name: Feature request
|
|
2
|
+
description: Propose a focused open-source improvement
|
|
3
|
+
body:
|
|
4
|
+
- type: textarea
|
|
5
|
+
id: problem
|
|
6
|
+
attributes:
|
|
7
|
+
label: User problem
|
|
8
|
+
validations:
|
|
9
|
+
required: true
|
|
10
|
+
- type: textarea
|
|
11
|
+
id: proposal
|
|
12
|
+
attributes:
|
|
13
|
+
label: Proposed behavior
|
|
14
|
+
validations:
|
|
15
|
+
required: true
|
|
16
|
+
- type: textarea
|
|
17
|
+
id: adapters
|
|
18
|
+
attributes:
|
|
19
|
+
label: Adapter and compatibility implications
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
version: 2
|
|
2
|
+
updates:
|
|
3
|
+
# Read uv.lock so version-type exclusions also cover range-only requirements.
|
|
4
|
+
- package-ecosystem: uv
|
|
5
|
+
directory: /
|
|
6
|
+
schedule:
|
|
7
|
+
interval: weekly
|
|
8
|
+
# Deferred compatibility work: https://github.com/a-adamovich/dbt-dqm/issues/11
|
|
9
|
+
# Update-type exclusions preserve patch updates and security-update handling.
|
|
10
|
+
ignore:
|
|
11
|
+
# dbt minors are adopted deliberately: the protocol SQL must render identically and the full
|
|
12
|
+
# warehouse suites must pass before the pins and require-dbt-version move (see #11).
|
|
13
|
+
- dependency-name: dbt-core
|
|
14
|
+
update-types: ["version-update:semver-minor"]
|
|
15
|
+
- dependency-name: dbt-bigquery
|
|
16
|
+
update-types: ["version-update:semver-minor"]
|
|
17
|
+
- dependency-name: dbt-postgres
|
|
18
|
+
update-types: ["version-update:semver-minor"]
|
|
19
|
+
- dependency-name: pandas
|
|
20
|
+
update-types: ["version-update:semver-major"]
|
|
21
|
+
# pyarrow majors are adopted deliberately, with the app memory benchmark rerun at the
|
|
22
|
+
# 50k cache boundary (pinned to 25.x; see #11 and docs/scale.md).
|
|
23
|
+
- dependency-name: pyarrow
|
|
24
|
+
update-types: ["version-update:semver-major"]
|
|
25
|
+
groups:
|
|
26
|
+
python-minor-patches:
|
|
27
|
+
patterns: ["*"]
|
|
28
|
+
update-types: [minor, patch]
|
|
29
|
+
- package-ecosystem: github-actions
|
|
30
|
+
directory: /
|
|
31
|
+
schedule:
|
|
32
|
+
interval: weekly
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
## Summary
|
|
2
|
+
|
|
3
|
+
Describe the focused change and its user-visible behavior.
|
|
4
|
+
|
|
5
|
+
## Verification
|
|
6
|
+
|
|
7
|
+
- [ ] `ruff check .`
|
|
8
|
+
- [ ] `pytest -q`
|
|
9
|
+
- [ ] Relevant dbt adapter/demo lifecycle checks
|
|
10
|
+
- [ ] Documentation and changelog updated where needed
|
|
11
|
+
- [ ] Commits include DCO sign-off (`git commit -s`)
|
|
12
|
+
|
|
13
|
+
Do not include credentials, proprietary data, customer names, or complete failed-row contents.
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
name: Scheduled BigQuery lifecycle
|
|
2
|
+
|
|
3
|
+
# Weekly credentialed BigQuery verification. Keyless: GitHub's OIDC token is exchanged through
|
|
4
|
+
# Workload Identity Federation for a short-lived token of a dedicated CI service account, so no
|
|
5
|
+
# long-lived key is stored. Inert until the repository variables below are set.
|
|
6
|
+
on:
|
|
7
|
+
workflow_dispatch:
|
|
8
|
+
schedule:
|
|
9
|
+
- cron: "17 6 * * 1"
|
|
10
|
+
|
|
11
|
+
permissions:
|
|
12
|
+
contents: read
|
|
13
|
+
id-token: write
|
|
14
|
+
issues: write
|
|
15
|
+
|
|
16
|
+
jobs:
|
|
17
|
+
lifecycle:
|
|
18
|
+
concurrency:
|
|
19
|
+
group: dbt-dqm-${{ vars.DBT_DQM_BIGQUERY_PROJECT }}-dbt_dqm_scheduled_ci
|
|
20
|
+
cancel-in-progress: false
|
|
21
|
+
if: vars.BIGQUERY_LIFECYCLE_ENABLED == 'true' && github.ref == 'refs/heads/main'
|
|
22
|
+
runs-on: ubuntu-latest
|
|
23
|
+
env:
|
|
24
|
+
DBT_DQM_BIGQUERY_PROJECT: ${{ vars.DBT_DQM_BIGQUERY_PROJECT }}
|
|
25
|
+
DBT_DQM_BIGQUERY_TEST_PROJECT: ${{ vars.DBT_DQM_BIGQUERY_PROJECT }}
|
|
26
|
+
steps:
|
|
27
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
28
|
+
- name: Authenticate to Google Cloud (Workload Identity Federation)
|
|
29
|
+
uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0
|
|
30
|
+
with:
|
|
31
|
+
workload_identity_provider: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }}
|
|
32
|
+
service_account: ${{ vars.GCP_CI_SERVICE_ACCOUNT }}
|
|
33
|
+
- name: Install uv
|
|
34
|
+
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
|
|
35
|
+
with:
|
|
36
|
+
python-version: "3.11"
|
|
37
|
+
- name: Install dependencies
|
|
38
|
+
run: uv sync --locked --all-groups
|
|
39
|
+
- name: Write profile (Application Default Credentials from the step above)
|
|
40
|
+
run: |
|
|
41
|
+
mkdir -p ~/.dbt
|
|
42
|
+
cat > ~/.dbt/profiles.yml <<'PROFILES'
|
|
43
|
+
dbt_dqm_demo:
|
|
44
|
+
target: ci
|
|
45
|
+
outputs:
|
|
46
|
+
ci:
|
|
47
|
+
type: bigquery
|
|
48
|
+
method: oauth
|
|
49
|
+
project: "{{ env_var('DBT_DQM_BIGQUERY_PROJECT') }}"
|
|
50
|
+
dataset: dbt_dqm_scheduled_ci
|
|
51
|
+
location: US
|
|
52
|
+
threads: 4
|
|
53
|
+
PROFILES
|
|
54
|
+
- name: The CI identity cannot read the shared QA dataset
|
|
55
|
+
run: |
|
|
56
|
+
uv run python - <<'PY'
|
|
57
|
+
import os
|
|
58
|
+
from google.api_core.exceptions import Forbidden, NotFound
|
|
59
|
+
from google.cloud import bigquery
|
|
60
|
+
project = os.environ["DBT_DQM_BIGQUERY_PROJECT"]
|
|
61
|
+
try:
|
|
62
|
+
bigquery.Client(project=project).query(
|
|
63
|
+
f"select 1 from `{project}.dbt_dqm_qa.dqm_install` limit 1").result()
|
|
64
|
+
except (Forbidden, NotFound):
|
|
65
|
+
print("As expected, the CI service account has no access to dbt_dqm_qa.")
|
|
66
|
+
else:
|
|
67
|
+
raise SystemExit("The CI service account can read dbt_dqm_qa; narrow its roles.")
|
|
68
|
+
PY
|
|
69
|
+
- name: Full credentialed lifecycle (dedicated dataset)
|
|
70
|
+
working-directory: integration_tests/demo_bigquery
|
|
71
|
+
run: |
|
|
72
|
+
set -e
|
|
73
|
+
uv run dbt deps
|
|
74
|
+
uv run dbt run-operation reset_demo_dqm
|
|
75
|
+
uv run dbt seed
|
|
76
|
+
for scenario in initial passed recurrence; do
|
|
77
|
+
uv run dbt run --select demo_records --vars "{demo_scenario: $scenario}"
|
|
78
|
+
uv run dbt test --select tag:dqm --vars "{demo_scenario: $scenario}"
|
|
79
|
+
uv run dbt test --select assert_collection_guardrail_fixtures
|
|
80
|
+
uv run dbt build --select package:dbt_dqm --vars "{demo_scenario: $scenario}"
|
|
81
|
+
uv run dbt test --select assert_scenario_outcomes --vars "{demo_scenario: $scenario}"
|
|
82
|
+
done
|
|
83
|
+
- name: Credentialed acceptance suite (parallel, disposable datasets)
|
|
84
|
+
shell: bash
|
|
85
|
+
run: |
|
|
86
|
+
mkdir -p acceptance-results
|
|
87
|
+
set -o pipefail
|
|
88
|
+
uv run pytest -q -rA -n 6 --dist load integration_tests/acceptance/test_bigquery.py \
|
|
89
|
+
--junitxml=acceptance-results/bigquery.xml 2>&1 | tee acceptance-results/bigquery.log
|
|
90
|
+
- name: Sweep stale acceptance datasets
|
|
91
|
+
if: always()
|
|
92
|
+
run: uv run python integration_tests/acceptance/bigquery_datasets.py --project "$DBT_DQM_BIGQUERY_PROJECT" --older-than-hours 6
|
|
93
|
+
- name: Upload results
|
|
94
|
+
if: always()
|
|
95
|
+
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
|
96
|
+
with:
|
|
97
|
+
name: scheduled-bigquery-results
|
|
98
|
+
path: acceptance-results/
|
|
99
|
+
if-no-files-found: ignore
|
|
100
|
+
- name: Report failure in an issue
|
|
101
|
+
if: failure()
|
|
102
|
+
env:
|
|
103
|
+
GH_TOKEN: ${{ github.token }}
|
|
104
|
+
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
|
105
|
+
run: |
|
|
106
|
+
gh label create scheduled-bigquery --color B60205 --description "Scheduled BigQuery verification" 2>/dev/null || true
|
|
107
|
+
existing="$(gh issue list --label scheduled-bigquery --state open --json number --jq '.[0].number')"
|
|
108
|
+
if [ -n "$existing" ]; then
|
|
109
|
+
gh issue comment "$existing" --body "Scheduled BigQuery verification failed again: $RUN_URL"
|
|
110
|
+
else
|
|
111
|
+
gh issue create --label scheduled-bigquery --title "Scheduled BigQuery verification failed" \
|
|
112
|
+
--body "The weekly credentialed BigQuery run failed on \`main\`: $RUN_URL"
|
|
113
|
+
fi
|
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
permissions:
|
|
4
|
+
contents: read
|
|
5
|
+
|
|
6
|
+
on:
|
|
7
|
+
push:
|
|
8
|
+
branches: [main]
|
|
9
|
+
pull_request:
|
|
10
|
+
|
|
11
|
+
jobs:
|
|
12
|
+
lint-and-test:
|
|
13
|
+
runs-on: ubuntu-latest
|
|
14
|
+
steps:
|
|
15
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
16
|
+
|
|
17
|
+
- name: Install uv
|
|
18
|
+
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
|
|
19
|
+
with:
|
|
20
|
+
python-version: "3.11"
|
|
21
|
+
|
|
22
|
+
- name: Install dependencies
|
|
23
|
+
run: uv sync --all-groups
|
|
24
|
+
|
|
25
|
+
- name: Ruff
|
|
26
|
+
run: uv run ruff check .
|
|
27
|
+
|
|
28
|
+
- name: Pytest
|
|
29
|
+
run: uv run pytest -q
|
|
30
|
+
|
|
31
|
+
- name: Streamlit startup smoke test
|
|
32
|
+
run: |
|
|
33
|
+
mkdir -p /tmp/dbt-dqm-smoke-project /tmp/dbt-dqm-smoke-profiles
|
|
34
|
+
printf 'name: smoke\nprofile: smoke\n' > /tmp/dbt-dqm-smoke-project/dbt_project.yml
|
|
35
|
+
printf 'smoke:\n outputs:\n dev:\n type: postgres\n host: localhost\n user: smoke\n pass: smoke\n dbname: smoke\n schema: smoke\n' > /tmp/dbt-dqm-smoke-profiles/profiles.yml
|
|
36
|
+
timeout 20 uv run dbt-dqm app --project-dir /tmp/dbt-dqm-smoke-project --profiles-dir /tmp/dbt-dqm-smoke-profiles --target dev --port 8765 &
|
|
37
|
+
for attempt in 1 2 3 4 5 6 7 8 9 10; do
|
|
38
|
+
if curl --fail --silent http://127.0.0.1:8765/_stcore/health; then exit 0; fi
|
|
39
|
+
sleep 1
|
|
40
|
+
done
|
|
41
|
+
exit 1
|
|
42
|
+
|
|
43
|
+
# Validates the dbt package (models/, macros/) end to end against a real warehouse without
|
|
44
|
+
# needing any secrets: a Postgres service container plus the synthetic demo project. There is
|
|
45
|
+
# no equivalent BigQuery job here since that would require committing or configuring warehouse
|
|
46
|
+
# credentials as repo secrets; add one separately (e.g. gated on a repo secret's presence) if
|
|
47
|
+
# BigQuery-specific coverage in CI is wanted later.
|
|
48
|
+
package-smoke:
|
|
49
|
+
runs-on: ubuntu-latest
|
|
50
|
+
steps:
|
|
51
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
52
|
+
- name: Install uv
|
|
53
|
+
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
|
|
54
|
+
with:
|
|
55
|
+
python-version: "3.11"
|
|
56
|
+
- name: Build wheel and source distribution
|
|
57
|
+
run: uv build
|
|
58
|
+
- name: Check distribution metadata
|
|
59
|
+
run: uvx --from twine twine check dist/*
|
|
60
|
+
- name: Install the wheel into a clean environment and smoke-test the CLI
|
|
61
|
+
run: |
|
|
62
|
+
uv venv /tmp/dbt-dqm-wheel-smoke --python 3.11
|
|
63
|
+
uv pip install --python /tmp/dbt-dqm-wheel-smoke/bin/python dist/*.whl
|
|
64
|
+
/tmp/dbt-dqm-wheel-smoke/bin/dbt-dqm --help
|
|
65
|
+
- name: Wheel carries the security floors (#24)
|
|
66
|
+
run: DBT_DQM_WHEEL="$(ls dist/*.whl)" uv run pytest -q tests/test_dependency_floors.py
|
|
67
|
+
- name: Installing the wheel upgrades older vulnerable versions
|
|
68
|
+
run: |
|
|
69
|
+
wheel="$(pwd)/$(ls dist/*.whl)"
|
|
70
|
+
uv venv /tmp/dbt-dqm-old-deps --python 3.11
|
|
71
|
+
uv pip install --python /tmp/dbt-dqm-old-deps/bin/python sqlparse==0.5.5 urllib3==2.7.0 oauthlib==3.3.1
|
|
72
|
+
cd /tmp
|
|
73
|
+
uv pip install --python /tmp/dbt-dqm-old-deps/bin/python "$wheel"
|
|
74
|
+
uv pip check --python /tmp/dbt-dqm-old-deps/bin/python
|
|
75
|
+
/tmp/dbt-dqm-old-deps/bin/python - <<'PY'
|
|
76
|
+
from importlib.metadata import version
|
|
77
|
+
from packaging.version import Version
|
|
78
|
+
floors = {"sqlparse": "0.6.0", "urllib3": "2.8.0", "oauthlib": "4.0.0"}
|
|
79
|
+
installed = {name: version(name) for name in floors}
|
|
80
|
+
print(installed)
|
|
81
|
+
assert all(Version(installed[n]) >= Version(v) for n, v in floors.items()), installed
|
|
82
|
+
PY
|
|
83
|
+
|
|
84
|
+
dbt-postgres:
|
|
85
|
+
runs-on: ubuntu-latest
|
|
86
|
+
strategy:
|
|
87
|
+
fail-fast: false
|
|
88
|
+
matrix:
|
|
89
|
+
postgres: ["14", "16"]
|
|
90
|
+
services:
|
|
91
|
+
postgres:
|
|
92
|
+
image: postgres:${{ matrix.postgres }}
|
|
93
|
+
env:
|
|
94
|
+
POSTGRES_PASSWORD: dbtdqm
|
|
95
|
+
POSTGRES_DB: dbt_dqm_ci
|
|
96
|
+
ports:
|
|
97
|
+
- 5432:5432
|
|
98
|
+
options: >-
|
|
99
|
+
--health-cmd pg_isready
|
|
100
|
+
--health-interval 10s
|
|
101
|
+
--health-timeout 5s
|
|
102
|
+
--health-retries 5
|
|
103
|
+
steps:
|
|
104
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
105
|
+
|
|
106
|
+
- name: Install uv
|
|
107
|
+
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
|
|
108
|
+
with:
|
|
109
|
+
python-version: "3.11"
|
|
110
|
+
|
|
111
|
+
- name: Install dependencies
|
|
112
|
+
run: uv sync --all-groups
|
|
113
|
+
|
|
114
|
+
- name: Enable pgcrypto
|
|
115
|
+
run: psql -h localhost -U postgres -d dbt_dqm_ci -c "create extension if not exists pgcrypto;"
|
|
116
|
+
env:
|
|
117
|
+
PGPASSWORD: dbtdqm
|
|
118
|
+
|
|
119
|
+
- name: Write profiles.yml
|
|
120
|
+
run: |
|
|
121
|
+
mkdir -p ~/.dbt
|
|
122
|
+
cat > ~/.dbt/profiles.yml <<'PROFILES'
|
|
123
|
+
dbt_dqm_demo_postgres:
|
|
124
|
+
target: ci
|
|
125
|
+
outputs:
|
|
126
|
+
ci:
|
|
127
|
+
type: postgres
|
|
128
|
+
host: localhost
|
|
129
|
+
port: 5432
|
|
130
|
+
user: postgres
|
|
131
|
+
pass: dbtdqm
|
|
132
|
+
dbname: dbt_dqm_ci
|
|
133
|
+
schema: dbt_dqm_demo
|
|
134
|
+
threads: 4
|
|
135
|
+
PROFILES
|
|
136
|
+
|
|
137
|
+
- name: dbt deps
|
|
138
|
+
working-directory: integration_tests/demo_postgres
|
|
139
|
+
run: uv run dbt deps
|
|
140
|
+
|
|
141
|
+
- name: Full lifecycle walkthrough (initial -> passed -> recurrence)
|
|
142
|
+
working-directory: integration_tests/demo_postgres
|
|
143
|
+
run: |
|
|
144
|
+
set -e
|
|
145
|
+
uv run dbt run-operation reset_demo_dqm
|
|
146
|
+
uv run dbt seed
|
|
147
|
+
for scenario in initial passed recurrence; do
|
|
148
|
+
uv run dbt run --select demo_records --vars "{demo_scenario: $scenario}"
|
|
149
|
+
uv run dbt test --select tag:dqm --vars "{demo_scenario: $scenario}"
|
|
150
|
+
uv run dbt test --select assert_collection_guardrail_fixtures
|
|
151
|
+
uv run dbt build --select package:dbt_dqm --vars "{demo_scenario: $scenario}"
|
|
152
|
+
uv run dbt test --select assert_scenario_outcomes --vars "{demo_scenario: $scenario}"
|
|
153
|
+
done
|
|
154
|
+
|
|
155
|
+
- name: Replay multiple executions between reconciliations
|
|
156
|
+
working-directory: integration_tests/demo_postgres
|
|
157
|
+
run: |
|
|
158
|
+
set -e
|
|
159
|
+
uv run dbt run-operation reset_demo_dqm
|
|
160
|
+
uv run dbt run --select demo_records --vars '{demo_scenario: initial}'
|
|
161
|
+
uv run dbt test --select tag:dqm --vars '{demo_scenario: initial}'
|
|
162
|
+
uv run dbt run --select demo_records --vars '{demo_scenario: passed}'
|
|
163
|
+
uv run dbt test --select tag:dqm --vars '{demo_scenario: passed}'
|
|
164
|
+
uv run dbt build --select package:dbt_dqm --vars '{demo_scenario: passed}'
|
|
165
|
+
uv run dbt test --select assert_scenario_outcomes --vars '{demo_scenario: passed}'
|
|
166
|
+
uv run dbt run-operation reset_demo_dqm
|
|
167
|
+
for scenario in initial passed recurrence; do
|
|
168
|
+
uv run dbt run --select demo_records --vars "{demo_scenario: $scenario}"
|
|
169
|
+
uv run dbt test --select tag:dqm --vars "{demo_scenario: $scenario}"
|
|
170
|
+
done
|
|
171
|
+
uv run dbt build --select package:dbt_dqm --vars '{demo_scenario: recurrence}'
|
|
172
|
+
uv run dbt test --select assert_scenario_outcomes --vars '{demo_scenario: recurrence}'
|
|
173
|
+
uv run dbt run-operation assert_postgres_query_plans
|
|
174
|
+
|
|
175
|
+
- name: Transactional acceptance and application interfaces
|
|
176
|
+
shell: bash
|
|
177
|
+
run: |
|
|
178
|
+
set -o pipefail
|
|
179
|
+
mkdir -p acceptance-results
|
|
180
|
+
uv run pytest -q -rA integration_tests/acceptance/test_postgres.py \
|
|
181
|
+
--junitxml=acceptance-results/junit.xml 2>&1 | tee acceptance-results/pytest.log
|
|
182
|
+
env:
|
|
183
|
+
DBT_DQM_TEST_DSN: host=localhost port=5432 dbname=dbt_dqm_ci user=postgres password=dbtdqm
|
|
184
|
+
|
|
185
|
+
# Annotations only report the exit code; keep the full pytest output for diagnosis.
|
|
186
|
+
- name: Upload acceptance results
|
|
187
|
+
if: ${{ !cancelled() }}
|
|
188
|
+
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
|
189
|
+
with:
|
|
190
|
+
name: postgres-${{ matrix.postgres }}-acceptance
|
|
191
|
+
path: acceptance-results/
|
|
192
|
+
if-no-files-found: ignore
|
|
193
|
+
|
|
194
|
+
bigquery-compile:
|
|
195
|
+
runs-on: ubuntu-latest
|
|
196
|
+
steps:
|
|
197
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
198
|
+
- name: Install uv
|
|
199
|
+
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
|
|
200
|
+
with:
|
|
201
|
+
python-version: "3.11"
|
|
202
|
+
- name: Install dependencies
|
|
203
|
+
run: uv sync --all-groups
|
|
204
|
+
- name: Write compile-only profile
|
|
205
|
+
run: |
|
|
206
|
+
mkdir -p ~/.dbt
|
|
207
|
+
cat > ~/.dbt/profiles.yml <<'PROFILES'
|
|
208
|
+
dbt_dqm_demo:
|
|
209
|
+
target: ci
|
|
210
|
+
outputs:
|
|
211
|
+
ci:
|
|
212
|
+
type: bigquery
|
|
213
|
+
method: oauth
|
|
214
|
+
project: dbt-dqm-compile-only
|
|
215
|
+
dataset: dbt_dqm_demo
|
|
216
|
+
location: US
|
|
217
|
+
threads: 1
|
|
218
|
+
PROFILES
|
|
219
|
+
- name: Parse and compile without warehouse introspection
|
|
220
|
+
working-directory: integration_tests/demo_bigquery
|
|
221
|
+
run: |
|
|
222
|
+
uv run dbt deps
|
|
223
|
+
uv run dbt parse
|
|
224
|
+
uv run python ../../scripts/compile_bigquery.py
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags: ["v*"]
|
|
6
|
+
|
|
7
|
+
permissions:
|
|
8
|
+
contents: write
|
|
9
|
+
id-token: write
|
|
10
|
+
attestations: write
|
|
11
|
+
|
|
12
|
+
jobs:
|
|
13
|
+
release:
|
|
14
|
+
runs-on: ubuntu-latest
|
|
15
|
+
environment: pypi
|
|
16
|
+
services:
|
|
17
|
+
postgres:
|
|
18
|
+
image: postgres:16
|
|
19
|
+
env:
|
|
20
|
+
POSTGRES_PASSWORD: dbtdqm
|
|
21
|
+
POSTGRES_DB: dbt_dqm_release
|
|
22
|
+
ports:
|
|
23
|
+
- 5432:5432
|
|
24
|
+
options: >-
|
|
25
|
+
--health-cmd pg_isready
|
|
26
|
+
--health-interval 10s
|
|
27
|
+
--health-timeout 5s
|
|
28
|
+
--health-retries 5
|
|
29
|
+
steps:
|
|
30
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
31
|
+
- name: Check the tag matches the package versions
|
|
32
|
+
shell: bash
|
|
33
|
+
run: |
|
|
34
|
+
set -euo pipefail
|
|
35
|
+
python_version="$(python3 -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')"
|
|
36
|
+
dbt_version="$(python3 -c 'import re; print(re.search(r"^version:\s*[\x27\x22]?([^\x27\x22\s]+)", open("dbt_project.yml").read(), re.M).group(1))')"
|
|
37
|
+
# dbt only accepts SemVer, so a Python pre-release 0.2.0a1 is spelled 0.2.0-a1 there.
|
|
38
|
+
expected_dbt_version="$(python3 -c 'import re, sys; print(re.sub(r"^(\d+\.\d+\.\d+)((?:a|b|rc)\d+)$", r"\1-\2", sys.argv[1]))' "$python_version")"
|
|
39
|
+
if [[ "$dbt_version" != "$expected_dbt_version" ]]; then
|
|
40
|
+
echo "pyproject.toml ($python_version) and dbt_project.yml ($dbt_version) versions differ; expected dbt version $expected_dbt_version." >&2
|
|
41
|
+
exit 1
|
|
42
|
+
fi
|
|
43
|
+
if [[ "$GITHUB_REF_NAME" != "v$python_version" ]]; then
|
|
44
|
+
echo "Tag $GITHUB_REF_NAME doesn't match package version v$python_version." >&2
|
|
45
|
+
exit 1
|
|
46
|
+
fi
|
|
47
|
+
- name: Install uv
|
|
48
|
+
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
|
|
49
|
+
with:
|
|
50
|
+
python-version: "3.11"
|
|
51
|
+
- name: Verify, build, and inspect distributions
|
|
52
|
+
run: |
|
|
53
|
+
uv sync --locked --all-groups
|
|
54
|
+
uv run ruff check .
|
|
55
|
+
uv run pytest -q
|
|
56
|
+
uv build --no-create-gitignore
|
|
57
|
+
uvx --from twine twine check dist/*
|
|
58
|
+
- name: Configure release-gate Postgres
|
|
59
|
+
run: |
|
|
60
|
+
psql -h localhost -U postgres -d dbt_dqm_release -c "create extension if not exists pgcrypto;"
|
|
61
|
+
mkdir -p ~/.dbt
|
|
62
|
+
cat > ~/.dbt/profiles.yml <<'PROFILES'
|
|
63
|
+
dbt_dqm_demo_postgres:
|
|
64
|
+
target: release
|
|
65
|
+
outputs:
|
|
66
|
+
release:
|
|
67
|
+
type: postgres
|
|
68
|
+
host: localhost
|
|
69
|
+
port: 5432
|
|
70
|
+
user: postgres
|
|
71
|
+
pass: dbtdqm
|
|
72
|
+
dbname: dbt_dqm_release
|
|
73
|
+
schema: dbt_dqm_demo
|
|
74
|
+
threads: 4
|
|
75
|
+
PROFILES
|
|
76
|
+
env:
|
|
77
|
+
PGPASSWORD: dbtdqm
|
|
78
|
+
- name: Full Postgres release lifecycle
|
|
79
|
+
working-directory: integration_tests/demo_postgres
|
|
80
|
+
run: |
|
|
81
|
+
uv run dbt deps
|
|
82
|
+
uv run dbt run-operation reset_demo_dqm
|
|
83
|
+
uv run dbt seed
|
|
84
|
+
for scenario in initial passed recurrence; do
|
|
85
|
+
uv run dbt run --select demo_records --vars "{demo_scenario: $scenario}"
|
|
86
|
+
uv run dbt test --select tag:dqm --vars "{demo_scenario: $scenario}"
|
|
87
|
+
uv run dbt test --select assert_collection_guardrail_fixtures
|
|
88
|
+
uv run dbt build --select package:dbt_dqm --vars "{demo_scenario: $scenario}"
|
|
89
|
+
uv run dbt test --select assert_scenario_outcomes --vars "{demo_scenario: $scenario}"
|
|
90
|
+
done
|
|
91
|
+
- name: Attest distribution provenance
|
|
92
|
+
uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2
|
|
93
|
+
with:
|
|
94
|
+
subject-path: dist/*
|
|
95
|
+
- name: Verify distribution provenance
|
|
96
|
+
shell: bash
|
|
97
|
+
env:
|
|
98
|
+
GH_TOKEN: ${{ github.token }}
|
|
99
|
+
run: |
|
|
100
|
+
set -euo pipefail
|
|
101
|
+
shopt -s nullglob
|
|
102
|
+
wheels=(dist/*.whl)
|
|
103
|
+
sdists=(dist/*.tar.gz)
|
|
104
|
+
files=(dist/* dist/.[!.]* dist/..?*)
|
|
105
|
+
if (( ${#wheels[@]} != 1 || ${#sdists[@]} != 1 || ${#files[@]} != 2 )); then
|
|
106
|
+
echo 'Expected exactly one wheel and one source archive, with no other distribution files.' >&2
|
|
107
|
+
exit 1
|
|
108
|
+
fi
|
|
109
|
+
for artifact in "${wheels[@]}" "${sdists[@]}"; do
|
|
110
|
+
if [[ ! -f "$artifact" || -L "$artifact" ]]; then
|
|
111
|
+
echo "Expected a regular distribution file: $artifact" >&2
|
|
112
|
+
exit 1
|
|
113
|
+
fi
|
|
114
|
+
verified=false
|
|
115
|
+
for attempt in 1 2 3; do
|
|
116
|
+
if gh attestation verify "$artifact" \
|
|
117
|
+
--repo "$GITHUB_REPOSITORY" \
|
|
118
|
+
--signer-workflow "$GITHUB_REPOSITORY/.github/workflows/release.yml" \
|
|
119
|
+
--source-digest "$GITHUB_SHA" \
|
|
120
|
+
--source-ref "$GITHUB_REF" \
|
|
121
|
+
--deny-self-hosted-runners; then
|
|
122
|
+
verified=true
|
|
123
|
+
break
|
|
124
|
+
fi
|
|
125
|
+
if (( attempt < 3 )); then
|
|
126
|
+
sleep 5
|
|
127
|
+
fi
|
|
128
|
+
done
|
|
129
|
+
if [[ "$verified" != true ]]; then
|
|
130
|
+
echo "Provenance verification failed after three attempts: $artifact" >&2
|
|
131
|
+
exit 1
|
|
132
|
+
fi
|
|
133
|
+
done
|
|
134
|
+
- name: Publish to PyPI through trusted publishing
|
|
135
|
+
run: uv publish
|
|
136
|
+
- name: Create GitHub release
|
|
137
|
+
run: |
|
|
138
|
+
set -euo pipefail
|
|
139
|
+
version="${GITHUB_REF_NAME#v}"
|
|
140
|
+
if [[ "$version" =~ (a|b|rc)[0-9]+$ ]]; then
|
|
141
|
+
dbt_version="$(python3 -c 'import re, sys; print(re.sub(r"^(\d+\.\d+\.\d+)((?:a|b|rc)\d+)$", r"\1-\2", sys.argv[1]))' "$version")"
|
|
142
|
+
gh release create "${GITHUB_REF_NAME}" dist/* --generate-notes --verify-tag \
|
|
143
|
+
--prerelease --title "dbt-dqm ${GITHUB_REF_NAME} (pre-release)" \
|
|
144
|
+
--notes "Pre-release: interfaces may change. Install the Python app with pip install dbt-dqm==${version} and install the dbt package separately from Git revision ${GITHUB_REF_NAME} (its dbt_project.yml spells the version ${dbt_version}). Upgrades from 0.1 require a fresh DQM schema. See README.md and CHANGELOG.md at this tag for supported versions and operating limits."
|
|
145
|
+
else
|
|
146
|
+
gh release create "${GITHUB_REF_NAME}" dist/* --generate-notes --verify-tag
|
|
147
|
+
fi
|
|
148
|
+
env:
|
|
149
|
+
GH_TOKEN: ${{ github.token }}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
.venv/
|
|
2
|
+
target/
|
|
3
|
+
dbt_packages/
|
|
4
|
+
logs/
|
|
5
|
+
*.sqlite*
|
|
6
|
+
__pycache__/
|
|
7
|
+
.pytest_cache/
|
|
8
|
+
.ruff_cache/
|
|
9
|
+
.DS_Store
|
|
10
|
+
*.json
|
|
11
|
+
!integration_tests/demo_bigquery/seeds/*.csv
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
# Benchmark results referenced by docs/scale.md (synthetic data only).
|
|
15
|
+
!docs/scale-results/**/*.json
|
|
16
|
+
!docs/scale-results/*.json
|