dbt-dqm 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dbt_dqm-0.2.0/.github/ISSUE_TEMPLATE/bug_report.yml +28 -0
- dbt_dqm-0.2.0/.github/ISSUE_TEMPLATE/feature_request.yml +19 -0
- dbt_dqm-0.2.0/.github/dependabot.yml +32 -0
- dbt_dqm-0.2.0/.github/pull_request_template.md +13 -0
- dbt_dqm-0.2.0/.github/workflows/bigquery-scheduled.yml +111 -0
- dbt_dqm-0.2.0/.github/workflows/ci.yml +267 -0
- dbt_dqm-0.2.0/.github/workflows/release.yml +149 -0
- dbt_dqm-0.2.0/.gitignore +16 -0
- dbt_dqm-0.2.0/CHANGELOG.md +285 -0
- dbt_dqm-0.2.0/CODE_OF_CONDUCT.md +58 -0
- dbt_dqm-0.2.0/CONTRIBUTING.md +17 -0
- dbt_dqm-0.2.0/LICENSE.md +190 -0
- dbt_dqm-0.2.0/PKG-INFO +553 -0
- dbt_dqm-0.2.0/README.md +512 -0
- dbt_dqm-0.2.0/SECURITY.md +30 -0
- dbt_dqm-0.2.0/dbt_project.yml +31 -0
- dbt_dqm-0.2.0/docs/design/warehouse-browsing.md +163 -0
- dbt_dqm-0.2.0/docs/functional-requirements.md +99 -0
- dbt_dqm-0.2.0/docs/implementation-0.2.md +152 -0
- dbt_dqm-0.2.0/docs/project-review.md +245 -0
- dbt_dqm-0.2.0/docs/qa-release-safeguards.md +118 -0
- dbt_dqm-0.2.0/docs/release-0.2.0.md +23 -0
- dbt_dqm-0.2.0/docs/release-0.2.0a1.md +109 -0
- dbt_dqm-0.2.0/docs/release-0.2.0a2.md +79 -0
- dbt_dqm-0.2.0/docs/release-0.2.0a3.md +57 -0
- dbt_dqm-0.2.0/docs/release-0.2.0a4.md +106 -0
- dbt_dqm-0.2.0/docs/release-0.2.0a5.md +80 -0
- dbt_dqm-0.2.0/docs/release-0.2.0a6.md +18 -0
- dbt_dqm-0.2.0/docs/release-0.2.0a7.md +31 -0
- dbt_dqm-0.2.0/docs/release-0.2.0a8.md +27 -0
- dbt_dqm-0.2.0/docs/release-checklist.md +196 -0
- dbt_dqm-0.2.0/docs/release-notes-0.2.0.md +25 -0
- dbt_dqm-0.2.0/docs/roadmap.md +59 -0
- dbt_dqm-0.2.0/docs/scale-results/allocator-ab-2026-10-10.json +202 -0
- dbt_dqm-0.2.0/docs/scale-results/bigquery-active-scope.json +67 -0
- dbt_dqm-0.2.0/docs/scale-results/bigquery-baseline.json +67 -0
- dbt_dqm-0.2.0/docs/scale-results/bigquery-observation-floor.json +67 -0
- dbt_dqm-0.2.0/docs/scale-results/browse-2026-10-11/bigquery-1m-chunks-0fd69f7.json +606 -0
- dbt_dqm-0.2.0/docs/scale-results/browse-2026-10-11/bigquery-1m-ef2e406.json +584 -0
- dbt_dqm-0.2.0/docs/scale-results/browse-2026-10-11/postgres-1m-ef2e406.json +387 -0
- dbt_dqm-0.2.0/docs/scale-results/browse-2026-10-11/postgres-plan-indexed-count_all.txt +23 -0
- dbt_dqm-0.2.0/docs/scale-results/browse-2026-10-11/postgres-plan-indexed-first_page.txt +16 -0
- dbt_dqm-0.2.0/docs/scale-results/browse-2026-10-11/postgres-plan-indexed-metrics.txt +25 -0
- dbt_dqm-0.2.0/docs/scale-results/browse-2026-10-11/postgres-plan-indexed-status_blocked.txt +27 -0
- dbt_dqm-0.2.0/docs/scale-results/browse-2026-10-11/postgres-plan-no-index-count_all.txt +23 -0
- dbt_dqm-0.2.0/docs/scale-results/browse-2026-10-11/postgres-plan-no-index-first_page.txt +27 -0
- dbt_dqm-0.2.0/docs/scale-results/browse-2026-10-11/postgres-plan-no-index-metrics.txt +25 -0
- dbt_dqm-0.2.0/docs/scale-results/browse-2026-10-11/postgres-plan-no-index-status_blocked.txt +27 -0
- dbt_dqm-0.2.0/docs/scale-results/cache-boundary-safeguards.json +10 -0
- dbt_dqm-0.2.0/docs/scale-results/candidate-11afd08-cache-boundary.json +22 -0
- dbt_dqm-0.2.0/docs/scale-results/candidate-4c36226-cache-boundary.json +10 -0
- dbt_dqm-0.2.0/docs/scale-results/candidate-55efa28-cache-boundary.json +22 -0
- dbt_dqm-0.2.0/docs/scale-results/candidate-608ce23-cache-boundary.json +10 -0
- dbt_dqm-0.2.0/docs/scale-results/candidate-608ce23-capture-benchmark.json +194 -0
- dbt_dqm-0.2.0/docs/scale-results/candidate-613e089-cache-boundary.json +38 -0
- dbt_dqm-0.2.0/docs/scale-results/candidate-69703f7-cache-boundary.json +37 -0
- dbt_dqm-0.2.0/docs/scale-results/candidate-6b5f0bd-cache-boundary.json +10 -0
- dbt_dqm-0.2.0/docs/scale-results/candidate-708b6c2-cache-boundary.json +10 -0
- dbt_dqm-0.2.0/docs/scale-results/candidate-955e6dc-cache-boundary.json +22 -0
- dbt_dqm-0.2.0/docs/scale-results/candidate-c9fa528-cache-boundary.json +10 -0
- dbt_dqm-0.2.0/docs/scale-results/candidate-fe629f1-cache-boundary.json +22 -0
- dbt_dqm-0.2.0/docs/scale-results/memory-interval-2026-10-10.json +463 -0
- dbt_dqm-0.2.0/docs/scale-results/pandas-3-2026-10-11.json +260 -0
- dbt_dqm-0.2.0/docs/scale-results/postgres-memory-1000000-payload8.json +29 -0
- dbt_dqm-0.2.0/docs/scale-results/postgres-memory-250000-payload8.json +29 -0
- dbt_dqm-0.2.0/docs/scale-results/postgres-memory-500000-payload8.json +29 -0
- dbt_dqm-0.2.0/docs/scale-results/postgres-planner/before-active-scope-postgres-1000000-payload8.json +41 -0
- dbt_dqm-0.2.0/docs/scale-results/postgres-planner/postgres-1000000-payload8.json +41 -0
- dbt_dqm-0.2.0/docs/scale-results/postgres-planner/postgres-20000-payload8.json +42 -0
- dbt_dqm-0.2.0/docs/scale-results/postgres-planner/postgres-200000-payload8.json +41 -0
- dbt_dqm-0.2.0/docs/scale-results/postgres-planner/postgres-plan-mass_pass-nestloop-off-change_set.txt +472 -0
- dbt_dqm-0.2.0/docs/scale-results/postgres-planner/postgres-plan-mass_pass-nestloop-on-change_set.txt +269 -0
- dbt_dqm-0.2.0/docs/scale-results/postgres-planner/postgres-plan-steady-nestloop-off-change_set.txt +449 -0
- dbt_dqm-0.2.0/docs/scale-results/postgres-planner/postgres-plan-steady-nestloop-on-change_set.txt +445 -0
- dbt_dqm-0.2.0/docs/scale-results/pyarrow-23-cache-boundary.json +43 -0
- dbt_dqm-0.2.0/docs/scale-results/pyarrow-25-2026-10-11.json +81 -0
- dbt_dqm-0.2.0/docs/scale.md +468 -0
- dbt_dqm-0.2.0/docs/scheduled-bigquery.md +69 -0
- dbt_dqm-0.2.0/docs/warehouse-interfaces.md +275 -0
- dbt_dqm-0.2.0/integration_tests/acceptance/alias_checks.py +66 -0
- dbt_dqm-0.2.0/integration_tests/acceptance/benchmark_capture.py +189 -0
- dbt_dqm-0.2.0/integration_tests/acceptance/bigquery_datasets.py +70 -0
- dbt_dqm-0.2.0/integration_tests/acceptance/browse_checks.py +91 -0
- dbt_dqm-0.2.0/integration_tests/acceptance/conftest.py +60 -0
- dbt_dqm-0.2.0/integration_tests/acceptance/explicit_capture.py +60 -0
- dbt_dqm-0.2.0/integration_tests/acceptance/namespace_checks.py +130 -0
- dbt_dqm-0.2.0/integration_tests/acceptance/reporting_checks.py +70 -0
- dbt_dqm-0.2.0/integration_tests/acceptance/review_checks.py +44 -0
- dbt_dqm-0.2.0/integration_tests/acceptance/test_bigquery.py +1321 -0
- dbt_dqm-0.2.0/integration_tests/acceptance/test_bigquery_permissions.py +201 -0
- dbt_dqm-0.2.0/integration_tests/acceptance/test_postgres.py +1726 -0
- dbt_dqm-0.2.0/integration_tests/demo_bigquery/dbt_project.yml +14 -0
- dbt_dqm-0.2.0/integration_tests/demo_bigquery/macros/acceptance_protocol.sql +90 -0
- dbt_dqm-0.2.0/integration_tests/demo_bigquery/macros/generate_schema_name.sql +12 -0
- dbt_dqm-0.2.0/integration_tests/demo_bigquery/macros/reset_demo_dqm.sql +42 -0
- dbt_dqm-0.2.0/integration_tests/demo_bigquery/macros/schema.yml +19 -0
- dbt_dqm-0.2.0/integration_tests/demo_bigquery/models/demo_records.sql +17 -0
- dbt_dqm-0.2.0/integration_tests/demo_bigquery/models/schema.yml +66 -0
- dbt_dqm-0.2.0/integration_tests/demo_bigquery/package-lock.yml +4 -0
- dbt_dqm-0.2.0/integration_tests/demo_bigquery/packages.yml +3 -0
- dbt_dqm-0.2.0/integration_tests/demo_bigquery/profiles.yml.example +11 -0
- dbt_dqm-0.2.0/integration_tests/demo_bigquery/seeds/demo_cases.csv +13 -0
- dbt_dqm-0.2.0/integration_tests/demo_bigquery/tests/assert_collection_guardrail_fixtures.sql +35 -0
- dbt_dqm-0.2.0/integration_tests/demo_bigquery/tests/assert_scenario_outcomes.sql +86 -0
- dbt_dqm-0.2.0/integration_tests/demo_bigquery/tests/demo_customer_email_invalid.sql +23 -0
- dbt_dqm-0.2.0/integration_tests/demo_bigquery/tests/demo_invalid_duplicate_grain.sql +10 -0
- dbt_dqm-0.2.0/integration_tests/demo_bigquery/tests/demo_invalid_empty_grain.sql +10 -0
- dbt_dqm-0.2.0/integration_tests/demo_bigquery/tests/demo_invalid_fail_calc.sql +2 -0
- dbt_dqm-0.2.0/integration_tests/demo_bigquery/tests/demo_invalid_limit.sql +2 -0
- dbt_dqm-0.2.0/integration_tests/demo_bigquery/tests/demo_invalid_missing_column.sql +10 -0
- dbt_dqm-0.2.0/integration_tests/demo_bigquery/tests/demo_invalid_non_string_grain.sql +10 -0
- dbt_dqm-0.2.0/integration_tests/demo_bigquery/tests/demo_invalid_owner.sql +2 -0
- dbt_dqm-0.2.0/integration_tests/demo_bigquery/tests/demo_invalid_priority.sql +2 -0
- dbt_dqm-0.2.0/integration_tests/demo_bigquery/tests/demo_invalid_view_storage.sql +2 -0
- dbt_dqm-0.2.0/integration_tests/demo_bigquery/tests/demo_order_amount_invalid.sql +21 -0
- dbt_dqm-0.2.0/integration_tests/demo_bigquery/tests/demo_order_line_invalid.sql +28 -0
- dbt_dqm-0.2.0/integration_tests/demo_bigquery/tests/schema.yml +17 -0
- dbt_dqm-0.2.0/integration_tests/demo_postgres/dbt_project.yml +13 -0
- dbt_dqm-0.2.0/integration_tests/demo_postgres/macros/assert_postgres_query_plans.sql +21 -0
- dbt_dqm-0.2.0/integration_tests/demo_postgres/macros/reset_demo_dqm.sql +46 -0
- dbt_dqm-0.2.0/integration_tests/demo_postgres/macros/schema.yml +11 -0
- dbt_dqm-0.2.0/integration_tests/demo_postgres/models/demo_records.sql +17 -0
- dbt_dqm-0.2.0/integration_tests/demo_postgres/models/schema.yml +66 -0
- dbt_dqm-0.2.0/integration_tests/demo_postgres/package-lock.yml +4 -0
- dbt_dqm-0.2.0/integration_tests/demo_postgres/packages.yml +2 -0
- dbt_dqm-0.2.0/integration_tests/demo_postgres/profiles.yml.example +17 -0
- dbt_dqm-0.2.0/integration_tests/demo_postgres/seeds/demo_cases.csv +13 -0
- dbt_dqm-0.2.0/integration_tests/demo_postgres/tests/assert_collection_guardrail_fixtures.sql +35 -0
- dbt_dqm-0.2.0/integration_tests/demo_postgres/tests/assert_scenario_outcomes.sql +80 -0
- dbt_dqm-0.2.0/integration_tests/demo_postgres/tests/demo_customer_email_invalid.sql +23 -0
- dbt_dqm-0.2.0/integration_tests/demo_postgres/tests/demo_invalid_duplicate_grain.sql +10 -0
- dbt_dqm-0.2.0/integration_tests/demo_postgres/tests/demo_invalid_empty_grain.sql +10 -0
- dbt_dqm-0.2.0/integration_tests/demo_postgres/tests/demo_invalid_fail_calc.sql +2 -0
- dbt_dqm-0.2.0/integration_tests/demo_postgres/tests/demo_invalid_limit.sql +2 -0
- dbt_dqm-0.2.0/integration_tests/demo_postgres/tests/demo_invalid_missing_column.sql +10 -0
- dbt_dqm-0.2.0/integration_tests/demo_postgres/tests/demo_invalid_non_string_grain.sql +10 -0
- dbt_dqm-0.2.0/integration_tests/demo_postgres/tests/demo_invalid_owner.sql +2 -0
- dbt_dqm-0.2.0/integration_tests/demo_postgres/tests/demo_invalid_priority.sql +2 -0
- dbt_dqm-0.2.0/integration_tests/demo_postgres/tests/demo_invalid_view_storage.sql +2 -0
- dbt_dqm-0.2.0/integration_tests/demo_postgres/tests/demo_order_amount_invalid.sql +21 -0
- dbt_dqm-0.2.0/integration_tests/demo_postgres/tests/demo_order_line_invalid.sql +28 -0
- dbt_dqm-0.2.0/integration_tests/demo_postgres/tests/schema.yml +17 -0
- dbt_dqm-0.2.0/integration_tests/scale/app_memory.py +88 -0
- dbt_dqm-0.2.0/integration_tests/scale/bigquery_scale.py +322 -0
- dbt_dqm-0.2.0/integration_tests/scale/browse_probe.py +199 -0
- dbt_dqm-0.2.0/integration_tests/scale/cache_boundary.py +54 -0
- dbt_dqm-0.2.0/integration_tests/scale/postgres_scale.py +452 -0
- dbt_dqm-0.2.0/macros/adapters.sql +249 -0
- dbt_dqm-0.2.0/macros/capture_test_results.sql +388 -0
- dbt_dqm-0.2.0/macros/health.sql +48 -0
- dbt_dqm-0.2.0/macros/helpers.sql +152 -0
- dbt_dqm-0.2.0/macros/history.sql +15 -0
- dbt_dqm-0.2.0/macros/maintenance.sql +387 -0
- dbt_dqm-0.2.0/macros/materializations.sql +58 -0
- dbt_dqm-0.2.0/macros/reconciliation.sql +216 -0
- dbt_dqm-0.2.0/macros/recovery.sql +147 -0
- dbt_dqm-0.2.0/macros/refresh.sql +17 -0
- dbt_dqm-0.2.0/macros/replay.sql +410 -0
- dbt_dqm-0.2.0/macros/schema.yml +188 -0
- dbt_dqm-0.2.0/macros/schema_management.sql +443 -0
- dbt_dqm-0.2.0/macros/table_schemas.sql +157 -0
- dbt_dqm-0.2.0/macros/test_materialization.sql +36 -0
- dbt_dqm-0.2.0/macros/workflow.sql +85 -0
- dbt_dqm-0.2.0/models/dqm_all_issues.sql +3 -0
- dbt_dqm-0.2.0/models/dqm_annotation_changes.sql +25 -0
- dbt_dqm-0.2.0/models/dqm_area_health.sql +39 -0
- dbt_dqm-0.2.0/models/dqm_current_issues.sql +3 -0
- dbt_dqm-0.2.0/models/dqm_issue_timeline.sql +6 -0
- dbt_dqm-0.2.0/models/dqm_maintenance_health.sql +136 -0
- dbt_dqm-0.2.0/models/dqm_missed_issues.sql +7 -0
- dbt_dqm-0.2.0/models/dqm_reconcile.sql +5 -0
- dbt_dqm-0.2.0/models/dqm_test_health.sql +113 -0
- dbt_dqm-0.2.0/models/schema.yml +163 -0
- dbt_dqm-0.2.0/packages.yml +2 -0
- dbt_dqm-0.2.0/pyproject.toml +74 -0
- dbt_dqm-0.2.0/scripts/check_optional_install.py +98 -0
- dbt_dqm-0.2.0/scripts/check_wheel_variants.py +31 -0
- dbt_dqm-0.2.0/scripts/compile_bigquery.py +41 -0
- dbt_dqm-0.2.0/selectors.yml +7 -0
- dbt_dqm-0.2.0/src/dbt_dqm_app/__init__.py +3 -0
- dbt_dqm-0.2.0/src/dbt_dqm_app/adapters.py +35 -0
- dbt_dqm-0.2.0/src/dbt_dqm_app/browse.py +293 -0
- dbt_dqm-0.2.0/src/dbt_dqm_app/cli.py +166 -0
- dbt_dqm-0.2.0/src/dbt_dqm_app/config.py +239 -0
- dbt_dqm-0.2.0/src/dbt_dqm_app/display.py +241 -0
- dbt_dqm-0.2.0/src/dbt_dqm_app/errors.py +121 -0
- dbt_dqm-0.2.0/src/dbt_dqm_app/exports.py +310 -0
- dbt_dqm-0.2.0/src/dbt_dqm_app/frames.py +19 -0
- dbt_dqm-0.2.0/src/dbt_dqm_app/limits.py +34 -0
- dbt_dqm-0.2.0/src/dbt_dqm_app/refresh.py +227 -0
- dbt_dqm-0.2.0/src/dbt_dqm_app/relations.py +210 -0
- dbt_dqm-0.2.0/src/dbt_dqm_app/runner.py +425 -0
- dbt_dqm-0.2.0/src/dbt_dqm_app/selection.py +33 -0
- dbt_dqm-0.2.0/src/dbt_dqm_app/store.py +600 -0
- dbt_dqm-0.2.0/src/dbt_dqm_app/streamlit_app.py +1207 -0
- dbt_dqm-0.2.0/src/dbt_dqm_app/summary.py +88 -0
- dbt_dqm-0.2.0/src/dbt_dqm_app/warehouse.py +1030 -0
- dbt_dqm-0.2.0/tests/assert_at_most_one_active_occurrence.sql +11 -0
- dbt_dqm-0.2.0/tests/assert_collection_evidence_guardrails.sql +14 -0
- dbt_dqm-0.2.0/tests/assert_event_id_unique.sql +5 -0
- dbt_dqm-0.2.0/tests/assert_identity_hash_vectors.sql +50 -0
- dbt_dqm-0.2.0/tests/assert_occurrence_id_unique.sql +5 -0
- dbt_dqm-0.2.0/tests/assert_receipt_unique.sql +5 -0
- dbt_dqm-0.2.0/tests/test_acceptance_sweep.py +58 -0
- dbt_dqm-0.2.0/tests/test_annotation_staging.py +83 -0
- dbt_dqm-0.2.0/tests/test_browse.py +312 -0
- dbt_dqm-0.2.0/tests/test_browse_warehouse.py +210 -0
- dbt_dqm-0.2.0/tests/test_cache_limits.py +190 -0
- dbt_dqm-0.2.0/tests/test_cli.py +94 -0
- dbt_dqm-0.2.0/tests/test_config.py +113 -0
- dbt_dqm-0.2.0/tests/test_credentials.py +114 -0
- dbt_dqm-0.2.0/tests/test_dbt_hints.py +32 -0
- dbt_dqm-0.2.0/tests/test_dependency_floors.py +66 -0
- dbt_dqm-0.2.0/tests/test_docs_consistency.py +66 -0
- dbt_dqm-0.2.0/tests/test_exports.py +200 -0
- dbt_dqm-0.2.0/tests/test_features.py +195 -0
- dbt_dqm-0.2.0/tests/test_frames.py +18 -0
- dbt_dqm-0.2.0/tests/test_initialization_permissions.py +31 -0
- dbt_dqm-0.2.0/tests/test_maintenance_health.py +100 -0
- dbt_dqm-0.2.0/tests/test_missing_values.py +47 -0
- dbt_dqm-0.2.0/tests/test_oauthlib_compat.py +89 -0
- dbt_dqm-0.2.0/tests/test_optional_adapters.py +47 -0
- dbt_dqm-0.2.0/tests/test_portability.py +286 -0
- dbt_dqm-0.2.0/tests/test_protocol_render.py +598 -0
- dbt_dqm-0.2.0/tests/test_python_compatibility.py +60 -0
- dbt_dqm-0.2.0/tests/test_relations.py +74 -0
- dbt_dqm-0.2.0/tests/test_release_workflow.py +258 -0
- dbt_dqm-0.2.0/tests/test_review_actions.py +113 -0
- dbt_dqm-0.2.0/tests/test_runner.py +393 -0
- dbt_dqm-0.2.0/tests/test_scheduled_workflow.py +32 -0
- dbt_dqm-0.2.0/tests/test_store.py +336 -0
- dbt_dqm-0.2.0/tests/test_summary.py +136 -0
- dbt_dqm-0.2.0/tests/test_sync_and_errors.py +160 -0
- dbt_dqm-0.2.0/tests/test_ui.py +163 -0
- dbt_dqm-0.2.0/tests/test_ui_browse.py +397 -0
- dbt_dqm-0.2.0/tests/test_wif.py +149 -0
- dbt_dqm-0.2.0/uv.lock +2702 -0
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
name: Bug report
|
|
2
|
+
description: Report incorrect collection, lifecycle, adapter, or app behavior
|
|
3
|
+
body:
|
|
4
|
+
- type: textarea
|
|
5
|
+
id: behavior
|
|
6
|
+
attributes:
|
|
7
|
+
label: Observed behavior
|
|
8
|
+
validations:
|
|
9
|
+
required: true
|
|
10
|
+
- type: textarea
|
|
11
|
+
id: reproduction
|
|
12
|
+
attributes:
|
|
13
|
+
label: Minimal reproduction
|
|
14
|
+
validations:
|
|
15
|
+
required: true
|
|
16
|
+
- type: input
|
|
17
|
+
id: versions
|
|
18
|
+
attributes:
|
|
19
|
+
label: dbt-dqm, dbt, adapter, warehouse, and Python versions
|
|
20
|
+
validations:
|
|
21
|
+
required: true
|
|
22
|
+
- type: checkboxes
|
|
23
|
+
id: safety
|
|
24
|
+
attributes:
|
|
25
|
+
label: Data safety
|
|
26
|
+
options:
|
|
27
|
+
- label: I removed credentials, customer data, and complete failed-row contents.
|
|
28
|
+
required: true
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
name: Feature request
|
|
2
|
+
description: Propose a focused open-source improvement
|
|
3
|
+
body:
|
|
4
|
+
- type: textarea
|
|
5
|
+
id: problem
|
|
6
|
+
attributes:
|
|
7
|
+
label: User problem
|
|
8
|
+
validations:
|
|
9
|
+
required: true
|
|
10
|
+
- type: textarea
|
|
11
|
+
id: proposal
|
|
12
|
+
attributes:
|
|
13
|
+
label: Proposed behavior
|
|
14
|
+
validations:
|
|
15
|
+
required: true
|
|
16
|
+
- type: textarea
|
|
17
|
+
id: adapters
|
|
18
|
+
attributes:
|
|
19
|
+
label: Adapter and compatibility implications
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
version: 2
|
|
2
|
+
updates:
|
|
3
|
+
# Read uv.lock so version-type exclusions also cover range-only requirements.
|
|
4
|
+
- package-ecosystem: uv
|
|
5
|
+
directory: /
|
|
6
|
+
schedule:
|
|
7
|
+
interval: weekly
|
|
8
|
+
# Deferred compatibility work: https://github.com/a-adamovich/dbt-dqm/issues/11
|
|
9
|
+
# Update-type exclusions preserve patch updates and security-update handling.
|
|
10
|
+
ignore:
|
|
11
|
+
# dbt minors are adopted deliberately: the protocol SQL must render identically and the full
|
|
12
|
+
# warehouse suites must pass before the pins and require-dbt-version move (see #11).
|
|
13
|
+
- dependency-name: dbt-core
|
|
14
|
+
update-types: ["version-update:semver-minor"]
|
|
15
|
+
- dependency-name: dbt-bigquery
|
|
16
|
+
update-types: ["version-update:semver-minor"]
|
|
17
|
+
- dependency-name: dbt-postgres
|
|
18
|
+
update-types: ["version-update:semver-minor"]
|
|
19
|
+
- dependency-name: pandas
|
|
20
|
+
update-types: ["version-update:semver-major"]
|
|
21
|
+
# pyarrow majors are adopted deliberately, with the app memory benchmark rerun at the
|
|
22
|
+
# 50k cache boundary (pinned to 25.x; see #11 and docs/scale.md).
|
|
23
|
+
- dependency-name: pyarrow
|
|
24
|
+
update-types: ["version-update:semver-major"]
|
|
25
|
+
groups:
|
|
26
|
+
python-minor-patches:
|
|
27
|
+
patterns: ["*"]
|
|
28
|
+
update-types: [minor, patch]
|
|
29
|
+
- package-ecosystem: github-actions
|
|
30
|
+
directory: /
|
|
31
|
+
schedule:
|
|
32
|
+
interval: weekly
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
## Summary
|
|
2
|
+
|
|
3
|
+
Describe the focused change and its user-visible behavior.
|
|
4
|
+
|
|
5
|
+
## Verification
|
|
6
|
+
|
|
7
|
+
- [ ] `ruff check .`
|
|
8
|
+
- [ ] `pytest -q`
|
|
9
|
+
- [ ] Relevant dbt adapter/demo lifecycle checks
|
|
10
|
+
- [ ] Documentation and changelog updated where needed
|
|
11
|
+
- [ ] Commits include DCO sign-off (`git commit -s`)
|
|
12
|
+
|
|
13
|
+
Do not include credentials, proprietary data, customer names, or complete failed-row contents.
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
name: Scheduled BigQuery lifecycle
|
|
2
|
+
|
|
3
|
+
# Weekly credentialed BigQuery verification. Keyless: GitHub's OIDC token is exchanged through
|
|
4
|
+
# Workload Identity Federation for a short-lived token of a dedicated CI service account, so no
|
|
5
|
+
# long-lived key is stored. Inert until the repository variables below are set.
|
|
6
|
+
on:
|
|
7
|
+
workflow_dispatch:
|
|
8
|
+
schedule:
|
|
9
|
+
- cron: "17 6 * * 1"
|
|
10
|
+
|
|
11
|
+
permissions:
|
|
12
|
+
contents: read
|
|
13
|
+
id-token: write
|
|
14
|
+
issues: write
|
|
15
|
+
|
|
16
|
+
jobs:
|
|
17
|
+
lifecycle:
|
|
18
|
+
concurrency:
|
|
19
|
+
group: dbt-dqm-${{ vars.DBT_DQM_BIGQUERY_PROJECT }}-dbt_dqm_scheduled_ci
|
|
20
|
+
cancel-in-progress: false
|
|
21
|
+
if: vars.BIGQUERY_LIFECYCLE_ENABLED == 'true' && github.ref == 'refs/heads/main'
|
|
22
|
+
runs-on: ubuntu-latest
|
|
23
|
+
env:
|
|
24
|
+
DBT_DQM_BIGQUERY_PROJECT: ${{ vars.DBT_DQM_BIGQUERY_PROJECT }}
|
|
25
|
+
DBT_DQM_BIGQUERY_TEST_PROJECT: ${{ vars.DBT_DQM_BIGQUERY_PROJECT }}
|
|
26
|
+
steps:
|
|
27
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
28
|
+
- name: Authenticate to Google Cloud (Workload Identity Federation)
|
|
29
|
+
uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0
|
|
30
|
+
with:
|
|
31
|
+
workload_identity_provider: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }}
|
|
32
|
+
service_account: ${{ vars.GCP_CI_SERVICE_ACCOUNT }}
|
|
33
|
+
- name: Install uv
|
|
34
|
+
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
|
|
35
|
+
with:
|
|
36
|
+
python-version: "3.14"
|
|
37
|
+
- name: Install dependencies
|
|
38
|
+
run: uv sync --locked --all-groups --extra bigquery
|
|
39
|
+
- name: Write profile (Application Default Credentials from the step above)
|
|
40
|
+
run: |
|
|
41
|
+
mkdir -p ~/.dbt
|
|
42
|
+
cat > ~/.dbt/profiles.yml <<'PROFILES'
|
|
43
|
+
dbt_dqm_demo:
|
|
44
|
+
target: ci
|
|
45
|
+
outputs:
|
|
46
|
+
ci:
|
|
47
|
+
type: bigquery
|
|
48
|
+
method: oauth
|
|
49
|
+
project: "{{ env_var('DBT_DQM_BIGQUERY_PROJECT') }}"
|
|
50
|
+
dataset: dbt_dqm_scheduled_ci
|
|
51
|
+
location: US
|
|
52
|
+
threads: 4
|
|
53
|
+
PROFILES
|
|
54
|
+
- name: The CI identity cannot read the shared QA dataset
|
|
55
|
+
run: |
|
|
56
|
+
uv run python - <<'PY'
|
|
57
|
+
import os
|
|
58
|
+
from google.api_core.exceptions import Forbidden, NotFound
|
|
59
|
+
from google.cloud import bigquery
|
|
60
|
+
project = os.environ["DBT_DQM_BIGQUERY_PROJECT"]
|
|
61
|
+
try:
|
|
62
|
+
bigquery.Client(project=project).query(
|
|
63
|
+
f"select 1 from `{project}.dbt_dqm_qa.dqm_install` limit 1").result()
|
|
64
|
+
except (Forbidden, NotFound):
|
|
65
|
+
print("As expected, the CI service account has no access to dbt_dqm_qa.")
|
|
66
|
+
else:
|
|
67
|
+
raise SystemExit("The CI service account can read dbt_dqm_qa; narrow its roles.")
|
|
68
|
+
PY
|
|
69
|
+
- name: Full credentialed lifecycle (dedicated dataset)
|
|
70
|
+
working-directory: integration_tests/demo_bigquery
|
|
71
|
+
run: |
|
|
72
|
+
set -e
|
|
73
|
+
uv run dbt deps
|
|
74
|
+
uv run dbt run-operation reset_demo_dqm
|
|
75
|
+
uv run dbt seed
|
|
76
|
+
for scenario in initial passed recurrence; do
|
|
77
|
+
uv run dbt run --select demo_records --vars "{demo_scenario: $scenario}"
|
|
78
|
+
uv run dbt-dqm run --vars "{demo_scenario: $scenario}"
|
|
79
|
+
uv run dbt test --select assert_scenario_outcomes --vars "{demo_scenario: $scenario, dqm_enabled: true}"
|
|
80
|
+
done
|
|
81
|
+
- name: Credentialed acceptance suite (parallel, disposable datasets)
|
|
82
|
+
shell: bash
|
|
83
|
+
run: |
|
|
84
|
+
mkdir -p acceptance-results
|
|
85
|
+
set -o pipefail
|
|
86
|
+
uv run pytest -q -rA -n 6 --dist load integration_tests/acceptance/test_bigquery.py \
|
|
87
|
+
--junitxml=acceptance-results/bigquery.xml 2>&1 | tee acceptance-results/bigquery.log
|
|
88
|
+
- name: Sweep stale acceptance datasets
|
|
89
|
+
if: always()
|
|
90
|
+
run: uv run python integration_tests/acceptance/bigquery_datasets.py --project "$DBT_DQM_BIGQUERY_PROJECT" --older-than-hours 6
|
|
91
|
+
- name: Upload results
|
|
92
|
+
if: always()
|
|
93
|
+
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
|
94
|
+
with:
|
|
95
|
+
name: scheduled-bigquery-results
|
|
96
|
+
path: acceptance-results/
|
|
97
|
+
if-no-files-found: ignore
|
|
98
|
+
- name: Report failure in an issue
|
|
99
|
+
if: failure()
|
|
100
|
+
env:
|
|
101
|
+
GH_TOKEN: ${{ github.token }}
|
|
102
|
+
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
|
103
|
+
run: |
|
|
104
|
+
gh label create scheduled-bigquery --color B60205 --description "Scheduled BigQuery verification" 2>/dev/null || true
|
|
105
|
+
existing="$(gh issue list --label scheduled-bigquery --state open --json number --jq '.[0].number')"
|
|
106
|
+
if [ -n "$existing" ]; then
|
|
107
|
+
gh issue comment "$existing" --body "Scheduled BigQuery verification failed again: $RUN_URL"
|
|
108
|
+
else
|
|
109
|
+
gh issue create --label scheduled-bigquery --title "Scheduled BigQuery verification failed" \
|
|
110
|
+
--body "The weekly credentialed BigQuery run failed on \`main\`: $RUN_URL"
|
|
111
|
+
fi
|
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
permissions:
|
|
4
|
+
contents: read
|
|
5
|
+
|
|
6
|
+
on:
|
|
7
|
+
push:
|
|
8
|
+
branches: [main]
|
|
9
|
+
pull_request:
|
|
10
|
+
|
|
11
|
+
jobs:
|
|
12
|
+
lint-and-test:
|
|
13
|
+
runs-on: ubuntu-latest
|
|
14
|
+
strategy:
|
|
15
|
+
fail-fast: false
|
|
16
|
+
matrix:
|
|
17
|
+
python: ["3.11", "3.12", "3.13", "3.14"]
|
|
18
|
+
steps:
|
|
19
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
20
|
+
|
|
21
|
+
- name: Install uv
|
|
22
|
+
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
|
|
23
|
+
with:
|
|
24
|
+
python-version: ${{ matrix.python }}
|
|
25
|
+
|
|
26
|
+
- name: Install dependencies
|
|
27
|
+
run: uv sync --locked --all-groups --all-extras
|
|
28
|
+
- name: Check dependency consistency
|
|
29
|
+
run: uv pip check
|
|
30
|
+
|
|
31
|
+
- name: Ruff
|
|
32
|
+
run: uv run ruff check .
|
|
33
|
+
|
|
34
|
+
- name: Pytest
|
|
35
|
+
run: uv run pytest -q
|
|
36
|
+
|
|
37
|
+
- name: Streamlit startup smoke test
|
|
38
|
+
run: |
|
|
39
|
+
mkdir -p /tmp/dbt-dqm-smoke-project /tmp/dbt-dqm-smoke-profiles
|
|
40
|
+
printf 'name: smoke\nprofile: smoke\n' > /tmp/dbt-dqm-smoke-project/dbt_project.yml
|
|
41
|
+
printf 'smoke:\n outputs:\n dev:\n type: postgres\n host: localhost\n user: smoke\n pass: smoke\n dbname: smoke\n schema: smoke\n' > /tmp/dbt-dqm-smoke-profiles/profiles.yml
|
|
42
|
+
timeout 20 uv run dbt-dqm app --project-dir /tmp/dbt-dqm-smoke-project --profiles-dir /tmp/dbt-dqm-smoke-profiles --target dev --port 8765 &
|
|
43
|
+
for attempt in 1 2 3 4 5 6 7 8 9 10; do
|
|
44
|
+
if curl --fail --silent http://127.0.0.1:8765/_stcore/health; then exit 0; fi
|
|
45
|
+
sleep 1
|
|
46
|
+
done
|
|
47
|
+
exit 1
|
|
48
|
+
|
|
49
|
+
# Validates the dbt package (models/, macros/) end to end against a real warehouse without
|
|
50
|
+
# needing any secrets: a Postgres service container plus the synthetic demo project. There is
|
|
51
|
+
# no equivalent BigQuery job here since that would require committing or configuring warehouse
|
|
52
|
+
# credentials as repo secrets; add one separately (e.g. gated on a repo secret's presence) if
|
|
53
|
+
# BigQuery-specific coverage in CI is wanted later.
|
|
54
|
+
package-smoke:
|
|
55
|
+
runs-on: ubuntu-latest
|
|
56
|
+
strategy:
|
|
57
|
+
fail-fast: false
|
|
58
|
+
matrix:
|
|
59
|
+
python: ["3.11", "3.12", "3.13", "3.14"]
|
|
60
|
+
steps:
|
|
61
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
62
|
+
- name: Install uv
|
|
63
|
+
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
|
|
64
|
+
with:
|
|
65
|
+
python-version: ${{ matrix.python }}
|
|
66
|
+
- name: Install dependencies
|
|
67
|
+
run: uv sync --locked --all-groups --all-extras
|
|
68
|
+
- name: Check dependency consistency
|
|
69
|
+
run: uv pip check
|
|
70
|
+
- name: Build wheel and source distribution
|
|
71
|
+
run: uv build
|
|
72
|
+
- name: Check distribution metadata
|
|
73
|
+
run: uvx --from twine twine check dist/*
|
|
74
|
+
- name: Verify base and optional-adapter wheel installations
|
|
75
|
+
run: |
|
|
76
|
+
wheel="$(pwd)/$(ls dist/*.whl)"
|
|
77
|
+
smoke="$(pwd)/scripts/check_optional_install.py"
|
|
78
|
+
for variant in base bigquery postgres both; do
|
|
79
|
+
environment="/tmp/dbt-dqm-wheel-${variant}"
|
|
80
|
+
requirement="$wheel"
|
|
81
|
+
case "$variant" in
|
|
82
|
+
bigquery|postgres) requirement="${wheel}[${variant}]" ;;
|
|
83
|
+
both) requirement="${wheel}[bigquery,postgres]" ;;
|
|
84
|
+
esac
|
|
85
|
+
uv venv "$environment" --python ${{ matrix.python }}
|
|
86
|
+
uv pip install --python "$environment/bin/python" "$requirement"
|
|
87
|
+
uv pip check --python "$environment/bin/python"
|
|
88
|
+
"$environment/bin/python" "$smoke" "$variant"
|
|
89
|
+
done
|
|
90
|
+
- name: Wheel carries the security floors (#24)
|
|
91
|
+
run: DBT_DQM_WHEEL="$(ls dist/*.whl)" uv run pytest -q tests/test_dependency_floors.py tests/test_python_compatibility.py
|
|
92
|
+
- name: Installing the wheel upgrades older vulnerable versions
|
|
93
|
+
run: |
|
|
94
|
+
wheel="$(pwd)/$(ls dist/*.whl)"
|
|
95
|
+
uv venv /tmp/dbt-dqm-old-deps --python ${{ matrix.python }}
|
|
96
|
+
uv pip install --python /tmp/dbt-dqm-old-deps/bin/python sqlparse==0.5.5 urllib3==2.7.0 oauthlib==3.3.1 mashumaro==3.14
|
|
97
|
+
cd /tmp
|
|
98
|
+
uv pip install --python /tmp/dbt-dqm-old-deps/bin/python "${wheel}[bigquery,postgres]"
|
|
99
|
+
uv pip check --python /tmp/dbt-dqm-old-deps/bin/python
|
|
100
|
+
/tmp/dbt-dqm-old-deps/bin/python - <<'PY'
|
|
101
|
+
from importlib.metadata import version
|
|
102
|
+
from packaging.version import Version
|
|
103
|
+
floors = {"sqlparse": "0.6.0", "urllib3": "2.8.0", "oauthlib": "4.0.0", "mashumaro": "3.17"}
|
|
104
|
+
installed = {name: version(name) for name in floors}
|
|
105
|
+
print(installed)
|
|
106
|
+
assert all(Version(installed[n]) >= Version(v) for n, v in floors.items()), installed
|
|
107
|
+
PY
|
|
108
|
+
|
|
109
|
+
dbt-postgres:
|
|
110
|
+
runs-on: ubuntu-latest
|
|
111
|
+
strategy:
|
|
112
|
+
fail-fast: false
|
|
113
|
+
matrix:
|
|
114
|
+
python: ["3.11", "3.12", "3.13", "3.14"]
|
|
115
|
+
postgres: ["14", "16"]
|
|
116
|
+
services:
|
|
117
|
+
postgres:
|
|
118
|
+
image: postgres:${{ matrix.postgres }}
|
|
119
|
+
env:
|
|
120
|
+
POSTGRES_PASSWORD: dbtdqm
|
|
121
|
+
POSTGRES_DB: dbt_dqm_ci
|
|
122
|
+
ports:
|
|
123
|
+
- 5432:5432
|
|
124
|
+
options: >-
|
|
125
|
+
--health-cmd pg_isready
|
|
126
|
+
--health-interval 10s
|
|
127
|
+
--health-timeout 5s
|
|
128
|
+
--health-retries 5
|
|
129
|
+
steps:
|
|
130
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
131
|
+
|
|
132
|
+
- name: Install uv
|
|
133
|
+
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
|
|
134
|
+
with:
|
|
135
|
+
python-version: ${{ matrix.python }}
|
|
136
|
+
|
|
137
|
+
- name: Install dependencies
|
|
138
|
+
run: uv sync --locked --all-groups --extra postgres
|
|
139
|
+
- name: Check dependency consistency
|
|
140
|
+
run: uv pip check
|
|
141
|
+
|
|
142
|
+
- name: Enable pgcrypto
|
|
143
|
+
run: psql -h localhost -U postgres -d dbt_dqm_ci -c "create extension if not exists pgcrypto;"
|
|
144
|
+
env:
|
|
145
|
+
PGPASSWORD: dbtdqm
|
|
146
|
+
|
|
147
|
+
- name: Write profiles.yml
|
|
148
|
+
run: |
|
|
149
|
+
mkdir -p ~/.dbt
|
|
150
|
+
cat > ~/.dbt/profiles.yml <<'PROFILES'
|
|
151
|
+
dbt_dqm_demo_postgres:
|
|
152
|
+
target: ci
|
|
153
|
+
outputs:
|
|
154
|
+
ci:
|
|
155
|
+
type: postgres
|
|
156
|
+
host: localhost
|
|
157
|
+
port: 5432
|
|
158
|
+
user: postgres
|
|
159
|
+
pass: dbtdqm
|
|
160
|
+
dbname: dbt_dqm_ci
|
|
161
|
+
schema: dbt_dqm_demo
|
|
162
|
+
threads: 4
|
|
163
|
+
PROFILES
|
|
164
|
+
|
|
165
|
+
- name: dbt deps
|
|
166
|
+
working-directory: integration_tests/demo_postgres
|
|
167
|
+
run: uv run dbt deps
|
|
168
|
+
|
|
169
|
+
- name: Full lifecycle walkthrough (initial -> passed -> recurrence)
|
|
170
|
+
working-directory: integration_tests/demo_postgres
|
|
171
|
+
run: |
|
|
172
|
+
set -e
|
|
173
|
+
uv run dbt run-operation reset_demo_dqm
|
|
174
|
+
uv run dbt seed
|
|
175
|
+
for scenario in initial passed recurrence; do
|
|
176
|
+
uv run dbt run --select demo_records --vars "{demo_scenario: $scenario}"
|
|
177
|
+
uv run dbt-dqm run --vars "{demo_scenario: $scenario}"
|
|
178
|
+
uv run dbt test --select assert_scenario_outcomes --vars "{demo_scenario: $scenario, dqm_enabled: true}"
|
|
179
|
+
done
|
|
180
|
+
|
|
181
|
+
- name: Disabled commands and dedicated-workflow acceptance
|
|
182
|
+
working-directory: integration_tests/demo_postgres
|
|
183
|
+
run: |
|
|
184
|
+
uv run dbt parse
|
|
185
|
+
uv run dbt ls --select package:dbt_dqm --resource-type model --output name
|
|
186
|
+
uv run dbt run-operation assert_postgres_query_plans --vars '{dqm_enabled: true}'
|
|
187
|
+
|
|
188
|
+
- name: Transactional acceptance and application interfaces
|
|
189
|
+
shell: bash
|
|
190
|
+
run: |
|
|
191
|
+
set -o pipefail
|
|
192
|
+
mkdir -p acceptance-results
|
|
193
|
+
uv run pytest -q -rA integration_tests/acceptance/test_postgres.py \
|
|
194
|
+
--junitxml=acceptance-results/junit.xml 2>&1 | tee acceptance-results/pytest.log
|
|
195
|
+
env:
|
|
196
|
+
DBT_DQM_TEST_DSN: host=localhost port=5432 dbname=dbt_dqm_ci user=postgres password=dbtdqm
|
|
197
|
+
|
|
198
|
+
# Annotations only report the exit code; keep the full pytest output for diagnosis.
|
|
199
|
+
- name: Upload acceptance results
|
|
200
|
+
if: ${{ !cancelled() }}
|
|
201
|
+
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
|
202
|
+
with:
|
|
203
|
+
name: python-${{ matrix.python }}-postgres-${{ matrix.postgres }}-acceptance
|
|
204
|
+
path: acceptance-results/
|
|
205
|
+
if-no-files-found: ignore
|
|
206
|
+
|
|
207
|
+
bigquery-compile:
|
|
208
|
+
runs-on: ubuntu-latest
|
|
209
|
+
strategy:
|
|
210
|
+
fail-fast: false
|
|
211
|
+
matrix:
|
|
212
|
+
python: ["3.11", "3.12", "3.13", "3.14"]
|
|
213
|
+
steps:
|
|
214
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
215
|
+
- name: Install uv
|
|
216
|
+
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
|
|
217
|
+
with:
|
|
218
|
+
python-version: ${{ matrix.python }}
|
|
219
|
+
- name: Install dependencies
|
|
220
|
+
run: uv sync --locked --all-groups --extra bigquery
|
|
221
|
+
- name: Check dependency consistency
|
|
222
|
+
run: uv pip check
|
|
223
|
+
- name: Write compile-only profile
|
|
224
|
+
run: |
|
|
225
|
+
mkdir -p ~/.dbt
|
|
226
|
+
cat > ~/.dbt/profiles.yml <<'PROFILES'
|
|
227
|
+
dbt_dqm_demo:
|
|
228
|
+
target: ci
|
|
229
|
+
outputs:
|
|
230
|
+
ci:
|
|
231
|
+
type: bigquery
|
|
232
|
+
method: oauth
|
|
233
|
+
project: dbt-dqm-compile-only
|
|
234
|
+
dataset: dbt_dqm_demo
|
|
235
|
+
location: US
|
|
236
|
+
threads: 1
|
|
237
|
+
PROFILES
|
|
238
|
+
- name: Parse and compile without warehouse introspection
|
|
239
|
+
working-directory: integration_tests/demo_bigquery
|
|
240
|
+
run: |
|
|
241
|
+
uv run dbt deps
|
|
242
|
+
uv run dbt parse
|
|
243
|
+
uv run python ../../scripts/compile_bigquery.py
|
|
244
|
+
|
|
245
|
+
portability:
|
|
246
|
+
runs-on: ${{ matrix.os }}
|
|
247
|
+
strategy:
|
|
248
|
+
fail-fast: false
|
|
249
|
+
matrix:
|
|
250
|
+
os: [windows-latest, macos-latest]
|
|
251
|
+
python: ["3.11", "3.12", "3.13", "3.14"]
|
|
252
|
+
defaults:
|
|
253
|
+
run:
|
|
254
|
+
shell: bash
|
|
255
|
+
steps:
|
|
256
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
257
|
+
- uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
|
|
258
|
+
with:
|
|
259
|
+
python-version: ${{ matrix.python }}
|
|
260
|
+
- name: Install adapters and development dependencies
|
|
261
|
+
run: uv sync --locked --all-groups --all-extras
|
|
262
|
+
- name: Native configuration, process pool and authentication checks
|
|
263
|
+
run: uv run pytest -q tests/test_runner.py tests/test_portability.py tests/test_config.py tests/test_cli.py tests/test_credentials.py tests/test_dependency_floors.py tests/test_python_compatibility.py tests/test_wif.py tests/test_review_actions.py tests/test_exports.py tests/test_store.py tests/test_ui.py tests/test_ui_browse.py
|
|
264
|
+
- name: Build and check each installed wheel variant
|
|
265
|
+
run: |
|
|
266
|
+
uv build --no-create-gitignore
|
|
267
|
+
uv run python scripts/check_wheel_variants.py dist/*.whl
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags: ["v*"]
|
|
6
|
+
|
|
7
|
+
permissions:
|
|
8
|
+
contents: write
|
|
9
|
+
id-token: write
|
|
10
|
+
attestations: write
|
|
11
|
+
|
|
12
|
+
jobs:
|
|
13
|
+
release:
|
|
14
|
+
runs-on: ubuntu-latest
|
|
15
|
+
environment: pypi
|
|
16
|
+
services:
|
|
17
|
+
postgres:
|
|
18
|
+
image: postgres:16
|
|
19
|
+
env:
|
|
20
|
+
POSTGRES_PASSWORD: dbtdqm
|
|
21
|
+
POSTGRES_DB: dbt_dqm_release
|
|
22
|
+
ports:
|
|
23
|
+
- 5432:5432
|
|
24
|
+
options: >-
|
|
25
|
+
--health-cmd pg_isready
|
|
26
|
+
--health-interval 10s
|
|
27
|
+
--health-timeout 5s
|
|
28
|
+
--health-retries 5
|
|
29
|
+
steps:
|
|
30
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
31
|
+
- name: Check the tag matches the package versions
|
|
32
|
+
shell: bash
|
|
33
|
+
run: |
|
|
34
|
+
set -euo pipefail
|
|
35
|
+
python_version="$(python3 -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')"
|
|
36
|
+
dbt_version="$(python3 -c 'import re; print(re.search(r"^version:\s*[\x27\x22]?([^\x27\x22\s]+)", open("dbt_project.yml").read(), re.M).group(1))')"
|
|
37
|
+
# dbt only accepts SemVer, so a Python pre-release 0.2.0a1 is spelled 0.2.0-a1 there.
|
|
38
|
+
expected_dbt_version="$(python3 -c 'import re, sys; print(re.sub(r"^(\d+\.\d+\.\d+)((?:a|b|rc)\d+)$", r"\1-\2", sys.argv[1]))' "$python_version")"
|
|
39
|
+
if [[ "$dbt_version" != "$expected_dbt_version" ]]; then
|
|
40
|
+
echo "pyproject.toml ($python_version) and dbt_project.yml ($dbt_version) versions differ; expected dbt version $expected_dbt_version." >&2
|
|
41
|
+
exit 1
|
|
42
|
+
fi
|
|
43
|
+
if [[ "$GITHUB_REF_NAME" != "v$python_version" ]]; then
|
|
44
|
+
echo "Tag $GITHUB_REF_NAME doesn't match package version v$python_version." >&2
|
|
45
|
+
exit 1
|
|
46
|
+
fi
|
|
47
|
+
- name: Install uv
|
|
48
|
+
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
|
|
49
|
+
with:
|
|
50
|
+
python-version: "3.14"
|
|
51
|
+
- name: Verify, build, and inspect distributions
|
|
52
|
+
run: |
|
|
53
|
+
uv sync --locked --all-groups --all-extras
|
|
54
|
+
uv run ruff check .
|
|
55
|
+
uv run pytest -q
|
|
56
|
+
uv build --no-create-gitignore
|
|
57
|
+
uvx --from twine twine check dist/*
|
|
58
|
+
- name: Configure release-gate Postgres
|
|
59
|
+
run: |
|
|
60
|
+
psql -h localhost -U postgres -d dbt_dqm_release -c "create extension if not exists pgcrypto;"
|
|
61
|
+
mkdir -p ~/.dbt
|
|
62
|
+
cat > ~/.dbt/profiles.yml <<'PROFILES'
|
|
63
|
+
dbt_dqm_demo_postgres:
|
|
64
|
+
target: release
|
|
65
|
+
outputs:
|
|
66
|
+
release:
|
|
67
|
+
type: postgres
|
|
68
|
+
host: localhost
|
|
69
|
+
port: 5432
|
|
70
|
+
user: postgres
|
|
71
|
+
pass: dbtdqm
|
|
72
|
+
dbname: dbt_dqm_release
|
|
73
|
+
schema: dbt_dqm_demo
|
|
74
|
+
threads: 4
|
|
75
|
+
PROFILES
|
|
76
|
+
env:
|
|
77
|
+
PGPASSWORD: dbtdqm
|
|
78
|
+
- name: Full Postgres release lifecycle
|
|
79
|
+
working-directory: integration_tests/demo_postgres
|
|
80
|
+
run: |
|
|
81
|
+
uv run dbt deps
|
|
82
|
+
uv run dbt run-operation reset_demo_dqm
|
|
83
|
+
uv run dbt seed
|
|
84
|
+
for scenario in initial passed recurrence; do
|
|
85
|
+
uv run dbt run --select demo_records --vars "{demo_scenario: $scenario}"
|
|
86
|
+
uv run dbt-dqm run --vars "{demo_scenario: $scenario}"
|
|
87
|
+
uv run dbt test --select assert_scenario_outcomes --vars "{demo_scenario: $scenario, dqm_enabled: true}"
|
|
88
|
+
done
|
|
89
|
+
- name: Attest distribution provenance
|
|
90
|
+
uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2
|
|
91
|
+
with:
|
|
92
|
+
subject-path: dist/*
|
|
93
|
+
- name: Verify distribution provenance
|
|
94
|
+
shell: bash
|
|
95
|
+
env:
|
|
96
|
+
GH_TOKEN: ${{ github.token }}
|
|
97
|
+
run: |
|
|
98
|
+
set -euo pipefail
|
|
99
|
+
shopt -s nullglob
|
|
100
|
+
wheels=(dist/*.whl)
|
|
101
|
+
sdists=(dist/*.tar.gz)
|
|
102
|
+
files=(dist/* dist/.[!.]* dist/..?*)
|
|
103
|
+
if (( ${#wheels[@]} != 1 || ${#sdists[@]} != 1 || ${#files[@]} != 2 )); then
|
|
104
|
+
echo 'Expected exactly one wheel and one source archive, with no other distribution files.' >&2
|
|
105
|
+
exit 1
|
|
106
|
+
fi
|
|
107
|
+
for artifact in "${wheels[@]}" "${sdists[@]}"; do
|
|
108
|
+
if [[ ! -f "$artifact" || -L "$artifact" ]]; then
|
|
109
|
+
echo "Expected a regular distribution file: $artifact" >&2
|
|
110
|
+
exit 1
|
|
111
|
+
fi
|
|
112
|
+
verified=false
|
|
113
|
+
for attempt in 1 2 3; do
|
|
114
|
+
if gh attestation verify "$artifact" \
|
|
115
|
+
--repo "$GITHUB_REPOSITORY" \
|
|
116
|
+
--signer-workflow "$GITHUB_REPOSITORY/.github/workflows/release.yml" \
|
|
117
|
+
--source-digest "$GITHUB_SHA" \
|
|
118
|
+
--source-ref "$GITHUB_REF" \
|
|
119
|
+
--deny-self-hosted-runners; then
|
|
120
|
+
verified=true
|
|
121
|
+
break
|
|
122
|
+
fi
|
|
123
|
+
if (( attempt < 3 )); then
|
|
124
|
+
sleep 5
|
|
125
|
+
fi
|
|
126
|
+
done
|
|
127
|
+
if [[ "$verified" != true ]]; then
|
|
128
|
+
echo "Provenance verification failed after three attempts: $artifact" >&2
|
|
129
|
+
exit 1
|
|
130
|
+
fi
|
|
131
|
+
done
|
|
132
|
+
- name: Publish to PyPI through trusted publishing
|
|
133
|
+
run: uv publish
|
|
134
|
+
- name: Create GitHub release
|
|
135
|
+
run: |
|
|
136
|
+
set -euo pipefail
|
|
137
|
+
version="${GITHUB_REF_NAME#v}"
|
|
138
|
+
if [[ "$version" =~ (a|b|rc)[0-9]+$ ]]; then
|
|
139
|
+
dbt_version="$(python3 -c 'import re, sys; print(re.sub(r"^(\d+\.\d+\.\d+)((?:a|b|rc)\d+)$", r"\1-\2", sys.argv[1]))' "$version")"
|
|
140
|
+
gh release create "${GITHUB_REF_NAME}" dist/* --generate-notes --verify-tag \
|
|
141
|
+
--prerelease --title "dbt-dqm ${GITHUB_REF_NAME} (pre-release)" \
|
|
142
|
+
--notes "Pre-release: interfaces may change. Install the Python app with pip install 'dbt-dqm[bigquery]==${version}' or 'dbt-dqm[postgres]==${version}' and install the dbt package separately from Git revision ${GITHUB_REF_NAME} (its dbt_project.yml spells the version ${dbt_version}). Version 0.2.0a8 preserves a4-a7 history when resolved names are unchanged. Alias naming now applies to state tables too: changed namespaces require a confirmed fresh start, leaving legacy shared history untouched. Pass matching naming vars to run, app and workspace. The optional --summary reports OPENED (NEW), RECURRING and ARCHIVED changes applied by the reconciliation alongside NEW (status) and ACTIVE workload by saved owner. Quiet test results sort by status and name, and completion reports collections reaching their native limit. Export columns follow review order and timestamp zones are labeled. Upgrades from earlier alphas still require the migration steps documented in README.md. Scheduled full refreshes now require --yes. The app supports cross-page bulk edits and filtered CSV/Excel exports of saved issues. DQM is disabled during ordinary commands. Use dbt-dqm run for full capture and bounded native test limits (default 100). Ownership uses dqm_owner with owner metadata fallback. See README.md and CHANGELOG.md at this tag for supported versions and operating limits."
|
|
143
|
+
elif [[ "$GITHUB_REF_NAME" == "v0.2.0" ]]; then
|
|
144
|
+
gh release create "${GITHUB_REF_NAME}" dist/* --notes-file docs/release-notes-0.2.0.md --verify-tag
|
|
145
|
+
else
|
|
146
|
+
gh release create "${GITHUB_REF_NAME}" dist/* --generate-notes --verify-tag
|
|
147
|
+
fi
|
|
148
|
+
env:
|
|
149
|
+
GH_TOKEN: ${{ github.token }}
|
dbt_dqm-0.2.0/.gitignore
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
.venv/
|
|
2
|
+
target/
|
|
3
|
+
dbt_packages/
|
|
4
|
+
logs/
|
|
5
|
+
*.sqlite*
|
|
6
|
+
__pycache__/
|
|
7
|
+
.pytest_cache/
|
|
8
|
+
.ruff_cache/
|
|
9
|
+
.DS_Store
|
|
10
|
+
*.json
|
|
11
|
+
!integration_tests/demo_bigquery/seeds/*.csv
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
# Benchmark results referenced by docs/scale.md (synthetic data only).
|
|
15
|
+
!docs/scale-results/**/*.json
|
|
16
|
+
!docs/scale-results/*.json
|