dbt-dqm 0.2.0a1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (191) hide show
  1. dbt_dqm-0.2.0a1/.github/ISSUE_TEMPLATE/bug_report.yml +28 -0
  2. dbt_dqm-0.2.0a1/.github/ISSUE_TEMPLATE/feature_request.yml +19 -0
  3. dbt_dqm-0.2.0a1/.github/dependabot.yml +32 -0
  4. dbt_dqm-0.2.0a1/.github/pull_request_template.md +13 -0
  5. dbt_dqm-0.2.0a1/.github/workflows/bigquery-scheduled.yml +113 -0
  6. dbt_dqm-0.2.0a1/.github/workflows/ci.yml +224 -0
  7. dbt_dqm-0.2.0a1/.github/workflows/release.yml +149 -0
  8. dbt_dqm-0.2.0a1/.gitignore +16 -0
  9. dbt_dqm-0.2.0a1/CHANGELOG.md +184 -0
  10. dbt_dqm-0.2.0a1/CODE_OF_CONDUCT.md +58 -0
  11. dbt_dqm-0.2.0a1/CONTRIBUTING.md +12 -0
  12. dbt_dqm-0.2.0a1/LICENSE.md +190 -0
  13. dbt_dqm-0.2.0a1/PKG-INFO +304 -0
  14. dbt_dqm-0.2.0a1/README.md +270 -0
  15. dbt_dqm-0.2.0a1/SECURITY.md +30 -0
  16. dbt_dqm-0.2.0a1/dbt_project.yml +35 -0
  17. dbt_dqm-0.2.0a1/docs/design/warehouse-browsing.md +163 -0
  18. dbt_dqm-0.2.0a1/docs/functional-requirements.md +99 -0
  19. dbt_dqm-0.2.0a1/docs/implementation-0.2.md +152 -0
  20. dbt_dqm-0.2.0a1/docs/project-review.md +245 -0
  21. dbt_dqm-0.2.0a1/docs/qa-release-safeguards.md +118 -0
  22. dbt_dqm-0.2.0a1/docs/release-0.2.0a1.md +109 -0
  23. dbt_dqm-0.2.0a1/docs/release-checklist.md +84 -0
  24. dbt_dqm-0.2.0a1/docs/roadmap.md +59 -0
  25. dbt_dqm-0.2.0a1/docs/scale-results/allocator-ab-2026-10-10.json +202 -0
  26. dbt_dqm-0.2.0a1/docs/scale-results/bigquery-active-scope.json +67 -0
  27. dbt_dqm-0.2.0a1/docs/scale-results/bigquery-baseline.json +67 -0
  28. dbt_dqm-0.2.0a1/docs/scale-results/bigquery-observation-floor.json +67 -0
  29. dbt_dqm-0.2.0a1/docs/scale-results/browse-2026-10-11/bigquery-1m-chunks-0fd69f7.json +606 -0
  30. dbt_dqm-0.2.0a1/docs/scale-results/browse-2026-10-11/bigquery-1m-ef2e406.json +584 -0
  31. dbt_dqm-0.2.0a1/docs/scale-results/browse-2026-10-11/postgres-1m-ef2e406.json +387 -0
  32. dbt_dqm-0.2.0a1/docs/scale-results/browse-2026-10-11/postgres-plan-indexed-count_all.txt +23 -0
  33. dbt_dqm-0.2.0a1/docs/scale-results/browse-2026-10-11/postgres-plan-indexed-first_page.txt +16 -0
  34. dbt_dqm-0.2.0a1/docs/scale-results/browse-2026-10-11/postgres-plan-indexed-metrics.txt +25 -0
  35. dbt_dqm-0.2.0a1/docs/scale-results/browse-2026-10-11/postgres-plan-indexed-status_blocked.txt +27 -0
  36. dbt_dqm-0.2.0a1/docs/scale-results/browse-2026-10-11/postgres-plan-no-index-count_all.txt +23 -0
  37. dbt_dqm-0.2.0a1/docs/scale-results/browse-2026-10-11/postgres-plan-no-index-first_page.txt +27 -0
  38. dbt_dqm-0.2.0a1/docs/scale-results/browse-2026-10-11/postgres-plan-no-index-metrics.txt +25 -0
  39. dbt_dqm-0.2.0a1/docs/scale-results/browse-2026-10-11/postgres-plan-no-index-status_blocked.txt +27 -0
  40. dbt_dqm-0.2.0a1/docs/scale-results/cache-boundary-safeguards.json +10 -0
  41. dbt_dqm-0.2.0a1/docs/scale-results/candidate-613e089-cache-boundary.json +38 -0
  42. dbt_dqm-0.2.0a1/docs/scale-results/candidate-69703f7-cache-boundary.json +37 -0
  43. dbt_dqm-0.2.0a1/docs/scale-results/memory-interval-2026-10-10.json +463 -0
  44. dbt_dqm-0.2.0a1/docs/scale-results/pandas-3-2026-10-11.json +260 -0
  45. dbt_dqm-0.2.0a1/docs/scale-results/postgres-memory-1000000-payload8.json +29 -0
  46. dbt_dqm-0.2.0a1/docs/scale-results/postgres-memory-250000-payload8.json +29 -0
  47. dbt_dqm-0.2.0a1/docs/scale-results/postgres-memory-500000-payload8.json +29 -0
  48. dbt_dqm-0.2.0a1/docs/scale-results/postgres-planner/before-active-scope-postgres-1000000-payload8.json +41 -0
  49. dbt_dqm-0.2.0a1/docs/scale-results/postgres-planner/postgres-1000000-payload8.json +41 -0
  50. dbt_dqm-0.2.0a1/docs/scale-results/postgres-planner/postgres-20000-payload8.json +42 -0
  51. dbt_dqm-0.2.0a1/docs/scale-results/postgres-planner/postgres-200000-payload8.json +41 -0
  52. dbt_dqm-0.2.0a1/docs/scale-results/postgres-planner/postgres-plan-mass_pass-nestloop-off-change_set.txt +472 -0
  53. dbt_dqm-0.2.0a1/docs/scale-results/postgres-planner/postgres-plan-mass_pass-nestloop-on-change_set.txt +269 -0
  54. dbt_dqm-0.2.0a1/docs/scale-results/postgres-planner/postgres-plan-steady-nestloop-off-change_set.txt +449 -0
  55. dbt_dqm-0.2.0a1/docs/scale-results/postgres-planner/postgres-plan-steady-nestloop-on-change_set.txt +445 -0
  56. dbt_dqm-0.2.0a1/docs/scale-results/pyarrow-23-cache-boundary.json +43 -0
  57. dbt_dqm-0.2.0a1/docs/scale-results/pyarrow-25-2026-10-11.json +81 -0
  58. dbt_dqm-0.2.0a1/docs/scale.md +375 -0
  59. dbt_dqm-0.2.0a1/docs/scheduled-bigquery.md +69 -0
  60. dbt_dqm-0.2.0a1/docs/warehouse-interfaces.md +227 -0
  61. dbt_dqm-0.2.0a1/integration_tests/acceptance/bigquery_datasets.py +70 -0
  62. dbt_dqm-0.2.0a1/integration_tests/acceptance/browse_checks.py +91 -0
  63. dbt_dqm-0.2.0a1/integration_tests/acceptance/conftest.py +61 -0
  64. dbt_dqm-0.2.0a1/integration_tests/acceptance/test_bigquery.py +1202 -0
  65. dbt_dqm-0.2.0a1/integration_tests/acceptance/test_bigquery_permissions.py +185 -0
  66. dbt_dqm-0.2.0a1/integration_tests/acceptance/test_postgres.py +1475 -0
  67. dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/dbt_project.yml +15 -0
  68. dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/macros/acceptance_protocol.sql +90 -0
  69. dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/macros/generate_schema_name.sql +12 -0
  70. dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/macros/reset_demo_dqm.sql +42 -0
  71. dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/macros/schema.yml +19 -0
  72. dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/models/demo_records.sql +17 -0
  73. dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/models/schema.yml +66 -0
  74. dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/package-lock.yml +4 -0
  75. dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/packages.yml +3 -0
  76. dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/profiles.yml.example +11 -0
  77. dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/seeds/demo_cases.csv +13 -0
  78. dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/tests/assert_collection_guardrail_fixtures.sql +34 -0
  79. dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/tests/assert_scenario_outcomes.sql +85 -0
  80. dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/tests/demo_customer_email_invalid.sql +25 -0
  81. dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/tests/demo_invalid_duplicate_grain.sql +10 -0
  82. dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/tests/demo_invalid_empty_grain.sql +10 -0
  83. dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/tests/demo_invalid_fail_calc.sql +2 -0
  84. dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/tests/demo_invalid_limit.sql +2 -0
  85. dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/tests/demo_invalid_missing_column.sql +10 -0
  86. dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/tests/demo_invalid_non_string_grain.sql +10 -0
  87. dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/tests/demo_invalid_owner.sql +2 -0
  88. dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/tests/demo_invalid_priority.sql +2 -0
  89. dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/tests/demo_invalid_view_storage.sql +2 -0
  90. dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/tests/demo_order_amount_invalid.sql +24 -0
  91. dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/tests/demo_order_line_invalid.sql +30 -0
  92. dbt_dqm-0.2.0a1/integration_tests/demo_bigquery/tests/schema.yml +17 -0
  93. dbt_dqm-0.2.0a1/integration_tests/demo_postgres/dbt_project.yml +14 -0
  94. dbt_dqm-0.2.0a1/integration_tests/demo_postgres/macros/assert_postgres_query_plans.sql +21 -0
  95. dbt_dqm-0.2.0a1/integration_tests/demo_postgres/macros/reset_demo_dqm.sql +46 -0
  96. dbt_dqm-0.2.0a1/integration_tests/demo_postgres/macros/schema.yml +11 -0
  97. dbt_dqm-0.2.0a1/integration_tests/demo_postgres/models/demo_records.sql +17 -0
  98. dbt_dqm-0.2.0a1/integration_tests/demo_postgres/models/schema.yml +66 -0
  99. dbt_dqm-0.2.0a1/integration_tests/demo_postgres/package-lock.yml +4 -0
  100. dbt_dqm-0.2.0a1/integration_tests/demo_postgres/packages.yml +2 -0
  101. dbt_dqm-0.2.0a1/integration_tests/demo_postgres/profiles.yml.example +17 -0
  102. dbt_dqm-0.2.0a1/integration_tests/demo_postgres/seeds/demo_cases.csv +13 -0
  103. dbt_dqm-0.2.0a1/integration_tests/demo_postgres/tests/assert_collection_guardrail_fixtures.sql +34 -0
  104. dbt_dqm-0.2.0a1/integration_tests/demo_postgres/tests/assert_scenario_outcomes.sql +79 -0
  105. dbt_dqm-0.2.0a1/integration_tests/demo_postgres/tests/demo_customer_email_invalid.sql +25 -0
  106. dbt_dqm-0.2.0a1/integration_tests/demo_postgres/tests/demo_invalid_duplicate_grain.sql +10 -0
  107. dbt_dqm-0.2.0a1/integration_tests/demo_postgres/tests/demo_invalid_empty_grain.sql +10 -0
  108. dbt_dqm-0.2.0a1/integration_tests/demo_postgres/tests/demo_invalid_fail_calc.sql +2 -0
  109. dbt_dqm-0.2.0a1/integration_tests/demo_postgres/tests/demo_invalid_limit.sql +2 -0
  110. dbt_dqm-0.2.0a1/integration_tests/demo_postgres/tests/demo_invalid_missing_column.sql +10 -0
  111. dbt_dqm-0.2.0a1/integration_tests/demo_postgres/tests/demo_invalid_non_string_grain.sql +10 -0
  112. dbt_dqm-0.2.0a1/integration_tests/demo_postgres/tests/demo_invalid_owner.sql +2 -0
  113. dbt_dqm-0.2.0a1/integration_tests/demo_postgres/tests/demo_invalid_priority.sql +2 -0
  114. dbt_dqm-0.2.0a1/integration_tests/demo_postgres/tests/demo_invalid_view_storage.sql +2 -0
  115. dbt_dqm-0.2.0a1/integration_tests/demo_postgres/tests/demo_order_amount_invalid.sql +24 -0
  116. dbt_dqm-0.2.0a1/integration_tests/demo_postgres/tests/demo_order_line_invalid.sql +30 -0
  117. dbt_dqm-0.2.0a1/integration_tests/demo_postgres/tests/schema.yml +17 -0
  118. dbt_dqm-0.2.0a1/integration_tests/scale/app_memory.py +77 -0
  119. dbt_dqm-0.2.0a1/integration_tests/scale/bigquery_scale.py +316 -0
  120. dbt_dqm-0.2.0a1/integration_tests/scale/browse_probe.py +199 -0
  121. dbt_dqm-0.2.0a1/integration_tests/scale/cache_boundary.py +54 -0
  122. dbt_dqm-0.2.0a1/integration_tests/scale/postgres_scale.py +448 -0
  123. dbt_dqm-0.2.0a1/macros/adapters.sql +249 -0
  124. dbt_dqm-0.2.0a1/macros/capture_test_results.sql +421 -0
  125. dbt_dqm-0.2.0a1/macros/health.sql +48 -0
  126. dbt_dqm-0.2.0a1/macros/helpers.sql +70 -0
  127. dbt_dqm-0.2.0a1/macros/history.sql +15 -0
  128. dbt_dqm-0.2.0a1/macros/maintenance.sql +301 -0
  129. dbt_dqm-0.2.0a1/macros/materializations.sql +58 -0
  130. dbt_dqm-0.2.0a1/macros/reconciliation.sql +204 -0
  131. dbt_dqm-0.2.0a1/macros/recovery.sql +147 -0
  132. dbt_dqm-0.2.0a1/macros/replay.sql +409 -0
  133. dbt_dqm-0.2.0a1/macros/schema.yml +182 -0
  134. dbt_dqm-0.2.0a1/macros/schema_management.sql +404 -0
  135. dbt_dqm-0.2.0a1/macros/table_schemas.sql +156 -0
  136. dbt_dqm-0.2.0a1/models/dqm_all_issues.sql +3 -0
  137. dbt_dqm-0.2.0a1/models/dqm_annotation_changes.sql +25 -0
  138. dbt_dqm-0.2.0a1/models/dqm_area_health.sql +39 -0
  139. dbt_dqm-0.2.0a1/models/dqm_current_issues.sql +3 -0
  140. dbt_dqm-0.2.0a1/models/dqm_issue_timeline.sql +6 -0
  141. dbt_dqm-0.2.0a1/models/dqm_maintenance_health.sql +136 -0
  142. dbt_dqm-0.2.0a1/models/dqm_missed_issues.sql +6 -0
  143. dbt_dqm-0.2.0a1/models/dqm_reconcile.sql +4 -0
  144. dbt_dqm-0.2.0a1/models/dqm_test_health.sql +103 -0
  145. dbt_dqm-0.2.0a1/models/schema.yml +163 -0
  146. dbt_dqm-0.2.0a1/packages.yml +2 -0
  147. dbt_dqm-0.2.0a1/pyproject.toml +64 -0
  148. dbt_dqm-0.2.0a1/scripts/compile_bigquery.py +40 -0
  149. dbt_dqm-0.2.0a1/selectors.yml +7 -0
  150. dbt_dqm-0.2.0a1/src/dbt_dqm_app/__init__.py +3 -0
  151. dbt_dqm-0.2.0a1/src/dbt_dqm_app/browse.py +293 -0
  152. dbt_dqm-0.2.0a1/src/dbt_dqm_app/cli.py +127 -0
  153. dbt_dqm-0.2.0a1/src/dbt_dqm_app/config.py +146 -0
  154. dbt_dqm-0.2.0a1/src/dbt_dqm_app/display.py +241 -0
  155. dbt_dqm-0.2.0a1/src/dbt_dqm_app/errors.py +107 -0
  156. dbt_dqm-0.2.0a1/src/dbt_dqm_app/frames.py +19 -0
  157. dbt_dqm-0.2.0a1/src/dbt_dqm_app/limits.py +34 -0
  158. dbt_dqm-0.2.0a1/src/dbt_dqm_app/store.py +476 -0
  159. dbt_dqm-0.2.0a1/src/dbt_dqm_app/streamlit_app.py +1051 -0
  160. dbt_dqm-0.2.0a1/src/dbt_dqm_app/warehouse.py +852 -0
  161. dbt_dqm-0.2.0a1/tests/assert_at_most_one_active_occurrence.sql +10 -0
  162. dbt_dqm-0.2.0a1/tests/assert_collection_evidence_guardrails.sql +13 -0
  163. dbt_dqm-0.2.0a1/tests/assert_event_id_unique.sql +4 -0
  164. dbt_dqm-0.2.0a1/tests/assert_identity_hash_vectors.sql +49 -0
  165. dbt_dqm-0.2.0a1/tests/assert_occurrence_id_unique.sql +4 -0
  166. dbt_dqm-0.2.0a1/tests/assert_receipt_unique.sql +4 -0
  167. dbt_dqm-0.2.0a1/tests/test_acceptance_sweep.py +58 -0
  168. dbt_dqm-0.2.0a1/tests/test_annotation_staging.py +81 -0
  169. dbt_dqm-0.2.0a1/tests/test_browse.py +314 -0
  170. dbt_dqm-0.2.0a1/tests/test_browse_warehouse.py +180 -0
  171. dbt_dqm-0.2.0a1/tests/test_cache_limits.py +190 -0
  172. dbt_dqm-0.2.0a1/tests/test_cli.py +56 -0
  173. dbt_dqm-0.2.0a1/tests/test_config.py +98 -0
  174. dbt_dqm-0.2.0a1/tests/test_credentials.py +114 -0
  175. dbt_dqm-0.2.0a1/tests/test_dbt_hints.py +32 -0
  176. dbt_dqm-0.2.0a1/tests/test_dependency_floors.py +57 -0
  177. dbt_dqm-0.2.0a1/tests/test_docs_consistency.py +66 -0
  178. dbt_dqm-0.2.0a1/tests/test_features.py +188 -0
  179. dbt_dqm-0.2.0a1/tests/test_frames.py +18 -0
  180. dbt_dqm-0.2.0a1/tests/test_maintenance_health.py +93 -0
  181. dbt_dqm-0.2.0a1/tests/test_missing_values.py +47 -0
  182. dbt_dqm-0.2.0a1/tests/test_oauthlib_compat.py +89 -0
  183. dbt_dqm-0.2.0a1/tests/test_protocol_render.py +292 -0
  184. dbt_dqm-0.2.0a1/tests/test_release_workflow.py +255 -0
  185. dbt_dqm-0.2.0a1/tests/test_scheduled_workflow.py +32 -0
  186. dbt_dqm-0.2.0a1/tests/test_store.py +310 -0
  187. dbt_dqm-0.2.0a1/tests/test_sync_and_errors.py +153 -0
  188. dbt_dqm-0.2.0a1/tests/test_ui.py +135 -0
  189. dbt_dqm-0.2.0a1/tests/test_ui_browse.py +269 -0
  190. dbt_dqm-0.2.0a1/tests/test_wif.py +145 -0
  191. dbt_dqm-0.2.0a1/uv.lock +1900 -0
@@ -0,0 +1,28 @@
1
+ name: Bug report
2
+ description: Report incorrect collection, lifecycle, adapter, or app behavior
3
+ body:
4
+ - type: textarea
5
+ id: behavior
6
+ attributes:
7
+ label: Observed behavior
8
+ validations:
9
+ required: true
10
+ - type: textarea
11
+ id: reproduction
12
+ attributes:
13
+ label: Minimal reproduction
14
+ validations:
15
+ required: true
16
+ - type: input
17
+ id: versions
18
+ attributes:
19
+ label: dbt-dqm, dbt, adapter, warehouse, and Python versions
20
+ validations:
21
+ required: true
22
+ - type: checkboxes
23
+ id: safety
24
+ attributes:
25
+ label: Data safety
26
+ options:
27
+ - label: I removed credentials, customer data, and complete failed-row contents.
28
+ required: true
@@ -0,0 +1,19 @@
1
+ name: Feature request
2
+ description: Propose a focused open-source improvement
3
+ body:
4
+ - type: textarea
5
+ id: problem
6
+ attributes:
7
+ label: User problem
8
+ validations:
9
+ required: true
10
+ - type: textarea
11
+ id: proposal
12
+ attributes:
13
+ label: Proposed behavior
14
+ validations:
15
+ required: true
16
+ - type: textarea
17
+ id: adapters
18
+ attributes:
19
+ label: Adapter and compatibility implications
@@ -0,0 +1,32 @@
1
+ version: 2
2
+ updates:
3
+ # Read uv.lock so version-type exclusions also cover range-only requirements.
4
+ - package-ecosystem: uv
5
+ directory: /
6
+ schedule:
7
+ interval: weekly
8
+ # Deferred compatibility work: https://github.com/a-adamovich/dbt-dqm/issues/11
9
+ # Update-type exclusions preserve patch updates and security-update handling.
10
+ ignore:
11
+ # dbt minors are adopted deliberately: the protocol SQL must render identically and the full
12
+ # warehouse suites must pass before the pins and require-dbt-version move (see #11).
13
+ - dependency-name: dbt-core
14
+ update-types: ["version-update:semver-minor"]
15
+ - dependency-name: dbt-bigquery
16
+ update-types: ["version-update:semver-minor"]
17
+ - dependency-name: dbt-postgres
18
+ update-types: ["version-update:semver-minor"]
19
+ - dependency-name: pandas
20
+ update-types: ["version-update:semver-major"]
21
+ # pyarrow majors are adopted deliberately, with the app memory benchmark rerun at the
22
+ # 50k cache boundary (pinned to 25.x; see #11 and docs/scale.md).
23
+ - dependency-name: pyarrow
24
+ update-types: ["version-update:semver-major"]
25
+ groups:
26
+ python-minor-patches:
27
+ patterns: ["*"]
28
+ update-types: [minor, patch]
29
+ - package-ecosystem: github-actions
30
+ directory: /
31
+ schedule:
32
+ interval: weekly
@@ -0,0 +1,13 @@
1
+ ## Summary
2
+
3
+ Describe the focused change and its user-visible behavior.
4
+
5
+ ## Verification
6
+
7
+ - [ ] `ruff check .`
8
+ - [ ] `pytest -q`
9
+ - [ ] Relevant dbt adapter/demo lifecycle checks
10
+ - [ ] Documentation and changelog updated where needed
11
+ - [ ] Commits include DCO sign-off (`git commit -s`)
12
+
13
+ Do not include credentials, proprietary data, customer names, or complete failed-row contents.
@@ -0,0 +1,113 @@
1
+ name: Scheduled BigQuery lifecycle
2
+
3
+ # Weekly credentialed BigQuery verification. Keyless: GitHub's OIDC token is exchanged through
4
+ # Workload Identity Federation for a short-lived token of a dedicated CI service account, so no
5
+ # long-lived key is stored. Inert until the repository variables below are set.
6
+ on:
7
+ workflow_dispatch:
8
+ schedule:
9
+ - cron: "17 6 * * 1"
10
+
11
+ permissions:
12
+ contents: read
13
+ id-token: write
14
+ issues: write
15
+
16
+ jobs:
17
+ lifecycle:
18
+ concurrency:
19
+ group: dbt-dqm-${{ vars.DBT_DQM_BIGQUERY_PROJECT }}-dbt_dqm_scheduled_ci
20
+ cancel-in-progress: false
21
+ if: vars.BIGQUERY_LIFECYCLE_ENABLED == 'true' && github.ref == 'refs/heads/main'
22
+ runs-on: ubuntu-latest
23
+ env:
24
+ DBT_DQM_BIGQUERY_PROJECT: ${{ vars.DBT_DQM_BIGQUERY_PROJECT }}
25
+ DBT_DQM_BIGQUERY_TEST_PROJECT: ${{ vars.DBT_DQM_BIGQUERY_PROJECT }}
26
+ steps:
27
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
28
+ - name: Authenticate to Google Cloud (Workload Identity Federation)
29
+ uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0
30
+ with:
31
+ workload_identity_provider: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }}
32
+ service_account: ${{ vars.GCP_CI_SERVICE_ACCOUNT }}
33
+ - name: Install uv
34
+ uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
35
+ with:
36
+ python-version: "3.11"
37
+ - name: Install dependencies
38
+ run: uv sync --locked --all-groups
39
+ - name: Write profile (Application Default Credentials from the step above)
40
+ run: |
41
+ mkdir -p ~/.dbt
42
+ cat > ~/.dbt/profiles.yml <<'PROFILES'
43
+ dbt_dqm_demo:
44
+ target: ci
45
+ outputs:
46
+ ci:
47
+ type: bigquery
48
+ method: oauth
49
+ project: "{{ env_var('DBT_DQM_BIGQUERY_PROJECT') }}"
50
+ dataset: dbt_dqm_scheduled_ci
51
+ location: US
52
+ threads: 4
53
+ PROFILES
54
+ - name: The CI identity cannot read the shared QA dataset
55
+ run: |
56
+ uv run python - <<'PY'
57
+ import os
58
+ from google.api_core.exceptions import Forbidden, NotFound
59
+ from google.cloud import bigquery
60
+ project = os.environ["DBT_DQM_BIGQUERY_PROJECT"]
61
+ try:
62
+ bigquery.Client(project=project).query(
63
+ f"select 1 from `{project}.dbt_dqm_qa.dqm_install` limit 1").result()
64
+ except (Forbidden, NotFound):
65
+ print("As expected, the CI service account has no access to dbt_dqm_qa.")
66
+ else:
67
+ raise SystemExit("The CI service account can read dbt_dqm_qa; narrow its roles.")
68
+ PY
69
+ - name: Full credentialed lifecycle (dedicated dataset)
70
+ working-directory: integration_tests/demo_bigquery
71
+ run: |
72
+ set -e
73
+ uv run dbt deps
74
+ uv run dbt run-operation reset_demo_dqm
75
+ uv run dbt seed
76
+ for scenario in initial passed recurrence; do
77
+ uv run dbt run --select demo_records --vars "{demo_scenario: $scenario}"
78
+ uv run dbt test --select tag:dqm --vars "{demo_scenario: $scenario}"
79
+ uv run dbt test --select assert_collection_guardrail_fixtures
80
+ uv run dbt build --select package:dbt_dqm --vars "{demo_scenario: $scenario}"
81
+ uv run dbt test --select assert_scenario_outcomes --vars "{demo_scenario: $scenario}"
82
+ done
83
+ - name: Credentialed acceptance suite (parallel, disposable datasets)
84
+ shell: bash
85
+ run: |
86
+ mkdir -p acceptance-results
87
+ set -o pipefail
88
+ uv run pytest -q -rA -n 6 --dist load integration_tests/acceptance/test_bigquery.py \
89
+ --junitxml=acceptance-results/bigquery.xml 2>&1 | tee acceptance-results/bigquery.log
90
+ - name: Sweep stale acceptance datasets
91
+ if: always()
92
+ run: uv run python integration_tests/acceptance/bigquery_datasets.py --project "$DBT_DQM_BIGQUERY_PROJECT" --older-than-hours 6
93
+ - name: Upload results
94
+ if: always()
95
+ uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
96
+ with:
97
+ name: scheduled-bigquery-results
98
+ path: acceptance-results/
99
+ if-no-files-found: ignore
100
+ - name: Report failure in an issue
101
+ if: failure()
102
+ env:
103
+ GH_TOKEN: ${{ github.token }}
104
+ RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
105
+ run: |
106
+ gh label create scheduled-bigquery --color B60205 --description "Scheduled BigQuery verification" 2>/dev/null || true
107
+ existing="$(gh issue list --label scheduled-bigquery --state open --json number --jq '.[0].number')"
108
+ if [ -n "$existing" ]; then
109
+ gh issue comment "$existing" --body "Scheduled BigQuery verification failed again: $RUN_URL"
110
+ else
111
+ gh issue create --label scheduled-bigquery --title "Scheduled BigQuery verification failed" \
112
+ --body "The weekly credentialed BigQuery run failed on \`main\`: $RUN_URL"
113
+ fi
@@ -0,0 +1,224 @@
1
+ name: CI
2
+
3
+ permissions:
4
+ contents: read
5
+
6
+ on:
7
+ push:
8
+ branches: [main]
9
+ pull_request:
10
+
11
+ jobs:
12
+ lint-and-test:
13
+ runs-on: ubuntu-latest
14
+ steps:
15
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
16
+
17
+ - name: Install uv
18
+ uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
19
+ with:
20
+ python-version: "3.11"
21
+
22
+ - name: Install dependencies
23
+ run: uv sync --all-groups
24
+
25
+ - name: Ruff
26
+ run: uv run ruff check .
27
+
28
+ - name: Pytest
29
+ run: uv run pytest -q
30
+
31
+ - name: Streamlit startup smoke test
32
+ run: |
33
+ mkdir -p /tmp/dbt-dqm-smoke-project /tmp/dbt-dqm-smoke-profiles
34
+ printf 'name: smoke\nprofile: smoke\n' > /tmp/dbt-dqm-smoke-project/dbt_project.yml
35
+ printf 'smoke:\n outputs:\n dev:\n type: postgres\n host: localhost\n user: smoke\n pass: smoke\n dbname: smoke\n schema: smoke\n' > /tmp/dbt-dqm-smoke-profiles/profiles.yml
36
+ timeout 20 uv run dbt-dqm app --project-dir /tmp/dbt-dqm-smoke-project --profiles-dir /tmp/dbt-dqm-smoke-profiles --target dev --port 8765 &
37
+ for attempt in 1 2 3 4 5 6 7 8 9 10; do
38
+ if curl --fail --silent http://127.0.0.1:8765/_stcore/health; then exit 0; fi
39
+ sleep 1
40
+ done
41
+ exit 1
42
+
43
+ # Validates the dbt package (models/, macros/) end to end against a real warehouse without
44
+ # needing any secrets: a Postgres service container plus the synthetic demo project. There is
45
+ # no equivalent BigQuery job here since that would require committing or configuring warehouse
46
+ # credentials as repo secrets; add one separately (e.g. gated on a repo secret's presence) if
47
+ # BigQuery-specific coverage in CI is wanted later.
48
+ package-smoke:
49
+ runs-on: ubuntu-latest
50
+ steps:
51
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
52
+ - name: Install uv
53
+ uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
54
+ with:
55
+ python-version: "3.11"
56
+ - name: Build wheel and source distribution
57
+ run: uv build
58
+ - name: Check distribution metadata
59
+ run: uvx --from twine twine check dist/*
60
+ - name: Install the wheel into a clean environment and smoke-test the CLI
61
+ run: |
62
+ uv venv /tmp/dbt-dqm-wheel-smoke --python 3.11
63
+ uv pip install --python /tmp/dbt-dqm-wheel-smoke/bin/python dist/*.whl
64
+ /tmp/dbt-dqm-wheel-smoke/bin/dbt-dqm --help
65
+ - name: Wheel carries the security floors (#24)
66
+ run: DBT_DQM_WHEEL="$(ls dist/*.whl)" uv run pytest -q tests/test_dependency_floors.py
67
+ - name: Installing the wheel upgrades older vulnerable versions
68
+ run: |
69
+ wheel="$(pwd)/$(ls dist/*.whl)"
70
+ uv venv /tmp/dbt-dqm-old-deps --python 3.11
71
+ uv pip install --python /tmp/dbt-dqm-old-deps/bin/python sqlparse==0.5.5 urllib3==2.7.0 oauthlib==3.3.1
72
+ cd /tmp
73
+ uv pip install --python /tmp/dbt-dqm-old-deps/bin/python "$wheel"
74
+ uv pip check --python /tmp/dbt-dqm-old-deps/bin/python
75
+ /tmp/dbt-dqm-old-deps/bin/python - <<'PY'
76
+ from importlib.metadata import version
77
+ from packaging.version import Version
78
+ floors = {"sqlparse": "0.6.0", "urllib3": "2.8.0", "oauthlib": "4.0.0"}
79
+ installed = {name: version(name) for name in floors}
80
+ print(installed)
81
+ assert all(Version(installed[n]) >= Version(v) for n, v in floors.items()), installed
82
+ PY
83
+
84
+ dbt-postgres:
85
+ runs-on: ubuntu-latest
86
+ strategy:
87
+ fail-fast: false
88
+ matrix:
89
+ postgres: ["14", "16"]
90
+ services:
91
+ postgres:
92
+ image: postgres:${{ matrix.postgres }}
93
+ env:
94
+ POSTGRES_PASSWORD: dbtdqm
95
+ POSTGRES_DB: dbt_dqm_ci
96
+ ports:
97
+ - 5432:5432
98
+ options: >-
99
+ --health-cmd pg_isready
100
+ --health-interval 10s
101
+ --health-timeout 5s
102
+ --health-retries 5
103
+ steps:
104
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
105
+
106
+ - name: Install uv
107
+ uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
108
+ with:
109
+ python-version: "3.11"
110
+
111
+ - name: Install dependencies
112
+ run: uv sync --all-groups
113
+
114
+ - name: Enable pgcrypto
115
+ run: psql -h localhost -U postgres -d dbt_dqm_ci -c "create extension if not exists pgcrypto;"
116
+ env:
117
+ PGPASSWORD: dbtdqm
118
+
119
+ - name: Write profiles.yml
120
+ run: |
121
+ mkdir -p ~/.dbt
122
+ cat > ~/.dbt/profiles.yml <<'PROFILES'
123
+ dbt_dqm_demo_postgres:
124
+ target: ci
125
+ outputs:
126
+ ci:
127
+ type: postgres
128
+ host: localhost
129
+ port: 5432
130
+ user: postgres
131
+ pass: dbtdqm
132
+ dbname: dbt_dqm_ci
133
+ schema: dbt_dqm_demo
134
+ threads: 4
135
+ PROFILES
136
+
137
+ - name: dbt deps
138
+ working-directory: integration_tests/demo_postgres
139
+ run: uv run dbt deps
140
+
141
+ - name: Full lifecycle walkthrough (initial -> passed -> recurrence)
142
+ working-directory: integration_tests/demo_postgres
143
+ run: |
144
+ set -e
145
+ uv run dbt run-operation reset_demo_dqm
146
+ uv run dbt seed
147
+ for scenario in initial passed recurrence; do
148
+ uv run dbt run --select demo_records --vars "{demo_scenario: $scenario}"
149
+ uv run dbt test --select tag:dqm --vars "{demo_scenario: $scenario}"
150
+ uv run dbt test --select assert_collection_guardrail_fixtures
151
+ uv run dbt build --select package:dbt_dqm --vars "{demo_scenario: $scenario}"
152
+ uv run dbt test --select assert_scenario_outcomes --vars "{demo_scenario: $scenario}"
153
+ done
154
+
155
+ - name: Replay multiple executions between reconciliations
156
+ working-directory: integration_tests/demo_postgres
157
+ run: |
158
+ set -e
159
+ uv run dbt run-operation reset_demo_dqm
160
+ uv run dbt run --select demo_records --vars '{demo_scenario: initial}'
161
+ uv run dbt test --select tag:dqm --vars '{demo_scenario: initial}'
162
+ uv run dbt run --select demo_records --vars '{demo_scenario: passed}'
163
+ uv run dbt test --select tag:dqm --vars '{demo_scenario: passed}'
164
+ uv run dbt build --select package:dbt_dqm --vars '{demo_scenario: passed}'
165
+ uv run dbt test --select assert_scenario_outcomes --vars '{demo_scenario: passed}'
166
+ uv run dbt run-operation reset_demo_dqm
167
+ for scenario in initial passed recurrence; do
168
+ uv run dbt run --select demo_records --vars "{demo_scenario: $scenario}"
169
+ uv run dbt test --select tag:dqm --vars "{demo_scenario: $scenario}"
170
+ done
171
+ uv run dbt build --select package:dbt_dqm --vars '{demo_scenario: recurrence}'
172
+ uv run dbt test --select assert_scenario_outcomes --vars '{demo_scenario: recurrence}'
173
+ uv run dbt run-operation assert_postgres_query_plans
174
+
175
+ - name: Transactional acceptance and application interfaces
176
+ shell: bash
177
+ run: |
178
+ set -o pipefail
179
+ mkdir -p acceptance-results
180
+ uv run pytest -q -rA integration_tests/acceptance/test_postgres.py \
181
+ --junitxml=acceptance-results/junit.xml 2>&1 | tee acceptance-results/pytest.log
182
+ env:
183
+ DBT_DQM_TEST_DSN: host=localhost port=5432 dbname=dbt_dqm_ci user=postgres password=dbtdqm
184
+
185
+ # Annotations only report the exit code; keep the full pytest output for diagnosis.
186
+ - name: Upload acceptance results
187
+ if: ${{ !cancelled() }}
188
+ uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
189
+ with:
190
+ name: postgres-${{ matrix.postgres }}-acceptance
191
+ path: acceptance-results/
192
+ if-no-files-found: ignore
193
+
194
+ bigquery-compile:
195
+ runs-on: ubuntu-latest
196
+ steps:
197
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
198
+ - name: Install uv
199
+ uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
200
+ with:
201
+ python-version: "3.11"
202
+ - name: Install dependencies
203
+ run: uv sync --all-groups
204
+ - name: Write compile-only profile
205
+ run: |
206
+ mkdir -p ~/.dbt
207
+ cat > ~/.dbt/profiles.yml <<'PROFILES'
208
+ dbt_dqm_demo:
209
+ target: ci
210
+ outputs:
211
+ ci:
212
+ type: bigquery
213
+ method: oauth
214
+ project: dbt-dqm-compile-only
215
+ dataset: dbt_dqm_demo
216
+ location: US
217
+ threads: 1
218
+ PROFILES
219
+ - name: Parse and compile without warehouse introspection
220
+ working-directory: integration_tests/demo_bigquery
221
+ run: |
222
+ uv run dbt deps
223
+ uv run dbt parse
224
+ uv run python ../../scripts/compile_bigquery.py
@@ -0,0 +1,149 @@
1
+ name: Release
2
+
3
+ on:
4
+ push:
5
+ tags: ["v*"]
6
+
7
+ permissions:
8
+ contents: write
9
+ id-token: write
10
+ attestations: write
11
+
12
+ jobs:
13
+ release:
14
+ runs-on: ubuntu-latest
15
+ environment: pypi
16
+ services:
17
+ postgres:
18
+ image: postgres:16
19
+ env:
20
+ POSTGRES_PASSWORD: dbtdqm
21
+ POSTGRES_DB: dbt_dqm_release
22
+ ports:
23
+ - 5432:5432
24
+ options: >-
25
+ --health-cmd pg_isready
26
+ --health-interval 10s
27
+ --health-timeout 5s
28
+ --health-retries 5
29
+ steps:
30
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
31
+ - name: Check the tag matches the package versions
32
+ shell: bash
33
+ run: |
34
+ set -euo pipefail
35
+ python_version="$(python3 -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')"
36
+ dbt_version="$(python3 -c 'import re; print(re.search(r"^version:\s*[\x27\x22]?([^\x27\x22\s]+)", open("dbt_project.yml").read(), re.M).group(1))')"
37
+ # dbt only accepts SemVer, so a Python pre-release 0.2.0a1 is spelled 0.2.0-a1 there.
38
+ expected_dbt_version="$(python3 -c 'import re, sys; print(re.sub(r"^(\d+\.\d+\.\d+)((?:a|b|rc)\d+)$", r"\1-\2", sys.argv[1]))' "$python_version")"
39
+ if [[ "$dbt_version" != "$expected_dbt_version" ]]; then
40
+ echo "pyproject.toml ($python_version) and dbt_project.yml ($dbt_version) versions differ; expected dbt version $expected_dbt_version." >&2
41
+ exit 1
42
+ fi
43
+ if [[ "$GITHUB_REF_NAME" != "v$python_version" ]]; then
44
+ echo "Tag $GITHUB_REF_NAME doesn't match package version v$python_version." >&2
45
+ exit 1
46
+ fi
47
+ - name: Install uv
48
+ uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
49
+ with:
50
+ python-version: "3.11"
51
+ - name: Verify, build, and inspect distributions
52
+ run: |
53
+ uv sync --locked --all-groups
54
+ uv run ruff check .
55
+ uv run pytest -q
56
+ uv build --no-create-gitignore
57
+ uvx --from twine twine check dist/*
58
+ - name: Configure release-gate Postgres
59
+ run: |
60
+ psql -h localhost -U postgres -d dbt_dqm_release -c "create extension if not exists pgcrypto;"
61
+ mkdir -p ~/.dbt
62
+ cat > ~/.dbt/profiles.yml <<'PROFILES'
63
+ dbt_dqm_demo_postgres:
64
+ target: release
65
+ outputs:
66
+ release:
67
+ type: postgres
68
+ host: localhost
69
+ port: 5432
70
+ user: postgres
71
+ pass: dbtdqm
72
+ dbname: dbt_dqm_release
73
+ schema: dbt_dqm_demo
74
+ threads: 4
75
+ PROFILES
76
+ env:
77
+ PGPASSWORD: dbtdqm
78
+ - name: Full Postgres release lifecycle
79
+ working-directory: integration_tests/demo_postgres
80
+ run: |
81
+ uv run dbt deps
82
+ uv run dbt run-operation reset_demo_dqm
83
+ uv run dbt seed
84
+ for scenario in initial passed recurrence; do
85
+ uv run dbt run --select demo_records --vars "{demo_scenario: $scenario}"
86
+ uv run dbt test --select tag:dqm --vars "{demo_scenario: $scenario}"
87
+ uv run dbt test --select assert_collection_guardrail_fixtures
88
+ uv run dbt build --select package:dbt_dqm --vars "{demo_scenario: $scenario}"
89
+ uv run dbt test --select assert_scenario_outcomes --vars "{demo_scenario: $scenario}"
90
+ done
91
+ - name: Attest distribution provenance
92
+ uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2
93
+ with:
94
+ subject-path: dist/*
95
+ - name: Verify distribution provenance
96
+ shell: bash
97
+ env:
98
+ GH_TOKEN: ${{ github.token }}
99
+ run: |
100
+ set -euo pipefail
101
+ shopt -s nullglob
102
+ wheels=(dist/*.whl)
103
+ sdists=(dist/*.tar.gz)
104
+ files=(dist/* dist/.[!.]* dist/..?*)
105
+ if (( ${#wheels[@]} != 1 || ${#sdists[@]} != 1 || ${#files[@]} != 2 )); then
106
+ echo 'Expected exactly one wheel and one source archive, with no other distribution files.' >&2
107
+ exit 1
108
+ fi
109
+ for artifact in "${wheels[@]}" "${sdists[@]}"; do
110
+ if [[ ! -f "$artifact" || -L "$artifact" ]]; then
111
+ echo "Expected a regular distribution file: $artifact" >&2
112
+ exit 1
113
+ fi
114
+ verified=false
115
+ for attempt in 1 2 3; do
116
+ if gh attestation verify "$artifact" \
117
+ --repo "$GITHUB_REPOSITORY" \
118
+ --signer-workflow "$GITHUB_REPOSITORY/.github/workflows/release.yml" \
119
+ --source-digest "$GITHUB_SHA" \
120
+ --source-ref "$GITHUB_REF" \
121
+ --deny-self-hosted-runners; then
122
+ verified=true
123
+ break
124
+ fi
125
+ if (( attempt < 3 )); then
126
+ sleep 5
127
+ fi
128
+ done
129
+ if [[ "$verified" != true ]]; then
130
+ echo "Provenance verification failed after three attempts: $artifact" >&2
131
+ exit 1
132
+ fi
133
+ done
134
+ - name: Publish to PyPI through trusted publishing
135
+ run: uv publish
136
+ - name: Create GitHub release
137
+ run: |
138
+ set -euo pipefail
139
+ version="${GITHUB_REF_NAME#v}"
140
+ if [[ "$version" =~ (a|b|rc)[0-9]+$ ]]; then
141
+ dbt_version="$(python3 -c 'import re, sys; print(re.sub(r"^(\d+\.\d+\.\d+)((?:a|b|rc)\d+)$", r"\1-\2", sys.argv[1]))' "$version")"
142
+ gh release create "${GITHUB_REF_NAME}" dist/* --generate-notes --verify-tag \
143
+ --prerelease --title "dbt-dqm ${GITHUB_REF_NAME} (pre-release)" \
144
+ --notes "Pre-release: interfaces may change. Install the Python app with pip install dbt-dqm==${version} and install the dbt package separately from Git revision ${GITHUB_REF_NAME} (its dbt_project.yml spells the version ${dbt_version}). Upgrades from 0.1 require a fresh DQM schema. See README.md and CHANGELOG.md at this tag for supported versions and operating limits."
145
+ else
146
+ gh release create "${GITHUB_REF_NAME}" dist/* --generate-notes --verify-tag
147
+ fi
148
+ env:
149
+ GH_TOKEN: ${{ github.token }}
@@ -0,0 +1,16 @@
1
+ .venv/
2
+ target/
3
+ dbt_packages/
4
+ logs/
5
+ *.sqlite*
6
+ __pycache__/
7
+ .pytest_cache/
8
+ .ruff_cache/
9
+ .DS_Store
10
+ *.json
11
+ !integration_tests/demo_bigquery/seeds/*.csv
12
+
13
+
14
+ # Benchmark results referenced by docs/scale.md (synthetic data only).
15
+ !docs/scale-results/**/*.json
16
+ !docs/scale-results/*.json