culvert 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (135) hide show
  1. culvert-0.1.0/.gitignore +131 -0
  2. culvert-0.1.0/PKG-INFO +112 -0
  3. culvert-0.1.0/README.md +56 -0
  4. culvert-0.1.0/hatch_build.py +48 -0
  5. culvert-0.1.0/pyproject.toml +118 -0
  6. culvert-0.1.0/src/culvert/__init__.py +27 -0
  7. culvert-0.1.0/src/data_pipeline_contract_tests/__init__.py +28 -0
  8. culvert-0.1.0/src/data_pipeline_contract_tests/blob_store.py +34 -0
  9. culvert-0.1.0/src/data_pipeline_contract_tests/secrets.py +41 -0
  10. culvert-0.1.0/src/data_pipeline_contract_tests/stage_metrics_hook.py +109 -0
  11. culvert-0.1.0/src/data_pipeline_contract_tests/warehouse.py +36 -0
  12. culvert-0.1.0/src/data_pipeline_core/__init__.py +60 -0
  13. culvert-0.1.0/src/data_pipeline_core/audit/__init__.py +5 -0
  14. culvert-0.1.0/src/data_pipeline_core/audit/records.py +34 -0
  15. culvert-0.1.0/src/data_pipeline_core/autoconfig.py +144 -0
  16. culvert-0.1.0/src/data_pipeline_core/contracts/__init__.py +41 -0
  17. culvert-0.1.0/src/data_pipeline_core/contracts/audit.py +37 -0
  18. culvert-0.1.0/src/data_pipeline_core/contracts/blob_store.py +81 -0
  19. culvert-0.1.0/src/data_pipeline_core/contracts/finops.py +33 -0
  20. culvert-0.1.0/src/data_pipeline_core/contracts/governance.py +46 -0
  21. culvert-0.1.0/src/data_pipeline_core/contracts/job_control.py +125 -0
  22. culvert-0.1.0/src/data_pipeline_core/contracts/lineage.py +28 -0
  23. culvert-0.1.0/src/data_pipeline_core/contracts/observability.py +78 -0
  24. culvert-0.1.0/src/data_pipeline_core/contracts/pipeline.py +46 -0
  25. culvert-0.1.0/src/data_pipeline_core/contracts/runtime.py +97 -0
  26. culvert-0.1.0/src/data_pipeline_core/contracts/secrets.py +31 -0
  27. culvert-0.1.0/src/data_pipeline_core/contracts/source.py +60 -0
  28. culvert-0.1.0/src/data_pipeline_core/contracts/stage_metrics.py +100 -0
  29. culvert-0.1.0/src/data_pipeline_core/contracts/warehouse.py +94 -0
  30. culvert-0.1.0/src/data_pipeline_core/dataquality/__init__.py +44 -0
  31. culvert-0.1.0/src/data_pipeline_core/dataquality/data_quality_transform.py +368 -0
  32. culvert-0.1.0/src/data_pipeline_core/dataquality/field_violation.py +48 -0
  33. culvert-0.1.0/src/data_pipeline_core/dataquality/numeric_range.py +54 -0
  34. culvert-0.1.0/src/data_pipeline_core/dataquality/validation_result.py +90 -0
  35. culvert-0.1.0/src/data_pipeline_core/dataquality/violation_kind.py +35 -0
  36. culvert-0.1.0/src/data_pipeline_core/decorators.py +77 -0
  37. culvert-0.1.0/src/data_pipeline_core/finops_api/__init__.py +27 -0
  38. culvert-0.1.0/src/data_pipeline_core/finops_api/budget.py +235 -0
  39. culvert-0.1.0/src/data_pipeline_core/finops_api/labels.py +30 -0
  40. culvert-0.1.0/src/data_pipeline_core/finops_api/models.py +40 -0
  41. culvert-0.1.0/src/data_pipeline_core/governance_api/__init__.py +21 -0
  42. culvert-0.1.0/src/data_pipeline_core/governance_api/classification.py +18 -0
  43. culvert-0.1.0/src/data_pipeline_core/governance_api/masker.py +102 -0
  44. culvert-0.1.0/src/data_pipeline_core/governance_api/pii_masking_governance_policy.py +173 -0
  45. culvert-0.1.0/src/data_pipeline_core/governance_api/policies.py +56 -0
  46. culvert-0.1.0/src/data_pipeline_core/job_control_api/__init__.py +26 -0
  47. culvert-0.1.0/src/data_pipeline_core/job_control_api/models.py +106 -0
  48. culvert-0.1.0/src/data_pipeline_core/job_control_api/types.py +44 -0
  49. culvert-0.1.0/src/data_pipeline_core/lineage/__init__.py +5 -0
  50. culvert-0.1.0/src/data_pipeline_core/lineage/events.py +67 -0
  51. culvert-0.1.0/src/data_pipeline_core/py.typed +0 -0
  52. culvert-0.1.0/src/data_pipeline_core/runtime.py +414 -0
  53. culvert-0.1.0/src/data_pipeline_core/schema/__init__.py +5 -0
  54. culvert-0.1.0/src/data_pipeline_core/schema/entity.py +70 -0
  55. culvert-0.1.0/src/data_pipeline_gcp_bigquery/__init__.py +18 -0
  56. culvert-0.1.0/src/data_pipeline_gcp_bigquery/cost_tracker.py +240 -0
  57. culvert-0.1.0/src/data_pipeline_gcp_bigquery/finops_sink.py +158 -0
  58. culvert-0.1.0/src/data_pipeline_gcp_bigquery/warehouse.py +133 -0
  59. culvert-0.1.0/src/data_pipeline_gcp_gcs/__init__.py +17 -0
  60. culvert-0.1.0/src/data_pipeline_gcp_gcs/blob_store.py +108 -0
  61. culvert-0.1.0/src/data_pipeline_gcp_gcs/cost_tracker.py +249 -0
  62. culvert-0.1.0/src/data_pipeline_gcp_observability/__init__.py +53 -0
  63. culvert-0.1.0/src/data_pipeline_gcp_observability/cloud_monitoring_metrics_hook.py +287 -0
  64. culvert-0.1.0/src/data_pipeline_gcp_observability/cloud_trace_observability_hook.py +236 -0
  65. culvert-0.1.0/src/data_pipeline_gcp_observability/culvert_mdc_populator.py +186 -0
  66. culvert-0.1.0/src/data_pipeline_gcp_observability/data_catalog_lineage_emitter.py +189 -0
  67. culvert-0.1.0/src/data_pipeline_gcp_pubsub/__init__.py +18 -0
  68. culvert-0.1.0/src/data_pipeline_gcp_pubsub/cost_tracker.py +205 -0
  69. culvert-0.1.0/src/data_pipeline_gcp_pubsub/io.py +93 -0
  70. culvert-0.1.0/src/data_pipeline_gcp_secrets/__init__.py +13 -0
  71. culvert-0.1.0/src/data_pipeline_gcp_secrets/secret_manager_provider.py +164 -0
  72. culvert-0.1.0/src/data_pipeline_orchestration/__init__.py +105 -0
  73. culvert-0.1.0/src/data_pipeline_orchestration/_job_control.py +91 -0
  74. culvert-0.1.0/src/data_pipeline_orchestration/callbacks/__init__.py +54 -0
  75. culvert-0.1.0/src/data_pipeline_orchestration/callbacks/dlq.py +174 -0
  76. culvert-0.1.0/src/data_pipeline_orchestration/callbacks/factory.py +128 -0
  77. culvert-0.1.0/src/data_pipeline_orchestration/callbacks/handlers.py +223 -0
  78. culvert-0.1.0/src/data_pipeline_orchestration/callbacks/quarantine.py +100 -0
  79. culvert-0.1.0/src/data_pipeline_orchestration/callbacks/types.py +66 -0
  80. culvert-0.1.0/src/data_pipeline_orchestration/dependency.py +250 -0
  81. culvert-0.1.0/src/data_pipeline_orchestration/factories/__init__.py +91 -0
  82. culvert-0.1.0/src/data_pipeline_orchestration/factories/_dag_builders.py +917 -0
  83. culvert-0.1.0/src/data_pipeline_orchestration/factories/base_dag_factory.py +206 -0
  84. culvert-0.1.0/src/data_pipeline_orchestration/factories/config.py +372 -0
  85. culvert-0.1.0/src/data_pipeline_orchestration/factories/dag_factory.py +146 -0
  86. culvert-0.1.0/src/data_pipeline_orchestration/factories/dag_factory_alias.py +154 -0
  87. culvert-0.1.0/src/data_pipeline_orchestration/factories/validators.py +366 -0
  88. culvert-0.1.0/src/data_pipeline_orchestration/hooks/secrets.py +84 -0
  89. culvert-0.1.0/src/data_pipeline_orchestration/operators/__init__.py +24 -0
  90. culvert-0.1.0/src/data_pipeline_orchestration/operators/dataflow.py +592 -0
  91. culvert-0.1.0/src/data_pipeline_orchestration/routing/__init__.py +54 -0
  92. culvert-0.1.0/src/data_pipeline_orchestration/routing/config.py +70 -0
  93. culvert-0.1.0/src/data_pipeline_orchestration/routing/router.py +167 -0
  94. culvert-0.1.0/src/data_pipeline_orchestration/routing/yaml_selector.py +228 -0
  95. culvert-0.1.0/src/data_pipeline_orchestration/sensors/__init__.py +15 -0
  96. culvert-0.1.0/src/data_pipeline_orchestration/sensors/dataflow.py +90 -0
  97. culvert-0.1.0/src/data_pipeline_orchestration/sensors/pubsub.py +311 -0
  98. culvert-0.1.0/src/data_pipeline_tester/__init__.py +41 -0
  99. culvert-0.1.0/src/data_pipeline_tester/assertions/__init__.py +79 -0
  100. culvert-0.1.0/src/data_pipeline_tester/assertions/beam_assertions.py +88 -0
  101. culvert-0.1.0/src/data_pipeline_tester/assertions/pipeline_assertions.py +106 -0
  102. culvert-0.1.0/src/data_pipeline_tester/assertions/record_assertions.py +109 -0
  103. culvert-0.1.0/src/data_pipeline_tester/base/__init__.py +37 -0
  104. culvert-0.1.0/src/data_pipeline_tester/base/beam_test.py +152 -0
  105. culvert-0.1.0/src/data_pipeline_tester/base/gdw_test.py +1 -0
  106. culvert-0.1.0/src/data_pipeline_tester/base/pipeline_test.py +134 -0
  107. culvert-0.1.0/src/data_pipeline_tester/base/result.py +70 -0
  108. culvert-0.1.0/src/data_pipeline_tester/base/scenario_test.py +107 -0
  109. culvert-0.1.0/src/data_pipeline_tester/base/validation_test.py +143 -0
  110. culvert-0.1.0/src/data_pipeline_tester/bdd/__init__.py +6 -0
  111. culvert-0.1.0/src/data_pipeline_tester/bdd/base.py +38 -0
  112. culvert-0.1.0/src/data_pipeline_tester/bdd/steps/__init__.py +0 -0
  113. culvert-0.1.0/src/data_pipeline_tester/bdd/steps/common_steps.py +18 -0
  114. culvert-0.1.0/src/data_pipeline_tester/bdd/steps/dq_steps.py +51 -0
  115. culvert-0.1.0/src/data_pipeline_tester/bdd/steps/pipeline_steps.py +54 -0
  116. culvert-0.1.0/src/data_pipeline_tester/builders/__init__.py +41 -0
  117. culvert-0.1.0/src/data_pipeline_tester/builders/config_builder.py +128 -0
  118. culvert-0.1.0/src/data_pipeline_tester/builders/pipeline_builder.py +102 -0
  119. culvert-0.1.0/src/data_pipeline_tester/builders/record_builder.py +166 -0
  120. culvert-0.1.0/src/data_pipeline_tester/comparison/__init__.py +16 -0
  121. culvert-0.1.0/src/data_pipeline_tester/comparison/dual_run.py +373 -0
  122. culvert-0.1.0/src/data_pipeline_tester/fixtures/__init__.py +88 -0
  123. culvert-0.1.0/src/data_pipeline_tester/fixtures/beam.py +64 -0
  124. culvert-0.1.0/src/data_pipeline_tester/fixtures/bigquery.py +83 -0
  125. culvert-0.1.0/src/data_pipeline_tester/fixtures/common.py +123 -0
  126. culvert-0.1.0/src/data_pipeline_tester/fixtures/gcs.py +79 -0
  127. culvert-0.1.0/src/data_pipeline_tester/mocks/__init__.py +39 -0
  128. culvert-0.1.0/src/data_pipeline_tester/mocks/bigquery_mock.py +99 -0
  129. culvert-0.1.0/src/data_pipeline_tester/mocks/gcs_mock.py +95 -0
  130. culvert-0.1.0/src/data_pipeline_tester/mocks/pubsub_mock.py +384 -0
  131. culvert-0.1.0/src/data_pipeline_transform/__init__.py +4 -0
  132. culvert-0.1.0/src/data_pipeline_transform/dbt_shared/macros/audit_columns.sql +50 -0
  133. culvert-0.1.0/src/data_pipeline_transform/dbt_shared/macros/data_quality_check.sql +121 -0
  134. culvert-0.1.0/src/data_pipeline_transform/dbt_shared/macros/enrichment.sql +48 -0
  135. culvert-0.1.0/src/data_pipeline_transform/dbt_shared/macros/pii_masking.sql +181 -0
@@ -0,0 +1,131 @@
1
+ # Python
2
+ __pycache__/
3
+ *.py[cod]
4
+ *$py.class
5
+ *.so
6
+ .Python
7
+ build/
8
+ develop-eggs/
9
+ dist/
10
+ downloads/
11
+ eggs/
12
+ .eggs/
13
+ lib/
14
+ lib64/
15
+ parts/
16
+ sdist/
17
+ var/
18
+ wheels/
19
+ pip-wheel-metadata/
20
+ share/python-wheels/
21
+ *.egg-info/
22
+ .installed.cfg
23
+ *.egg
24
+ MANIFEST
25
+
26
+ # Virtual Environment
27
+ venv/
28
+ ENV/
29
+ env/
30
+ .venv
31
+
32
+ # IDEs
33
+ .vscode/
34
+ .idea/
35
+ *.swp
36
+ *.swo
37
+ *~
38
+ .DS_Store
39
+
40
+ # Jupyter Notebooks
41
+ .ipynb_checkpoints
42
+ *.ipynb
43
+
44
+ # Testing
45
+ .pytest_cache/
46
+ .coverage
47
+ htmlcov/
48
+ .tox/
49
+ .nox/
50
+ .pytest_results*.txt
51
+
52
+ # Environment variables
53
+ .env
54
+ .env.local
55
+
56
+ # GCP
57
+ .gcp/
58
+ credentials.json
59
+ service-account.json
60
+
61
+ # Logs
62
+ *.log
63
+ logs/
64
+
65
+ # Temporary files
66
+ tmp/
67
+ temp/
68
+ *.tmp
69
+
70
+ # Data files (large datasets should not be committed)
71
+ data/output/*
72
+ data/archive/*
73
+ !data/input/applications_20250119_1.csv # Keep sample data
74
+ *.png
75
+ !blueprint/docs/08-reference/Tasks.png
76
+
77
+ # Terraform
78
+ .terraform/
79
+ *.tfstate
80
+ *.tfstate.*
81
+ crash.log
82
+ *.tfvars
83
+ *.tfvars.json
84
+ override.tf
85
+ override.tf.json
86
+ _override.tf
87
+ _override.tf.json
88
+ .terraformrc
89
+ terraform.rc
90
+
91
+ # DBT
92
+ dbt_packages/
93
+ dbt_modules/
94
+ target/
95
+ profiles.yml
96
+ !deployments/*/dbt/profiles.yml
97
+ shared_macros/
98
+
99
+ # Compiled files
100
+ *.pyc
101
+ *.pyo
102
+
103
+ # Local prompts and session scratchpads
104
+ NEXT_PROMPT.md
105
+ ANALYSIS_AND_PROMPT.md
106
+ docs/COPILOT_NEXT_SESSION_PROMPT.md
107
+
108
+ # Claude Code local settings (per-user, not shared)
109
+ .claude/
110
+
111
+ # Scratch CLI output and test logs that should never be committed
112
+ /1pip
113
+ /transform_log.txt
114
+ **/tester_tests.log
115
+
116
+
117
+ # Additional venv directories
118
+ .venv_*/
119
+ venv*/
120
+ .venv*/
121
+ infrastructure/terraform/systems/generic/.terraform.lock.hcl
122
+ dependency-reduced-pom.xml
123
+
124
+ # Helm vendored subchart tarballs (Chart.lock is the source of truth)
125
+ deployments/*/helm/charts/*.tgz
126
+
127
+ # Per-deployment terraform provider lock (single-platform; regenerated on init)
128
+ **/.terraform.lock.hcl
129
+
130
+ # culvert distribution build artifacts
131
+ python-culvert/dist/
culvert-0.1.0/PKG-INFO ADDED
@@ -0,0 +1,112 @@
1
+ Metadata-Version: 2.4
2
+ Name: culvert
3
+ Version: 0.1.0
4
+ Summary: Cloud-agnostic, polyglot data-pipeline framework: one contract set, adapters per cloud. Python side of Culvert (Java twin on Maven Central: com.enrichmeai.culvert).
5
+ Project-URL: Homepage, https://github.com/enrichmeai/culvert
6
+ Project-URL: Repository, https://github.com/enrichmeai/culvert
7
+ Project-URL: Issues, https://github.com/enrichmeai/culvert/issues
8
+ Author: Joseph Aruja
9
+ License: MIT
10
+ Keywords: airflow,bigquery,cloud-agnostic,contracts,data-pipeline,dbt,framework,gcp
11
+ Classifier: Development Status :: 4 - Beta
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: License :: OSI Approved :: MIT License
14
+ Classifier: Programming Language :: Python :: 3.10
15
+ Classifier: Programming Language :: Python :: 3.11
16
+ Classifier: Programming Language :: Python :: 3.12
17
+ Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
18
+ Requires-Python: >=3.10
19
+ Requires-Dist: typing-extensions>=4.7
20
+ Provides-Extra: all
21
+ Requires-Dist: dbt-bigquery>=1.5.0; extra == 'all'
22
+ Requires-Dist: google-cloud-bigquery>=3.11.0; extra == 'all'
23
+ Requires-Dist: google-cloud-datacatalog>=3.14.0; extra == 'all'
24
+ Requires-Dist: google-cloud-monitoring>=2.14.0; extra == 'all'
25
+ Requires-Dist: google-cloud-pubsub>=2.18.0; extra == 'all'
26
+ Requires-Dist: google-cloud-secret-manager>=2.16.0; extra == 'all'
27
+ Requires-Dist: google-cloud-storage>=2.10.0; extra == 'all'
28
+ Requires-Dist: opentelemetry-api>=1.20.0; extra == 'all'
29
+ Requires-Dist: pyyaml>=6.0; extra == 'all'
30
+ Provides-Extra: contract-tests
31
+ Requires-Dist: pytest>=7.4; extra == 'contract-tests'
32
+ Provides-Extra: gcp
33
+ Requires-Dist: google-cloud-bigquery>=3.11.0; extra == 'gcp'
34
+ Requires-Dist: google-cloud-datacatalog>=3.14.0; extra == 'gcp'
35
+ Requires-Dist: google-cloud-monitoring>=2.14.0; extra == 'gcp'
36
+ Requires-Dist: google-cloud-pubsub>=2.18.0; extra == 'gcp'
37
+ Requires-Dist: google-cloud-secret-manager>=2.16.0; extra == 'gcp'
38
+ Requires-Dist: google-cloud-storage>=2.10.0; extra == 'gcp'
39
+ Requires-Dist: opentelemetry-api>=1.20.0; extra == 'gcp'
40
+ Provides-Extra: orchestration
41
+ Requires-Dist: google-cloud-bigquery>=3.11.0; extra == 'orchestration'
42
+ Requires-Dist: google-cloud-pubsub>=2.18.0; extra == 'orchestration'
43
+ Requires-Dist: google-cloud-storage>=2.10.0; extra == 'orchestration'
44
+ Requires-Dist: pyyaml>=6.0; extra == 'orchestration'
45
+ Provides-Extra: tester
46
+ Requires-Dist: apache-beam[gcp]==2.56.0; extra == 'tester'
47
+ Requires-Dist: google-cloud-bigquery>=3.11.0; extra == 'tester'
48
+ Requires-Dist: google-cloud-pubsub>=2.18.0; extra == 'tester'
49
+ Requires-Dist: google-cloud-storage>=2.10.0; extra == 'tester'
50
+ Requires-Dist: pytest-cov>=4.0.0; extra == 'tester'
51
+ Requires-Dist: pytest-mock>=3.10.0; extra == 'tester'
52
+ Requires-Dist: pytest>=7.0.0; extra == 'tester'
53
+ Provides-Extra: transform
54
+ Requires-Dist: dbt-bigquery>=1.5.0; extra == 'transform'
55
+ Description-Content-Type: text/markdown
56
+
57
+ # Culvert
58
+
59
+ **A cloud-agnostic, polyglot data-pipeline framework.** One contract set —
60
+ `Source`, `Sink`, `Transform`, `Pipeline`, `RuntimeContext`, `BlobStore`,
61
+ `Warehouse`, `JobControlRepository`, and friends — realised in **Java and
62
+ Python**, with cloud specifics behind adapters. Define the pipeline once
63
+ against contracts; point it at an emulator on your laptop, a dev project, or
64
+ production by wiring, not rewriting.
65
+
66
+ - **Contract-driven** — the [language-neutral spec](https://github.com/enrichmeai/culvert/blob/main/docs/CONTRACT.md)
67
+ is the portability boundary. Business logic depends only on contracts.
68
+ - **Two clouds today** — GCP is the first full implementation; AWS is a real
69
+ (Java) adapter family with the same pipeline proven on both. Azure is on the
70
+ roadmap.
71
+ - **No application framework in the core** — plain Python Protocols +
72
+ entry-point auto-discovery; composes into Beam pipelines, Airflow DAGs, or a
73
+ script on your laptop.
74
+ - **Local-first** — the whole stack runs against emulators with zero cloud
75
+ account; cloud is where you prove and ship, not where you develop.
76
+
77
+ ## Install
78
+
79
+ ```bash
80
+ pip install culvert # core contracts only (no cloud SDKs)
81
+ pip install culvert[gcp] # + BigQuery, GCS, Pub/Sub, Secret Manager, observability
82
+ pip install culvert[orchestration] # + Airflow-side DAG factory, operators, sensors
83
+ pip install culvert[transform] # + dbt integration
84
+ pip install culvert[all]
85
+ ```
86
+
87
+ Python ≥ 3.10. For 0.1.0 the import packages keep their library names — the
88
+ contracts live in `data_pipeline_core`:
89
+
90
+ ```python
91
+ from data_pipeline_core import autoconfig
92
+
93
+ config = autoconfig.discover() # entry-point adapter discovery
94
+ blob_store = config.blob_store() # GcsBlobStore if culvert[gcp] installed
95
+ ```
96
+
97
+ The Java twin ships as `com.enrichmeai.culvert:*` on Maven Central — same
98
+ contracts, Java owns the Beam/Dataflow execution layer; Python owns the
99
+ Airflow runtime and dbt packaging.
100
+
101
+ ## Status
102
+
103
+ `0.1.0` — first public release. Built and validated against a real GCP
104
+ project (Cloud Run, BigQuery, Pub/Sub, event-driven end-to-end) before
105
+ publishing. GCP adapters are production-shaped; the AWS family is Java-side;
106
+ Azure is a roadmap skeleton. Honest limitations are documented per adapter in
107
+ the source.
108
+
109
+ Docs, architecture, worked example deployments, and the engineering story:
110
+ **https://github.com/enrichmeai/culvert**
111
+
112
+ MIT licensed.
@@ -0,0 +1,56 @@
1
+ # Culvert
2
+
3
+ **A cloud-agnostic, polyglot data-pipeline framework.** One contract set —
4
+ `Source`, `Sink`, `Transform`, `Pipeline`, `RuntimeContext`, `BlobStore`,
5
+ `Warehouse`, `JobControlRepository`, and friends — realised in **Java and
6
+ Python**, with cloud specifics behind adapters. Define the pipeline once
7
+ against contracts; point it at an emulator on your laptop, a dev project, or
8
+ production by wiring, not rewriting.
9
+
10
+ - **Contract-driven** — the [language-neutral spec](https://github.com/enrichmeai/culvert/blob/main/docs/CONTRACT.md)
11
+ is the portability boundary. Business logic depends only on contracts.
12
+ - **Two clouds today** — GCP is the first full implementation; AWS is a real
13
+ (Java) adapter family with the same pipeline proven on both. Azure is on the
14
+ roadmap.
15
+ - **No application framework in the core** — plain Python Protocols +
16
+ entry-point auto-discovery; composes into Beam pipelines, Airflow DAGs, or a
17
+ script on your laptop.
18
+ - **Local-first** — the whole stack runs against emulators with zero cloud
19
+ account; cloud is where you prove and ship, not where you develop.
20
+
21
+ ## Install
22
+
23
+ ```bash
24
+ pip install culvert # core contracts only (no cloud SDKs)
25
+ pip install culvert[gcp] # + BigQuery, GCS, Pub/Sub, Secret Manager, observability
26
+ pip install culvert[orchestration] # + Airflow-side DAG factory, operators, sensors
27
+ pip install culvert[transform] # + dbt integration
28
+ pip install culvert[all]
29
+ ```
30
+
31
+ Python ≥ 3.10. For 0.1.0 the import packages keep their library names — the
32
+ contracts live in `data_pipeline_core`:
33
+
34
+ ```python
35
+ from data_pipeline_core import autoconfig
36
+
37
+ config = autoconfig.discover() # entry-point adapter discovery
38
+ blob_store = config.blob_store() # GcsBlobStore if culvert[gcp] installed
39
+ ```
40
+
41
+ The Java twin ships as `com.enrichmeai.culvert:*` on Maven Central — same
42
+ contracts, Java owns the Beam/Dataflow execution layer; Python owns the
43
+ Airflow runtime and dbt packaging.
44
+
45
+ ## Status
46
+
47
+ `0.1.0` — first public release. Built and validated against a real GCP
48
+ project (Cloud Run, BigQuery, Pub/Sub, event-driven end-to-end) before
49
+ publishing. GCP adapters are production-shaped; the AWS family is Java-side;
50
+ Azure is a roadmap skeleton. Honest limitations are documented per adapter in
51
+ the source.
52
+
53
+ Docs, architecture, worked example deployments, and the engineering story:
54
+ **https://github.com/enrichmeai/culvert**
55
+
56
+ MIT licensed.
@@ -0,0 +1,48 @@
1
+ """Hatchling build hook: locate the library packages in either context.
2
+
3
+ The wheel is built from two places:
4
+ - the repo (CI / local): packages live at ../data-pipeline-libraries/*/src/
5
+ - an extracted sdist (pip building from source): packages were packed into
6
+ this project's own src/ by the sdist force-include
7
+
8
+ A static [tool.hatch.build.targets.wheel.force-include] can only express one
9
+ of those, so this hook resolves each package at build time.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ from pathlib import Path
15
+
16
+ from hatchling.builders.hooks.plugin.interface import BuildHookInterface
17
+
18
+ # import package -> distribution dir under data-pipeline-libraries/
19
+ PACKAGES = {
20
+ "data_pipeline_core": "data-pipeline-core",
21
+ "data_pipeline_gcp_bigquery": "data-pipeline-gcp-bigquery",
22
+ "data_pipeline_gcp_gcs": "data-pipeline-gcp-gcs",
23
+ "data_pipeline_gcp_pubsub": "data-pipeline-gcp-pubsub",
24
+ "data_pipeline_gcp_secrets": "data-pipeline-gcp-secrets",
25
+ "data_pipeline_gcp_observability": "data-pipeline-gcp-observability",
26
+ "data_pipeline_orchestration": "data-pipeline-orchestration",
27
+ "data_pipeline_transform": "data-pipeline-transform",
28
+ "data_pipeline_tester": "data-pipeline-tester",
29
+ "data_pipeline_contract_tests": "data-pipeline-contract-tests",
30
+ }
31
+
32
+
33
+ class LibraryPackagesHook(BuildHookInterface):
34
+ PLUGIN_NAME = "library-packages"
35
+
36
+ def initialize(self, version, build_data):
37
+ if self.target_name != "wheel":
38
+ return
39
+ root = Path(self.root)
40
+ force_include = build_data.setdefault("force_include", {})
41
+ for pkg, dist in PACKAGES.items():
42
+ repo_src = root.parent / "data-pipeline-libraries" / dist / "src" / pkg
43
+ sdist_src = root / "src" / pkg
44
+ src = repo_src if repo_src.is_dir() else sdist_src
45
+ if not src.is_dir():
46
+ raise FileNotFoundError(
47
+ f"package {pkg} not found at {repo_src} or {sdist_src}")
48
+ force_include[str(src)] = pkg
@@ -0,0 +1,118 @@
1
+ # The `culvert` PyPI distribution (D1 decision, 2026-07: ONE distribution with
2
+ # extras — the Maven side stays modular; Python presents one front door).
3
+ #
4
+ # This pyproject assembles the data-pipeline-libraries/* import packages into a
5
+ # single wheel via hatchling force-include. Import names stay `data_pipeline_*`
6
+ # for 0.1.0 so every path:line citation in the book remains true; the `culvert`
7
+ # import package is a thin front door (version + re-exports).
8
+ #
9
+ # Excluded on purpose: data_pipeline_framework (the legacy bundle — predecessor
10
+ # deps; removed entirely at the F1 legacy cleanup).
11
+ #
12
+ # Entry points: the per-library `data_pipeline_core.adapters` registrations are
13
+ # merged here — without them AutoConfig discovery finds nothing. autoconfig
14
+ # skips entry points whose deps aren't installed (core-only install is safe).
15
+
16
+ [build-system]
17
+ requires = ["hatchling"]
18
+ build-backend = "hatchling.build"
19
+
20
+ [project]
21
+ name = "culvert"
22
+ version = "0.1.0"
23
+ description = "Cloud-agnostic, polyglot data-pipeline framework: one contract set, adapters per cloud. Python side of Culvert (Java twin on Maven Central: com.enrichmeai.culvert)."
24
+ readme = "README.md"
25
+ license = { text = "MIT" }
26
+ requires-python = ">=3.10"
27
+ authors = [{ name = "Joseph Aruja" }]
28
+ keywords = ["data-pipeline", "framework", "cloud-agnostic", "gcp", "bigquery", "dbt", "airflow", "contracts"]
29
+ classifiers = [
30
+ "Development Status :: 4 - Beta",
31
+ "Intended Audience :: Developers",
32
+ "License :: OSI Approved :: MIT License",
33
+ "Programming Language :: Python :: 3.10",
34
+ "Programming Language :: Python :: 3.11",
35
+ "Programming Language :: Python :: 3.12",
36
+ "Topic :: Software Development :: Libraries :: Application Frameworks",
37
+ ]
38
+ dependencies = [
39
+ "typing-extensions>=4.7",
40
+ ]
41
+
42
+ [project.urls]
43
+ Homepage = "https://github.com/enrichmeai/culvert"
44
+ Repository = "https://github.com/enrichmeai/culvert"
45
+ Issues = "https://github.com/enrichmeai/culvert/issues"
46
+
47
+ [project.optional-dependencies]
48
+ gcp = [
49
+ "google-cloud-bigquery>=3.11.0",
50
+ "google-cloud-storage>=2.10.0",
51
+ "google-cloud-pubsub>=2.18.0",
52
+ "google-cloud-secret-manager>=2.16.0",
53
+ "google-cloud-monitoring>=2.14.0",
54
+ "google-cloud-datacatalog>=3.14.0",
55
+ "opentelemetry-api>=1.20.0",
56
+ ]
57
+ orchestration = [
58
+ "PyYAML>=6.0",
59
+ "google-cloud-storage>=2.10.0",
60
+ "google-cloud-pubsub>=2.18.0",
61
+ "google-cloud-bigquery>=3.11.0",
62
+ ]
63
+ transform = [
64
+ "dbt-bigquery>=1.5.0",
65
+ ]
66
+ tester = [
67
+ "pytest>=7.0.0",
68
+ "pytest-cov>=4.0.0",
69
+ "pytest-mock>=3.10.0",
70
+ "apache-beam[gcp]==2.56.0",
71
+ "google-cloud-bigquery>=3.11.0",
72
+ "google-cloud-storage>=2.10.0",
73
+ "google-cloud-pubsub>=2.18.0",
74
+ ]
75
+ contract-tests = [
76
+ "pytest>=7.4",
77
+ ]
78
+ all = [
79
+ "culvert[gcp,orchestration,transform]",
80
+ ]
81
+
82
+ [project.entry-points."data_pipeline_core.adapters"]
83
+ blob_store = "data_pipeline_gcp_gcs:GcsBlobStore"
84
+ warehouse = "data_pipeline_gcp_bigquery:BigQueryWarehouse"
85
+ finops = "data_pipeline_gcp_bigquery:BigQueryFinOpsSink"
86
+ source = "data_pipeline_gcp_pubsub:PubSubSource"
87
+ sink = "data_pipeline_gcp_pubsub:PubSubSink"
88
+ secrets = "data_pipeline_gcp_secrets:SecretManagerProvider"
89
+ observability = "data_pipeline_gcp_observability:CloudTraceObservabilityHook"
90
+ stage_metrics = "data_pipeline_gcp_observability:CloudMonitoringMetricsHook"
91
+ lineage = "data_pipeline_gcp_observability:DataCatalogLineageEmitter"
92
+
93
+ [tool.hatch.build.targets.wheel]
94
+ packages = ["src/culvert"]
95
+
96
+ # Library packages are force-included by the build hook (hatch_build.py),
97
+ # which resolves them from the repo or from an extracted sdist.
98
+ [tool.hatch.build.targets.wheel.hooks.custom]
99
+ path = "hatch_build.py"
100
+
101
+
102
+ [tool.hatch.build.targets.sdist]
103
+ include = ["src/culvert", "README.md", "pyproject.toml", "hatch_build.py"]
104
+
105
+ [tool.hatch.build.targets.sdist.force-include]
106
+ "../data-pipeline-libraries/data-pipeline-core/src/data_pipeline_core" = "src/data_pipeline_core"
107
+ "../data-pipeline-libraries/data-pipeline-gcp-bigquery/src/data_pipeline_gcp_bigquery" = "src/data_pipeline_gcp_bigquery"
108
+ "../data-pipeline-libraries/data-pipeline-gcp-gcs/src/data_pipeline_gcp_gcs" = "src/data_pipeline_gcp_gcs"
109
+ "../data-pipeline-libraries/data-pipeline-gcp-pubsub/src/data_pipeline_gcp_pubsub" = "src/data_pipeline_gcp_pubsub"
110
+ "../data-pipeline-libraries/data-pipeline-gcp-secrets/src/data_pipeline_gcp_secrets" = "src/data_pipeline_gcp_secrets"
111
+ "../data-pipeline-libraries/data-pipeline-gcp-observability/src/data_pipeline_gcp_observability" = "src/data_pipeline_gcp_observability"
112
+ "../data-pipeline-libraries/data-pipeline-orchestration/src/data_pipeline_orchestration" = "src/data_pipeline_orchestration"
113
+ "../data-pipeline-libraries/data-pipeline-transform/src/data_pipeline_transform" = "src/data_pipeline_transform"
114
+ "../data-pipeline-libraries/data-pipeline-tester/src/data_pipeline_tester" = "src/data_pipeline_tester"
115
+ "../data-pipeline-libraries/data-pipeline-contract-tests/src/data_pipeline_contract_tests" = "src/data_pipeline_contract_tests"
116
+
117
+ [tool.hatch.build]
118
+ exclude = ["**/__pycache__", "**/*.pyc", "**/.pytest_cache", "**/*.egg-info"]
@@ -0,0 +1,27 @@
1
+ """Culvert — a cloud-agnostic, polyglot data-pipeline framework.
2
+
3
+ One language-neutral contract set (Source, Sink, Transform, Pipeline,
4
+ RuntimeContext, BlobStore, Warehouse, JobControlRepository, ...), realised in
5
+ Java and Python, with cloud specifics behind adapters. This distribution is the
6
+ Python side; the Java twin ships as ``com.enrichmeai.culvert:*`` on Maven
7
+ Central.
8
+
9
+ For 0.1.0 the import packages keep their library names — start at
10
+ ``data_pipeline_core`` for the contracts::
11
+
12
+ from data_pipeline_core import autoconfig
13
+ from data_pipeline_core.contracts.blob_store import BlobStore
14
+
15
+ Install extras for the adapters you need::
16
+
17
+ pip install culvert[gcp] # BigQuery, GCS, Pub/Sub, Secret Manager, observability
18
+ pip install culvert[orchestration] # Airflow-side DAG factory, operators, sensors
19
+ pip install culvert[transform] # dbt runner assets
20
+ pip install culvert[all]
21
+
22
+ Docs and worked examples: https://github.com/enrichmeai/culvert
23
+ """
24
+
25
+ __version__ = "0.1.0"
26
+
27
+ __all__ = ["__version__"]
@@ -0,0 +1,28 @@
1
+ """Abstract pytest contract tests for the Culvert data pipeline framework.
2
+
3
+ Each cloud adapter library inherits the relevant mixin and supplies the
4
+ adapter under test. The mixin tests prove the adapter honours the
5
+ Protocol's documented behaviour.
6
+
7
+ Mirror of the Java ``data-pipeline-contract-tests`` library.
8
+
9
+ Sprint-5 deliverable. Sprint-17 (T17.3): added
10
+ :class:`StageMetricsHookContract` and brought existing mixins to 1:1
11
+ method-level parity with their Java counterparts (null-rejection tests).
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ from data_pipeline_contract_tests.blob_store import BlobStoreContract
17
+ from data_pipeline_contract_tests.secrets import SecretProviderContract
18
+ from data_pipeline_contract_tests.stage_metrics_hook import StageMetricsHookContract
19
+ from data_pipeline_contract_tests.warehouse import WarehouseContract
20
+
21
+ __version__ = "0.1.0"
22
+
23
+ __all__ = [
24
+ "BlobStoreContract",
25
+ "SecretProviderContract",
26
+ "StageMetricsHookContract",
27
+ "WarehouseContract",
28
+ ]
@@ -0,0 +1,34 @@
1
+ """BlobStore contract test mixin.
2
+
3
+ Java mirror: ``com.enrichmeai.culvert.contracttests.BlobStoreContractTest``
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ import pytest
9
+
10
+
11
+ class BlobStoreContract:
12
+ """Mixin — subclasses provide ``store``, ``known_uri``, ``missing_uri``
13
+ fixtures. ``known_uri`` resolves to bytes equal to ``b"hello"``.
14
+
15
+ Java mirror: ``BlobStoreContractTest`` (Sprint-5 deliverable).
16
+ """
17
+
18
+ def test_get_known_returns_bytes(self, store, known_uri):
19
+ assert store.get(known_uri) == b"hello"
20
+
21
+ def test_exists_known_true(self, store, known_uri):
22
+ assert store.exists(known_uri) is True
23
+
24
+ def test_exists_missing_false(self, store, missing_uri):
25
+ assert store.exists(missing_uri) is False
26
+
27
+ def test_delete_missing_idempotent(self, store, missing_uri):
28
+ # Should not raise.
29
+ store.delete(missing_uri)
30
+
31
+ def test_null_arguments_rejected(self, store):
32
+ # Java: ``nullArgumentsRejected`` — get(null) must raise.
33
+ with pytest.raises((TypeError, ValueError)):
34
+ store.get(None)
@@ -0,0 +1,41 @@
1
+ """SecretProvider contract test mixin.
2
+
3
+ Java mirror: ``com.enrichmeai.culvert.contracttests.SecretProviderContractTest``
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ import pytest
9
+
10
+
11
+ class SecretProviderContract:
12
+ """Mixin — subclasses provide ``self.provider`` via a pytest fixture
13
+ named ``provider`` (and configure it so ``get("known")`` returns
14
+ ``"the-secret"`` and ``get("missing")`` raises ``KeyError``).
15
+
16
+ Java mirror: ``SecretProviderContractTest`` (Sprint-5 deliverable).
17
+
18
+ Example wiring in a cloud adapter's test sources:
19
+
20
+ .. code-block:: python
21
+
22
+ from data_pipeline_contract_tests import SecretProviderContract
23
+
24
+ class TestMySecretProvider(SecretProviderContract):
25
+ @pytest.fixture
26
+ def provider(self):
27
+ client = make_mock_client({"known": "the-secret"})
28
+ return MySecretProvider("my-project", client)
29
+ """
30
+
31
+ def test_get_known_returns_value(self, provider):
32
+ assert provider.get("known") == "the-secret"
33
+
34
+ def test_get_missing_raises(self, provider):
35
+ with pytest.raises((KeyError, LookupError)):
36
+ provider.get("missing")
37
+
38
+ def test_null_name_rejected(self, provider):
39
+ # Java: ``nullNameRejected`` — get(null, ...) must raise.
40
+ with pytest.raises((TypeError, ValueError)):
41
+ provider.get(None)