culvert 0.1.0__py3-none-any.whl

culvert/__init__.py

@@ -0,0 +1,27 @@
+"""Culvert — a cloud-agnostic, polyglot data-pipeline framework.
+
+One language-neutral contract set (Source, Sink, Transform, Pipeline,
+RuntimeContext, BlobStore, Warehouse, JobControlRepository, ...), realised in
+Java and Python, with cloud specifics behind adapters. This distribution is the
+Python side; the Java twin ships as ``com.enrichmeai.culvert:*`` on Maven
+Central.
+
+For 0.1.0 the import packages keep their library names — start at
+``data_pipeline_core`` for the contracts::
+
+    from data_pipeline_core import autoconfig
+    from data_pipeline_core.contracts.blob_store import BlobStore
+
+Install extras for the adapters you need::
+
+    pip install culvert[gcp]            # BigQuery, GCS, Pub/Sub, Secret Manager, observability
+    pip install culvert[orchestration]  # Airflow-side DAG factory, operators, sensors
+    pip install culvert[transform]      # dbt runner assets
+    pip install culvert[all]
+
+Docs and worked examples: https://github.com/enrichmeai/culvert
+"""
+
+__version__ = "0.1.0"
+
+__all__ = ["__version__"]

culvert-0.1.0.dist-info/METADATA

@@ -0,0 +1,112 @@
+Metadata-Version: 2.4
+Name: culvert
+Version: 0.1.0
+Summary: Cloud-agnostic, polyglot data-pipeline framework: one contract set, adapters per cloud. Python side of Culvert (Java twin on Maven Central: com.enrichmeai.culvert).
+Project-URL: Homepage, https://github.com/enrichmeai/culvert
+Project-URL: Repository, https://github.com/enrichmeai/culvert
+Project-URL: Issues, https://github.com/enrichmeai/culvert/issues
+Author: Joseph Aruja
+License: MIT
+Keywords: airflow,bigquery,cloud-agnostic,contracts,data-pipeline,dbt,framework,gcp
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
+Requires-Python: >=3.10
+Requires-Dist: typing-extensions>=4.7
+Provides-Extra: all
+Requires-Dist: dbt-bigquery>=1.5.0; extra == 'all'
+Requires-Dist: google-cloud-bigquery>=3.11.0; extra == 'all'
+Requires-Dist: google-cloud-datacatalog>=3.14.0; extra == 'all'
+Requires-Dist: google-cloud-monitoring>=2.14.0; extra == 'all'
+Requires-Dist: google-cloud-pubsub>=2.18.0; extra == 'all'
+Requires-Dist: google-cloud-secret-manager>=2.16.0; extra == 'all'
+Requires-Dist: google-cloud-storage>=2.10.0; extra == 'all'
+Requires-Dist: opentelemetry-api>=1.20.0; extra == 'all'
+Requires-Dist: pyyaml>=6.0; extra == 'all'
+Provides-Extra: contract-tests
+Requires-Dist: pytest>=7.4; extra == 'contract-tests'
+Provides-Extra: gcp
+Requires-Dist: google-cloud-bigquery>=3.11.0; extra == 'gcp'
+Requires-Dist: google-cloud-datacatalog>=3.14.0; extra == 'gcp'
+Requires-Dist: google-cloud-monitoring>=2.14.0; extra == 'gcp'
+Requires-Dist: google-cloud-pubsub>=2.18.0; extra == 'gcp'
+Requires-Dist: google-cloud-secret-manager>=2.16.0; extra == 'gcp'
+Requires-Dist: google-cloud-storage>=2.10.0; extra == 'gcp'
+Requires-Dist: opentelemetry-api>=1.20.0; extra == 'gcp'
+Provides-Extra: orchestration
+Requires-Dist: google-cloud-bigquery>=3.11.0; extra == 'orchestration'
+Requires-Dist: google-cloud-pubsub>=2.18.0; extra == 'orchestration'
+Requires-Dist: google-cloud-storage>=2.10.0; extra == 'orchestration'
+Requires-Dist: pyyaml>=6.0; extra == 'orchestration'
+Provides-Extra: tester
+Requires-Dist: apache-beam[gcp]==2.56.0; extra == 'tester'
+Requires-Dist: google-cloud-bigquery>=3.11.0; extra == 'tester'
+Requires-Dist: google-cloud-pubsub>=2.18.0; extra == 'tester'
+Requires-Dist: google-cloud-storage>=2.10.0; extra == 'tester'
+Requires-Dist: pytest-cov>=4.0.0; extra == 'tester'
+Requires-Dist: pytest-mock>=3.10.0; extra == 'tester'
+Requires-Dist: pytest>=7.0.0; extra == 'tester'
+Provides-Extra: transform
+Requires-Dist: dbt-bigquery>=1.5.0; extra == 'transform'
+Description-Content-Type: text/markdown
+
+# Culvert
+
+**A cloud-agnostic, polyglot data-pipeline framework.** One contract set —
+`Source`, `Sink`, `Transform`, `Pipeline`, `RuntimeContext`, `BlobStore`,
+`Warehouse`, `JobControlRepository`, and friends — realised in **Java and
+Python**, with cloud specifics behind adapters. Define the pipeline once
+against contracts; point it at an emulator on your laptop, a dev project, or
+production by wiring, not rewriting.
+
+- **Contract-driven** — the [language-neutral spec](https://github.com/enrichmeai/culvert/blob/main/docs/CONTRACT.md)
+  is the portability boundary. Business logic depends only on contracts.
+- **Two clouds today** — GCP is the first full implementation; AWS is a real
+  (Java) adapter family with the same pipeline proven on both. Azure is on the
+  roadmap.
+- **No application framework in the core** — plain Python Protocols +
+  entry-point auto-discovery; composes into Beam pipelines, Airflow DAGs, or a
+  script on your laptop.
+- **Local-first** — the whole stack runs against emulators with zero cloud
+  account; cloud is where you prove and ship, not where you develop.
+
+## Install
+
+```bash
+pip install culvert                 # core contracts only (no cloud SDKs)
+pip install culvert[gcp]            # + BigQuery, GCS, Pub/Sub, Secret Manager, observability
+pip install culvert[orchestration]  # + Airflow-side DAG factory, operators, sensors
+pip install culvert[transform]      # + dbt integration
+pip install culvert[all]
+```
+
+Python ≥ 3.10. For 0.1.0 the import packages keep their library names — the
+contracts live in `data_pipeline_core`:
+
+```python
+from data_pipeline_core import autoconfig
+
+config = autoconfig.discover()          # entry-point adapter discovery
+blob_store = config.blob_store()        # GcsBlobStore if culvert[gcp] installed
+```
+
+The Java twin ships as `com.enrichmeai.culvert:*` on Maven Central — same
+contracts, Java owns the Beam/Dataflow execution layer; Python owns the
+Airflow runtime and dbt packaging.
+
+## Status
+
+`0.1.0` — first public release. Built and validated against a real GCP
+project (Cloud Run, BigQuery, Pub/Sub, event-driven end-to-end) before
+publishing. GCP adapters are production-shaped; the AWS family is Java-side;
+Azure is a roadmap skeleton. Honest limitations are documented per adapter in
+the source.
+
+Docs, architecture, worked example deployments, and the engineering story:
+**https://github.com/enrichmeai/culvert**
+
+MIT licensed.

culvert-0.1.0.dist-info/RECORD

@@ -0,0 +1,134 @@
+culvert/__init__.py,sha256=I0wbpJgTo-c0xEif_5i_tfNqklyXf13KH-NSnDNMM2s,1040
+data_pipeline_contract_tests/__init__.py,sha256=VcJjW6mrWX0HO64gozESfO076CZijWI52zTJeHt1sn8,986
+data_pipeline_contract_tests/blob_store.py,sha256=JZUcZs1gt0WULzNUEIHZdCdYyv06rDlx3x1r6ayh9Oo,1073
+data_pipeline_contract_tests/secrets.py,sha256=pzlJnl_-x6VfcHpcAtCJ9OmrkhEmzj46-loNaE46SwI,1354
+data_pipeline_contract_tests/stage_metrics_hook.py,sha256=tCFxrD5BT8nq-NhtNkI9MujVHKSnX0gjwHV1uaKyOVc,4372
+data_pipeline_contract_tests/warehouse.py,sha256=5B10dA9y46oI_qI7-pfch24oBiimVnAx6vjRr0AHDtc,1323
+data_pipeline_core/__init__.py,sha256=ScNCKd2CwkaDL6YOim2BCDylwTaP-jtn9HeOgyRKkYw,2329
+data_pipeline_core/autoconfig.py,sha256=IOmwu11AhuFsy8yn3SOiqDCDBa8h_k3x_JsNV_dOf7A,5406
+data_pipeline_core/decorators.py,sha256=2yvjDcnDWH4w2PxFVecBURpp8kQgfI0BTV5cc9FBv3k,2512
+data_pipeline_core/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+data_pipeline_core/runtime.py,sha256=-u7ncxGAE2ibh8nJ9mmXMcJ13sBfnJ4dz7XsXSb2GcY,15728
+data_pipeline_core/audit/__init__.py,sha256=OHQCTUlOGMer4IYyr5TRNTV7NR6alPIIodnHKe7BiSo,168
+data_pipeline_core/audit/records.py,sha256=cKwd4x8pgNze7Q9U1Czao2Mu59OsMK-m0srWYRHCkek,997
+data_pipeline_core/contracts/__init__.py,sha256=qs6RUsA95Vdy6vstNiR891-BCquw_22Scy_IjJ3gcLg,1575
+data_pipeline_core/contracts/audit.py,sha256=XCx-HeI1BgGy082gT0lhmPAboaPChSjwMrW-rdD9hlo,1237
+data_pipeline_core/contracts/blob_store.py,sha256=JV_YAxE_WDZihGU95zY-m3CtE-F9xAF-5tpZLcYy9_s,2641
+data_pipeline_core/contracts/finops.py,sha256=ym8QDiI9EA012p5M_sOngSITcdVz-hITJfo1T9Jco5c,1233
+data_pipeline_core/contracts/governance.py,sha256=EkXpuJfOOWqCU1zpCs7ma_bZKAErF2KlW3xQI0F4Dpk,1633
+data_pipeline_core/contracts/job_control.py,sha256=WouoYykydGfjrxDdbRZAfNinQZXhQkPWCam89Di8lTc,4254
+data_pipeline_core/contracts/lineage.py,sha256=994hyvoO_LJZSzEFZD35cmCXAN6gRrZAHuoSq28lcJg,946
+data_pipeline_core/contracts/observability.py,sha256=CWZzaj1BdNPRK-9m91HY-vabIDuRGXkJs7MfNG_Xy80,2481
+data_pipeline_core/contracts/pipeline.py,sha256=3bhCvi9Luw5LspFzVoxNfE6s4hDWmAN5CEP3SQJ0x8s,1369
+data_pipeline_core/contracts/runtime.py,sha256=PVYaOYHClOZdEdyjUAOZS7iKyzdz61EXgwkl78efvxk,4317
+data_pipeline_core/contracts/secrets.py,sha256=ZQkU-leLqGAqPgkKFOuBrb1M7jXBBZLhejBPtXG4jJQ,1143
+data_pipeline_core/contracts/source.py,sha256=1-Ngdd0iyPqHh73cIweP4MErtSWS74JAzujRUZv5xaQ,2037
+data_pipeline_core/contracts/stage_metrics.py,sha256=OXbF5Ie2vgr8eGAhJqgEkwJjgpy1p-OX0C_MEZhm6kY,3900
+data_pipeline_core/contracts/warehouse.py,sha256=StJiDm6ySK1vVxVT9hKTJiN5qEbcADs-eUeq2M5rUdk,3144
+data_pipeline_core/dataquality/__init__.py,sha256=igfv9RvTQJ_nkl5c_Sx4Aw4GEXhVw3clEDU304su400,1507
+data_pipeline_core/dataquality/data_quality_transform.py,sha256=3iookCH7oI-l7_D1Nnb27zxJZ8A0XNZTvPaqWbueqRU,15370
+data_pipeline_core/dataquality/field_violation.py,sha256=eeaRkxsknJ-VuZLX5e9n9K8ri-B30Fq9YNi-fYnYDcQ,1672
+data_pipeline_core/dataquality/numeric_range.py,sha256=cLocX6GvWteo-jU76lEv7080yqufgv8_yjpBdu-4QXY,1862
+data_pipeline_core/dataquality/validation_result.py,sha256=H2m2M3I0w6mspCPTjD4lOGmcXVJUpdAIFeNT60KtQ6s,3124
+data_pipeline_core/dataquality/violation_kind.py,sha256=mzbhQI-igbML5ErF5doR4fhHjk8lxyhfU1Rl4_HQVyY,1220
+data_pipeline_core/finops_api/__init__.py,sha256=LF7h5aDW5mL_E2nPXoxaunId5aGGvKHFfaNPUABIVaw,855
+data_pipeline_core/finops_api/budget.py,sha256=yDrkG4IxOXqFJ8kaG2AkQ_8sR9hSI-HLfy-S5DBe9fw,9170
+data_pipeline_core/finops_api/labels.py,sha256=4TGX-Z4M0vuu3VxtSmaupmhuUSSbGm5PHJf5ERHhsc8,961
+data_pipeline_core/finops_api/models.py,sha256=WUoZvycA3TvIuq6dhTQrjN3KP8pvdC_3H_oJCqGfX4Q,1374
+data_pipeline_core/governance_api/__init__.py,sha256=z-GSVht8drj1ppuIx8qP0KmfwADKkrTKtobwvhnQtnQ,596
+data_pipeline_core/governance_api/classification.py,sha256=R4bXcad0NXTq0MEmE78U7f6OXJuaNwSQbioaEdcNMfA,572
+data_pipeline_core/governance_api/masker.py,sha256=RZurFdESsc2s3cUzT9zHwSkzJEt5jtZQqmszpUn7s3k,3522
+data_pipeline_core/governance_api/pii_masking_governance_policy.py,sha256=epuJ7W7nvvzjGbHy31qqy2GqY11xXWN7QFzF_RRKb4c,6910
+data_pipeline_core/governance_api/policies.py,sha256=qgDvXKB95G3fV1YdX__5YUzT-ZN97tBWKITgtN7jlno,1686
+data_pipeline_core/job_control_api/__init__.py,sha256=0iYej02zSMa4_-1Rr-gbm8II2MfyJesp_9nP8F0m4fQ,561
+data_pipeline_core/job_control_api/models.py,sha256=kMgwW9F9lZswgAdDf9IfbAsHSCiYgrvZsRIxdLBoqTs,3219
+data_pipeline_core/job_control_api/types.py,sha256=sttpoVR3A2r3HcGsU1qPQFse-eGnY36kc9FNmW-HsE4,1086
+data_pipeline_core/lineage/__init__.py,sha256=shMojrPXu2hgJCnmfGlOVTJhMFD2sHEM1CmbEqeimzI,171
+data_pipeline_core/lineage/events.py,sha256=6BZ2bJB_nfCSGIYwdJCpjGukq-3KZL36yDVhVQdzHGc,1982
+data_pipeline_core/schema/__init__.py,sha256=FDFlB6K1yw4VsJ-XjfSwo91b4wiTmI7aNREZ1BPgiPo,178
+data_pipeline_core/schema/entity.py,sha256=Na-fS5GaQC0k5Nz5jKdfYk22cEtVQNdZ3t1Hwh_ZZf0,2866
+data_pipeline_gcp_bigquery/__init__.py,sha256=-fOkVAvRw7Lcy9h999qRR6-K-tRCahhyKE_Gkif4RYE,526
+data_pipeline_gcp_bigquery/cost_tracker.py,sha256=9CuxpNVpsxDJPXMe0jNlJYubOGtijHTdVTecJeLQUno,9366
+data_pipeline_gcp_bigquery/finops_sink.py,sha256=3Z1yt6s6yjOM-vsa5n9ndMjfKS-Zz-9kLVIyVDQCz7I,6327
+data_pipeline_gcp_bigquery/warehouse.py,sha256=3uXuB5wfitpdgcOe3j4osuQ03-0Kk9aWW4Ou1kAyrD4,5076
+data_pipeline_gcp_gcs/__init__.py,sha256=Yzj81NMiv8CslB2qT8DHGSzh1koWKQksyJQ2ccUAx4g,426
+data_pipeline_gcp_gcs/blob_store.py,sha256=Fn5cCf_zQ1heSsK6og2lT5H5WUMUTY_9tHUtlZSOWKA,3918
+data_pipeline_gcp_gcs/cost_tracker.py,sha256=15w6jXJdD6rhGJ8f2YDZONgWNxoSOErFwEGb-Irgyl0,9246
+data_pipeline_gcp_observability/__init__.py,sha256=y4HMRDeGhncSEoJvJxbb6cpxYUFeehFTjOOdX0DaIiA,1783
+data_pipeline_gcp_observability/cloud_monitoring_metrics_hook.py,sha256=ScjYnAESrelqA7TR3J4lqcDu4hli81RMWGzrzyCRmoY,10937
+data_pipeline_gcp_observability/cloud_trace_observability_hook.py,sha256=Tw1RPV4UczeCmsCje185PF3zzXivbpplVXdLEbJ4fvA,8967
+data_pipeline_gcp_observability/culvert_mdc_populator.py,sha256=7M2lv8AuXGujMaKmF_1UvRN8A1hmBiV5jXfud1Gc9Wo,6003
+data_pipeline_gcp_observability/data_catalog_lineage_emitter.py,sha256=8mU583_MC-x0OjysLeEEqB3DyE3UVpSK_joHVeT5aVU,7015
+data_pipeline_gcp_pubsub/__init__.py,sha256=GPtmBp4bo0MEkOjLjJ-1gtYbwCJpzfmH0wYPD-JhZzk,470
+data_pipeline_gcp_pubsub/cost_tracker.py,sha256=NRDJIGp52ToZK47vVDarCL3cVFA-nUUf19KEToL9ME4,7573
+data_pipeline_gcp_pubsub/io.py,sha256=yfndM1HvACzyEhHxURKifPtZjRjVaphOi55TgmdKbw8,3462
+data_pipeline_gcp_secrets/__init__.py,sha256=qFAge9_PFFmmG7671E6pj0WYcoXlhK4V9tresL-CAZQ,389
+data_pipeline_gcp_secrets/secret_manager_provider.py,sha256=6syIkm9rwykJ_DvxtaKMffGtRJledOjKLJyBGNqj6eg,6444
+data_pipeline_orchestration/__init__.py,sha256=JPG-vKX9vN5p27FC6tLPoHImouLwxc96Ukf4edEdcWE,2496
+data_pipeline_orchestration/_job_control.py,sha256=KEG0yES5SXvy7zVnWt6MncS7IXWjRalvvKYCK_1WLv0,3696
+data_pipeline_orchestration/dependency.py,sha256=lH-wyMIlO9FNHE7gRn-YH36ZPcLZdWyRJo25xl_WxqQ,9231
+data_pipeline_orchestration/callbacks/__init__.py,sha256=B6ec6OSY1ovhXZQ5hnjBSgSgH16DiJNZaCnHak4HnPo,961
+data_pipeline_orchestration/callbacks/dlq.py,sha256=VUGq7V_8wKhEieDz3CksJN7ivsKPrEFvq2CbjEw8gXs,5529
+data_pipeline_orchestration/callbacks/factory.py,sha256=pSmoevqtTCABpvQseQLQWVQGCP_kAv43m8wzqlunBrY,3676
+data_pipeline_orchestration/callbacks/handlers.py,sha256=d6m-Pn5j03DBMWpuOHnU39XEvmjt09-4VKNv5OJIjNU,6582
+data_pipeline_orchestration/callbacks/quarantine.py,sha256=CM5xoFzF7iP-qYDCSb2VObqkcfK0LU_linDyaYlaNao,2972
+data_pipeline_orchestration/callbacks/types.py,sha256=6qByCaLPIibhFY3yyYRkcpWNt7HvekG_npBU00QFrKU,1791
+data_pipeline_orchestration/factories/__init__.py,sha256=Cx585Uz0N5dug_ML25K4S1UCbh6PRryHlV2tDnqwPQc,2213
+data_pipeline_orchestration/factories/_dag_builders.py,sha256=4AsspjaWOWhB3_ThSrNFiLoYUgpsAfSoSUYorzNyOGg,38663
+data_pipeline_orchestration/factories/base_dag_factory.py,sha256=XciK6QffNdH6UK5MQ8ZrcEK3tFHIxtaeJh2KOoku1U8,5830
+data_pipeline_orchestration/factories/config.py,sha256=LJHqbEQcI_MmNVwwDhGSUCyMUayWD_uJ6cI2irJjaPg,12737
+data_pipeline_orchestration/factories/dag_factory.py,sha256=NUoK7X5aKsg_6BJxAKhwO9xFVPmF-cTUjyPpxsOXSXU,5987
+data_pipeline_orchestration/factories/dag_factory_alias.py,sha256=ec220Ai0fIdkgwZ0Wukx58ahhR9P7F9iPxaRZLTZEls,5938
+data_pipeline_orchestration/factories/validators.py,sha256=0pOKRXHpVVsNEQ8SfUNMEfpid2Lwn8OOGRLVB-Gy-Rg,12316
+data_pipeline_orchestration/hooks/secrets.py,sha256=UVAfJucwHxs7zfmFll5DbRQ6Sn3SmD8jQteD-4RWtgA,2652
+data_pipeline_orchestration/operators/__init__.py,sha256=51zUKPKdJIqJ_ZVqbERhD8Tcn_vd_ASuNWXPEBOFqoA,454
+data_pipeline_orchestration/operators/dataflow.py,sha256=u4pl0ff7D0WEkWlKmiQGd5crnXxRDxslqxP91jyqtVs,20927
+data_pipeline_orchestration/routing/__init__.py,sha256=SzahekNMWT5xuYlTcjQUbk9zHvqx7afGD-UXvLwsNjE,1393
+data_pipeline_orchestration/routing/config.py,sha256=cWu7W84oXcxYWlDu5GbXo9-FhDamx1q8ip8poBuC32Q,1710
+data_pipeline_orchestration/routing/router.py,sha256=y16wVUCJvQ94th6diqa8ZXBRUeK7R8cp94Gh1RgtG6c,4633
+data_pipeline_orchestration/routing/yaml_selector.py,sha256=xMhuA__MLYVYDNM-jKSWVV7cPll7EjVPPxYZ2gJduFQ,7354
+data_pipeline_orchestration/sensors/__init__.py,sha256=xf91EHczJouGtbskWIZZvhDEq7bNnLTvFlP1GtP3miQ,284
+data_pipeline_orchestration/sensors/dataflow.py,sha256=AVN_F16dNxjtuOY0HVYKBDsYc4OzmCM6UHjCFFMzz4s,3091
+data_pipeline_orchestration/sensors/pubsub.py,sha256=RJVEmCFfh05hXdE6vrlyqa1xJ1bO2R2vkHC5sPy0C8k,11470
+data_pipeline_tester/__init__.py,sha256=NaopYycE680DfS3M3qeJD7AS6lx-bwo2K6KJagZnsmA,966
+data_pipeline_tester/assertions/__init__.py,sha256=tDUPUiQJ3_mSPsFkkaIf6khiBMkzgUFz0ZLi2IWcHeU,1975
+data_pipeline_tester/assertions/beam_assertions.py,sha256=UkHrizI07kt2yCiHcujyM33fviKMtTRwQ6WGyQAPtyc,2443
+data_pipeline_tester/assertions/pipeline_assertions.py,sha256=zjkAaLqqXZDj7FxbmPOJStzyZajp_YcdLEq_yYIn-b0,3016
+data_pipeline_tester/assertions/record_assertions.py,sha256=YgTOeXIg0Cl7rSVafQumCs4i2Aixi6LBYFv1HDGXYks,2886
+data_pipeline_tester/base/__init__.py,sha256=rCkw2DUZKT5dLPP61fjiMWbFDTX60LJ7AiVve6OGH5g,1207
+data_pipeline_tester/base/beam_test.py,sha256=G5f7XuHFawQiodas1cYWYPmBQqHJY5CUqT_jDSge9js,4999
+data_pipeline_tester/base/gdw_test.py,sha256=DILKFeGoVm_G5gOsN5bOvlI9ILjiwfcGKHp_X2_FHCk,66
+data_pipeline_tester/base/pipeline_test.py,sha256=IqCCqsgmEVRYZruYKsZAylQYiRcf7mF4zFs8A2rCwdM,4119
+data_pipeline_tester/base/result.py,sha256=O2kbl9k--V3mF6vWqYoCU-1HLAV0guddo4Nxbb8T--M,1695
+data_pipeline_tester/base/scenario_test.py,sha256=vKKjw9RhMwpa8ERjNtGeQ3Ldak6nnnN8vQAOWOVPrvw,3175
+data_pipeline_tester/base/validation_test.py,sha256=DsP7-rm-KEEdXUzEu4HrZ5UBB2Hjpkp5SXXNdthANp4,4706
+data_pipeline_tester/bdd/__init__.py,sha256=rT6bDPHNEAR2uPO802KCZ3efQNtCFii6f-GMRsUQveg,124
+data_pipeline_tester/bdd/base.py,sha256=7gp82365icc1qqSisV367Ht_4u6SIx_xtTWJbp3od50,1042
+data_pipeline_tester/bdd/steps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+data_pipeline_tester/bdd/steps/common_steps.py,sha256=aPXY3VcHaSm4EIRGxPNoO8bYPNf8zHzr--O8tuHUJDk,642
+data_pipeline_tester/bdd/steps/dq_steps.py,sha256=T12OnBj_uQrojAMq4DjLN2n2n_Y12Nblp0eNt2fEvvg,1936
+data_pipeline_tester/bdd/steps/pipeline_steps.py,sha256=BR1d950ydJwGQOnmGxZFc10U6hxKlnt4wJkiMYlPksM,2113
+data_pipeline_tester/builders/__init__.py,sha256=WQiGMw41iH2S1nIUles4-CATqSs3Aj6qwlnM2pBecng,1141
+data_pipeline_tester/builders/config_builder.py,sha256=CIVrGodzrQuUiAp9kWvn-1ti9D0Rwj_j946L16ronhc,3804
+data_pipeline_tester/builders/pipeline_builder.py,sha256=ohVzeOXiNQeZTsrwuZP3LYWIgjRbfVcfl5XZ0zOIFHU,2579
+data_pipeline_tester/builders/record_builder.py,sha256=LWy_klIox7ggAlL9ZKJiJCZULVqOdYvlt-69d8Kfjds,4000
+data_pipeline_tester/comparison/__init__.py,sha256=HWQawx5sIpW8DstAsk-tIJ2O2BowAVjAYpq58LTAE0g,238
+data_pipeline_tester/comparison/dual_run.py,sha256=R_lvryr3RFu5R3FxklHZo7X7fkVzV6900Avp8g6la7E,12155
+data_pipeline_tester/fixtures/__init__.py,sha256=Z_jUwDI3JEeBWQcHDDkzheBYYhOVajAvJcbByzJ2e3Q,2194
+data_pipeline_tester/fixtures/beam.py,sha256=5ySAIl2ONwgJSooR7D-31INswDLucMeF4rg28jPvzQc,1438
+data_pipeline_tester/fixtures/bigquery.py,sha256=3ef6mC2rsCyIfHGMZj1_EzGIHC3AzjkxWziTNP0MSfw,2127
+data_pipeline_tester/fixtures/common.py,sha256=FeVIWbdu6tI6Mry02uYIG8NFAljNG81F5GYUY6bhAio,3125
+data_pipeline_tester/fixtures/gcs.py,sha256=oZnSXDz_5D035YcOt6j1fUn8F3FTO8AVnKQh-K6De2Q,1650
+data_pipeline_tester/mocks/__init__.py,sha256=jM9LwxLCTB4vyMOZ7gMr2zL0bJ-TMen1mUuglZ7v1tw,1097
+data_pipeline_tester/mocks/bigquery_mock.py,sha256=9i4LtDnkdVU3OGCEAQwNJO66AgEC1J61xIMcvV76iiE,2639
+data_pipeline_tester/mocks/gcs_mock.py,sha256=bOG2EUXlvHUBYDMfdVDqa3vPW8i6VCEM_4A7kSywZso,2558
+data_pipeline_tester/mocks/pubsub_mock.py,sha256=mAfZ08ypGlRJWch6rwva5IKgCgqcLMFDA3YmbgsnOQk,11839
+data_pipeline_transform/__init__.py,sha256=ltAvCHCTw0ylJorp_yI363VF7xsxr6fNyGFSm-Nx87g,61
+data_pipeline_transform/dbt_shared/macros/audit_columns.sql,sha256=KilvtCY65KmQV_me4qKPV5-591pjt1S7ZKGmAz1anAg,1468
+data_pipeline_transform/dbt_shared/macros/data_quality_check.sql,sha256=7_uO9kNfIFc4ocM0OzT-gGdTLU55Qw0N-MZndx1LVAg,4456
+data_pipeline_transform/dbt_shared/macros/enrichment.sql,sha256=EjXAnM0SSkSs2RqCR1FP4g7YE3hFhsiVgxiD7pJxS8s,2414
+data_pipeline_transform/dbt_shared/macros/pii_masking.sql,sha256=MGmtrLdYiVfZJFdBx69dkeubapmOUMoxYEJEkMzxX9c,6155
+culvert-0.1.0.dist-info/METADATA,sha256=WAB1KLIcu0DDS0zLbkKhfkSi2KiymWi3zTdbAnXd0Fg,5331
+culvert-0.1.0.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
+culvert-0.1.0.dist-info/entry_points.txt,sha256=BRGRVTNRy5mXOv8VdZckmGKMm4W34s9E0lw_5n8kvIM,557
+culvert-0.1.0.dist-info/RECORD,,

culvert-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.31.0
+Root-Is-Purelib: true
+Tag: py3-none-any

culvert-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,10 @@
+[data_pipeline_core.adapters]
+blob_store = data_pipeline_gcp_gcs:GcsBlobStore
+finops = data_pipeline_gcp_bigquery:BigQueryFinOpsSink
+lineage = data_pipeline_gcp_observability:DataCatalogLineageEmitter
+observability = data_pipeline_gcp_observability:CloudTraceObservabilityHook
+secrets = data_pipeline_gcp_secrets:SecretManagerProvider
+sink = data_pipeline_gcp_pubsub:PubSubSink
+source = data_pipeline_gcp_pubsub:PubSubSource
+stage_metrics = data_pipeline_gcp_observability:CloudMonitoringMetricsHook
+warehouse = data_pipeline_gcp_bigquery:BigQueryWarehouse

data_pipeline_contract_tests/__init__.py

@@ -0,0 +1,28 @@
+"""Abstract pytest contract tests for the Culvert data pipeline framework.
+
+Each cloud adapter library inherits the relevant mixin and supplies the
+adapter under test. The mixin tests prove the adapter honours the
+Protocol's documented behaviour.
+
+Mirror of the Java ``data-pipeline-contract-tests`` library.
+
+Sprint-5 deliverable. Sprint-17 (T17.3): added
+:class:`StageMetricsHookContract` and brought existing mixins to 1:1
+method-level parity with their Java counterparts (null-rejection tests).
+"""
+
+from __future__ import annotations
+
+from data_pipeline_contract_tests.blob_store import BlobStoreContract
+from data_pipeline_contract_tests.secrets import SecretProviderContract
+from data_pipeline_contract_tests.stage_metrics_hook import StageMetricsHookContract
+from data_pipeline_contract_tests.warehouse import WarehouseContract
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "BlobStoreContract",
+    "SecretProviderContract",
+    "StageMetricsHookContract",
+    "WarehouseContract",
+]

data_pipeline_contract_tests/blob_store.py

@@ -0,0 +1,34 @@
+"""BlobStore contract test mixin.
+
+Java mirror: ``com.enrichmeai.culvert.contracttests.BlobStoreContractTest``
+"""
+
+from __future__ import annotations
+
+import pytest
+
+
+class BlobStoreContract:
+    """Mixin — subclasses provide ``store``, ``known_uri``, ``missing_uri``
+    fixtures. ``known_uri`` resolves to bytes equal to ``b"hello"``.
+
+    Java mirror: ``BlobStoreContractTest`` (Sprint-5 deliverable).
+    """
+
+    def test_get_known_returns_bytes(self, store, known_uri):
+        assert store.get(known_uri) == b"hello"
+
+    def test_exists_known_true(self, store, known_uri):
+        assert store.exists(known_uri) is True
+
+    def test_exists_missing_false(self, store, missing_uri):
+        assert store.exists(missing_uri) is False
+
+    def test_delete_missing_idempotent(self, store, missing_uri):
+        # Should not raise.
+        store.delete(missing_uri)
+
+    def test_null_arguments_rejected(self, store):
+        # Java: ``nullArgumentsRejected`` — get(null) must raise.
+        with pytest.raises((TypeError, ValueError)):
+            store.get(None)

data_pipeline_contract_tests/secrets.py

@@ -0,0 +1,41 @@
+"""SecretProvider contract test mixin.
+
+Java mirror: ``com.enrichmeai.culvert.contracttests.SecretProviderContractTest``
+"""
+
+from __future__ import annotations
+
+import pytest
+
+
+class SecretProviderContract:
+    """Mixin — subclasses provide ``self.provider`` via a pytest fixture
+    named ``provider`` (and configure it so ``get("known")`` returns
+    ``"the-secret"`` and ``get("missing")`` raises ``KeyError``).
+
+    Java mirror: ``SecretProviderContractTest`` (Sprint-5 deliverable).
+
+    Example wiring in a cloud adapter's test sources:
+
+    .. code-block:: python
+
+        from data_pipeline_contract_tests import SecretProviderContract
+
+        class TestMySecretProvider(SecretProviderContract):
+            @pytest.fixture
+            def provider(self):
+                client = make_mock_client({"known": "the-secret"})
+                return MySecretProvider("my-project", client)
+    """
+
+    def test_get_known_returns_value(self, provider):
+        assert provider.get("known") == "the-secret"
+
+    def test_get_missing_raises(self, provider):
+        with pytest.raises((KeyError, LookupError)):
+            provider.get("missing")
+
+    def test_null_name_rejected(self, provider):
+        # Java: ``nullNameRejected`` — get(null, ...) must raise.
+        with pytest.raises((TypeError, ValueError)):
+            provider.get(None)

data_pipeline_contract_tests/stage_metrics_hook.py

@@ -0,0 +1,109 @@
+"""StageMetricsHook contract test mixin.
+
+Java mirror: no ``Abstract*ContractTest`` class exists on the Java side —
+this mixin is derived directly from the ``StageMetricsHook`` interface contract
+and its accompanying ``StageMetrics`` record
+(``com.enrichmeai.culvert.contracts.StageMetricsHook``,
+``com.enrichmeai.culvert.contracts.StageMetrics``, Sprint-12 / issue #65).
+
+The defining guarantee of ``StageMetricsHook`` is:
+  "Implementations must not propagate monitoring-backend failures to the
+   caller. If the backend is unavailable the implementation logs and
+   swallows the exception so the pipeline continues uninterrupted."
+
+That guarantee is the primary behavioral assertion this mixin checks.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from data_pipeline_core.contracts.stage_metrics import StageMetrics
+
+
+def _valid_metrics(**overrides) -> StageMetrics:
+    """Build a minimal valid :class:`StageMetrics` snapshot."""
+    defaults = dict(
+        pipeline_id="pipe-1",
+        run_id="run-abc",
+        stage_name="my-stage",
+        rows_processed=100,
+        stage_latency_ms=42.5,
+        error_count=0,
+    )
+    defaults.update(overrides)
+    return StageMetrics(**defaults)
+
+
+class StageMetricsHookContract:
+    """Mixin — subclasses provide two pytest fixtures:
+
+    - ``hook`` — the :class:`~data_pipeline_core.contracts.stage_metrics.StageMetricsHook`
+      implementation under test (happy-path instance).
+    - ``failing_hook`` — the same implementation wired to a backend that
+      raises an exception on every ``record_stage_metrics`` call, used to
+      verify the swallow-and-continue guarantee.
+
+    Java mirror: no ``AbstractStageMetricsHookContractTest`` exists in
+    ``data-pipeline-contract-tests-java``; guarantees are derived directly
+    from the interface contract in ``StageMetricsHook.java`` (Sprint-12
+    / issue #65).
+
+    Example wiring:
+
+    .. code-block:: python
+
+        from data_pipeline_contract_tests import StageMetricsHookContract
+        from data_pipeline_core.contracts.stage_metrics import StageMetrics
+        from unittest.mock import MagicMock
+
+        class TestMyHook(StageMetricsHookContract):
+            @pytest.fixture
+            def hook(self):
+                client = make_mock_monitoring_client()
+                return MyCloudMetricsHook(client)
+
+            @pytest.fixture
+            def failing_hook(self):
+                client = MagicMock()
+                client.create_time_series.side_effect = RuntimeError("backend down")
+                return MyCloudMetricsHook(client)
+    """
+
+    def test_record_stage_metrics_returns_none(self, hook):
+        """Happy-path: ``record_stage_metrics`` with a valid snapshot returns
+        ``None`` (i.e. does not raise and has no meaningful return value).
+        """
+        result = hook.record_stage_metrics(_valid_metrics())
+        assert result is None
+
+    def test_record_stage_metrics_all_fields(self, hook):
+        """Happy-path: all three label fields and all three metric fields are
+        accepted, including edge values (zero rows, zero errors, non-zero latency).
+        """
+        hook.record_stage_metrics(
+            _valid_metrics(rows_processed=0, stage_latency_ms=0.0, error_count=0)
+        )
+
+    def test_record_stage_metrics_with_errors(self, hook):
+        """Happy-path: non-zero ``error_count`` is accepted without raising."""
+        hook.record_stage_metrics(_valid_metrics(error_count=5))
+
+    def test_backend_failure_is_swallowed(self, failing_hook):
+        """Core contract guarantee: if the monitoring backend is unavailable
+        the implementation must NOT propagate the exception to the caller.
+        The pipeline continues uninterrupted.
+
+        This is the primary behavioral invariant stated in both
+        ``StageMetricsHook.java`` and the Python ``StageMetricsHook`` Protocol.
+        """
+        # Must not raise even though the backend raises internally.
+        failing_hook.record_stage_metrics(_valid_metrics())
+
+    def test_null_metrics_rejected(self, hook):
+        """``record_stage_metrics(None)`` must raise ``TypeError`` or
+        ``ValueError`` — implementations should not silently accept ``None``
+        in place of a ``StageMetrics`` snapshot.
+        """
+        with pytest.raises((TypeError, ValueError)):
+            hook.record_stage_metrics(None)

data_pipeline_contract_tests/warehouse.py

@@ -0,0 +1,36 @@
+"""Warehouse contract test mixin.
+
+Java mirror: ``com.enrichmeai.culvert.contracttests.WarehouseContractTest``
+"""
+
+from __future__ import annotations
+
+import pytest
+
+
+class WarehouseContract:
+    """Mixin — subclasses provide ``warehouse`` fixture configured so that
+    ``query("SELECT id FROM contract_test_table")`` yields ``{"id": 1}``,
+    ``table_exists("contract_test_table")`` is True, and
+    ``table_exists("contract_missing_table")`` is False.
+
+    Java mirror: ``WarehouseContractTest`` (Sprint-5 deliverable; T15.4
+    added ``known_table``/``missing_table`` hook — Python subclasses
+    override the fixture to supply qualified names if needed).
+    """
+
+    def test_query_streams_rows(self, warehouse):
+        rows = list(warehouse.query("SELECT id FROM contract_test_table"))
+        assert len(rows) >= 1
+        assert "id" in rows[0]
+
+    def test_table_exists_known_true(self, warehouse):
+        assert warehouse.table_exists("contract_test_table") is True
+
+    def test_table_exists_missing_false(self, warehouse):
+        assert warehouse.table_exists("contract_missing_table") is False
+
+    def test_null_sql_rejected(self, warehouse):
+        # Java: ``nullSqlRejected`` — query(null, ...) must raise.
+        with pytest.raises((TypeError, ValueError)):
+            warehouse.query(None)

data_pipeline_core/__init__.py

@@ -0,0 +1,60 @@
+"""data-pipeline-core: the cloud-neutral contracts for the Culvert framework.
+
+This package contains only Protocols, dataclasses, enums, and TypedDicts.
+Zero dependencies on google.cloud, boto3, or azure. Cloud-specific
+implementations of the Protocols live in sibling distributions
+(`data-pipeline-gcp-*`, future `data-pipeline-aws-*`, `data-pipeline-azure-*`).
+
+The public surface is the `contracts` package plus the supporting types.
+Most users will import the Protocols from the top level:
+
+    from data_pipeline_core import (
+        Source, Sink, Transform, Pipeline, PipelineStage, RuntimeContext,
+        JobControlRepository, BlobStore, Warehouse,
+        AuditEventPublisher, GovernancePolicy, LineageEmitter,
+        ObservabilityHook, FinOpsSink, SecretProvider,
+        StageMetrics, StageMetricsHook,
+    )
+"""
+
+from data_pipeline_core.contracts.audit import AuditEventPublisher
+from data_pipeline_core.contracts.blob_store import BlobStore
+from data_pipeline_core.contracts.finops import FinOpsSink
+from data_pipeline_core.contracts.governance import GovernancePolicy
+from data_pipeline_core.contracts.job_control import JobControlRepository
+from data_pipeline_core.contracts.lineage import LineageEmitter
+from data_pipeline_core.contracts.observability import ObservabilityHook
+from data_pipeline_core.contracts.pipeline import Pipeline, PipelineStage
+from data_pipeline_core.contracts.runtime import RuntimeContext
+from data_pipeline_core.contracts.secrets import SecretProvider
+from data_pipeline_core.contracts.source import Sink, Source, Transform
+from data_pipeline_core.contracts.stage_metrics import StageMetrics, StageMetricsHook
+from data_pipeline_core.contracts.warehouse import Warehouse
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "__version__",
+    # Source / Sink / Transform — the I/O primitives
+    "Source",
+    "Sink",
+    "Transform",
+    # Pipeline composition
+    "Pipeline",
+    "PipelineStage",
+    # The DI container / run metadata carrier
+    "RuntimeContext",
+    # Cloud-pluggable contracts
+    "JobControlRepository",
+    "BlobStore",
+    "Warehouse",
+    "AuditEventPublisher",
+    "GovernancePolicy",
+    "LineageEmitter",
+    "ObservabilityHook",
+    "FinOpsSink",
+    "SecretProvider",
+    # Typed per-stage metrics (Sprint-12 / T17.1)
+    "StageMetrics",
+    "StageMetricsHook",
+]

data_pipeline_core/audit/__init__.py

@@ -0,0 +1,5 @@
+"""Audit records — the cloud-neutral data shape an AuditEventPublisher emits."""
+
+from data_pipeline_core.audit.records import AuditRecord
+
+__all__ = ["AuditRecord"]

data_pipeline_core/audit/records.py

@@ -0,0 +1,34 @@
+"""The AuditRecord dataclass — the unit of audit emission.
+
+Mirrors the existing `gcp_pipeline_core.audit.records.AuditRecord` shape
+verbatim (eleven fields). Stage 1 will collapse the two into one.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any, Dict
+
+
+@dataclass
+class AuditRecord:
+    """A single audit event emitted at a pipeline boundary (e.g. an
+    ingestion stage completing).
+
+    `audit_hash` is a deterministic content hash that downstream
+    reconciliation uses to dedupe replays. `metadata` is the catch-all
+    for stage-specific context (table identifiers, partition keys, etc.)
+    """
+
+    run_id: str
+    pipeline_name: str
+    entity_type: str
+    source_file: str
+    record_count: int
+    processed_timestamp: datetime
+    processing_duration_seconds: float
+    success: bool
+    error_count: int = 0
+    audit_hash: str = ""
+    metadata: Dict[str, Any] = field(default_factory=dict)