acryl-datahub-actions 1.3.1.5rc9__py3-none-any.whl → 1.3.1.5rc10__py3-none-any.whl

{acryl_datahub_actions-1.3.1.5rc9.dist-info → acryl_datahub_actions-1.3.1.5rc10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: acryl-datahub-actions
-Version: 1.3.1.5rc9
+Version: 1.3.1.5rc10
 Summary: An action framework to work with DataHub real time changes.
 Home-page: https://docs.datahub.com/
 License: Apache-2.0
@@ -21,201 +21,216 @@ Classifier: Environment :: MacOS X
 Classifier: Topic :: Software Development
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: pydantic<3.0.0,>=2.0.0
-Requires-Dist: toml>=0.10.0
 Requires-Dist: h11>=0.16
+Requires-Dist: PyYAML
+Requires-Dist: typing-inspect
+Requires-Dist: entrypoints
 Requires-Dist: stackprinter
-Requires-Dist: click>=6.0.0
-Requires-Dist: click-default-group
-Requires-Dist: azure-identity==1.21.0
+Requires-Dist: httpcore>=1.0.9
+Requires-Dist: prometheus-client
+Requires-Dist: python-dateutil>=2.8.0
+Requires-Dist: acryl-datahub[datahub-kafka]==1.3.1.5rc10
 Requires-Dist: aws-msk-iam-sasl-signer-python==1.0.2
+Requires-Dist: click>=6.0.0
 Requires-Dist: progressbar2
-Requires-Dist: entrypoints
-Requires-Dist: PyYAML
-Requires-Dist: python-dateutil>=2.8.0
-Requires-Dist: httpcore>=1.0.9
-Requires-Dist: acryl-datahub[datahub-kafka]==1.3.1.5rc9
 Requires-Dist: tenacity
-Requires-Dist: prometheus-client
+Requires-Dist: toml>=0.10.0
+Requires-Dist: click-default-group
+Requires-Dist: pydantic<3.0.0,>=2.0.0
 Requires-Dist: ratelimit
-Requires-Dist: typing-inspect
+Requires-Dist: azure-identity==1.21.0
 Provides-Extra: base
-Requires-Dist: toml>=0.10.0; extra == "base"
+Requires-Dist: PyYAML; extra == "base"
+Requires-Dist: entrypoints; extra == "base"
 Requires-Dist: stackprinter; extra == "base"
+Requires-Dist: prometheus-client; extra == "base"
+Requires-Dist: python-dateutil>=2.8.0; extra == "base"
 Requires-Dist: click>=6.0.0; extra == "base"
+Requires-Dist: toml>=0.10.0; extra == "base"
 Requires-Dist: click-default-group; extra == "base"
 Requires-Dist: progressbar2; extra == "base"
-Requires-Dist: entrypoints; extra == "base"
-Requires-Dist: PyYAML; extra == "base"
-Requires-Dist: python-dateutil>=2.8.0; extra == "base"
 Requires-Dist: tenacity; extra == "base"
-Requires-Dist: prometheus-client; extra == "base"
 Provides-Extra: kafka
-Requires-Dist: toml>=0.10.0; extra == "kafka"
+Requires-Dist: PyYAML; extra == "kafka"
+Requires-Dist: entrypoints; extra == "kafka"
 Requires-Dist: stackprinter; extra == "kafka"
+Requires-Dist: prometheus-client; extra == "kafka"
+Requires-Dist: python-dateutil>=2.8.0; extra == "kafka"
 Requires-Dist: click>=6.0.0; extra == "kafka"
+Requires-Dist: toml>=0.10.0; extra == "kafka"
 Requires-Dist: click-default-group; extra == "kafka"
-Requires-Dist: progressbar2; extra == "kafka"
-Requires-Dist: entrypoints; extra == "kafka"
-Requires-Dist: PyYAML; extra == "kafka"
 Requires-Dist: confluent-kafka[schemaregistry]; extra == "kafka"
-Requires-Dist: python-dateutil>=2.8.0; extra == "kafka"
+Requires-Dist: progressbar2; extra == "kafka"
 Requires-Dist: tenacity; extra == "kafka"
-Requires-Dist: prometheus-client; extra == "kafka"
 Provides-Extra: executor
-Requires-Dist: toml>=0.10.0; extra == "executor"
+Requires-Dist: PyYAML; extra == "executor"
+Requires-Dist: entrypoints; extra == "executor"
 Requires-Dist: stackprinter; extra == "executor"
+Requires-Dist: prometheus-client; extra == "executor"
+Requires-Dist: python-dateutil>=2.8.0; extra == "executor"
 Requires-Dist: acryl-executor==0.3.2; extra == "executor"
 Requires-Dist: click>=6.0.0; extra == "executor"
+Requires-Dist: toml>=0.10.0; extra == "executor"
 Requires-Dist: click-default-group; extra == "executor"
 Requires-Dist: progressbar2; extra == "executor"
-Requires-Dist: entrypoints; extra == "executor"
-Requires-Dist: PyYAML; extra == "executor"
-Requires-Dist: python-dateutil>=2.8.0; extra == "executor"
 Requires-Dist: tenacity; extra == "executor"
-Requires-Dist: prometheus-client; extra == "executor"
 Provides-Extra: slack
-Requires-Dist: slack-bolt>=1.15.5; extra == "slack"
-Requires-Dist: toml>=0.10.0; extra == "slack"
+Requires-Dist: PyYAML; extra == "slack"
+Requires-Dist: entrypoints; extra == "slack"
 Requires-Dist: stackprinter; extra == "slack"
+Requires-Dist: prometheus-client; extra == "slack"
+Requires-Dist: python-dateutil>=2.8.0; extra == "slack"
 Requires-Dist: click>=6.0.0; extra == "slack"
+Requires-Dist: toml>=0.10.0; extra == "slack"
 Requires-Dist: click-default-group; extra == "slack"
 Requires-Dist: progressbar2; extra == "slack"
-Requires-Dist: entrypoints; extra == "slack"
-Requires-Dist: PyYAML; extra == "slack"
-Requires-Dist: python-dateutil>=2.8.0; extra == "slack"
+Requires-Dist: slack-bolt>=1.15.5; extra == "slack"
 Requires-Dist: tenacity; extra == "slack"
-Requires-Dist: prometheus-client; extra == "slack"
 Provides-Extra: teams
-Requires-Dist: toml>=0.10.0; extra == "teams"
+Requires-Dist: PyYAML; extra == "teams"
+Requires-Dist: entrypoints; extra == "teams"
 Requires-Dist: stackprinter; extra == "teams"
+Requires-Dist: prometheus-client; extra == "teams"
+Requires-Dist: python-dateutil>=2.8.0; extra == "teams"
+Requires-Dist: pymsteams>=0.2.2; extra == "teams"
 Requires-Dist: click>=6.0.0; extra == "teams"
+Requires-Dist: toml>=0.10.0; extra == "teams"
 Requires-Dist: click-default-group; extra == "teams"
 Requires-Dist: progressbar2; extra == "teams"
-Requires-Dist: entrypoints; extra == "teams"
-Requires-Dist: PyYAML; extra == "teams"
-Requires-Dist: python-dateutil>=2.8.0; extra == "teams"
 Requires-Dist: tenacity; extra == "teams"
-Requires-Dist: pymsteams>=0.2.2; extra == "teams"
-Requires-Dist: prometheus-client; extra == "teams"
 Provides-Extra: tag-propagation
-Requires-Dist: toml>=0.10.0; extra == "tag-propagation"
+Requires-Dist: PyYAML; extra == "tag-propagation"
+Requires-Dist: entrypoints; extra == "tag-propagation"
 Requires-Dist: stackprinter; extra == "tag-propagation"
+Requires-Dist: prometheus-client; extra == "tag-propagation"
+Requires-Dist: python-dateutil>=2.8.0; extra == "tag-propagation"
 Requires-Dist: click>=6.0.0; extra == "tag-propagation"
+Requires-Dist: toml>=0.10.0; extra == "tag-propagation"
 Requires-Dist: click-default-group; extra == "tag-propagation"
 Requires-Dist: progressbar2; extra == "tag-propagation"
-Requires-Dist: entrypoints; extra == "tag-propagation"
-Requires-Dist: PyYAML; extra == "tag-propagation"
-Requires-Dist: python-dateutil>=2.8.0; extra == "tag-propagation"
 Requires-Dist: tenacity; extra == "tag-propagation"
-Requires-Dist: prometheus-client; extra == "tag-propagation"
 Provides-Extra: term-propagation
-Requires-Dist: toml>=0.10.0; extra == "term-propagation"
+Requires-Dist: PyYAML; extra == "term-propagation"
+Requires-Dist: entrypoints; extra == "term-propagation"
 Requires-Dist: stackprinter; extra == "term-propagation"
+Requires-Dist: prometheus-client; extra == "term-propagation"
+Requires-Dist: python-dateutil>=2.8.0; extra == "term-propagation"
 Requires-Dist: click>=6.0.0; extra == "term-propagation"
+Requires-Dist: toml>=0.10.0; extra == "term-propagation"
 Requires-Dist: click-default-group; extra == "term-propagation"
 Requires-Dist: progressbar2; extra == "term-propagation"
-Requires-Dist: entrypoints; extra == "term-propagation"
-Requires-Dist: PyYAML; extra == "term-propagation"
-Requires-Dist: python-dateutil>=2.8.0; extra == "term-propagation"
 Requires-Dist: tenacity; extra == "term-propagation"
-Requires-Dist: prometheus-client; extra == "term-propagation"
 Provides-Extra: snowflake-tag-propagation
-Requires-Dist: toml>=0.10.0; extra == "snowflake-tag-propagation"
+Requires-Dist: PyYAML; extra == "snowflake-tag-propagation"
+Requires-Dist: entrypoints; extra == "snowflake-tag-propagation"
 Requires-Dist: stackprinter; extra == "snowflake-tag-propagation"
-Requires-Dist: acryl-datahub[snowflake-slim]==1.3.1.5rc9; extra == "snowflake-tag-propagation"
+Requires-Dist: prometheus-client; extra == "snowflake-tag-propagation"
+Requires-Dist: python-dateutil>=2.8.0; extra == "snowflake-tag-propagation"
 Requires-Dist: click>=6.0.0; extra == "snowflake-tag-propagation"
+Requires-Dist: toml>=0.10.0; extra == "snowflake-tag-propagation"
 Requires-Dist: click-default-group; extra == "snowflake-tag-propagation"
+Requires-Dist: acryl-datahub[snowflake-slim]==1.3.1.5rc10; extra == "snowflake-tag-propagation"
 Requires-Dist: progressbar2; extra == "snowflake-tag-propagation"
-Requires-Dist: entrypoints; extra == "snowflake-tag-propagation"
-Requires-Dist: PyYAML; extra == "snowflake-tag-propagation"
-Requires-Dist: python-dateutil>=2.8.0; extra == "snowflake-tag-propagation"
 Requires-Dist: tenacity; extra == "snowflake-tag-propagation"
-Requires-Dist: prometheus-client; extra == "snowflake-tag-propagation"
 Provides-Extra: doc-propagation
-Requires-Dist: toml>=0.10.0; extra == "doc-propagation"
+Requires-Dist: PyYAML; extra == "doc-propagation"
+Requires-Dist: entrypoints; extra == "doc-propagation"
 Requires-Dist: stackprinter; extra == "doc-propagation"
+Requires-Dist: prometheus-client; extra == "doc-propagation"
+Requires-Dist: python-dateutil>=2.8.0; extra == "doc-propagation"
 Requires-Dist: click>=6.0.0; extra == "doc-propagation"
+Requires-Dist: toml>=0.10.0; extra == "doc-propagation"
 Requires-Dist: click-default-group; extra == "doc-propagation"
 Requires-Dist: progressbar2; extra == "doc-propagation"
-Requires-Dist: entrypoints; extra == "doc-propagation"
-Requires-Dist: PyYAML; extra == "doc-propagation"
-Requires-Dist: python-dateutil>=2.8.0; extra == "doc-propagation"
 Requires-Dist: tenacity; extra == "doc-propagation"
-Requires-Dist: prometheus-client; extra == "doc-propagation"
+Provides-Extra: observability
+Requires-Dist: PyYAML; extra == "observability"
+Requires-Dist: entrypoints; extra == "observability"
+Requires-Dist: stackprinter; extra == "observability"
+Requires-Dist: prometheus-client; extra == "observability"
+Requires-Dist: python-dateutil>=2.8.0; extra == "observability"
+Requires-Dist: opentelemetry-sdk>=1.20.0; extra == "observability"
+Requires-Dist: click>=6.0.0; extra == "observability"
+Requires-Dist: toml>=0.10.0; extra == "observability"
+Requires-Dist: click-default-group; extra == "observability"
+Requires-Dist: progressbar2; extra == "observability"
+Requires-Dist: opentelemetry-api>=1.20.0; extra == "observability"
+Requires-Dist: tenacity; extra == "observability"
 Provides-Extra: all
-Requires-Dist: slack-bolt>=1.15.5; extra == "all"
-Requires-Dist: toml>=0.10.0; extra == "all"
+Requires-Dist: PyYAML; extra == "all"
+Requires-Dist: entrypoints; extra == "all"
 Requires-Dist: stackprinter; extra == "all"
+Requires-Dist: prometheus-client; extra == "all"
+Requires-Dist: python-dateutil>=2.8.0; extra == "all"
 Requires-Dist: acryl-executor==0.3.2; extra == "all"
-Requires-Dist: acryl-datahub[snowflake-slim]==1.3.1.5rc9; extra == "all"
+Requires-Dist: pymsteams>=0.2.2; extra == "all"
+Requires-Dist: opentelemetry-sdk>=1.20.0; extra == "all"
 Requires-Dist: click>=6.0.0; extra == "all"
+Requires-Dist: toml>=0.10.0; extra == "all"
 Requires-Dist: click-default-group; extra == "all"
-Requires-Dist: progressbar2; extra == "all"
-Requires-Dist: entrypoints; extra == "all"
-Requires-Dist: PyYAML; extra == "all"
 Requires-Dist: confluent-kafka[schemaregistry]; extra == "all"
-Requires-Dist: python-dateutil>=2.8.0; extra == "all"
+Requires-Dist: acryl-datahub[snowflake-slim]==1.3.1.5rc10; extra == "all"
+Requires-Dist: progressbar2; extra == "all"
+Requires-Dist: opentelemetry-api>=1.20.0; extra == "all"
+Requires-Dist: slack-bolt>=1.15.5; extra == "all"
 Requires-Dist: tenacity; extra == "all"
-Requires-Dist: pymsteams>=0.2.2; extra == "all"
-Requires-Dist: prometheus-client; extra == "all"
 Provides-Extra: dev
-Requires-Dist: toml>=0.10.0; extra == "dev"
-Requires-Dist: types-PyYAML; extra == "dev"
-Requires-Dist: acryl-executor==0.3.2; extra == "dev"
-Requires-Dist: sqlalchemy-stubs; extra == "dev"
-Requires-Dist: click-default-group; extra == "dev"
-Requires-Dist: types-python-dateutil; extra == "dev"
-Requires-Dist: pytest-docker>=0.10.3; extra == "dev"
-Requires-Dist: types-toml; extra == "dev"
-Requires-Dist: deepdiff; extra == "dev"
+Requires-Dist: h11>=0.16; extra == "dev"
+Requires-Dist: freezegun; extra == "dev"
+Requires-Dist: prometheus-client; extra == "dev"
+Requires-Dist: types-freezegun; extra == "dev"
+Requires-Dist: requests-mock; extra == "dev"
+Requires-Dist: types-dataclasses; extra == "dev"
+Requires-Dist: confluent-kafka[schemaregistry]; extra == "dev"
+Requires-Dist: pytest-dependency>=0.5.1; extra == "dev"
+Requires-Dist: types-cachetools; extra == "dev"
+Requires-Dist: ratelimit; extra == "dev"
+Requires-Dist: tenacity; extra == "dev"
+Requires-Dist: PyYAML; extra == "dev"
 Requires-Dist: types-requests; extra == "dev"
+Requires-Dist: types-setuptools; extra == "dev"
+Requires-Dist: python-dateutil>=2.8.0; extra == "dev"
+Requires-Dist: types-pytz; extra == "dev"
+Requires-Dist: httpcore>=1.0.9; extra == "dev"
+Requires-Dist: types-click==0.1.12; extra == "dev"
+Requires-Dist: aws-msk-iam-sasl-signer-python==1.0.2; extra == "dev"
+Requires-Dist: mypy==1.17.1; extra == "dev"
 Requires-Dist: types-six; extra == "dev"
-Requires-Dist: types-dataclasses; extra == "dev"
+Requires-Dist: build; extra == "dev"
+Requires-Dist: acryl-datahub[datahub-kafka]==1.3.1.5rc10; extra == "dev"
 Requires-Dist: types-PyMySQL; extra == "dev"
+Requires-Dist: typing-inspect; extra == "dev"
+Requires-Dist: entrypoints; extra == "dev"
 Requires-Dist: stackprinter; extra == "dev"
-Requires-Dist: build; extra == "dev"
-Requires-Dist: aws-msk-iam-sasl-signer-python==1.0.2; extra == "dev"
-Requires-Dist: PyYAML; extra == "dev"
-Requires-Dist: confluent-kafka[schemaregistry]; extra == "dev"
-Requires-Dist: pytest>=6.2.2; extra == "dev"
+Requires-Dist: pymsteams>=0.2.2; extra == "dev"
+Requires-Dist: ruff==0.11.7; extra == "dev"
+Requires-Dist: click>=6.0.0; extra == "dev"
+Requires-Dist: types-python-dateutil; extra == "dev"
+Requires-Dist: toml>=0.10.0; extra == "dev"
+Requires-Dist: click-default-group; extra == "dev"
+Requires-Dist: pydantic<3.0.0,>=2.0.0; extra == "dev"
+Requires-Dist: progressbar2; extra == "dev"
 Requires-Dist: pytest-cov>=2.8.1; extra == "dev"
-Requires-Dist: python-dateutil>=2.8.0; extra == "dev"
+Requires-Dist: types-toml; extra == "dev"
+Requires-Dist: pytest>=6.2.2; extra == "dev"
+Requires-Dist: acryl-executor==0.3.2; extra == "dev"
+Requires-Dist: deepdiff; extra == "dev"
 Requires-Dist: tox; extra == "dev"
-Requires-Dist: types-setuptools; extra == "dev"
-Requires-Dist: coverage>=5.1; extra == "dev"
 Requires-Dist: twine; extra == "dev"
-Requires-Dist: types-click==0.1.12; extra == "dev"
 Requires-Dist: jsonpickle; extra == "dev"
-Requires-Dist: types-pytz; extra == "dev"
-Requires-Dist: pytest-dependency>=0.5.1; extra == "dev"
-Requires-Dist: prometheus-client; extra == "dev"
-Requires-Dist: pydantic<3.0.0,>=2.0.0; extra == "dev"
+Requires-Dist: pytest-docker>=0.10.3; extra == "dev"
+Requires-Dist: acryl-datahub[snowflake-slim]==1.3.1.5rc10; extra == "dev"
+Requires-Dist: types-PyYAML; extra == "dev"
+Requires-Dist: coverage>=5.1; extra == "dev"
 Requires-Dist: slack-bolt>=1.15.5; extra == "dev"
-Requires-Dist: ruff==0.11.7; extra == "dev"
-Requires-Dist: types-cachetools; extra == "dev"
-Requires-Dist: click>=6.0.0; extra == "dev"
-Requires-Dist: progressbar2; extra == "dev"
-Requires-Dist: entrypoints; extra == "dev"
-Requires-Dist: requests-mock; extra == "dev"
-Requires-Dist: mypy==1.17.1; extra == "dev"
-Requires-Dist: acryl-datahub[datahub-kafka]==1.3.1.5rc9; extra == "dev"
-Requires-Dist: types-freezegun; extra == "dev"
-Requires-Dist: ratelimit; extra == "dev"
-Requires-Dist: typing-inspect; extra == "dev"
-Requires-Dist: h11>=0.16; extra == "dev"
-Requires-Dist: freezegun; extra == "dev"
-Requires-Dist: acryl-datahub[snowflake-slim]==1.3.1.5rc9; extra == "dev"
+Requires-Dist: sqlalchemy-stubs; extra == "dev"
 Requires-Dist: azure-identity==1.21.0; extra == "dev"
-Requires-Dist: httpcore>=1.0.9; extra == "dev"
-Requires-Dist: tenacity; extra == "dev"
-Requires-Dist: pymsteams>=0.2.2; extra == "dev"
 Provides-Extra: integration-tests
-Requires-Dist: slack-bolt>=1.15.5; extra == "integration-tests"
 Requires-Dist: acryl-executor==0.3.2; extra == "integration-tests"
-Requires-Dist: acryl-datahub[snowflake-slim]==1.3.1.5rc9; extra == "integration-tests"
-Requires-Dist: confluent-kafka[schemaregistry]; extra == "integration-tests"
 Requires-Dist: pymsteams>=0.2.2; extra == "integration-tests"
+Requires-Dist: confluent-kafka[schemaregistry]; extra == "integration-tests"
+Requires-Dist: acryl-datahub[snowflake-slim]==1.3.1.5rc10; extra == "integration-tests"
+Requires-Dist: slack-bolt>=1.15.5; extra == "integration-tests"
 Dynamic: classifier
 Dynamic: description
 Dynamic: description-content-type

{acryl_datahub_actions-1.3.1.5rc9.dist-info → acryl_datahub_actions-1.3.1.5rc10.dist-info}/RECORD

@@ -1,5 +1,5 @@
 datahub_actions/__init__.py,sha256=Pn9UTDbqYPt6jY_acE7MQIveX_Nzdfl5oGmi-Ze8CHs,647
-datahub_actions/_version.py,sha256=eWryIWAU9j5gdTM9B7DMIXhXxXht0_2bc4gKRc9cU60,339
+datahub_actions/_version.py,sha256=yYoKMr5I5pxzVQxB11xhjKWxbFgE0iycOeBEygdOepA,340
 datahub_actions/entrypoints.py,sha256=_6NOpKhlfXuSUdPhDpPya7d9kJmwoRGrunxcNPMQE9k,4743
 datahub_actions/action/__init__.py,sha256=KYWPHGi7sDM0DXrrXmhlR6_zhym1qNbtFhjYk1Ug6ss,579
 datahub_actions/action/action.py,sha256=ET1fpeRn6KVD9diJ9ZOObsojrN9y6Vfn4tK7jzBQKHg,1537
@@ -13,6 +13,8 @@ datahub_actions/event/__init__.py,sha256=KYWPHGi7sDM0DXrrXmhlR6_zhym1qNbtFhjYk1U
 datahub_actions/event/event.py,sha256=2Eyizwzbf3fXsUV4n5p7gsYZS_CjEE5y9m1YvkmKOKU,990
 datahub_actions/event/event_envelope.py,sha256=x1QfDetMM7k5SLecD0Nb-duxMxKWU0rmeLroScvkicY,2258
 datahub_actions/event/event_registry.py,sha256=bWV2n9u1n8p9Onu9G2AVgZIfOxCjaBT0pKg2eOQdaig,4663
+datahub_actions/observability/__init__.py,sha256=llKqfushdo7d6RNAtIx9ofS8fqFltea-BHkkzZCUyAA,643
+datahub_actions/observability/kafka_lag_monitor.py,sha256=UrGSC18iUqvKCpxN8HzZ5gpvDjdNtAwuDgXgd77pfpE,7865
 datahub_actions/pipeline/__init__.py,sha256=KYWPHGi7sDM0DXrrXmhlR6_zhym1qNbtFhjYk1Ug6ss,579
 datahub_actions/pipeline/pipeline.py,sha256=6Bod5W3QJNAV0kXymooBxxJVuvAYv3mpvAa6zp-9u5c,12194
 datahub_actions/pipeline/pipeline_config.py,sha256=6mJKK0J9EuXaxPVjl7UuE0qzACpPGjQLr6yNv_-O7Xg,2347
@@ -55,7 +57,7 @@ datahub_actions/plugin/source/acryl/datahub_cloud_events_ack_manager.py,sha256=k
 datahub_actions/plugin/source/acryl/datahub_cloud_events_consumer.py,sha256=1j6s-uTyZcRlF74toHTrdNeVNq5MMh9_Fp8FOsV-8a0,7116
 datahub_actions/plugin/source/acryl/datahub_cloud_events_consumer_offsets_store.py,sha256=5m_VR_5yHjt4_YZEi1die5sr0ngl9dVobX10AjIodvA,3969
 datahub_actions/plugin/source/kafka/__init__.py,sha256=KYWPHGi7sDM0DXrrXmhlR6_zhym1qNbtFhjYk1Ug6ss,579
-datahub_actions/plugin/source/kafka/kafka_event_source.py,sha256=0SunE49cKmqEGaazyYTYMVZ-NsIDP89QcBa0Xs-JkdQ,11594
+datahub_actions/plugin/source/kafka/kafka_event_source.py,sha256=5Nj4coUbboK7ZFtH0JzIvzzacdqWd9a1nHLBtpx3Uog,13380
 datahub_actions/plugin/source/kafka/utils.py,sha256=EEqBnv8Zd05zSg9T3f2FHaARaStD2j2M_xiSeaQBplA,758
 datahub_actions/plugin/transform/__init__.py,sha256=KYWPHGi7sDM0DXrrXmhlR6_zhym1qNbtFhjYk1Ug6ss,579
 datahub_actions/plugin/transform/filter/__init__.py,sha256=KYWPHGi7sDM0DXrrXmhlR6_zhym1qNbtFhjYk1Ug6ss,579
@@ -75,8 +77,8 @@ datahub_actions/utils/event_util.py,sha256=VluTOeyFcot48moK9qLmYL1ADAjsau0346Ngi
 datahub_actions/utils/kafka_msk_iam.py,sha256=JWg0MBEMcsG2AmW4yXiHvH_dnnsQDIRASdlvDXGTVcI,1013
 datahub_actions/utils/name_resolver.py,sha256=uXICSpy1IUe5uyFUiRk4vDQ9_G0JytPgKPSnqMA6fZk,10540
 datahub_actions/utils/social_util.py,sha256=FI_3qDjayX9LKlDjf43QHafnOznQk3v5Vp3Xyhq-lno,5271
-acryl_datahub_actions-1.3.1.5rc9.dist-info/METADATA,sha256=eBH8Q0aKNcELG4XylRTmF-cKWhZN47F_c0r4fjZw9_Y,18124
-acryl_datahub_actions-1.3.1.5rc9.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-acryl_datahub_actions-1.3.1.5rc9.dist-info/entry_points.txt,sha256=Gbvj36kOFWrsJ1meJVFB7zYgrKbIGgufOpZDurJbehU,866
-acryl_datahub_actions-1.3.1.5rc9.dist-info/top_level.txt,sha256=93StcIqRM0PfcJoT06TFhcCjPnIw-CyFgBaF-4vqCKY,16
-acryl_datahub_actions-1.3.1.5rc9.dist-info/RECORD,,
+acryl_datahub_actions-1.3.1.5rc10.dist-info/METADATA,sha256=dLcgOWiBJxMdiQqcG-pkqoufuxW4hG8wQka-9wJbzbs,18960
+acryl_datahub_actions-1.3.1.5rc10.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+acryl_datahub_actions-1.3.1.5rc10.dist-info/entry_points.txt,sha256=Gbvj36kOFWrsJ1meJVFB7zYgrKbIGgufOpZDurJbehU,866
+acryl_datahub_actions-1.3.1.5rc10.dist-info/top_level.txt,sha256=93StcIqRM0PfcJoT06TFhcCjPnIw-CyFgBaF-4vqCKY,16
+acryl_datahub_actions-1.3.1.5rc10.dist-info/RECORD,,

datahub_actions/_version.py

@@ -1,6 +1,6 @@
 # Published at https://pypi.org/project/acryl-datahub-actions/.
 __package_name__ = "acryl-datahub-actions"
-__version__ = "1.3.1.5rc9"
+__version__ = "1.3.1.5rc10"
 
 
 def is_dev_mode() -> bool:

datahub_actions/observability/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2021 Acryl Data, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Observability and metrics utilities for datahub-actions."""

datahub_actions/observability/kafka_lag_monitor.py

@@ -0,0 +1,230 @@
+# Copyright 2021 Acryl Data, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Background thread for monitoring Kafka consumer lag.
+
+This module provides a KafkaLagMonitor class that periodically calculates
+and reports Kafka consumer lag metrics to Prometheus.
+"""
+
+import logging
+import threading
+from dataclasses import dataclass
+from typing import Dict, Optional
+
+from confluent_kafka import Consumer, KafkaException, TopicPartition
+from prometheus_client import Gauge
+
+logger = logging.getLogger(__name__)
+
+# Prometheus metrics
+KAFKA_LAG_GAUGE = Gauge(
+    name="kafka_consumer_lag",
+    documentation="Kafka consumer lag aggregated per topic",
+    labelnames=["topic", "pipeline_name"],
+)
+
+
+@dataclass
+class LagStats:
+    """Statistics for a topic's consumer lag."""
+
+    topic: str
+    total_lag: int
+    partition_lags: Dict[int, int]  # partition_id -> lag
+
+
+class KafkaLagMonitor:
+    """Background thread that periodically reports Kafka consumer lag.
+
+    This monitor:
+    1. Queries assigned partitions from the Kafka consumer
+    2. Gets high water marks for each partition
+    3. Gets committed offsets for each partition
+    4. Calculates lag = high_water_mark - committed_offset
+    5. Aggregates per-topic lag (sum across partitions)
+    6. Updates Prometheus Gauge metrics
+    7. Optionally updates OpenTelemetry metrics if available
+    """
+
+    def __init__(
+        self,
+        consumer: Consumer,
+        pipeline_name: str,
+        interval_seconds: float = 30.0,
+        timeout_seconds: float = 5.0,
+    ):
+        """Initialize lag monitor.
+
+        Args:
+            consumer: confluent_kafka.Consumer instance to monitor
+            pipeline_name: Name of the action pipeline (for metric labels)
+            interval_seconds: How often to report lag (default: 30s)
+            timeout_seconds: Timeout for Kafka API calls (default: 5s)
+        """
+        self.consumer = consumer
+        self.pipeline_name = pipeline_name
+        self.interval_seconds = interval_seconds
+        self.timeout_seconds = timeout_seconds
+
+        self._stop_event = threading.Event()
+        self._thread: Optional[threading.Thread] = None
+
+    def start(self) -> None:
+        """Start the background monitoring thread."""
+        if self._thread is not None:
+            logger.warning("Lag monitor already started")
+            return
+
+        self._stop_event.clear()
+        self._thread = threading.Thread(
+            target=self._monitor_loop,
+            name=f"kafka-lag-monitor-{self.pipeline_name}",
+            daemon=True,  # Daemon thread exits when main thread exits
+        )
+        self._thread.start()
+        logger.info(
+            f"Kafka lag monitor started for pipeline '{self.pipeline_name}' "
+            f"(interval={self.interval_seconds}s)"
+        )
+
+    def stop(self) -> None:
+        """Stop the background monitoring thread."""
+        if self._thread is None:
+            return
+
+        logger.info(f"Stopping Kafka lag monitor for pipeline '{self.pipeline_name}'")
+        self._stop_event.set()
+        self._thread.join(timeout=10.0)
+        self._thread = None
+
+    def _monitor_loop(self) -> None:
+        """Main monitoring loop that runs in background thread."""
+        while not self._stop_event.is_set():
+            try:
+                self._collect_and_report_lag()
+            except Exception as e:
+                # Log error but don't crash - monitoring should be resilient
+                logger.error(
+                    f"Error collecting lag for pipeline '{self.pipeline_name}': {e}",
+                    exc_info=True,
+                )
+
+            # Sleep with interrupt support
+            self._stop_event.wait(timeout=self.interval_seconds)
+
+    def _collect_and_report_lag(self) -> None:
+        """Collect lag statistics and update metrics."""
+        # Get assigned partitions
+        assignment = self.consumer.assignment()
+        if not assignment:
+            logger.debug(f"No partitions assigned to pipeline '{self.pipeline_name}'")
+            return
+
+        # Group partitions by topic
+        topic_partitions: Dict[str, list[TopicPartition]] = {}
+        for tp in assignment:
+            if tp.topic not in topic_partitions:
+                topic_partitions[tp.topic] = []
+            topic_partitions[tp.topic].append(tp)
+
+        # Calculate lag per topic
+        for topic, partitions in topic_partitions.items():
+            lag_stats = self._calculate_topic_lag(topic, partitions)
+            if lag_stats:
+                self._report_lag(lag_stats)
+
+    def _calculate_topic_lag(
+        self, topic: str, partitions: list[TopicPartition]
+    ) -> Optional[LagStats]:
+        """Calculate lag for all partitions of a topic.
+
+        Args:
+            topic: Topic name
+            partitions: List of TopicPartition objects for this topic
+
+        Returns:
+            LagStats with aggregated lag, or None if calculation failed
+        """
+        partition_lags: Dict[int, int] = {}
+
+        # Get committed offsets for all partitions at once
+        try:
+            committed_partitions = self.consumer.committed(
+                partitions, timeout=self.timeout_seconds
+            )
+        except KafkaException as e:
+            logger.warning(f"Failed to get committed offsets for topic '{topic}': {e}")
+            return None
+
+        # Calculate lag for each partition
+        for tp in committed_partitions:
+            try:
+                # Get high water mark
+                watermarks = self.consumer.get_watermark_offsets(
+                    tp, timeout=self.timeout_seconds, cached=False
+                )
+                if watermarks is None:
+                    logger.warning(
+                        f"Failed to get watermarks for {topic}[{tp.partition}]"
+                    )
+                    continue
+
+                low, high = watermarks
+
+                # Calculate lag
+                if tp.offset < 0:
+                    # No committed offset yet - show total available messages as lag
+                    lag = high - low
+                else:
+                    # Normal case: lag = high water mark - committed offset
+                    lag = high - tp.offset
+
+                # Ensure non-negative lag
+                lag = max(0, lag)
+                partition_lags[tp.partition] = lag
+
+            except KafkaException as e:
+                logger.warning(
+                    f"Error calculating lag for {topic}[{tp.partition}]: {e}"
+                )
+                continue
+
+        if not partition_lags:
+            return None
+
+        total_lag = sum(partition_lags.values())
+        return LagStats(
+            topic=topic,
+            total_lag=total_lag,
+            partition_lags=partition_lags,
+        )
+
+    def _report_lag(self, lag_stats: LagStats) -> None:
+        """Report lag statistics to metrics backends.
+
+        Args:
+            lag_stats: Lag statistics to report
+        """
+        # Always update Prometheus (base requirement)
+        KAFKA_LAG_GAUGE.labels(
+            topic=lag_stats.topic,
+            pipeline_name=self.pipeline_name,
+        ).set(lag_stats.total_lag)
+
+        logger.debug(
+            f"Pipeline '{self.pipeline_name}' topic '{lag_stats.topic}': "
+            f"lag={lag_stats.total_lag} "
+            f"(partitions: {lag_stats.partition_lags})"
+        )

datahub_actions/plugin/source/kafka/kafka_event_source.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import logging
+import os
 from dataclasses import dataclass
 from typing import Any, Callable, Dict, Iterable, Optional
 
@@ -41,6 +42,7 @@ from datahub_actions.event.event_registry import (
 )
 
 # May or may not need these.
+from datahub_actions.observability.kafka_lag_monitor import KafkaLagMonitor
 from datahub_actions.pipeline.pipeline_context import PipelineContext
 from datahub_actions.plugin.source.kafka.utils import with_retry
 from datahub_actions.source.event_source import EventSource
@@ -124,6 +126,7 @@ def kafka_messages_observer(pipeline_name: str) -> Callable:
 class KafkaEventSource(EventSource):
     running = False
     source_config: KafkaEventSourceConfig
+    _lag_monitor: Optional[KafkaLagMonitor] = None
 
     def __init__(self, config: KafkaEventSourceConfig, ctx: PipelineContext):
         self.source_config = config
@@ -159,6 +162,41 @@ class KafkaEventSource(EventSource):
         )
         self._observe_message: Callable = kafka_messages_observer(ctx.pipeline_name)
 
+        # Initialize lag monitoring (if enabled)
+        if self._is_lag_monitoring_enabled():
+            lag_interval = float(
+                os.environ.get("DATAHUB_ACTIONS_KAFKA_LAG_INTERVAL_SECONDS", "30")
+            )
+            lag_timeout = float(
+                os.environ.get("DATAHUB_ACTIONS_KAFKA_LAG_TIMEOUT_SECONDS", "5")
+            )
+            self._lag_monitor = KafkaLagMonitor(
+                consumer=self.consumer,
+                pipeline_name=ctx.pipeline_name,
+                interval_seconds=lag_interval,
+                timeout_seconds=lag_timeout,
+            )
+            logger.info(
+                f"Kafka lag monitoring enabled for '{ctx.pipeline_name}' "
+                f"(interval={lag_interval}s, timeout={lag_timeout}s)"
+            )
+        else:
+            logger.debug(
+                f"Kafka lag monitoring disabled for pipeline '{ctx.pipeline_name}'"
+            )
+
+    @staticmethod
+    def _is_lag_monitoring_enabled() -> bool:
+        """Check if Kafka lag monitoring should be enabled.
+
+        Lag monitoring is enabled if:
+        1. DATAHUB_ACTIONS_KAFKA_LAG_ENABLED=true (case-insensitive)
+
+        Default: False (conservative default for OSS rollout)
+        """
+        enabled_str = os.environ.get("DATAHUB_ACTIONS_KAFKA_LAG_ENABLED", "false")
+        return enabled_str.lower() in ("true", "1", "yes")
+
     @classmethod
     def create(cls, config_dict: dict, ctx: PipelineContext) -> "EventSource":
         config = KafkaEventSourceConfig.model_validate(config_dict)
@@ -169,6 +207,11 @@ class KafkaEventSource(EventSource):
         topics_to_subscribe = list(topic_routes.values())
         logger.debug(f"Subscribing to the following topics: {topics_to_subscribe}")
         self.consumer.subscribe(topics_to_subscribe)
+
+        # Start lag monitoring after subscription
+        if self._lag_monitor is not None:
+            self._lag_monitor.start()
+
         self.running = True
         while self.running:
             try:
@@ -229,6 +272,11 @@ class KafkaEventSource(EventSource):
             yield EventEnvelope(RELATIONSHIP_CHANGE_EVENT_V1_TYPE, rce, kafka_meta)
 
     def close(self) -> None:
+        # Stop lag monitoring first
+        if self._lag_monitor is not None:
+            self._lag_monitor.stop()
+
+        # Then close consumer
         if self.consumer:
             self.running = False
             self.consumer.close()