acryl-datahub-actions 1.3.1.4__py3-none-any.whl → 1.3.1.5__py3-none-any.whl

{acryl_datahub_actions-1.3.1.4.dist-info → acryl_datahub_actions-1.3.1.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: acryl-datahub-actions
-Version: 1.3.1.4
+Version: 1.3.1.5
 Summary: An action framework to work with DataHub real time changes.
 Home-page: https://docs.datahub.com/
 License: Apache-2.0
@@ -21,201 +21,216 @@ Classifier: Environment :: MacOS X
 Classifier: Topic :: Software Development
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: h11>=0.16
-Requires-Dist: click-default-group
-Requires-Dist: typing-inspect
-Requires-Dist: toml>=0.10.0
-Requires-Dist: progressbar2
-Requires-Dist: azure-identity==1.21.0
+Requires-Dist: httpcore>=1.0.9
 Requires-Dist: python-dateutil>=2.8.0
-Requires-Dist: aws-msk-iam-sasl-signer-python==1.0.2
-Requires-Dist: stackprinter
 Requires-Dist: click>=6.0.0
+Requires-Dist: ratelimit
 Requires-Dist: entrypoints
-Requires-Dist: httpcore>=1.0.9
-Requires-Dist: pydantic<3.0.0,>=2.0.0
-Requires-Dist: PyYAML
+Requires-Dist: progressbar2
+Requires-Dist: h11>=0.16
 Requires-Dist: tenacity
-Requires-Dist: ratelimit
 Requires-Dist: prometheus-client
-Requires-Dist: acryl-datahub[datahub-kafka]==1.3.1.4
+Requires-Dist: PyYAML
+Requires-Dist: click-default-group
+Requires-Dist: toml>=0.10.0
+Requires-Dist: stackprinter
+Requires-Dist: pydantic<3.0.0,>=2.0.0
+Requires-Dist: typing-inspect
+Requires-Dist: acryl-datahub[datahub-kafka]==1.3.1.5
+Requires-Dist: azure-identity==1.21.0
+Requires-Dist: aws-msk-iam-sasl-signer-python==1.0.2
 Provides-Extra: base
-Requires-Dist: click-default-group; extra == "base"
-Requires-Dist: toml>=0.10.0; extra == "base"
-Requires-Dist: progressbar2; extra == "base"
 Requires-Dist: python-dateutil>=2.8.0; extra == "base"
-Requires-Dist: stackprinter; extra == "base"
 Requires-Dist: click>=6.0.0; extra == "base"
 Requires-Dist: entrypoints; extra == "base"
-Requires-Dist: PyYAML; extra == "base"
+Requires-Dist: progressbar2; extra == "base"
 Requires-Dist: tenacity; extra == "base"
 Requires-Dist: prometheus-client; extra == "base"
+Requires-Dist: PyYAML; extra == "base"
+Requires-Dist: click-default-group; extra == "base"
+Requires-Dist: toml>=0.10.0; extra == "base"
+Requires-Dist: stackprinter; extra == "base"
 Provides-Extra: kafka
-Requires-Dist: click-default-group; extra == "kafka"
-Requires-Dist: toml>=0.10.0; extra == "kafka"
-Requires-Dist: progressbar2; extra == "kafka"
 Requires-Dist: python-dateutil>=2.8.0; extra == "kafka"
-Requires-Dist: confluent-kafka[schemaregistry]; extra == "kafka"
-Requires-Dist: stackprinter; extra == "kafka"
 Requires-Dist: click>=6.0.0; extra == "kafka"
 Requires-Dist: entrypoints; extra == "kafka"
-Requires-Dist: PyYAML; extra == "kafka"
+Requires-Dist: progressbar2; extra == "kafka"
+Requires-Dist: confluent-kafka[schemaregistry]; extra == "kafka"
 Requires-Dist: tenacity; extra == "kafka"
 Requires-Dist: prometheus-client; extra == "kafka"
+Requires-Dist: PyYAML; extra == "kafka"
+Requires-Dist: click-default-group; extra == "kafka"
+Requires-Dist: toml>=0.10.0; extra == "kafka"
+Requires-Dist: stackprinter; extra == "kafka"
 Provides-Extra: executor
-Requires-Dist: click-default-group; extra == "executor"
-Requires-Dist: toml>=0.10.0; extra == "executor"
-Requires-Dist: progressbar2; extra == "executor"
-Requires-Dist: acryl-executor==0.3.2; extra == "executor"
 Requires-Dist: python-dateutil>=2.8.0; extra == "executor"
-Requires-Dist: stackprinter; extra == "executor"
 Requires-Dist: click>=6.0.0; extra == "executor"
 Requires-Dist: entrypoints; extra == "executor"
-Requires-Dist: PyYAML; extra == "executor"
+Requires-Dist: progressbar2; extra == "executor"
 Requires-Dist: tenacity; extra == "executor"
 Requires-Dist: prometheus-client; extra == "executor"
+Requires-Dist: PyYAML; extra == "executor"
+Requires-Dist: click-default-group; extra == "executor"
+Requires-Dist: toml>=0.10.0; extra == "executor"
+Requires-Dist: stackprinter; extra == "executor"
+Requires-Dist: acryl-executor==0.3.2; extra == "executor"
 Provides-Extra: slack
-Requires-Dist: click-default-group; extra == "slack"
-Requires-Dist: toml>=0.10.0; extra == "slack"
-Requires-Dist: progressbar2; extra == "slack"
 Requires-Dist: python-dateutil>=2.8.0; extra == "slack"
-Requires-Dist: slack-bolt>=1.15.5; extra == "slack"
-Requires-Dist: stackprinter; extra == "slack"
 Requires-Dist: click>=6.0.0; extra == "slack"
 Requires-Dist: entrypoints; extra == "slack"
-Requires-Dist: PyYAML; extra == "slack"
+Requires-Dist: progressbar2; extra == "slack"
 Requires-Dist: tenacity; extra == "slack"
 Requires-Dist: prometheus-client; extra == "slack"
+Requires-Dist: PyYAML; extra == "slack"
+Requires-Dist: click-default-group; extra == "slack"
+Requires-Dist: toml>=0.10.0; extra == "slack"
+Requires-Dist: stackprinter; extra == "slack"
+Requires-Dist: slack-bolt>=1.15.5; extra == "slack"
 Provides-Extra: teams
-Requires-Dist: click-default-group; extra == "teams"
-Requires-Dist: toml>=0.10.0; extra == "teams"
-Requires-Dist: progressbar2; extra == "teams"
 Requires-Dist: python-dateutil>=2.8.0; extra == "teams"
-Requires-Dist: stackprinter; extra == "teams"
 Requires-Dist: click>=6.0.0; extra == "teams"
 Requires-Dist: entrypoints; extra == "teams"
-Requires-Dist: PyYAML; extra == "teams"
-Requires-Dist: tenacity; extra == "teams"
+Requires-Dist: progressbar2; extra == "teams"
 Requires-Dist: pymsteams>=0.2.2; extra == "teams"
+Requires-Dist: tenacity; extra == "teams"
 Requires-Dist: prometheus-client; extra == "teams"
+Requires-Dist: PyYAML; extra == "teams"
+Requires-Dist: click-default-group; extra == "teams"
+Requires-Dist: toml>=0.10.0; extra == "teams"
+Requires-Dist: stackprinter; extra == "teams"
 Provides-Extra: tag-propagation
-Requires-Dist: click-default-group; extra == "tag-propagation"
-Requires-Dist: toml>=0.10.0; extra == "tag-propagation"
-Requires-Dist: progressbar2; extra == "tag-propagation"
 Requires-Dist: python-dateutil>=2.8.0; extra == "tag-propagation"
-Requires-Dist: stackprinter; extra == "tag-propagation"
 Requires-Dist: click>=6.0.0; extra == "tag-propagation"
 Requires-Dist: entrypoints; extra == "tag-propagation"
-Requires-Dist: PyYAML; extra == "tag-propagation"
+Requires-Dist: progressbar2; extra == "tag-propagation"
 Requires-Dist: tenacity; extra == "tag-propagation"
 Requires-Dist: prometheus-client; extra == "tag-propagation"
+Requires-Dist: PyYAML; extra == "tag-propagation"
+Requires-Dist: click-default-group; extra == "tag-propagation"
+Requires-Dist: toml>=0.10.0; extra == "tag-propagation"
+Requires-Dist: stackprinter; extra == "tag-propagation"
 Provides-Extra: term-propagation
-Requires-Dist: click-default-group; extra == "term-propagation"
-Requires-Dist: toml>=0.10.0; extra == "term-propagation"
-Requires-Dist: progressbar2; extra == "term-propagation"
 Requires-Dist: python-dateutil>=2.8.0; extra == "term-propagation"
-Requires-Dist: stackprinter; extra == "term-propagation"
 Requires-Dist: click>=6.0.0; extra == "term-propagation"
 Requires-Dist: entrypoints; extra == "term-propagation"
-Requires-Dist: PyYAML; extra == "term-propagation"
+Requires-Dist: progressbar2; extra == "term-propagation"
 Requires-Dist: tenacity; extra == "term-propagation"
 Requires-Dist: prometheus-client; extra == "term-propagation"
+Requires-Dist: PyYAML; extra == "term-propagation"
+Requires-Dist: click-default-group; extra == "term-propagation"
+Requires-Dist: toml>=0.10.0; extra == "term-propagation"
+Requires-Dist: stackprinter; extra == "term-propagation"
 Provides-Extra: snowflake-tag-propagation
-Requires-Dist: click-default-group; extra == "snowflake-tag-propagation"
-Requires-Dist: toml>=0.10.0; extra == "snowflake-tag-propagation"
-Requires-Dist: progressbar2; extra == "snowflake-tag-propagation"
 Requires-Dist: python-dateutil>=2.8.0; extra == "snowflake-tag-propagation"
-Requires-Dist: stackprinter; extra == "snowflake-tag-propagation"
-Requires-Dist: acryl-datahub[snowflake-slim]==1.3.1.4; extra == "snowflake-tag-propagation"
 Requires-Dist: click>=6.0.0; extra == "snowflake-tag-propagation"
 Requires-Dist: entrypoints; extra == "snowflake-tag-propagation"
-Requires-Dist: PyYAML; extra == "snowflake-tag-propagation"
+Requires-Dist: progressbar2; extra == "snowflake-tag-propagation"
 Requires-Dist: tenacity; extra == "snowflake-tag-propagation"
 Requires-Dist: prometheus-client; extra == "snowflake-tag-propagation"
+Requires-Dist: PyYAML; extra == "snowflake-tag-propagation"
+Requires-Dist: click-default-group; extra == "snowflake-tag-propagation"
+Requires-Dist: toml>=0.10.0; extra == "snowflake-tag-propagation"
+Requires-Dist: stackprinter; extra == "snowflake-tag-propagation"
+Requires-Dist: acryl-datahub[snowflake-slim]==1.3.1.5; extra == "snowflake-tag-propagation"
 Provides-Extra: doc-propagation
-Requires-Dist: click-default-group; extra == "doc-propagation"
-Requires-Dist: toml>=0.10.0; extra == "doc-propagation"
-Requires-Dist: progressbar2; extra == "doc-propagation"
 Requires-Dist: python-dateutil>=2.8.0; extra == "doc-propagation"
-Requires-Dist: stackprinter; extra == "doc-propagation"
 Requires-Dist: click>=6.0.0; extra == "doc-propagation"
 Requires-Dist: entrypoints; extra == "doc-propagation"
-Requires-Dist: PyYAML; extra == "doc-propagation"
+Requires-Dist: progressbar2; extra == "doc-propagation"
 Requires-Dist: tenacity; extra == "doc-propagation"
 Requires-Dist: prometheus-client; extra == "doc-propagation"
+Requires-Dist: PyYAML; extra == "doc-propagation"
+Requires-Dist: click-default-group; extra == "doc-propagation"
+Requires-Dist: toml>=0.10.0; extra == "doc-propagation"
+Requires-Dist: stackprinter; extra == "doc-propagation"
+Provides-Extra: observability
+Requires-Dist: python-dateutil>=2.8.0; extra == "observability"
+Requires-Dist: click>=6.0.0; extra == "observability"
+Requires-Dist: entrypoints; extra == "observability"
+Requires-Dist: progressbar2; extra == "observability"
+Requires-Dist: tenacity; extra == "observability"
+Requires-Dist: prometheus-client; extra == "observability"
+Requires-Dist: PyYAML; extra == "observability"
+Requires-Dist: click-default-group; extra == "observability"
+Requires-Dist: toml>=0.10.0; extra == "observability"
+Requires-Dist: stackprinter; extra == "observability"
+Requires-Dist: opentelemetry-api>=1.20.0; extra == "observability"
+Requires-Dist: opentelemetry-sdk>=1.20.0; extra == "observability"
 Provides-Extra: all
-Requires-Dist: click-default-group; extra == "all"
-Requires-Dist: toml>=0.10.0; extra == "all"
-Requires-Dist: progressbar2; extra == "all"
-Requires-Dist: acryl-executor==0.3.2; extra == "all"
 Requires-Dist: python-dateutil>=2.8.0; extra == "all"
-Requires-Dist: confluent-kafka[schemaregistry]; extra == "all"
-Requires-Dist: slack-bolt>=1.15.5; extra == "all"
-Requires-Dist: stackprinter; extra == "all"
-Requires-Dist: acryl-datahub[snowflake-slim]==1.3.1.4; extra == "all"
 Requires-Dist: click>=6.0.0; extra == "all"
 Requires-Dist: entrypoints; extra == "all"
-Requires-Dist: PyYAML; extra == "all"
-Requires-Dist: tenacity; extra == "all"
+Requires-Dist: progressbar2; extra == "all"
 Requires-Dist: pymsteams>=0.2.2; extra == "all"
+Requires-Dist: confluent-kafka[schemaregistry]; extra == "all"
+Requires-Dist: tenacity; extra == "all"
 Requires-Dist: prometheus-client; extra == "all"
+Requires-Dist: PyYAML; extra == "all"
+Requires-Dist: click-default-group; extra == "all"
+Requires-Dist: toml>=0.10.0; extra == "all"
+Requires-Dist: stackprinter; extra == "all"
+Requires-Dist: slack-bolt>=1.15.5; extra == "all"
+Requires-Dist: acryl-datahub[snowflake-slim]==1.3.1.5; extra == "all"
+Requires-Dist: opentelemetry-api>=1.20.0; extra == "all"
+Requires-Dist: opentelemetry-sdk>=1.20.0; extra == "all"
+Requires-Dist: acryl-executor==0.3.2; extra == "all"
 Provides-Extra: dev
-Requires-Dist: h11>=0.16; extra == "dev"
-Requires-Dist: typing-inspect; extra == "dev"
-Requires-Dist: toml>=0.10.0; extra == "dev"
-Requires-Dist: types-six; extra == "dev"
-Requires-Dist: aws-msk-iam-sasl-signer-python==1.0.2; extra == "dev"
-Requires-Dist: pydantic<3.0.0,>=2.0.0; extra == "dev"
-Requires-Dist: pytest>=6.2.2; extra == "dev"
-Requires-Dist: prometheus-client; extra == "dev"
+Requires-Dist: python-dateutil>=2.8.0; extra == "dev"
 Requires-Dist: mypy==1.17.1; extra == "dev"
-Requires-Dist: types-freezegun; extra == "dev"
-Requires-Dist: requests-mock; extra == "dev"
+Requires-Dist: types-toml; extra == "dev"
 Requires-Dist: types-dataclasses; extra == "dev"
-Requires-Dist: progressbar2; extra == "dev"
-Requires-Dist: azure-identity==1.21.0; extra == "dev"
-Requires-Dist: confluent-kafka[schemaregistry]; extra == "dev"
-Requires-Dist: build; extra == "dev"
-Requires-Dist: slack-bolt>=1.15.5; extra == "dev"
-Requires-Dist: jsonpickle; extra == "dev"
-Requires-Dist: acryl-datahub[snowflake-slim]==1.3.1.4; extra == "dev"
-Requires-Dist: entrypoints; extra == "dev"
-Requires-Dist: PyYAML; extra == "dev"
-Requires-Dist: ruff==0.11.7; extra == "dev"
-Requires-Dist: types-pytz; extra == "dev"
 Requires-Dist: click-default-group; extra == "dev"
 Requires-Dist: types-setuptools; extra == "dev"
-Requires-Dist: acryl-executor==0.3.2; extra == "dev"
-Requires-Dist: python-dateutil>=2.8.0; extra == "dev"
-Requires-Dist: types-PyYAML; extra == "dev"
-Requires-Dist: pytest-docker>=0.10.3; extra == "dev"
 Requires-Dist: deepdiff; extra == "dev"
-Requires-Dist: types-PyMySQL; extra == "dev"
-Requires-Dist: types-click==0.1.12; extra == "dev"
-Requires-Dist: click>=6.0.0; extra == "dev"
+Requires-Dist: pytest-cov>=2.8.1; extra == "dev"
+Requires-Dist: tox; extra == "dev"
+Requires-Dist: types-python-dateutil; extra == "dev"
+Requires-Dist: requests-mock; extra == "dev"
+Requires-Dist: freezegun; extra == "dev"
+Requires-Dist: progressbar2; extra == "dev"
+Requires-Dist: pymsteams>=0.2.2; extra == "dev"
+Requires-Dist: coverage>=5.1; extra == "dev"
+Requires-Dist: h11>=0.16; extra == "dev"
+Requires-Dist: types-pytz; extra == "dev"
 Requires-Dist: pytest-dependency>=0.5.1; extra == "dev"
-Requires-Dist: httpcore>=1.0.9; extra == "dev"
 Requires-Dist: tenacity; extra == "dev"
+Requires-Dist: slack-bolt>=1.15.5; extra == "dev"
+Requires-Dist: toml>=0.10.0; extra == "dev"
+Requires-Dist: azure-identity==1.21.0; extra == "dev"
+Requires-Dist: types-six; extra == "dev"
+Requires-Dist: types-PyYAML; extra == "dev"
+Requires-Dist: twine; extra == "dev"
 Requires-Dist: ratelimit; extra == "dev"
-Requires-Dist: pytest-cov>=2.8.1; extra == "dev"
-Requires-Dist: types-toml; extra == "dev"
-Requires-Dist: freezegun; extra == "dev"
+Requires-Dist: entrypoints; extra == "dev"
+Requires-Dist: ruff==0.11.7; extra == "dev"
+Requires-Dist: pytest>=6.2.2; extra == "dev"
+Requires-Dist: stackprinter; extra == "dev"
+Requires-Dist: pydantic<3.0.0,>=2.0.0; extra == "dev"
+Requires-Dist: types-click==0.1.12; extra == "dev"
+Requires-Dist: acryl-executor==0.3.2; extra == "dev"
 Requires-Dist: types-requests; extra == "dev"
+Requires-Dist: aws-msk-iam-sasl-signer-python==1.0.2; extra == "dev"
+Requires-Dist: httpcore>=1.0.9; extra == "dev"
+Requires-Dist: click>=6.0.0; extra == "dev"
+Requires-Dist: jsonpickle; extra == "dev"
+Requires-Dist: build; extra == "dev"
+Requires-Dist: types-PyMySQL; extra == "dev"
+Requires-Dist: prometheus-client; extra == "dev"
+Requires-Dist: confluent-kafka[schemaregistry]; extra == "dev"
+Requires-Dist: types-freezegun; extra == "dev"
+Requires-Dist: PyYAML; extra == "dev"
+Requires-Dist: acryl-datahub[snowflake-slim]==1.3.1.5; extra == "dev"
+Requires-Dist: typing-inspect; extra == "dev"
+Requires-Dist: pytest-docker>=0.10.3; extra == "dev"
+Requires-Dist: acryl-datahub[datahub-kafka]==1.3.1.5; extra == "dev"
 Requires-Dist: sqlalchemy-stubs; extra == "dev"
 Requires-Dist: types-cachetools; extra == "dev"
-Requires-Dist: stackprinter; extra == "dev"
-Requires-Dist: types-python-dateutil; extra == "dev"
-Requires-Dist: pymsteams>=0.2.2; extra == "dev"
-Requires-Dist: twine; extra == "dev"
-Requires-Dist: coverage>=5.1; extra == "dev"
-Requires-Dist: acryl-datahub[datahub-kafka]==1.3.1.4; extra == "dev"
-Requires-Dist: tox; extra == "dev"
 Provides-Extra: integration-tests
-Requires-Dist: acryl-executor==0.3.2; extra == "integration-tests"
+Requires-Dist: pymsteams>=0.2.2; extra == "integration-tests"
 Requires-Dist: confluent-kafka[schemaregistry]; extra == "integration-tests"
 Requires-Dist: slack-bolt>=1.15.5; extra == "integration-tests"
-Requires-Dist: acryl-datahub[snowflake-slim]==1.3.1.4; extra == "integration-tests"
-Requires-Dist: pymsteams>=0.2.2; extra == "integration-tests"
+Requires-Dist: acryl-datahub[snowflake-slim]==1.3.1.5; extra == "integration-tests"
+Requires-Dist: acryl-executor==0.3.2; extra == "integration-tests"
 Dynamic: classifier
 Dynamic: description
 Dynamic: description-content-type

{acryl_datahub_actions-1.3.1.4.dist-info → acryl_datahub_actions-1.3.1.5.dist-info}/RECORD

@@ -1,5 +1,5 @@
 datahub_actions/__init__.py,sha256=Pn9UTDbqYPt6jY_acE7MQIveX_Nzdfl5oGmi-Ze8CHs,647
-datahub_actions/_version.py,sha256=_vcRytCcexYixG4GcDRb-zlkn5jVenw_BKurlFMKTwk,336
+datahub_actions/_version.py,sha256=hJKt_5TzC3gmpk1gMO_bZIrwgZ5C_WNJH7va8i2jA6Q,336
 datahub_actions/entrypoints.py,sha256=_6NOpKhlfXuSUdPhDpPya7d9kJmwoRGrunxcNPMQE9k,4743
 datahub_actions/action/__init__.py,sha256=KYWPHGi7sDM0DXrrXmhlR6_zhym1qNbtFhjYk1Ug6ss,579
 datahub_actions/action/action.py,sha256=ET1fpeRn6KVD9diJ9ZOObsojrN9y6Vfn4tK7jzBQKHg,1537
@@ -13,9 +13,11 @@ datahub_actions/event/__init__.py,sha256=KYWPHGi7sDM0DXrrXmhlR6_zhym1qNbtFhjYk1U
 datahub_actions/event/event.py,sha256=2Eyizwzbf3fXsUV4n5p7gsYZS_CjEE5y9m1YvkmKOKU,990
 datahub_actions/event/event_envelope.py,sha256=x1QfDetMM7k5SLecD0Nb-duxMxKWU0rmeLroScvkicY,2258
 datahub_actions/event/event_registry.py,sha256=bWV2n9u1n8p9Onu9G2AVgZIfOxCjaBT0pKg2eOQdaig,4663
+datahub_actions/observability/__init__.py,sha256=llKqfushdo7d6RNAtIx9ofS8fqFltea-BHkkzZCUyAA,643
+datahub_actions/observability/kafka_lag_monitor.py,sha256=UrGSC18iUqvKCpxN8HzZ5gpvDjdNtAwuDgXgd77pfpE,7865
 datahub_actions/pipeline/__init__.py,sha256=KYWPHGi7sDM0DXrrXmhlR6_zhym1qNbtFhjYk1Ug6ss,579
 datahub_actions/pipeline/pipeline.py,sha256=6Bod5W3QJNAV0kXymooBxxJVuvAYv3mpvAa6zp-9u5c,12194
-datahub_actions/pipeline/pipeline_config.py,sha256=ikJYdDpBv0PI0lpbtubseh8SsuK0032i4Gb6Uum2jck,2208
+datahub_actions/pipeline/pipeline_config.py,sha256=6mJKK0J9EuXaxPVjl7UuE0qzACpPGjQLr6yNv_-O7Xg,2347
 datahub_actions/pipeline/pipeline_context.py,sha256=RollJBpjyw_BChsPPDaXC8_t97Di5eYaAYchGcBKWjs,964
 datahub_actions/pipeline/pipeline_manager.py,sha256=vFqcq1PY2SWtWZElntWzF8P_5wgmZBqTWjE_1NiHGyY,4039
 datahub_actions/pipeline/pipeline_stats.py,sha256=p6R6y4NlaHv6MKkMkAAM5ZgMye1wRK0y6Dlk5eCmbZ8,5062
@@ -33,12 +35,12 @@ datahub_actions/plugin/action/metadata_change_sync/metadata_change_sync.py,sha25
 datahub_actions/plugin/action/propagation/__init__.py,sha256=KYWPHGi7sDM0DXrrXmhlR6_zhym1qNbtFhjYk1Ug6ss,579
 datahub_actions/plugin/action/propagation/propagation_utils.py,sha256=411ojU1UzOLURecxCXhdcmi1v0tfuIQyLrRuwT1fpc4,10810
 datahub_actions/plugin/action/propagation/docs/__init__.py,sha256=KYWPHGi7sDM0DXrrXmhlR6_zhym1qNbtFhjYk1Ug6ss,579
-datahub_actions/plugin/action/propagation/docs/propagation_action.py,sha256=mtK-sZwERmqOe3xkwFqtelX3vWzF9NMQXZ-P-LK-vsI,37067
+datahub_actions/plugin/action/propagation/docs/propagation_action.py,sha256=Q1Fvb-1DAKS0Ktlpf0ydK1jXu14nSxYP-mWkdmhpo_M,37161
 datahub_actions/plugin/action/slack/__init__.py,sha256=KYWPHGi7sDM0DXrrXmhlR6_zhym1qNbtFhjYk1Ug6ss,579
 datahub_actions/plugin/action/slack/slack.py,sha256=YT9kTQsX0FzsB3pglaDl2kIFFnNF7cWx8HcC_xzq9eM,5175
 datahub_actions/plugin/action/snowflake/__init__.py,sha256=KYWPHGi7sDM0DXrrXmhlR6_zhym1qNbtFhjYk1Ug6ss,579
 datahub_actions/plugin/action/snowflake/snowflake_util.py,sha256=ZJW7Wsb65-3JrA15EVfv0ojgvFSC-sV0a_LZ9QyP_bw,5190
-datahub_actions/plugin/action/snowflake/tag_propagator.py,sha256=GgyQHnLtZcmUPyxCNO2eTziMhXizC1N3dkUP_EdxUaw,5052
+datahub_actions/plugin/action/snowflake/tag_propagator.py,sha256=fQ1BzarIQcI5QziydSleknIV-KQF2oEQ3M0KoE3WFLk,5110
 datahub_actions/plugin/action/tag/__init__.py,sha256=KYWPHGi7sDM0DXrrXmhlR6_zhym1qNbtFhjYk1Ug6ss,579
 datahub_actions/plugin/action/tag/tag_propagation_action.py,sha256=2iLG4oJURrpuwnS28ePDtBGKvvtk-2BWB8PHGLyttw4,6448
 datahub_actions/plugin/action/teams/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -50,16 +52,16 @@ datahub_actions/plugin/action/utils/term_resolver.py,sha256=J3_u-iGFXxsGCPtaeV_p
 datahub_actions/plugin/source/__init__.py,sha256=KYWPHGi7sDM0DXrrXmhlR6_zhym1qNbtFhjYk1Ug6ss,579
 datahub_actions/plugin/source/acryl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 datahub_actions/plugin/source/acryl/constants.py,sha256=f6vEsF6SdD0B1Vs90lCRglRQOCCYjiIlGanNtC7OsnY,307
-datahub_actions/plugin/source/acryl/datahub_cloud_event_source.py,sha256=izoNCkZa3DZwW_2wPBfVgXrfqPIFCoBUYFzw6OyIU6w,12538
+datahub_actions/plugin/source/acryl/datahub_cloud_event_source.py,sha256=61kyW--hZD6gzs3_OO5a0XcBNuhNkLBo_7h1oX-hBek,12814
 datahub_actions/plugin/source/acryl/datahub_cloud_events_ack_manager.py,sha256=ky15ibq5lfYdPIwufv4w92XOpp9C6cRvlhPRbJ_cs10,993
-datahub_actions/plugin/source/acryl/datahub_cloud_events_consumer.py,sha256=hjjDNC-fcv-JVWI5oC3KA3mueFKeoQv8MZsWBtrVkkc,6144
+datahub_actions/plugin/source/acryl/datahub_cloud_events_consumer.py,sha256=1j6s-uTyZcRlF74toHTrdNeVNq5MMh9_Fp8FOsV-8a0,7116
 datahub_actions/plugin/source/acryl/datahub_cloud_events_consumer_offsets_store.py,sha256=5m_VR_5yHjt4_YZEi1die5sr0ngl9dVobX10AjIodvA,3969
 datahub_actions/plugin/source/kafka/__init__.py,sha256=KYWPHGi7sDM0DXrrXmhlR6_zhym1qNbtFhjYk1Ug6ss,579
-datahub_actions/plugin/source/kafka/kafka_event_source.py,sha256=i3JqjXuAhjg0fwK99vbOH92sQHex8BzcplZDcrSclZ0,11545
+datahub_actions/plugin/source/kafka/kafka_event_source.py,sha256=5Nj4coUbboK7ZFtH0JzIvzzacdqWd9a1nHLBtpx3Uog,13380
 datahub_actions/plugin/source/kafka/utils.py,sha256=EEqBnv8Zd05zSg9T3f2FHaARaStD2j2M_xiSeaQBplA,758
 datahub_actions/plugin/transform/__init__.py,sha256=KYWPHGi7sDM0DXrrXmhlR6_zhym1qNbtFhjYk1Ug6ss,579
 datahub_actions/plugin/transform/filter/__init__.py,sha256=KYWPHGi7sDM0DXrrXmhlR6_zhym1qNbtFhjYk1Ug6ss,579
-datahub_actions/plugin/transform/filter/filter_transformer.py,sha256=aF5sTy5FuRV1odZckibLIR0uGYbLmehq5pRCZ2LYIwE,3136
+datahub_actions/plugin/transform/filter/filter_transformer.py,sha256=Dc5xeM9CPxqhvNP2bHPcDrJ6NFDI2KUjVm8Q4Lt-KqA,3186
 datahub_actions/source/__init__.py,sha256=KYWPHGi7sDM0DXrrXmhlR6_zhym1qNbtFhjYk1Ug6ss,579
 datahub_actions/source/event_source.py,sha256=_7ufs1WyeAFQ1b4BxGypvN8mpW006U2kA-WCyZndf7c,1947
 datahub_actions/source/event_source_registry.py,sha256=5Nmz8mNE1G_zJ8zJ-WnNt3JMrJdteuKHdr4mKe3t6Ig,1147
@@ -75,8 +77,8 @@ datahub_actions/utils/event_util.py,sha256=VluTOeyFcot48moK9qLmYL1ADAjsau0346Ngi
 datahub_actions/utils/kafka_msk_iam.py,sha256=JWg0MBEMcsG2AmW4yXiHvH_dnnsQDIRASdlvDXGTVcI,1013
 datahub_actions/utils/name_resolver.py,sha256=uXICSpy1IUe5uyFUiRk4vDQ9_G0JytPgKPSnqMA6fZk,10540
 datahub_actions/utils/social_util.py,sha256=FI_3qDjayX9LKlDjf43QHafnOznQk3v5Vp3Xyhq-lno,5271
-acryl_datahub_actions-1.3.1.4.dist-info/METADATA,sha256=BIKglhe6wTJTtwymoipYsHHe7s-ScXQKkOQ0iLwOwIs,18103
-acryl_datahub_actions-1.3.1.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-acryl_datahub_actions-1.3.1.4.dist-info/entry_points.txt,sha256=Gbvj36kOFWrsJ1meJVFB7zYgrKbIGgufOpZDurJbehU,866
-acryl_datahub_actions-1.3.1.4.dist-info/top_level.txt,sha256=93StcIqRM0PfcJoT06TFhcCjPnIw-CyFgBaF-4vqCKY,16
-acryl_datahub_actions-1.3.1.4.dist-info/RECORD,,
+acryl_datahub_actions-1.3.1.5.dist-info/METADATA,sha256=lqq2CebXYdT4MAVOEOtalLC2dsPgOmB8rjpi0YI_XAg,18932
+acryl_datahub_actions-1.3.1.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+acryl_datahub_actions-1.3.1.5.dist-info/entry_points.txt,sha256=Gbvj36kOFWrsJ1meJVFB7zYgrKbIGgufOpZDurJbehU,866
+acryl_datahub_actions-1.3.1.5.dist-info/top_level.txt,sha256=93StcIqRM0PfcJoT06TFhcCjPnIw-CyFgBaF-4vqCKY,16
+acryl_datahub_actions-1.3.1.5.dist-info/RECORD,,

datahub_actions/_version.py

@@ -1,6 +1,6 @@
 # Published at https://pypi.org/project/acryl-datahub-actions/.
 __package_name__ = "acryl-datahub-actions"
-__version__ = "1.3.1.4"
+__version__ = "1.3.1.5"
 
 
 def is_dev_mode() -> bool:

datahub_actions/observability/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2021 Acryl Data, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Observability and metrics utilities for datahub-actions."""

datahub_actions/observability/kafka_lag_monitor.py

@@ -0,0 +1,230 @@
+# Copyright 2021 Acryl Data, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Background thread for monitoring Kafka consumer lag.
+
+This module provides a KafkaLagMonitor class that periodically calculates
+and reports Kafka consumer lag metrics to Prometheus.
+"""
+
+import logging
+import threading
+from dataclasses import dataclass
+from typing import Dict, Optional
+
+from confluent_kafka import Consumer, KafkaException, TopicPartition
+from prometheus_client import Gauge
+
+logger = logging.getLogger(__name__)
+
+# Prometheus metrics
+KAFKA_LAG_GAUGE = Gauge(
+    name="kafka_consumer_lag",
+    documentation="Kafka consumer lag aggregated per topic",
+    labelnames=["topic", "pipeline_name"],
+)
+
+
+@dataclass
+class LagStats:
+    """Statistics for a topic's consumer lag."""
+
+    topic: str
+    total_lag: int
+    partition_lags: Dict[int, int]  # partition_id -> lag
+
+
+class KafkaLagMonitor:
+    """Background thread that periodically reports Kafka consumer lag.
+
+    This monitor:
+    1. Queries assigned partitions from the Kafka consumer
+    2. Gets high water marks for each partition
+    3. Gets committed offsets for each partition
+    4. Calculates lag = high_water_mark - committed_offset
+    5. Aggregates per-topic lag (sum across partitions)
+    6. Updates Prometheus Gauge metrics
+    7. Optionally updates OpenTelemetry metrics if available
+    """
+
+    def __init__(
+        self,
+        consumer: Consumer,
+        pipeline_name: str,
+        interval_seconds: float = 30.0,
+        timeout_seconds: float = 5.0,
+    ):
+        """Initialize lag monitor.
+
+        Args:
+            consumer: confluent_kafka.Consumer instance to monitor
+            pipeline_name: Name of the action pipeline (for metric labels)
+            interval_seconds: How often to report lag (default: 30s)
+            timeout_seconds: Timeout for Kafka API calls (default: 5s)
+        """
+        self.consumer = consumer
+        self.pipeline_name = pipeline_name
+        self.interval_seconds = interval_seconds
+        self.timeout_seconds = timeout_seconds
+
+        self._stop_event = threading.Event()
+        self._thread: Optional[threading.Thread] = None
+
+    def start(self) -> None:
+        """Start the background monitoring thread."""
+        if self._thread is not None:
+            logger.warning("Lag monitor already started")
+            return
+
+        self._stop_event.clear()
+        self._thread = threading.Thread(
+            target=self._monitor_loop,
+            name=f"kafka-lag-monitor-{self.pipeline_name}",
+            daemon=True,  # Daemon thread exits when main thread exits
+        )
+        self._thread.start()
+        logger.info(
+            f"Kafka lag monitor started for pipeline '{self.pipeline_name}' "
+            f"(interval={self.interval_seconds}s)"
+        )
+
+    def stop(self) -> None:
+        """Stop the background monitoring thread."""
+        if self._thread is None:
+            return
+
+        logger.info(f"Stopping Kafka lag monitor for pipeline '{self.pipeline_name}'")
+        self._stop_event.set()
+        self._thread.join(timeout=10.0)
+        self._thread = None
+
+    def _monitor_loop(self) -> None:
+        """Main monitoring loop that runs in background thread."""
+        while not self._stop_event.is_set():
+            try:
+                self._collect_and_report_lag()
+            except Exception as e:
+                # Log error but don't crash - monitoring should be resilient
+                logger.error(
+                    f"Error collecting lag for pipeline '{self.pipeline_name}': {e}",
+                    exc_info=True,
+                )
+
+            # Sleep with interrupt support
+            self._stop_event.wait(timeout=self.interval_seconds)
+
+    def _collect_and_report_lag(self) -> None:
+        """Collect lag statistics and update metrics."""
+        # Get assigned partitions
+        assignment = self.consumer.assignment()
+        if not assignment:
+            logger.debug(f"No partitions assigned to pipeline '{self.pipeline_name}'")
+            return
+
+        # Group partitions by topic
+        topic_partitions: Dict[str, list[TopicPartition]] = {}
+        for tp in assignment:
+            if tp.topic not in topic_partitions:
+                topic_partitions[tp.topic] = []
+            topic_partitions[tp.topic].append(tp)
+
+        # Calculate lag per topic
+        for topic, partitions in topic_partitions.items():
+            lag_stats = self._calculate_topic_lag(topic, partitions)
+            if lag_stats:
+                self._report_lag(lag_stats)
+
+    def _calculate_topic_lag(
+        self, topic: str, partitions: list[TopicPartition]
+    ) -> Optional[LagStats]:
+        """Calculate lag for all partitions of a topic.
+
+        Args:
+            topic: Topic name
+            partitions: List of TopicPartition objects for this topic
+
+        Returns:
+            LagStats with aggregated lag, or None if calculation failed
+        """
+        partition_lags: Dict[int, int] = {}
+
+        # Get committed offsets for all partitions at once
+        try:
+            committed_partitions = self.consumer.committed(
+                partitions, timeout=self.timeout_seconds
+            )
+        except KafkaException as e:
+            logger.warning(f"Failed to get committed offsets for topic '{topic}': {e}")
+            return None
+
+        # Calculate lag for each partition
+        for tp in committed_partitions:
+            try:
+                # Get high water mark
+                watermarks = self.consumer.get_watermark_offsets(
+                    tp, timeout=self.timeout_seconds, cached=False
+                )
+                if watermarks is None:
+                    logger.warning(
+                        f"Failed to get watermarks for {topic}[{tp.partition}]"
+                    )
+                    continue
+
+                low, high = watermarks
+
+                # Calculate lag
+                if tp.offset < 0:
+                    # No committed offset yet - show total available messages as lag
+                    lag = high - low
+                else:
+                    # Normal case: lag = high water mark - committed offset
+                    lag = high - tp.offset
+
+                # Ensure non-negative lag
+                lag = max(0, lag)
+                partition_lags[tp.partition] = lag
+
+            except KafkaException as e:
+                logger.warning(
+                    f"Error calculating lag for {topic}[{tp.partition}]: {e}"
+                )
+                continue
+
+        if not partition_lags:
+            return None
+
+        total_lag = sum(partition_lags.values())
+        return LagStats(
+            topic=topic,
+            total_lag=total_lag,
+            partition_lags=partition_lags,
+        )
+
+    def _report_lag(self, lag_stats: LagStats) -> None:
+        """Report lag statistics to metrics backends.
+
+        Args:
+            lag_stats: Lag statistics to report
+        """
+        # Always update Prometheus (base requirement)
+        KAFKA_LAG_GAUGE.labels(
+            topic=lag_stats.topic,
+            pipeline_name=self.pipeline_name,
+        ).set(lag_stats.total_lag)
+
+        logger.debug(
+            f"Pipeline '{self.pipeline_name}' topic '{lag_stats.topic}': "
+            f"lag={lag_stats.total_lag} "
+            f"(partitions: {lag_stats.partition_lags})"
+        )

datahub_actions/pipeline/pipeline_config.py

@@ -14,7 +14,7 @@
 
 from typing import Any, Dict, List, Optional, Union
 
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
 
 from datahub.configuration import ConfigModel
 from datahub.configuration.common import ConfigEnum
@@ -30,29 +30,29 @@ class FailureMode(ConfigEnum):
 
 class SourceConfig(ConfigModel):
     type: str
-    config: Optional[Dict[str, Any]] = None
+    config: Optional[Dict[str, Any]] = Field(default=None)
 
 
 class TransformConfig(ConfigModel):
     type: str
-    config: Optional[Dict[str, Any]] = None
+    config: Optional[Dict[str, Any]] = Field(default=None)
 
 
 class FilterConfig(ConfigModel):
     event_type: Union[str, List[str]]
-    event: Optional[Dict[str, Any]] = None
+    event: Optional[Dict[str, Any]] = Field(default=None)
 
 
 class ActionConfig(ConfigModel):
     type: str
-    config: Optional[dict]
+    config: Optional[Dict[str, Any]] = Field(default=None)
 
 
 class PipelineOptions(BaseModel):
-    retry_count: Optional[int] = None
-    failure_mode: Optional[FailureMode] = None
-    failed_events_dir: Optional[str] = (
-        None  # The path where failed events should be logged.
+    retry_count: Optional[int] = Field(default=None)
+    failure_mode: Optional[FailureMode] = Field(default=None)
+    failed_events_dir: Optional[str] = Field(
+        default=None, description="The path where failed events should be logged."
     )
 
 

datahub_actions/plugin/action/propagation/docs/propagation_action.py

@@ -132,10 +132,11 @@ class DocPropagationAction(Action):
     def __init__(self, config: DocPropagationConfig, ctx: PipelineContext):
         super().__init__()
         self.action_urn: str
-        if not ctx.pipeline_name.startswith("urn:li:dataHubAction"):
-            self.action_urn = f"urn:li:dataHubAction:{ctx.pipeline_name}"
+        if "urn:li:dataHubAction:" in ctx.pipeline_name:
+            action_urn_part = ctx.pipeline_name.split("urn:li:dataHubAction:")[1]
+            self.action_urn = f"urn:li:dataHubAction:{action_urn_part}"
         else:
-            self.action_urn = ctx.pipeline_name
+            self.action_urn = f"urn:li:dataHubAction:{ctx.pipeline_name}"
 
         self.config: DocPropagationConfig = config
         self.last_config_refresh: float = 0

datahub_actions/plugin/action/snowflake/tag_propagator.py

@@ -15,6 +15,8 @@
 import logging
 from typing import Optional
 
+from pydantic import Field
+
 from datahub.configuration.common import ConfigModel
 from datahub.ingestion.source.snowflake.snowflake_config import SnowflakeV2Config
 from datahub_actions.action.action import Action
@@ -36,8 +38,8 @@ logger = logging.getLogger(__name__)
 
 class SnowflakeTagPropagatorConfig(ConfigModel):
     snowflake: SnowflakeV2Config
-    tag_propagation: Optional[TagPropagationConfig] = None
-    term_propagation: Optional[TermPropagationConfig] = None
+    tag_propagation: Optional[TagPropagationConfig] = Field(default=None)
+    term_propagation: Optional[TermPropagationConfig] = Field(default=None)
 
 
 class SnowflakeTagPropagatorAction(Action):

datahub_actions/plugin/source/acryl/datahub_cloud_event_source.py

@@ -4,6 +4,8 @@ import time
 from dataclasses import dataclass
 from typing import Dict, Iterable, List, Optional, Union, cast
 
+from pydantic import Field
+
 from datahub.configuration import ConfigModel
 from datahub.emitter.serialization_helper import post_json_transform
 from datahub.ingestion.graph.client import DataHubGraph
@@ -59,9 +61,12 @@ def build_metadata_change_log_event(msg: ExternalEvent) -> MetadataChangeLogEven
 
 class DataHubEventsSourceConfig(ConfigModel):
     topics: Union[str, List[str]] = PLATFORM_EVENT_TOPIC_NAME
-    consumer_id: Optional[str] = None  # Used to store offset for the consumer.
-    lookback_days: Optional[int] = None
+    consumer_id: Optional[str] = Field(
+        default=None, description="Used to store offset for the consumer."
+    )
+    lookback_days: Optional[int] = Field(default=None)
     reset_offsets: Optional[bool] = False
+    infinite_retry: Optional[bool] = False
 
     # Time and Exit Conditions.
     kill_after_idle_timeout: bool = False
@@ -106,6 +111,7 @@ class DataHubEventSource(EventSource):
             graph=self.ctx.graph.graph,
             lookback_days=self.source_config.lookback_days,
             reset_offsets=self.source_config.reset_offsets,
+            infinite_retry=self.source_config.infinite_retry,
         )
 
         self.ack_manager = AckManager()
@@ -120,6 +126,7 @@ class DataHubEventSource(EventSource):
         graph: DataHubGraph,
         lookback_days: Optional[int],
         reset_offsets: Optional[bool],
+        infinite_retry: Optional[bool],
     ) -> Dict[str, DataHubEventsConsumer]:
         """
         Initialize DataHub consumers for each topic with appropriate consumer IDs.
@@ -156,6 +163,7 @@ class DataHubEventSource(EventSource):
                 consumer_id=topic_consumer_id,
                 lookback_days=lookback_days,
                 reset_offsets=reset_offsets,
+                infinite_retry=infinite_retry,
             )
 
         return topic_consumers

datahub_actions/plugin/source/acryl/datahub_cloud_events_consumer.py

@@ -11,9 +11,10 @@ from requests.exceptions import (
     Timeout,
 )
 from tenacity import (
-    retry,
+    Retrying,
     retry_if_exception_type,
     stop_after_attempt,
+    stop_never,
     wait_exponential,
 )
 
@@ -49,6 +50,7 @@ class DataHubEventsConsumer:
         offset_id: Optional[str] = None,
         lookback_days: Optional[int] = None,
         reset_offsets: Optional[bool] = False,
+        infinite_retry: Optional[bool] = False,
     ):
         # 1) Always set self.consumer_id, even if None, so tests can assert it safely.
         self.consumer_id: Optional[str] = consumer_id
@@ -57,6 +59,9 @@ class DataHubEventsConsumer:
         self.graph: DataHubGraph = graph
         self.offset_id: Optional[str] = offset_id
         self.default_lookback_days: Optional[int] = lookback_days
+        self.infinite_retry: bool = (
+            infinite_retry if infinite_retry is not None else False
+        )
         self.offsets_store: Optional[
             DataHubEventsConsumerPlatformResourceOffsetsStore
         ] = None
@@ -83,14 +88,6 @@ class DataHubEventsConsumer:
         else:
             logger.debug("Starting DataHub Events Consumer with no consumer ID.")
 
-    @retry(
-        retry=retry_if_exception_type(
-            (HTTPError, ConnectionError, ChunkedEncodingError, Timeout)
-        ),
-        wait=wait_exponential(multiplier=1, min=2, max=30),
-        stop=stop_after_attempt(3),
-        reraise=True,
-    )
     def poll_events(
         self,
         topic: str,
@@ -101,6 +98,36 @@ class DataHubEventsConsumer:
         """
         Fetch events for a specific topic.
         """
+        stop_condition = stop_never if self.infinite_retry else stop_after_attempt(15)
+
+        retry_strategy = Retrying(
+            retry=retry_if_exception_type(
+                (HTTPError, ConnectionError, ChunkedEncodingError, Timeout)
+            ),
+            wait=wait_exponential(multiplier=1, min=2, max=60),
+            stop=stop_condition,
+            reraise=True,
+        )
+
+        for attempt in retry_strategy:
+            with attempt:
+                return self._poll_events_impl(
+                    topic, offset_id, limit, poll_timeout_seconds
+                )
+
+        # This should never be reached due to reraise=True, but mypy needs it
+        raise RuntimeError("Retry strategy exhausted without returning or raising")
+
+    def _poll_events_impl(
+        self,
+        topic: str,
+        offset_id: Optional[str] = None,
+        limit: Optional[int] = None,
+        poll_timeout_seconds: Optional[int] = None,
+    ) -> ExternalEventsResponse:
+        """
+        Internal implementation of poll_events.
+        """
         endpoint = f"{self.base_url}/v1/events/poll"
 
         # If the caller provided an offset_id, use it; otherwise fall back to self.offset_id.

datahub_actions/plugin/source/kafka/kafka_event_source.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import logging
+import os
 from dataclasses import dataclass
 from typing import Any, Callable, Dict, Iterable, Optional
 
@@ -22,6 +23,7 @@ from confluent_kafka import KafkaError, KafkaException, TopicPartition
 from confluent_kafka.schema_registry.avro import AvroDeserializer
 from confluent_kafka.schema_registry.schema_registry_client import SchemaRegistryClient
 from prometheus_client import Counter, Gauge
+from pydantic import Field
 
 from datahub.configuration import ConfigModel
 from datahub.configuration.kafka import KafkaConsumerConnectionConfig
@@ -40,6 +42,7 @@ from datahub_actions.event.event_registry import (
 )
 
 # May or may not need these.
+from datahub_actions.observability.kafka_lag_monitor import KafkaLagMonitor
 from datahub_actions.pipeline.pipeline_context import PipelineContext
 from datahub_actions.plugin.source.kafka.utils import with_retry
 from datahub_actions.source.event_source import EventSource
@@ -94,7 +97,7 @@ def build_entity_change_event(payload: GenericPayloadClass) -> EntityChangeEvent
 
 class KafkaEventSourceConfig(ConfigModel):
     connection: KafkaConsumerConnectionConfig = KafkaConsumerConnectionConfig()
-    topic_routes: Optional[Dict[str, str]]
+    topic_routes: Optional[Dict[str, str]] = Field(default=None)
     async_commit_enabled: bool = False
     async_commit_interval: int = 10000
     commit_retry_count: int = 5
@@ -123,6 +126,7 @@ def kafka_messages_observer(pipeline_name: str) -> Callable:
 class KafkaEventSource(EventSource):
     running = False
     source_config: KafkaEventSourceConfig
+    _lag_monitor: Optional[KafkaLagMonitor] = None
 
     def __init__(self, config: KafkaEventSourceConfig, ctx: PipelineContext):
         self.source_config = config
@@ -158,6 +162,41 @@ class KafkaEventSource(EventSource):
         )
         self._observe_message: Callable = kafka_messages_observer(ctx.pipeline_name)
 
+        # Initialize lag monitoring (if enabled)
+        if self._is_lag_monitoring_enabled():
+            lag_interval = float(
+                os.environ.get("DATAHUB_ACTIONS_KAFKA_LAG_INTERVAL_SECONDS", "30")
+            )
+            lag_timeout = float(
+                os.environ.get("DATAHUB_ACTIONS_KAFKA_LAG_TIMEOUT_SECONDS", "5")
+            )
+            self._lag_monitor = KafkaLagMonitor(
+                consumer=self.consumer,
+                pipeline_name=ctx.pipeline_name,
+                interval_seconds=lag_interval,
+                timeout_seconds=lag_timeout,
+            )
+            logger.info(
+                f"Kafka lag monitoring enabled for '{ctx.pipeline_name}' "
+                f"(interval={lag_interval}s, timeout={lag_timeout}s)"
+            )
+        else:
+            logger.debug(
+                f"Kafka lag monitoring disabled for pipeline '{ctx.pipeline_name}'"
+            )
+
+    @staticmethod
+    def _is_lag_monitoring_enabled() -> bool:
+        """Check if Kafka lag monitoring should be enabled.
+
+        Lag monitoring is enabled if:
+        1. DATAHUB_ACTIONS_KAFKA_LAG_ENABLED=true (case-insensitive)
+
+        Default: False (conservative default for OSS rollout)
+        """
+        enabled_str = os.environ.get("DATAHUB_ACTIONS_KAFKA_LAG_ENABLED", "false")
+        return enabled_str.lower() in ("true", "1", "yes")
+
     @classmethod
     def create(cls, config_dict: dict, ctx: PipelineContext) -> "EventSource":
         config = KafkaEventSourceConfig.model_validate(config_dict)
@@ -168,6 +207,11 @@ class KafkaEventSource(EventSource):
         topics_to_subscribe = list(topic_routes.values())
         logger.debug(f"Subscribing to the following topics: {topics_to_subscribe}")
         self.consumer.subscribe(topics_to_subscribe)
+
+        # Start lag monitoring after subscription
+        if self._lag_monitor is not None:
+            self._lag_monitor.start()
+
         self.running = True
         while self.running:
             try:
@@ -228,6 +272,11 @@ class KafkaEventSource(EventSource):
             yield EventEnvelope(RELATIONSHIP_CHANGE_EVENT_V1_TYPE, rce, kafka_meta)
 
     def close(self) -> None:
+        # Stop lag monitoring first
+        if self._lag_monitor is not None:
+            self._lag_monitor.stop()
+
+        # Then close consumer
         if self.consumer:
             self.running = False
             self.consumer.close()

datahub_actions/plugin/transform/filter/filter_transformer.py

@@ -16,6 +16,8 @@ import json
 import logging
 from typing import Any, Dict, List, Optional, Union
 
+from pydantic import Field
+
 from datahub.configuration import ConfigModel
 from datahub_actions.event.event_envelope import EventEnvelope
 from datahub_actions.pipeline.pipeline_context import PipelineContext
@@ -26,7 +28,7 @@ logger = logging.getLogger(__name__)
 
 class FilterTransformerConfig(ConfigModel):
     event_type: Union[str, List[str]]
-    event: Optional[Dict[str, Any]]
+    event: Optional[Dict[str, Any]] = Field(default=None)
 
 
 class FilterTransformer(Transformer):