acryl-datahub-actions 1.3.1.5__py3-none-any.whl → 1.3.1.5rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {acryl_datahub_actions-1.3.1.5.dist-info → acryl_datahub_actions-1.3.1.5rc1.dist-info}/METADATA +99 -114
- {acryl_datahub_actions-1.3.1.5.dist-info → acryl_datahub_actions-1.3.1.5rc1.dist-info}/RECORD +12 -14
- datahub_actions/_version.py +1 -1
- datahub_actions/pipeline/pipeline_config.py +9 -9
- datahub_actions/plugin/action/propagation/docs/propagation_action.py +3 -4
- datahub_actions/plugin/action/snowflake/tag_propagator.py +2 -4
- datahub_actions/plugin/source/acryl/datahub_cloud_event_source.py +2 -6
- datahub_actions/plugin/source/kafka/kafka_event_source.py +1 -50
- datahub_actions/plugin/transform/filter/filter_transformer.py +1 -3
- datahub_actions/observability/__init__.py +0 -15
- datahub_actions/observability/kafka_lag_monitor.py +0 -230
- {acryl_datahub_actions-1.3.1.5.dist-info → acryl_datahub_actions-1.3.1.5rc1.dist-info}/WHEEL +0 -0
- {acryl_datahub_actions-1.3.1.5.dist-info → acryl_datahub_actions-1.3.1.5rc1.dist-info}/entry_points.txt +0 -0
- {acryl_datahub_actions-1.3.1.5.dist-info → acryl_datahub_actions-1.3.1.5rc1.dist-info}/top_level.txt +0 -0
{acryl_datahub_actions-1.3.1.5.dist-info → acryl_datahub_actions-1.3.1.5rc1.dist-info}/METADATA
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: acryl-datahub-actions
|
|
3
|
-
Version: 1.3.1.
|
|
3
|
+
Version: 1.3.1.5rc1
|
|
4
4
|
Summary: An action framework to work with DataHub real time changes.
|
|
5
5
|
Home-page: https://docs.datahub.com/
|
|
6
6
|
License: Apache-2.0
|
|
@@ -21,216 +21,201 @@ Classifier: Environment :: MacOS X
|
|
|
21
21
|
Classifier: Topic :: Software Development
|
|
22
22
|
Requires-Python: >=3.9
|
|
23
23
|
Description-Content-Type: text/markdown
|
|
24
|
-
Requires-Dist: httpcore>=1.0.9
|
|
25
|
-
Requires-Dist: python-dateutil>=2.8.0
|
|
26
|
-
Requires-Dist: click>=6.0.0
|
|
27
24
|
Requires-Dist: ratelimit
|
|
25
|
+
Requires-Dist: python-dateutil>=2.8.0
|
|
26
|
+
Requires-Dist: acryl-datahub[datahub-kafka]==1.3.1.5rc1
|
|
28
27
|
Requires-Dist: entrypoints
|
|
29
|
-
Requires-Dist:
|
|
30
|
-
Requires-Dist:
|
|
28
|
+
Requires-Dist: azure-identity==1.21.0
|
|
29
|
+
Requires-Dist: PyYAML
|
|
30
|
+
Requires-Dist: toml>=0.10.0
|
|
31
31
|
Requires-Dist: tenacity
|
|
32
|
+
Requires-Dist: progressbar2
|
|
32
33
|
Requires-Dist: prometheus-client
|
|
33
|
-
Requires-Dist: PyYAML
|
|
34
34
|
Requires-Dist: click-default-group
|
|
35
|
-
Requires-Dist: toml>=0.10.0
|
|
36
|
-
Requires-Dist: stackprinter
|
|
37
35
|
Requires-Dist: pydantic<3.0.0,>=2.0.0
|
|
36
|
+
Requires-Dist: httpcore>=1.0.9
|
|
38
37
|
Requires-Dist: typing-inspect
|
|
39
|
-
Requires-Dist:
|
|
40
|
-
Requires-Dist:
|
|
38
|
+
Requires-Dist: stackprinter
|
|
39
|
+
Requires-Dist: h11>=0.16
|
|
40
|
+
Requires-Dist: click>=6.0.0
|
|
41
41
|
Requires-Dist: aws-msk-iam-sasl-signer-python==1.0.2
|
|
42
42
|
Provides-Extra: base
|
|
43
43
|
Requires-Dist: python-dateutil>=2.8.0; extra == "base"
|
|
44
|
-
Requires-Dist: click>=6.0.0; extra == "base"
|
|
45
44
|
Requires-Dist: entrypoints; extra == "base"
|
|
46
|
-
Requires-Dist:
|
|
45
|
+
Requires-Dist: PyYAML; extra == "base"
|
|
46
|
+
Requires-Dist: toml>=0.10.0; extra == "base"
|
|
47
47
|
Requires-Dist: tenacity; extra == "base"
|
|
48
|
+
Requires-Dist: progressbar2; extra == "base"
|
|
48
49
|
Requires-Dist: prometheus-client; extra == "base"
|
|
49
|
-
Requires-Dist: PyYAML; extra == "base"
|
|
50
50
|
Requires-Dist: click-default-group; extra == "base"
|
|
51
|
-
Requires-Dist: toml>=0.10.0; extra == "base"
|
|
52
51
|
Requires-Dist: stackprinter; extra == "base"
|
|
52
|
+
Requires-Dist: click>=6.0.0; extra == "base"
|
|
53
53
|
Provides-Extra: kafka
|
|
54
54
|
Requires-Dist: python-dateutil>=2.8.0; extra == "kafka"
|
|
55
|
-
Requires-Dist: click>=6.0.0; extra == "kafka"
|
|
56
55
|
Requires-Dist: entrypoints; extra == "kafka"
|
|
57
|
-
Requires-Dist:
|
|
58
|
-
Requires-Dist:
|
|
56
|
+
Requires-Dist: PyYAML; extra == "kafka"
|
|
57
|
+
Requires-Dist: toml>=0.10.0; extra == "kafka"
|
|
59
58
|
Requires-Dist: tenacity; extra == "kafka"
|
|
59
|
+
Requires-Dist: progressbar2; extra == "kafka"
|
|
60
60
|
Requires-Dist: prometheus-client; extra == "kafka"
|
|
61
|
-
Requires-Dist: PyYAML; extra == "kafka"
|
|
62
61
|
Requires-Dist: click-default-group; extra == "kafka"
|
|
63
|
-
Requires-Dist: toml>=0.10.0; extra == "kafka"
|
|
64
62
|
Requires-Dist: stackprinter; extra == "kafka"
|
|
63
|
+
Requires-Dist: confluent-kafka[schemaregistry]; extra == "kafka"
|
|
64
|
+
Requires-Dist: click>=6.0.0; extra == "kafka"
|
|
65
65
|
Provides-Extra: executor
|
|
66
66
|
Requires-Dist: python-dateutil>=2.8.0; extra == "executor"
|
|
67
|
-
Requires-Dist: click>=6.0.0; extra == "executor"
|
|
68
67
|
Requires-Dist: entrypoints; extra == "executor"
|
|
69
|
-
Requires-Dist:
|
|
68
|
+
Requires-Dist: PyYAML; extra == "executor"
|
|
69
|
+
Requires-Dist: toml>=0.10.0; extra == "executor"
|
|
70
70
|
Requires-Dist: tenacity; extra == "executor"
|
|
71
|
+
Requires-Dist: progressbar2; extra == "executor"
|
|
72
|
+
Requires-Dist: acryl-executor==0.3.2; extra == "executor"
|
|
71
73
|
Requires-Dist: prometheus-client; extra == "executor"
|
|
72
|
-
Requires-Dist: PyYAML; extra == "executor"
|
|
73
74
|
Requires-Dist: click-default-group; extra == "executor"
|
|
74
|
-
Requires-Dist: toml>=0.10.0; extra == "executor"
|
|
75
75
|
Requires-Dist: stackprinter; extra == "executor"
|
|
76
|
-
Requires-Dist:
|
|
76
|
+
Requires-Dist: click>=6.0.0; extra == "executor"
|
|
77
77
|
Provides-Extra: slack
|
|
78
78
|
Requires-Dist: python-dateutil>=2.8.0; extra == "slack"
|
|
79
|
-
Requires-Dist: click>=6.0.0; extra == "slack"
|
|
80
79
|
Requires-Dist: entrypoints; extra == "slack"
|
|
81
|
-
Requires-Dist:
|
|
80
|
+
Requires-Dist: PyYAML; extra == "slack"
|
|
81
|
+
Requires-Dist: toml>=0.10.0; extra == "slack"
|
|
82
82
|
Requires-Dist: tenacity; extra == "slack"
|
|
83
|
+
Requires-Dist: progressbar2; extra == "slack"
|
|
83
84
|
Requires-Dist: prometheus-client; extra == "slack"
|
|
84
|
-
Requires-Dist: PyYAML; extra == "slack"
|
|
85
85
|
Requires-Dist: click-default-group; extra == "slack"
|
|
86
|
-
Requires-Dist: toml>=0.10.0; extra == "slack"
|
|
87
86
|
Requires-Dist: stackprinter; extra == "slack"
|
|
87
|
+
Requires-Dist: click>=6.0.0; extra == "slack"
|
|
88
88
|
Requires-Dist: slack-bolt>=1.15.5; extra == "slack"
|
|
89
89
|
Provides-Extra: teams
|
|
90
90
|
Requires-Dist: python-dateutil>=2.8.0; extra == "teams"
|
|
91
|
-
Requires-Dist: click>=6.0.0; extra == "teams"
|
|
92
91
|
Requires-Dist: entrypoints; extra == "teams"
|
|
92
|
+
Requires-Dist: PyYAML; extra == "teams"
|
|
93
|
+
Requires-Dist: toml>=0.10.0; extra == "teams"
|
|
94
|
+
Requires-Dist: tenacity; extra == "teams"
|
|
93
95
|
Requires-Dist: progressbar2; extra == "teams"
|
|
94
96
|
Requires-Dist: pymsteams>=0.2.2; extra == "teams"
|
|
95
|
-
Requires-Dist: tenacity; extra == "teams"
|
|
96
97
|
Requires-Dist: prometheus-client; extra == "teams"
|
|
97
|
-
Requires-Dist: PyYAML; extra == "teams"
|
|
98
98
|
Requires-Dist: click-default-group; extra == "teams"
|
|
99
|
-
Requires-Dist: toml>=0.10.0; extra == "teams"
|
|
100
99
|
Requires-Dist: stackprinter; extra == "teams"
|
|
100
|
+
Requires-Dist: click>=6.0.0; extra == "teams"
|
|
101
101
|
Provides-Extra: tag-propagation
|
|
102
102
|
Requires-Dist: python-dateutil>=2.8.0; extra == "tag-propagation"
|
|
103
|
-
Requires-Dist: click>=6.0.0; extra == "tag-propagation"
|
|
104
103
|
Requires-Dist: entrypoints; extra == "tag-propagation"
|
|
105
|
-
Requires-Dist:
|
|
104
|
+
Requires-Dist: PyYAML; extra == "tag-propagation"
|
|
105
|
+
Requires-Dist: toml>=0.10.0; extra == "tag-propagation"
|
|
106
106
|
Requires-Dist: tenacity; extra == "tag-propagation"
|
|
107
|
+
Requires-Dist: progressbar2; extra == "tag-propagation"
|
|
107
108
|
Requires-Dist: prometheus-client; extra == "tag-propagation"
|
|
108
|
-
Requires-Dist: PyYAML; extra == "tag-propagation"
|
|
109
109
|
Requires-Dist: click-default-group; extra == "tag-propagation"
|
|
110
|
-
Requires-Dist: toml>=0.10.0; extra == "tag-propagation"
|
|
111
110
|
Requires-Dist: stackprinter; extra == "tag-propagation"
|
|
111
|
+
Requires-Dist: click>=6.0.0; extra == "tag-propagation"
|
|
112
112
|
Provides-Extra: term-propagation
|
|
113
113
|
Requires-Dist: python-dateutil>=2.8.0; extra == "term-propagation"
|
|
114
|
-
Requires-Dist: click>=6.0.0; extra == "term-propagation"
|
|
115
114
|
Requires-Dist: entrypoints; extra == "term-propagation"
|
|
116
|
-
Requires-Dist:
|
|
115
|
+
Requires-Dist: PyYAML; extra == "term-propagation"
|
|
116
|
+
Requires-Dist: toml>=0.10.0; extra == "term-propagation"
|
|
117
117
|
Requires-Dist: tenacity; extra == "term-propagation"
|
|
118
|
+
Requires-Dist: progressbar2; extra == "term-propagation"
|
|
118
119
|
Requires-Dist: prometheus-client; extra == "term-propagation"
|
|
119
|
-
Requires-Dist: PyYAML; extra == "term-propagation"
|
|
120
120
|
Requires-Dist: click-default-group; extra == "term-propagation"
|
|
121
|
-
Requires-Dist: toml>=0.10.0; extra == "term-propagation"
|
|
122
121
|
Requires-Dist: stackprinter; extra == "term-propagation"
|
|
122
|
+
Requires-Dist: click>=6.0.0; extra == "term-propagation"
|
|
123
123
|
Provides-Extra: snowflake-tag-propagation
|
|
124
124
|
Requires-Dist: python-dateutil>=2.8.0; extra == "snowflake-tag-propagation"
|
|
125
|
-
Requires-Dist: click>=6.0.0; extra == "snowflake-tag-propagation"
|
|
126
125
|
Requires-Dist: entrypoints; extra == "snowflake-tag-propagation"
|
|
127
|
-
Requires-Dist:
|
|
126
|
+
Requires-Dist: PyYAML; extra == "snowflake-tag-propagation"
|
|
127
|
+
Requires-Dist: toml>=0.10.0; extra == "snowflake-tag-propagation"
|
|
128
128
|
Requires-Dist: tenacity; extra == "snowflake-tag-propagation"
|
|
129
|
+
Requires-Dist: progressbar2; extra == "snowflake-tag-propagation"
|
|
129
130
|
Requires-Dist: prometheus-client; extra == "snowflake-tag-propagation"
|
|
130
|
-
Requires-Dist: PyYAML; extra == "snowflake-tag-propagation"
|
|
131
131
|
Requires-Dist: click-default-group; extra == "snowflake-tag-propagation"
|
|
132
|
-
Requires-Dist:
|
|
132
|
+
Requires-Dist: acryl-datahub[snowflake-slim]==1.3.1.5rc1; extra == "snowflake-tag-propagation"
|
|
133
133
|
Requires-Dist: stackprinter; extra == "snowflake-tag-propagation"
|
|
134
|
-
Requires-Dist:
|
|
134
|
+
Requires-Dist: click>=6.0.0; extra == "snowflake-tag-propagation"
|
|
135
135
|
Provides-Extra: doc-propagation
|
|
136
136
|
Requires-Dist: python-dateutil>=2.8.0; extra == "doc-propagation"
|
|
137
|
-
Requires-Dist: click>=6.0.0; extra == "doc-propagation"
|
|
138
137
|
Requires-Dist: entrypoints; extra == "doc-propagation"
|
|
139
|
-
Requires-Dist:
|
|
138
|
+
Requires-Dist: PyYAML; extra == "doc-propagation"
|
|
139
|
+
Requires-Dist: toml>=0.10.0; extra == "doc-propagation"
|
|
140
140
|
Requires-Dist: tenacity; extra == "doc-propagation"
|
|
141
|
+
Requires-Dist: progressbar2; extra == "doc-propagation"
|
|
141
142
|
Requires-Dist: prometheus-client; extra == "doc-propagation"
|
|
142
|
-
Requires-Dist: PyYAML; extra == "doc-propagation"
|
|
143
143
|
Requires-Dist: click-default-group; extra == "doc-propagation"
|
|
144
|
-
Requires-Dist: toml>=0.10.0; extra == "doc-propagation"
|
|
145
144
|
Requires-Dist: stackprinter; extra == "doc-propagation"
|
|
146
|
-
|
|
147
|
-
Requires-Dist: python-dateutil>=2.8.0; extra == "observability"
|
|
148
|
-
Requires-Dist: click>=6.0.0; extra == "observability"
|
|
149
|
-
Requires-Dist: entrypoints; extra == "observability"
|
|
150
|
-
Requires-Dist: progressbar2; extra == "observability"
|
|
151
|
-
Requires-Dist: tenacity; extra == "observability"
|
|
152
|
-
Requires-Dist: prometheus-client; extra == "observability"
|
|
153
|
-
Requires-Dist: PyYAML; extra == "observability"
|
|
154
|
-
Requires-Dist: click-default-group; extra == "observability"
|
|
155
|
-
Requires-Dist: toml>=0.10.0; extra == "observability"
|
|
156
|
-
Requires-Dist: stackprinter; extra == "observability"
|
|
157
|
-
Requires-Dist: opentelemetry-api>=1.20.0; extra == "observability"
|
|
158
|
-
Requires-Dist: opentelemetry-sdk>=1.20.0; extra == "observability"
|
|
145
|
+
Requires-Dist: click>=6.0.0; extra == "doc-propagation"
|
|
159
146
|
Provides-Extra: all
|
|
160
147
|
Requires-Dist: python-dateutil>=2.8.0; extra == "all"
|
|
161
|
-
Requires-Dist: click>=6.0.0; extra == "all"
|
|
162
148
|
Requires-Dist: entrypoints; extra == "all"
|
|
149
|
+
Requires-Dist: PyYAML; extra == "all"
|
|
150
|
+
Requires-Dist: toml>=0.10.0; extra == "all"
|
|
151
|
+
Requires-Dist: tenacity; extra == "all"
|
|
163
152
|
Requires-Dist: progressbar2; extra == "all"
|
|
153
|
+
Requires-Dist: acryl-executor==0.3.2; extra == "all"
|
|
164
154
|
Requires-Dist: pymsteams>=0.2.2; extra == "all"
|
|
165
|
-
Requires-Dist: confluent-kafka[schemaregistry]; extra == "all"
|
|
166
|
-
Requires-Dist: tenacity; extra == "all"
|
|
167
155
|
Requires-Dist: prometheus-client; extra == "all"
|
|
168
|
-
Requires-Dist: PyYAML; extra == "all"
|
|
169
156
|
Requires-Dist: click-default-group; extra == "all"
|
|
170
|
-
Requires-Dist:
|
|
157
|
+
Requires-Dist: acryl-datahub[snowflake-slim]==1.3.1.5rc1; extra == "all"
|
|
171
158
|
Requires-Dist: stackprinter; extra == "all"
|
|
159
|
+
Requires-Dist: confluent-kafka[schemaregistry]; extra == "all"
|
|
160
|
+
Requires-Dist: click>=6.0.0; extra == "all"
|
|
172
161
|
Requires-Dist: slack-bolt>=1.15.5; extra == "all"
|
|
173
|
-
Requires-Dist: acryl-datahub[snowflake-slim]==1.3.1.5; extra == "all"
|
|
174
|
-
Requires-Dist: opentelemetry-api>=1.20.0; extra == "all"
|
|
175
|
-
Requires-Dist: opentelemetry-sdk>=1.20.0; extra == "all"
|
|
176
|
-
Requires-Dist: acryl-executor==0.3.2; extra == "all"
|
|
177
162
|
Provides-Extra: dev
|
|
163
|
+
Requires-Dist: requests-mock; extra == "dev"
|
|
164
|
+
Requires-Dist: build; extra == "dev"
|
|
165
|
+
Requires-Dist: twine; extra == "dev"
|
|
166
|
+
Requires-Dist: tox; extra == "dev"
|
|
167
|
+
Requires-Dist: acryl-datahub[snowflake-slim]==1.3.1.5rc1; extra == "dev"
|
|
168
|
+
Requires-Dist: typing-inspect; extra == "dev"
|
|
169
|
+
Requires-Dist: types-click==0.1.12; extra == "dev"
|
|
170
|
+
Requires-Dist: click>=6.0.0; extra == "dev"
|
|
171
|
+
Requires-Dist: types-PyYAML; extra == "dev"
|
|
178
172
|
Requires-Dist: python-dateutil>=2.8.0; extra == "dev"
|
|
179
|
-
Requires-Dist:
|
|
180
|
-
Requires-Dist: types-
|
|
181
|
-
Requires-Dist:
|
|
173
|
+
Requires-Dist: types-six; extra == "dev"
|
|
174
|
+
Requires-Dist: types-PyMySQL; extra == "dev"
|
|
175
|
+
Requires-Dist: progressbar2; extra == "dev"
|
|
182
176
|
Requires-Dist: click-default-group; extra == "dev"
|
|
183
|
-
Requires-Dist:
|
|
184
|
-
Requires-Dist:
|
|
185
|
-
Requires-Dist:
|
|
186
|
-
Requires-Dist:
|
|
177
|
+
Requires-Dist: pydantic<3.0.0,>=2.0.0; extra == "dev"
|
|
178
|
+
Requires-Dist: sqlalchemy-stubs; extra == "dev"
|
|
179
|
+
Requires-Dist: confluent-kafka[schemaregistry]; extra == "dev"
|
|
180
|
+
Requires-Dist: types-cachetools; extra == "dev"
|
|
181
|
+
Requires-Dist: pytest>=6.2.2; extra == "dev"
|
|
182
|
+
Requires-Dist: mypy==1.17.1; extra == "dev"
|
|
187
183
|
Requires-Dist: types-python-dateutil; extra == "dev"
|
|
188
|
-
Requires-Dist:
|
|
189
|
-
Requires-Dist:
|
|
190
|
-
Requires-Dist: progressbar2; extra == "dev"
|
|
191
|
-
Requires-Dist: pymsteams>=0.2.2; extra == "dev"
|
|
192
|
-
Requires-Dist: coverage>=5.1; extra == "dev"
|
|
193
|
-
Requires-Dist: h11>=0.16; extra == "dev"
|
|
194
|
-
Requires-Dist: types-pytz; extra == "dev"
|
|
195
|
-
Requires-Dist: pytest-dependency>=0.5.1; extra == "dev"
|
|
196
|
-
Requires-Dist: tenacity; extra == "dev"
|
|
184
|
+
Requires-Dist: types-setuptools; extra == "dev"
|
|
185
|
+
Requires-Dist: aws-msk-iam-sasl-signer-python==1.0.2; extra == "dev"
|
|
197
186
|
Requires-Dist: slack-bolt>=1.15.5; extra == "dev"
|
|
198
|
-
Requires-Dist: toml>=0.10.0; extra == "dev"
|
|
199
|
-
Requires-Dist: azure-identity==1.21.0; extra == "dev"
|
|
200
|
-
Requires-Dist: types-six; extra == "dev"
|
|
201
|
-
Requires-Dist: types-PyYAML; extra == "dev"
|
|
202
|
-
Requires-Dist: twine; extra == "dev"
|
|
203
187
|
Requires-Dist: ratelimit; extra == "dev"
|
|
188
|
+
Requires-Dist: acryl-datahub[datahub-kafka]==1.3.1.5rc1; extra == "dev"
|
|
204
189
|
Requires-Dist: entrypoints; extra == "dev"
|
|
190
|
+
Requires-Dist: PyYAML; extra == "dev"
|
|
191
|
+
Requires-Dist: tenacity; extra == "dev"
|
|
192
|
+
Requires-Dist: types-pytz; extra == "dev"
|
|
205
193
|
Requires-Dist: ruff==0.11.7; extra == "dev"
|
|
206
|
-
Requires-Dist:
|
|
207
|
-
Requires-Dist: stackprinter; extra == "dev"
|
|
208
|
-
Requires-Dist: pydantic<3.0.0,>=2.0.0; extra == "dev"
|
|
209
|
-
Requires-Dist: types-click==0.1.12; extra == "dev"
|
|
210
|
-
Requires-Dist: acryl-executor==0.3.2; extra == "dev"
|
|
211
|
-
Requires-Dist: types-requests; extra == "dev"
|
|
212
|
-
Requires-Dist: aws-msk-iam-sasl-signer-python==1.0.2; extra == "dev"
|
|
213
|
-
Requires-Dist: httpcore>=1.0.9; extra == "dev"
|
|
214
|
-
Requires-Dist: click>=6.0.0; extra == "dev"
|
|
215
|
-
Requires-Dist: jsonpickle; extra == "dev"
|
|
216
|
-
Requires-Dist: build; extra == "dev"
|
|
217
|
-
Requires-Dist: types-PyMySQL; extra == "dev"
|
|
194
|
+
Requires-Dist: pymsteams>=0.2.2; extra == "dev"
|
|
218
195
|
Requires-Dist: prometheus-client; extra == "dev"
|
|
219
|
-
Requires-Dist:
|
|
196
|
+
Requires-Dist: coverage>=5.1; extra == "dev"
|
|
197
|
+
Requires-Dist: pytest-dependency>=0.5.1; extra == "dev"
|
|
198
|
+
Requires-Dist: types-dataclasses; extra == "dev"
|
|
199
|
+
Requires-Dist: types-toml; extra == "dev"
|
|
200
|
+
Requires-Dist: deepdiff; extra == "dev"
|
|
201
|
+
Requires-Dist: h11>=0.16; extra == "dev"
|
|
220
202
|
Requires-Dist: types-freezegun; extra == "dev"
|
|
221
|
-
Requires-Dist: PyYAML; extra == "dev"
|
|
222
|
-
Requires-Dist: acryl-datahub[snowflake-slim]==1.3.1.5; extra == "dev"
|
|
223
|
-
Requires-Dist: typing-inspect; extra == "dev"
|
|
224
203
|
Requires-Dist: pytest-docker>=0.10.3; extra == "dev"
|
|
225
|
-
Requires-Dist:
|
|
226
|
-
Requires-Dist:
|
|
227
|
-
Requires-Dist:
|
|
204
|
+
Requires-Dist: azure-identity==1.21.0; extra == "dev"
|
|
205
|
+
Requires-Dist: freezegun; extra == "dev"
|
|
206
|
+
Requires-Dist: toml>=0.10.0; extra == "dev"
|
|
207
|
+
Requires-Dist: acryl-executor==0.3.2; extra == "dev"
|
|
208
|
+
Requires-Dist: httpcore>=1.0.9; extra == "dev"
|
|
209
|
+
Requires-Dist: stackprinter; extra == "dev"
|
|
210
|
+
Requires-Dist: types-requests; extra == "dev"
|
|
211
|
+
Requires-Dist: jsonpickle; extra == "dev"
|
|
212
|
+
Requires-Dist: pytest-cov>=2.8.1; extra == "dev"
|
|
228
213
|
Provides-Extra: integration-tests
|
|
214
|
+
Requires-Dist: acryl-executor==0.3.2; extra == "integration-tests"
|
|
229
215
|
Requires-Dist: pymsteams>=0.2.2; extra == "integration-tests"
|
|
216
|
+
Requires-Dist: acryl-datahub[snowflake-slim]==1.3.1.5rc1; extra == "integration-tests"
|
|
230
217
|
Requires-Dist: confluent-kafka[schemaregistry]; extra == "integration-tests"
|
|
231
218
|
Requires-Dist: slack-bolt>=1.15.5; extra == "integration-tests"
|
|
232
|
-
Requires-Dist: acryl-datahub[snowflake-slim]==1.3.1.5; extra == "integration-tests"
|
|
233
|
-
Requires-Dist: acryl-executor==0.3.2; extra == "integration-tests"
|
|
234
219
|
Dynamic: classifier
|
|
235
220
|
Dynamic: description
|
|
236
221
|
Dynamic: description-content-type
|
{acryl_datahub_actions-1.3.1.5.dist-info → acryl_datahub_actions-1.3.1.5rc1.dist-info}/RECORD
RENAMED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
datahub_actions/__init__.py,sha256=Pn9UTDbqYPt6jY_acE7MQIveX_Nzdfl5oGmi-Ze8CHs,647
|
|
2
|
-
datahub_actions/_version.py,sha256=
|
|
2
|
+
datahub_actions/_version.py,sha256=20DRLe6b8OXfpqDG8VyphSaTXezWC6nxlLh54Ji37to,339
|
|
3
3
|
datahub_actions/entrypoints.py,sha256=_6NOpKhlfXuSUdPhDpPya7d9kJmwoRGrunxcNPMQE9k,4743
|
|
4
4
|
datahub_actions/action/__init__.py,sha256=KYWPHGi7sDM0DXrrXmhlR6_zhym1qNbtFhjYk1Ug6ss,579
|
|
5
5
|
datahub_actions/action/action.py,sha256=ET1fpeRn6KVD9diJ9ZOObsojrN9y6Vfn4tK7jzBQKHg,1537
|
|
@@ -13,11 +13,9 @@ datahub_actions/event/__init__.py,sha256=KYWPHGi7sDM0DXrrXmhlR6_zhym1qNbtFhjYk1U
|
|
|
13
13
|
datahub_actions/event/event.py,sha256=2Eyizwzbf3fXsUV4n5p7gsYZS_CjEE5y9m1YvkmKOKU,990
|
|
14
14
|
datahub_actions/event/event_envelope.py,sha256=x1QfDetMM7k5SLecD0Nb-duxMxKWU0rmeLroScvkicY,2258
|
|
15
15
|
datahub_actions/event/event_registry.py,sha256=bWV2n9u1n8p9Onu9G2AVgZIfOxCjaBT0pKg2eOQdaig,4663
|
|
16
|
-
datahub_actions/observability/__init__.py,sha256=llKqfushdo7d6RNAtIx9ofS8fqFltea-BHkkzZCUyAA,643
|
|
17
|
-
datahub_actions/observability/kafka_lag_monitor.py,sha256=UrGSC18iUqvKCpxN8HzZ5gpvDjdNtAwuDgXgd77pfpE,7865
|
|
18
16
|
datahub_actions/pipeline/__init__.py,sha256=KYWPHGi7sDM0DXrrXmhlR6_zhym1qNbtFhjYk1Ug6ss,579
|
|
19
17
|
datahub_actions/pipeline/pipeline.py,sha256=6Bod5W3QJNAV0kXymooBxxJVuvAYv3mpvAa6zp-9u5c,12194
|
|
20
|
-
datahub_actions/pipeline/pipeline_config.py,sha256=
|
|
18
|
+
datahub_actions/pipeline/pipeline_config.py,sha256=PpO6DED9LD-7fwIiwp7eMDJmSmmoRbIS2OZJcVfJwy0,2225
|
|
21
19
|
datahub_actions/pipeline/pipeline_context.py,sha256=RollJBpjyw_BChsPPDaXC8_t97Di5eYaAYchGcBKWjs,964
|
|
22
20
|
datahub_actions/pipeline/pipeline_manager.py,sha256=vFqcq1PY2SWtWZElntWzF8P_5wgmZBqTWjE_1NiHGyY,4039
|
|
23
21
|
datahub_actions/pipeline/pipeline_stats.py,sha256=p6R6y4NlaHv6MKkMkAAM5ZgMye1wRK0y6Dlk5eCmbZ8,5062
|
|
@@ -35,12 +33,12 @@ datahub_actions/plugin/action/metadata_change_sync/metadata_change_sync.py,sha25
|
|
|
35
33
|
datahub_actions/plugin/action/propagation/__init__.py,sha256=KYWPHGi7sDM0DXrrXmhlR6_zhym1qNbtFhjYk1Ug6ss,579
|
|
36
34
|
datahub_actions/plugin/action/propagation/propagation_utils.py,sha256=411ojU1UzOLURecxCXhdcmi1v0tfuIQyLrRuwT1fpc4,10810
|
|
37
35
|
datahub_actions/plugin/action/propagation/docs/__init__.py,sha256=KYWPHGi7sDM0DXrrXmhlR6_zhym1qNbtFhjYk1Ug6ss,579
|
|
38
|
-
datahub_actions/plugin/action/propagation/docs/propagation_action.py,sha256=
|
|
36
|
+
datahub_actions/plugin/action/propagation/docs/propagation_action.py,sha256=mtK-sZwERmqOe3xkwFqtelX3vWzF9NMQXZ-P-LK-vsI,37067
|
|
39
37
|
datahub_actions/plugin/action/slack/__init__.py,sha256=KYWPHGi7sDM0DXrrXmhlR6_zhym1qNbtFhjYk1Ug6ss,579
|
|
40
38
|
datahub_actions/plugin/action/slack/slack.py,sha256=YT9kTQsX0FzsB3pglaDl2kIFFnNF7cWx8HcC_xzq9eM,5175
|
|
41
39
|
datahub_actions/plugin/action/snowflake/__init__.py,sha256=KYWPHGi7sDM0DXrrXmhlR6_zhym1qNbtFhjYk1Ug6ss,579
|
|
42
40
|
datahub_actions/plugin/action/snowflake/snowflake_util.py,sha256=ZJW7Wsb65-3JrA15EVfv0ojgvFSC-sV0a_LZ9QyP_bw,5190
|
|
43
|
-
datahub_actions/plugin/action/snowflake/tag_propagator.py,sha256=
|
|
41
|
+
datahub_actions/plugin/action/snowflake/tag_propagator.py,sha256=GgyQHnLtZcmUPyxCNO2eTziMhXizC1N3dkUP_EdxUaw,5052
|
|
44
42
|
datahub_actions/plugin/action/tag/__init__.py,sha256=KYWPHGi7sDM0DXrrXmhlR6_zhym1qNbtFhjYk1Ug6ss,579
|
|
45
43
|
datahub_actions/plugin/action/tag/tag_propagation_action.py,sha256=2iLG4oJURrpuwnS28ePDtBGKvvtk-2BWB8PHGLyttw4,6448
|
|
46
44
|
datahub_actions/plugin/action/teams/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -52,16 +50,16 @@ datahub_actions/plugin/action/utils/term_resolver.py,sha256=J3_u-iGFXxsGCPtaeV_p
|
|
|
52
50
|
datahub_actions/plugin/source/__init__.py,sha256=KYWPHGi7sDM0DXrrXmhlR6_zhym1qNbtFhjYk1Ug6ss,579
|
|
53
51
|
datahub_actions/plugin/source/acryl/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
54
52
|
datahub_actions/plugin/source/acryl/constants.py,sha256=f6vEsF6SdD0B1Vs90lCRglRQOCCYjiIlGanNtC7OsnY,307
|
|
55
|
-
datahub_actions/plugin/source/acryl/datahub_cloud_event_source.py,sha256=
|
|
53
|
+
datahub_actions/plugin/source/acryl/datahub_cloud_event_source.py,sha256=MA6OlR3mNx69hgprlFifVApSwuW_qOBaHQSlyD4Hm38,12730
|
|
56
54
|
datahub_actions/plugin/source/acryl/datahub_cloud_events_ack_manager.py,sha256=ky15ibq5lfYdPIwufv4w92XOpp9C6cRvlhPRbJ_cs10,993
|
|
57
55
|
datahub_actions/plugin/source/acryl/datahub_cloud_events_consumer.py,sha256=1j6s-uTyZcRlF74toHTrdNeVNq5MMh9_Fp8FOsV-8a0,7116
|
|
58
56
|
datahub_actions/plugin/source/acryl/datahub_cloud_events_consumer_offsets_store.py,sha256=5m_VR_5yHjt4_YZEi1die5sr0ngl9dVobX10AjIodvA,3969
|
|
59
57
|
datahub_actions/plugin/source/kafka/__init__.py,sha256=KYWPHGi7sDM0DXrrXmhlR6_zhym1qNbtFhjYk1Ug6ss,579
|
|
60
|
-
datahub_actions/plugin/source/kafka/kafka_event_source.py,sha256=
|
|
58
|
+
datahub_actions/plugin/source/kafka/kafka_event_source.py,sha256=i3JqjXuAhjg0fwK99vbOH92sQHex8BzcplZDcrSclZ0,11545
|
|
61
59
|
datahub_actions/plugin/source/kafka/utils.py,sha256=EEqBnv8Zd05zSg9T3f2FHaARaStD2j2M_xiSeaQBplA,758
|
|
62
60
|
datahub_actions/plugin/transform/__init__.py,sha256=KYWPHGi7sDM0DXrrXmhlR6_zhym1qNbtFhjYk1Ug6ss,579
|
|
63
61
|
datahub_actions/plugin/transform/filter/__init__.py,sha256=KYWPHGi7sDM0DXrrXmhlR6_zhym1qNbtFhjYk1Ug6ss,579
|
|
64
|
-
datahub_actions/plugin/transform/filter/filter_transformer.py,sha256=
|
|
62
|
+
datahub_actions/plugin/transform/filter/filter_transformer.py,sha256=aF5sTy5FuRV1odZckibLIR0uGYbLmehq5pRCZ2LYIwE,3136
|
|
65
63
|
datahub_actions/source/__init__.py,sha256=KYWPHGi7sDM0DXrrXmhlR6_zhym1qNbtFhjYk1Ug6ss,579
|
|
66
64
|
datahub_actions/source/event_source.py,sha256=_7ufs1WyeAFQ1b4BxGypvN8mpW006U2kA-WCyZndf7c,1947
|
|
67
65
|
datahub_actions/source/event_source_registry.py,sha256=5Nmz8mNE1G_zJ8zJ-WnNt3JMrJdteuKHdr4mKe3t6Ig,1147
|
|
@@ -77,8 +75,8 @@ datahub_actions/utils/event_util.py,sha256=VluTOeyFcot48moK9qLmYL1ADAjsau0346Ngi
|
|
|
77
75
|
datahub_actions/utils/kafka_msk_iam.py,sha256=JWg0MBEMcsG2AmW4yXiHvH_dnnsQDIRASdlvDXGTVcI,1013
|
|
78
76
|
datahub_actions/utils/name_resolver.py,sha256=uXICSpy1IUe5uyFUiRk4vDQ9_G0JytPgKPSnqMA6fZk,10540
|
|
79
77
|
datahub_actions/utils/social_util.py,sha256=FI_3qDjayX9LKlDjf43QHafnOznQk3v5Vp3Xyhq-lno,5271
|
|
80
|
-
acryl_datahub_actions-1.3.1.
|
|
81
|
-
acryl_datahub_actions-1.3.1.
|
|
82
|
-
acryl_datahub_actions-1.3.1.
|
|
83
|
-
acryl_datahub_actions-1.3.1.
|
|
84
|
-
acryl_datahub_actions-1.3.1.
|
|
78
|
+
acryl_datahub_actions-1.3.1.5rc1.dist-info/METADATA,sha256=pjhuLQpTMB9wXwVxK8hm2jGqpPdryQZ9L8o_sfh8_m4,18124
|
|
79
|
+
acryl_datahub_actions-1.3.1.5rc1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
80
|
+
acryl_datahub_actions-1.3.1.5rc1.dist-info/entry_points.txt,sha256=Gbvj36kOFWrsJ1meJVFB7zYgrKbIGgufOpZDurJbehU,866
|
|
81
|
+
acryl_datahub_actions-1.3.1.5rc1.dist-info/top_level.txt,sha256=93StcIqRM0PfcJoT06TFhcCjPnIw-CyFgBaF-4vqCKY,16
|
|
82
|
+
acryl_datahub_actions-1.3.1.5rc1.dist-info/RECORD,,
|
datahub_actions/_version.py
CHANGED
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
|
|
15
15
|
from typing import Any, Dict, List, Optional, Union
|
|
16
16
|
|
|
17
|
-
from pydantic import BaseModel
|
|
17
|
+
from pydantic import BaseModel
|
|
18
18
|
|
|
19
19
|
from datahub.configuration import ConfigModel
|
|
20
20
|
from datahub.configuration.common import ConfigEnum
|
|
@@ -30,29 +30,29 @@ class FailureMode(ConfigEnum):
|
|
|
30
30
|
|
|
31
31
|
class SourceConfig(ConfigModel):
|
|
32
32
|
type: str
|
|
33
|
-
config: Optional[Dict[str, Any]] =
|
|
33
|
+
config: Optional[Dict[str, Any]] = None
|
|
34
34
|
|
|
35
35
|
|
|
36
36
|
class TransformConfig(ConfigModel):
|
|
37
37
|
type: str
|
|
38
|
-
config: Optional[Dict[str, Any]] =
|
|
38
|
+
config: Optional[Dict[str, Any]] = None
|
|
39
39
|
|
|
40
40
|
|
|
41
41
|
class FilterConfig(ConfigModel):
|
|
42
42
|
event_type: Union[str, List[str]]
|
|
43
|
-
event: Optional[Dict[str, Any]] =
|
|
43
|
+
event: Optional[Dict[str, Any]] = None
|
|
44
44
|
|
|
45
45
|
|
|
46
46
|
class ActionConfig(ConfigModel):
|
|
47
47
|
type: str
|
|
48
|
-
config: Optional[Dict[str, Any]] =
|
|
48
|
+
config: Optional[Dict[str, Any]] = None
|
|
49
49
|
|
|
50
50
|
|
|
51
51
|
class PipelineOptions(BaseModel):
|
|
52
|
-
retry_count: Optional[int] =
|
|
53
|
-
failure_mode: Optional[FailureMode] =
|
|
54
|
-
failed_events_dir: Optional[str] =
|
|
55
|
-
|
|
52
|
+
retry_count: Optional[int] = None
|
|
53
|
+
failure_mode: Optional[FailureMode] = None
|
|
54
|
+
failed_events_dir: Optional[str] = (
|
|
55
|
+
None # The path where failed events should be logged.
|
|
56
56
|
)
|
|
57
57
|
|
|
58
58
|
|
|
@@ -132,11 +132,10 @@ class DocPropagationAction(Action):
|
|
|
132
132
|
def __init__(self, config: DocPropagationConfig, ctx: PipelineContext):
|
|
133
133
|
super().__init__()
|
|
134
134
|
self.action_urn: str
|
|
135
|
-
if "urn:li:dataHubAction
|
|
136
|
-
action_urn_part = ctx.pipeline_name.split("urn:li:dataHubAction:")[1]
|
|
137
|
-
self.action_urn = f"urn:li:dataHubAction:{action_urn_part}"
|
|
138
|
-
else:
|
|
135
|
+
if not ctx.pipeline_name.startswith("urn:li:dataHubAction"):
|
|
139
136
|
self.action_urn = f"urn:li:dataHubAction:{ctx.pipeline_name}"
|
|
137
|
+
else:
|
|
138
|
+
self.action_urn = ctx.pipeline_name
|
|
140
139
|
|
|
141
140
|
self.config: DocPropagationConfig = config
|
|
142
141
|
self.last_config_refresh: float = 0
|
|
@@ -15,8 +15,6 @@
|
|
|
15
15
|
import logging
|
|
16
16
|
from typing import Optional
|
|
17
17
|
|
|
18
|
-
from pydantic import Field
|
|
19
|
-
|
|
20
18
|
from datahub.configuration.common import ConfigModel
|
|
21
19
|
from datahub.ingestion.source.snowflake.snowflake_config import SnowflakeV2Config
|
|
22
20
|
from datahub_actions.action.action import Action
|
|
@@ -38,8 +36,8 @@ logger = logging.getLogger(__name__)
|
|
|
38
36
|
|
|
39
37
|
class SnowflakeTagPropagatorConfig(ConfigModel):
|
|
40
38
|
snowflake: SnowflakeV2Config
|
|
41
|
-
tag_propagation: Optional[TagPropagationConfig] =
|
|
42
|
-
term_propagation: Optional[TermPropagationConfig] =
|
|
39
|
+
tag_propagation: Optional[TagPropagationConfig] = None
|
|
40
|
+
term_propagation: Optional[TermPropagationConfig] = None
|
|
43
41
|
|
|
44
42
|
|
|
45
43
|
class SnowflakeTagPropagatorAction(Action):
|
|
@@ -4,8 +4,6 @@ import time
|
|
|
4
4
|
from dataclasses import dataclass
|
|
5
5
|
from typing import Dict, Iterable, List, Optional, Union, cast
|
|
6
6
|
|
|
7
|
-
from pydantic import Field
|
|
8
|
-
|
|
9
7
|
from datahub.configuration import ConfigModel
|
|
10
8
|
from datahub.emitter.serialization_helper import post_json_transform
|
|
11
9
|
from datahub.ingestion.graph.client import DataHubGraph
|
|
@@ -61,10 +59,8 @@ def build_metadata_change_log_event(msg: ExternalEvent) -> MetadataChangeLogEven
|
|
|
61
59
|
|
|
62
60
|
class DataHubEventsSourceConfig(ConfigModel):
|
|
63
61
|
topics: Union[str, List[str]] = PLATFORM_EVENT_TOPIC_NAME
|
|
64
|
-
consumer_id: Optional[str] =
|
|
65
|
-
|
|
66
|
-
)
|
|
67
|
-
lookback_days: Optional[int] = Field(default=None)
|
|
62
|
+
consumer_id: Optional[str] = None # Used to store offset for the consumer.
|
|
63
|
+
lookback_days: Optional[int] = None
|
|
68
64
|
reset_offsets: Optional[bool] = False
|
|
69
65
|
infinite_retry: Optional[bool] = False
|
|
70
66
|
|
|
@@ -13,7 +13,6 @@
|
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
15
|
import logging
|
|
16
|
-
import os
|
|
17
16
|
from dataclasses import dataclass
|
|
18
17
|
from typing import Any, Callable, Dict, Iterable, Optional
|
|
19
18
|
|
|
@@ -23,7 +22,6 @@ from confluent_kafka import KafkaError, KafkaException, TopicPartition
|
|
|
23
22
|
from confluent_kafka.schema_registry.avro import AvroDeserializer
|
|
24
23
|
from confluent_kafka.schema_registry.schema_registry_client import SchemaRegistryClient
|
|
25
24
|
from prometheus_client import Counter, Gauge
|
|
26
|
-
from pydantic import Field
|
|
27
25
|
|
|
28
26
|
from datahub.configuration import ConfigModel
|
|
29
27
|
from datahub.configuration.kafka import KafkaConsumerConnectionConfig
|
|
@@ -42,7 +40,6 @@ from datahub_actions.event.event_registry import (
|
|
|
42
40
|
)
|
|
43
41
|
|
|
44
42
|
# May or may not need these.
|
|
45
|
-
from datahub_actions.observability.kafka_lag_monitor import KafkaLagMonitor
|
|
46
43
|
from datahub_actions.pipeline.pipeline_context import PipelineContext
|
|
47
44
|
from datahub_actions.plugin.source.kafka.utils import with_retry
|
|
48
45
|
from datahub_actions.source.event_source import EventSource
|
|
@@ -97,7 +94,7 @@ def build_entity_change_event(payload: GenericPayloadClass) -> EntityChangeEvent
|
|
|
97
94
|
|
|
98
95
|
class KafkaEventSourceConfig(ConfigModel):
|
|
99
96
|
connection: KafkaConsumerConnectionConfig = KafkaConsumerConnectionConfig()
|
|
100
|
-
topic_routes: Optional[Dict[str, str]]
|
|
97
|
+
topic_routes: Optional[Dict[str, str]]
|
|
101
98
|
async_commit_enabled: bool = False
|
|
102
99
|
async_commit_interval: int = 10000
|
|
103
100
|
commit_retry_count: int = 5
|
|
@@ -126,7 +123,6 @@ def kafka_messages_observer(pipeline_name: str) -> Callable:
|
|
|
126
123
|
class KafkaEventSource(EventSource):
|
|
127
124
|
running = False
|
|
128
125
|
source_config: KafkaEventSourceConfig
|
|
129
|
-
_lag_monitor: Optional[KafkaLagMonitor] = None
|
|
130
126
|
|
|
131
127
|
def __init__(self, config: KafkaEventSourceConfig, ctx: PipelineContext):
|
|
132
128
|
self.source_config = config
|
|
@@ -162,41 +158,6 @@ class KafkaEventSource(EventSource):
|
|
|
162
158
|
)
|
|
163
159
|
self._observe_message: Callable = kafka_messages_observer(ctx.pipeline_name)
|
|
164
160
|
|
|
165
|
-
# Initialize lag monitoring (if enabled)
|
|
166
|
-
if self._is_lag_monitoring_enabled():
|
|
167
|
-
lag_interval = float(
|
|
168
|
-
os.environ.get("DATAHUB_ACTIONS_KAFKA_LAG_INTERVAL_SECONDS", "30")
|
|
169
|
-
)
|
|
170
|
-
lag_timeout = float(
|
|
171
|
-
os.environ.get("DATAHUB_ACTIONS_KAFKA_LAG_TIMEOUT_SECONDS", "5")
|
|
172
|
-
)
|
|
173
|
-
self._lag_monitor = KafkaLagMonitor(
|
|
174
|
-
consumer=self.consumer,
|
|
175
|
-
pipeline_name=ctx.pipeline_name,
|
|
176
|
-
interval_seconds=lag_interval,
|
|
177
|
-
timeout_seconds=lag_timeout,
|
|
178
|
-
)
|
|
179
|
-
logger.info(
|
|
180
|
-
f"Kafka lag monitoring enabled for '{ctx.pipeline_name}' "
|
|
181
|
-
f"(interval={lag_interval}s, timeout={lag_timeout}s)"
|
|
182
|
-
)
|
|
183
|
-
else:
|
|
184
|
-
logger.debug(
|
|
185
|
-
f"Kafka lag monitoring disabled for pipeline '{ctx.pipeline_name}'"
|
|
186
|
-
)
|
|
187
|
-
|
|
188
|
-
@staticmethod
|
|
189
|
-
def _is_lag_monitoring_enabled() -> bool:
|
|
190
|
-
"""Check if Kafka lag monitoring should be enabled.
|
|
191
|
-
|
|
192
|
-
Lag monitoring is enabled if:
|
|
193
|
-
1. DATAHUB_ACTIONS_KAFKA_LAG_ENABLED=true (case-insensitive)
|
|
194
|
-
|
|
195
|
-
Default: False (conservative default for OSS rollout)
|
|
196
|
-
"""
|
|
197
|
-
enabled_str = os.environ.get("DATAHUB_ACTIONS_KAFKA_LAG_ENABLED", "false")
|
|
198
|
-
return enabled_str.lower() in ("true", "1", "yes")
|
|
199
|
-
|
|
200
161
|
@classmethod
|
|
201
162
|
def create(cls, config_dict: dict, ctx: PipelineContext) -> "EventSource":
|
|
202
163
|
config = KafkaEventSourceConfig.model_validate(config_dict)
|
|
@@ -207,11 +168,6 @@ class KafkaEventSource(EventSource):
|
|
|
207
168
|
topics_to_subscribe = list(topic_routes.values())
|
|
208
169
|
logger.debug(f"Subscribing to the following topics: {topics_to_subscribe}")
|
|
209
170
|
self.consumer.subscribe(topics_to_subscribe)
|
|
210
|
-
|
|
211
|
-
# Start lag monitoring after subscription
|
|
212
|
-
if self._lag_monitor is not None:
|
|
213
|
-
self._lag_monitor.start()
|
|
214
|
-
|
|
215
171
|
self.running = True
|
|
216
172
|
while self.running:
|
|
217
173
|
try:
|
|
@@ -272,11 +228,6 @@ class KafkaEventSource(EventSource):
|
|
|
272
228
|
yield EventEnvelope(RELATIONSHIP_CHANGE_EVENT_V1_TYPE, rce, kafka_meta)
|
|
273
229
|
|
|
274
230
|
def close(self) -> None:
|
|
275
|
-
# Stop lag monitoring first
|
|
276
|
-
if self._lag_monitor is not None:
|
|
277
|
-
self._lag_monitor.stop()
|
|
278
|
-
|
|
279
|
-
# Then close consumer
|
|
280
231
|
if self.consumer:
|
|
281
232
|
self.running = False
|
|
282
233
|
self.consumer.close()
|
|
@@ -16,8 +16,6 @@ import json
|
|
|
16
16
|
import logging
|
|
17
17
|
from typing import Any, Dict, List, Optional, Union
|
|
18
18
|
|
|
19
|
-
from pydantic import Field
|
|
20
|
-
|
|
21
19
|
from datahub.configuration import ConfigModel
|
|
22
20
|
from datahub_actions.event.event_envelope import EventEnvelope
|
|
23
21
|
from datahub_actions.pipeline.pipeline_context import PipelineContext
|
|
@@ -28,7 +26,7 @@ logger = logging.getLogger(__name__)
|
|
|
28
26
|
|
|
29
27
|
class FilterTransformerConfig(ConfigModel):
|
|
30
28
|
event_type: Union[str, List[str]]
|
|
31
|
-
event: Optional[Dict[str, Any]]
|
|
29
|
+
event: Optional[Dict[str, Any]]
|
|
32
30
|
|
|
33
31
|
|
|
34
32
|
class FilterTransformer(Transformer):
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
# Copyright 2021 Acryl Data, Inc.
|
|
2
|
-
#
|
|
3
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
-
# you may not use this file except in compliance with the License.
|
|
5
|
-
# You may obtain a copy of the License at
|
|
6
|
-
#
|
|
7
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
-
#
|
|
9
|
-
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
-
# See the License for the specific language governing permissions and
|
|
13
|
-
# limitations under the License.
|
|
14
|
-
|
|
15
|
-
"""Observability and metrics utilities for datahub-actions."""
|
|
@@ -1,230 +0,0 @@
|
|
|
1
|
-
# Copyright 2021 Acryl Data, Inc.
|
|
2
|
-
#
|
|
3
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
-
# you may not use this file except in compliance with the License.
|
|
5
|
-
# You may obtain a copy of the License at
|
|
6
|
-
#
|
|
7
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
-
#
|
|
9
|
-
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
-
# See the License for the specific language governing permissions and
|
|
13
|
-
# limitations under the License.
|
|
14
|
-
|
|
15
|
-
"""Background thread for monitoring Kafka consumer lag.
|
|
16
|
-
|
|
17
|
-
This module provides a KafkaLagMonitor class that periodically calculates
|
|
18
|
-
and reports Kafka consumer lag metrics to Prometheus.
|
|
19
|
-
"""
|
|
20
|
-
|
|
21
|
-
import logging
|
|
22
|
-
import threading
|
|
23
|
-
from dataclasses import dataclass
|
|
24
|
-
from typing import Dict, Optional
|
|
25
|
-
|
|
26
|
-
from confluent_kafka import Consumer, KafkaException, TopicPartition
|
|
27
|
-
from prometheus_client import Gauge
|
|
28
|
-
|
|
29
|
-
logger = logging.getLogger(__name__)
|
|
30
|
-
|
|
31
|
-
# Prometheus metrics
|
|
32
|
-
KAFKA_LAG_GAUGE = Gauge(
|
|
33
|
-
name="kafka_consumer_lag",
|
|
34
|
-
documentation="Kafka consumer lag aggregated per topic",
|
|
35
|
-
labelnames=["topic", "pipeline_name"],
|
|
36
|
-
)
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
@dataclass
|
|
40
|
-
class LagStats:
|
|
41
|
-
"""Statistics for a topic's consumer lag."""
|
|
42
|
-
|
|
43
|
-
topic: str
|
|
44
|
-
total_lag: int
|
|
45
|
-
partition_lags: Dict[int, int] # partition_id -> lag
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
class KafkaLagMonitor:
|
|
49
|
-
"""Background thread that periodically reports Kafka consumer lag.
|
|
50
|
-
|
|
51
|
-
This monitor:
|
|
52
|
-
1. Queries assigned partitions from the Kafka consumer
|
|
53
|
-
2. Gets high water marks for each partition
|
|
54
|
-
3. Gets committed offsets for each partition
|
|
55
|
-
4. Calculates lag = high_water_mark - committed_offset
|
|
56
|
-
5. Aggregates per-topic lag (sum across partitions)
|
|
57
|
-
6. Updates Prometheus Gauge metrics
|
|
58
|
-
7. Optionally updates OpenTelemetry metrics if available
|
|
59
|
-
"""
|
|
60
|
-
|
|
61
|
-
def __init__(
|
|
62
|
-
self,
|
|
63
|
-
consumer: Consumer,
|
|
64
|
-
pipeline_name: str,
|
|
65
|
-
interval_seconds: float = 30.0,
|
|
66
|
-
timeout_seconds: float = 5.0,
|
|
67
|
-
):
|
|
68
|
-
"""Initialize lag monitor.
|
|
69
|
-
|
|
70
|
-
Args:
|
|
71
|
-
consumer: confluent_kafka.Consumer instance to monitor
|
|
72
|
-
pipeline_name: Name of the action pipeline (for metric labels)
|
|
73
|
-
interval_seconds: How often to report lag (default: 30s)
|
|
74
|
-
timeout_seconds: Timeout for Kafka API calls (default: 5s)
|
|
75
|
-
"""
|
|
76
|
-
self.consumer = consumer
|
|
77
|
-
self.pipeline_name = pipeline_name
|
|
78
|
-
self.interval_seconds = interval_seconds
|
|
79
|
-
self.timeout_seconds = timeout_seconds
|
|
80
|
-
|
|
81
|
-
self._stop_event = threading.Event()
|
|
82
|
-
self._thread: Optional[threading.Thread] = None
|
|
83
|
-
|
|
84
|
-
def start(self) -> None:
|
|
85
|
-
"""Start the background monitoring thread."""
|
|
86
|
-
if self._thread is not None:
|
|
87
|
-
logger.warning("Lag monitor already started")
|
|
88
|
-
return
|
|
89
|
-
|
|
90
|
-
self._stop_event.clear()
|
|
91
|
-
self._thread = threading.Thread(
|
|
92
|
-
target=self._monitor_loop,
|
|
93
|
-
name=f"kafka-lag-monitor-{self.pipeline_name}",
|
|
94
|
-
daemon=True, # Daemon thread exits when main thread exits
|
|
95
|
-
)
|
|
96
|
-
self._thread.start()
|
|
97
|
-
logger.info(
|
|
98
|
-
f"Kafka lag monitor started for pipeline '{self.pipeline_name}' "
|
|
99
|
-
f"(interval={self.interval_seconds}s)"
|
|
100
|
-
)
|
|
101
|
-
|
|
102
|
-
def stop(self) -> None:
|
|
103
|
-
"""Stop the background monitoring thread."""
|
|
104
|
-
if self._thread is None:
|
|
105
|
-
return
|
|
106
|
-
|
|
107
|
-
logger.info(f"Stopping Kafka lag monitor for pipeline '{self.pipeline_name}'")
|
|
108
|
-
self._stop_event.set()
|
|
109
|
-
self._thread.join(timeout=10.0)
|
|
110
|
-
self._thread = None
|
|
111
|
-
|
|
112
|
-
def _monitor_loop(self) -> None:
|
|
113
|
-
"""Main monitoring loop that runs in background thread."""
|
|
114
|
-
while not self._stop_event.is_set():
|
|
115
|
-
try:
|
|
116
|
-
self._collect_and_report_lag()
|
|
117
|
-
except Exception as e:
|
|
118
|
-
# Log error but don't crash - monitoring should be resilient
|
|
119
|
-
logger.error(
|
|
120
|
-
f"Error collecting lag for pipeline '{self.pipeline_name}': {e}",
|
|
121
|
-
exc_info=True,
|
|
122
|
-
)
|
|
123
|
-
|
|
124
|
-
# Sleep with interrupt support
|
|
125
|
-
self._stop_event.wait(timeout=self.interval_seconds)
|
|
126
|
-
|
|
127
|
-
def _collect_and_report_lag(self) -> None:
|
|
128
|
-
"""Collect lag statistics and update metrics."""
|
|
129
|
-
# Get assigned partitions
|
|
130
|
-
assignment = self.consumer.assignment()
|
|
131
|
-
if not assignment:
|
|
132
|
-
logger.debug(f"No partitions assigned to pipeline '{self.pipeline_name}'")
|
|
133
|
-
return
|
|
134
|
-
|
|
135
|
-
# Group partitions by topic
|
|
136
|
-
topic_partitions: Dict[str, list[TopicPartition]] = {}
|
|
137
|
-
for tp in assignment:
|
|
138
|
-
if tp.topic not in topic_partitions:
|
|
139
|
-
topic_partitions[tp.topic] = []
|
|
140
|
-
topic_partitions[tp.topic].append(tp)
|
|
141
|
-
|
|
142
|
-
# Calculate lag per topic
|
|
143
|
-
for topic, partitions in topic_partitions.items():
|
|
144
|
-
lag_stats = self._calculate_topic_lag(topic, partitions)
|
|
145
|
-
if lag_stats:
|
|
146
|
-
self._report_lag(lag_stats)
|
|
147
|
-
|
|
148
|
-
def _calculate_topic_lag(
|
|
149
|
-
self, topic: str, partitions: list[TopicPartition]
|
|
150
|
-
) -> Optional[LagStats]:
|
|
151
|
-
"""Calculate lag for all partitions of a topic.
|
|
152
|
-
|
|
153
|
-
Args:
|
|
154
|
-
topic: Topic name
|
|
155
|
-
partitions: List of TopicPartition objects for this topic
|
|
156
|
-
|
|
157
|
-
Returns:
|
|
158
|
-
LagStats with aggregated lag, or None if calculation failed
|
|
159
|
-
"""
|
|
160
|
-
partition_lags: Dict[int, int] = {}
|
|
161
|
-
|
|
162
|
-
# Get committed offsets for all partitions at once
|
|
163
|
-
try:
|
|
164
|
-
committed_partitions = self.consumer.committed(
|
|
165
|
-
partitions, timeout=self.timeout_seconds
|
|
166
|
-
)
|
|
167
|
-
except KafkaException as e:
|
|
168
|
-
logger.warning(f"Failed to get committed offsets for topic '{topic}': {e}")
|
|
169
|
-
return None
|
|
170
|
-
|
|
171
|
-
# Calculate lag for each partition
|
|
172
|
-
for tp in committed_partitions:
|
|
173
|
-
try:
|
|
174
|
-
# Get high water mark
|
|
175
|
-
watermarks = self.consumer.get_watermark_offsets(
|
|
176
|
-
tp, timeout=self.timeout_seconds, cached=False
|
|
177
|
-
)
|
|
178
|
-
if watermarks is None:
|
|
179
|
-
logger.warning(
|
|
180
|
-
f"Failed to get watermarks for {topic}[{tp.partition}]"
|
|
181
|
-
)
|
|
182
|
-
continue
|
|
183
|
-
|
|
184
|
-
low, high = watermarks
|
|
185
|
-
|
|
186
|
-
# Calculate lag
|
|
187
|
-
if tp.offset < 0:
|
|
188
|
-
# No committed offset yet - show total available messages as lag
|
|
189
|
-
lag = high - low
|
|
190
|
-
else:
|
|
191
|
-
# Normal case: lag = high water mark - committed offset
|
|
192
|
-
lag = high - tp.offset
|
|
193
|
-
|
|
194
|
-
# Ensure non-negative lag
|
|
195
|
-
lag = max(0, lag)
|
|
196
|
-
partition_lags[tp.partition] = lag
|
|
197
|
-
|
|
198
|
-
except KafkaException as e:
|
|
199
|
-
logger.warning(
|
|
200
|
-
f"Error calculating lag for {topic}[{tp.partition}]: {e}"
|
|
201
|
-
)
|
|
202
|
-
continue
|
|
203
|
-
|
|
204
|
-
if not partition_lags:
|
|
205
|
-
return None
|
|
206
|
-
|
|
207
|
-
total_lag = sum(partition_lags.values())
|
|
208
|
-
return LagStats(
|
|
209
|
-
topic=topic,
|
|
210
|
-
total_lag=total_lag,
|
|
211
|
-
partition_lags=partition_lags,
|
|
212
|
-
)
|
|
213
|
-
|
|
214
|
-
def _report_lag(self, lag_stats: LagStats) -> None:
|
|
215
|
-
"""Report lag statistics to metrics backends.
|
|
216
|
-
|
|
217
|
-
Args:
|
|
218
|
-
lag_stats: Lag statistics to report
|
|
219
|
-
"""
|
|
220
|
-
# Always update Prometheus (base requirement)
|
|
221
|
-
KAFKA_LAG_GAUGE.labels(
|
|
222
|
-
topic=lag_stats.topic,
|
|
223
|
-
pipeline_name=self.pipeline_name,
|
|
224
|
-
).set(lag_stats.total_lag)
|
|
225
|
-
|
|
226
|
-
logger.debug(
|
|
227
|
-
f"Pipeline '{self.pipeline_name}' topic '{lag_stats.topic}': "
|
|
228
|
-
f"lag={lag_stats.total_lag} "
|
|
229
|
-
f"(partitions: {lag_stats.partition_lags})"
|
|
230
|
-
)
|
{acryl_datahub_actions-1.3.1.5.dist-info → acryl_datahub_actions-1.3.1.5rc1.dist-info}/WHEEL
RENAMED
|
File without changes
|
|
File without changes
|
{acryl_datahub_actions-1.3.1.5.dist-info → acryl_datahub_actions-1.3.1.5rc1.dist-info}/top_level.txt
RENAMED
|
File without changes
|