acryl-datahub-airflow-plugin 1.3.1.4__py3-none-any.whl → 1.3.1.5rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {acryl_datahub_airflow_plugin-1.3.1.4.dist-info → acryl_datahub_airflow_plugin-1.3.1.5rc1.dist-info}/METADATA +30 -29
- {acryl_datahub_airflow_plugin-1.3.1.4.dist-info → acryl_datahub_airflow_plugin-1.3.1.5rc1.dist-info}/RECORD +7 -7
- datahub_airflow_plugin/_extractors.py +30 -1
- datahub_airflow_plugin/_version.py +1 -1
- {acryl_datahub_airflow_plugin-1.3.1.4.dist-info → acryl_datahub_airflow_plugin-1.3.1.5rc1.dist-info}/WHEEL +0 -0
- {acryl_datahub_airflow_plugin-1.3.1.4.dist-info → acryl_datahub_airflow_plugin-1.3.1.5rc1.dist-info}/entry_points.txt +0 -0
- {acryl_datahub_airflow_plugin-1.3.1.4.dist-info → acryl_datahub_airflow_plugin-1.3.1.5rc1.dist-info}/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: acryl-datahub-airflow-plugin
|
|
3
|
-
Version: 1.3.1.
|
|
3
|
+
Version: 1.3.1.5rc1
|
|
4
4
|
Summary: Datahub Airflow plugin to capture executions and send to Datahub
|
|
5
5
|
Home-page: https://docs.datahub.com/
|
|
6
6
|
License: Apache-2.0
|
|
@@ -21,53 +21,54 @@ Classifier: Environment :: MacOS X
|
|
|
21
21
|
Classifier: Topic :: Software Development
|
|
22
22
|
Requires-Python: >=3.9
|
|
23
23
|
Description-Content-Type: text/markdown
|
|
24
|
-
Requires-Dist: apache-airflow<3,>=2.7.0
|
|
25
24
|
Requires-Dist: openlineage-airflow<=1.30.1,>=1.2.0
|
|
26
|
-
Requires-Dist: acryl-datahub[datahub-rest]==1.3.1.
|
|
27
|
-
Requires-Dist: acryl-datahub[datahub-rest,sql-parser]==1.3.1.4
|
|
25
|
+
Requires-Dist: acryl-datahub[datahub-rest]==1.3.1.5rc1
|
|
28
26
|
Requires-Dist: pydantic>=2.4.0
|
|
27
|
+
Requires-Dist: apache-airflow<3,>=2.7.0
|
|
28
|
+
Requires-Dist: acryl-datahub[datahub-rest,sql-parser]==1.3.1.5rc1
|
|
29
29
|
Provides-Extra: ignore
|
|
30
30
|
Provides-Extra: datahub-rest
|
|
31
|
-
Requires-Dist: acryl-datahub[datahub-rest]==1.3.1.
|
|
31
|
+
Requires-Dist: acryl-datahub[datahub-rest]==1.3.1.5rc1; extra == "datahub-rest"
|
|
32
32
|
Provides-Extra: datahub-kafka
|
|
33
|
-
Requires-Dist: acryl-datahub[datahub-kafka]==1.3.1.
|
|
33
|
+
Requires-Dist: acryl-datahub[datahub-kafka]==1.3.1.5rc1; extra == "datahub-kafka"
|
|
34
34
|
Provides-Extra: datahub-file
|
|
35
|
-
Requires-Dist: acryl-datahub[sync-file-emitter]==1.3.1.
|
|
35
|
+
Requires-Dist: acryl-datahub[sync-file-emitter]==1.3.1.5rc1; extra == "datahub-file"
|
|
36
36
|
Provides-Extra: dev
|
|
37
|
-
Requires-Dist: ruff==0.11.7; extra == "dev"
|
|
38
|
-
Requires-Dist: apache-airflow<3,>=2.7.0; extra == "dev"
|
|
39
|
-
Requires-Dist: packaging; extra == "dev"
|
|
40
37
|
Requires-Dist: openlineage-airflow<=1.30.1,>=1.2.0; extra == "dev"
|
|
41
|
-
Requires-Dist: tenacity; extra == "dev"
|
|
42
|
-
Requires-Dist: tox; extra == "dev"
|
|
43
38
|
Requires-Dist: types-PyYAML; extra == "dev"
|
|
39
|
+
Requires-Dist: pydantic>=2.4.0; extra == "dev"
|
|
44
40
|
Requires-Dist: mypy==1.17.1; extra == "dev"
|
|
45
41
|
Requires-Dist: tox-uv; extra == "dev"
|
|
46
|
-
Requires-Dist:
|
|
47
|
-
Requires-Dist:
|
|
48
|
-
Requires-Dist:
|
|
42
|
+
Requires-Dist: packaging; extra == "dev"
|
|
43
|
+
Requires-Dist: sqlalchemy-stubs; extra == "dev"
|
|
44
|
+
Requires-Dist: acryl-datahub[datahub-rest]==1.3.1.5rc1; extra == "dev"
|
|
45
|
+
Requires-Dist: types-cachetools; extra == "dev"
|
|
49
46
|
Requires-Dist: coverage>=5.1; extra == "dev"
|
|
47
|
+
Requires-Dist: types-six; extra == "dev"
|
|
48
|
+
Requires-Dist: types-requests; extra == "dev"
|
|
49
|
+
Requires-Dist: types-toml; extra == "dev"
|
|
50
50
|
Requires-Dist: pytest-cov>=2.8.1; extra == "dev"
|
|
51
|
+
Requires-Dist: pytest>=6.2.2; extra == "dev"
|
|
52
|
+
Requires-Dist: tenacity; extra == "dev"
|
|
51
53
|
Requires-Dist: types-setuptools; extra == "dev"
|
|
52
|
-
Requires-Dist: pydantic>=2.4.0; extra == "dev"
|
|
53
|
-
Requires-Dist: types-dataclasses; extra == "dev"
|
|
54
54
|
Requires-Dist: build; extra == "dev"
|
|
55
|
-
Requires-Dist:
|
|
55
|
+
Requires-Dist: deepdiff!=8.0.0; extra == "dev"
|
|
56
|
+
Requires-Dist: types-click==0.1.12; extra == "dev"
|
|
57
|
+
Requires-Dist: types-dataclasses; extra == "dev"
|
|
58
|
+
Requires-Dist: ruff==0.11.7; extra == "dev"
|
|
59
|
+
Requires-Dist: tox; extra == "dev"
|
|
60
|
+
Requires-Dist: apache-airflow<3,>=2.7.0; extra == "dev"
|
|
56
61
|
Requires-Dist: types-tabulate; extra == "dev"
|
|
57
|
-
Requires-Dist: types-six; extra == "dev"
|
|
58
|
-
Requires-Dist: types-requests; extra == "dev"
|
|
59
|
-
Requires-Dist: types-cachetools; extra == "dev"
|
|
60
|
-
Requires-Dist: acryl-datahub[datahub-rest,sql-parser]==1.3.1.4; extra == "dev"
|
|
61
|
-
Requires-Dist: sqlalchemy-stubs; extra == "dev"
|
|
62
|
-
Requires-Dist: pytest>=6.2.2; extra == "dev"
|
|
63
|
-
Requires-Dist: types-toml; extra == "dev"
|
|
64
62
|
Requires-Dist: types-python-dateutil; extra == "dev"
|
|
63
|
+
Requires-Dist: twine; extra == "dev"
|
|
64
|
+
Requires-Dist: acryl-datahub[datahub-rest,sql-parser]==1.3.1.5rc1; extra == "dev"
|
|
65
65
|
Provides-Extra: integration-tests
|
|
66
|
-
Requires-Dist:
|
|
67
|
-
Requires-Dist: apache-airflow[amazon,google,snowflake]>=2.0.2; extra == "integration-tests"
|
|
68
|
-
Requires-Dist: acryl-datahub[testing-utils]==1.3.1.4; extra == "integration-tests"
|
|
69
|
-
Requires-Dist: acryl-datahub[datahub-kafka]==1.3.1.4; extra == "integration-tests"
|
|
66
|
+
Requires-Dist: apache-airflow-providers-teradata; extra == "integration-tests"
|
|
70
67
|
Requires-Dist: snowflake-connector-python>=2.7.10; extra == "integration-tests"
|
|
68
|
+
Requires-Dist: acryl-datahub[testing-utils]==1.3.1.5rc1; extra == "integration-tests"
|
|
69
|
+
Requires-Dist: apache-airflow[amazon,google,snowflake]>=2.0.2; extra == "integration-tests"
|
|
70
|
+
Requires-Dist: acryl-datahub[datahub-kafka]==1.3.1.5rc1; extra == "integration-tests"
|
|
71
|
+
Requires-Dist: acryl-datahub[sync-file-emitter]==1.3.1.5rc1; extra == "integration-tests"
|
|
71
72
|
Requires-Dist: apache-airflow-providers-sqlite; extra == "integration-tests"
|
|
72
73
|
Requires-Dist: virtualenv; extra == "integration-tests"
|
|
73
74
|
Dynamic: classifier
|
|
@@ -2,8 +2,8 @@ datahub_airflow_plugin/__init__.py,sha256=NScUtA8N-m66Pyg0DO--YbPkrl48PK3UevpdQV
|
|
|
2
2
|
datahub_airflow_plugin/_airflow_shims.py,sha256=hLMTkANJzmH9sEcaiwNb0EZgD11vh-XnDBqFQ9yqjr4,1613
|
|
3
3
|
datahub_airflow_plugin/_config.py,sha256=qNbNC6YUGHf06RfOMsU7jpeNeE2ttxjUQIMEpxIhvyM,5221
|
|
4
4
|
datahub_airflow_plugin/_datahub_ol_adapter.py,sha256=RuzMyWZo7MeJzAFoBfkT4cdDw5g1iWshB_nXG7jLnR0,545
|
|
5
|
-
datahub_airflow_plugin/_extractors.py,sha256=
|
|
6
|
-
datahub_airflow_plugin/_version.py,sha256=
|
|
5
|
+
datahub_airflow_plugin/_extractors.py,sha256=o2amnv1ram58zYNiWpleQcpSREthin33LQ4yRLwoGxA,12759
|
|
6
|
+
datahub_airflow_plugin/_version.py,sha256=mFUiAN31A5PUxNLnvOGfHnDXajphGmegPa0MM9whufg,148
|
|
7
7
|
datahub_airflow_plugin/datahub_listener.py,sha256=plAU-DZ7EQmC1Ec4gzOaksqHnJLxaCYHkm2jEPTkgvc,31100
|
|
8
8
|
datahub_airflow_plugin/datahub_plugin.py,sha256=rbZhs7s5O3_MlkQw5aZToC2W5mMic_EpI3oybHB0ofw,1224
|
|
9
9
|
datahub_airflow_plugin/entities.py,sha256=xDZ-mZH7hjUkZbatWYUwI43_9B40wGiotlyQhiO8rEM,1987
|
|
@@ -26,8 +26,8 @@ datahub_airflow_plugin/operators/datahub_assertion_operator.py,sha256=j_P9M1a5qM
|
|
|
26
26
|
datahub_airflow_plugin/operators/datahub_assertion_sensor.py,sha256=QJIZZYQhqscj3bhBN5Sei-ABMRRAl2KiQxXTXcZQ51Q,2917
|
|
27
27
|
datahub_airflow_plugin/operators/datahub_operation_operator.py,sha256=KJ8M8jJ7UWW6kNbiS-rELc-kqCPkZ3ck7z51oAXGPSI,3351
|
|
28
28
|
datahub_airflow_plugin/operators/datahub_operation_sensor.py,sha256=U19fi5DpjBRWm_1ljXcjnspUzfa3mqYfOQZHjLk-ufI,3618
|
|
29
|
-
acryl_datahub_airflow_plugin-1.3.1.
|
|
30
|
-
acryl_datahub_airflow_plugin-1.3.1.
|
|
31
|
-
acryl_datahub_airflow_plugin-1.3.1.
|
|
32
|
-
acryl_datahub_airflow_plugin-1.3.1.
|
|
33
|
-
acryl_datahub_airflow_plugin-1.3.1.
|
|
29
|
+
acryl_datahub_airflow_plugin-1.3.1.5rc1.dist-info/METADATA,sha256=qSybyzWbxjUDvL88MWC5HVumK3DTVtsT4Ezl4fTnyWw,4088
|
|
30
|
+
acryl_datahub_airflow_plugin-1.3.1.5rc1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
31
|
+
acryl_datahub_airflow_plugin-1.3.1.5rc1.dist-info/entry_points.txt,sha256=HqmajDHtrsz0b5Lswe1-eeuObxdtucd9YoxH77jJBA8,179
|
|
32
|
+
acryl_datahub_airflow_plugin-1.3.1.5rc1.dist-info/top_level.txt,sha256=VBzisOQfzqL1WRbNyItaruf3kTigXltjzgqzbheaFp0,23
|
|
33
|
+
acryl_datahub_airflow_plugin-1.3.1.5rc1.dist-info/RECORD,,
|
|
@@ -67,6 +67,10 @@ class ExtractorManager(OLExtractorManager):
|
|
|
67
67
|
BigQueryInsertJobOperatorExtractor
|
|
68
68
|
)
|
|
69
69
|
|
|
70
|
+
self.task_to_extractor.extractors["TeradataOperator"] = (
|
|
71
|
+
TeradataOperatorExtractor
|
|
72
|
+
)
|
|
73
|
+
|
|
70
74
|
self._graph: Optional["DataHubGraph"] = None
|
|
71
75
|
|
|
72
76
|
@contextlib.contextmanager
|
|
@@ -333,4 +337,29 @@ def _snowflake_default_schema(self: "SnowflakeExtractor") -> Optional[str]:
|
|
|
333
337
|
or self.conn.schema
|
|
334
338
|
)
|
|
335
339
|
# TODO: Should we try a fallback of:
|
|
336
|
-
# execute_query_on_hook(self.hook, "SELECT current_schema();")
|
|
340
|
+
# execute_query_on_hook(self.hook, "SELECT current_schema();")
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
class TeradataOperatorExtractor(BaseExtractor):
|
|
344
|
+
"""Extractor for Teradata SQL operations.
|
|
345
|
+
|
|
346
|
+
Extracts lineage from TeradataOperator tasks by parsing the SQL queries
|
|
347
|
+
and understanding Teradata's two-tier database.table naming convention.
|
|
348
|
+
"""
|
|
349
|
+
|
|
350
|
+
def extract(self) -> Optional[TaskMetadata]:
|
|
351
|
+
from airflow.providers.teradata.operators.teradata import TeradataOperator
|
|
352
|
+
|
|
353
|
+
operator: "TeradataOperator" = self.operator
|
|
354
|
+
sql = operator.sql
|
|
355
|
+
if not sql:
|
|
356
|
+
self.log.warning("No query found in TeradataOperator")
|
|
357
|
+
return None
|
|
358
|
+
|
|
359
|
+
return _parse_sql_into_task_metadata(
|
|
360
|
+
self,
|
|
361
|
+
sql,
|
|
362
|
+
platform="teradata",
|
|
363
|
+
default_database=None,
|
|
364
|
+
default_schema=None,
|
|
365
|
+
)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|