apache-airflow-providers-snowflake 6.2.2__tar.gz → 6.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of apache-airflow-providers-snowflake might be problematic. Click here for more details.
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/PKG-INFO +11 -11
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/README.rst +5 -5
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/docs/changelog.rst +47 -0
- apache_airflow_providers_snowflake-6.3.0/docs/commits.rst +35 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/docs/connections/snowflake.rst +11 -2
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/docs/index.rst +6 -6
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/provider.yaml +6 -2
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/pyproject.toml +6 -6
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/src/airflow/providers/snowflake/__init__.py +3 -3
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/src/airflow/providers/snowflake/hooks/snowflake.py +49 -2
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/src/airflow/providers/snowflake/hooks/snowflake_sql_api.py +11 -37
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/src/airflow/providers/snowflake/utils/openlineage.py +45 -27
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/src/airflow/providers/snowflake/version_compat.py +0 -1
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/tests/unit/snowflake/hooks/test_snowflake.py +109 -8
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/tests/unit/snowflake/hooks/test_snowflake_sql_api.py +26 -14
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/tests/unit/snowflake/utils/test_openlineage.py +28 -12
- apache_airflow_providers_snowflake-6.2.2/docs/commits.rst +0 -979
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/docs/.latest-doc-only-change.txt +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/docs/conf.py +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/docs/decorators/index.rst +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/docs/decorators/snowpark.rst +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/docs/installing-providers-from-sources.rst +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/docs/integration-logos/Snowflake.png +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/docs/operators/copy_into_snowflake.rst +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/docs/operators/index.rst +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/docs/operators/snowflake.rst +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/docs/operators/snowpark.rst +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/docs/security.rst +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/src/airflow/__init__.py +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/src/airflow/providers/__init__.py +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/src/airflow/providers/snowflake/LICENSE +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/src/airflow/providers/snowflake/decorators/__init__.py +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/src/airflow/providers/snowflake/decorators/snowpark.py +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/src/airflow/providers/snowflake/get_provider_info.py +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/src/airflow/providers/snowflake/hooks/__init__.py +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/src/airflow/providers/snowflake/operators/__init__.py +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/src/airflow/providers/snowflake/operators/snowflake.py +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/src/airflow/providers/snowflake/operators/snowpark.py +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/src/airflow/providers/snowflake/transfers/__init__.py +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/src/airflow/providers/snowflake/transfers/copy_into_snowflake.py +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/src/airflow/providers/snowflake/triggers/__init__.py +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/src/airflow/providers/snowflake/triggers/snowflake_trigger.py +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/src/airflow/providers/snowflake/utils/__init__.py +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/src/airflow/providers/snowflake/utils/common.py +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/src/airflow/providers/snowflake/utils/snowpark.py +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/src/airflow/providers/snowflake/utils/sql_api_generate_jwt.py +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/tests/conftest.py +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/tests/system/__init__.py +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/tests/system/snowflake/__init__.py +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/tests/system/snowflake/example_copy_into_snowflake.py +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/tests/system/snowflake/example_snowflake.py +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/tests/system/snowflake/example_snowflake_snowflake_op_template_file.sql +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/tests/system/snowflake/example_snowpark_decorator.py +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/tests/system/snowflake/example_snowpark_operator.py +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/tests/unit/__init__.py +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/tests/unit/snowflake/__init__.py +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/tests/unit/snowflake/decorators/__init__.py +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/tests/unit/snowflake/decorators/test_snowpark.py +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/tests/unit/snowflake/hooks/__init__.py +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/tests/unit/snowflake/hooks/test_sql.py +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/tests/unit/snowflake/operators/__init__.py +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/tests/unit/snowflake/operators/test_snowflake.py +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/tests/unit/snowflake/operators/test_snowflake_sql.py +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/tests/unit/snowflake/operators/test_snowpark.py +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/tests/unit/snowflake/transfers/__init__.py +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/tests/unit/snowflake/transfers/test_copy_into_snowflake.py +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/tests/unit/snowflake/triggers/__init__.py +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/tests/unit/snowflake/triggers/test_snowflake.py +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/tests/unit/snowflake/utils/__init__.py +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/tests/unit/snowflake/utils/test_common.py +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/tests/unit/snowflake/utils/test_snowpark.py +0 -0
- {apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/tests/unit/snowflake/utils/test_sql_api_generate_jwt.py +0 -0
{apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/PKG-INFO
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: apache-airflow-providers-snowflake
|
|
3
|
-
Version: 6.
|
|
3
|
+
Version: 6.3.0
|
|
4
4
|
Summary: Provider package apache-airflow-providers-snowflake for Apache Airflow
|
|
5
5
|
Keywords: airflow-provider,snowflake,airflow,integration
|
|
6
6
|
Author-email: Apache Software Foundation <dev@airflow.apache.org>
|
|
@@ -20,18 +20,18 @@ Classifier: Programming Language :: Python :: 3.10
|
|
|
20
20
|
Classifier: Programming Language :: Python :: 3.11
|
|
21
21
|
Classifier: Programming Language :: Python :: 3.12
|
|
22
22
|
Classifier: Topic :: System :: Monitoring
|
|
23
|
-
Requires-Dist: apache-airflow>=2.
|
|
23
|
+
Requires-Dist: apache-airflow>=2.10.0
|
|
24
24
|
Requires-Dist: apache-airflow-providers-common-compat>=1.6.0
|
|
25
|
-
Requires-Dist: apache-airflow-providers-common-sql>=1.
|
|
25
|
+
Requires-Dist: apache-airflow-providers-common-sql>=1.21.0
|
|
26
26
|
Requires-Dist: pandas>=2.1.2,<2.2
|
|
27
27
|
Requires-Dist: pyarrow>=14.0.1
|
|
28
28
|
Requires-Dist: snowflake-connector-python>=3.7.1
|
|
29
29
|
Requires-Dist: snowflake-sqlalchemy>=1.4.0
|
|
30
30
|
Requires-Dist: snowflake-snowpark-python>=1.17.0;python_version<'3.12'
|
|
31
|
-
Requires-Dist: apache-airflow-providers-openlineage ; extra == "openlineage"
|
|
31
|
+
Requires-Dist: apache-airflow-providers-openlineage>=2.3.0 ; extra == "openlineage"
|
|
32
32
|
Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
|
|
33
|
-
Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-snowflake/6.
|
|
34
|
-
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-snowflake/6.
|
|
33
|
+
Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-snowflake/6.3.0/changelog.html
|
|
34
|
+
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-snowflake/6.3.0
|
|
35
35
|
Project-URL: Mastodon, https://fosstodon.org/@airflow
|
|
36
36
|
Project-URL: Slack Chat, https://s.apache.org/airflow-slack
|
|
37
37
|
Project-URL: Source Code, https://github.com/apache/airflow
|
|
@@ -63,7 +63,7 @@ Provides-Extra: openlineage
|
|
|
63
63
|
|
|
64
64
|
Package ``apache-airflow-providers-snowflake``
|
|
65
65
|
|
|
66
|
-
Release: ``6.
|
|
66
|
+
Release: ``6.3.0``
|
|
67
67
|
|
|
68
68
|
|
|
69
69
|
`Snowflake <https://www.snowflake.com/>`__
|
|
@@ -76,7 +76,7 @@ This is a provider package for ``snowflake`` provider. All classes for this prov
|
|
|
76
76
|
are in ``airflow.providers.snowflake`` python package.
|
|
77
77
|
|
|
78
78
|
You can find package information and changelog for the provider
|
|
79
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-snowflake/6.
|
|
79
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-snowflake/6.3.0/>`_.
|
|
80
80
|
|
|
81
81
|
Installation
|
|
82
82
|
------------
|
|
@@ -93,9 +93,9 @@ Requirements
|
|
|
93
93
|
========================================== =====================================
|
|
94
94
|
PIP package Version required
|
|
95
95
|
========================================== =====================================
|
|
96
|
-
``apache-airflow`` ``>=2.
|
|
96
|
+
``apache-airflow`` ``>=2.10.0``
|
|
97
97
|
``apache-airflow-providers-common-compat`` ``>=1.6.0``
|
|
98
|
-
``apache-airflow-providers-common-sql`` ``>=1.
|
|
98
|
+
``apache-airflow-providers-common-sql`` ``>=1.21.0``
|
|
99
99
|
``pandas`` ``>=2.1.2,<2.2``
|
|
100
100
|
``pyarrow`` ``>=14.0.1``
|
|
101
101
|
``snowflake-connector-python`` ``>=3.7.1``
|
|
@@ -125,5 +125,5 @@ Dependent package
|
|
|
125
125
|
================================================================================================================== =================
|
|
126
126
|
|
|
127
127
|
The changelog for the provider package can be found in the
|
|
128
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-snowflake/6.
|
|
128
|
+
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-snowflake/6.3.0/changelog.html>`_.
|
|
129
129
|
|
{apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/README.rst
RENAMED
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
|
|
24
24
|
Package ``apache-airflow-providers-snowflake``
|
|
25
25
|
|
|
26
|
-
Release: ``6.
|
|
26
|
+
Release: ``6.3.0``
|
|
27
27
|
|
|
28
28
|
|
|
29
29
|
`Snowflake <https://www.snowflake.com/>`__
|
|
@@ -36,7 +36,7 @@ This is a provider package for ``snowflake`` provider. All classes for this prov
|
|
|
36
36
|
are in ``airflow.providers.snowflake`` python package.
|
|
37
37
|
|
|
38
38
|
You can find package information and changelog for the provider
|
|
39
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-snowflake/6.
|
|
39
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-snowflake/6.3.0/>`_.
|
|
40
40
|
|
|
41
41
|
Installation
|
|
42
42
|
------------
|
|
@@ -53,9 +53,9 @@ Requirements
|
|
|
53
53
|
========================================== =====================================
|
|
54
54
|
PIP package Version required
|
|
55
55
|
========================================== =====================================
|
|
56
|
-
``apache-airflow`` ``>=2.
|
|
56
|
+
``apache-airflow`` ``>=2.10.0``
|
|
57
57
|
``apache-airflow-providers-common-compat`` ``>=1.6.0``
|
|
58
|
-
``apache-airflow-providers-common-sql`` ``>=1.
|
|
58
|
+
``apache-airflow-providers-common-sql`` ``>=1.21.0``
|
|
59
59
|
``pandas`` ``>=2.1.2,<2.2``
|
|
60
60
|
``pyarrow`` ``>=14.0.1``
|
|
61
61
|
``snowflake-connector-python`` ``>=3.7.1``
|
|
@@ -85,4 +85,4 @@ Dependent package
|
|
|
85
85
|
================================================================================================================== =================
|
|
86
86
|
|
|
87
87
|
The changelog for the provider package can be found in the
|
|
88
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-snowflake/6.
|
|
88
|
+
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-snowflake/6.3.0/changelog.html>`_.
|
|
@@ -27,6 +27,53 @@
|
|
|
27
27
|
Changelog
|
|
28
28
|
---------
|
|
29
29
|
|
|
30
|
+
6.3.0
|
|
31
|
+
.....
|
|
32
|
+
|
|
33
|
+
.. note::
|
|
34
|
+
This release of provider is only available for Airflow 2.10+ as explained in the
|
|
35
|
+
`Apache Airflow providers support policy <https://github.com/apache/airflow/blob/main/PROVIDERS.rst#minimum-supported-version-of-airflow-for-community-managed-providers>`_.
|
|
36
|
+
|
|
37
|
+
.. note::
|
|
38
|
+
``private_key_content`` in Snowflake connection should now be base64 encoded. To encode your private key, you can use the following Python snippet:
|
|
39
|
+
|
|
40
|
+
.. code-block:: python
|
|
41
|
+
|
|
42
|
+
import base64
|
|
43
|
+
|
|
44
|
+
with open("path/to/your/private_key.pem", "rb") as key_file:
|
|
45
|
+
encoded_key = base64.b64encode(key_file.read()).decode("utf-8")
|
|
46
|
+
print(encoded_key)
|
|
47
|
+
|
|
48
|
+
Features
|
|
49
|
+
~~~~~~~~
|
|
50
|
+
|
|
51
|
+
* ``Adding OAuth support for SnowflakeHook (#47191)``
|
|
52
|
+
|
|
53
|
+
Bug Fixes
|
|
54
|
+
~~~~~~~~~
|
|
55
|
+
|
|
56
|
+
* ``make 'private_key_content' in snowflake connection to be a base64 encoded string (#49467)``
|
|
57
|
+
* ``Fix SnowflakeSqlApiHook backwards compatibility for get_oauth_token method (#49482)``
|
|
58
|
+
* ``Fix mypy for get_oauth_token signature in SnowflakeSqlApiHook (#49449)``
|
|
59
|
+
* ``Fix infinite recursive call of _get_conn_params while getting oauth token from snowflake (#50344)``
|
|
60
|
+
* ``Fix: adjust dag_run extraction for Airflow 3 in OL utils (#50346)``
|
|
61
|
+
|
|
62
|
+
Misc
|
|
63
|
+
~~~~
|
|
64
|
+
|
|
65
|
+
* ``Remove AIRFLOW_2_10_PLUS conditions (#49877)``
|
|
66
|
+
* ``Bump min Airflow version in providers to 2.10 (#49843)``
|
|
67
|
+
* ``enhance: logs SQL before execution in 'snowflake' and 'databricks_sql' (#48942)``
|
|
68
|
+
* ``chore: import paths use the stable functions (#49460)``
|
|
69
|
+
* ``add root parent information to OpenLineage events (#49237)``
|
|
70
|
+
|
|
71
|
+
.. Below changes are excluded from the changelog. Move them to
|
|
72
|
+
appropriate section above if needed. Do not delete the lines(!):
|
|
73
|
+
* ``Avoid committing history for providers (#49907)``
|
|
74
|
+
* ``Update description of provider.yaml dependencies (#50231)``
|
|
75
|
+
* ``Prepare ad hoc release for providers May 2025 (#50166)``
|
|
76
|
+
|
|
30
77
|
6.2.2
|
|
31
78
|
.....
|
|
32
79
|
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
|
|
2
|
+
.. Licensed to the Apache Software Foundation (ASF) under one
|
|
3
|
+
or more contributor license agreements. See the NOTICE file
|
|
4
|
+
distributed with this work for additional information
|
|
5
|
+
regarding copyright ownership. The ASF licenses this file
|
|
6
|
+
to you under the Apache License, Version 2.0 (the
|
|
7
|
+
"License"); you may not use this file except in compliance
|
|
8
|
+
with the License. You may obtain a copy of the License at
|
|
9
|
+
|
|
10
|
+
.. http://www.apache.org/licenses/LICENSE-2.0
|
|
11
|
+
|
|
12
|
+
.. Unless required by applicable law or agreed to in writing,
|
|
13
|
+
software distributed under the License is distributed on an
|
|
14
|
+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
15
|
+
KIND, either express or implied. See the License for the
|
|
16
|
+
specific language governing permissions and limitations
|
|
17
|
+
under the License.
|
|
18
|
+
|
|
19
|
+
.. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN!
|
|
20
|
+
|
|
21
|
+
.. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
|
|
22
|
+
`PROVIDER_COMMITS_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
|
|
23
|
+
|
|
24
|
+
.. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN!
|
|
25
|
+
|
|
26
|
+
Package apache-airflow-providers-snowflake
|
|
27
|
+
------------------------------------------------------
|
|
28
|
+
|
|
29
|
+
`Snowflake <https://www.snowflake.com/>`__
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
This is detailed commit list of changes for versions provider package: ``snowflake``.
|
|
33
|
+
For high-level changelog, see :doc:`package information including changelog <index>`.
|
|
34
|
+
|
|
35
|
+
.. airflow-providers-commits::
|
|
@@ -39,10 +39,11 @@ Configuring the Connection
|
|
|
39
39
|
--------------------------
|
|
40
40
|
|
|
41
41
|
Login
|
|
42
|
-
Specify the snowflake username.
|
|
42
|
+
Specify the snowflake username. For OAuth, the OAuth Client ID.
|
|
43
43
|
|
|
44
44
|
Password
|
|
45
45
|
Specify the snowflake password. For public key authentication, the passphrase for the private key.
|
|
46
|
+
For OAuth, the OAuth Client Secret.
|
|
46
47
|
|
|
47
48
|
Schema (optional)
|
|
48
49
|
Specify the snowflake schema to be used.
|
|
@@ -59,7 +60,15 @@ Extra (optional)
|
|
|
59
60
|
* ``authenticator``: To connect using OAuth set this parameter ``oauth``.
|
|
60
61
|
* ``refresh_token``: Specify refresh_token for OAuth connection.
|
|
61
62
|
* ``private_key_file``: Specify the path to the private key file.
|
|
62
|
-
* ``private_key_content``: Specify the content of the private key file.
|
|
63
|
+
* ``private_key_content``: Specify the content of the private key file in base64 encoded format. You can use the following Python code to encode the private key:
|
|
64
|
+
|
|
65
|
+
.. code-block:: python
|
|
66
|
+
|
|
67
|
+
import base64
|
|
68
|
+
|
|
69
|
+
with open("path/to/private_key.pem", "rb") as key_file:
|
|
70
|
+
private_key_content = base64.b64encode(key_file.read()).decode("utf-8")
|
|
71
|
+
print(private_key_content)
|
|
63
72
|
* ``session_parameters``: Specify `session level parameters <https://docs.snowflake.com/en/user-guide/python-connector-example.html#setting-session-parameters>`_.
|
|
64
73
|
* ``insecure_mode``: Turn off OCSP certificate checks. For details, see: `How To: Turn Off OCSP Checking in Snowflake Client Drivers - Snowflake Community <https://community.snowflake.com/s/article/How-to-turn-off-OCSP-checking-in-Snowflake-client-drivers>`_.
|
|
65
74
|
* ``host``: Target Snowflake hostname to connect to (e.g., for local testing with LocalStack).
|
{apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/docs/index.rst
RENAMED
|
@@ -78,7 +78,7 @@ apache-airflow-providers-snowflake package
|
|
|
78
78
|
`Snowflake <https://www.snowflake.com/>`__
|
|
79
79
|
|
|
80
80
|
|
|
81
|
-
Release: 6.
|
|
81
|
+
Release: 6.3.0
|
|
82
82
|
|
|
83
83
|
Provider package
|
|
84
84
|
----------------
|
|
@@ -96,14 +96,14 @@ For the minimum Airflow version supported, see ``Requirements`` below.
|
|
|
96
96
|
Requirements
|
|
97
97
|
------------
|
|
98
98
|
|
|
99
|
-
The minimum Apache Airflow version supported by this provider distribution is ``2.
|
|
99
|
+
The minimum Apache Airflow version supported by this provider distribution is ``2.10.0``.
|
|
100
100
|
|
|
101
101
|
========================================== =====================================
|
|
102
102
|
PIP package Version required
|
|
103
103
|
========================================== =====================================
|
|
104
|
-
``apache-airflow`` ``>=2.
|
|
104
|
+
``apache-airflow`` ``>=2.10.0``
|
|
105
105
|
``apache-airflow-providers-common-compat`` ``>=1.6.0``
|
|
106
|
-
``apache-airflow-providers-common-sql`` ``>=1.
|
|
106
|
+
``apache-airflow-providers-common-sql`` ``>=1.21.0``
|
|
107
107
|
``pandas`` ``>=2.1.2,<2.2``
|
|
108
108
|
``pyarrow`` ``>=14.0.1``
|
|
109
109
|
``snowflake-connector-python`` ``>=3.7.1``
|
|
@@ -138,5 +138,5 @@ Downloading official packages
|
|
|
138
138
|
You can download officially released packages and verify their checksums and signatures from the
|
|
139
139
|
`Official Apache Download site <https://downloads.apache.org/airflow/providers/>`_
|
|
140
140
|
|
|
141
|
-
* `The apache-airflow-providers-snowflake 6.
|
|
142
|
-
* `The apache-airflow-providers-snowflake 6.
|
|
141
|
+
* `The apache-airflow-providers-snowflake 6.3.0 sdist package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_snowflake-6.3.0.tar.gz>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_snowflake-6.3.0.tar.gz.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_snowflake-6.3.0.tar.gz.sha512>`__)
|
|
142
|
+
* `The apache-airflow-providers-snowflake 6.3.0 wheel package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_snowflake-6.3.0-py3-none-any.whl>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_snowflake-6.3.0-py3-none-any.whl.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_snowflake-6.3.0-py3-none-any.whl.sha512>`__)
|
{apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/provider.yaml
RENAMED
|
@@ -22,9 +22,13 @@ description: |
|
|
|
22
22
|
`Snowflake <https://www.snowflake.com/>`__
|
|
23
23
|
|
|
24
24
|
state: ready
|
|
25
|
-
source-date-epoch:
|
|
26
|
-
#
|
|
25
|
+
source-date-epoch: 1746344752
|
|
26
|
+
# Note that those versions are maintained by release manager - do not update them manually
|
|
27
|
+
# with the exception of case where other provider in sources has >= new provider version.
|
|
28
|
+
# In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have
|
|
29
|
+
# to be done in the same PR
|
|
27
30
|
versions:
|
|
31
|
+
- 6.3.0
|
|
28
32
|
- 6.2.2
|
|
29
33
|
- 6.2.1
|
|
30
34
|
- 6.2.0
|
{apache_airflow_providers_snowflake-6.2.2 → apache_airflow_providers_snowflake-6.3.0}/pyproject.toml
RENAMED
|
@@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi"
|
|
|
25
25
|
|
|
26
26
|
[project]
|
|
27
27
|
name = "apache-airflow-providers-snowflake"
|
|
28
|
-
version = "6.
|
|
28
|
+
version = "6.3.0"
|
|
29
29
|
description = "Provider package apache-airflow-providers-snowflake for Apache Airflow"
|
|
30
30
|
readme = "README.rst"
|
|
31
31
|
authors = [
|
|
@@ -57,9 +57,9 @@ requires-python = "~=3.9"
|
|
|
57
57
|
# Make sure to run ``breeze static-checks --type update-providers-dependencies --all-files``
|
|
58
58
|
# After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build``
|
|
59
59
|
dependencies = [
|
|
60
|
-
"apache-airflow>=2.
|
|
60
|
+
"apache-airflow>=2.10.0",
|
|
61
61
|
"apache-airflow-providers-common-compat>=1.6.0",
|
|
62
|
-
"apache-airflow-providers-common-sql>=1.
|
|
62
|
+
"apache-airflow-providers-common-sql>=1.21.0",
|
|
63
63
|
# In pandas 2.2 minimal version of the sqlalchemy is 2.0
|
|
64
64
|
# https://pandas.pydata.org/docs/whatsnew/v2.2.0.html#increased-minimum-versions-for-dependencies
|
|
65
65
|
# However Airflow not fully supports it yet: https://github.com/apache/airflow/issues/28723
|
|
@@ -75,7 +75,7 @@ dependencies = [
|
|
|
75
75
|
# Any change in the dependencies is preserved when the file is regenerated
|
|
76
76
|
[project.optional-dependencies]
|
|
77
77
|
"openlineage" = [
|
|
78
|
-
"apache-airflow-providers-openlineage"
|
|
78
|
+
"apache-airflow-providers-openlineage>=2.3.0"
|
|
79
79
|
]
|
|
80
80
|
|
|
81
81
|
[dependency-groups]
|
|
@@ -116,8 +116,8 @@ apache-airflow-providers-common-sql = {workspace = true}
|
|
|
116
116
|
apache-airflow-providers-standard = {workspace = true}
|
|
117
117
|
|
|
118
118
|
[project.urls]
|
|
119
|
-
"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-snowflake/6.
|
|
120
|
-
"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-snowflake/6.
|
|
119
|
+
"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-snowflake/6.3.0"
|
|
120
|
+
"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-snowflake/6.3.0/changelog.html"
|
|
121
121
|
"Bug Tracker" = "https://github.com/apache/airflow/issues"
|
|
122
122
|
"Source Code" = "https://github.com/apache/airflow"
|
|
123
123
|
"Slack Chat" = "https://s.apache.org/airflow-slack"
|
|
@@ -29,11 +29,11 @@ from airflow import __version__ as airflow_version
|
|
|
29
29
|
|
|
30
30
|
__all__ = ["__version__"]
|
|
31
31
|
|
|
32
|
-
__version__ = "6.
|
|
32
|
+
__version__ = "6.3.0"
|
|
33
33
|
|
|
34
34
|
if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
|
|
35
|
-
"2.
|
|
35
|
+
"2.10.0"
|
|
36
36
|
):
|
|
37
37
|
raise RuntimeError(
|
|
38
|
-
f"The package `apache-airflow-providers-snowflake:{__version__}` needs Apache Airflow 2.
|
|
38
|
+
f"The package `apache-airflow-providers-snowflake:{__version__}` needs Apache Airflow 2.10.0+"
|
|
39
39
|
)
|
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
# under the License.
|
|
18
18
|
from __future__ import annotations
|
|
19
19
|
|
|
20
|
+
import base64
|
|
20
21
|
import os
|
|
21
22
|
from collections.abc import Iterable, Mapping
|
|
22
23
|
from contextlib import closing, contextmanager
|
|
@@ -26,15 +27,18 @@ from pathlib import Path
|
|
|
26
27
|
from typing import TYPE_CHECKING, Any, Callable, TypeVar, overload
|
|
27
28
|
from urllib.parse import urlparse
|
|
28
29
|
|
|
30
|
+
import requests
|
|
29
31
|
from cryptography.hazmat.backends import default_backend
|
|
30
32
|
from cryptography.hazmat.primitives import serialization
|
|
33
|
+
from requests.auth import HTTPBasicAuth
|
|
31
34
|
from snowflake import connector
|
|
32
35
|
from snowflake.connector import DictCursor, SnowflakeConnection, util_text
|
|
33
36
|
from snowflake.sqlalchemy import URL
|
|
34
37
|
from sqlalchemy import create_engine
|
|
35
38
|
|
|
36
39
|
from airflow.exceptions import AirflowException
|
|
37
|
-
from airflow.providers.common.sql.hooks.
|
|
40
|
+
from airflow.providers.common.sql.hooks.handlers import return_single_query_results
|
|
41
|
+
from airflow.providers.common.sql.hooks.sql import DbApiHook
|
|
38
42
|
from airflow.providers.snowflake.utils.openlineage import fix_snowflake_sqlalchemy_uri
|
|
39
43
|
from airflow.utils.strings import to_boolean
|
|
40
44
|
|
|
@@ -185,6 +189,45 @@ class SnowflakeHook(DbApiHook):
|
|
|
185
189
|
return extra_dict[field_name] or None
|
|
186
190
|
return extra_dict.get(backcompat_key) or None
|
|
187
191
|
|
|
192
|
+
@property
|
|
193
|
+
def account_identifier(self) -> str:
|
|
194
|
+
"""Get snowflake account identifier."""
|
|
195
|
+
conn_config = self._get_conn_params
|
|
196
|
+
account_identifier = f"https://{conn_config['account']}"
|
|
197
|
+
|
|
198
|
+
if conn_config["region"]:
|
|
199
|
+
account_identifier += f".{conn_config['region']}"
|
|
200
|
+
|
|
201
|
+
return account_identifier
|
|
202
|
+
|
|
203
|
+
def get_oauth_token(self, conn_config: dict | None = None) -> str:
|
|
204
|
+
"""Generate temporary OAuth access token using refresh token in connection details."""
|
|
205
|
+
if conn_config is None:
|
|
206
|
+
conn_config = self._get_conn_params
|
|
207
|
+
|
|
208
|
+
url = f"https://{conn_config['account']}.snowflakecomputing.com/oauth/token-request"
|
|
209
|
+
|
|
210
|
+
data = {
|
|
211
|
+
"grant_type": "refresh_token",
|
|
212
|
+
"refresh_token": conn_config["refresh_token"],
|
|
213
|
+
"redirect_uri": conn_config.get("redirect_uri", "https://localhost.com"),
|
|
214
|
+
}
|
|
215
|
+
response = requests.post(
|
|
216
|
+
url,
|
|
217
|
+
data=data,
|
|
218
|
+
headers={
|
|
219
|
+
"Content-Type": "application/x-www-form-urlencoded",
|
|
220
|
+
},
|
|
221
|
+
auth=HTTPBasicAuth(conn_config["client_id"], conn_config["client_secret"]), # type: ignore[arg-type]
|
|
222
|
+
)
|
|
223
|
+
|
|
224
|
+
try:
|
|
225
|
+
response.raise_for_status()
|
|
226
|
+
except requests.exceptions.HTTPError as e: # pragma: no cover
|
|
227
|
+
msg = f"Response: {e.response.content.decode()} Status Code: {e.response.status_code}"
|
|
228
|
+
raise AirflowException(msg)
|
|
229
|
+
return response.json()["access_token"]
|
|
230
|
+
|
|
188
231
|
@cached_property
|
|
189
232
|
def _get_conn_params(self) -> dict[str, str | None]:
|
|
190
233
|
"""
|
|
@@ -262,7 +305,7 @@ class SnowflakeHook(DbApiHook):
|
|
|
262
305
|
raise ValueError("The private_key_file size is too big. Please keep it less than 4 KB.")
|
|
263
306
|
private_key_pem = Path(private_key_file_path).read_bytes()
|
|
264
307
|
elif private_key_content:
|
|
265
|
-
private_key_pem =
|
|
308
|
+
private_key_pem = base64.b64decode(private_key_content)
|
|
266
309
|
|
|
267
310
|
if private_key_pem:
|
|
268
311
|
passphrase = None
|
|
@@ -289,8 +332,11 @@ class SnowflakeHook(DbApiHook):
|
|
|
289
332
|
conn_config["client_id"] = conn.login
|
|
290
333
|
conn_config["client_secret"] = conn.password
|
|
291
334
|
conn_config.pop("login", None)
|
|
335
|
+
conn_config.pop("user", None)
|
|
292
336
|
conn_config.pop("password", None)
|
|
293
337
|
|
|
338
|
+
conn_config["token"] = self.get_oauth_token(conn_config=conn_config)
|
|
339
|
+
|
|
294
340
|
# configure custom target hostname and port, if specified
|
|
295
341
|
snowflake_host = extra_dict.get("host")
|
|
296
342
|
snowflake_port = extra_dict.get("port")
|
|
@@ -472,6 +518,7 @@ class SnowflakeHook(DbApiHook):
|
|
|
472
518
|
with self._get_cursor(conn, return_dictionaries) as cur:
|
|
473
519
|
results = []
|
|
474
520
|
for sql_statement in sql_list:
|
|
521
|
+
self.log.info("Running statement: %s, parameters: %s", sql_statement, parameters)
|
|
475
522
|
self._run_command(cur, sql_statement, parameters) # type: ignore[attr-defined]
|
|
476
523
|
|
|
477
524
|
if handler is not None:
|
|
@@ -16,7 +16,9 @@
|
|
|
16
16
|
# under the License.
|
|
17
17
|
from __future__ import annotations
|
|
18
18
|
|
|
19
|
+
import base64
|
|
19
20
|
import uuid
|
|
21
|
+
import warnings
|
|
20
22
|
from datetime import timedelta
|
|
21
23
|
from pathlib import Path
|
|
22
24
|
from typing import Any
|
|
@@ -25,9 +27,8 @@ import aiohttp
|
|
|
25
27
|
import requests
|
|
26
28
|
from cryptography.hazmat.backends import default_backend
|
|
27
29
|
from cryptography.hazmat.primitives import serialization
|
|
28
|
-
from requests.auth import HTTPBasicAuth
|
|
29
30
|
|
|
30
|
-
from airflow.exceptions import AirflowException
|
|
31
|
+
from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
|
|
31
32
|
from airflow.providers.snowflake.hooks.snowflake import SnowflakeHook
|
|
32
33
|
from airflow.providers.snowflake.utils.sql_api_generate_jwt import JWTGenerator
|
|
33
34
|
|
|
@@ -83,17 +84,6 @@ class SnowflakeSqlApiHook(SnowflakeHook):
|
|
|
83
84
|
super().__init__(snowflake_conn_id, *args, **kwargs)
|
|
84
85
|
self.private_key: Any = None
|
|
85
86
|
|
|
86
|
-
@property
|
|
87
|
-
def account_identifier(self) -> str:
|
|
88
|
-
"""Returns snowflake account identifier."""
|
|
89
|
-
conn_config = self._get_conn_params
|
|
90
|
-
account_identifier = f"https://{conn_config['account']}"
|
|
91
|
-
|
|
92
|
-
if conn_config["region"]:
|
|
93
|
-
account_identifier += f".{conn_config['region']}"
|
|
94
|
-
|
|
95
|
-
return account_identifier
|
|
96
|
-
|
|
97
87
|
def get_private_key(self) -> None:
|
|
98
88
|
"""Get the private key from snowflake connection."""
|
|
99
89
|
conn = self.get_connection(self.snowflake_conn_id)
|
|
@@ -120,7 +110,7 @@ class SnowflakeSqlApiHook(SnowflakeHook):
|
|
|
120
110
|
if private_key_file:
|
|
121
111
|
private_key_pem = Path(private_key_file).read_bytes()
|
|
122
112
|
elif private_key_content:
|
|
123
|
-
private_key_pem =
|
|
113
|
+
private_key_pem = base64.b64decode(private_key_content)
|
|
124
114
|
|
|
125
115
|
if private_key_pem:
|
|
126
116
|
passphrase = None
|
|
@@ -201,7 +191,7 @@ class SnowflakeSqlApiHook(SnowflakeHook):
|
|
|
201
191
|
if all(
|
|
202
192
|
[conn_config.get("refresh_token"), conn_config.get("client_id"), conn_config.get("client_secret")]
|
|
203
193
|
):
|
|
204
|
-
oauth_token = self.get_oauth_token()
|
|
194
|
+
oauth_token = self.get_oauth_token(conn_config=conn_config)
|
|
205
195
|
headers = {
|
|
206
196
|
"Content-Type": "application/json",
|
|
207
197
|
"Authorization": f"Bearer {oauth_token}",
|
|
@@ -232,30 +222,14 @@ class SnowflakeSqlApiHook(SnowflakeHook):
|
|
|
232
222
|
}
|
|
233
223
|
return headers
|
|
234
224
|
|
|
235
|
-
def get_oauth_token(self) -> str:
|
|
225
|
+
def get_oauth_token(self, conn_config: dict[str, Any] | None = None) -> str:
|
|
236
226
|
"""Generate temporary OAuth access token using refresh token in connection details."""
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
"refresh_token": conn_config["refresh_token"],
|
|
242
|
-
"redirect_uri": conn_config.get("redirect_uri", "https://localhost.com"),
|
|
243
|
-
}
|
|
244
|
-
response = requests.post(
|
|
245
|
-
url,
|
|
246
|
-
data=data,
|
|
247
|
-
headers={
|
|
248
|
-
"Content-Type": "application/x-www-form-urlencoded",
|
|
249
|
-
},
|
|
250
|
-
auth=HTTPBasicAuth(conn_config["client_id"], conn_config["client_secret"]), # type: ignore[arg-type]
|
|
227
|
+
warnings.warn(
|
|
228
|
+
"This method is deprecated. Please use `get_oauth_token` method from `SnowflakeHook` instead. ",
|
|
229
|
+
AirflowProviderDeprecationWarning,
|
|
230
|
+
stacklevel=2,
|
|
251
231
|
)
|
|
252
|
-
|
|
253
|
-
try:
|
|
254
|
-
response.raise_for_status()
|
|
255
|
-
except requests.exceptions.HTTPError as e: # pragma: no cover
|
|
256
|
-
msg = f"Response: {e.response.content.decode()} Status Code: {e.response.status_code}"
|
|
257
|
-
raise AirflowException(msg)
|
|
258
|
-
return response.json()["access_token"]
|
|
232
|
+
return super().get_oauth_token(conn_config=conn_config)
|
|
259
233
|
|
|
260
234
|
def get_request_url_header_params(self, query_id: str) -> tuple[dict[str, Any], dict[str, Any], str]:
|
|
261
235
|
"""
|
|
@@ -23,9 +23,8 @@ from typing import TYPE_CHECKING
|
|
|
23
23
|
from urllib.parse import quote, urlparse, urlunparse
|
|
24
24
|
|
|
25
25
|
from airflow.providers.common.compat.openlineage.check import require_openlineage_version
|
|
26
|
-
from airflow.providers.snowflake.version_compat import
|
|
26
|
+
from airflow.providers.snowflake.version_compat import AIRFLOW_V_3_0_PLUS
|
|
27
27
|
from airflow.utils import timezone
|
|
28
|
-
from airflow.utils.state import TaskInstanceState
|
|
29
28
|
|
|
30
29
|
if TYPE_CHECKING:
|
|
31
30
|
from openlineage.client.event_v2 import RunEvent
|
|
@@ -98,6 +97,28 @@ def fix_snowflake_sqlalchemy_uri(uri: str) -> str:
|
|
|
98
97
|
return urlunparse((parts.scheme, hostname, parts.path, parts.params, parts.query, parts.fragment))
|
|
99
98
|
|
|
100
99
|
|
|
100
|
+
def _get_logical_date(task_instance):
|
|
101
|
+
# todo: remove when min airflow version >= 3.0
|
|
102
|
+
if AIRFLOW_V_3_0_PLUS:
|
|
103
|
+
dagrun = task_instance.get_template_context()["dag_run"]
|
|
104
|
+
return dagrun.logical_date or dagrun.run_after
|
|
105
|
+
|
|
106
|
+
if hasattr(task_instance, "logical_date"):
|
|
107
|
+
date = task_instance.logical_date
|
|
108
|
+
else:
|
|
109
|
+
date = task_instance.execution_date
|
|
110
|
+
|
|
111
|
+
return date
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _get_dag_run_clear_number(task_instance):
|
|
115
|
+
# todo: remove when min airflow version >= 3.0
|
|
116
|
+
if AIRFLOW_V_3_0_PLUS:
|
|
117
|
+
dagrun = task_instance.get_template_context()["dag_run"]
|
|
118
|
+
return dagrun.clear_number
|
|
119
|
+
return task_instance.dag_run.clear_number
|
|
120
|
+
|
|
121
|
+
|
|
101
122
|
# todo: move this run_id logic into OpenLineage's listener to avoid differences
|
|
102
123
|
def _get_ol_run_id(task_instance) -> str:
|
|
103
124
|
"""
|
|
@@ -109,38 +130,27 @@ def _get_ol_run_id(task_instance) -> str:
|
|
|
109
130
|
"""
|
|
110
131
|
from airflow.providers.openlineage.plugins.adapter import OpenLineageAdapter
|
|
111
132
|
|
|
112
|
-
def _get_logical_date():
|
|
113
|
-
# todo: remove when min airflow version >= 3.0
|
|
114
|
-
if AIRFLOW_V_3_0_PLUS:
|
|
115
|
-
dagrun = task_instance.get_template_context()["dag_run"]
|
|
116
|
-
return dagrun.logical_date or dagrun.run_after
|
|
117
|
-
|
|
118
|
-
if hasattr(task_instance, "logical_date"):
|
|
119
|
-
date = task_instance.logical_date
|
|
120
|
-
else:
|
|
121
|
-
date = task_instance.execution_date
|
|
122
|
-
|
|
123
|
-
return date
|
|
124
|
-
|
|
125
|
-
def _get_try_number_success():
|
|
126
|
-
"""We are running this in the _on_complete, so need to adjust for try_num changes."""
|
|
127
|
-
# todo: remove when min airflow version >= 2.10.0
|
|
128
|
-
if AIRFLOW_V_2_10_PLUS:
|
|
129
|
-
return task_instance.try_number
|
|
130
|
-
if task_instance.state == TaskInstanceState.SUCCESS:
|
|
131
|
-
return task_instance.try_number - 1
|
|
132
|
-
return task_instance.try_number
|
|
133
|
-
|
|
134
133
|
# Generate same OL run id as is generated for current task instance
|
|
135
134
|
return OpenLineageAdapter.build_task_instance_run_id(
|
|
136
135
|
dag_id=task_instance.dag_id,
|
|
137
136
|
task_id=task_instance.task_id,
|
|
138
|
-
logical_date=_get_logical_date(),
|
|
139
|
-
try_number=
|
|
137
|
+
logical_date=_get_logical_date(task_instance),
|
|
138
|
+
try_number=task_instance.try_number,
|
|
140
139
|
map_index=task_instance.map_index,
|
|
141
140
|
)
|
|
142
141
|
|
|
143
142
|
|
|
143
|
+
# todo: move this run_id logic into OpenLineage's listener to avoid differences
|
|
144
|
+
def _get_ol_dag_run_id(task_instance) -> str:
|
|
145
|
+
from airflow.providers.openlineage.plugins.adapter import OpenLineageAdapter
|
|
146
|
+
|
|
147
|
+
return OpenLineageAdapter.build_dag_run_id(
|
|
148
|
+
dag_id=task_instance.dag_id,
|
|
149
|
+
logical_date=_get_logical_date(task_instance),
|
|
150
|
+
clear_number=_get_dag_run_clear_number(task_instance),
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
|
|
144
154
|
def _get_parent_run_facet(task_instance):
|
|
145
155
|
"""
|
|
146
156
|
Retrieve the ParentRunFacet associated with a specific Airflow task instance.
|
|
@@ -154,6 +164,7 @@ def _get_parent_run_facet(task_instance):
|
|
|
154
164
|
from airflow.providers.openlineage.conf import namespace
|
|
155
165
|
|
|
156
166
|
parent_run_id = _get_ol_run_id(task_instance)
|
|
167
|
+
root_parent_run_id = _get_ol_dag_run_id(task_instance)
|
|
157
168
|
|
|
158
169
|
return parent_run.ParentRunFacet(
|
|
159
170
|
run=parent_run.Run(runId=parent_run_id),
|
|
@@ -161,6 +172,13 @@ def _get_parent_run_facet(task_instance):
|
|
|
161
172
|
namespace=namespace(),
|
|
162
173
|
name=f"{task_instance.dag_id}.{task_instance.task_id}",
|
|
163
174
|
),
|
|
175
|
+
root=parent_run.Root(
|
|
176
|
+
run=parent_run.RootRun(runId=root_parent_run_id),
|
|
177
|
+
job=parent_run.RootJob(
|
|
178
|
+
name=task_instance.dag_id,
|
|
179
|
+
namespace=namespace(),
|
|
180
|
+
),
|
|
181
|
+
),
|
|
164
182
|
)
|
|
165
183
|
|
|
166
184
|
|
|
@@ -228,7 +246,7 @@ def _create_snowflake_event_pair(
|
|
|
228
246
|
return start, end
|
|
229
247
|
|
|
230
248
|
|
|
231
|
-
@require_openlineage_version(provider_min_version="2.
|
|
249
|
+
@require_openlineage_version(provider_min_version="2.3.0")
|
|
232
250
|
def emit_openlineage_events_for_snowflake_queries(
|
|
233
251
|
query_ids: list[str],
|
|
234
252
|
query_source_namespace: str,
|
|
@@ -32,5 +32,4 @@ def get_base_airflow_version_tuple() -> tuple[int, int, int]:
|
|
|
32
32
|
return airflow_version.major, airflow_version.minor, airflow_version.micro
|
|
33
33
|
|
|
34
34
|
|
|
35
|
-
AIRFLOW_V_2_10_PLUS = get_base_airflow_version_tuple() >= (2, 10, 0)
|
|
36
35
|
AIRFLOW_V_3_0_PLUS = get_base_airflow_version_tuple() >= (3, 0, 0)
|