apache-airflow-providers-snowflake 5.7.1rc1__tar.gz → 5.8.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of apache-airflow-providers-snowflake might be problematic. Click here for more details.
- {apache_airflow_providers_snowflake-5.7.1rc1 → apache_airflow_providers_snowflake-5.8.0}/PKG-INFO +13 -12
- {apache_airflow_providers_snowflake-5.7.1rc1 → apache_airflow_providers_snowflake-5.8.0}/README.rst +5 -4
- {apache_airflow_providers_snowflake-5.7.1rc1 → apache_airflow_providers_snowflake-5.8.0}/airflow/providers/snowflake/__init__.py +1 -1
- apache_airflow_providers_snowflake-5.8.0/airflow/providers/snowflake/decorators/snowpark.py +124 -0
- {apache_airflow_providers_snowflake-5.7.1rc1 → apache_airflow_providers_snowflake-5.8.0}/airflow/providers/snowflake/get_provider_info.py +17 -3
- {apache_airflow_providers_snowflake-5.7.1rc1 → apache_airflow_providers_snowflake-5.8.0}/airflow/providers/snowflake/hooks/snowflake.py +29 -4
- {apache_airflow_providers_snowflake-5.7.1rc1 → apache_airflow_providers_snowflake-5.8.0}/airflow/providers/snowflake/hooks/snowflake_sql_api.py +8 -2
- apache_airflow_providers_snowflake-5.8.0/airflow/providers/snowflake/operators/snowpark.py +133 -0
- apache_airflow_providers_snowflake-5.8.0/airflow/providers/snowflake/utils/__init__.py +16 -0
- apache_airflow_providers_snowflake-5.8.0/airflow/providers/snowflake/utils/snowpark.py +44 -0
- {apache_airflow_providers_snowflake-5.7.1rc1 → apache_airflow_providers_snowflake-5.8.0}/pyproject.toml +8 -8
- {apache_airflow_providers_snowflake-5.7.1rc1 → apache_airflow_providers_snowflake-5.8.0}/airflow/providers/snowflake/LICENSE +0 -0
- {apache_airflow_providers_snowflake-5.7.1rc1/airflow/providers/snowflake/hooks → apache_airflow_providers_snowflake-5.8.0/airflow/providers/snowflake/decorators}/__init__.py +0 -0
- {apache_airflow_providers_snowflake-5.7.1rc1/airflow/providers/snowflake/operators → apache_airflow_providers_snowflake-5.8.0/airflow/providers/snowflake/hooks}/__init__.py +0 -0
- {apache_airflow_providers_snowflake-5.7.1rc1/airflow/providers/snowflake/transfers → apache_airflow_providers_snowflake-5.8.0/airflow/providers/snowflake/operators}/__init__.py +0 -0
- {apache_airflow_providers_snowflake-5.7.1rc1 → apache_airflow_providers_snowflake-5.8.0}/airflow/providers/snowflake/operators/snowflake.py +0 -0
- {apache_airflow_providers_snowflake-5.7.1rc1/airflow/providers/snowflake/triggers → apache_airflow_providers_snowflake-5.8.0/airflow/providers/snowflake/transfers}/__init__.py +0 -0
- {apache_airflow_providers_snowflake-5.7.1rc1 → apache_airflow_providers_snowflake-5.8.0}/airflow/providers/snowflake/transfers/copy_into_snowflake.py +0 -0
- {apache_airflow_providers_snowflake-5.7.1rc1/airflow/providers/snowflake/utils → apache_airflow_providers_snowflake-5.8.0/airflow/providers/snowflake/triggers}/__init__.py +0 -0
- {apache_airflow_providers_snowflake-5.7.1rc1 → apache_airflow_providers_snowflake-5.8.0}/airflow/providers/snowflake/triggers/snowflake_trigger.py +0 -0
- {apache_airflow_providers_snowflake-5.7.1rc1 → apache_airflow_providers_snowflake-5.8.0}/airflow/providers/snowflake/utils/common.py +0 -0
- {apache_airflow_providers_snowflake-5.7.1rc1 → apache_airflow_providers_snowflake-5.8.0}/airflow/providers/snowflake/utils/openlineage.py +0 -0
- {apache_airflow_providers_snowflake-5.7.1rc1 → apache_airflow_providers_snowflake-5.8.0}/airflow/providers/snowflake/utils/sql_api_generate_jwt.py +0 -0
{apache_airflow_providers_snowflake-5.7.1rc1 → apache_airflow_providers_snowflake-5.8.0}/PKG-INFO
RENAMED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: apache-airflow-providers-snowflake
|
|
3
|
-
Version: 5.
|
|
3
|
+
Version: 5.8.0
|
|
4
4
|
Summary: Provider package apache-airflow-providers-snowflake for Apache Airflow
|
|
5
5
|
Keywords: airflow-provider,snowflake,airflow,integration
|
|
6
6
|
Author-email: Apache Software Foundation <dev@airflow.apache.org>
|
|
7
7
|
Maintainer-email: Apache Software Foundation <dev@airflow.apache.org>
|
|
8
|
-
Requires-Python: ~=3.
|
|
8
|
+
Requires-Python: ~=3.9
|
|
9
9
|
Description-Content-Type: text/x-rst
|
|
10
10
|
Classifier: Development Status :: 5 - Production/Stable
|
|
11
11
|
Classifier: Environment :: Console
|
|
@@ -15,26 +15,26 @@ Classifier: Intended Audience :: System Administrators
|
|
|
15
15
|
Classifier: Framework :: Apache Airflow
|
|
16
16
|
Classifier: Framework :: Apache Airflow :: Provider
|
|
17
17
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
18
|
-
Classifier: Programming Language :: Python :: 3.8
|
|
19
18
|
Classifier: Programming Language :: Python :: 3.9
|
|
20
19
|
Classifier: Programming Language :: Python :: 3.10
|
|
21
20
|
Classifier: Programming Language :: Python :: 3.11
|
|
22
21
|
Classifier: Programming Language :: Python :: 3.12
|
|
23
22
|
Classifier: Topic :: System :: Monitoring
|
|
24
|
-
Requires-Dist: apache-airflow-providers-common-compat>=1.1.
|
|
25
|
-
Requires-Dist: apache-airflow-providers-common-sql>=1.14.
|
|
26
|
-
Requires-Dist: apache-airflow>=2.8.
|
|
23
|
+
Requires-Dist: apache-airflow-providers-common-compat>=1.1.0
|
|
24
|
+
Requires-Dist: apache-airflow-providers-common-sql>=1.14.1
|
|
25
|
+
Requires-Dist: apache-airflow>=2.8.0
|
|
27
26
|
Requires-Dist: pandas>=1.5.3,<2.2;python_version<"3.9"
|
|
28
27
|
Requires-Dist: pandas>=2.1.2,<2.2;python_version>="3.9"
|
|
29
28
|
Requires-Dist: pyarrow>=14.0.1
|
|
30
29
|
Requires-Dist: snowflake-connector-python>=3.7.1
|
|
30
|
+
Requires-Dist: snowflake-snowpark-python>=1.17.0;python_version<"3.12"
|
|
31
31
|
Requires-Dist: snowflake-sqlalchemy>=1.4.0
|
|
32
32
|
Requires-Dist: apache-airflow-providers-common-compat ; extra == "common.compat"
|
|
33
33
|
Requires-Dist: apache-airflow-providers-common-sql ; extra == "common.sql"
|
|
34
34
|
Requires-Dist: apache-airflow-providers-openlineage ; extra == "openlineage"
|
|
35
35
|
Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
|
|
36
|
-
Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-snowflake/5.
|
|
37
|
-
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-snowflake/5.
|
|
36
|
+
Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-snowflake/5.8.0/changelog.html
|
|
37
|
+
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-snowflake/5.8.0
|
|
38
38
|
Project-URL: Slack Chat, https://s.apache.org/airflow-slack
|
|
39
39
|
Project-URL: Source Code, https://github.com/apache/airflow
|
|
40
40
|
Project-URL: Twitter, https://twitter.com/ApacheAirflow
|
|
@@ -87,7 +87,7 @@ Provides-Extra: openlineage
|
|
|
87
87
|
|
|
88
88
|
Package ``apache-airflow-providers-snowflake``
|
|
89
89
|
|
|
90
|
-
Release: ``5.
|
|
90
|
+
Release: ``5.8.0``
|
|
91
91
|
|
|
92
92
|
|
|
93
93
|
`Snowflake <https://www.snowflake.com/>`__
|
|
@@ -100,7 +100,7 @@ This is a provider package for ``snowflake`` provider. All classes for this prov
|
|
|
100
100
|
are in ``airflow.providers.snowflake`` python package.
|
|
101
101
|
|
|
102
102
|
You can find package information and changelog for the provider
|
|
103
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-snowflake/5.
|
|
103
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-snowflake/5.8.0/>`_.
|
|
104
104
|
|
|
105
105
|
Installation
|
|
106
106
|
------------
|
|
@@ -109,7 +109,7 @@ You can install this package on top of an existing Airflow 2 installation (see `
|
|
|
109
109
|
for the minimum Airflow version supported) via
|
|
110
110
|
``pip install apache-airflow-providers-snowflake``
|
|
111
111
|
|
|
112
|
-
The package supports the following python versions: 3.
|
|
112
|
+
The package supports the following python versions: 3.9,3.10,3.11,3.12
|
|
113
113
|
|
|
114
114
|
Requirements
|
|
115
115
|
------------
|
|
@@ -125,6 +125,7 @@ PIP package Version required
|
|
|
125
125
|
``pyarrow`` ``>=14.0.1``
|
|
126
126
|
``snowflake-connector-python`` ``>=3.7.1``
|
|
127
127
|
``snowflake-sqlalchemy`` ``>=1.4.0``
|
|
128
|
+
``snowflake-snowpark-python`` ``>=1.17.0; python_version < "3.12"``
|
|
128
129
|
========================================== =========================================
|
|
129
130
|
|
|
130
131
|
Cross provider package dependencies
|
|
@@ -149,4 +150,4 @@ Dependent package
|
|
|
149
150
|
================================================================================================================== =================
|
|
150
151
|
|
|
151
152
|
The changelog for the provider package can be found in the
|
|
152
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-snowflake/5.
|
|
153
|
+
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-snowflake/5.8.0/changelog.html>`_.
|
{apache_airflow_providers_snowflake-5.7.1rc1 → apache_airflow_providers_snowflake-5.8.0}/README.rst
RENAMED
|
@@ -42,7 +42,7 @@
|
|
|
42
42
|
|
|
43
43
|
Package ``apache-airflow-providers-snowflake``
|
|
44
44
|
|
|
45
|
-
Release: ``5.
|
|
45
|
+
Release: ``5.8.0``
|
|
46
46
|
|
|
47
47
|
|
|
48
48
|
`Snowflake <https://www.snowflake.com/>`__
|
|
@@ -55,7 +55,7 @@ This is a provider package for ``snowflake`` provider. All classes for this prov
|
|
|
55
55
|
are in ``airflow.providers.snowflake`` python package.
|
|
56
56
|
|
|
57
57
|
You can find package information and changelog for the provider
|
|
58
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-snowflake/5.
|
|
58
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-snowflake/5.8.0/>`_.
|
|
59
59
|
|
|
60
60
|
Installation
|
|
61
61
|
------------
|
|
@@ -64,7 +64,7 @@ You can install this package on top of an existing Airflow 2 installation (see `
|
|
|
64
64
|
for the minimum Airflow version supported) via
|
|
65
65
|
``pip install apache-airflow-providers-snowflake``
|
|
66
66
|
|
|
67
|
-
The package supports the following python versions: 3.
|
|
67
|
+
The package supports the following python versions: 3.9,3.10,3.11,3.12
|
|
68
68
|
|
|
69
69
|
Requirements
|
|
70
70
|
------------
|
|
@@ -80,6 +80,7 @@ PIP package Version required
|
|
|
80
80
|
``pyarrow`` ``>=14.0.1``
|
|
81
81
|
``snowflake-connector-python`` ``>=3.7.1``
|
|
82
82
|
``snowflake-sqlalchemy`` ``>=1.4.0``
|
|
83
|
+
``snowflake-snowpark-python`` ``>=1.17.0; python_version < "3.12"``
|
|
83
84
|
========================================== =========================================
|
|
84
85
|
|
|
85
86
|
Cross provider package dependencies
|
|
@@ -104,4 +105,4 @@ Dependent package
|
|
|
104
105
|
================================================================================================================== =================
|
|
105
106
|
|
|
106
107
|
The changelog for the provider package can be found in the
|
|
107
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-snowflake/5.
|
|
108
|
+
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-snowflake/5.8.0/changelog.html>`_.
|
|
@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
|
|
|
29
29
|
|
|
30
30
|
__all__ = ["__version__"]
|
|
31
31
|
|
|
32
|
-
__version__ = "5.
|
|
32
|
+
__version__ = "5.8.0"
|
|
33
33
|
|
|
34
34
|
if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
|
|
35
35
|
"2.8.0"
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
|
2
|
+
# or more contributor license agreements. See the NOTICE file
|
|
3
|
+
# distributed with this work for additional information
|
|
4
|
+
# regarding copyright ownership. The ASF licenses this file
|
|
5
|
+
# to you under the Apache License, Version 2.0 (the
|
|
6
|
+
# "License"); you may not use this file except in compliance
|
|
7
|
+
# with the License. You may obtain a copy of the License at
|
|
8
|
+
#
|
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
#
|
|
11
|
+
# Unless required by applicable law or agreed to in writing,
|
|
12
|
+
# software distributed under the License is distributed on an
|
|
13
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
14
|
+
# KIND, either express or implied. See the License for the
|
|
15
|
+
# specific language governing permissions and limitations
|
|
16
|
+
# under the License.
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
from typing import TYPE_CHECKING, Callable, Sequence
|
|
21
|
+
|
|
22
|
+
from airflow.decorators.base import DecoratedOperator, task_decorator_factory
|
|
23
|
+
from airflow.providers.snowflake.operators.snowpark import SnowparkOperator
|
|
24
|
+
from airflow.providers.snowflake.utils.snowpark import inject_session_into_op_kwargs
|
|
25
|
+
|
|
26
|
+
if TYPE_CHECKING:
|
|
27
|
+
from airflow.decorators.base import TaskDecorator
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class _SnowparkDecoratedOperator(DecoratedOperator, SnowparkOperator):
|
|
31
|
+
"""
|
|
32
|
+
Wraps a Python callable that contains Snowpark code and captures args/kwargs when called for execution.
|
|
33
|
+
|
|
34
|
+
:param snowflake_conn_id: Reference to
|
|
35
|
+
:ref:`Snowflake connection id<howto/connection:snowflake>`
|
|
36
|
+
:param python_callable: A reference to an object that is callable
|
|
37
|
+
:param op_args: a list of positional arguments that will get unpacked when
|
|
38
|
+
calling your callable
|
|
39
|
+
:param op_kwargs: a dictionary of keyword arguments that will get unpacked
|
|
40
|
+
in your function
|
|
41
|
+
:param warehouse: name of warehouse (will overwrite any warehouse
|
|
42
|
+
defined in the connection's extra JSON)
|
|
43
|
+
:param database: name of database (will overwrite database defined
|
|
44
|
+
in connection)
|
|
45
|
+
:param schema: name of schema (will overwrite schema defined in
|
|
46
|
+
connection)
|
|
47
|
+
:param role: name of role (will overwrite any role defined in
|
|
48
|
+
connection's extra JSON)
|
|
49
|
+
:param authenticator: authenticator for Snowflake.
|
|
50
|
+
'snowflake' (default) to use the internal Snowflake authenticator
|
|
51
|
+
'externalbrowser' to authenticate using your web browser and
|
|
52
|
+
Okta, ADFS or any other SAML 2.0-compliant identify provider
|
|
53
|
+
(IdP) that has been defined for your account
|
|
54
|
+
'https://<your_okta_account_name>.okta.com' to authenticate
|
|
55
|
+
through native Okta.
|
|
56
|
+
:param session_parameters: You can set session-level parameters at
|
|
57
|
+
the time you connect to Snowflake
|
|
58
|
+
:param multiple_outputs: If set to True, the decorated function's return value will be unrolled to
|
|
59
|
+
multiple XCom values. Dict will unroll to XCom values with its keys as XCom keys. Defaults to False.
|
|
60
|
+
"""
|
|
61
|
+
|
|
62
|
+
custom_operator_name = "@task.snowpark"
|
|
63
|
+
|
|
64
|
+
def __init__(
|
|
65
|
+
self,
|
|
66
|
+
*,
|
|
67
|
+
snowflake_conn_id: str = "snowflake_default",
|
|
68
|
+
python_callable: Callable,
|
|
69
|
+
op_args: Sequence | None = None,
|
|
70
|
+
op_kwargs: dict | None = None,
|
|
71
|
+
warehouse: str | None = None,
|
|
72
|
+
database: str | None = None,
|
|
73
|
+
role: str | None = None,
|
|
74
|
+
schema: str | None = None,
|
|
75
|
+
authenticator: str | None = None,
|
|
76
|
+
session_parameters: dict | None = None,
|
|
77
|
+
**kwargs,
|
|
78
|
+
) -> None:
|
|
79
|
+
kwargs_to_upstream = {
|
|
80
|
+
"python_callable": python_callable,
|
|
81
|
+
"op_args": op_args,
|
|
82
|
+
"op_kwargs": op_kwargs,
|
|
83
|
+
}
|
|
84
|
+
super().__init__(
|
|
85
|
+
kwargs_to_upstream=kwargs_to_upstream,
|
|
86
|
+
snowflake_conn_id=snowflake_conn_id,
|
|
87
|
+
python_callable=python_callable,
|
|
88
|
+
op_args=op_args,
|
|
89
|
+
# airflow.decorators.base.DecoratedOperator checks if the functions are bindable, so we have to
|
|
90
|
+
# add an artificial value to pass the validation if there is a keyword argument named `session`
|
|
91
|
+
# in the signature of the python callable. The real value is determined at runtime.
|
|
92
|
+
op_kwargs=inject_session_into_op_kwargs(python_callable, op_kwargs, None)
|
|
93
|
+
if op_kwargs is not None
|
|
94
|
+
else op_kwargs,
|
|
95
|
+
warehouse=warehouse,
|
|
96
|
+
database=database,
|
|
97
|
+
role=role,
|
|
98
|
+
schema=schema,
|
|
99
|
+
authenticator=authenticator,
|
|
100
|
+
session_parameters=session_parameters,
|
|
101
|
+
**kwargs,
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def snowpark_task(
|
|
106
|
+
python_callable: Callable | None = None,
|
|
107
|
+
multiple_outputs: bool | None = None,
|
|
108
|
+
**kwargs,
|
|
109
|
+
) -> TaskDecorator:
|
|
110
|
+
"""
|
|
111
|
+
Wrap a function that contains Snowpark code into an Airflow operator.
|
|
112
|
+
|
|
113
|
+
Accepts kwargs for operator kwarg. Can be reused in a single DAG.
|
|
114
|
+
|
|
115
|
+
:param python_callable: Function to decorate
|
|
116
|
+
:param multiple_outputs: If set to True, the decorated function's return value will be unrolled to
|
|
117
|
+
multiple XCom values. Dict will unroll to XCom values with its keys as XCom keys. Defaults to False.
|
|
118
|
+
"""
|
|
119
|
+
return task_decorator_factory(
|
|
120
|
+
python_callable=python_callable,
|
|
121
|
+
multiple_outputs=multiple_outputs,
|
|
122
|
+
decorated_operator_class=_SnowparkDecoratedOperator,
|
|
123
|
+
**kwargs,
|
|
124
|
+
)
|
|
@@ -28,8 +28,9 @@ def get_provider_info():
|
|
|
28
28
|
"name": "Snowflake",
|
|
29
29
|
"description": "`Snowflake <https://www.snowflake.com/>`__\n",
|
|
30
30
|
"state": "ready",
|
|
31
|
-
"source-date-epoch":
|
|
31
|
+
"source-date-epoch": 1728485406,
|
|
32
32
|
"versions": [
|
|
33
|
+
"5.8.0",
|
|
33
34
|
"5.7.1",
|
|
34
35
|
"5.7.0",
|
|
35
36
|
"5.6.1",
|
|
@@ -91,12 +92,16 @@ def get_provider_info():
|
|
|
91
92
|
"pyarrow>=14.0.1",
|
|
92
93
|
"snowflake-connector-python>=3.7.1",
|
|
93
94
|
"snowflake-sqlalchemy>=1.4.0",
|
|
95
|
+
'snowflake-snowpark-python>=1.17.0;python_version<"3.12"',
|
|
94
96
|
],
|
|
95
97
|
"integrations": [
|
|
96
98
|
{
|
|
97
99
|
"integration-name": "Snowflake",
|
|
98
100
|
"external-doc-url": "https://snowflake.com/",
|
|
99
|
-
"how-to-guide": [
|
|
101
|
+
"how-to-guide": [
|
|
102
|
+
"/docs/apache-airflow-providers-snowflake/operators/snowflake.rst",
|
|
103
|
+
"/docs/apache-airflow-providers-snowflake/operators/snowpark.rst",
|
|
104
|
+
],
|
|
100
105
|
"logo": "/integration-logos/snowflake/Snowflake.png",
|
|
101
106
|
"tags": ["service"],
|
|
102
107
|
}
|
|
@@ -104,7 +109,16 @@ def get_provider_info():
|
|
|
104
109
|
"operators": [
|
|
105
110
|
{
|
|
106
111
|
"integration-name": "Snowflake",
|
|
107
|
-
"python-modules": [
|
|
112
|
+
"python-modules": [
|
|
113
|
+
"airflow.providers.snowflake.operators.snowflake",
|
|
114
|
+
"airflow.providers.snowflake.operators.snowpark",
|
|
115
|
+
],
|
|
116
|
+
}
|
|
117
|
+
],
|
|
118
|
+
"task-decorators": [
|
|
119
|
+
{
|
|
120
|
+
"class-name": "airflow.providers.snowflake.decorators.snowpark.snowpark_task",
|
|
121
|
+
"name": "snowpark",
|
|
108
122
|
}
|
|
109
123
|
],
|
|
110
124
|
"hooks": [
|
|
@@ -96,9 +96,12 @@ class SnowflakeHook(DbApiHook):
|
|
|
96
96
|
@classmethod
|
|
97
97
|
def get_connection_form_widgets(cls) -> dict[str, Any]:
|
|
98
98
|
"""Return connection widgets to add to connection form."""
|
|
99
|
-
from flask_appbuilder.fieldwidgets import
|
|
99
|
+
from flask_appbuilder.fieldwidgets import (
|
|
100
|
+
BS3PasswordFieldWidget,
|
|
101
|
+
BS3TextFieldWidget,
|
|
102
|
+
)
|
|
100
103
|
from flask_babel import lazy_gettext
|
|
101
|
-
from wtforms import BooleanField, StringField
|
|
104
|
+
from wtforms import BooleanField, PasswordField, StringField
|
|
102
105
|
|
|
103
106
|
return {
|
|
104
107
|
"account": StringField(lazy_gettext("Account"), widget=BS3TextFieldWidget()),
|
|
@@ -107,8 +110,8 @@ class SnowflakeHook(DbApiHook):
|
|
|
107
110
|
"region": StringField(lazy_gettext("Region"), widget=BS3TextFieldWidget()),
|
|
108
111
|
"role": StringField(lazy_gettext("Role"), widget=BS3TextFieldWidget()),
|
|
109
112
|
"private_key_file": StringField(lazy_gettext("Private key (Path)"), widget=BS3TextFieldWidget()),
|
|
110
|
-
"private_key_content":
|
|
111
|
-
lazy_gettext("Private key (Text)"), widget=
|
|
113
|
+
"private_key_content": PasswordField(
|
|
114
|
+
lazy_gettext("Private key (Text)"), widget=BS3PasswordFieldWidget()
|
|
112
115
|
),
|
|
113
116
|
"insecure_mode": BooleanField(
|
|
114
117
|
label=lazy_gettext("Insecure mode"), description="Turns off OCSP certificate checks"
|
|
@@ -318,6 +321,28 @@ class SnowflakeHook(DbApiHook):
|
|
|
318
321
|
engine_kwargs["connect_args"][key] = conn_params[key]
|
|
319
322
|
return create_engine(self._conn_params_to_sqlalchemy_uri(conn_params), **engine_kwargs)
|
|
320
323
|
|
|
324
|
+
def get_snowpark_session(self):
|
|
325
|
+
"""
|
|
326
|
+
Get a Snowpark session object.
|
|
327
|
+
|
|
328
|
+
:return: the created session.
|
|
329
|
+
"""
|
|
330
|
+
from snowflake.snowpark import Session
|
|
331
|
+
|
|
332
|
+
from airflow import __version__ as airflow_version
|
|
333
|
+
from airflow.providers.snowflake import __version__ as provider_version
|
|
334
|
+
|
|
335
|
+
conn_config = self._get_conn_params
|
|
336
|
+
session = Session.builder.configs(conn_config).create()
|
|
337
|
+
# add query tag for observability
|
|
338
|
+
session.update_query_tag(
|
|
339
|
+
{
|
|
340
|
+
"airflow_version": airflow_version,
|
|
341
|
+
"airflow_provider_version": provider_version,
|
|
342
|
+
}
|
|
343
|
+
)
|
|
344
|
+
return session
|
|
345
|
+
|
|
321
346
|
def set_autocommit(self, conn, autocommit: Any) -> None:
|
|
322
347
|
conn.autocommit(autocommit)
|
|
323
348
|
conn.autocommit_mode = autocommit
|
|
@@ -153,8 +153,14 @@ class SnowflakeSqlApiHook(SnowflakeHook):
|
|
|
153
153
|
url = f"{self.account_identifier}.snowflakecomputing.com/api/v2/statements"
|
|
154
154
|
params: dict[str, Any] | None = {"requestId": str(req_id), "async": True, "pageSize": 10}
|
|
155
155
|
headers = self.get_headers()
|
|
156
|
-
|
|
157
|
-
|
|
156
|
+
sql_is_multi_stmt = ";" in sql.strip()
|
|
157
|
+
if not isinstance(bindings, dict) and bindings is not None:
|
|
158
|
+
raise AirflowException("Bindings should be a dictionary or None.")
|
|
159
|
+
if bindings and sql_is_multi_stmt:
|
|
160
|
+
self.log.warning(
|
|
161
|
+
"Bindings are not supported for multi-statement queries. Bindings will be ignored."
|
|
162
|
+
)
|
|
163
|
+
bindings = bindings or {}
|
|
158
164
|
data = {
|
|
159
165
|
"statement": sql,
|
|
160
166
|
"resultSetMetaData": {"format": "json"},
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
|
2
|
+
# or more contributor license agreements. See the NOTICE file
|
|
3
|
+
# distributed with this work for additional information
|
|
4
|
+
# regarding copyright ownership. The ASF licenses this file
|
|
5
|
+
# to you under the Apache License, Version 2.0 (the
|
|
6
|
+
# "License"); you may not use this file except in compliance
|
|
7
|
+
# with the License. You may obtain a copy of the License at
|
|
8
|
+
#
|
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
#
|
|
11
|
+
# Unless required by applicable law or agreed to in writing,
|
|
12
|
+
# software distributed under the License is distributed on an
|
|
13
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
14
|
+
# KIND, either express or implied. See the License for the
|
|
15
|
+
# specific language governing permissions and limitations
|
|
16
|
+
# under the License.
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
from typing import Any, Callable, Collection, Mapping, Sequence
|
|
21
|
+
|
|
22
|
+
from airflow.operators.python import PythonOperator, get_current_context
|
|
23
|
+
from airflow.providers.snowflake.hooks.snowflake import SnowflakeHook
|
|
24
|
+
from airflow.providers.snowflake.utils.snowpark import inject_session_into_op_kwargs
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class SnowparkOperator(PythonOperator):
|
|
28
|
+
"""
|
|
29
|
+
Executes a Python function with Snowpark Python code.
|
|
30
|
+
|
|
31
|
+
.. seealso::
|
|
32
|
+
For more information on how to use this operator, take a look at the guide:
|
|
33
|
+
:ref:`howto/operator:SnowparkOperator`
|
|
34
|
+
|
|
35
|
+
:param snowflake_conn_id: Reference to
|
|
36
|
+
:ref:`Snowflake connection id<howto/connection:snowflake>`
|
|
37
|
+
:param python_callable: A reference to an object that is callable
|
|
38
|
+
:param op_args: a list of positional arguments that will get unpacked when
|
|
39
|
+
calling your callable
|
|
40
|
+
:param op_kwargs: a dictionary of keyword arguments that will get unpacked
|
|
41
|
+
in your function
|
|
42
|
+
:param templates_dict: a dictionary where the values are templates that
|
|
43
|
+
will get templated by the Airflow engine sometime between
|
|
44
|
+
``__init__`` and ``execute`` takes place and are made available
|
|
45
|
+
in your callable's context after the template has been applied. (templated)
|
|
46
|
+
:param templates_exts: a list of file extensions to resolve while
|
|
47
|
+
processing templated fields, for examples ``['.sql', '.hql']``
|
|
48
|
+
:param show_return_value_in_logs: a bool value whether to show return_value
|
|
49
|
+
logs. Defaults to True, which allows return value log output.
|
|
50
|
+
It can be set to False to prevent log output of return value when you return huge data
|
|
51
|
+
such as transmission a large amount of XCom to TaskAPI.
|
|
52
|
+
:param warehouse: name of warehouse (will overwrite any warehouse
|
|
53
|
+
defined in the connection's extra JSON)
|
|
54
|
+
:param database: name of database (will overwrite database defined
|
|
55
|
+
in connection)
|
|
56
|
+
:param schema: name of schema (will overwrite schema defined in
|
|
57
|
+
connection)
|
|
58
|
+
:param role: name of role (will overwrite any role defined in
|
|
59
|
+
connection's extra JSON)
|
|
60
|
+
:param authenticator: authenticator for Snowflake.
|
|
61
|
+
'snowflake' (default) to use the internal Snowflake authenticator
|
|
62
|
+
'externalbrowser' to authenticate using your web browser and
|
|
63
|
+
Okta, ADFS or any other SAML 2.0-compliant identify provider
|
|
64
|
+
(IdP) that has been defined for your account
|
|
65
|
+
'https://<your_okta_account_name>.okta.com' to authenticate
|
|
66
|
+
through native Okta.
|
|
67
|
+
:param session_parameters: You can set session-level parameters at
|
|
68
|
+
the time you connect to Snowflake
|
|
69
|
+
"""
|
|
70
|
+
|
|
71
|
+
def __init__(
|
|
72
|
+
self,
|
|
73
|
+
*,
|
|
74
|
+
snowflake_conn_id: str = "snowflake_default",
|
|
75
|
+
python_callable: Callable,
|
|
76
|
+
op_args: Collection[Any] | None = None,
|
|
77
|
+
op_kwargs: Mapping[str, Any] | None = None,
|
|
78
|
+
templates_dict: dict[str, Any] | None = None,
|
|
79
|
+
templates_exts: Sequence[str] | None = None,
|
|
80
|
+
show_return_value_in_logs: bool = True,
|
|
81
|
+
warehouse: str | None = None,
|
|
82
|
+
database: str | None = None,
|
|
83
|
+
schema: str | None = None,
|
|
84
|
+
role: str | None = None,
|
|
85
|
+
authenticator: str | None = None,
|
|
86
|
+
session_parameters: dict | None = None,
|
|
87
|
+
**kwargs,
|
|
88
|
+
):
|
|
89
|
+
super().__init__(
|
|
90
|
+
python_callable=python_callable,
|
|
91
|
+
op_args=op_args,
|
|
92
|
+
op_kwargs=op_kwargs,
|
|
93
|
+
templates_dict=templates_dict,
|
|
94
|
+
templates_exts=templates_exts,
|
|
95
|
+
show_return_value_in_logs=show_return_value_in_logs,
|
|
96
|
+
**kwargs,
|
|
97
|
+
)
|
|
98
|
+
self.snowflake_conn_id = snowflake_conn_id
|
|
99
|
+
self.warehouse = warehouse
|
|
100
|
+
self.database = database
|
|
101
|
+
self.schema = schema
|
|
102
|
+
self.role = role
|
|
103
|
+
self.authenticator = authenticator
|
|
104
|
+
self.session_parameters = session_parameters
|
|
105
|
+
|
|
106
|
+
def execute_callable(self):
|
|
107
|
+
hook = SnowflakeHook(
|
|
108
|
+
snowflake_conn_id=self.snowflake_conn_id,
|
|
109
|
+
warehouse=self.warehouse,
|
|
110
|
+
database=self.database,
|
|
111
|
+
role=self.role,
|
|
112
|
+
schema=self.schema,
|
|
113
|
+
authenticator=self.authenticator,
|
|
114
|
+
session_parameters=self.session_parameters,
|
|
115
|
+
)
|
|
116
|
+
session = hook.get_snowpark_session()
|
|
117
|
+
context = get_current_context()
|
|
118
|
+
session.update_query_tag(
|
|
119
|
+
{
|
|
120
|
+
"dag_id": context["dag_run"].dag_id,
|
|
121
|
+
"dag_run_id": context["dag_run"].run_id,
|
|
122
|
+
"task_id": context["task_instance"].task_id,
|
|
123
|
+
"operator": self.__class__.__name__,
|
|
124
|
+
}
|
|
125
|
+
)
|
|
126
|
+
try:
|
|
127
|
+
# inject session object if the function has "session" keyword as an argument
|
|
128
|
+
self.op_kwargs = inject_session_into_op_kwargs(
|
|
129
|
+
self.python_callable, dict(self.op_kwargs), session
|
|
130
|
+
)
|
|
131
|
+
return super().execute_callable()
|
|
132
|
+
finally:
|
|
133
|
+
session.close()
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
|
2
|
+
# or more contributor license agreements. See the NOTICE file
|
|
3
|
+
# distributed with this work for additional information
|
|
4
|
+
# regarding copyright ownership. The ASF licenses this file
|
|
5
|
+
# to you under the Apache License, Version 2.0 (the
|
|
6
|
+
# "License"); you may not use this file except in compliance
|
|
7
|
+
# with the License. You may obtain a copy of the License at
|
|
8
|
+
#
|
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
#
|
|
11
|
+
# Unless required by applicable law or agreed to in writing,
|
|
12
|
+
# software distributed under the License is distributed on an
|
|
13
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
14
|
+
# KIND, either express or implied. See the License for the
|
|
15
|
+
# specific language governing permissions and limitations
|
|
16
|
+
# under the License.
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
|
2
|
+
# or more contributor license agreements. See the NOTICE file
|
|
3
|
+
# distributed with this work for additional information
|
|
4
|
+
# regarding copyright ownership. The ASF licenses this file
|
|
5
|
+
# to you under the Apache License, Version 2.0 (the
|
|
6
|
+
# "License"); you may not use this file except in compliance
|
|
7
|
+
# with the License. You may obtain a copy of the License at
|
|
8
|
+
#
|
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
#
|
|
11
|
+
# Unless required by applicable law or agreed to in writing,
|
|
12
|
+
# software distributed under the License is distributed on an
|
|
13
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
14
|
+
# KIND, either express or implied. See the License for the
|
|
15
|
+
# specific language governing permissions and limitations
|
|
16
|
+
# under the License.
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import inspect
|
|
21
|
+
from typing import TYPE_CHECKING, Callable
|
|
22
|
+
|
|
23
|
+
if TYPE_CHECKING:
|
|
24
|
+
from snowflake.snowpark import Session
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def inject_session_into_op_kwargs(
|
|
28
|
+
python_callable: Callable, op_kwargs: dict, session: Session | None
|
|
29
|
+
) -> dict:
|
|
30
|
+
"""
|
|
31
|
+
Inject Snowpark session into operator kwargs based on signature of python callable.
|
|
32
|
+
|
|
33
|
+
If there is a keyword argument named `session` in the signature of the python callable,
|
|
34
|
+
a Snowpark session object will be injected into kwargs.
|
|
35
|
+
|
|
36
|
+
:param python_callable: Python callable
|
|
37
|
+
:param op_kwargs: Operator kwargs
|
|
38
|
+
:param session: Snowpark session
|
|
39
|
+
"""
|
|
40
|
+
signature = inspect.signature(python_callable)
|
|
41
|
+
if "session" in signature.parameters:
|
|
42
|
+
return {**op_kwargs, "session": session}
|
|
43
|
+
else:
|
|
44
|
+
return op_kwargs
|
|
@@ -28,7 +28,7 @@ build-backend = "flit_core.buildapi"
|
|
|
28
28
|
|
|
29
29
|
[project]
|
|
30
30
|
name = "apache-airflow-providers-snowflake"
|
|
31
|
-
version = "5.
|
|
31
|
+
version = "5.8.0"
|
|
32
32
|
description = "Provider package apache-airflow-providers-snowflake for Apache Airflow"
|
|
33
33
|
readme = "README.rst"
|
|
34
34
|
authors = [
|
|
@@ -47,28 +47,28 @@ classifiers = [
|
|
|
47
47
|
"Framework :: Apache Airflow",
|
|
48
48
|
"Framework :: Apache Airflow :: Provider",
|
|
49
49
|
"License :: OSI Approved :: Apache Software License",
|
|
50
|
-
"Programming Language :: Python :: 3.8",
|
|
51
50
|
"Programming Language :: Python :: 3.9",
|
|
52
51
|
"Programming Language :: Python :: 3.10",
|
|
53
52
|
"Programming Language :: Python :: 3.11",
|
|
54
53
|
"Programming Language :: Python :: 3.12",
|
|
55
54
|
"Topic :: System :: Monitoring",
|
|
56
55
|
]
|
|
57
|
-
requires-python = "~=3.
|
|
56
|
+
requires-python = "~=3.9"
|
|
58
57
|
dependencies = [
|
|
59
|
-
"apache-airflow-providers-common-compat>=1.1.
|
|
60
|
-
"apache-airflow-providers-common-sql>=1.14.
|
|
61
|
-
"apache-airflow>=2.8.
|
|
58
|
+
"apache-airflow-providers-common-compat>=1.1.0",
|
|
59
|
+
"apache-airflow-providers-common-sql>=1.14.1",
|
|
60
|
+
"apache-airflow>=2.8.0",
|
|
62
61
|
"pandas>=1.5.3,<2.2;python_version<\"3.9\"",
|
|
63
62
|
"pandas>=2.1.2,<2.2;python_version>=\"3.9\"",
|
|
64
63
|
"pyarrow>=14.0.1",
|
|
65
64
|
"snowflake-connector-python>=3.7.1",
|
|
65
|
+
"snowflake-snowpark-python>=1.17.0;python_version<\"3.12\"",
|
|
66
66
|
"snowflake-sqlalchemy>=1.4.0",
|
|
67
67
|
]
|
|
68
68
|
|
|
69
69
|
[project.urls]
|
|
70
|
-
"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-snowflake/5.
|
|
71
|
-
"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-snowflake/5.
|
|
70
|
+
"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-snowflake/5.8.0"
|
|
71
|
+
"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-snowflake/5.8.0/changelog.html"
|
|
72
72
|
"Bug Tracker" = "https://github.com/apache/airflow/issues"
|
|
73
73
|
"Source Code" = "https://github.com/apache/airflow"
|
|
74
74
|
"Slack Chat" = "https://s.apache.org/airflow-slack"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|