apache-airflow-providers-yandex 4.1.1rc1__py3-none-any.whl → 4.2.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/yandex/__init__.py +1 -1
- airflow/providers/yandex/hooks/yandex.py +1 -5
- airflow/providers/yandex/operators/dataproc.py +16 -4
- airflow/providers/yandex/version_compat.py +9 -0
- {apache_airflow_providers_yandex-4.1.1rc1.dist-info → apache_airflow_providers_yandex-4.2.0rc1.dist-info}/METADATA +15 -33
- {apache_airflow_providers_yandex-4.1.1rc1.dist-info → apache_airflow_providers_yandex-4.2.0rc1.dist-info}/RECORD +8 -8
- {apache_airflow_providers_yandex-4.1.1rc1.dist-info → apache_airflow_providers_yandex-4.2.0rc1.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_yandex-4.1.1rc1.dist-info → apache_airflow_providers_yandex-4.2.0rc1.dist-info}/entry_points.txt +0 -0
@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
|
|
29
29
|
|
30
30
|
__all__ = ["__version__"]
|
31
31
|
|
32
|
-
__version__ = "4.
|
32
|
+
__version__ = "4.2.0"
|
33
33
|
|
34
34
|
if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
|
35
35
|
"2.10.0"
|
@@ -28,11 +28,7 @@ from airflow.providers.yandex.utils.credentials import (
|
|
28
28
|
from airflow.providers.yandex.utils.defaults import conn_name_attr, conn_type, default_conn_name, hook_name
|
29
29
|
from airflow.providers.yandex.utils.fields import get_field_from_extras
|
30
30
|
from airflow.providers.yandex.utils.user_agent import provider_user_agent
|
31
|
-
|
32
|
-
try:
|
33
|
-
from airflow.sdk import BaseHook
|
34
|
-
except ImportError:
|
35
|
-
from airflow.hooks.base import BaseHook # type: ignore[attr-defined,no-redef]
|
31
|
+
from airflow.providers.yandex.version_compat import BaseHook
|
36
32
|
|
37
33
|
|
38
34
|
class YandexCloudBaseHook(BaseHook):
|
@@ -14,12 +14,15 @@
|
|
14
14
|
# KIND, either express or implied. See the License for the
|
15
15
|
# specific language governing permissions and limitations
|
16
16
|
# under the License.
|
17
|
+
|
17
18
|
from __future__ import annotations
|
18
19
|
|
19
20
|
from collections.abc import Iterable, Sequence
|
20
21
|
from dataclasses import dataclass
|
21
22
|
from typing import TYPE_CHECKING
|
22
23
|
|
24
|
+
import yandexcloud
|
25
|
+
|
23
26
|
from airflow.providers.yandex.hooks.dataproc import DataprocHook
|
24
27
|
from airflow.providers.yandex.version_compat import BaseOperator
|
25
28
|
|
@@ -54,6 +57,7 @@ class DataprocCreateClusterOperator(BaseOperator):
|
|
54
57
|
Currently there are ru-central1-a, ru-central1-b and ru-central1-c.
|
55
58
|
:param service_account_id: Service account id for the cluster.
|
56
59
|
Service account can be created inside the folder.
|
60
|
+
:param environment: Environment for the cluster. Possible options: PRODUCTION, PRESTABLE.
|
57
61
|
:param masternode_resource_preset: Resources preset (CPU+RAM configuration)
|
58
62
|
for the primary node of the cluster.
|
59
63
|
:param masternode_disk_size: Masternode storage size in GiB.
|
@@ -96,6 +100,7 @@ class DataprocCreateClusterOperator(BaseOperator):
|
|
96
100
|
Docs: https://cloud.yandex.com/docs/data-proc/concepts/logs
|
97
101
|
:param initialization_actions: Set of init-actions to run when cluster starts.
|
98
102
|
Docs: https://cloud.yandex.com/docs/data-proc/concepts/init-action
|
103
|
+
:param oslogin_enabled: Enable authorization via OS Login for cluster.
|
99
104
|
:param labels: Cluster labels as key:value pairs. No more than 64 per resource.
|
100
105
|
Docs: https://cloud.yandex.com/docs/resource-manager/concepts/labels
|
101
106
|
"""
|
@@ -109,10 +114,11 @@ class DataprocCreateClusterOperator(BaseOperator):
|
|
109
114
|
cluster_image_version: str | None = None,
|
110
115
|
ssh_public_keys: str | Iterable[str] | None = None,
|
111
116
|
subnet_id: str | None = None,
|
112
|
-
services: Iterable[str] = ("HDFS", "YARN", "MAPREDUCE", "HIVE", "SPARK"),
|
117
|
+
services: Iterable[str] | None = ("HDFS", "YARN", "MAPREDUCE", "HIVE", "SPARK"),
|
113
118
|
s3_bucket: str | None = None,
|
114
119
|
zone: str = "ru-central1-b",
|
115
120
|
service_account_id: str | None = None,
|
121
|
+
environment: str | None = None,
|
116
122
|
masternode_resource_preset: str | None = None,
|
117
123
|
masternode_disk_size: int | None = None,
|
118
124
|
masternode_disk_type: str | None = None,
|
@@ -138,6 +144,7 @@ class DataprocCreateClusterOperator(BaseOperator):
|
|
138
144
|
security_group_ids: Iterable[str] | None = None,
|
139
145
|
log_group_id: str | None = None,
|
140
146
|
initialization_actions: Iterable[InitializationAction] | None = None,
|
147
|
+
oslogin_enabled: bool = False,
|
141
148
|
labels: dict[str, str] | None = None,
|
142
149
|
**kwargs,
|
143
150
|
) -> None:
|
@@ -145,9 +152,6 @@ class DataprocCreateClusterOperator(BaseOperator):
|
|
145
152
|
if ssh_public_keys is None:
|
146
153
|
ssh_public_keys = []
|
147
154
|
|
148
|
-
if services is None:
|
149
|
-
services = []
|
150
|
-
|
151
155
|
self.folder_id = folder_id
|
152
156
|
self.yandex_conn_id = connection_id
|
153
157
|
self.cluster_name = cluster_name
|
@@ -159,6 +163,7 @@ class DataprocCreateClusterOperator(BaseOperator):
|
|
159
163
|
self.s3_bucket = s3_bucket
|
160
164
|
self.zone = zone
|
161
165
|
self.service_account_id = service_account_id
|
166
|
+
self.environment = environment
|
162
167
|
self.masternode_resource_preset = masternode_resource_preset
|
163
168
|
self.masternode_disk_size = masternode_disk_size
|
164
169
|
self.masternode_disk_type = masternode_disk_type
|
@@ -183,6 +188,7 @@ class DataprocCreateClusterOperator(BaseOperator):
|
|
183
188
|
self.security_group_ids = security_group_ids
|
184
189
|
self.log_group_id = log_group_id
|
185
190
|
self.initialization_actions = initialization_actions
|
191
|
+
self.oslogin_enabled = oslogin_enabled
|
186
192
|
self.labels = labels
|
187
193
|
|
188
194
|
self.hook: DataprocHook | None = None
|
@@ -191,6 +197,11 @@ class DataprocCreateClusterOperator(BaseOperator):
|
|
191
197
|
self.hook = DataprocHook(
|
192
198
|
yandex_conn_id=self.yandex_conn_id,
|
193
199
|
)
|
200
|
+
kwargs_depends_on_version = {}
|
201
|
+
if yandexcloud.__version__ >= "0.350.0":
|
202
|
+
kwargs_depends_on_version.update(
|
203
|
+
{"oslogin_enabled": self.oslogin_enabled, "environment": self.environment}
|
204
|
+
)
|
194
205
|
operation_result = self.hook.dataproc_client.create_cluster(
|
195
206
|
folder_id=self.folder_id,
|
196
207
|
cluster_name=self.cluster_name,
|
@@ -236,6 +247,7 @@ class DataprocCreateClusterOperator(BaseOperator):
|
|
236
247
|
]
|
237
248
|
if self.initialization_actions
|
238
249
|
else None,
|
250
|
+
**kwargs_depends_on_version,
|
239
251
|
)
|
240
252
|
cluster_id = operation_result.response.id
|
241
253
|
|
@@ -28,6 +28,13 @@ def get_base_airflow_version_tuple() -> tuple[int, int, int]:
|
|
28
28
|
|
29
29
|
|
30
30
|
AIRFLOW_V_3_0_PLUS = get_base_airflow_version_tuple() >= (3, 0, 0)
|
31
|
+
AIRFLOW_V_3_1_PLUS: bool = get_base_airflow_version_tuple() >= (3, 1, 0)
|
32
|
+
|
33
|
+
|
34
|
+
if AIRFLOW_V_3_1_PLUS:
|
35
|
+
from airflow.sdk import BaseHook
|
36
|
+
else:
|
37
|
+
from airflow.hooks.base import BaseHook # type: ignore[attr-defined,no-redef]
|
31
38
|
|
32
39
|
if AIRFLOW_V_3_0_PLUS:
|
33
40
|
from airflow.sdk import BaseOperator, BaseOperatorLink
|
@@ -41,6 +48,8 @@ else:
|
|
41
48
|
|
42
49
|
__all__ = [
|
43
50
|
"AIRFLOW_V_3_0_PLUS",
|
51
|
+
"AIRFLOW_V_3_1_PLUS",
|
52
|
+
"BaseHook",
|
44
53
|
"BaseOperator",
|
45
54
|
"BaseOperatorLink",
|
46
55
|
"Context",
|
@@ -1,11 +1,11 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: apache-airflow-providers-yandex
|
3
|
-
Version: 4.
|
3
|
+
Version: 4.2.0rc1
|
4
4
|
Summary: Provider package apache-airflow-providers-yandex for Apache Airflow
|
5
5
|
Keywords: airflow-provider,yandex,airflow,integration
|
6
6
|
Author-email: Apache Software Foundation <dev@airflow.apache.org>
|
7
7
|
Maintainer-email: Apache Software Foundation <dev@airflow.apache.org>
|
8
|
-
Requires-Python:
|
8
|
+
Requires-Python: >=3.10,!=3.13
|
9
9
|
Description-Content-Type: text/x-rst
|
10
10
|
Classifier: Development Status :: 5 - Production/Stable
|
11
11
|
Classifier: Environment :: Console
|
@@ -20,12 +20,12 @@ Classifier: Programming Language :: Python :: 3.11
|
|
20
20
|
Classifier: Programming Language :: Python :: 3.12
|
21
21
|
Classifier: Topic :: System :: Monitoring
|
22
22
|
Requires-Dist: apache-airflow>=2.10.0rc1
|
23
|
-
Requires-Dist: yandexcloud>=0.308.0
|
24
|
-
Requires-Dist: yandex-query-client>=0.1.4
|
23
|
+
Requires-Dist: yandexcloud>=0.308.0; python_version < '3.13'
|
24
|
+
Requires-Dist: yandex-query-client>=0.1.4; python_version < '3.13'
|
25
25
|
Requires-Dist: apache-airflow-providers-common-compat ; extra == "common-compat"
|
26
26
|
Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
|
27
|
-
Project-URL: Changelog, https://airflow.staged.apache.org/docs/apache-airflow-providers-yandex/4.
|
28
|
-
Project-URL: Documentation, https://airflow.staged.apache.org/docs/apache-airflow-providers-yandex/4.
|
27
|
+
Project-URL: Changelog, https://airflow.staged.apache.org/docs/apache-airflow-providers-yandex/4.2.0/changelog.html
|
28
|
+
Project-URL: Documentation, https://airflow.staged.apache.org/docs/apache-airflow-providers-yandex/4.2.0
|
29
29
|
Project-URL: Mastodon, https://fosstodon.org/@airflow
|
30
30
|
Project-URL: Slack Chat, https://s.apache.org/airflow-slack
|
31
31
|
Project-URL: Source Code, https://github.com/apache/airflow
|
@@ -57,8 +57,9 @@ Provides-Extra: common-compat
|
|
57
57
|
|
58
58
|
Package ``apache-airflow-providers-yandex``
|
59
59
|
|
60
|
-
Release: ``4.
|
60
|
+
Release: ``4.2.0``
|
61
61
|
|
62
|
+
Release Date: ``|PypiReleaseDate|``
|
62
63
|
|
63
64
|
This package is for Yandex, including:
|
64
65
|
|
@@ -72,7 +73,7 @@ This is a provider package for ``yandex`` provider. All classes for this provide
|
|
72
73
|
are in ``airflow.providers.yandex`` python package.
|
73
74
|
|
74
75
|
You can find package information and changelog for the provider
|
75
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-yandex/4.
|
76
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-yandex/4.2.0/>`_.
|
76
77
|
|
77
78
|
Installation
|
78
79
|
------------
|
@@ -86,33 +87,14 @@ The package supports the following python versions: 3.10,3.11,3.12
|
|
86
87
|
Requirements
|
87
88
|
------------
|
88
89
|
|
89
|
-
=======================
|
90
|
+
======================= ======================================
|
90
91
|
PIP package Version required
|
91
|
-
=======================
|
92
|
+
======================= ======================================
|
92
93
|
``apache-airflow`` ``>=2.10.0``
|
93
|
-
``yandexcloud`` ``>=0.308.0``
|
94
|
-
``yandex-query-client`` ``>=0.1.4``
|
95
|
-
=======================
|
96
|
-
|
97
|
-
Cross provider package dependencies
|
98
|
-
-----------------------------------
|
99
|
-
|
100
|
-
Those are dependencies that might be needed in order to use all the features of the package.
|
101
|
-
You need to install the specified providers in order to use them.
|
102
|
-
|
103
|
-
You can install such cross-provider dependencies when installing from PyPI. For example:
|
104
|
-
|
105
|
-
.. code-block:: bash
|
106
|
-
|
107
|
-
pip install apache-airflow-providers-yandex[common.compat]
|
108
|
-
|
109
|
-
|
110
|
-
================================================================================================================== =================
|
111
|
-
Dependent package Extra
|
112
|
-
================================================================================================================== =================
|
113
|
-
`apache-airflow-providers-common-compat <https://airflow.apache.org/docs/apache-airflow-providers-common-compat>`_ ``common.compat``
|
114
|
-
================================================================================================================== =================
|
94
|
+
``yandexcloud`` ``>=0.308.0; python_version < "3.13"``
|
95
|
+
``yandex-query-client`` ``>=0.1.4; python_version < "3.13"``
|
96
|
+
======================= ======================================
|
115
97
|
|
116
98
|
The changelog for the provider package can be found in the
|
117
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-yandex/4.
|
99
|
+
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-yandex/4.2.0/changelog.html>`_.
|
118
100
|
|
@@ -1,15 +1,15 @@
|
|
1
1
|
airflow/providers/yandex/LICENSE,sha256=gXPVwptPlW1TJ4HSuG5OMPg-a3h43OGMkZRR1rpwfJA,10850
|
2
|
-
airflow/providers/yandex/__init__.py,sha256=
|
2
|
+
airflow/providers/yandex/__init__.py,sha256=YDygjzuZt5SxJ-wZ31Re-ffhJg4a4FnO-zWznCwvWeU,1495
|
3
3
|
airflow/providers/yandex/get_provider_info.py,sha256=eihBGmOy9ljwxZQLES-Fi0RdSKWGze07VtFBRUITLP0,4015
|
4
|
-
airflow/providers/yandex/version_compat.py,sha256=
|
4
|
+
airflow/providers/yandex/version_compat.py,sha256=wLjcfyobIcLCh2pfVeTZftTpMnkbWOKGqlfu09bdGMo,1920
|
5
5
|
airflow/providers/yandex/hooks/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
6
6
|
airflow/providers/yandex/hooks/dataproc.py,sha256=8ykGvlZUAjAKHpypiOYcWVJk7u-WNjfEohuyOy1Okss,1944
|
7
|
-
airflow/providers/yandex/hooks/yandex.py,sha256=
|
7
|
+
airflow/providers/yandex/hooks/yandex.py,sha256=6IsaImhCHYcu6aJSe6u0Yqf29nlTORLnQput3OzFJIM,6455
|
8
8
|
airflow/providers/yandex/hooks/yq.py,sha256=Qh1ZTp8OVKvQ6sFzmKUMe3kbkYT5v7D4qEq6VsKtB2k,3503
|
9
9
|
airflow/providers/yandex/links/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
10
10
|
airflow/providers/yandex/links/yq.py,sha256=I-Tp0EUb5-DhMmiQJcHSPiOK__7MkIYFIAfLxDlS4p0,1553
|
11
11
|
airflow/providers/yandex/operators/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
12
|
-
airflow/providers/yandex/operators/dataproc.py,sha256=
|
12
|
+
airflow/providers/yandex/operators/dataproc.py,sha256=vWvxsoBU6Z6CJ0YquwRnAy1MYi231QDPVh_MzZQaUJU,26283
|
13
13
|
airflow/providers/yandex/operators/yq.py,sha256=csvfEv1yXmc_bBIgMjs4Nh2yngk0tiQRM1GpgeJM5og,3287
|
14
14
|
airflow/providers/yandex/secrets/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
15
15
|
airflow/providers/yandex/secrets/lockbox.py,sha256=gUYWJE2qEvAZRz35qv0iKYK33BGnWesahZFuQgK5_kM,12168
|
@@ -18,7 +18,7 @@ airflow/providers/yandex/utils/credentials.py,sha256=l-8lIkQaIXTsNP_hMfP_tVADM54
|
|
18
18
|
airflow/providers/yandex/utils/defaults.py,sha256=CXt75MhGJe8echoDpl1vR4VG5bEvYDDjIHmFqckDh2w,950
|
19
19
|
airflow/providers/yandex/utils/fields.py,sha256=1D8SDWH8h0djj5Hnk50w6BpPeNJyP-689Qfjpkr-yCg,1728
|
20
20
|
airflow/providers/yandex/utils/user_agent.py,sha256=AC-WEzhjxkgUYOy4LdX2-nnUZdMhKRRUCJ2_TjfNm6k,1839
|
21
|
-
apache_airflow_providers_yandex-4.
|
22
|
-
apache_airflow_providers_yandex-4.
|
23
|
-
apache_airflow_providers_yandex-4.
|
24
|
-
apache_airflow_providers_yandex-4.
|
21
|
+
apache_airflow_providers_yandex-4.2.0rc1.dist-info/entry_points.txt,sha256=ApXKRkvdgU2QNSQovjewC0b-LptwfBGBnJB3LTgBNx8,102
|
22
|
+
apache_airflow_providers_yandex-4.2.0rc1.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
|
23
|
+
apache_airflow_providers_yandex-4.2.0rc1.dist-info/METADATA,sha256=HSBEuYQU3yq4rZoe-0RHhuBf77Se0a_Bv89zYInJkpk,4291
|
24
|
+
apache_airflow_providers_yandex-4.2.0rc1.dist-info/RECORD,,
|
File without changes
|