apache-airflow-providers-yandex 4.1.1__py3-none-any.whl → 4.2.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
29
29
 
30
30
  __all__ = ["__version__"]
31
31
 
32
- __version__ = "4.1.1"
32
+ __version__ = "4.2.0"
33
33
 
34
34
  if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
35
35
  "2.10.0"
@@ -28,11 +28,7 @@ from airflow.providers.yandex.utils.credentials import (
28
28
  from airflow.providers.yandex.utils.defaults import conn_name_attr, conn_type, default_conn_name, hook_name
29
29
  from airflow.providers.yandex.utils.fields import get_field_from_extras
30
30
  from airflow.providers.yandex.utils.user_agent import provider_user_agent
31
-
32
- try:
33
- from airflow.sdk import BaseHook
34
- except ImportError:
35
- from airflow.hooks.base import BaseHook # type: ignore[attr-defined,no-redef]
31
+ from airflow.providers.yandex.version_compat import BaseHook
36
32
 
37
33
 
38
34
  class YandexCloudBaseHook(BaseHook):
@@ -14,12 +14,15 @@
14
14
  # KIND, either express or implied. See the License for the
15
15
  # specific language governing permissions and limitations
16
16
  # under the License.
17
+
17
18
  from __future__ import annotations
18
19
 
19
20
  from collections.abc import Iterable, Sequence
20
21
  from dataclasses import dataclass
21
22
  from typing import TYPE_CHECKING
22
23
 
24
+ import yandexcloud
25
+
23
26
  from airflow.providers.yandex.hooks.dataproc import DataprocHook
24
27
  from airflow.providers.yandex.version_compat import BaseOperator
25
28
 
@@ -54,6 +57,7 @@ class DataprocCreateClusterOperator(BaseOperator):
54
57
  Currently there are ru-central1-a, ru-central1-b and ru-central1-c.
55
58
  :param service_account_id: Service account id for the cluster.
56
59
  Service account can be created inside the folder.
60
+ :param environment: Environment for the cluster. Possible options: PRODUCTION, PRESTABLE.
57
61
  :param masternode_resource_preset: Resources preset (CPU+RAM configuration)
58
62
  for the primary node of the cluster.
59
63
  :param masternode_disk_size: Masternode storage size in GiB.
@@ -96,6 +100,7 @@ class DataprocCreateClusterOperator(BaseOperator):
96
100
  Docs: https://cloud.yandex.com/docs/data-proc/concepts/logs
97
101
  :param initialization_actions: Set of init-actions to run when cluster starts.
98
102
  Docs: https://cloud.yandex.com/docs/data-proc/concepts/init-action
103
+ :param oslogin_enabled: Enable authorization via OS Login for cluster.
99
104
  :param labels: Cluster labels as key:value pairs. No more than 64 per resource.
100
105
  Docs: https://cloud.yandex.com/docs/resource-manager/concepts/labels
101
106
  """
@@ -109,10 +114,11 @@ class DataprocCreateClusterOperator(BaseOperator):
109
114
  cluster_image_version: str | None = None,
110
115
  ssh_public_keys: str | Iterable[str] | None = None,
111
116
  subnet_id: str | None = None,
112
- services: Iterable[str] = ("HDFS", "YARN", "MAPREDUCE", "HIVE", "SPARK"),
117
+ services: Iterable[str] | None = ("HDFS", "YARN", "MAPREDUCE", "HIVE", "SPARK"),
113
118
  s3_bucket: str | None = None,
114
119
  zone: str = "ru-central1-b",
115
120
  service_account_id: str | None = None,
121
+ environment: str | None = None,
116
122
  masternode_resource_preset: str | None = None,
117
123
  masternode_disk_size: int | None = None,
118
124
  masternode_disk_type: str | None = None,
@@ -138,6 +144,7 @@ class DataprocCreateClusterOperator(BaseOperator):
138
144
  security_group_ids: Iterable[str] | None = None,
139
145
  log_group_id: str | None = None,
140
146
  initialization_actions: Iterable[InitializationAction] | None = None,
147
+ oslogin_enabled: bool = False,
141
148
  labels: dict[str, str] | None = None,
142
149
  **kwargs,
143
150
  ) -> None:
@@ -145,9 +152,6 @@ class DataprocCreateClusterOperator(BaseOperator):
145
152
  if ssh_public_keys is None:
146
153
  ssh_public_keys = []
147
154
 
148
- if services is None:
149
- services = []
150
-
151
155
  self.folder_id = folder_id
152
156
  self.yandex_conn_id = connection_id
153
157
  self.cluster_name = cluster_name
@@ -159,6 +163,7 @@ class DataprocCreateClusterOperator(BaseOperator):
159
163
  self.s3_bucket = s3_bucket
160
164
  self.zone = zone
161
165
  self.service_account_id = service_account_id
166
+ self.environment = environment
162
167
  self.masternode_resource_preset = masternode_resource_preset
163
168
  self.masternode_disk_size = masternode_disk_size
164
169
  self.masternode_disk_type = masternode_disk_type
@@ -183,6 +188,7 @@ class DataprocCreateClusterOperator(BaseOperator):
183
188
  self.security_group_ids = security_group_ids
184
189
  self.log_group_id = log_group_id
185
190
  self.initialization_actions = initialization_actions
191
+ self.oslogin_enabled = oslogin_enabled
186
192
  self.labels = labels
187
193
 
188
194
  self.hook: DataprocHook | None = None
@@ -191,6 +197,11 @@ class DataprocCreateClusterOperator(BaseOperator):
191
197
  self.hook = DataprocHook(
192
198
  yandex_conn_id=self.yandex_conn_id,
193
199
  )
200
+ kwargs_depends_on_version = {}
201
+ if yandexcloud.__version__ >= "0.350.0":
202
+ kwargs_depends_on_version.update(
203
+ {"oslogin_enabled": self.oslogin_enabled, "environment": self.environment}
204
+ )
194
205
  operation_result = self.hook.dataproc_client.create_cluster(
195
206
  folder_id=self.folder_id,
196
207
  cluster_name=self.cluster_name,
@@ -236,6 +247,7 @@ class DataprocCreateClusterOperator(BaseOperator):
236
247
  ]
237
248
  if self.initialization_actions
238
249
  else None,
250
+ **kwargs_depends_on_version,
239
251
  )
240
252
  cluster_id = operation_result.response.id
241
253
 
@@ -28,6 +28,13 @@ def get_base_airflow_version_tuple() -> tuple[int, int, int]:
28
28
 
29
29
 
30
30
  AIRFLOW_V_3_0_PLUS = get_base_airflow_version_tuple() >= (3, 0, 0)
31
+ AIRFLOW_V_3_1_PLUS: bool = get_base_airflow_version_tuple() >= (3, 1, 0)
32
+
33
+
34
+ if AIRFLOW_V_3_1_PLUS:
35
+ from airflow.sdk import BaseHook
36
+ else:
37
+ from airflow.hooks.base import BaseHook # type: ignore[attr-defined,no-redef]
31
38
 
32
39
  if AIRFLOW_V_3_0_PLUS:
33
40
  from airflow.sdk import BaseOperator, BaseOperatorLink
@@ -41,6 +48,8 @@ else:
41
48
 
42
49
  __all__ = [
43
50
  "AIRFLOW_V_3_0_PLUS",
51
+ "AIRFLOW_V_3_1_PLUS",
52
+ "BaseHook",
44
53
  "BaseOperator",
45
54
  "BaseOperatorLink",
46
55
  "Context",
@@ -1,11 +1,11 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: apache-airflow-providers-yandex
3
- Version: 4.1.1
3
+ Version: 4.2.0rc1
4
4
  Summary: Provider package apache-airflow-providers-yandex for Apache Airflow
5
5
  Keywords: airflow-provider,yandex,airflow,integration
6
6
  Author-email: Apache Software Foundation <dev@airflow.apache.org>
7
7
  Maintainer-email: Apache Software Foundation <dev@airflow.apache.org>
8
- Requires-Python: ~=3.10
8
+ Requires-Python: >=3.10,!=3.13
9
9
  Description-Content-Type: text/x-rst
10
10
  Classifier: Development Status :: 5 - Production/Stable
11
11
  Classifier: Environment :: Console
@@ -19,13 +19,13 @@ Classifier: Programming Language :: Python :: 3.10
19
19
  Classifier: Programming Language :: Python :: 3.11
20
20
  Classifier: Programming Language :: Python :: 3.12
21
21
  Classifier: Topic :: System :: Monitoring
22
- Requires-Dist: apache-airflow>=2.10.0
23
- Requires-Dist: yandexcloud>=0.308.0
24
- Requires-Dist: yandex-query-client>=0.1.4
22
+ Requires-Dist: apache-airflow>=2.10.0rc1
23
+ Requires-Dist: yandexcloud>=0.308.0; python_version < '3.13'
24
+ Requires-Dist: yandex-query-client>=0.1.4; python_version < '3.13'
25
25
  Requires-Dist: apache-airflow-providers-common-compat ; extra == "common-compat"
26
26
  Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
27
- Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-yandex/4.1.1/changelog.html
28
- Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-yandex/4.1.1
27
+ Project-URL: Changelog, https://airflow.staged.apache.org/docs/apache-airflow-providers-yandex/4.2.0/changelog.html
28
+ Project-URL: Documentation, https://airflow.staged.apache.org/docs/apache-airflow-providers-yandex/4.2.0
29
29
  Project-URL: Mastodon, https://fosstodon.org/@airflow
30
30
  Project-URL: Slack Chat, https://s.apache.org/airflow-slack
31
31
  Project-URL: Source Code, https://github.com/apache/airflow
@@ -57,8 +57,9 @@ Provides-Extra: common-compat
57
57
 
58
58
  Package ``apache-airflow-providers-yandex``
59
59
 
60
- Release: ``4.1.1``
60
+ Release: ``4.2.0``
61
61
 
62
+ Release Date: ``|PypiReleaseDate|``
62
63
 
63
64
  This package is for Yandex, including:
64
65
 
@@ -72,7 +73,7 @@ This is a provider package for ``yandex`` provider. All classes for this provide
72
73
  are in ``airflow.providers.yandex`` python package.
73
74
 
74
75
  You can find package information and changelog for the provider
75
- in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-yandex/4.1.1/>`_.
76
+ in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-yandex/4.2.0/>`_.
76
77
 
77
78
  Installation
78
79
  ------------
@@ -86,33 +87,14 @@ The package supports the following python versions: 3.10,3.11,3.12
86
87
  Requirements
87
88
  ------------
88
89
 
89
- ======================= ==================
90
+ ======================= ======================================
90
91
  PIP package Version required
91
- ======================= ==================
92
+ ======================= ======================================
92
93
  ``apache-airflow`` ``>=2.10.0``
93
- ``yandexcloud`` ``>=0.308.0``
94
- ``yandex-query-client`` ``>=0.1.4``
95
- ======================= ==================
96
-
97
- Cross provider package dependencies
98
- -----------------------------------
99
-
100
- Those are dependencies that might be needed in order to use all the features of the package.
101
- You need to install the specified providers in order to use them.
102
-
103
- You can install such cross-provider dependencies when installing from PyPI. For example:
104
-
105
- .. code-block:: bash
106
-
107
- pip install apache-airflow-providers-yandex[common.compat]
108
-
109
-
110
- ================================================================================================================== =================
111
- Dependent package Extra
112
- ================================================================================================================== =================
113
- `apache-airflow-providers-common-compat <https://airflow.apache.org/docs/apache-airflow-providers-common-compat>`_ ``common.compat``
114
- ================================================================================================================== =================
94
+ ``yandexcloud`` ``>=0.308.0; python_version < "3.13"``
95
+ ``yandex-query-client`` ``>=0.1.4; python_version < "3.13"``
96
+ ======================= ======================================
115
97
 
116
98
  The changelog for the provider package can be found in the
117
- `changelog <https://airflow.apache.org/docs/apache-airflow-providers-yandex/4.1.1/changelog.html>`_.
99
+ `changelog <https://airflow.apache.org/docs/apache-airflow-providers-yandex/4.2.0/changelog.html>`_.
118
100
 
@@ -1,15 +1,15 @@
1
1
  airflow/providers/yandex/LICENSE,sha256=gXPVwptPlW1TJ4HSuG5OMPg-a3h43OGMkZRR1rpwfJA,10850
2
- airflow/providers/yandex/__init__.py,sha256=67ndbApW7fBf1vwhVnXolKf9bnWuQLAZoa8nAswC2pg,1495
2
+ airflow/providers/yandex/__init__.py,sha256=YDygjzuZt5SxJ-wZ31Re-ffhJg4a4FnO-zWznCwvWeU,1495
3
3
  airflow/providers/yandex/get_provider_info.py,sha256=eihBGmOy9ljwxZQLES-Fi0RdSKWGze07VtFBRUITLP0,4015
4
- airflow/providers/yandex/version_compat.py,sha256=6IDC7JAL1RaxOlEouNZTzv2fHP8a7yiBBw3nPDucBrA,1654
4
+ airflow/providers/yandex/version_compat.py,sha256=wLjcfyobIcLCh2pfVeTZftTpMnkbWOKGqlfu09bdGMo,1920
5
5
  airflow/providers/yandex/hooks/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
6
6
  airflow/providers/yandex/hooks/dataproc.py,sha256=8ykGvlZUAjAKHpypiOYcWVJk7u-WNjfEohuyOy1Okss,1944
7
- airflow/providers/yandex/hooks/yandex.py,sha256=YmcA7fUfdWlAWHN52C44rvictxSF9jBB1Dc1Sg7FgGA,6540
7
+ airflow/providers/yandex/hooks/yandex.py,sha256=6IsaImhCHYcu6aJSe6u0Yqf29nlTORLnQput3OzFJIM,6455
8
8
  airflow/providers/yandex/hooks/yq.py,sha256=Qh1ZTp8OVKvQ6sFzmKUMe3kbkYT5v7D4qEq6VsKtB2k,3503
9
9
  airflow/providers/yandex/links/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
10
10
  airflow/providers/yandex/links/yq.py,sha256=I-Tp0EUb5-DhMmiQJcHSPiOK__7MkIYFIAfLxDlS4p0,1553
11
11
  airflow/providers/yandex/operators/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
12
- airflow/providers/yandex/operators/dataproc.py,sha256=wVNc7vHApsHhDAU_YZ_CEVMZTljk7X4PqIDVPBIPOQA,25697
12
+ airflow/providers/yandex/operators/dataproc.py,sha256=vWvxsoBU6Z6CJ0YquwRnAy1MYi231QDPVh_MzZQaUJU,26283
13
13
  airflow/providers/yandex/operators/yq.py,sha256=csvfEv1yXmc_bBIgMjs4Nh2yngk0tiQRM1GpgeJM5og,3287
14
14
  airflow/providers/yandex/secrets/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
15
15
  airflow/providers/yandex/secrets/lockbox.py,sha256=gUYWJE2qEvAZRz35qv0iKYK33BGnWesahZFuQgK5_kM,12168
@@ -18,7 +18,7 @@ airflow/providers/yandex/utils/credentials.py,sha256=l-8lIkQaIXTsNP_hMfP_tVADM54
18
18
  airflow/providers/yandex/utils/defaults.py,sha256=CXt75MhGJe8echoDpl1vR4VG5bEvYDDjIHmFqckDh2w,950
19
19
  airflow/providers/yandex/utils/fields.py,sha256=1D8SDWH8h0djj5Hnk50w6BpPeNJyP-689Qfjpkr-yCg,1728
20
20
  airflow/providers/yandex/utils/user_agent.py,sha256=AC-WEzhjxkgUYOy4LdX2-nnUZdMhKRRUCJ2_TjfNm6k,1839
21
- apache_airflow_providers_yandex-4.1.1.dist-info/entry_points.txt,sha256=ApXKRkvdgU2QNSQovjewC0b-LptwfBGBnJB3LTgBNx8,102
22
- apache_airflow_providers_yandex-4.1.1.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
23
- apache_airflow_providers_yandex-4.1.1.dist-info/METADATA,sha256=PkB-ZRmSKxOFZ6hjG-KsV5aq2p_EsBy_aTabbnuEH_s,5137
24
- apache_airflow_providers_yandex-4.1.1.dist-info/RECORD,,
21
+ apache_airflow_providers_yandex-4.2.0rc1.dist-info/entry_points.txt,sha256=ApXKRkvdgU2QNSQovjewC0b-LptwfBGBnJB3LTgBNx8,102
22
+ apache_airflow_providers_yandex-4.2.0rc1.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
23
+ apache_airflow_providers_yandex-4.2.0rc1.dist-info/METADATA,sha256=HSBEuYQU3yq4rZoe-0RHhuBf77Se0a_Bv89zYInJkpk,4291
24
+ apache_airflow_providers_yandex-4.2.0rc1.dist-info/RECORD,,