apache-airflow-providers-yandex 3.11.2rc1__py3-none-any.whl → 3.12.0rc1__py3-none-any.whl

airflow/providers/yandex/__init__.py

@@ -29,11 +29,11 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "3.11.2"
+__version__ = "3.12.0"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
-    "2.7.0"
+    "2.8.0"
 ):
     raise RuntimeError(
-        f"The package `apache-airflow-providers-yandex:{__version__}` needs Apache Airflow 2.7.0+"
+        f"The package `apache-airflow-providers-yandex:{__version__}` needs Apache Airflow 2.8.0+"
     )

airflow/providers/yandex/get_provider_info.py

@@ -28,8 +28,9 @@ def get_provider_info():
         "name": "Yandex",
         "description": "This package is for Yandex, including:\n\n    - `Yandex.Cloud <https://cloud.yandex.com/>`__\n",
         "state": "ready",
-        "source-date-epoch": 1718605606,
+        "source-date-epoch": 1723970668,
         "versions": [
+            "3.12.0",
             "3.11.2",
             "3.11.1",
             "3.11.0",
@@ -54,11 +55,7 @@ def get_provider_info():
             "1.0.1",
             "1.0.0",
         ],
-        "dependencies": [
-            "apache-airflow>=2.7.0",
-            "yandexcloud>=0.278.0,!=0.289.0,!=0.290.0,<0.292.0",
-            "yandex-query-client>=0.1.4",
-        ],
+        "dependencies": ["apache-airflow>=2.8.0", "yandexcloud>=0.308.0", "yandex-query-client>=0.1.4"],
         "integrations": [
             {
                 "integration-name": "Yandex.Cloud",

airflow/providers/yandex/hooks/dataproc.py

@@ -16,8 +16,16 @@
 # under the License.
 from __future__ import annotations
 
+from typing import TYPE_CHECKING
+
+from deprecated import deprecated
+
+from airflow.exceptions import AirflowProviderDeprecationWarning
 from airflow.providers.yandex.hooks.yandex import YandexCloudBaseHook
 
+if TYPE_CHECKING:
+    from yandexcloud._wrappers.dataproc import Dataproc
+
 
 class DataprocHook(YandexCloudBaseHook):
     """
@@ -29,7 +37,19 @@ class DataprocHook(YandexCloudBaseHook):
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
         self.cluster_id = None
-        self.client = self.sdk.wrappers.Dataproc(
+        self.dataproc_client: Dataproc = self.sdk.wrappers.Dataproc(
             default_folder_id=self.default_folder_id,
             default_public_ssh_key=self.default_public_ssh_key,
         )
+
+    @property
+    @deprecated(
+        reason="`client` deprecated and will be removed in the future. Use `dataproc_client` instead",
+        category=AirflowProviderDeprecationWarning,
+    )
+    def client(self):
+        return self.dataproc_client
+
+    @client.setter
+    def client(self, value):
+        self.dataproc_client = value

airflow/providers/yandex/hooks/yandex.py

@@ -24,6 +24,7 @@ import yandexcloud
 from airflow.exceptions import AirflowProviderDeprecationWarning
 from airflow.hooks.base import BaseHook
 from airflow.providers.yandex.utils.credentials import (
+    CredentialsType,
     get_credentials,
     get_service_account_id,
 )
@@ -132,13 +133,18 @@ class YandexCloudBaseHook(BaseHook):
         self.connection_id = yandex_conn_id or connection_id or default_conn_name
         self.connection = self.get_connection(self.connection_id)
         self.extras = self.connection.extra_dejson
-        self.credentials = get_credentials(
+        self.credentials: CredentialsType = get_credentials(
             oauth_token=self._get_field("oauth"),
             service_account_json=self._get_field("service_account_json"),
             service_account_json_path=self._get_field("service_account_json_path"),
         )
         sdk_config = self._get_endpoint()
-        self.sdk = yandexcloud.SDK(user_agent=provider_user_agent(), **sdk_config, **self.credentials)
+        self.sdk = yandexcloud.SDK(
+            user_agent=provider_user_agent(),
+            token=self.credentials.get("token"),
+            service_account_key=self.credentials.get("service_account_key"),
+            endpoint=sdk_config.get("endpoint"),
+        )
         self.default_folder_id = default_folder_id or self._get_field("folder_id")
         self.default_public_ssh_key = default_public_ssh_key or self._get_field("public_ssh_key")
         self.default_service_account_id = default_service_account_id or get_service_account_id(

airflow/providers/yandex/hooks/yq.py

@@ -43,7 +43,8 @@ class YQHook(YandexCloudBaseHook):
         self.client.close()
 
     def create_query(self, query_text: str | None, name: str | None = None) -> str:
-        """Create and run query.
+        """
+        Create and run query.
 
         :param query_text: SQL text.
         :param name: name for the query
@@ -54,7 +55,8 @@ class YQHook(YandexCloudBaseHook):
         )
 
     def wait_results(self, query_id: str, execution_timeout: timedelta = timedelta(minutes=30)) -> Any:
-        """Wait for query complete and get results.
+        """
+        Wait for query complete and get results.
 
         :param query_id: ID of query.
         :param execution_timeout: how long to wait for the query to complete.
@@ -66,36 +68,39 @@ class YQHook(YandexCloudBaseHook):
         return self.client.get_query_all_result_sets(query_id=query_id, result_set_count=result_set_count)
 
     def stop_query(self, query_id: str) -> None:
-        """Stop the query.
+        """
+        Stop the query.
 
         :param query_id: ID of the query.
         """
         self.client.stop_query(query_id)
 
     def get_query(self, query_id: str) -> Any:
-        """Get query info.
+        """
+        Get query info.
 
         :param query_id: ID of the query.
         """
         return self.client.get_query(query_id)
 
     def get_query_status(self, query_id: str) -> str:
-        """Get status of the query.
+        """
+        Get status of the query.
 
         :param query_id: ID of query.
         """
         return self.client.get_query_status(query_id)
 
     def compose_query_web_link(self, query_id: str):
-        """Compose web link to query in Yandex Query UI.
+        """
+        Compose web link to query in Yandex Query UI.
 
         :param query_id: ID of query.
         """
         return self.client.compose_query_web_link(query_id)
 
     def _get_iam_token(self) -> str:
-        iam_token = self.credentials.get("token")
-        if iam_token is not None:
-            return iam_token
+        if "token" in self.credentials:
+            return self.credentials["token"]
 
         return yc_auth.get_auth_token(service_account_key=self.credentials.get("service_account_key"))

airflow/providers/yandex/operators/dataproc.py

@@ -33,12 +33,13 @@ class InitializationAction:
     """Data for initialization action to be run at start of DataProc cluster."""
 
     uri: str  # Uri of the executable file
-    args: Sequence[str]  # Arguments to the initialization action
+    args: Iterable[str]  # Arguments to the initialization action
     timeout: int  # Execution timeout
 
 
 class DataprocCreateClusterOperator(BaseOperator):
-    """Creates Yandex.Cloud Data Proc cluster.
+    """
+    Creates Yandex.Cloud Data Proc cluster.
 
     :param folder_id: ID of the folder in which cluster should be created.
     :param cluster_name: Cluster name. Must be unique inside the folder.
@@ -142,6 +143,12 @@ class DataprocCreateClusterOperator(BaseOperator):
         **kwargs,
     ) -> None:
         super().__init__(**kwargs)
+        if ssh_public_keys is None:
+            ssh_public_keys = []
+
+        if services is None:
+            services = []
+
         self.folder_id = folder_id
         self.yandex_conn_id = connection_id
         self.cluster_name = cluster_name
@@ -185,7 +192,7 @@ class DataprocCreateClusterOperator(BaseOperator):
         self.hook = DataprocHook(
             yandex_conn_id=self.yandex_conn_id,
         )
-        operation_result = self.hook.client.create_cluster(
+        operation_result = self.hook.dataproc_client.create_cluster(
             folder_id=self.folder_id,
             cluster_name=self.cluster_name,
             cluster_description=self.cluster_description,
@@ -220,15 +227,16 @@ class DataprocCreateClusterOperator(BaseOperator):
             security_group_ids=self.security_group_ids,
             log_group_id=self.log_group_id,
             labels=self.labels,
-            initialization_actions=self.initialization_actions
-            and [
+            initialization_actions=[
                 self.hook.sdk.wrappers.InitializationAction(
                     uri=init_action.uri,
                     args=init_action.args,
                     timeout=init_action.timeout,
                 )
                 for init_action in self.initialization_actions
-            ],
+            ]
+            if self.initialization_actions
+            else None,
         )
         cluster_id = operation_result.response.id
 
@@ -243,7 +251,8 @@ class DataprocCreateClusterOperator(BaseOperator):
 
 
 class DataprocBaseOperator(BaseOperator):
-    """Base class for DataProc operators working with given cluster.
+    """
+    Base class for DataProc operators working with given cluster.
 
     :param connection_id: ID of the Yandex.Cloud Airflow connection.
     :param cluster_id: ID of the cluster to remove. (templated)
@@ -276,7 +285,8 @@ class DataprocBaseOperator(BaseOperator):
 
 
 class DataprocDeleteClusterOperator(DataprocBaseOperator):
-    """Deletes Yandex.Cloud Data Proc cluster.
+    """
+    Deletes Yandex.Cloud Data Proc cluster.
 
     :param connection_id: ID of the Yandex.Cloud Airflow connection.
     :param cluster_id: ID of the cluster to remove. (templated)
@@ -287,11 +297,12 @@ class DataprocDeleteClusterOperator(DataprocBaseOperator):
 
     def execute(self, context: Context) -> None:
         hook = self._setup(context)
-        hook.client.delete_cluster(self.cluster_id)
+        hook.dataproc_client.delete_cluster(self.cluster_id)
 
 
 class DataprocCreateHiveJobOperator(DataprocBaseOperator):
-    """Runs Hive job in Data Proc cluster.
+    """
+    Runs Hive job in Data Proc cluster.
 
     :param query: Hive query.
     :param query_file_uri: URI of the script that contains Hive queries. Can be placed in HDFS or S3.
@@ -327,7 +338,7 @@ class DataprocCreateHiveJobOperator(DataprocBaseOperator):
 
     def execute(self, context: Context) -> None:
         hook = self._setup(context)
-        hook.client.create_hive_job(
+        hook.dataproc_client.create_hive_job(
             query=self.query,
             query_file_uri=self.query_file_uri,
             script_variables=self.script_variables,
@@ -339,7 +350,8 @@ class DataprocCreateHiveJobOperator(DataprocBaseOperator):
 
 
 class DataprocCreateMapReduceJobOperator(DataprocBaseOperator):
-    """Runs Mapreduce job in Data Proc cluster.
+    """
+    Runs Mapreduce job in Data Proc cluster.
 
     :param main_jar_file_uri: URI of jar file with job.
                               Can be placed in HDFS or S3. Can be specified instead of main_class.
@@ -382,7 +394,7 @@ class DataprocCreateMapReduceJobOperator(DataprocBaseOperator):
 
     def execute(self, context: Context) -> None:
         hook = self._setup(context)
-        hook.client.create_mapreduce_job(
+        hook.dataproc_client.create_mapreduce_job(
             main_class=self.main_class,
             main_jar_file_uri=self.main_jar_file_uri,
             jar_file_uris=self.jar_file_uris,
@@ -396,7 +408,8 @@ class DataprocCreateMapReduceJobOperator(DataprocBaseOperator):
 
 
 class DataprocCreateSparkJobOperator(DataprocBaseOperator):
-    """Runs Spark job in Data Proc cluster.
+    """
+    Runs Spark job in Data Proc cluster.
 
     :param main_jar_file_uri: URI of jar file with job. Can be placed in HDFS or S3.
     :param main_class: Name of the main class of the job.
@@ -449,7 +462,7 @@ class DataprocCreateSparkJobOperator(DataprocBaseOperator):
 
     def execute(self, context: Context) -> None:
         hook = self._setup(context)
-        hook.client.create_spark_job(
+        hook.dataproc_client.create_spark_job(
             main_class=self.main_class,
             main_jar_file_uri=self.main_jar_file_uri,
             jar_file_uris=self.jar_file_uris,
@@ -466,7 +479,8 @@ class DataprocCreateSparkJobOperator(DataprocBaseOperator):
 
 
 class DataprocCreatePysparkJobOperator(DataprocBaseOperator):
-    """Runs Pyspark job in Data Proc cluster.
+    """
+    Runs Pyspark job in Data Proc cluster.
 
     :param main_python_file_uri: URI of python file with job. Can be placed in HDFS or S3.
     :param python_file_uris: URIs of python files used in the job. Can be placed in HDFS or S3.
@@ -519,7 +533,7 @@ class DataprocCreatePysparkJobOperator(DataprocBaseOperator):
 
     def execute(self, context: Context) -> None:
         hook = self._setup(context)
-        hook.client.create_pyspark_job(
+        hook.dataproc_client.create_pyspark_job(
             main_python_file_uri=self.main_python_file_uri,
             python_file_uris=self.python_file_uris,
             jar_file_uris=self.jar_file_uris,

airflow/providers/yandex/secrets/lockbox.py

@@ -242,7 +242,7 @@ class LockboxSecretBackend(BaseSecretsBackend, LoggingMixin):
         return f"{prefix}{self.sep}{key}"
 
     def _get_secret_value(self, prefix: str, key: str) -> str | None:
-        secret: secret_pb.Secret = None
+        secret: secret_pb.Secret | None = None
         for s in self._get_secrets():
             if s.name == self._build_secret_name(prefix=prefix, key=key):
                 secret = s

airflow/providers/yandex/utils/credentials.py

@@ -18,16 +18,23 @@ from __future__ import annotations
 
 import json
 import logging
-from typing import Any
+from typing import TypedDict
 
 log = logging.getLogger(__name__)
 
 
+class CredentialsType(TypedDict, total=False):
+    """Credentials dict description."""
+
+    token: str
+    service_account_key: dict[str, str]
+
+
 def get_credentials(
     oauth_token: str | None = None,
     service_account_json: dict | str | None = None,
     service_account_json_path: str | None = None,
-) -> dict[str, Any]:
+) -> CredentialsType:
     """
     Return credentials JSON for Yandex Cloud SDK based on credentials.
 

{apache_airflow_providers_yandex-3.11.2rc1.dist-info → apache_airflow_providers_yandex-3.12.0rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apache-airflow-providers-yandex
-Version: 3.11.2rc1
+Version: 3.12.0rc1
 Summary: Provider package apache-airflow-providers-yandex for Apache Airflow
 Keywords: airflow-provider,yandex,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -21,12 +21,12 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: System :: Monitoring
-Requires-Dist: apache-airflow>=2.7.0rc0
+Requires-Dist: apache-airflow>=2.8.0rc0
 Requires-Dist: yandex-query-client>=0.1.4
-Requires-Dist: yandexcloud>=0.278.0,!=0.289.0,!=0.290.0,<0.292.0
+Requires-Dist: yandexcloud>=0.308.0
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-yandex/3.11.2/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-yandex/3.11.2
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-yandex/3.12.0/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-yandex/3.12.0
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Twitter, https://twitter.com/ApacheAirflow
@@ -76,7 +76,7 @@ Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
 
 Package ``apache-airflow-providers-yandex``
 
-Release: ``3.11.2.rc1``
+Release: ``3.12.0.rc1``
 
 
 This package is for Yandex, including:
@@ -91,7 +91,7 @@ This is a provider package for ``yandex`` provider. All classes for this provide
 are in ``airflow.providers.yandex`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-yandex/3.11.2/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-yandex/3.12.0/>`_.
 
 Installation
 ------------
@@ -105,13 +105,13 @@ The package supports the following python versions: 3.8,3.9,3.10,3.11,3.12
 Requirements
 ------------
 
-=======================  ==========================================
+=======================  ==================
 PIP package              Version required
-=======================  ==========================================
-``apache-airflow``       ``>=2.7.0``
-``yandexcloud``          ``>=0.278.0,!=0.289.0,!=0.290.0,<0.292.0``
+=======================  ==================
+``apache-airflow``       ``>=2.8.0``
+``yandexcloud``          ``>=0.308.0``
 ``yandex-query-client``  ``>=0.1.4``
-=======================  ==========================================
+=======================  ==================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-yandex/3.11.2/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-yandex/3.12.0/changelog.html>`_.

{apache_airflow_providers_yandex-3.11.2rc1.dist-info → apache_airflow_providers_yandex-3.12.0rc1.dist-info}/RECORD

@@ -1,25 +1,25 @@
 airflow/providers/yandex/LICENSE,sha256=FFb4jd2AXnOOf7XLP04pQW6jbdhG49TxlGY6fFpCV1Y,13609
-airflow/providers/yandex/__init__.py,sha256=G8AzvXtrAVvJmVp2TEsaHPbRLEvfzmZILZfeTHipHuc,1494
-airflow/providers/yandex/get_provider_info.py,sha256=wO1VKFNN4YeAGx_bF_LcbrDlBi_WPqQi02RD4nK0144,4815
+airflow/providers/yandex/__init__.py,sha256=icjIfW_ya1vvqiD0HpHgBf7Jq1HuS_li4-8Kd7GKwck,1494
+airflow/providers/yandex/get_provider_info.py,sha256=Bf4RoWGLV7qisIZH-_J9bi2MQLnCxziXdMGxpN3siwg,4761
 airflow/providers/yandex/hooks/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/yandex/hooks/dataproc.py,sha256=1UdqxDMI7uL6fNkG6oU6l2tFITF_nHXiV1VUgRqF7KY,1379
-airflow/providers/yandex/hooks/yandex.py,sha256=xJMUzGo0sNpb5-LQvgq6jDxWHK3XkNzlpoeEELREeow,7097
+airflow/providers/yandex/hooks/dataproc.py,sha256=8ykGvlZUAjAKHpypiOYcWVJk7u-WNjfEohuyOy1Okss,1944
+airflow/providers/yandex/hooks/yandex.py,sha256=z8HYXIUkBxhDL2vQUUrNu4xkAPf3rjMv0SkP187odTk,7299
 airflow/providers/yandex/hooks/yandexcloud_dataproc.py,sha256=-JVJm3YLkDbJZKauCR1oCnWNkdLUJa1Fj_5HmZq1f44,1243
-airflow/providers/yandex/hooks/yq.py,sha256=KPOXEIfjW6m-MYZMjg0J852t-gVxXZNqRQtIgVaGZO0,3477
+airflow/providers/yandex/hooks/yq.py,sha256=Qh1ZTp8OVKvQ6sFzmKUMe3kbkYT5v7D4qEq6VsKtB2k,3503
 airflow/providers/yandex/links/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
 airflow/providers/yandex/links/yq.py,sha256=jsy3liqQFk1eSSdK9YDbor0Epp7ng_q2ueVIwsD2i-8,1578
 airflow/providers/yandex/operators/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/yandex/operators/dataproc.py,sha256=QJc7UvBNPhAUBsuYQ4H8Wf0LpZP_-kCw7RdI0n3P_Bs,25945
+airflow/providers/yandex/operators/dataproc.py,sha256=JGBwX_ae3OZ4XtPVdvYi_XFC4sibI94mhBMbDDn0KT4,26181
 airflow/providers/yandex/operators/yandexcloud_dataproc.py,sha256=bDLMwevS5spRfVEtixdKhQTC9gqDMm9himLrRohJwKQ,1255
 airflow/providers/yandex/operators/yq.py,sha256=lGqbogakylV4s5D5movQRL4v3IU2Qt1JHH8ygo3Hd2Q,3223
 airflow/providers/yandex/secrets/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/yandex/secrets/lockbox.py,sha256=9Vi95RXd6VT36Rh1PVMUfFzm42oyrlMl77DoL9ivxVc,12161
+airflow/providers/yandex/secrets/lockbox.py,sha256=gUYWJE2qEvAZRz35qv0iKYK33BGnWesahZFuQgK5_kM,12168
 airflow/providers/yandex/utils/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/yandex/utils/credentials.py,sha256=6McJIitAuTROJRUSKTdWChfcZ9o4WthP6GmBJ4uz-j8,3335
+airflow/providers/yandex/utils/credentials.py,sha256=l-8lIkQaIXTsNP_hMfP_tVADM54ZdV70J4563Ig7r4M,3487
 airflow/providers/yandex/utils/defaults.py,sha256=CXt75MhGJe8echoDpl1vR4VG5bEvYDDjIHmFqckDh2w,950
 airflow/providers/yandex/utils/fields.py,sha256=1D8SDWH8h0djj5Hnk50w6BpPeNJyP-689Qfjpkr-yCg,1728
 airflow/providers/yandex/utils/user_agent.py,sha256=AC-WEzhjxkgUYOy4LdX2-nnUZdMhKRRUCJ2_TjfNm6k,1839
-apache_airflow_providers_yandex-3.11.2rc1.dist-info/entry_points.txt,sha256=ApXKRkvdgU2QNSQovjewC0b-LptwfBGBnJB3LTgBNx8,102
-apache_airflow_providers_yandex-3.11.2rc1.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
-apache_airflow_providers_yandex-3.11.2rc1.dist-info/METADATA,sha256=eUUr7WsQMyJuXCZ-2hI9sW5XJF5kCCVb710yno_SHG8,5049
-apache_airflow_providers_yandex-3.11.2rc1.dist-info/RECORD,,
+apache_airflow_providers_yandex-3.12.0rc1.dist-info/entry_points.txt,sha256=ApXKRkvdgU2QNSQovjewC0b-LptwfBGBnJB3LTgBNx8,102
+apache_airflow_providers_yandex-3.12.0rc1.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
+apache_airflow_providers_yandex-3.12.0rc1.dist-info/METADATA,sha256=cdmIvDbR-WYqvFvYvXjby5_KBRgvah7i0kjTgOw4EFo,4919
+apache_airflow_providers_yandex-3.12.0rc1.dist-info/RECORD,,