apache-airflow-providers-cncf-kubernetes 8.0.1rc1__tar.gz → 8.1.0rc1__tar.gz

{apache_airflow_providers_cncf_kubernetes-8.0.1rc1 → apache_airflow_providers_cncf_kubernetes-8.1.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apache-airflow-providers-cncf-kubernetes
-Version: 8.0.1rc1
+Version: 8.1.0rc1
 Summary: Provider package apache-airflow-providers-cncf-kubernetes for Apache Airflow
 Keywords: airflow-provider,cncf.kubernetes,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -19,17 +19,18 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: System :: Monitoring
 Requires-Dist: aiofiles>=23.2.0
-Requires-Dist: apache-airflow>=2.6.0.dev0
+Requires-Dist: apache-airflow>=2.6.0rc0
 Requires-Dist: asgiref>=3.5.2
 Requires-Dist: cryptography>=2.0.0
 Requires-Dist: google-re2>=1.0
 Requires-Dist: kubernetes>=28.1.0,<=29.0.0
 Requires-Dist: kubernetes_asyncio>=28.1.0,<=29.0.0
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.0.1/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.0.1
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.1.0/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.1.0
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Twitter, https://twitter.com/ApacheAirflow
@@ -79,7 +80,7 @@ Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
 
 Package ``apache-airflow-providers-cncf-kubernetes``
 
-Release: ``8.0.1.rc1``
+Release: ``8.1.0.rc1``
 
 
 `Kubernetes <https://kubernetes.io/>`__
@@ -92,7 +93,7 @@ This is a provider package for ``cncf.kubernetes`` provider. All classes for thi
 are in ``airflow.providers.cncf.kubernetes`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.0.1/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.1.0/>`_.
 
 Installation
 ------------
@@ -101,7 +102,7 @@ You can install this package on top of an existing Airflow 2 installation (see `
 for the minimum Airflow version supported) via
 ``pip install apache-airflow-providers-cncf-kubernetes``
 
-The package supports the following python versions: 3.8,3.9,3.10,3.11
+The package supports the following python versions: 3.8,3.9,3.10,3.11,3.12
 
 Requirements
 ------------
@@ -119,4 +120,4 @@ PIP package             Version required
 ======================  =====================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.0.1/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.1.0/changelog.html>`_.

{apache_airflow_providers_cncf_kubernetes-8.0.1rc1 → apache_airflow_providers_cncf_kubernetes-8.1.0rc1}/README.rst

@@ -42,7 +42,7 @@
 
 Package ``apache-airflow-providers-cncf-kubernetes``
 
-Release: ``8.0.1.rc1``
+Release: ``8.1.0.rc1``
 
 
 `Kubernetes <https://kubernetes.io/>`__
@@ -55,7 +55,7 @@ This is a provider package for ``cncf.kubernetes`` provider. All classes for thi
 are in ``airflow.providers.cncf.kubernetes`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.0.1/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.1.0/>`_.
 
 Installation
 ------------
@@ -64,7 +64,7 @@ You can install this package on top of an existing Airflow 2 installation (see `
 for the minimum Airflow version supported) via
 ``pip install apache-airflow-providers-cncf-kubernetes``
 
-The package supports the following python versions: 3.8,3.9,3.10,3.11
+The package supports the following python versions: 3.8,3.9,3.10,3.11,3.12
 
 Requirements
 ------------
@@ -82,4 +82,4 @@ PIP package             Version required
 ======================  =====================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.0.1/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.1.0/changelog.html>`_.

{apache_airflow_providers_cncf_kubernetes-8.0.1rc1 → apache_airflow_providers_cncf_kubernetes-8.1.0rc1}/airflow/providers/cncf/kubernetes/__init__.py

@@ -27,7 +27,7 @@ import packaging.version
 
 __all__ = ["__version__"]
 
-__version__ = "8.0.1"
+__version__ = "8.1.0"
 
 try:
     from airflow import __version__ as airflow_version

{apache_airflow_providers_cncf_kubernetes-8.0.1rc1 → apache_airflow_providers_cncf_kubernetes-8.1.0rc1}/airflow/providers/cncf/kubernetes/backcompat/backwards_compat_converters.py

@@ -15,6 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 """Executes task in a Kubernetes POD."""
+
 from __future__ import annotations
 
 from kubernetes.client import ApiClient, models as k8s

{apache_airflow_providers_cncf_kubernetes-8.0.1rc1 → apache_airflow_providers_cncf_kubernetes-8.1.0rc1}/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py

@@ -21,6 +21,7 @@ KubernetesExecutor.
     For more information on how the KubernetesExecutor works, take a look at the guide:
     :doc:`/kubernetes_executor`
 """
+
 from __future__ import annotations
 
 import contextlib

{apache_airflow_providers_cncf_kubernetes-8.0.1rc1 → apache_airflow_providers_cncf_kubernetes-8.1.0rc1}/airflow/providers/cncf/kubernetes/get_provider_info.py

@@ -28,8 +28,9 @@ def get_provider_info():
         "name": "Kubernetes",
         "description": "`Kubernetes <https://kubernetes.io/>`__\n",
         "state": "ready",
-        "source-date-epoch": 1709555165,
+        "source-date-epoch": 1712665231,
         "versions": [
+            "8.1.0",
             "8.0.1",
             "8.0.0",
             "7.14.0",
@@ -135,7 +136,10 @@ def get_provider_info():
         "triggers": [
             {
                 "integration-name": "Kubernetes",
-                "python-modules": ["airflow.providers.cncf.kubernetes.triggers.pod"],
+                "python-modules": [
+                    "airflow.providers.cncf.kubernetes.triggers.pod",
+                    "airflow.providers.cncf.kubernetes.triggers.job",
+                ],
             }
         ],
         "connection-types": [

{apache_airflow_providers_cncf_kubernetes-8.0.1rc1 → apache_airflow_providers_cncf_kubernetes-8.1.0rc1}/airflow/providers/cncf/kubernetes/hooks/kubernetes.py

@@ -16,10 +16,12 @@
 # under the License.
 from __future__ import annotations
 
+import asyncio
 import contextlib
 import json
 import tempfile
 from functools import cached_property
+from time import sleep
 from typing import TYPE_CHECKING, Any, Generator
 
 import aiofiles
@@ -37,10 +39,18 @@ from airflow.providers.cncf.kubernetes.utils.pod_manager import PodOperatorHookP
 from airflow.utils import yaml
 
 if TYPE_CHECKING:
+    from kubernetes.client import V1JobList
     from kubernetes.client.models import V1Deployment, V1Job, V1Pod
 
 LOADING_KUBE_CONFIG_FILE_RESOURCE = "Loading Kubernetes configuration file kube_config from {}..."
 
+JOB_FINAL_STATUS_CONDITION_TYPES = {
+    "Complete",
+    "Failed",
+}
+
+JOB_STATUS_CONDITION_TYPES = JOB_FINAL_STATUS_CONDITION_TYPES | {"Suspended"}
+
 
 def _load_body_to_dict(body: str) -> dict:
     try:
@@ -502,6 +512,114 @@ class KubernetesHook(BaseHook, PodOperatorHookProtocol):
             raise e
         return resp
 
+    def get_job(self, job_name: str, namespace: str) -> V1Job:
+        """Get Job of specified name and namespace.
+
+        :param job_name: Name of Job to fetch.
+        :param namespace: Namespace of the Job.
+        :return: Job object
+        """
+        return self.batch_v1_client.read_namespaced_job(name=job_name, namespace=namespace, pretty=True)
+
+    def get_job_status(self, job_name: str, namespace: str) -> V1Job:
+        """Get job with status of specified name and namespace.
+
+        :param job_name: Name of Job to fetch.
+        :param namespace: Namespace of the Job.
+        :return: Job object
+        """
+        return self.batch_v1_client.read_namespaced_job_status(
+            name=job_name, namespace=namespace, pretty=True
+        )
+
+    def wait_until_job_complete(self, job_name: str, namespace: str, job_poll_interval: float = 10) -> V1Job:
+        """Block job of specified name and namespace until it is complete or failed.
+
+        :param job_name: Name of Job to fetch.
+        :param namespace: Namespace of the Job.
+        :param job_poll_interval: Interval in seconds between polling the job status
+        :return: Job object
+        """
+        while True:
+            self.log.info("Requesting status for the job '%s' ", job_name)
+            job: V1Job = self.get_job_status(job_name=job_name, namespace=namespace)
+            if self.is_job_complete(job=job):
+                return job
+            self.log.info("The job '%s' is incomplete. Sleeping for %i sec.", job_name, job_poll_interval)
+            sleep(job_poll_interval)
+
+    def list_jobs_all_namespaces(self) -> V1JobList:
+        """Get list of Jobs from all namespaces.
+
+        :return: V1JobList object
+        """
+        return self.batch_v1_client.list_job_for_all_namespaces(pretty=True)
+
+    def list_jobs_from_namespace(self, namespace: str) -> V1JobList:
+        """Get list of Jobs from dedicated namespace.
+
+        :param namespace: Namespace of the Job.
+        :return: V1JobList object
+        """
+        return self.batch_v1_client.list_namespaced_job(namespace=namespace, pretty=True)
+
+    def is_job_complete(self, job: V1Job) -> bool:
+        """Check whether the given job is complete (with success or fail).
+
+        :return: Boolean indicating that the given job is complete.
+        """
+        if status := job.status:
+            if conditions := status.conditions:
+                if final_condition_types := list(
+                    c for c in conditions if c.type in JOB_FINAL_STATUS_CONDITION_TYPES and c.status
+                ):
+                    s = "s" if len(final_condition_types) > 1 else ""
+                    self.log.info(
+                        "The job '%s' state%s: %s",
+                        job.metadata.name,
+                        s,
+                        ", ".join(f"{c.type} at {c.last_transition_time}" for c in final_condition_types),
+                    )
+                    return True
+        return False
+
+    @staticmethod
+    def is_job_failed(job: V1Job) -> str | bool:
+        """Check whether the given job is failed.
+
+        :return: Error message if the job is failed, and False otherwise.
+        """
+        if status := job.status:
+            conditions = status.conditions or []
+            if fail_condition := next((c for c in conditions if c.type == "Failed" and c.status), None):
+                return fail_condition.reason
+        return False
+
+    @staticmethod
+    def is_job_successful(job: V1Job) -> str | bool:
+        """Check whether the given job is completed successfully..
+
+        :return: Error message if the job is failed, and False otherwise.
+        """
+        if status := job.status:
+            conditions = status.conditions or []
+            return bool(next((c for c in conditions if c.type == "Complete" and c.status), None))
+        return False
+
+    def patch_namespaced_job(self, job_name: str, namespace: str, body: object) -> V1Job:
+        """
+        Update the specified Job.
+
+        :param job_name: name of the Job
+        :param namespace: the namespace to run within kubernetes
+        :param body: json object with parameters for update
+        """
+        return self.batch_v1_client.patch_namespaced_job(
+            name=job_name,
+            namespace=namespace,
+            body=body,
+        )
+
 
 def _get_bool(val) -> bool | None:
     """Convert val to bool if can be done with certainty; if we cannot infer intention we return None."""
@@ -667,3 +785,34 @@ class AsyncKubernetesHook(KubernetesHook):
             except HTTPError:
                 self.log.exception("There was an error reading the kubernetes API.")
                 raise
+
+    async def get_job_status(self, name: str, namespace: str) -> V1Job:
+        """
+        Get job's status object.
+
+        :param name: Name of the pod.
+        :param namespace: Name of the pod's namespace.
+        """
+        async with self.get_conn() as connection:
+            v1_api = async_client.BatchV1Api(connection)
+            job: V1Job = await v1_api.read_namespaced_job_status(
+                name=name,
+                namespace=namespace,
+            )
+        return job
+
+    async def wait_until_job_complete(self, name: str, namespace: str, poll_interval: float = 10) -> V1Job:
+        """Block job of specified name and namespace until it is complete or failed.
+
+        :param name: Name of Job to fetch.
+        :param namespace: Namespace of the Job.
+        :param poll_interval: Interval in seconds between polling the job status
+        :return: Job object
+        """
+        while True:
+            self.log.info("Requesting status for the job '%s' ", name)
+            job: V1Job = await self.get_job_status(name=name, namespace=namespace)
+            if self.is_job_complete(job=job):
+                return job
+            self.log.info("The job '%s' is incomplete. Sleeping for %i sec.", name, poll_interval)
+            await asyncio.sleep(poll_interval)

{apache_airflow_providers_cncf_kubernetes-8.0.1rc1 → apache_airflow_providers_cncf_kubernetes-8.1.0rc1}/airflow/providers/cncf/kubernetes/k8s_model.py

@@ -15,6 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 """Classes for interacting with Kubernetes API."""
+
 from __future__ import annotations
 
 from abc import ABC, abstractmethod

{apache_airflow_providers_cncf_kubernetes-8.0.1rc1 → apache_airflow_providers_cncf_kubernetes-8.1.0rc1}/airflow/providers/cncf/kubernetes/kube_client.py

@@ -15,6 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 """Client for kubernetes communication."""
+
 from __future__ import annotations
 
 import logging

{apache_airflow_providers_cncf_kubernetes-8.0.1rc1 → apache_airflow_providers_cncf_kubernetes-8.1.0rc1}/airflow/providers/cncf/kubernetes/kubernetes_helper_functions.py

@@ -104,7 +104,7 @@ def create_unique_id(
         name += task_id
     base_name = slugify(name, lowercase=True)[:max_length].strip(".-")
     if unique:
-        return add_pod_suffix(pod_name=base_name, rand_len=8, max_len=max_length)
+        return add_unique_suffix(name=base_name, rand_len=8, max_len=max_length)
     else:
         return base_name
 

{apache_airflow_providers_cncf_kubernetes-8.0.1rc1 → apache_airflow_providers_cncf_kubernetes-8.1.0rc1}/airflow/providers/cncf/kubernetes/operators/custom_object_launcher.py

@@ -15,6 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 """Launches Custom object."""
+
 from __future__ import annotations
 
 import time
@@ -59,9 +60,9 @@ class SparkJobSpec:
         if self.spec.get("dynamicAllocation", {}).get("enabled"):
             if not all(
                 [
-                    self.spec["dynamicAllocation"]["initialExecutors"],
-                    self.spec["dynamicAllocation"]["minExecutors"],
-                    self.spec["dynamicAllocation"]["maxExecutors"],
+                    self.spec["dynamicAllocation"].get("initialExecutors"),
+                    self.spec["dynamicAllocation"].get("minExecutors"),
+                    self.spec["dynamicAllocation"].get("maxExecutors"),
                 ]
             ):
                 raise AirflowException("Make sure initial/min/max value for dynamic allocation is passed")