apache-airflow-providers-cncf-kubernetes 8.0.0rc3__tar.gz → 8.0.1rc1__tar.gz

{apache_airflow_providers_cncf_kubernetes-8.0.0rc3 → apache_airflow_providers_cncf_kubernetes-8.0.1rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apache-airflow-providers-cncf-kubernetes
-Version: 8.0.0rc3
+Version: 8.0.1rc1
 Summary: Provider package apache-airflow-providers-cncf-kubernetes for Apache Airflow
 Keywords: airflow-provider,cncf.kubernetes,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -28,8 +28,8 @@ Requires-Dist: google-re2>=1.0
 Requires-Dist: kubernetes>=28.1.0,<=29.0.0
 Requires-Dist: kubernetes_asyncio>=28.1.0,<=29.0.0
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.0.0/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.0.0
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.0.1/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.0.1
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Twitter, https://twitter.com/ApacheAirflow
@@ -79,7 +79,7 @@ Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
 
 Package ``apache-airflow-providers-cncf-kubernetes``
 
-Release: ``8.0.0.rc3``
+Release: ``8.0.1.rc1``
 
 
 `Kubernetes <https://kubernetes.io/>`__
@@ -92,7 +92,7 @@ This is a provider package for ``cncf.kubernetes`` provider. All classes for thi
 are in ``airflow.providers.cncf.kubernetes`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.0.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.0.1/>`_.
 
 Installation
 ------------
@@ -119,4 +119,4 @@ PIP package             Version required
 ======================  =====================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.0.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.0.1/changelog.html>`_.

{apache_airflow_providers_cncf_kubernetes-8.0.0rc3 → apache_airflow_providers_cncf_kubernetes-8.0.1rc1}/README.rst

@@ -42,7 +42,7 @@
 
 Package ``apache-airflow-providers-cncf-kubernetes``
 
-Release: ``8.0.0.rc3``
+Release: ``8.0.1.rc1``
 
 
 `Kubernetes <https://kubernetes.io/>`__
@@ -55,7 +55,7 @@ This is a provider package for ``cncf.kubernetes`` provider. All classes for thi
 are in ``airflow.providers.cncf.kubernetes`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.0.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.0.1/>`_.
 
 Installation
 ------------
@@ -82,4 +82,4 @@ PIP package             Version required
 ======================  =====================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.0.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.0.1/changelog.html>`_.

{apache_airflow_providers_cncf_kubernetes-8.0.0rc3 → apache_airflow_providers_cncf_kubernetes-8.0.1rc1}/airflow/providers/cncf/kubernetes/__init__.py

@@ -27,7 +27,7 @@ import packaging.version
 
 __all__ = ["__version__"]
 
-__version__ = "8.0.0"
+__version__ = "8.0.1"
 
 try:
     from airflow import __version__ as airflow_version

{apache_airflow_providers_cncf_kubernetes-8.0.0rc3 → apache_airflow_providers_cncf_kubernetes-8.0.1rc1}/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py

@@ -19,7 +19,7 @@ KubernetesExecutor.
 
 .. seealso::
     For more information on how the KubernetesExecutor works, take a look at the guide:
-    :ref:`executor:KubernetesExecutor`
+    :doc:`/kubernetes_executor`
 """
 from __future__ import annotations
 
@@ -431,10 +431,9 @@ class KubernetesExecutor(BaseExecutor):
                     self.kube_scheduler.run_next(task)
                     self.task_publish_retries.pop(key, None)
                 except PodReconciliationError as e:
-                    self.log.error(
+                    self.log.exception(
                         "Pod reconciliation failed, likely due to kubernetes library upgrade. "
                         "Try clearing the task to re-run.",
-                        exc_info=True,
                     )
                     self.fail(task[0], e)
                 except ApiException as e:

{apache_airflow_providers_cncf_kubernetes-8.0.0rc3 → apache_airflow_providers_cncf_kubernetes-8.0.1rc1}/airflow/providers/cncf/kubernetes/executors/kubernetes_executor_utils.py

@@ -235,6 +235,34 @@ class KubernetesJobWatcher(multiprocessing.Process, LoggingMixin):
                 self.watcher_queue.put(
                     (pod_name, namespace, TaskInstanceState.FAILED, annotations, resource_version)
                 )
+            elif (
+                self.kube_config.worker_pod_pending_fatal_container_state_reasons
+                and "status" in event["raw_object"]
+            ):
+                self.log.info("Event: %s Pending, annotations: %s", pod_name, annotations_string)
+                # Init containers and base container statuses to check.
+                # Skipping the other containers statuses check.
+                container_statuses_to_check = []
+                if "initContainerStatuses" in event["raw_object"]["status"]:
+                    container_statuses_to_check.extend(event["raw_object"]["status"]["initContainerStatuses"])
+                if "containerStatuses" in event["raw_object"]["status"]:
+                    container_statuses_to_check.append(event["raw_object"]["status"]["containerStatuses"][0])
+                for container_status in container_statuses_to_check:
+                    container_status_state = container_status["state"]
+                    if "waiting" in container_status_state:
+                        if (
+                            container_status_state["waiting"]["reason"]
+                            in self.kube_config.worker_pod_pending_fatal_container_state_reasons
+                        ):
+                            if (
+                                container_status_state["waiting"]["reason"] == "ErrImagePull"
+                                and container_status_state["waiting"]["message"] == "pull QPS exceeded"
+                            ):
+                                continue
+                            self.watcher_queue.put(
+                                (pod_name, namespace, TaskInstanceState.FAILED, annotations, resource_version)
+                            )
+                            break
             else:
                 self.log.debug("Event: %s Pending, annotations: %s", pod_name, annotations_string)
         elif status == "Failed":

{apache_airflow_providers_cncf_kubernetes-8.0.0rc3 → apache_airflow_providers_cncf_kubernetes-8.0.1rc1}/airflow/providers/cncf/kubernetes/get_provider_info.py

@@ -28,8 +28,9 @@ def get_provider_info():
         "name": "Kubernetes",
         "description": "`Kubernetes <https://kubernetes.io/>`__\n",
         "state": "ready",
-        "source-date-epoch": 1707637063,
+        "source-date-epoch": 1709555165,
         "versions": [
+            "8.0.1",
             "8.0.0",
             "7.14.0",
             "7.13.0",
@@ -222,6 +223,13 @@ def get_provider_info():
                         "example": None,
                         "default": "False",
                     },
+                    "worker_pod_pending_fatal_container_state_reasons": {
+                        "description": "If the worker pods are in a pending state due to a fatal container\nstate reasons, then fail the task and delete the worker pod\nif delete_worker_pods is True and delete_worker_pods_on_failure is True.\n",
+                        "version_added": "8.1.0",
+                        "type": "string",
+                        "example": None,
+                        "default": "CreateContainerConfigError,ErrImagePull,CreateContainerError,ImageInspectError, InvalidImageName",
+                    },
                     "worker_pods_creation_batch_size": {
                         "description": 'Number of Kubernetes Worker Pod creation calls per scheduler loop.\nNote that the current default of "1" will only launch a single pod\nper-heartbeat. It is HIGHLY recommended that users increase this\nnumber to match the tolerance of their kubernetes cluster for\nbetter performance.\n',
                         "version_added": None,

{apache_airflow_providers_cncf_kubernetes-8.0.0rc3 → apache_airflow_providers_cncf_kubernetes-8.0.1rc1}/airflow/providers/cncf/kubernetes/kube_config.py

@@ -40,6 +40,12 @@ class KubeConfig:
         self.delete_worker_pods_on_failure = conf.getboolean(
             self.kubernetes_section, "delete_worker_pods_on_failure"
         )
+        self.worker_pod_pending_fatal_container_state_reasons = []
+        if conf.get(self.kubernetes_section, "worker_pod_pending_fatal_container_state_reasons", fallback=""):
+            self.worker_pod_pending_fatal_container_state_reasons = conf.get(
+                self.kubernetes_section, "worker_pod_pending_fatal_container_state_reasons"
+            ).split(",")
+
         self.worker_pods_creation_batch_size = conf.getint(
             self.kubernetes_section, "worker_pods_creation_batch_size"
         )

{apache_airflow_providers_cncf_kubernetes-8.0.0rc3 → apache_airflow_providers_cncf_kubernetes-8.0.1rc1}/airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py

@@ -21,7 +21,7 @@ from __future__ import annotations
 import warnings
 
 from airflow.exceptions import AirflowProviderDeprecationWarning
-from airflow.providers.cncf.kubernetes.operators.pod import *  # noqa
+from airflow.providers.cncf.kubernetes.operators.pod import *  # noqa: F403
 
 warnings.warn(
     "This module is deprecated. Please use `airflow.providers.cncf.kubernetes.operators.pod` instead.",

{apache_airflow_providers_cncf_kubernetes-8.0.0rc3 → apache_airflow_providers_cncf_kubernetes-8.0.1rc1}/airflow/providers/cncf/kubernetes/operators/pod.py

@@ -793,9 +793,11 @@ class KubernetesPodOperator(BaseOperator):
             self.callbacks.on_pod_cleanup(pod=pod, client=self.client, mode=ExecutionMode.SYNC)
 
     def cleanup(self, pod: k8s.V1Pod, remote_pod: k8s.V1Pod):
-        # If a task got marked as failed, "on_kill" method would be called and the pod will be cleaned up
+        # Skip cleaning the pod in the following scenarios.
+        # 1. If a task got marked as failed, "on_kill" method would be called and the pod will be cleaned up
         # there. Cleaning it up again will raise an exception (which might cause retry).
-        if self._killed:
+        # 2. remote pod is null (ex: pod creation failed)
+        if self._killed or not remote_pod:
             return
 
         istio_enabled = self.is_istio_enabled(remote_pod)

{apache_airflow_providers_cncf_kubernetes-8.0.0rc3 → apache_airflow_providers_cncf_kubernetes-8.0.1rc1}/airflow/providers/cncf/kubernetes/triggers/kubernetes_pod.py

@@ -21,7 +21,7 @@ from __future__ import annotations
 import warnings
 
 from airflow.exceptions import AirflowProviderDeprecationWarning
-from airflow.providers.cncf.kubernetes.triggers.pod import *  # noqa
+from airflow.providers.cncf.kubernetes.triggers.pod import *  # noqa: F403
 
 warnings.warn(
     "This module is deprecated. Please use `airflow.providers.cncf.kubernetes.triggers.pod` instead.",

{apache_airflow_providers_cncf_kubernetes-8.0.0rc3 → apache_airflow_providers_cncf_kubernetes-8.0.1rc1}/pyproject.toml

@@ -28,7 +28,7 @@ build-backend = "flit_core.buildapi"
 
 [project]
 name = "apache-airflow-providers-cncf-kubernetes"
-version = "8.0.0.rc3"
+version = "8.0.1.rc1"
 description = "Provider package apache-airflow-providers-cncf-kubernetes for Apache Airflow"
 readme = "README.rst"
 authors = [
@@ -65,8 +65,8 @@ dependencies = [
 ]
 
 [project.urls]
-"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.0.0"
-"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.0.0/changelog.html"
+"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.0.1"
+"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.0.1/changelog.html"
 "Bug Tracker" = "https://github.com/apache/airflow/issues"
 "Source Code" = "https://github.com/apache/airflow"
 "Slack Chat" = "https://s.apache.org/airflow-slack"