apache-airflow-providers-cncf-kubernetes 8.1.1rc1__tar.gz → 8.2.0rc1__tar.gz

{apache_airflow_providers_cncf_kubernetes-8.1.1rc1 → apache_airflow_providers_cncf_kubernetes-8.2.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apache-airflow-providers-cncf-kubernetes
-Version: 8.1.1rc1
+Version: 8.2.0rc1
 Summary: Provider package apache-airflow-providers-cncf-kubernetes for Apache Airflow
 Keywords: airflow-provider,cncf.kubernetes,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -22,15 +22,15 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: System :: Monitoring
 Requires-Dist: aiofiles>=23.2.0
-Requires-Dist: apache-airflow>=2.6.0rc0
+Requires-Dist: apache-airflow>=2.7.0rc0
 Requires-Dist: asgiref>=3.5.2
 Requires-Dist: cryptography>=2.0.0
 Requires-Dist: google-re2>=1.0
 Requires-Dist: kubernetes>=28.1.0,<=29.0.0
 Requires-Dist: kubernetes_asyncio>=28.1.0,<=29.0.0
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.1.1/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.1.1
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.2.0/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.2.0
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Twitter, https://twitter.com/ApacheAirflow
@@ -80,7 +80,7 @@ Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
 
 Package ``apache-airflow-providers-cncf-kubernetes``
 
-Release: ``8.1.1.rc1``
+Release: ``8.2.0.rc1``
 
 
 `Kubernetes <https://kubernetes.io/>`__
@@ -93,7 +93,7 @@ This is a provider package for ``cncf.kubernetes`` provider. All classes for thi
 are in ``airflow.providers.cncf.kubernetes`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.1.1/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.2.0/>`_.
 
 Installation
 ------------
@@ -111,7 +111,7 @@ Requirements
 PIP package             Version required
 ======================  =====================
 ``aiofiles``            ``>=23.2.0``
-``apache-airflow``      ``>=2.6.0``
+``apache-airflow``      ``>=2.7.0``
 ``asgiref``             ``>=3.5.2``
 ``cryptography``        ``>=2.0.0``
 ``kubernetes``          ``>=28.1.0,<=29.0.0``
@@ -120,4 +120,4 @@ PIP package             Version required
 ======================  =====================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.1.1/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.2.0/changelog.html>`_.

{apache_airflow_providers_cncf_kubernetes-8.1.1rc1 → apache_airflow_providers_cncf_kubernetes-8.2.0rc1}/README.rst

@@ -42,7 +42,7 @@
 
 Package ``apache-airflow-providers-cncf-kubernetes``
 
-Release: ``8.1.1.rc1``
+Release: ``8.2.0.rc1``
 
 
 `Kubernetes <https://kubernetes.io/>`__
@@ -55,7 +55,7 @@ This is a provider package for ``cncf.kubernetes`` provider. All classes for thi
 are in ``airflow.providers.cncf.kubernetes`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.1.1/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.2.0/>`_.
 
 Installation
 ------------
@@ -73,7 +73,7 @@ Requirements
 PIP package             Version required
 ======================  =====================
 ``aiofiles``            ``>=23.2.0``
-``apache-airflow``      ``>=2.6.0``
+``apache-airflow``      ``>=2.7.0``
 ``asgiref``             ``>=3.5.2``
 ``cryptography``        ``>=2.0.0``
 ``kubernetes``          ``>=28.1.0,<=29.0.0``
@@ -82,4 +82,4 @@ PIP package             Version required
 ======================  =====================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.1.1/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.2.0/changelog.html>`_.

{apache_airflow_providers_cncf_kubernetes-8.1.1rc1 → apache_airflow_providers_cncf_kubernetes-8.2.0rc1}/airflow/providers/cncf/kubernetes/__init__.py

@@ -27,7 +27,7 @@ import packaging.version
 
 __all__ = ["__version__"]
 
-__version__ = "8.1.1"
+__version__ = "8.2.0"
 
 try:
     from airflow import __version__ as airflow_version
@@ -35,8 +35,8 @@ except ImportError:
     from airflow.version import version as airflow_version
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
-    "2.6.0"
+    "2.7.0"
 ):
     raise RuntimeError(
-        f"The package `apache-airflow-providers-cncf-kubernetes:{__version__}` needs Apache Airflow 2.6.0+"
+        f"The package `apache-airflow-providers-cncf-kubernetes:{__version__}` needs Apache Airflow 2.7.0+"
     )

{apache_airflow_providers_cncf_kubernetes-8.1.1rc1 → apache_airflow_providers_cncf_kubernetes-8.2.0rc1}/airflow/providers/cncf/kubernetes/backcompat/backwards_compat_converters.py

@@ -76,10 +76,20 @@ def convert_env_vars(env_vars: list[k8s.V1EnvVar] | dict[str, str]) -> list[k8s.
 
     If the collection is a str-str dict, convert it into a list of ``V1EnvVar``s.
     """
-    if isinstance(env_vars, list):
-        return env_vars
     if isinstance(env_vars, dict):
         return [k8s.V1EnvVar(name=k, value=v) for k, v in env_vars.items()]
+    return env_vars
+
+
+def convert_env_vars_or_raise_error(env_vars: list[k8s.V1EnvVar] | dict[str, str]) -> list[k8s.V1EnvVar]:
+    """
+    Separate function to convert env var collection for kubernetes and then raise an error if it is still the wrong type.
+
+    This is used after the template strings have been rendered.
+    """
+    env_vars = convert_env_vars(env_vars)
+    if isinstance(env_vars, list):
+        return env_vars
     raise AirflowException(f"Expected dict or list, got {type(env_vars)}")
 
 

{apache_airflow_providers_cncf_kubernetes-8.1.1rc1 → apache_airflow_providers_cncf_kubernetes-8.2.0rc1}/airflow/providers/cncf/kubernetes/get_provider_info.py

@@ -28,8 +28,9 @@ def get_provider_info():
         "name": "Kubernetes",
         "description": "`Kubernetes <https://kubernetes.io/>`__\n",
         "state": "ready",
-        "source-date-epoch": 1713253055,
+        "source-date-epoch": 1714476004,
         "versions": [
+            "8.2.0",
             "8.1.1",
             "8.1.0",
             "8.0.1",
@@ -88,7 +89,7 @@ def get_provider_info():
         ],
         "dependencies": [
             "aiofiles>=23.2.0",
-            "apache-airflow>=2.6.0",
+            "apache-airflow>=2.7.0",
             "asgiref>=3.5.2",
             "cryptography>=2.0.0",
             "kubernetes>=28.1.0,<=29.0.0",

{apache_airflow_providers_cncf_kubernetes-8.1.1rc1 → apache_airflow_providers_cncf_kubernetes-8.2.0rc1}/airflow/providers/cncf/kubernetes/operators/custom_object_launcher.py

@@ -60,12 +60,11 @@ class SparkJobSpec:
         if self.spec.get("dynamicAllocation", {}).get("enabled"):
             if not all(
                 [
-                    self.spec["dynamicAllocation"].get("initialExecutors"),
                     self.spec["dynamicAllocation"].get("minExecutors"),
                     self.spec["dynamicAllocation"].get("maxExecutors"),
                 ]
             ):
-                raise AirflowException("Make sure initial/min/max value for dynamic allocation is passed")
+                raise AirflowException("Make sure min/max value for dynamic allocation is passed")
 
     def update_resources(self):
         if self.spec["driver"].get("container_resources"):
@@ -345,7 +344,7 @@ class CustomObjectLauncher(LoggingMixin):
             waiting_message = waiting_status.message
         except Exception:
             return
-        if waiting_reason != "ContainerCreating":
+        if waiting_reason not in ("ContainerCreating", "PodInitializing"):
             raise AirflowException(f"Spark Job Failed. Status: {waiting_reason}, Error: {waiting_message}")
 
     def delete_spark_job(self, spark_job_name=None):

{apache_airflow_providers_cncf_kubernetes-8.1.1rc1 → apache_airflow_providers_cncf_kubernetes-8.2.0rc1}/airflow/providers/cncf/kubernetes/operators/pod.py

@@ -33,6 +33,7 @@ from functools import cached_property
 from typing import TYPE_CHECKING, Any, Callable, Iterable, Sequence
 
 import kubernetes
+import tenacity
 from deprecated import deprecated
 from kubernetes.client import CoreV1Api, V1Pod, models as k8s
 from kubernetes.stream import stream
@@ -51,6 +52,7 @@ from airflow.providers.cncf.kubernetes.backcompat.backwards_compat_converters im
     convert_affinity,
     convert_configmap,
     convert_env_vars,
+    convert_env_vars_or_raise_error,
     convert_image_pull_secrets,
     convert_pod_runtime_info_env,
     convert_port,
@@ -76,6 +78,7 @@ from airflow.providers.cncf.kubernetes.utils.pod_manager import (
     PodNotFoundException,
     PodOperatorHookProtocol,
     PodPhase,
+    check_exception_is_kubernetes_api_unauthorized,
     container_is_succeeded,
     get_container_termination_message,
 )
@@ -332,8 +335,10 @@ class KubernetesPodOperator(BaseOperator):
         self.startup_check_interval_seconds = startup_check_interval_seconds
         env_vars = convert_env_vars(env_vars) if env_vars else []
         self.env_vars = env_vars
-        if pod_runtime_info_envs:
-            self.env_vars.extend([convert_pod_runtime_info_env(p) for p in pod_runtime_info_envs])
+        pod_runtime_info_envs = (
+            [convert_pod_runtime_info_env(p) for p in pod_runtime_info_envs] if pod_runtime_info_envs else []
+        )
+        self.pod_runtime_info_envs = pod_runtime_info_envs
         self.env_from = env_from or []
         if configmaps:
             self.env_from.extend([convert_configmap(c) for c in configmaps])
@@ -615,9 +620,7 @@ class KubernetesPodOperator(BaseOperator):
             if not self.get_logs or (
                 self.container_logs is not True and self.base_container_name not in self.container_logs
             ):
-                self.pod_manager.await_container_completion(
-                    pod=self.pod, container_name=self.base_container_name
-                )
+                self.await_container_completion(pod=self.pod, container_name=self.base_container_name)
             if self.callbacks:
                 self.callbacks.on_pod_completion(
                     pod=self.find_pod(self.pod.metadata.namespace, context=context),
@@ -644,6 +647,28 @@ class KubernetesPodOperator(BaseOperator):
         if self.do_xcom_push:
             return result
 
+    @tenacity.retry(
+        stop=tenacity.stop_after_attempt(3),
+        wait=tenacity.wait_exponential(max=15),
+        retry=tenacity.retry_if_exception(lambda exc: check_exception_is_kubernetes_api_unauthorized(exc)),
+        reraise=True,
+    )
+    def await_container_completion(self, pod: k8s.V1Pod, container_name: str):
+        try:
+            self.pod_manager.await_container_completion(pod=pod, container_name=container_name)
+        except kubernetes.client.exceptions.ApiException as exc:
+            if exc.status and str(exc.status) == "401":
+                self.log.warning(
+                    "Failed to check container status due to permission error. Refreshing credentials and retrying."
+                )
+                self._refresh_cached_properties()
+            raise exc
+
+    def _refresh_cached_properties(self):
+        del self.hook
+        del self.client
+        del self.pod_manager
+
     def execute_async(self, context: Context):
         self.pod_request_obj = self.build_pod_request_obj(context)
         self.pod = self.get_or_create_pod(  # must set `self.pod` for `on_kill`
@@ -985,6 +1010,11 @@ class KubernetesPodOperator(BaseOperator):
         template file.
         """
         self.log.debug("Creating pod for KubernetesPodOperator task %s", self.task_id)
+
+        self.env_vars = convert_env_vars_or_raise_error(self.env_vars) if self.env_vars else []
+        if self.pod_runtime_info_envs:
+            self.env_vars.extend(self.pod_runtime_info_envs)
+
         if self.pod_template_file:
             self.log.debug("Pod template file found, will parse for base pod")
             pod_template = pod_generator.PodGenerator.deserialize_model_file(self.pod_template_file)

{apache_airflow_providers_cncf_kubernetes-8.1.1rc1 → apache_airflow_providers_cncf_kubernetes-8.2.0rc1}/airflow/providers/cncf/kubernetes/pod_generator.py

@@ -50,7 +50,7 @@ from airflow.providers.cncf.kubernetes.kubernetes_helper_functions import (
     rand_str,
 )
 from airflow.providers.cncf.kubernetes.pod_generator_deprecated import (
-    PodDefaults,
+    PodDefaults as PodDefaultsDeprecated,
     PodGenerator as PodGeneratorDeprecated,
 )
 from airflow.utils import yaml
@@ -180,10 +180,10 @@ class PodGenerator:
         """Add sidecar."""
         pod_cp = copy.deepcopy(pod)
         pod_cp.spec.volumes = pod.spec.volumes or []
-        pod_cp.spec.volumes.insert(0, PodDefaults.VOLUME)
+        pod_cp.spec.volumes.insert(0, PodDefaultsDeprecated.VOLUME)
         pod_cp.spec.containers[0].volume_mounts = pod_cp.spec.containers[0].volume_mounts or []
-        pod_cp.spec.containers[0].volume_mounts.insert(0, PodDefaults.VOLUME_MOUNT)
-        pod_cp.spec.containers.append(PodDefaults.SIDECAR_CONTAINER)
+        pod_cp.spec.containers[0].volume_mounts.insert(0, PodDefaultsDeprecated.VOLUME_MOUNT)
+        pod_cp.spec.containers.append(PodDefaultsDeprecated.SIDECAR_CONTAINER)
 
         return pod_cp
 

{apache_airflow_providers_cncf_kubernetes-8.1.1rc1 → apache_airflow_providers_cncf_kubernetes-8.2.0rc1}/airflow/providers/cncf/kubernetes/pod_generator_deprecated.py

@@ -31,8 +31,6 @@ import uuid
 import re2
 from kubernetes.client import models as k8s
 
-from airflow.utils.hashlib_wrapper import md5
-
 MAX_POD_ID_LEN = 253
 
 MAX_LABEL_LEN = 63
@@ -71,6 +69,8 @@ def make_safe_label_value(string):
     way from the original value sent to this function, then we need to truncate to
     53 chars, and append it with a unique hash.
     """
+    from airflow.utils.hashlib_wrapper import md5
+
     safe_label = re2.sub(r"^[^a-z0-9A-Z]*|[^a-zA-Z0-9_\-\.]|[^a-z0-9A-Z]*$", "", string)
 
     if len(safe_label) > MAX_LABEL_LEN or string != safe_label:

{apache_airflow_providers_cncf_kubernetes-8.1.1rc1 → apache_airflow_providers_cncf_kubernetes-8.2.0rc1}/airflow/providers/cncf/kubernetes/pod_launcher_deprecated.py

@@ -33,7 +33,7 @@ from requests.exceptions import HTTPError
 
 from airflow.exceptions import AirflowException, RemovedInAirflow3Warning
 from airflow.providers.cncf.kubernetes.kube_client import get_kube_client
-from airflow.providers.cncf.kubernetes.pod_generator import PodDefaults
+from airflow.providers.cncf.kubernetes.pod_generator import PodDefaultsDeprecated
 from airflow.settings import pod_mutation_hook
 from airflow.utils.log.logging_mixin import LoggingMixin
 from airflow.utils.state import State
@@ -272,7 +272,7 @@ class PodLauncher(LoggingMixin):
             self._client.connect_get_namespaced_pod_exec,
             pod.metadata.name,
             pod.metadata.namespace,
-            container=PodDefaults.SIDECAR_CONTAINER_NAME,
+            container=PodDefaultsDeprecated.SIDECAR_CONTAINER_NAME,
             command=["/bin/sh"],
             stdin=True,
             stdout=True,
@@ -281,7 +281,7 @@ class PodLauncher(LoggingMixin):
             _preload_content=False,
         )
         try:
-            result = self._exec_pod_command(resp, f"cat {PodDefaults.XCOM_MOUNT_PATH}/return.json")
+            result = self._exec_pod_command(resp, f"cat {PodDefaultsDeprecated.XCOM_MOUNT_PATH}/return.json")
             self._exec_pod_command(resp, "kill -s SIGINT 1")
         finally:
             resp.close()

{apache_airflow_providers_cncf_kubernetes-8.1.1rc1 → apache_airflow_providers_cncf_kubernetes-8.2.0rc1}/airflow/providers/cncf/kubernetes/triggers/pod.py

@@ -262,6 +262,7 @@ class KubernetesPodTrigger(BaseTrigger):
                         "last_log_time": self.last_log_time,
                     }
                 )
+            self.log.debug("Container is not completed and still working.")
             if time_get_more_logs and datetime.datetime.now(tz=datetime.timezone.utc) > time_get_more_logs:
                 return TriggerEvent(
                     {
@@ -271,6 +272,7 @@ class KubernetesPodTrigger(BaseTrigger):
                         "name": self.pod_name,
                     }
                 )
+            self.log.debug("Sleeping for %s seconds.", self.poll_interval)
             await asyncio.sleep(self.poll_interval)
 
     def _get_async_hook(self) -> AsyncKubernetesHook:

{apache_airflow_providers_cncf_kubernetes-8.1.1rc1 → apache_airflow_providers_cncf_kubernetes-8.2.0rc1}/airflow/providers/cncf/kubernetes/utils/pod_manager.py

@@ -42,7 +42,7 @@ from urllib3.exceptions import HTTPError, TimeoutError
 
 from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
 from airflow.providers.cncf.kubernetes.callbacks import ExecutionMode, KubernetesPodOperatorCallback
-from airflow.providers.cncf.kubernetes.pod_generator import PodDefaults
+from airflow.providers.cncf.kubernetes.utils.xcom_sidecar import PodDefaults
 from airflow.utils.log.logging_mixin import LoggingMixin
 from airflow.utils.timezone import utcnow
 
@@ -190,6 +190,10 @@ def get_container_termination_message(pod: V1Pod, container_name: str):
         return container_status.state.terminated.message if container_status else None
 
 
+def check_exception_is_kubernetes_api_unauthorized(exc: BaseException):
+    return isinstance(exc, ApiException) and exc.status and str(exc.status) == "401"
+
+
 class PodLaunchTimeoutException(AirflowException):
     """When pod does not leave the ``Pending`` phase within specified timeout."""
 

{apache_airflow_providers_cncf_kubernetes-8.1.1rc1 → apache_airflow_providers_cncf_kubernetes-8.2.0rc1}/pyproject.toml

@@ -28,7 +28,7 @@ build-backend = "flit_core.buildapi"
 
 [project]
 name = "apache-airflow-providers-cncf-kubernetes"
-version = "8.1.1.rc1"
+version = "8.2.0.rc1"
 description = "Provider package apache-airflow-providers-cncf-kubernetes for Apache Airflow"
 readme = "README.rst"
 authors = [
@@ -57,7 +57,7 @@ classifiers = [
 requires-python = "~=3.8"
 dependencies = [
     "aiofiles>=23.2.0",
-    "apache-airflow>=2.6.0rc0",
+    "apache-airflow>=2.7.0rc0",
     "asgiref>=3.5.2",
     "cryptography>=2.0.0",
     "google-re2>=1.0",
@@ -66,8 +66,8 @@ dependencies = [
 ]
 
 [project.urls]
-"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.1.1"
-"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.1.1/changelog.html"
+"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.2.0"
+"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.2.0/changelog.html"
 "Bug Tracker" = "https://github.com/apache/airflow/issues"
 "Source Code" = "https://github.com/apache/airflow"
 "Slack Chat" = "https://s.apache.org/airflow-slack"