apache-airflow-providers-cncf-kubernetes 10.0.1rc1__tar.gz → 10.1.0rc1__tar.gz

{apache_airflow_providers_cncf_kubernetes-10.0.1rc1 → apache_airflow_providers_cncf_kubernetes-10.1.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: apache-airflow-providers-cncf-kubernetes
-Version: 10.0.1rc1
+Version: 10.1.0rc1
 Summary: Provider package apache-airflow-providers-cncf-kubernetes for Apache Airflow
 Keywords: airflow-provider,cncf.kubernetes,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -21,18 +21,18 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: System :: Monitoring
 Requires-Dist: aiofiles>=23.2.0
-Requires-Dist: apache-airflow>=2.8.0rc0
+Requires-Dist: apache-airflow>=2.9.0rc0
 Requires-Dist: asgiref>=3.5.2
 Requires-Dist: cryptography>=41.0.0
 Requires-Dist: google-re2>=1.0
 Requires-Dist: kubernetes>=29.0.0,<=31.0.0
 Requires-Dist: kubernetes_asyncio>=29.0.0,<=31.0.0
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.0.1/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.0.1
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.1.0/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.1.0
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
-Project-URL: Twitter, https://twitter.com/ApacheAirflow
+Project-URL: Twitter, https://x.com/ApacheAirflow
 Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
 
 
@@ -79,7 +79,7 @@ Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
 
 Package ``apache-airflow-providers-cncf-kubernetes``
 
-Release: ``10.0.1.rc1``
+Release: ``10.1.0.rc1``
 
 
 `Kubernetes <https://kubernetes.io/>`__
@@ -92,7 +92,7 @@ This is a provider package for ``cncf.kubernetes`` provider. All classes for thi
 are in ``airflow.providers.cncf.kubernetes`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.0.1/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.1.0/>`_.
 
 Installation
 ------------
@@ -110,7 +110,7 @@ Requirements
 PIP package             Version required
 ======================  =====================
 ``aiofiles``            ``>=23.2.0``
-``apache-airflow``      ``>=2.8.0``
+``apache-airflow``      ``>=2.9.0``
 ``asgiref``             ``>=3.5.2``
 ``cryptography``        ``>=41.0.0``
 ``kubernetes``          ``>=29.0.0,<=31.0.0``
@@ -119,4 +119,4 @@ PIP package             Version required
 ======================  =====================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.0.1/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.1.0/changelog.html>`_.

{apache_airflow_providers_cncf_kubernetes-10.0.1rc1 → apache_airflow_providers_cncf_kubernetes-10.1.0rc1}/README.rst

@@ -42,7 +42,7 @@
 
 Package ``apache-airflow-providers-cncf-kubernetes``
 
-Release: ``10.0.1.rc1``
+Release: ``10.1.0.rc1``
 
 
 `Kubernetes <https://kubernetes.io/>`__
@@ -55,7 +55,7 @@ This is a provider package for ``cncf.kubernetes`` provider. All classes for thi
 are in ``airflow.providers.cncf.kubernetes`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.0.1/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.1.0/>`_.
 
 Installation
 ------------
@@ -73,7 +73,7 @@ Requirements
 PIP package             Version required
 ======================  =====================
 ``aiofiles``            ``>=23.2.0``
-``apache-airflow``      ``>=2.8.0``
+``apache-airflow``      ``>=2.9.0``
 ``asgiref``             ``>=3.5.2``
 ``cryptography``        ``>=41.0.0``
 ``kubernetes``          ``>=29.0.0,<=31.0.0``
@@ -82,4 +82,4 @@ PIP package             Version required
 ======================  =====================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.0.1/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.1.0/changelog.html>`_.

{apache_airflow_providers_cncf_kubernetes-10.0.1rc1 → apache_airflow_providers_cncf_kubernetes-10.1.0rc1}/airflow/providers/cncf/kubernetes/__init__.py

@@ -29,11 +29,11 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "10.0.1"
+__version__ = "10.1.0"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
-    "2.8.0"
+    "2.9.0"
 ):
     raise RuntimeError(
-        f"The package `apache-airflow-providers-cncf-kubernetes:{__version__}` needs Apache Airflow 2.8.0+"
+        f"The package `apache-airflow-providers-cncf-kubernetes:{__version__}` needs Apache Airflow 2.9.0+"
     )

{apache_airflow_providers_cncf_kubernetes-10.0.1rc1 → apache_airflow_providers_cncf_kubernetes-10.1.0rc1}/airflow/providers/cncf/kubernetes/cli/kubernetes_command.py

@@ -25,22 +25,18 @@ from datetime import datetime, timedelta
 from kubernetes import client
 from kubernetes.client.api_client import ApiClient
 from kubernetes.client.rest import ApiException
-from packaging.version import Version
 
-from airflow import __version__ as airflow_version
 from airflow.models import DagRun, TaskInstance
 from airflow.providers.cncf.kubernetes import pod_generator
 from airflow.providers.cncf.kubernetes.executors.kubernetes_executor import KubeConfig
 from airflow.providers.cncf.kubernetes.kube_client import get_kube_client
 from airflow.providers.cncf.kubernetes.kubernetes_helper_functions import create_unique_id
 from airflow.providers.cncf.kubernetes.pod_generator import PodGenerator
+from airflow.providers.cncf.kubernetes.version_compat import AIRFLOW_V_3_0_PLUS
 from airflow.utils import cli as cli_utils, yaml
 from airflow.utils.cli import get_dag
 from airflow.utils.providers_configuration_loader import providers_configuration_loaded
 
-AIRFLOW_VERSION = Version(airflow_version)
-AIRFLOW_V_3_0_PLUS = Version(AIRFLOW_VERSION.base_version) >= Version("3.0.0")
-
 
 @cli_utils.action_cli
 @providers_configuration_loaded

{apache_airflow_providers_cncf_kubernetes-10.0.1rc1 → apache_airflow_providers_cncf_kubernetes-10.1.0rc1}/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py

@@ -239,11 +239,12 @@ class KubernetesExecutor(BaseExecutor):
         from airflow.models.taskinstance import TaskInstance
 
         hybrid_executor_enabled = hasattr(TaskInstance, "executor")
-        default_executor = None
+        default_executor_alias = None
         if hybrid_executor_enabled:
             from airflow.executors.executor_loader import ExecutorLoader
 
-            default_executor = str(ExecutorLoader.get_default_executor_name())
+            default_executor_name = ExecutorLoader.get_default_executor_name()
+            default_executor_alias = default_executor_name.alias
 
         with Stats.timer("kubernetes_executor.clear_not_launched_queued_tasks.duration"):
             self.log.debug("Clearing tasks that have not been launched")
@@ -253,7 +254,10 @@ class KubernetesExecutor(BaseExecutor):
             )
             if self.kubernetes_queue:
                 query = query.where(TaskInstance.queue == self.kubernetes_queue)
-            elif hybrid_executor_enabled and KUBERNETES_EXECUTOR == default_executor:
+            # KUBERNETES_EXECUTOR is the string name/alias of the "core" executor represented by this
+            # module. The ExecutorName for "core" executors always contains an alias and cannot be modified
+            # to be different from the constant (in this case KUBERNETES_EXECUTOR).
+            elif hybrid_executor_enabled and default_executor_alias == KUBERNETES_EXECUTOR:
                 query = query.where(
                     or_(
                         TaskInstance.executor == KUBERNETES_EXECUTOR,

{apache_airflow_providers_cncf_kubernetes-10.0.1rc1 → apache_airflow_providers_cncf_kubernetes-10.1.0rc1}/airflow/providers/cncf/kubernetes/get_provider_info.py

@@ -28,8 +28,9 @@ def get_provider_info():
         "name": "Kubernetes",
         "description": "`Kubernetes <https://kubernetes.io/>`__\n",
         "state": "ready",
-        "source-date-epoch": 1732429220,
+        "source-date-epoch": 1734537609,
         "versions": [
+            "10.1.0",
             "10.0.1",
             "10.0.0",
             "9.0.1",
@@ -101,7 +102,7 @@ def get_provider_info():
         ],
         "dependencies": [
             "aiofiles>=23.2.0",
-            "apache-airflow>=2.8.0",
+            "apache-airflow>=2.9.0",
             "asgiref>=3.5.2",
             "cryptography>=41.0.0",
             "kubernetes>=29.0.0,<=31.0.0",
@@ -128,6 +129,7 @@ def get_provider_info():
                 "integration-name": "Kubernetes",
                 "python-modules": [
                     "airflow.providers.cncf.kubernetes.operators.custom_object_launcher",
+                    "airflow.providers.cncf.kubernetes.operators.kueue",
                     "airflow.providers.cncf.kubernetes.operators.pod",
                     "airflow.providers.cncf.kubernetes.operators.spark_kubernetes",
                     "airflow.providers.cncf.kubernetes.operators.resource",

{apache_airflow_providers_cncf_kubernetes-10.0.1rc1 → apache_airflow_providers_cncf_kubernetes-10.1.0rc1}/airflow/providers/cncf/kubernetes/hooks/kubernetes.py

@@ -26,9 +26,11 @@ from time import sleep
 from typing import TYPE_CHECKING, Any
 
 import aiofiles
+import requests
 import tenacity
 from asgiref.sync import sync_to_async
-from kubernetes import client, config, watch
+from kubernetes import client, config, utils, watch
+from kubernetes.client.models import V1Deployment
 from kubernetes.config import ConfigException
 from kubernetes_asyncio import client as async_client, config as async_config
 from urllib3.exceptions import HTTPError
@@ -47,7 +49,7 @@ from airflow.utils import yaml
 
 if TYPE_CHECKING:
     from kubernetes.client import V1JobList
-    from kubernetes.client.models import V1Deployment, V1Job, V1Pod
+    from kubernetes.client.models import V1Job, V1Pod
 
 LOADING_KUBE_CONFIG_FILE_RESOURCE = "Loading Kubernetes configuration file kube_config from {}..."
 
@@ -489,12 +491,9 @@ class KubernetesHook(BaseHook, PodOperatorHookProtocol):
         :param name: Name of Deployment to retrieve
         :param namespace: Deployment namespace
         """
-        try:
-            return self.apps_v1_client.read_namespaced_deployment_status(
-                name=name, namespace=namespace, pretty=True, **kwargs
-            )
-        except Exception as exc:
-            raise exc
+        return self.apps_v1_client.read_namespaced_deployment_status(
+            name=name, namespace=namespace, pretty=True, **kwargs
+        )
 
     @tenacity.retry(
         stop=tenacity.stop_after_attempt(3),
@@ -644,6 +643,71 @@ class KubernetesHook(BaseHook, PodOperatorHookProtocol):
             body=body,
         )
 
+    def apply_from_yaml_file(
+        self,
+        api_client: Any = None,
+        yaml_file: str | None = None,
+        yaml_objects: list[dict] | None = None,
+        verbose: bool = False,
+        namespace: str = "default",
+    ):
+        """
+        Perform an action from a yaml file.
+
+        :param api_client: A Kubernetes client application.
+        :param yaml_file: Contains the path to yaml file.
+        :param yaml_objects: List of YAML objects; used instead of reading the yaml_file.
+        :param verbose: If True, print confirmation from create action. Default is False.
+        :param namespace: Contains the namespace to create all resources inside. The namespace must
+            preexist otherwise the resource creation will fail.
+        """
+        utils.create_from_yaml(
+            k8s_client=api_client or self.api_client,
+            yaml_objects=yaml_objects,
+            yaml_file=yaml_file,
+            verbose=verbose,
+            namespace=namespace or self.get_namespace(),
+        )
+
+    def check_kueue_deployment_running(
+        self, name: str, namespace: str, timeout: float = 300.0, polling_period_seconds: float = 2.0
+    ) -> None:
+        _timeout = timeout
+        while _timeout > 0:
+            try:
+                deployment = self.get_deployment_status(name=name, namespace=namespace)
+            except Exception as e:
+                self.log.exception("Exception occurred while checking for Deployment status.")
+                raise e
+
+            deployment_status = V1Deployment.to_dict(deployment)["status"]
+            replicas = deployment_status["replicas"]
+            ready_replicas = deployment_status["ready_replicas"]
+            unavailable_replicas = deployment_status["unavailable_replicas"]
+            if (
+                replicas is not None
+                and ready_replicas is not None
+                and unavailable_replicas is None
+                and replicas == ready_replicas
+            ):
+                return
+            else:
+                self.log.info("Waiting until Deployment will be ready...")
+                sleep(polling_period_seconds)
+
+            _timeout -= polling_period_seconds
+
+        raise AirflowException("Deployment timed out")
+
+    @staticmethod
+    def get_yaml_content_from_file(kueue_yaml_url) -> list[dict]:
+        """Download content of YAML file and separate it into several dictionaries."""
+        response = requests.get(kueue_yaml_url, allow_redirects=True)
+        if response.status_code != 200:
+            raise AirflowException("Was not able to read the yaml file from given URL")
+
+        return list(yaml.safe_load_all(response.text))
+
 
 def _get_bool(val) -> bool | None:
     """Convert val to bool if can be done with certainty; if we cannot infer intention we return None."""

apache_airflow_providers_cncf_kubernetes-10.1.0rc1/airflow/providers/cncf/kubernetes/operators/kueue.py

@@ -0,0 +1,111 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Manage a Kubernetes Kueue."""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Sequence
+from functools import cached_property
+
+from kubernetes.utils import FailToCreateError
+
+from airflow.exceptions import AirflowException
+from airflow.models import BaseOperator
+from airflow.providers.cncf.kubernetes.hooks.kubernetes import KubernetesHook
+from airflow.providers.cncf.kubernetes.operators.job import KubernetesJobOperator
+
+
+class KubernetesInstallKueueOperator(BaseOperator):
+    """
+    Installs a Kubernetes Kueue.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:KubernetesInstallKueueOperator`
+
+    :param kueue_version: The Kubernetes Kueue version to install.
+    :param kubernetes_conn_id: The :ref:`kubernetes connection id <howto/connection:kubernetes>`
+        for the Kubernetes cluster.
+    """
+
+    template_fields: Sequence[str] = (
+        "kueue_version",
+        "kubernetes_conn_id",
+    )
+
+    def __init__(self, kueue_version: str, kubernetes_conn_id: str = "kubernetes_default", *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.kubernetes_conn_id = kubernetes_conn_id
+        self.kueue_version = kueue_version
+        self._kueue_yaml_url = (
+            f"https://github.com/kubernetes-sigs/kueue/releases/download/{self.kueue_version}/manifests.yaml"
+        )
+
+    @cached_property
+    def hook(self) -> KubernetesHook:
+        return KubernetesHook(conn_id=self.kubernetes_conn_id)
+
+    def execute(self, context):
+        yaml_objects = self.hook.get_yaml_content_from_file(kueue_yaml_url=self._kueue_yaml_url)
+        try:
+            self.hook.apply_from_yaml_file(yaml_objects=yaml_objects)
+        except FailToCreateError as ex:
+            error_bodies = [json.loads(e.body) for e in ex.api_exceptions]
+            if next((e for e in error_bodies if e.get("reason") == "AlreadyExists"), None):
+                self.log.info("Kueue is already enabled for the cluster")
+
+            if errors := [e for e in error_bodies if e.get("reason") != "AlreadyExists"]:
+                error_message = "\n".join(e.get("body") for e in errors)
+                raise AirflowException(error_message)
+            return
+
+        self.hook.check_kueue_deployment_running(name="kueue-controller-manager", namespace="kueue-system")
+        self.log.info("Kueue installed successfully!")
+
+
+class KubernetesStartKueueJobOperator(KubernetesJobOperator):
+    """
+    Executes a Kubernetes Job in Kueue.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:KubernetesStartKueueJobOperator`
+
+    :param queue_name: The name of the Queue in the cluster
+    """
+
+    template_fields = tuple({"queue_name"} | set(KubernetesJobOperator.template_fields))
+
+    def __init__(self, queue_name: str, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.queue_name = queue_name
+
+        if self.suspend is False:
+            raise AirflowException(
+                "The `suspend` parameter can't be False. If you want to use Kueue for running Job"
+                " in a Kubernetes cluster, set the `suspend` parameter to True.",
+            )
+        elif self.suspend is None:
+            self.log.info(
+                "You have not set parameter `suspend` in class %s. "
+                "For running a Job in Kueue the `suspend` parameter has been set to True.",
+                self.__class__.__name__,
+            )
+            self.suspend = True
+        self.labels.update({"kueue.x-k8s.io/queue-name": self.queue_name})
+        self.annotations.update({"kueue.x-k8s.io/queue-name": self.queue_name})

{apache_airflow_providers_cncf_kubernetes-10.0.1rc1 → apache_airflow_providers_cncf_kubernetes-10.1.0rc1}/airflow/providers/cncf/kubernetes/operators/pod.py

@@ -155,6 +155,9 @@ class KubernetesPodOperator(BaseOperator):
     :param startup_timeout_seconds: timeout in seconds to startup the pod.
     :param startup_check_interval_seconds: interval in seconds to check if the pod has already started
     :param get_logs: get the stdout of the base container as logs of the tasks.
+    :param init_container_logs: list of init containers whose logs will be published to stdout
+        Takes a sequence of containers, a single container name or True. If True,
+        all the containers logs are published.
     :param container_logs: list of containers whose logs will be published to stdout
         Takes a sequence of containers, a single container name or True. If True,
         all the containers logs are published. Works in conjunction with get_logs param.
@@ -278,6 +281,7 @@ class KubernetesPodOperator(BaseOperator):
         startup_check_interval_seconds: int = 5,
         get_logs: bool = True,
         base_container_name: str | None = None,
+        init_container_logs: Iterable[str] | str | Literal[True] | None = None,
         container_logs: Iterable[str] | str | Literal[True] | None = None,
         image_pull_policy: str | None = None,
         annotations: dict | None = None,
@@ -352,6 +356,7 @@ class KubernetesPodOperator(BaseOperator):
         # Fallback to the class variable BASE_CONTAINER_NAME here instead of via default argument value
         # in the init method signature, to be compatible with subclasses overloading the class variable value.
         self.base_container_name = base_container_name or self.BASE_CONTAINER_NAME
+        self.init_container_logs = init_container_logs
         self.container_logs = container_logs or self.base_container_name
         self.image_pull_policy = image_pull_policy
         self.node_selector = node_selector or {}
@@ -396,7 +401,9 @@ class KubernetesPodOperator(BaseOperator):
         self.remote_pod: k8s.V1Pod | None = None
         self.log_pod_spec_on_failure = log_pod_spec_on_failure
         self.on_finish_action = OnFinishAction(on_finish_action)
-        self.is_delete_operator_pod = self.on_finish_action == OnFinishAction.DELETE_POD
+        # The `is_delete_operator_pod` parameter should have been removed in provider version 10.0.0.
+        # TODO: remove it from here and from the operator's parameters list when the next major version bumped
+        self._is_delete_operator_pod = self.on_finish_action == OnFinishAction.DELETE_POD
         self.termination_message_policy = termination_message_policy
         self.active_deadline_seconds = active_deadline_seconds
         self.logging_interval = logging_interval
@@ -598,6 +605,9 @@ class KubernetesPodOperator(BaseOperator):
                 self.callbacks.on_pod_creation(
                     pod=self.remote_pod, client=self.client, mode=ExecutionMode.SYNC
                 )
+
+            self.await_init_containers_completion(pod=self.pod)
+
             self.await_pod_start(pod=self.pod)
             if self.callbacks:
                 self.callbacks.on_pod_starting(
@@ -633,6 +643,22 @@ class KubernetesPodOperator(BaseOperator):
         if self.do_xcom_push:
             return result
 
+    @tenacity.retry(
+        wait=tenacity.wait_exponential(max=15),
+        retry=tenacity.retry_if_exception_type(PodCredentialsExpiredFailure),
+        reraise=True,
+    )
+    def await_init_containers_completion(self, pod: k8s.V1Pod):
+        try:
+            if self.init_container_logs:
+                self.pod_manager.fetch_requested_init_container_logs(
+                    pod=pod,
+                    init_containers=self.init_container_logs,
+                    follow_logs=True,
+                )
+        except kubernetes.client.exceptions.ApiException as exc:
+            self._handle_api_exception(exc, pod)
+
     @tenacity.retry(
         wait=tenacity.wait_exponential(max=15),
         retry=tenacity.retry_if_exception_type(PodCredentialsExpiredFailure),
@@ -651,16 +677,21 @@ class KubernetesPodOperator(BaseOperator):
             ):
                 self.pod_manager.await_container_completion(pod=pod, container_name=self.base_container_name)
         except kubernetes.client.exceptions.ApiException as exc:
-            if exc.status and str(exc.status) == "401":
-                self.log.warning(
-                    "Failed to check container status due to permission error. Refreshing credentials and retrying."
-                )
-                self._refresh_cached_properties()
-                self.pod_manager.read_pod(
-                    pod=pod
-                )  # attempt using refreshed credentials, raises if still invalid
-                raise PodCredentialsExpiredFailure("Kubernetes credentials expired, retrying after refresh.")
-            raise exc
+            self._handle_api_exception(exc, pod)
+
+    def _handle_api_exception(
+        self,
+        exc: kubernetes.client.exceptions.ApiException,
+        pod: k8s.V1Pod,
+    ):
+        if exc.status and str(exc.status) == "401":
+            self.log.warning(
+                "Failed to check container status due to permission error. Refreshing credentials and retrying."
+            )
+            self._refresh_cached_properties()
+            self.pod_manager.read_pod(pod=pod)  # attempt using refreshed credentials, raises if still invalid
+            raise PodCredentialsExpiredFailure("Kubernetes credentials expired, retrying after refresh.")
+        raise exc
 
     def _refresh_cached_properties(self):
         del self.hook

{apache_airflow_providers_cncf_kubernetes-10.0.1rc1 → apache_airflow_providers_cncf_kubernetes-10.1.0rc1}/airflow/providers/cncf/kubernetes/utils/pod_manager.py

@@ -19,6 +19,7 @@
 from __future__ import annotations
 
 import enum
+import itertools
 import json
 import math
 import time
@@ -117,7 +118,13 @@ class PodOperatorHookProtocol(Protocol):
 
 def get_container_status(pod: V1Pod, container_name: str) -> V1ContainerStatus | None:
     """Retrieve container status."""
-    container_statuses = pod.status.container_statuses if pod and pod.status else None
+    if pod and pod.status:
+        container_statuses = itertools.chain(
+            pod.status.container_statuses, pod.status.init_container_statuses
+        )
+    else:
+        container_statuses = None
+
     if container_statuses:
         # In general the variable container_statuses can store multiple items matching different containers.
         # The following generator expression yields all items that have name equal to the container_name.
@@ -166,6 +173,19 @@ def container_is_succeeded(pod: V1Pod, container_name: str) -> bool:
     return container_status.state.terminated.exit_code == 0
 
 
+def container_is_wait(pod: V1Pod, container_name: str) -> bool:
+    """
+    Examine V1Pod ``pod`` to determine whether ``container_name`` is waiting.
+
+    If that container is present and waiting, returns True.  Returns False otherwise.
+    """
+    container_status = get_container_status(pod, container_name)
+    if not container_status:
+        return False
+
+    return container_status.state.waiting is not None
+
+
 def container_is_terminated(pod: V1Pod, container_name: str) -> bool:
     """
     Examine V1Pod ``pod`` to determine whether ``container_name`` is terminated.
@@ -451,7 +471,10 @@ class PodManager(LoggingMixin):
                                             line=line, client=self._client, mode=ExecutionMode.SYNC
                                         )
                                 if message_to_log is not None:
-                                    self.log.info("[%s] %s", container_name, message_to_log)
+                                    if is_log_group_marker(message_to_log):
+                                        print(message_to_log)
+                                    else:
+                                        self.log.info("[%s] %s", container_name, message_to_log)
                                 last_captured_timestamp = message_timestamp
                                 message_to_log = message
                                 message_timestamp = line_timestamp
@@ -467,7 +490,10 @@ class PodManager(LoggingMixin):
                                 line=line, client=self._client, mode=ExecutionMode.SYNC
                             )
                     if message_to_log is not None:
-                        self.log.info("[%s] %s", container_name, message_to_log)
+                        if is_log_group_marker(message_to_log):
+                            print(message_to_log)
+                        else:
+                            self.log.info("[%s] %s", container_name, message_to_log)
                     last_captured_timestamp = message_timestamp
             except TimeoutError as e:
                 # in case of timeout, increment return time by 2 seconds to avoid
@@ -503,7 +529,7 @@ class PodManager(LoggingMixin):
                 time.sleep(1)
 
     def _reconcile_requested_log_containers(
-        self, requested: Iterable[str] | str | bool, actual: list[str], pod_name
+        self, requested: Iterable[str] | str | bool | None, actual: list[str], pod_name
     ) -> list[str]:
         """Return actual containers based on requested."""
         containers_to_log = []
@@ -546,6 +572,31 @@ class PodManager(LoggingMixin):
             self.log.error("Could not retrieve containers for the pod: %s", pod_name)
         return containers_to_log
 
+    def fetch_requested_init_container_logs(
+        self, pod: V1Pod, init_containers: Iterable[str] | str | Literal[True] | None, follow_logs=False
+    ) -> list[PodLoggingStatus]:
+        """
+        Follow the logs of containers in the specified pod and publish it to airflow logging.
+
+        Returns when all the containers exit.
+
+        :meta private:
+        """
+        pod_logging_statuses = []
+        all_containers = self.get_init_container_names(pod)
+        containers_to_log = self._reconcile_requested_log_containers(
+            requested=init_containers,
+            actual=all_containers,
+            pod_name=pod.metadata.name,
+        )
+        # sort by spec.initContainers because containers runs sequentially
+        containers_to_log = sorted(containers_to_log, key=lambda cn: all_containers.index(cn))
+        for c in containers_to_log:
+            self._await_init_container_start(pod=pod, container_name=c)
+            status = self.fetch_container_logs(pod=pod, container_name=c, follow=follow_logs)
+            pod_logging_statuses.append(status)
+        return pod_logging_statuses
+
     def fetch_requested_container_logs(
         self, pod: V1Pod, containers: Iterable[str] | str | Literal[True], follow_logs=False
     ) -> list[PodLoggingStatus]:
@@ -673,9 +724,22 @@ class PodManager(LoggingMixin):
             post_termination_timeout=post_termination_timeout,
         )
 
+    @tenacity.retry(stop=tenacity.stop_after_attempt(3), wait=tenacity.wait_exponential(), reraise=True)
+    def get_init_container_names(self, pod: V1Pod) -> list[str]:
+        """
+        Return container names from the POD except for the airflow-xcom-sidecar container.
+
+        :meta private:
+        """
+        return [container_spec.name for container_spec in pod.spec.init_containers]
+
     @tenacity.retry(stop=tenacity.stop_after_attempt(3), wait=tenacity.wait_exponential(), reraise=True)
     def get_container_names(self, pod: V1Pod) -> list[str]:
-        """Return container names from the POD except for the airflow-xcom-sidecar container."""
+        """
+        Return container names from the POD except for the airflow-xcom-sidecar container.
+
+        :meta private:
+        """
         pod_info = self.read_pod(pod)
         return [
             container_spec.name
@@ -791,7 +855,7 @@ class PodManager(LoggingMixin):
                 _preload_content=False,
             )
         ) as resp:
-            self._exec_pod_command(resp, "kill -2 1")
+            self._exec_pod_command(resp, "kill -2 $(pgrep -u $(whoami) -f trap)")
 
     def _exec_pod_command(self, resp, command: str) -> str | None:
         res = ""
@@ -813,6 +877,20 @@ class PodManager(LoggingMixin):
                 return res
         return None
 
+    def _await_init_container_start(self, pod: V1Pod, container_name: str):
+        while True:
+            remote_pod = self.read_pod(pod)
+
+            if (
+                remote_pod.status is not None
+                and remote_pod.status.phase != PodPhase.PENDING
+                and get_container_status(remote_pod, container_name) is not None
+                and not container_is_wait(remote_pod, container_name)
+            ):
+                return
+
+            time.sleep(1)
+
 
 class OnFinishAction(str, enum.Enum):
     """Action to take when the pod finishes."""
@@ -820,3 +898,8 @@ class OnFinishAction(str, enum.Enum):
     KEEP_POD = "keep_pod"
     DELETE_POD = "delete_pod"
     DELETE_SUCCEEDED_POD = "delete_succeeded_pod"
+
+
+def is_log_group_marker(line: str) -> bool:
+    """Check if the line is a log group marker like `::group::` or `::endgroup::`."""
+    return line.startswith("::group::") or line.startswith("::endgroup::")

apache_airflow_providers_cncf_kubernetes-10.1.0rc1/airflow/providers/cncf/kubernetes/version_compat.py

@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# NOTE! THIS FILE IS COPIED MANUALLY IN OTHER PROVIDERS DELIBERATELY TO AVOID ADDING UNNECESSARY
+# DEPENDENCIES BETWEEN PROVIDERS. IF YOU WANT TO ADD CONDITIONAL CODE IN YOUR PROVIDER THAT DEPENDS
+# ON AIRFLOW VERSION, PLEASE COPY THIS FILE TO THE ROOT PACKAGE OF YOUR PROVIDER AND IMPORT
+# THOSE CONSTANTS FROM IT RATHER THAN IMPORTING THEM FROM ANOTHER PROVIDER OR TEST CODE
+#
+from __future__ import annotations
+
+
+def get_base_airflow_version_tuple() -> tuple[int, int, int]:
+    from packaging.version import Version
+
+    from airflow import __version__
+
+    airflow_version = Version(__version__)
+    return airflow_version.major, airflow_version.minor, airflow_version.micro
+
+
+AIRFLOW_V_2_10_PLUS = get_base_airflow_version_tuple() >= (2, 10, 0)
+AIRFLOW_V_3_0_PLUS = get_base_airflow_version_tuple() >= (3, 0, 0)

{apache_airflow_providers_cncf_kubernetes-10.0.1rc1 → apache_airflow_providers_cncf_kubernetes-10.1.0rc1}/pyproject.toml

@@ -27,7 +27,7 @@ build-backend = "flit_core.buildapi"
 
 [project]
 name = "apache-airflow-providers-cncf-kubernetes"
-version = "10.0.1.rc1"
+version = "10.1.0.rc1"
 description = "Provider package apache-airflow-providers-cncf-kubernetes for Apache Airflow"
 readme = "README.rst"
 authors = [
@@ -55,7 +55,7 @@ classifiers = [
 requires-python = "~=3.9"
 dependencies = [
     "aiofiles>=23.2.0",
-    "apache-airflow>=2.8.0rc0",
+    "apache-airflow>=2.9.0rc0",
     "asgiref>=3.5.2",
     "cryptography>=41.0.0",
     "google-re2>=1.0",
@@ -64,12 +64,12 @@ dependencies = [
 ]
 
 [project.urls]
-"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.0.1"
-"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.0.1/changelog.html"
+"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.1.0"
+"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.1.0/changelog.html"
 "Bug Tracker" = "https://github.com/apache/airflow/issues"
 "Source Code" = "https://github.com/apache/airflow"
 "Slack Chat" = "https://s.apache.org/airflow-slack"
-"Twitter" = "https://twitter.com/ApacheAirflow"
+"Twitter" = "https://x.com/ApacheAirflow"
 "YouTube" = "https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/"
 
 [project.entry-points."apache_airflow_provider"]