apache-airflow-providers-cncf-kubernetes 10.9.0rc1__py3-none-any.whl → 10.11.0rc2__py3-none-any.whl

airflow/providers/cncf/kubernetes/__init__.py

@@ -29,11 +29,11 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "10.9.0"
+__version__ = "10.11.0"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
-    "2.10.0"
+    "2.11.0"
 ):
     raise RuntimeError(
-        f"The package `apache-airflow-providers-cncf-kubernetes:{__version__}` needs Apache Airflow 2.10.0+"
+        f"The package `apache-airflow-providers-cncf-kubernetes:{__version__}` needs Apache Airflow 2.11.0+"
     )

airflow/providers/cncf/kubernetes/exceptions.py

@@ -16,9 +16,7 @@
 # under the License.
 from __future__ import annotations
 
-from airflow.exceptions import (
-    AirflowException,
-)
+from airflow.exceptions import AirflowException
 
 
 class PodMutationHookException(AirflowException):
@@ -27,3 +25,11 @@ class PodMutationHookException(AirflowException):
 
 class PodReconciliationError(AirflowException):
     """Raised when an error is encountered while trying to merge pod configs."""
+
+
+class KubernetesApiError(AirflowException):
+    """Raised when an error is encountered while trying access Kubernetes API."""
+
+
+class KubernetesApiPermissionError(AirflowException):
+    """Raised when an error is encountered while trying access Kubernetes API."""

airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py

@@ -165,6 +165,7 @@ class KubernetesExecutor(BaseExecutor):
         self.task_publish_max_retries = conf.getint(
             "kubernetes_executor", "task_publish_max_retries", fallback=0
         )
+        self.completed: set[KubernetesResults] = set()
         super().__init__(parallelism=self.kube_config.parallelism)
 
     def _list_pods(self, query_kwargs):
@@ -343,6 +344,9 @@ class KubernetesExecutor(BaseExecutor):
                 finally:
                     self.result_queue.task_done()
 
+                for result in self.completed:
+                    self._change_state(result)
+
         from airflow.providers.cncf.kubernetes.executors.kubernetes_executor_utils import ResourceVersion
 
         resource_instance = ResourceVersion()
@@ -385,6 +389,7 @@ class KubernetesExecutor(BaseExecutor):
                     if (
                         (str(e.status) == "403" and "exceeded quota" in message)
                         or (str(e.status) == "409" and "object has been modified" in message)
+                        or str(e.status) == "500"
                     ) and (self.task_publish_max_retries == -1 or retries < self.task_publish_max_retries):
                         self.log.warning(
                             "[Try %s of %s] Kube ApiException for Task: (%s). Reason: %r. Message: %s",
@@ -501,7 +506,11 @@ class KubernetesExecutor(BaseExecutor):
         if state is None:
             from airflow.models.taskinstance import TaskInstance
 
-            state = session.scalar(select(TaskInstance.state).where(TaskInstance.filter_for_tis([key])))
+            filter_for_tis = TaskInstance.filter_for_tis([key])
+            if filter_for_tis is not None:
+                state = session.scalar(select(TaskInstance.state).where(filter_for_tis))
+            else:
+                state = None
             state = TaskInstanceState(state) if state else None
 
         self.event_buffer[key] = state, None
@@ -511,7 +520,8 @@ class KubernetesExecutor(BaseExecutor):
         pod_override = ti.executor_config.get("pod_override")
         namespace = None
         with suppress(Exception):
-            namespace = pod_override.metadata.namespace
+            if pod_override is not None:
+                namespace = pod_override.metadata.namespace
         return namespace or conf.get("kubernetes_executor", "namespace")
 
     def get_task_log(self, ti: TaskInstance, try_number: int) -> tuple[list[str], list[str]]:
@@ -565,7 +575,7 @@ class KubernetesExecutor(BaseExecutor):
             tis_to_flush_by_key = {ti.key: ti for ti in tis if ti.queued_by_job_id}
             kube_client: client.CoreV1Api = self.kube_client
             for scheduler_job_id in scheduler_job_ids:
-                scheduler_job_id = self._make_safe_label_value(str(scheduler_job_id))
+                scheduler_job_id_safe_label = self._make_safe_label_value(str(scheduler_job_id))
                 # We will look for any pods owned by the no-longer-running scheduler,
                 # but will exclude only successful pods, as those TIs will have a terminal state
                 # and not be up for adoption!
@@ -575,7 +585,7 @@ class KubernetesExecutor(BaseExecutor):
                     "field_selector": "status.phase!=Succeeded",
                     "label_selector": (
                         "kubernetes_executor=True,"
-                        f"airflow-worker={scheduler_job_id},{POD_EXECUTOR_DONE_KEY}!=True"
+                        f"airflow-worker={scheduler_job_id_safe_label},{POD_EXECUTOR_DONE_KEY}!=True"
                     ),
                 }
                 pod_list = self._list_pods(query_kwargs)
@@ -720,7 +730,16 @@ class KubernetesExecutor(BaseExecutor):
                 continue
 
             ti_id = annotations_to_key(pod.metadata.annotations)
-            self.running.add(ti_id)
+            self.completed.add(
+                KubernetesResults(
+                    key=ti_id,
+                    state="completed",
+                    pod_name=pod.metadata.name,
+                    namespace=pod.metadata.namespace,
+                    resource_version=pod.metadata.resource_version,
+                    failure_details=None,
+                )
+            )
 
     def _flush_task_queue(self) -> None:
         if TYPE_CHECKING:

airflow/providers/cncf/kubernetes/get_provider_info.py

@@ -135,6 +135,8 @@ def get_provider_info():
                         "type": "string",
                         "example": None,
                         "default": "",
+                        "deprecated": True,
+                        "deprecation_reason": "This configuration is deprecated. Use `pod_template_file` to specify container image instead.\n",
                     },
                     "worker_container_tag": {
                         "description": "The tag of the Kubernetes Image for the Worker to Run\n",
@@ -142,6 +144,8 @@ def get_provider_info():
                         "type": "string",
                         "example": None,
                         "default": "",
+                        "deprecated": True,
+                        "deprecation_reason": "This configuration is deprecated. Use `pod_template_file` to specify the image tag instead.\n",
                     },
                     "namespace": {
                         "description": "The Kubernetes namespace where airflow workers should be created. Defaults to ``default``\n",
@@ -149,6 +153,8 @@ def get_provider_info():
                         "type": "string",
                         "example": None,
                         "default": "default",
+                        "deprecated": True,
+                        "deprecation_reason": "This configuration is deprecated. Use `pod_template_file` to specify namespace instead.\n",
                     },
                     "delete_worker_pods": {
                         "description": "If True, all worker pods will be deleted upon termination\n",

airflow/providers/cncf/kubernetes/hooks/kubernetes.py

@@ -27,7 +27,6 @@ from typing import TYPE_CHECKING, Any, Protocol
 
 import aiofiles
 import requests
-import tenacity
 from asgiref.sync import sync_to_async
 from kubernetes import client, config, utils, watch
 from kubernetes.client.models import V1Deployment
@@ -37,8 +36,9 @@ from urllib3.exceptions import HTTPError
 
 from airflow.exceptions import AirflowException, AirflowNotFoundException
 from airflow.models import Connection
+from airflow.providers.cncf.kubernetes.exceptions import KubernetesApiError, KubernetesApiPermissionError
 from airflow.providers.cncf.kubernetes.kube_client import _disable_verify_ssl, _enable_tcp_keepalive
-from airflow.providers.cncf.kubernetes.kubernetes_helper_functions import should_retry_creation
+from airflow.providers.cncf.kubernetes.kubernetes_helper_functions import generic_api_retry
 from airflow.providers.cncf.kubernetes.utils.container import (
     container_is_completed,
     container_is_running,
@@ -48,7 +48,7 @@ from airflow.utils import yaml
 
 if TYPE_CHECKING:
     from kubernetes.client import V1JobList
-    from kubernetes.client.models import V1Job, V1Pod
+    from kubernetes.client.models import CoreV1EventList, V1Job, V1Pod
 
 LOADING_KUBE_CONFIG_FILE_RESOURCE = "Loading Kubernetes configuration file kube_config from {}..."
 
@@ -389,6 +389,7 @@ class KubernetesHook(BaseHook, PodOperatorHookProtocol):
         self.log.debug("Response: %s", response)
         return response
 
+    @generic_api_retry
     def get_custom_object(
         self, group: str, version: str, plural: str, name: str, namespace: str | None = None
     ):
@@ -411,6 +412,7 @@ class KubernetesHook(BaseHook, PodOperatorHookProtocol):
         )
         return response
 
+    @generic_api_retry
     def delete_custom_object(
         self, group: str, version: str, plural: str, name: str, namespace: str | None = None, **kwargs
     ):
@@ -539,12 +541,7 @@ class KubernetesHook(BaseHook, PodOperatorHookProtocol):
             name=name, namespace=namespace, pretty=True, **kwargs
         )
 
-    @tenacity.retry(
-        stop=tenacity.stop_after_attempt(3),
-        wait=tenacity.wait_random_exponential(),
-        reraise=True,
-        retry=tenacity.retry_if_exception(should_retry_creation),
-    )
+    @generic_api_retry
     def create_job(
         self,
         job: V1Job,
@@ -571,6 +568,7 @@ class KubernetesHook(BaseHook, PodOperatorHookProtocol):
             raise e
         return resp
 
+    @generic_api_retry
     def get_job(self, job_name: str, namespace: str) -> V1Job:
         """
         Get Job of specified name and namespace.
@@ -581,6 +579,7 @@ class KubernetesHook(BaseHook, PodOperatorHookProtocol):
         """
         return self.batch_v1_client.read_namespaced_job(name=job_name, namespace=namespace, pretty=True)
 
+    @generic_api_retry
     def get_job_status(self, job_name: str, namespace: str) -> V1Job:
         """
         Get job with status of specified name and namespace.
@@ -610,6 +609,7 @@ class KubernetesHook(BaseHook, PodOperatorHookProtocol):
             self.log.info("The job '%s' is incomplete. Sleeping for %i sec.", job_name, job_poll_interval)
             sleep(job_poll_interval)
 
+    @generic_api_retry
     def list_jobs_all_namespaces(self) -> V1JobList:
         """
         Get list of Jobs from all namespaces.
@@ -618,6 +618,7 @@ class KubernetesHook(BaseHook, PodOperatorHookProtocol):
         """
         return self.batch_v1_client.list_job_for_all_namespaces(pretty=True)
 
+    @generic_api_retry
     def list_jobs_from_namespace(self, namespace: str) -> V1JobList:
         """
         Get list of Jobs from dedicated namespace.
@@ -673,6 +674,7 @@ class KubernetesHook(BaseHook, PodOperatorHookProtocol):
             return bool(next((c for c in conditions if c.type == "Complete" and c.status), None))
         return False
 
+    @generic_api_retry
     def patch_namespaced_job(self, job_name: str, namespace: str, body: object) -> V1Job:
         """
         Update the specified Job.
@@ -830,6 +832,13 @@ class AsyncKubernetesHook(KubernetesHook):
                     "Reading kubernetes configuration file from connection "
                     "object and writing temporary config file with its content",
                 )
+                if isinstance(kubeconfig, dict):
+                    self.log.debug(
+                        LOADING_KUBE_CONFIG_FILE_RESOURCE.format(
+                            "connection kube_config dictionary (serializing)"
+                        )
+                    )
+                    kubeconfig = json.dumps(kubeconfig)
                 await temp_config.write(kubeconfig.encode())
                 await temp_config.flush()
                 self._is_in_cluster = False
@@ -871,6 +880,7 @@ class AsyncKubernetesHook(KubernetesHook):
             if kube_client is not None:
                 await kube_client.close()
 
+    @generic_api_retry
     async def get_pod(self, name: str, namespace: str) -> V1Pod:
         """
         Get pod's object.
@@ -879,13 +889,19 @@ class AsyncKubernetesHook(KubernetesHook):
         :param namespace: Name of the pod's namespace.
         """
         async with self.get_conn() as connection:
-            v1_api = async_client.CoreV1Api(connection)
-            pod: V1Pod = await v1_api.read_namespaced_pod(
-                name=name,
-                namespace=namespace,
-            )
-        return pod
+            try:
+                v1_api = async_client.CoreV1Api(connection)
+                pod: V1Pod = await v1_api.read_namespaced_pod(
+                    name=name,
+                    namespace=namespace,
+                )
+                return pod
+            except HTTPError as e:
+                if hasattr(e, "status") and e.status == 403:
+                    raise KubernetesApiPermissionError("Permission denied (403) from Kubernetes API.") from e
+                raise KubernetesApiError from e
 
+    @generic_api_retry
     async def delete_pod(self, name: str, namespace: str):
         """
         Delete pod's object.
@@ -904,7 +920,10 @@ class AsyncKubernetesHook(KubernetesHook):
                 if str(e.status) != "404":
                     raise
 
-    async def read_logs(self, name: str, namespace: str):
+    @generic_api_retry
+    async def read_logs(
+        self, name: str, namespace: str, container_name: str | None = None, since_seconds: int | None = None
+    ) -> list[str]:
         """
         Read logs inside the pod while starting containers inside.
 
@@ -915,6 +934,8 @@ class AsyncKubernetesHook(KubernetesHook):
 
         :param name: Name of the pod.
         :param namespace: Name of the pod's namespace.
+        :param container_name: Name of the container inside the pod.
+        :param since_seconds: Only return logs newer than a relative duration in seconds.
         """
         async with self.get_conn() as connection:
             try:
@@ -922,17 +943,33 @@ class AsyncKubernetesHook(KubernetesHook):
                 logs = await v1_api.read_namespaced_pod_log(
                     name=name,
                     namespace=namespace,
+                    container=container_name,
                     follow=False,
                     timestamps=True,
+                    since_seconds=since_seconds,
                 )
                 logs = logs.splitlines()
-                for line in logs:
-                    self.log.info("Container logs from %s", line)
                 return logs
-            except HTTPError:
-                self.log.exception("There was an error reading the kubernetes API.")
-                raise
+            except HTTPError as e:
+                raise KubernetesApiError from e
+
+    @generic_api_retry
+    async def get_pod_events(self, name: str, namespace: str) -> CoreV1EventList:
+        """Get pod's events."""
+        async with self.get_conn() as connection:
+            try:
+                v1_api = async_client.CoreV1Api(connection)
+                events: CoreV1EventList = await v1_api.list_namespaced_event(
+                    field_selector=f"involvedObject.name={name}",
+                    namespace=namespace,
+                )
+                return events
+            except HTTPError as e:
+                if hasattr(e, "status") and e.status == 403:
+                    raise KubernetesApiPermissionError("Permission denied (403) from Kubernetes API.") from e
+                raise KubernetesApiError from e
 
+    @generic_api_retry
     async def get_job_status(self, name: str, namespace: str) -> V1Job:
         """
         Get job's status object.

airflow/providers/cncf/kubernetes/kube_config.py

@@ -16,8 +16,10 @@
 # under the License.
 from __future__ import annotations
 
+import warnings
+
 from airflow.configuration import conf
-from airflow.exceptions import AirflowConfigException
+from airflow.exceptions import AirflowConfigException, AirflowProviderDeprecationWarning
 from airflow.settings import AIRFLOW_HOME
 
 
@@ -53,7 +55,21 @@ class KubeConfig:
             self.kubernetes_section, "worker_pods_creation_batch_size"
         )
         self.worker_container_repository = conf.get(self.kubernetes_section, "worker_container_repository")
+        if self.worker_container_repository:
+            warnings.warn(
+                "Configuration 'worker_container_repository' is deprecated. "
+                "Use 'pod_template_file' to specify the container image repository instead.",
+                AirflowProviderDeprecationWarning,
+                stacklevel=2,
+            )
         self.worker_container_tag = conf.get(self.kubernetes_section, "worker_container_tag")
+        if self.worker_container_tag:
+            warnings.warn(
+                "Configuration 'worker_container_tag' is deprecated. "
+                "Use 'pod_template_file' to specify the container image tag instead.",
+                AirflowProviderDeprecationWarning,
+                stacklevel=2,
+            )
         if self.worker_container_repository and self.worker_container_tag:
             self.kube_image = f"{self.worker_container_repository}:{self.worker_container_tag}"
         else:
@@ -64,6 +80,13 @@ class KubeConfig:
         # cluster has RBAC enabled, your scheduler may need service account permissions to
         # create, watch, get, and delete pods in this namespace.
         self.kube_namespace = conf.get(self.kubernetes_section, "namespace")
+        if self.kube_namespace and self.kube_namespace != "default":
+            warnings.warn(
+                "Configuration 'namespace' is deprecated. "
+                "Use 'pod_template_file' to specify the namespace instead.",
+                AirflowProviderDeprecationWarning,
+                stacklevel=2,
+            )
         self.multi_namespace_mode = conf.getboolean(self.kubernetes_section, "multi_namespace_mode")
         if self.multi_namespace_mode and conf.get(
             self.kubernetes_section, "multi_namespace_mode_namespace_list"

airflow/providers/cncf/kubernetes/kubernetes_helper_functions.py

@@ -23,10 +23,14 @@ from functools import cache
 from typing import TYPE_CHECKING
 
 import pendulum
-from kubernetes.client.rest import ApiException
+import tenacity
+from kubernetes.client.rest import ApiException as SyncApiException
+from kubernetes_asyncio.client.exceptions import ApiException as AsyncApiException
 from slugify import slugify
+from urllib3.exceptions import HTTPError
 
 from airflow.configuration import conf
+from airflow.exceptions import AirflowException
 from airflow.providers.cncf.kubernetes.backcompat import get_logical_date_key
 
 if TYPE_CHECKING:
@@ -39,6 +43,62 @@ alphanum_lower = string.ascii_lowercase + string.digits
 POD_NAME_MAX_LENGTH = 63  # Matches Linux kernel's HOST_NAME_MAX default value minus 1.
 
 
+class PodLaunchFailedException(AirflowException):
+    """When pod launching fails in KubernetesPodOperator."""
+
+
+class KubernetesApiException(AirflowException):
+    """When communication with kubernetes API fails."""
+
+
+API_RETRIES = conf.getint("workers", "api_retries", fallback=5)
+API_RETRY_WAIT_MIN = conf.getfloat("workers", "api_retry_wait_min", fallback=1)
+API_RETRY_WAIT_MAX = conf.getfloat("workers", "api_retry_wait_max", fallback=15)
+
+_default_wait = tenacity.wait_exponential(min=API_RETRY_WAIT_MIN, max=API_RETRY_WAIT_MAX)
+
+TRANSIENT_STATUS_CODES = {409, 429, 500, 502, 503, 504}
+
+
+def _should_retry_api(exc: BaseException) -> bool:
+    """Retry on selected ApiException status codes, plus plain HTTP/timeout errors."""
+    if isinstance(exc, (SyncApiException, AsyncApiException)):
+        return exc.status in TRANSIENT_STATUS_CODES
+    return isinstance(exc, (HTTPError, KubernetesApiException))
+
+
+class WaitRetryAfterOrExponential(tenacity.wait.wait_base):
+    """Wait strategy that honors Retry-After header on 429, else falls back to exponential backoff."""
+
+    def __call__(self, retry_state):
+        exc = retry_state.outcome.exception() if retry_state.outcome else None
+        if isinstance(exc, (SyncApiException, AsyncApiException)) and exc.status == 429:
+            retry_after = (exc.headers or {}).get("Retry-After")
+            if retry_after:
+                try:
+                    return float(int(retry_after))
+                except ValueError:
+                    pass
+        # Inline exponential fallback
+        return _default_wait(retry_state)
+
+
+def generic_api_retry(func):
+    """
+    Retry to Kubernetes API calls.
+
+    - Retries only transient ApiException status codes.
+    - Honors Retry-After on 429.
+    """
+    return tenacity.retry(
+        stop=tenacity.stop_after_attempt(API_RETRIES),
+        wait=WaitRetryAfterOrExponential(),
+        retry=tenacity.retry_if_exception(_should_retry_api),
+        reraise=True,
+        before_sleep=tenacity.before_sleep_log(log, logging.WARNING),
+    )(func)
+
+
 def rand_str(num):
     """
     Generate random lowercase alphanumeric string of length num.
@@ -111,6 +171,8 @@ def annotations_to_key(annotations: dict[str, str]) -> TaskInstanceKey:
     if not annotation_run_id and logical_date_key in annotations:
         logical_date = pendulum.parse(annotations[logical_date_key])
         # Do _not_ use create-session, we don't want to expunge
+        if Session is None:
+            raise RuntimeError("Session not configured. Call configure_orm() first.")
         session = Session()
 
         task_instance_run_id = (
@@ -146,18 +208,3 @@ def annotations_for_logging_task_metadata(annotation_set):
     else:
         annotations_for_logging = "<omitted>"
     return annotations_for_logging
-
-
-def should_retry_creation(exception: BaseException) -> bool:
-    """
-    Check if an Exception indicates a transient error and warrants retrying.
-
-    This function is needed for preventing 'No agent available' error. The error appears time to time
-    when users try to create a Resource or Job. This issue is inside kubernetes and in the current moment
-    has no solution. Like a temporary solution we decided to retry Job or Resource creation request each
-    time when this error appears.
-    More about this issue here: https://github.com/cert-manager/cert-manager/issues/6457
-    """
-    if isinstance(exception, ApiException):
-        return str(exception.status) == "500"
-    return False

airflow/providers/cncf/kubernetes/operators/job.py

@@ -35,6 +35,7 @@ from airflow.configuration import conf
 from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
 from airflow.providers.cncf.kubernetes.hooks.kubernetes import KubernetesHook
 from airflow.providers.cncf.kubernetes.kubernetes_helper_functions import (
+    POD_NAME_MAX_LENGTH,
     add_unique_suffix,
     create_unique_id,
 )
@@ -56,6 +57,8 @@ if TYPE_CHECKING:
 
 log = logging.getLogger(__name__)
 
+JOB_NAME_PREFIX = "job-"
+
 
 class KubernetesJobOperator(KubernetesPodOperator):
     """
@@ -378,15 +381,18 @@ class KubernetesJobOperator(KubernetesPodOperator):
 
         job = self.reconcile_jobs(job_template, job)
 
+        # Account for job name prefix when generating/truncating the name
+        max_base_length = POD_NAME_MAX_LENGTH - len(JOB_NAME_PREFIX)
+
         if not job.metadata.name:
             job.metadata.name = create_unique_id(
-                task_id=self.task_id, unique=self.random_name_suffix, max_length=80
+                task_id=self.task_id, unique=self.random_name_suffix, max_length=max_base_length
             )
         elif self.random_name_suffix:
             # user has supplied job name, we're just adding suffix
-            job.metadata.name = add_unique_suffix(name=job.metadata.name)
+            job.metadata.name = add_unique_suffix(name=job.metadata.name, max_len=max_base_length)
 
-        job.metadata.name = f"job-{job.metadata.name}"
+        job.metadata.name = f"{JOB_NAME_PREFIX}{job.metadata.name}"
 
         if not job.metadata.namespace:
             hook_namespace = self.hook.get_namespace()