apache-airflow-providers-cncf-kubernetes 8.2.0__tar.gz → 8.3.0__tar.gz

{apache_airflow_providers_cncf_kubernetes-8.2.0 → apache_airflow_providers_cncf_kubernetes-8.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apache-airflow-providers-cncf-kubernetes
-Version: 8.2.0
+Version: 8.3.0
 Summary: Provider package apache-airflow-providers-cncf-kubernetes for Apache Airflow
 Keywords: airflow-provider,cncf.kubernetes,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -29,8 +29,8 @@ Requires-Dist: google-re2>=1.0
 Requires-Dist: kubernetes>=28.1.0,<=29.0.0
 Requires-Dist: kubernetes_asyncio>=28.1.0,<=29.0.0
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.2.0/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.2.0
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.3.0/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.3.0
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Twitter, https://twitter.com/ApacheAirflow
@@ -80,7 +80,7 @@ Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
 
 Package ``apache-airflow-providers-cncf-kubernetes``
 
-Release: ``8.2.0``
+Release: ``8.3.0``
 
 
 `Kubernetes <https://kubernetes.io/>`__
@@ -93,7 +93,7 @@ This is a provider package for ``cncf.kubernetes`` provider. All classes for thi
 are in ``airflow.providers.cncf.kubernetes`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.2.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.3.0/>`_.
 
 Installation
 ------------
@@ -120,4 +120,4 @@ PIP package             Version required
 ======================  =====================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.2.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.3.0/changelog.html>`_.

{apache_airflow_providers_cncf_kubernetes-8.2.0 → apache_airflow_providers_cncf_kubernetes-8.3.0}/README.rst

@@ -42,7 +42,7 @@
 
 Package ``apache-airflow-providers-cncf-kubernetes``
 
-Release: ``8.2.0``
+Release: ``8.3.0``
 
 
 `Kubernetes <https://kubernetes.io/>`__
@@ -55,7 +55,7 @@ This is a provider package for ``cncf.kubernetes`` provider. All classes for thi
 are in ``airflow.providers.cncf.kubernetes`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.2.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.3.0/>`_.
 
 Installation
 ------------
@@ -82,4 +82,4 @@ PIP package             Version required
 ======================  =====================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.2.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.3.0/changelog.html>`_.

{apache_airflow_providers_cncf_kubernetes-8.2.0 → apache_airflow_providers_cncf_kubernetes-8.3.0}/airflow/providers/cncf/kubernetes/__init__.py

@@ -25,14 +25,11 @@ from __future__ import annotations
 
 import packaging.version
 
-__all__ = ["__version__"]
+from airflow import __version__ as airflow_version
 
-__version__ = "8.2.0"
+__all__ = ["__version__"]
 
-try:
-    from airflow import __version__ as airflow_version
-except ImportError:
-    from airflow.version import version as airflow_version
+__version__ = "8.3.0"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.7.0"

apache_airflow_providers_cncf_kubernetes-8.3.0/airflow/providers/cncf/kubernetes/cli/kubernetes_command.py

@@ -0,0 +1,164 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Kubernetes sub-commands."""
+
+from __future__ import annotations
+
+import os
+import sys
+from datetime import datetime, timedelta
+
+from kubernetes import client
+from kubernetes.client.api_client import ApiClient
+from kubernetes.client.rest import ApiException
+
+from airflow.models import DagRun, TaskInstance
+from airflow.providers.cncf.kubernetes import pod_generator
+from airflow.providers.cncf.kubernetes.executors.kubernetes_executor import KubeConfig
+from airflow.providers.cncf.kubernetes.kube_client import get_kube_client
+from airflow.providers.cncf.kubernetes.kubernetes_helper_functions import create_pod_id
+from airflow.providers.cncf.kubernetes.pod_generator import PodGenerator
+from airflow.utils import cli as cli_utils, yaml
+from airflow.utils.cli import get_dag
+from airflow.utils.providers_configuration_loader import providers_configuration_loaded
+
+
+@cli_utils.action_cli
+@providers_configuration_loaded
+def generate_pod_yaml(args):
+    """Generate yaml files for each task in the DAG. Used for testing output of KubernetesExecutor."""
+    execution_date = args.execution_date
+    dag = get_dag(subdir=args.subdir, dag_id=args.dag_id)
+    yaml_output_path = args.output_path
+    dr = DagRun(dag.dag_id, execution_date=execution_date)
+    kube_config = KubeConfig()
+    for task in dag.tasks:
+        ti = TaskInstance(task, None)
+        ti.dag_run = dr
+        pod = PodGenerator.construct_pod(
+            dag_id=args.dag_id,
+            task_id=ti.task_id,
+            pod_id=create_pod_id(args.dag_id, ti.task_id),
+            try_number=ti.try_number,
+            kube_image=kube_config.kube_image,
+            date=ti.execution_date,
+            args=ti.command_as_list(),
+            pod_override_object=PodGenerator.from_obj(ti.executor_config),
+            scheduler_job_id="worker-config",
+            namespace=kube_config.executor_namespace,
+            base_worker_pod=PodGenerator.deserialize_model_file(kube_config.pod_template_file),
+            with_mutation_hook=True,
+        )
+        api_client = ApiClient()
+        date_string = pod_generator.datetime_to_label_safe_datestring(execution_date)
+        yaml_file_name = f"{args.dag_id}_{ti.task_id}_{date_string}.yml"
+        os.makedirs(os.path.dirname(yaml_output_path + "/airflow_yaml_output/"), exist_ok=True)
+        with open(yaml_output_path + "/airflow_yaml_output/" + yaml_file_name, "w") as output:
+            sanitized_pod = api_client.sanitize_for_serialization(pod)
+            output.write(yaml.dump(sanitized_pod))
+    print(f"YAML output can be found at {yaml_output_path}/airflow_yaml_output/")
+
+
+@cli_utils.action_cli
+@providers_configuration_loaded
+def cleanup_pods(args):
+    """Clean up k8s pods in evicted/failed/succeeded/pending states."""
+    namespace = args.namespace
+
+    min_pending_minutes = args.min_pending_minutes
+    # protect newly created pods from deletion
+    if min_pending_minutes < 5:
+        min_pending_minutes = 5
+
+    # https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/
+    # All Containers in the Pod have terminated in success, and will not be restarted.
+    pod_succeeded = "succeeded"
+
+    # The Pod has been accepted by the Kubernetes cluster,
+    # but one or more of the containers has not been set up and made ready to run.
+    pod_pending = "pending"
+
+    # All Containers in the Pod have terminated, and at least one Container has terminated in failure.
+    # That is, the Container either exited with non-zero status or was terminated by the system.
+    pod_failed = "failed"
+
+    # https://kubernetes.io/docs/tasks/administer-cluster/out-of-resource/
+    pod_reason_evicted = "evicted"
+    # If pod is failed and restartPolicy is:
+    # * Always: Restart Container; Pod phase stays Running.
+    # * OnFailure: Restart Container; Pod phase stays Running.
+    # * Never: Pod phase becomes Failed.
+    pod_restart_policy_never = "never"
+
+    print("Loading Kubernetes configuration")
+    kube_client = get_kube_client()
+    print(f"Listing pods in namespace {namespace}")
+    airflow_pod_labels = [
+        "dag_id",
+        "task_id",
+        "try_number",
+        "airflow_version",
+    ]
+    list_kwargs = {"namespace": namespace, "limit": 500, "label_selector": ",".join(airflow_pod_labels)}
+
+    while True:
+        pod_list = kube_client.list_namespaced_pod(**list_kwargs)
+        for pod in pod_list.items:
+            pod_name = pod.metadata.name
+            print(f"Inspecting pod {pod_name}")
+            pod_phase = pod.status.phase.lower()
+            pod_reason = pod.status.reason.lower() if pod.status.reason else ""
+            pod_restart_policy = pod.spec.restart_policy.lower()
+            current_time = datetime.now(pod.metadata.creation_timestamp.tzinfo)
+
+            if (
+                pod_phase == pod_succeeded
+                or (pod_phase == pod_failed and pod_restart_policy == pod_restart_policy_never)
+                or (pod_reason == pod_reason_evicted)
+                or (
+                    pod_phase == pod_pending
+                    and current_time - pod.metadata.creation_timestamp
+                    > timedelta(minutes=min_pending_minutes)
+                )
+            ):
+                print(
+                    f'Deleting pod "{pod_name}" phase "{pod_phase}" and reason "{pod_reason}", '
+                    f'restart policy "{pod_restart_policy}"'
+                )
+                try:
+                    _delete_pod(pod.metadata.name, namespace)
+                except ApiException as e:
+                    print(f"Can't remove POD: {e}", file=sys.stderr)
+            else:
+                print(f"No action taken on pod {pod_name}")
+        continue_token = pod_list.metadata._continue
+        if not continue_token:
+            break
+        list_kwargs["_continue"] = continue_token
+
+
+def _delete_pod(name, namespace):
+    """
+    Delete a namespaced pod.
+
+    Helper Function for cleanup_pods.
+    """
+    kube_client = get_kube_client()
+    delete_options = client.V1DeleteOptions()
+    print(f'Deleting POD "{name}" from "{namespace}" namespace')
+    api_response = kube_client.delete_namespaced_pod(name=name, namespace=namespace, body=delete_options)
+    print(api_response)

{apache_airflow_providers_cncf_kubernetes-8.2.0 → apache_airflow_providers_cncf_kubernetes-8.3.0}/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py

@@ -38,42 +38,18 @@ from typing import TYPE_CHECKING, Any, Sequence
 from kubernetes.dynamic import DynamicClient
 from sqlalchemy import select, update
 
-from airflow.providers.cncf.kubernetes.pod_generator import PodMutationHookException, PodReconciliationError
-from airflow.stats import Stats
-
-try:
-    from airflow.cli.cli_config import (
-        ARG_DAG_ID,
-        ARG_EXECUTION_DATE,
-        ARG_OUTPUT_PATH,
-        ARG_SUBDIR,
-        ARG_VERBOSE,
-        ActionCommand,
-        Arg,
-        GroupCommand,
-        lazy_load_command,
-        positive_int,
-    )
-except ImportError:
-    try:
-        from airflow import __version__ as airflow_version
-    except ImportError:
-        from airflow.version import version as airflow_version
-
-    import packaging.version
-
-    from airflow.exceptions import AirflowOptionalProviderFeatureException
-
-    base_version = packaging.version.parse(airflow_version).base_version
-
-    if packaging.version.parse(base_version) < packaging.version.parse("2.7.0"):
-        raise AirflowOptionalProviderFeatureException(
-            "Kubernetes Executor from CNCF Provider should only be used with Airflow 2.7.0+.\n"
-            f"This is Airflow {airflow_version} and Kubernetes and CeleryKubernetesExecutor are "
-            f"available in the 'airflow.executors' package. You should not use "
-            f"the provider's executors in this version of Airflow."
-        )
-    raise
+from airflow.cli.cli_config import (
+    ARG_DAG_ID,
+    ARG_EXECUTION_DATE,
+    ARG_OUTPUT_PATH,
+    ARG_SUBDIR,
+    ARG_VERBOSE,
+    ActionCommand,
+    Arg,
+    GroupCommand,
+    lazy_load_command,
+    positive_int,
+)
 from airflow.configuration import conf
 from airflow.executors.base_executor import BaseExecutor
 from airflow.providers.cncf.kubernetes.executors.kubernetes_executor_types import (
@@ -82,6 +58,8 @@ from airflow.providers.cncf.kubernetes.executors.kubernetes_executor_types impor
 )
 from airflow.providers.cncf.kubernetes.kube_config import KubeConfig
 from airflow.providers.cncf.kubernetes.kubernetes_helper_functions import annotations_to_key
+from airflow.providers.cncf.kubernetes.pod_generator import PodMutationHookException, PodReconciliationError
+from airflow.stats import Stats
 from airflow.utils.event_scheduler import EventScheduler
 from airflow.utils.log.logging_mixin import remove_escape_codes
 from airflow.utils.session import NEW_SESSION, provide_session
@@ -132,14 +110,14 @@ KUBERNETES_COMMANDS = (
             "(created by KubernetesExecutor/KubernetesPodOperator) "
             "in evicted/failed/succeeded/pending states"
         ),
-        func=lazy_load_command("airflow.cli.commands.kubernetes_command.cleanup_pods"),
+        func=lazy_load_command("airflow.providers.cncf.kubernetes.cli.kubernetes_command.cleanup_pods"),
         args=(ARG_NAMESPACE, ARG_MIN_PENDING_MINUTES, ARG_VERBOSE),
     ),
     ActionCommand(
         name="generate-dag-yaml",
         help="Generate YAML files for all tasks in DAG. Useful for debugging tasks without "
         "launching into a cluster",
-        func=lazy_load_command("airflow.cli.commands.kubernetes_command.generate_pod_yaml"),
+        func=lazy_load_command("airflow.providers.cncf.kubernetes.cli.kubernetes_command.generate_pod_yaml"),
         args=(ARG_DAG_ID, ARG_EXECUTION_DATE, ARG_SUBDIR, ARG_OUTPUT_PATH, ARG_VERBOSE),
     ),
 )

{apache_airflow_providers_cncf_kubernetes-8.2.0 → apache_airflow_providers_cncf_kubernetes-8.3.0}/airflow/providers/cncf/kubernetes/executors/kubernetes_executor_utils.py

@@ -21,7 +21,7 @@ import json
 import multiprocessing
 import time
 from queue import Empty, Queue
-from typing import TYPE_CHECKING, Any, Generic, TypeVar
+from typing import TYPE_CHECKING, Any
 
 from kubernetes import client, watch
 from kubernetes.client.rest import ApiException
@@ -32,10 +32,11 @@ from airflow.providers.cncf.kubernetes.kube_client import get_kube_client
 from airflow.providers.cncf.kubernetes.kubernetes_helper_functions import (
     annotations_for_logging_task_metadata,
     annotations_to_key,
-    create_pod_id,
+    create_unique_id,
 )
 from airflow.providers.cncf.kubernetes.pod_generator import PodGenerator
 from airflow.utils.log.logging_mixin import LoggingMixin
+from airflow.utils.singleton import Singleton
 from airflow.utils.state import TaskInstanceState
 
 try:
@@ -60,22 +61,6 @@ if TYPE_CHECKING:
         KubernetesWatchType,
     )
 
-# Singleton here is duplicated version of airflow.utils.singleton.Singleton until
-# min-airflow version is 2.7.0 for the provider. then it can be imported from airflow.utils.singleton.
-
-T = TypeVar("T")
-
-
-class Singleton(type, Generic[T]):
-    """Metaclass that allows to implement singleton pattern."""
-
-    _instances: dict[Singleton[T], T] = {}
-
-    def __call__(cls: Singleton[T], *args, **kwargs) -> T:
-        if cls not in cls._instances:
-            cls._instances[cls] = super().__call__(*args, **kwargs)
-        return cls._instances[cls]
-
 
 class ResourceVersion(metaclass=Singleton):
     """Singleton for tracking resourceVersion from Kubernetes."""
@@ -113,9 +98,7 @@ class KubernetesJobWatcher(multiprocessing.Process, LoggingMixin):
                     kube_client, self.resource_version, self.scheduler_job_id, self.kube_config
                 )
             except ReadTimeoutError:
-                self.log.warning(
-                    "There was a timeout error accessing the Kube API. Retrying request.", exc_info=True
-                )
+                self.log.info("Kubernetes watch timed out waiting for events. Restarting watch.")
                 time.sleep(1)
             except Exception:
                 self.log.exception("Unknown error in KubernetesJobWatcher. Failing")
@@ -156,7 +139,7 @@ class KubernetesJobWatcher(multiprocessing.Process, LoggingMixin):
     ) -> str | None:
         self.log.info("Event: and now my watch begins starting at resource_version: %s", resource_version)
 
-        kwargs = {"label_selector": f"airflow-worker={scheduler_job_id}"}
+        kwargs: dict[str, Any] = {"label_selector": f"airflow-worker={scheduler_job_id}"}
         if resource_version:
             kwargs["resource_version"] = resource_version
         if kube_config.kube_client_request_args:
@@ -165,6 +148,14 @@ class KubernetesJobWatcher(multiprocessing.Process, LoggingMixin):
 
         last_resource_version: str | None = None
 
+        # For info about k8s timeout settings see
+        # https://github.com/kubernetes-client/python/blob/v29.0.0/examples/watch/timeout-settings.md
+        # and https://github.com/kubernetes-client/python/blob/v29.0.0/kubernetes/client/api_client.py#L336-L339
+        client_timeout = 30
+        server_conn_timeout = 3600
+        kwargs["_request_timeout"] = client_timeout
+        kwargs["timeout_seconds"] = server_conn_timeout
+
         for event in self._pod_events(kube_client=kube_client, query_kwargs=kwargs):
             task = event["object"]
             self.log.debug("Event: %s had an event of type %s", task.metadata.name, event["type"])
@@ -413,7 +404,7 @@ class AirflowKubernetesScheduler(LoggingMixin):
         pod = PodGenerator.construct_pod(
             namespace=self.namespace,
             scheduler_job_id=self.scheduler_job_id,
-            pod_id=create_pod_id(dag_id, task_id),
+            pod_id=create_unique_id(dag_id, task_id),
             dag_id=dag_id,
             task_id=task_id,
             kube_image=self.kube_config.kube_image,

{apache_airflow_providers_cncf_kubernetes-8.2.0 → apache_airflow_providers_cncf_kubernetes-8.3.0}/airflow/providers/cncf/kubernetes/get_provider_info.py

@@ -28,8 +28,9 @@ def get_provider_info():
         "name": "Kubernetes",
         "description": "`Kubernetes <https://kubernetes.io/>`__\n",
         "state": "ready",
-        "source-date-epoch": 1714476004,
+        "source-date-epoch": 1716287042,
         "versions": [
+            "8.3.0",
             "8.2.0",
             "8.1.1",
             "8.1.0",

{apache_airflow_providers_cncf_kubernetes-8.2.0 → apache_airflow_providers_cncf_kubernetes-8.3.0}/airflow/providers/cncf/kubernetes/hooks/kubernetes.py

@@ -25,6 +25,7 @@ from time import sleep
 from typing import TYPE_CHECKING, Any, Generator
 
 import aiofiles
+import tenacity
 from asgiref.sync import sync_to_async
 from kubernetes import client, config, watch
 from kubernetes.config import ConfigException
@@ -35,6 +36,7 @@ from airflow.exceptions import AirflowException, AirflowNotFoundException
 from airflow.hooks.base import BaseHook
 from airflow.models import Connection
 from airflow.providers.cncf.kubernetes.kube_client import _disable_verify_ssl, _enable_tcp_keepalive
+from airflow.providers.cncf.kubernetes.kubernetes_helper_functions import should_retry_creation
 from airflow.providers.cncf.kubernetes.utils.pod_manager import PodOperatorHookProtocol
 from airflow.utils import yaml
 
@@ -486,6 +488,12 @@ class KubernetesHook(BaseHook, PodOperatorHookProtocol):
         except Exception as exc:
             raise exc
 
+    @tenacity.retry(
+        stop=tenacity.stop_after_attempt(3),
+        wait=tenacity.wait_random_exponential(),
+        reraise=True,
+        retry=tenacity.retry_if_exception(should_retry_creation),
+    )
     def create_job(
         self,
         job: V1Job,

{apache_airflow_providers_cncf_kubernetes-8.2.0 → apache_airflow_providers_cncf_kubernetes-8.3.0}/airflow/providers/cncf/kubernetes/kubernetes_helper_functions.py

@@ -19,10 +19,11 @@ from __future__ import annotations
 import logging
 import secrets
 import string
-import warnings
 from typing import TYPE_CHECKING
 
 import pendulum
+from deprecated import deprecated
+from kubernetes.client.rest import ApiException
 from slugify import slugify
 
 from airflow.compat.functools import cache
@@ -59,6 +60,10 @@ def add_unique_suffix(*, name: str, rand_len: int = 8, max_len: int = POD_NAME_M
     return name[: max_len - len(suffix)].strip("-.") + suffix
 
 
+@deprecated(
+    reason="This function is deprecated. Please use `add_unique_suffix`",
+    category=AirflowProviderDeprecationWarning,
+)
 def add_pod_suffix(*, pod_name: str, rand_len: int = 8, max_len: int = POD_NAME_MAX_LENGTH) -> str:
     """Add random string to pod name while staying under max length.
 
@@ -67,14 +72,7 @@ def add_pod_suffix(*, pod_name: str, rand_len: int = 8, max_len: int = POD_NAME_
     :param max_len: maximum length of the pod name
     :meta private:
     """
-    warnings.warn(
-        "This function is deprecated. Please use `add_unique_suffix`.",
-        AirflowProviderDeprecationWarning,
-        stacklevel=2,
-    )
-
-    suffix = "-" + rand_str(rand_len)
-    return pod_name[: max_len - len(suffix)].strip("-.") + suffix
+    return add_unique_suffix(name=pod_name, rand_len=rand_len, max_len=max_len)
 
 
 def create_unique_id(
@@ -109,6 +107,10 @@ def create_unique_id(
         return base_name
 
 
+@deprecated(
+    reason="This function is deprecated. Please use `create_unique_id`.",
+    category=AirflowProviderDeprecationWarning,
+)
 def create_pod_id(
     dag_id: str | None = None,
     task_id: str | None = None,
@@ -125,26 +127,7 @@ def create_pod_id(
     :param unique: whether a random string suffix should be added
     :return: A valid identifier for a kubernetes pod name
     """
-    warnings.warn(
-        "This function is deprecated. Please use `create_unique_id`.",
-        AirflowProviderDeprecationWarning,
-        stacklevel=2,
-    )
-
-    if not (dag_id or task_id):
-        raise ValueError("Must supply either dag_id or task_id.")
-    name = ""
-    if dag_id:
-        name += dag_id
-    if task_id:
-        if name:
-            name += "-"
-        name += task_id
-    base_name = slugify(name, lowercase=True)[:max_length].strip(".-")
-    if unique:
-        return add_pod_suffix(pod_name=base_name, rand_len=8, max_len=max_length)
-    else:
-        return base_name
+    return create_unique_id(dag_id=dag_id, task_id=task_id, max_length=max_length, unique=unique)
 
 
 def annotations_to_key(annotations: dict[str, str]) -> TaskInstanceKey:
@@ -199,3 +182,18 @@ def annotations_for_logging_task_metadata(annotation_set):
     else:
         annotations_for_logging = "<omitted>"
     return annotations_for_logging
+
+
+def should_retry_creation(exception: BaseException) -> bool:
+    """
+    Check if an Exception indicates a transient error and warrants retrying.
+
+    This function is needed for preventing 'No agent available' error. The error appears time to time
+    when users try to create a Resource or Job. This issue is inside kubernetes and in the current moment
+    has no solution. Like a temporary solution we decided to retry Job or Resource creation request each
+    time when this error appears.
+    More about this issue here: https://github.com/cert-manager/cert-manager/issues/6457
+    """
+    if isinstance(exception, ApiException):
+        return str(exception.status) == "500"
+    return False

{apache_airflow_providers_cncf_kubernetes-8.2.0 → apache_airflow_providers_cncf_kubernetes-8.3.0}/airflow/providers/cncf/kubernetes/operators/pod.py

@@ -36,6 +36,7 @@ import kubernetes
 import tenacity
 from deprecated import deprecated
 from kubernetes.client import CoreV1Api, V1Pod, models as k8s
+from kubernetes.client.exceptions import ApiException
 from kubernetes.stream import stream
 from urllib3.exceptions import HTTPError
 
@@ -553,7 +554,7 @@ class KubernetesPodOperator(BaseOperator):
 
         return pod_request_obj
 
-    def await_pod_start(self, pod: k8s.V1Pod):
+    def await_pod_start(self, pod: k8s.V1Pod) -> None:
         try:
             self.pod_manager.await_pod_start(
                 pod=pod,
@@ -565,23 +566,23 @@ class KubernetesPodOperator(BaseOperator):
                 self._read_pod_events(pod, reraise=False)
             raise
 
-    def extract_xcom(self, pod: k8s.V1Pod):
+    def extract_xcom(self, pod: k8s.V1Pod) -> dict[Any, Any] | None:
         """Retrieve xcom value and kill xcom sidecar container."""
         result = self.pod_manager.extract_xcom(pod)
         if isinstance(result, str) and result.rstrip() == EMPTY_XCOM_RESULT:
             self.log.info("xcom result file is empty.")
             return None
-        else:
-            self.log.info("xcom result: \n%s", result)
-            return json.loads(result)
+
+        self.log.info("xcom result: \n%s", result)
+        return json.loads(result)
 
     def execute(self, context: Context):
         """Based on the deferrable parameter runs the pod asynchronously or synchronously."""
-        if self.deferrable:
-            self.execute_async(context)
-        else:
+        if not self.deferrable:
             return self.execute_sync(context)
 
+        self.execute_async(context)
+
     def execute_sync(self, context: Context):
         result = None
         try:
@@ -669,7 +670,7 @@ class KubernetesPodOperator(BaseOperator):
         del self.client
         del self.pod_manager
 
-    def execute_async(self, context: Context):
+    def execute_async(self, context: Context) -> None:
         self.pod_request_obj = self.build_pod_request_obj(context)
         self.pod = self.get_or_create_pod(  # must set `self.pod` for `on_kill`
             pod_request_obj=self.pod_request_obj,
@@ -687,7 +688,7 @@ class KubernetesPodOperator(BaseOperator):
 
         self.invoke_defer_method()
 
-    def invoke_defer_method(self, last_log_time: DateTime | None = None):
+    def invoke_defer_method(self, last_log_time: DateTime | None = None) -> None:
         """Redefine triggers which are being used in child classes."""
         trigger_start_time = datetime.datetime.now(tz=datetime.timezone.utc)
         self.defer(
@@ -742,7 +743,7 @@ class KubernetesPodOperator(BaseOperator):
             if event["status"] in ("error", "failed", "timeout"):
                 # fetch some logs when pod is failed
                 if self.get_logs:
-                    self.write_logs(self.pod, follow=follow, since_time=last_log_time)
+                    self._write_logs(self.pod, follow=follow, since_time=last_log_time)
 
                 if self.do_xcom_push:
                     _ = self.extract_xcom(pod=self.pod)
@@ -770,7 +771,7 @@ class KubernetesPodOperator(BaseOperator):
             elif event["status"] == "success":
                 # fetch some logs when pod is executed successfully
                 if self.get_logs:
-                    self.write_logs(self.pod, follow=follow, since_time=last_log_time)
+                    self._write_logs(self.pod, follow=follow, since_time=last_log_time)
 
                 if self.do_xcom_push:
                     xcom_sidecar_output = self.extract_xcom(pod=self.pod)
@@ -781,51 +782,59 @@ class KubernetesPodOperator(BaseOperator):
         finally:
             self._clean(event)
 
-    def _clean(self, event: dict[str, Any]):
+    def _clean(self, event: dict[str, Any]) -> None:
         if event["status"] == "running":
             return
         istio_enabled = self.is_istio_enabled(self.pod)
         # Skip await_pod_completion when the event is 'timeout' due to the pod can hang
         # on the ErrImagePull or ContainerCreating step and it will never complete
         if event["status"] != "timeout":
-            self.pod = self.pod_manager.await_pod_completion(
-                self.pod, istio_enabled, self.base_container_name
-            )
+            try:
+                self.pod = self.pod_manager.await_pod_completion(
+                    self.pod, istio_enabled, self.base_container_name
+                )
+            except ApiException as e:
+                if e.status == 404:
+                    self.pod = None
+                    self.log.warning(
+                        "Pod not found while waiting for completion. The last status was %r", event["status"]
+                    )
+                else:
+                    raise e
         if self.pod is not None:
             self.post_complete_action(
                 pod=self.pod,
                 remote_pod=self.pod,
             )
 
-    @deprecated(reason="use `trigger_reentry` instead.", category=AirflowProviderDeprecationWarning)
-    def execute_complete(self, context: Context, event: dict, **kwargs):
-        return self.trigger_reentry(context=context, event=event)
-
-    def write_logs(self, pod: k8s.V1Pod, follow: bool = False, since_time: DateTime | None = None):
+    def _write_logs(self, pod: k8s.V1Pod, follow: bool = False, since_time: DateTime | None = None) -> None:
         try:
             since_seconds = (
                 math.ceil((datetime.datetime.now(tz=datetime.timezone.utc) - since_time).total_seconds())
                 if since_time
                 else None
             )
-            logs = self.pod_manager.read_pod_logs(
-                pod=pod,
-                container_name=self.base_container_name,
+            logs = self.client.read_namespaced_pod_log(
+                name=pod.metadata.name,
+                namespace=pod.metadata.namespace,
+                container=self.base_container_name,
                 follow=follow,
+                timestamps=False,
                 since_seconds=since_seconds,
+                _preload_content=False,
             )
             for raw_line in logs:
                 line = raw_line.decode("utf-8", errors="backslashreplace").rstrip("\n")
                 if line:
-                    self.log.info("Container logs: %s", line)
-        except HTTPError as e:
+                    self.log.info("[%s] logs: %s", self.base_container_name, line)
+        except (HTTPError, ApiException) as e:
             self.log.warning(
                 "Reading of logs interrupted with error %r; will retry. "
                 "Set log level to DEBUG for traceback.",
-                e,
+                e if not isinstance(e, ApiException) else e.reason,
             )
 
-    def post_complete_action(self, *, pod, remote_pod, **kwargs):
+    def post_complete_action(self, *, pod, remote_pod, **kwargs) -> None:
         """Actions that must be done after operator finishes logic of the deferrable_execution."""
         self.cleanup(
             pod=pod,
@@ -893,7 +902,7 @@ class KubernetesPodOperator(BaseOperator):
                 )
             )
 
-    def _read_pod_events(self, pod, *, reraise=True):
+    def _read_pod_events(self, pod, *, reraise=True) -> None:
         """Will fetch and emit events from pod."""
         with _optionally_suppress(reraise=reraise):
             for event in self.pod_manager.read_pod_events(pod).items:
@@ -941,15 +950,11 @@ class KubernetesPodOperator(BaseOperator):
     def process_pod_deletion(self, pod: k8s.V1Pod, *, reraise=True):
         with _optionally_suppress(reraise=reraise):
             if pod is not None:
-                should_delete_pod = (
-                    (self.on_finish_action == OnFinishAction.DELETE_POD)
-                    or (
-                        self.on_finish_action == OnFinishAction.DELETE_SUCCEEDED_POD
-                        and pod.status.phase == PodPhase.SUCCEEDED
-                    )
-                    or (
-                        self.on_finish_action == OnFinishAction.DELETE_SUCCEEDED_POD
-                        and container_is_succeeded(pod, self.base_container_name)
+                should_delete_pod = (self.on_finish_action == OnFinishAction.DELETE_POD) or (
+                    self.on_finish_action == OnFinishAction.DELETE_SUCCEEDED_POD
+                    and (
+                        pod.status.phase == PodPhase.SUCCEEDED
+                        or container_is_succeeded(pod, self.base_container_name)
                     )
                 )
                 if should_delete_pod:
@@ -966,8 +971,8 @@ class KubernetesPodOperator(BaseOperator):
         label_strings = [f"{label_id}={label}" for label_id, label in sorted(labels.items())]
         labels_value = ",".join(label_strings)
         if exclude_checked:
-            labels_value += f",{self.POD_CHECKED_KEY}!=True"
-        labels_value += ",!airflow-worker"
+            labels_value = f"{labels_value},{self.POD_CHECKED_KEY}!=True"
+        labels_value = f"{labels_value},!airflow-worker"
         return labels_value
 
     @staticmethod
@@ -1129,6 +1134,10 @@ class KubernetesPodOperator(BaseOperator):
         pod = self.build_pod_request_obj()
         print(yaml.dump(prune_dict(pod.to_dict(), mode="strict")))
 
+    @deprecated(reason="use `trigger_reentry` instead.", category=AirflowProviderDeprecationWarning)
+    def execute_complete(self, context: Context, event: dict, **kwargs):
+        return self.trigger_reentry(context=context, event=event)
+
 
 class _optionally_suppress(AbstractContextManager):
     """
@@ -1142,7 +1151,7 @@ class _optionally_suppress(AbstractContextManager):
     :meta private:
     """
 
-    def __init__(self, *exceptions, reraise=False):
+    def __init__(self, *exceptions, reraise: bool = False) -> None:
         self._exceptions = exceptions or (Exception,)
         self.reraise = reraise
         self.exception = None
@@ -1150,7 +1159,7 @@ class _optionally_suppress(AbstractContextManager):
     def __enter__(self):
         return self
 
-    def __exit__(self, exctype, excinst, exctb):
+    def __exit__(self, exctype, excinst, exctb) -> bool:
         error = exctype is not None
         matching_error = error and issubclass(exctype, self._exceptions)
         if (error and not matching_error) or (matching_error and self.reraise):

{apache_airflow_providers_cncf_kubernetes-8.2.0 → apache_airflow_providers_cncf_kubernetes-8.3.0}/airflow/providers/cncf/kubernetes/operators/resource.py

@@ -22,12 +22,14 @@ import os
 from functools import cached_property
 from typing import TYPE_CHECKING, Sequence
 
+import tenacity
 import yaml
 from kubernetes.utils import create_from_yaml
 
 from airflow.exceptions import AirflowException
 from airflow.models import BaseOperator
 from airflow.providers.cncf.kubernetes.hooks.kubernetes import KubernetesHook
+from airflow.providers.cncf.kubernetes.kubernetes_helper_functions import should_retry_creation
 from airflow.providers.cncf.kubernetes.utils.delete_from import delete_from_yaml
 from airflow.providers.cncf.kubernetes.utils.k8s_resource_iterator import k8s_resource_iterator
 
@@ -126,7 +128,14 @@ class KubernetesCreateResourceOperator(KubernetesResourceBaseOperator):
         else:
             self.custom_object_client.create_cluster_custom_object(group, version, plural, body)
 
+    @tenacity.retry(
+        stop=tenacity.stop_after_attempt(3),
+        wait=tenacity.wait_random_exponential(),
+        reraise=True,
+        retry=tenacity.retry_if_exception(should_retry_creation),
+    )
     def _create_objects(self, objects):
+        self.log.info("Starting resource creation")
         if not self.custom_resource_definition:
             create_from_yaml(
                 k8s_client=self.client,
@@ -144,6 +153,7 @@ class KubernetesCreateResourceOperator(KubernetesResourceBaseOperator):
                 self._create_objects(yaml.safe_load_all(stream))
         else:
             raise AirflowException("File %s not found", self.yaml_conf_file)
+        self.log.info("Resource was created")
 
 
 class KubernetesDeleteResourceOperator(KubernetesResourceBaseOperator):

{apache_airflow_providers_cncf_kubernetes-8.2.0 → apache_airflow_providers_cncf_kubernetes-8.3.0}/airflow/providers/cncf/kubernetes/pod_generator.py

@@ -45,7 +45,6 @@ from airflow.exceptions import (
 )
 from airflow.providers.cncf.kubernetes.kubernetes_helper_functions import (
     POD_NAME_MAX_LENGTH,
-    add_pod_suffix,
     add_unique_suffix,
     rand_str,
 )
@@ -156,12 +155,15 @@ class PodGenerator:
         # Attach sidecar
         self.extract_xcom = extract_xcom
 
-    @deprecated(reason="This function is deprecated.", category=AirflowProviderDeprecationWarning)
+    @deprecated(
+        reason="This method is deprecated and will be removed in the future releases",
+        category=AirflowProviderDeprecationWarning,
+    )
     def gen_pod(self) -> k8s.V1Pod:
         """Generate pod."""
         result = self.ud_pod
 
-        result.metadata.name = add_pod_suffix(pod_name=result.metadata.name)
+        result.metadata.name = add_unique_suffix(name=result.metadata.name)
 
         if self.extract_xcom:
             result = self.add_xcom_sidecar(result)
@@ -210,8 +212,8 @@ class PodGenerator:
             return k8s_object
         elif isinstance(k8s_legacy_object, dict):
             warnings.warn(
-                "Using a dictionary for the executor_config is deprecated and will soon be removed."
-                'please use a `kubernetes.client.models.V1Pod` class with a "pod_override" key'
+                "Using a dictionary for the executor_config is deprecated and will soon be removed. "
+                'Please use a `kubernetes.client.models.V1Pod` class with a "pod_override" key'
                 " instead. ",
                 category=AirflowProviderDeprecationWarning,
                 stacklevel=2,
@@ -575,7 +577,7 @@ class PodGenerator:
 
     @staticmethod
     @deprecated(
-        reason="This function is deprecated. Use `add_pod_suffix` in `kubernetes_helper_functions`.",
+        reason="This method is deprecated. Use `add_pod_suffix` in `kubernetes_helper_functions`.",
         category=AirflowProviderDeprecationWarning,
     )
     def make_unique_pod_id(pod_id: str) -> str | None:
@@ -595,12 +597,6 @@ class PodGenerator:
         :param pod_id: requested pod name
         :return: ``str`` valid Pod name of appropriate length
         """
-        warnings.warn(
-            "This function is deprecated. Use `add_pod_suffix` in `kubernetes_helper_functions`.",
-            AirflowProviderDeprecationWarning,
-            stacklevel=2,
-        )
-
         if not pod_id:
             return None
 

{apache_airflow_providers_cncf_kubernetes-8.2.0 → apache_airflow_providers_cncf_kubernetes-8.3.0}/airflow/providers/cncf/kubernetes/template_rendering.py

@@ -24,9 +24,7 @@ from kubernetes.client.api_client import ApiClient
 
 from airflow.exceptions import AirflowException
 from airflow.providers.cncf.kubernetes.kube_config import KubeConfig
-from airflow.providers.cncf.kubernetes.kubernetes_helper_functions import (
-    create_pod_id,
-)
+from airflow.providers.cncf.kubernetes.kubernetes_helper_functions import create_unique_id
 from airflow.providers.cncf.kubernetes.pod_generator import PodGenerator
 from airflow.utils.session import NEW_SESSION, provide_session
 
@@ -43,7 +41,7 @@ def render_k8s_pod_yaml(task_instance: TaskInstance) -> dict | None:
         task_id=task_instance.task_id,
         map_index=task_instance.map_index,
         date=None,
-        pod_id=create_pod_id(task_instance.dag_id, task_instance.task_id),
+        pod_id=create_unique_id(task_instance.dag_id, task_instance.task_id),
         try_number=task_instance.try_number,
         kube_image=kube_config.kube_image,
         args=task_instance.command_as_list(),

apache_airflow_providers_cncf_kubernetes-8.3.0/airflow/providers/cncf/kubernetes/triggers/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

{apache_airflow_providers_cncf_kubernetes-8.2.0 → apache_airflow_providers_cncf_kubernetes-8.3.0}/airflow/providers/cncf/kubernetes/utils/pod_manager.py

@@ -516,7 +516,7 @@ class PodManager(LoggingMixin):
                 # a timeout is a normal thing and we ignore it and resume following logs
                 if not isinstance(exc, TimeoutError):
                     self.log.warning(
-                        "Pod %s log read interrupted but container %s still running",
+                        "Pod %s log read interrupted but container %s still running. Logs generated in the last one second might get duplicated.",
                         pod.metadata.name,
                         container_name,
                     )

{apache_airflow_providers_cncf_kubernetes-8.2.0 → apache_airflow_providers_cncf_kubernetes-8.3.0}/pyproject.toml

@@ -28,7 +28,7 @@ build-backend = "flit_core.buildapi"
 
 [project]
 name = "apache-airflow-providers-cncf-kubernetes"
-version = "8.2.0"
+version = "8.3.0"
 description = "Provider package apache-airflow-providers-cncf-kubernetes for Apache Airflow"
 readme = "README.rst"
 authors = [
@@ -66,8 +66,8 @@ dependencies = [
 ]
 
 [project.urls]
-"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.2.0"
-"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.2.0/changelog.html"
+"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.3.0"
+"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/8.3.0/changelog.html"
 "Bug Tracker" = "https://github.com/apache/airflow/issues"
 "Source Code" = "https://github.com/apache/airflow"
 "Slack Chat" = "https://s.apache.org/airflow-slack"