apache-airflow-providers-cncf-kubernetes 10.3.1rc1__py3-none-any.whl → 10.4.0rc1__py3-none-any.whl

airflow/providers/cncf/kubernetes/__init__.py

@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "10.3.1"
+__version__ = "10.4.0"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.9.0"

airflow/providers/cncf/kubernetes/get_provider_info.py

@@ -27,8 +27,9 @@ def get_provider_info():
         "name": "Kubernetes",
         "description": "`Kubernetes <https://kubernetes.io/>`__\n",
         "state": "ready",
-        "source-date-epoch": 1741508564,
+        "source-date-epoch": 1742480644,
         "versions": [
+            "10.4.0",
             "10.3.1",
             "10.3.0",
             "10.1.0",

airflow/providers/cncf/kubernetes/operators/pod.py

@@ -213,7 +213,9 @@ class KubernetesPodOperator(BaseOperator):
     :param base_container_name: The name of the base container in the pod. This container's logs
         will appear as part of this task's logs if get_logs is True. Defaults to None. If None,
         will consult the class variable BASE_CONTAINER_NAME (which defaults to "base") for the base
-        container name to use.
+        container name to use. (templated)
+    :param base_container_status_polling_interval: Polling period in seconds to check for the pod base
+        container status. Default to 1s.
     :param deferrable: Run operator in the deferrable mode.
     :param poll_interval: Polling period in seconds to check for the status. Used only in deferrable mode.
     :param log_pod_spec_on_failure: Log the pod's specification if a failure occurs
@@ -261,6 +263,7 @@ class KubernetesPodOperator(BaseOperator):
         "env_from",
         "node_selector",
         "kubernetes_conn_id",
+        "base_container_name",
     )
     template_fields_renderers = {"env_vars": "py"}
 
@@ -288,6 +291,7 @@ class KubernetesPodOperator(BaseOperator):
         startup_check_interval_seconds: int = 5,
         get_logs: bool = True,
         base_container_name: str | None = None,
+        base_container_status_polling_interval: float = 1,
         init_container_logs: Iterable[str] | str | Literal[True] | None = None,
         container_logs: Iterable[str] | str | Literal[True] | None = None,
         image_pull_policy: str | None = None,
@@ -365,6 +369,7 @@ class KubernetesPodOperator(BaseOperator):
         # Fallback to the class variable BASE_CONTAINER_NAME here instead of via default argument value
         # in the init method signature, to be compatible with subclasses overloading the class variable value.
         self.base_container_name = base_container_name or self.BASE_CONTAINER_NAME
+        self.base_container_status_polling_interval = base_container_status_polling_interval
         self.init_container_logs = init_container_logs
         self.container_logs = container_logs or self.base_container_name
         self.image_pull_policy = image_pull_policy
@@ -720,7 +725,11 @@ class KubernetesPodOperator(BaseOperator):
             if not self.get_logs or (
                 self.container_logs is not True and self.base_container_name not in self.container_logs
             ):
-                self.pod_manager.await_container_completion(pod=pod, container_name=self.base_container_name)
+                self.pod_manager.await_container_completion(
+                    pod=pod,
+                    container_name=self.base_container_name,
+                    polling_time=self.base_container_status_polling_interval,
+                )
         except kubernetes.client.exceptions.ApiException as exc:
             self._handle_api_exception(exc, pod)
 
@@ -834,6 +843,10 @@ class KubernetesPodOperator(BaseOperator):
             last_log_time = event.get("last_log_time")
 
             if event["status"] in ("error", "failed", "timeout"):
+                event_message = event.get("message", "No message provided")
+                self.log.error(
+                    "Trigger emitted an %s event, failing the task: %s", event["status"], event_message
+                )
                 # fetch some logs when pod is failed
                 if self.get_logs:
                     self._write_logs(self.pod, follow=follow, since_time=last_log_time)

airflow/providers/cncf/kubernetes/triggers/pod.py

@@ -22,7 +22,10 @@ import traceback
 from collections.abc import AsyncIterator
 from enum import Enum
 from functools import cached_property
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, cast
+
+import tenacity
+from kubernetes_asyncio.client.models import V1Pod
 
 from airflow.providers.cncf.kubernetes.hooks.kubernetes import AsyncKubernetesHook
 from airflow.providers.cncf.kubernetes.utils.pod_manager import (
@@ -33,7 +36,6 @@ from airflow.providers.cncf.kubernetes.utils.pod_manager import (
 from airflow.triggers.base import BaseTrigger, TriggerEvent
 
 if TYPE_CHECKING:
-    from kubernetes_asyncio.client.models import V1Pod
     from pendulum import DateTime
 
 
@@ -200,7 +202,7 @@ class KubernetesPodTrigger(BaseTrigger):
     async def _wait_for_pod_start(self) -> ContainerState:
         """Loops until pod phase leaves ``PENDING`` If timeout is reached, throws error."""
         while True:
-            pod = await self.hook.get_pod(self.pod_name, self.pod_namespace)
+            pod = await self._get_pod()
             if not pod.status.phase == "Pending":
                 return self.define_container_state(pod)
 
@@ -223,7 +225,7 @@ class KubernetesPodTrigger(BaseTrigger):
         if self.logging_interval is not None:
             time_get_more_logs = time_begin + datetime.timedelta(seconds=self.logging_interval)
         while True:
-            pod = await self.hook.get_pod(self.pod_name, self.pod_namespace)
+            pod = await self._get_pod()
             container_state = self.define_container_state(pod)
             if container_state == ContainerState.TERMINATED:
                 return TriggerEvent(
@@ -257,8 +259,16 @@ class KubernetesPodTrigger(BaseTrigger):
             self.log.debug("Sleeping for %s seconds.", self.poll_interval)
             await asyncio.sleep(self.poll_interval)
 
-    def _get_async_hook(self) -> AsyncKubernetesHook:
-        # TODO: Remove this method when the min version of kubernetes provider is 7.12.0 in Google provider.
+    @tenacity.retry(stop=tenacity.stop_after_attempt(3), wait=tenacity.wait_exponential(), reraise=True)
+    async def _get_pod(self) -> V1Pod:
+        """Get the pod from Kubernetes with retries."""
+        pod = await self.hook.get_pod(name=self.pod_name, namespace=self.pod_namespace)
+        # Due to AsyncKubernetesHook overriding get_pod, we need to cast the return
+        # value to kubernetes_asyncio.V1Pod, because it's perceived as different type
+        return cast(V1Pod, pod)
+
+    @cached_property
+    def hook(self) -> AsyncKubernetesHook:
         return AsyncKubernetesHook(
             conn_id=self.kubernetes_conn_id,
             in_cluster=self.in_cluster,
@@ -266,10 +276,6 @@ class KubernetesPodTrigger(BaseTrigger):
             cluster_context=self.cluster_context,
         )
 
-    @cached_property
-    def hook(self) -> AsyncKubernetesHook:
-        return self._get_async_hook()
-
     def define_container_state(self, pod: V1Pod) -> ContainerState:
         pod_containers = pod.status.container_statuses
 

airflow/providers/cncf/kubernetes/utils/pod_manager.py

@@ -619,12 +619,14 @@ class PodManager(LoggingMixin):
             pod_logging_statuses.append(status)
         return pod_logging_statuses
 
-    def await_container_completion(self, pod: V1Pod, container_name: str) -> None:
+    def await_container_completion(self, pod: V1Pod, container_name: str, polling_time: float = 1) -> None:
         """
         Wait for the given container in the given pod to be completed.
 
         :param pod: pod spec that will be monitored
         :param container_name: name of the container within the pod to monitor
+        :param polling_time: polling time between two container status checks.
+            Defaults to 1s.
         """
         while True:
             remote_pod = self.read_pod(pod)
@@ -632,7 +634,7 @@ class PodManager(LoggingMixin):
             if terminated:
                 break
             self.log.info("Waiting for container '%s' state to be completed", container_name)
-            time.sleep(1)
+            time.sleep(polling_time)
 
     def await_pod_completion(
         self, pod: V1Pod, istio_enabled: bool = False, container_name: str = "base"
@@ -805,26 +807,35 @@ class PodManager(LoggingMixin):
     )
     def extract_xcom_json(self, pod: V1Pod) -> str:
         """Retrieve XCom value and also check if xcom json is valid."""
+        command = (
+            f"if [ -s {PodDefaults.XCOM_MOUNT_PATH}/return.json ]; "
+            f"then cat {PodDefaults.XCOM_MOUNT_PATH}/return.json; "
+            f"else echo {EMPTY_XCOM_RESULT}; fi"
+        )
         with closing(
             kubernetes_stream(
                 self._client.connect_get_namespaced_pod_exec,
                 pod.metadata.name,
                 pod.metadata.namespace,
                 container=PodDefaults.SIDECAR_CONTAINER_NAME,
-                command=["/bin/sh"],
-                stdin=True,
+                command=[
+                    "/bin/sh",
+                    "-c",
+                    command,
+                ],
+                stdin=False,
                 stdout=True,
                 stderr=True,
                 tty=False,
                 _preload_content=False,
             )
-        ) as resp:
-            result = self._exec_pod_command(
-                resp,
-                f"if [ -s {PodDefaults.XCOM_MOUNT_PATH}/return.json ]; "
-                f"then cat {PodDefaults.XCOM_MOUNT_PATH}/return.json; "
-                f"else echo {EMPTY_XCOM_RESULT}; fi",
-            )
+        ) as client:
+            self.log.info("Running command... %s", command)
+            client.run_forever()
+            if client.peek_stderr():
+                stderr = client.read_stderr()
+                self.log.error("stderr from command: %s", stderr)
+            result = client.read_all()
             if result and result.rstrip() != EMPTY_XCOM_RESULT:
                 # Note: result string is parsed to check if its valid json.
                 # This function still returns a string which is converted into json in the calling method.

{apache_airflow_providers_cncf_kubernetes-10.3.1rc1.dist-info → apache_airflow_providers_cncf_kubernetes-10.4.0rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apache-airflow-providers-cncf-kubernetes
-Version: 10.3.1rc1
+Version: 10.4.0rc1
 Summary: Provider package apache-airflow-providers-cncf-kubernetes for Apache Airflow
 Keywords: airflow-provider,cncf.kubernetes,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -27,11 +27,11 @@ Requires-Dist: cryptography>=41.0.0
 Requires-Dist: kubernetes>=29.0.0,<=31.0.0
 Requires-Dist: kubernetes_asyncio>=29.0.0,<=31.0.0
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.3.1/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.3.1
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.4.0/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.4.0
+Project-URL: Mastodon, https://fosstodon.org/@airflow
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
-Project-URL: Twitter, https://x.com/ApacheAirflow
 Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
 
 
@@ -59,7 +59,7 @@ Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
 
 Package ``apache-airflow-providers-cncf-kubernetes``
 
-Release: ``10.3.1``
+Release: ``10.4.0``
 
 
 `Kubernetes <https://kubernetes.io/>`__
@@ -72,7 +72,7 @@ This is a provider package for ``cncf.kubernetes`` provider. All classes for thi
 are in ``airflow.providers.cncf.kubernetes`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.3.1/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.4.0/>`_.
 
 Installation
 ------------
@@ -98,5 +98,5 @@ PIP package             Version required
 ======================  =====================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.3.1/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.4.0/changelog.html>`_.
 

{apache_airflow_providers_cncf_kubernetes-10.3.1rc1.dist-info → apache_airflow_providers_cncf_kubernetes-10.4.0rc1.dist-info}/RECORD

@@ -1,8 +1,8 @@
 airflow/providers/cncf/kubernetes/LICENSE,sha256=gXPVwptPlW1TJ4HSuG5OMPg-a3h43OGMkZRR1rpwfJA,10850
-airflow/providers/cncf/kubernetes/__init__.py,sha256=dh4Bxuix8J3KKJlqsC94wTE9mXFj35O10tZtnu_B-nI,1503
+airflow/providers/cncf/kubernetes/__init__.py,sha256=J9O8rOoYu-94ZUrjAZ_noDh1WJF4nDtP6wIgWvgVpeI,1503
 airflow/providers/cncf/kubernetes/callbacks.py,sha256=5zGmQthojdT9iBEV3LIyBq-oKzjv2D4dOYCjYRbb61c,6076
 airflow/providers/cncf/kubernetes/exceptions.py,sha256=3cNEZTnrltBsqwzHiLfckwYYc_IWY1g4PcRs6zuMWWA,1137
-airflow/providers/cncf/kubernetes/get_provider_info.py,sha256=z5ETfTSpSYyt8ZQJrRSGYrWU8-gSfNx_K-fiMSH6Bl0,17748
+airflow/providers/cncf/kubernetes/get_provider_info.py,sha256=-jfmOWThKJwsuEFZbB-1zZfjfEFEDF4QQfAm0g0Y0_o,17770
 airflow/providers/cncf/kubernetes/k8s_model.py,sha256=xmdFhX29DjegoZ-cq8-KDL9soVYXf4OpU6fAGr3cPTU,2101
 airflow/providers/cncf/kubernetes/kube_client.py,sha256=yflZxLousXA9d7t67KrEy55qzb1cUhEyy6yCPkEem28,5329
 airflow/providers/cncf/kubernetes/kube_config.py,sha256=3qWdCp2z4g8gX_sIOProgwp52UxM5kAIYabkxaX297g,5079
@@ -32,7 +32,7 @@ airflow/providers/cncf/kubernetes/operators/__init__.py,sha256=mlJxuZLkd5x-iq2SB
 airflow/providers/cncf/kubernetes/operators/custom_object_launcher.py,sha256=CS8VNDq0ZBvq25AkLn90Au1L75PrWM6sdz81PNIYG0U,15315
 airflow/providers/cncf/kubernetes/operators/job.py,sha256=GXtTHoK2y7mG7rvGdHt0KIp_0V8d0wKzZ2rUV1zg56g,23773
 airflow/providers/cncf/kubernetes/operators/kueue.py,sha256=kBKw3R2IDMNOaAQ3oMNaerPBxWhCW_xJT4WxVVDhBGo,4565
-airflow/providers/cncf/kubernetes/operators/pod.py,sha256=OEyrFOc61yeoGD4AH6K6dyRM6u5gbK914bbrB8xt2W8,57108
+airflow/providers/cncf/kubernetes/operators/pod.py,sha256=RHLl00cOmnXRtGUDPqvDa4ay69c0IBDrnhTnr4mCkhE,57814
 airflow/providers/cncf/kubernetes/operators/resource.py,sha256=4nS-eiVEGlotp-gCkHlwRuenj3pnKhZ4khh9s2cjZms,7597
 airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py,sha256=z1CF8jRNVwUWBAWhP7Qgl_Wly24hBA14ONu2N9zEdFg,13847
 airflow/providers/cncf/kubernetes/pod_template_file_examples/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
@@ -47,13 +47,13 @@ airflow/providers/cncf/kubernetes/sensors/__init__.py,sha256=9hdXHABrVpkbpjZgUft
 airflow/providers/cncf/kubernetes/sensors/spark_kubernetes.py,sha256=q5VZPf037pbsunFUjUrtz7M5oJW7waZ1q8snNAugCIk,5380
 airflow/providers/cncf/kubernetes/triggers/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
 airflow/providers/cncf/kubernetes/triggers/job.py,sha256=DGbC1FZktBF-00Lb0pU9iIKQnmdW8HWklp5Wwq54OEY,6754
-airflow/providers/cncf/kubernetes/triggers/pod.py,sha256=7vLj9SvOOwh9p5XSZIr7mj47YWe_aPQgtN7uEZhma68,12653
+airflow/providers/cncf/kubernetes/triggers/pod.py,sha256=C3siy1RfpmN8dpTh1sZEanDRn37k3-hRVvh5MzxUmsc,12887
 airflow/providers/cncf/kubernetes/utils/__init__.py,sha256=ClZN0VPjWySdVwS_ktH7rrgL9VLAcs3OSJSB9s3zaYw,863
 airflow/providers/cncf/kubernetes/utils/delete_from.py,sha256=poObZSoEJwQyaYWilEURs8f4CDY2sn_pfwS31Lf579A,5195
 airflow/providers/cncf/kubernetes/utils/k8s_resource_iterator.py,sha256=DLypjkD_3YDixRTcsxEjgvHZNbbG9qamlz05eBqaWzU,1955
-airflow/providers/cncf/kubernetes/utils/pod_manager.py,sha256=JGyp9PbJZW6hgdbioodBzwTsu_jue2A0cFKYn5yWwo4,36474
+airflow/providers/cncf/kubernetes/utils/pod_manager.py,sha256=Lk4RWe1pa9KzSR4x_Mm9CFSuQwL74VNYqLHpQ5172gM,36931
 airflow/providers/cncf/kubernetes/utils/xcom_sidecar.py,sha256=k6bdmVJ21OrAwGmWwledRrAmaty9ZrmbuM-IbaI4mqo,2519
-apache_airflow_providers_cncf_kubernetes-10.3.1rc1.dist-info/entry_points.txt,sha256=ByD3QJJyP9CfmTYtpNI1953akD38RUDgpGXLaq9vpOw,111
-apache_airflow_providers_cncf_kubernetes-10.3.1rc1.dist-info/WHEEL,sha256=_2ozNFCLWc93bK4WKHCO-eDUENDlo-dgc9cU3qokYO4,82
-apache_airflow_providers_cncf_kubernetes-10.3.1rc1.dist-info/METADATA,sha256=bjIhcTachOTR_a0non6JVd3kJIC5_cnHyzikwn2XmJI,4308
-apache_airflow_providers_cncf_kubernetes-10.3.1rc1.dist-info/RECORD,,
+apache_airflow_providers_cncf_kubernetes-10.4.0rc1.dist-info/entry_points.txt,sha256=ByD3QJJyP9CfmTYtpNI1953akD38RUDgpGXLaq9vpOw,111
+apache_airflow_providers_cncf_kubernetes-10.4.0rc1.dist-info/WHEEL,sha256=_2ozNFCLWc93bK4WKHCO-eDUENDlo-dgc9cU3qokYO4,82
+apache_airflow_providers_cncf_kubernetes-10.4.0rc1.dist-info/METADATA,sha256=Sv4IRzUYFIJ9T9spK9Z1kKlwGogQpawgnHk5guioA8c,4312
+apache_airflow_providers_cncf_kubernetes-10.4.0rc1.dist-info/RECORD,,