apache-airflow-providers-cncf-kubernetes 9.0.0rc1__py3-none-any.whl → 9.0.1__py3-none-any.whl

airflow/providers/cncf/kubernetes/__init__.py

@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "9.0.0"
+__version__ = "9.0.1"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.8.0"

airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py

@@ -36,7 +36,7 @@ from queue import Empty, Queue
 from typing import TYPE_CHECKING, Any, Sequence
 
 from kubernetes.dynamic import DynamicClient
-from sqlalchemy import select, update
+from sqlalchemy import or_, select, update
 
 from airflow.cli.cli_config import (
     ARG_DAG_ID,
@@ -52,6 +52,7 @@ from airflow.cli.cli_config import (
 )
 from airflow.configuration import conf
 from airflow.executors.base_executor import BaseExecutor
+from airflow.executors.executor_constants import KUBERNETES_EXECUTOR
 from airflow.providers.cncf.kubernetes.executors.kubernetes_executor_types import (
     ADOPTED,
     POD_EXECUTOR_DONE_KEY,
@@ -229,13 +230,30 @@ class KubernetesExecutor(BaseExecutor):
             assert self.kube_client
         from airflow.models.taskinstance import TaskInstance
 
+        hybrid_executor_enabled = hasattr(TaskInstance, "executor")
+        default_executor = None
+        if hybrid_executor_enabled:
+            from airflow.executors.executor_loader import ExecutorLoader
+
+            default_executor = str(ExecutorLoader.get_default_executor_name())
+
         with Stats.timer("kubernetes_executor.clear_not_launched_queued_tasks.duration"):
             self.log.debug("Clearing tasks that have not been launched")
             query = select(TaskInstance).where(
-                TaskInstance.state == TaskInstanceState.QUEUED, TaskInstance.queued_by_job_id == self.job_id
+                TaskInstance.state == TaskInstanceState.QUEUED,
+                TaskInstance.queued_by_job_id == self.job_id,
             )
             if self.kubernetes_queue:
                 query = query.where(TaskInstance.queue == self.kubernetes_queue)
+            elif hybrid_executor_enabled and KUBERNETES_EXECUTOR == default_executor:
+                query = query.where(
+                    or_(
+                        TaskInstance.executor == KUBERNETES_EXECUTOR,
+                        TaskInstance.executor.is_(None),
+                    ),
+                )
+            elif hybrid_executor_enabled:
+                query = query.where(TaskInstance.executor == KUBERNETES_EXECUTOR)
             queued_tis: list[TaskInstance] = session.scalars(query).all()
             self.log.info("Found %s queued task instances", len(queued_tis))
 
@@ -482,6 +500,7 @@ class KubernetesExecutor(BaseExecutor):
             self.running.remove(key)
         except KeyError:
             self.log.debug("TI key not in running, not adding to event_buffer: %s", key)
+            return
 
         # If we don't have a TI state, look it up from the db. event_buffer expects the TI state
         if state is None:

airflow/providers/cncf/kubernetes/executors/kubernetes_executor_utils.py

@@ -273,16 +273,6 @@ class KubernetesJobWatcher(multiprocessing.Process, LoggingMixin):
                 (pod_name, namespace, TaskInstanceState.FAILED, annotations, resource_version)
             )
         elif status == "Succeeded":
-            # We get multiple events once the pod hits a terminal state, and we only want to
-            # send it along to the scheduler once.
-            # If our event type is DELETED, or the pod has a deletion timestamp, we've already
-            # seen the initial Succeeded event and sent it along to the scheduler.
-            if event["type"] == "DELETED" or pod.metadata.deletion_timestamp:
-                self.log.info(
-                    "Skipping event for Succeeded pod %s - event for this pod already sent to executor",
-                    pod_name,
-                )
-                return
             self.log.info("Event: %s Succeeded, annotations: %s", pod_name, annotations_string)
             self.watcher_queue.put((pod_name, namespace, None, annotations, resource_version))
         elif status == "Running":

airflow/providers/cncf/kubernetes/get_provider_info.py

@@ -28,8 +28,9 @@ def get_provider_info():
         "name": "Kubernetes",
         "description": "`Kubernetes <https://kubernetes.io/>`__\n",
         "state": "ready",
-        "source-date-epoch": 1726860352,
+        "source-date-epoch": 1730012271,
         "versions": [
+            "9.0.1",
             "9.0.0",
             "8.4.2",
             "8.4.1",
@@ -101,8 +102,8 @@ def get_provider_info():
             "apache-airflow>=2.8.0",
             "asgiref>=3.5.2",
             "cryptography>=41.0.0",
-            "kubernetes>=29.0.0,<=30.1.0",
-            "kubernetes_asyncio>=29.0.0,<=30.1.0",
+            "kubernetes>=29.0.0,<=31.0.0",
+            "kubernetes_asyncio>=29.0.0,<=31.0.0",
             "google-re2>=1.0",
         ],
         "integrations": [

airflow/providers/cncf/kubernetes/operators/pod.py

@@ -31,7 +31,7 @@ from collections.abc import Container
 from contextlib import AbstractContextManager
 from enum import Enum
 from functools import cached_property
-from typing import TYPE_CHECKING, Any, Callable, Iterable, Sequence
+from typing import TYPE_CHECKING, Any, Callable, Iterable, Literal, Sequence
 
 import kubernetes
 import tenacity
@@ -91,7 +91,6 @@ from airflow.version import version as airflow_version
 if TYPE_CHECKING:
     import jinja2
     from pendulum import DateTime
-    from typing_extensions import Literal
 
     from airflow.providers.cncf.kubernetes.secret import Secret
     from airflow.utils.context import Context
@@ -171,7 +170,7 @@ class KubernetesPodOperator(BaseOperator):
     :param affinity: affinity scheduling rules for the launched pod.
     :param config_file: The path to the Kubernetes config file. (templated)
         If not specified, default value is ``~/.kube/config``
-    :param node_selector: A dict containing a group of scheduling rules.
+    :param node_selector: A dict containing a group of scheduling rules. (templated)
     :param image_pull_secrets: Any image pull secrets to be given to the pod.
         If more than one secret is required, provide a
         comma separated list: secret_a,secret_b
@@ -259,6 +258,8 @@ class KubernetesPodOperator(BaseOperator):
         "volume_mounts",
         "cluster_context",
         "env_from",
+        "node_selector",
+        "kubernetes_conn_id",
     )
     template_fields_renderers = {"env_vars": "py"}
 
@@ -285,7 +286,8 @@ class KubernetesPodOperator(BaseOperator):
         startup_timeout_seconds: int = 120,
         startup_check_interval_seconds: int = 5,
         get_logs: bool = True,
-        container_logs: Iterable[str] | str | Literal[True] = BASE_CONTAINER_NAME,
+        base_container_name: str | None = None,
+        container_logs: Iterable[str] | str | Literal[True] | None = None,
         image_pull_policy: str | None = None,
         annotations: dict | None = None,
         container_resources: k8s.V1ResourceRequirements | None = None,
@@ -315,7 +317,6 @@ class KubernetesPodOperator(BaseOperator):
         termination_grace_period: int | None = None,
         configmaps: list[str] | None = None,
         skip_on_exit_code: int | Container[int] | None = None,
-        base_container_name: str | None = None,
         deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
         poll_interval: float = 2,
         log_pod_spec_on_failure: bool = True,
@@ -357,9 +358,10 @@ class KubernetesPodOperator(BaseOperator):
         self.cluster_context = cluster_context
         self.reattach_on_restart = reattach_on_restart
         self.get_logs = get_logs
-        self.container_logs = container_logs
-        if self.container_logs == KubernetesPodOperator.BASE_CONTAINER_NAME:
-            self.container_logs = base_container_name or self.BASE_CONTAINER_NAME
+        # Fallback to the class variable BASE_CONTAINER_NAME here instead of via default argument value
+        # in the init method signature, to be compatible with subclasses overloading the class variable value.
+        self.base_container_name = base_container_name or self.BASE_CONTAINER_NAME
+        self.container_logs = container_logs or self.base_container_name
         self.image_pull_policy = image_pull_policy
         self.node_selector = node_selector or {}
         self.annotations = annotations or {}
@@ -398,7 +400,6 @@ class KubernetesPodOperator(BaseOperator):
             if skip_on_exit_code is not None
             else []
         )
-        self.base_container_name = base_container_name or self.BASE_CONTAINER_NAME
         self.deferrable = deferrable
         self.poll_interval = poll_interval
         self.remote_pod: k8s.V1Pod | None = None
@@ -785,7 +786,7 @@ class KubernetesPodOperator(BaseOperator):
 
                     pod_log_status = self.pod_manager.fetch_container_logs(
                         pod=self.pod,
-                        container_name=self.BASE_CONTAINER_NAME,
+                        container_name=self.base_container_name,
                         follow=follow,
                         since_time=last_log_time,
                     )

airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py

@@ -69,7 +69,7 @@ class SparkKubernetesOperator(KubernetesPodOperator):
     :param kubernetes_conn_id: the connection to Kubernetes cluster
     """
 
-    template_fields = ["application_file", "namespace", "template_spec"]
+    template_fields = ["application_file", "namespace", "template_spec", "kubernetes_conn_id"]
     template_fields_renderers = {"template_spec": "py"}
     template_ext = ("yaml", "yml", "json")
     ui_color = "#f4a460"

{apache_airflow_providers_cncf_kubernetes-9.0.0rc1.dist-info → apache_airflow_providers_cncf_kubernetes-9.0.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apache-airflow-providers-cncf-kubernetes
-Version: 9.0.0rc1
+Version: 9.0.1
 Summary: Provider package apache-airflow-providers-cncf-kubernetes for Apache Airflow
 Keywords: airflow-provider,cncf.kubernetes,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -21,15 +21,15 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: System :: Monitoring
 Requires-Dist: aiofiles>=23.2.0
-Requires-Dist: apache-airflow>=2.8.0rc0
+Requires-Dist: apache-airflow>=2.8.0
 Requires-Dist: asgiref>=3.5.2
 Requires-Dist: cryptography>=41.0.0
 Requires-Dist: google-re2>=1.0
-Requires-Dist: kubernetes>=29.0.0,<=30.1.0
-Requires-Dist: kubernetes_asyncio>=29.0.0,<=30.1.0
+Requires-Dist: kubernetes>=29.0.0,<=31.0.0
+Requires-Dist: kubernetes_asyncio>=29.0.0,<=31.0.0
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/9.0.0/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/9.0.0
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/9.0.1/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/9.0.1
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Twitter, https://twitter.com/ApacheAirflow
@@ -79,7 +79,7 @@ Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
 
 Package ``apache-airflow-providers-cncf-kubernetes``
 
-Release: ``9.0.0.rc1``
+Release: ``9.0.1``
 
 
 `Kubernetes <https://kubernetes.io/>`__
@@ -92,7 +92,7 @@ This is a provider package for ``cncf.kubernetes`` provider. All classes for thi
 are in ``airflow.providers.cncf.kubernetes`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/9.0.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/9.0.1/>`_.
 
 Installation
 ------------
@@ -113,10 +113,10 @@ PIP package             Version required
 ``apache-airflow``      ``>=2.8.0``
 ``asgiref``             ``>=3.5.2``
 ``cryptography``        ``>=41.0.0``
-``kubernetes``          ``>=29.0.0,<=30.1.0``
-``kubernetes_asyncio``  ``>=29.0.0,<=30.1.0``
+``kubernetes``          ``>=29.0.0,<=31.0.0``
+``kubernetes_asyncio``  ``>=29.0.0,<=31.0.0``
 ``google-re2``          ``>=1.0``
 ======================  =====================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/9.0.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/9.0.1/changelog.html>`_.

{apache_airflow_providers_cncf_kubernetes-9.0.0rc1.dist-info → apache_airflow_providers_cncf_kubernetes-9.0.1.dist-info}/RECORD

@@ -1,7 +1,7 @@
 airflow/providers/cncf/kubernetes/LICENSE,sha256=FFb4jd2AXnOOf7XLP04pQW6jbdhG49TxlGY6fFpCV1Y,13609
-airflow/providers/cncf/kubernetes/__init__.py,sha256=VYEaCJTXEsqDXEr1WY7moklTkLqQgKhR1cWglnIlZGY,1502
+airflow/providers/cncf/kubernetes/__init__.py,sha256=p5-ifXMiybbsAM0J0xLXPmubE5pHXdZmpYD5CriC2Ek,1502
 airflow/providers/cncf/kubernetes/callbacks.py,sha256=SK_gKvGWuU-nxHfsqsYMlNQ8HZbHfpvyItOqieel2lc,4162
-airflow/providers/cncf/kubernetes/get_provider_info.py,sha256=dEbAwIRN2-XVVKuy25fpL3jQ7UN1U2VZMDc2hgIGbiE,17965
+airflow/providers/cncf/kubernetes/get_provider_info.py,sha256=B0FUhOfC6Xkj7ynRLYH2CJCEHMVNRQMStARc89kfEC4,17986
 airflow/providers/cncf/kubernetes/k8s_model.py,sha256=xmdFhX29DjegoZ-cq8-KDL9soVYXf4OpU6fAGr3cPTU,2101
 airflow/providers/cncf/kubernetes/kube_client.py,sha256=yflZxLousXA9d7t67KrEy55qzb1cUhEyy6yCPkEem28,5329
 airflow/providers/cncf/kubernetes/kube_config.py,sha256=FAmhZZ_Z2JtoVzL6wENSjcwrlwAenHttTX_Ild9aEms,5225
@@ -20,9 +20,9 @@ airflow/providers/cncf/kubernetes/cli/kubernetes_command.py,sha256=W49hsWZ3XWUJg
 airflow/providers/cncf/kubernetes/decorators/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
 airflow/providers/cncf/kubernetes/decorators/kubernetes.py,sha256=qfU_MDOhBsHc2leu1RW3YE5steDFOnWxcVswhjC6GXU,5785
 airflow/providers/cncf/kubernetes/executors/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
-airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py,sha256=leOd9r6YMkiIyGhqHfVVJXxQvkJ9UdgXppVDb94oegU,33166
+airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py,sha256=tco4nuKFjKmVA_pIsjli-NlB9DA31cq_845hD54N6mM,34006
 airflow/providers/cncf/kubernetes/executors/kubernetes_executor_types.py,sha256=9rRhfRuujGtSE73Ax0kC12whZCgWF2m6j5w9G9e0F_I,1673
-airflow/providers/cncf/kubernetes/executors/kubernetes_executor_utils.py,sha256=aGGtM7QE2cjdq3dRXh7k7c45ipnVfkjFy4SP35clJpc,23963
+airflow/providers/cncf/kubernetes/executors/kubernetes_executor_utils.py,sha256=f2FHTyQyfFR1jjQmZV9bfX7U4Vxk5h61KPJO0yMcGbM,23356
 airflow/providers/cncf/kubernetes/executors/local_kubernetes_executor.py,sha256=tlUMdInlQjw5pJTxtyQ1lBzq8HCVttNLvZsCtNtb1Io,10844
 airflow/providers/cncf/kubernetes/hooks/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
 airflow/providers/cncf/kubernetes/hooks/kubernetes.py,sha256=WSWQZrAzyrjN9y5TCS796X1L7oB-Sq_2BUzhT9xAE2k,33632
@@ -32,9 +32,9 @@ airflow/providers/cncf/kubernetes/operators/__init__.py,sha256=mlJxuZLkd5x-iq2SB
 airflow/providers/cncf/kubernetes/operators/custom_object_launcher.py,sha256=ZEXw_PqGItO47AEgGKqAqwFHFo-gb9-7jgEMRJgOfNU,15311
 airflow/providers/cncf/kubernetes/operators/job.py,sha256=RyPuT0yP7-tRrrS7uqDUBGI_xA9_YEg-JLUDJ5QcTA8,23809
 airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py,sha256=EFwHEdEqq4I18BeV9hgp_GFxrLI78ihZAccbyGLSjtY,1269
-airflow/providers/cncf/kubernetes/operators/pod.py,sha256=leGZOcEnqACp6AK5ZUt1jzP17pzGEHk6qW9wgYqwpNc,54693
+airflow/providers/cncf/kubernetes/operators/pod.py,sha256=MAHaLxy5ScgbY0LO5i3ejH3MHqYyXG3a9harRg-8guE,54803
 airflow/providers/cncf/kubernetes/operators/resource.py,sha256=ccbZQKB1B5N4Y-ruItacB5Q105Tc0uNSoEmUAaSCrGQ,7570
-airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py,sha256=tnRxHZSQTuqFQP0YFPMjuiatklkr45W3N9RPDXhf_-o,13646
+airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py,sha256=oi1JUCoRo7AzAOYhKgbCB60Hmg_slRdj3FsQKWQRhcI,13668
 airflow/providers/cncf/kubernetes/pod_template_file_examples/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
 airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_image_template.yaml,sha256=7JdppZ-XDBpv2Bnde2SthhcME8w3b8xQdPAK1fJGW60,2256
 airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_volume_template.yaml,sha256=-Pk_EwKpyWRYZKOnumUxVrDeAfFJ0nr3WZ7JNnvppzg,2442
@@ -54,7 +54,7 @@ airflow/providers/cncf/kubernetes/utils/delete_from.py,sha256=poObZSoEJwQyaYWilE
 airflow/providers/cncf/kubernetes/utils/k8s_resource_iterator.py,sha256=-Pgc5i2WEDl7ZBvtJZ4eWDqqlSj8WdULqwUyOWmsRp8,1928
 airflow/providers/cncf/kubernetes/utils/pod_manager.py,sha256=eITiFHU8lsCy3UwRUm-3uOQ3J1BgkshaXd1FJOJ3Q4A,34298
 airflow/providers/cncf/kubernetes/utils/xcom_sidecar.py,sha256=k6bdmVJ21OrAwGmWwledRrAmaty9ZrmbuM-IbaI4mqo,2519
-apache_airflow_providers_cncf_kubernetes-9.0.0rc1.dist-info/entry_points.txt,sha256=ByD3QJJyP9CfmTYtpNI1953akD38RUDgpGXLaq9vpOw,111
-apache_airflow_providers_cncf_kubernetes-9.0.0rc1.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
-apache_airflow_providers_cncf_kubernetes-9.0.0rc1.dist-info/METADATA,sha256=u4HfGJAdX7Q-DaBDqFyt54JzRaf9e9A3PvZMT6TtAtY,5216
-apache_airflow_providers_cncf_kubernetes-9.0.0rc1.dist-info/RECORD,,
+apache_airflow_providers_cncf_kubernetes-9.0.1.dist-info/entry_points.txt,sha256=ByD3QJJyP9CfmTYtpNI1953akD38RUDgpGXLaq9vpOw,111
+apache_airflow_providers_cncf_kubernetes-9.0.1.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
+apache_airflow_providers_cncf_kubernetes-9.0.1.dist-info/METADATA,sha256=TcqzgCgCkgrxxFlTSpaaOjhIiN747VjmE1fVa4hlTQY,5206
+apache_airflow_providers_cncf_kubernetes-9.0.1.dist-info/RECORD,,