apache-airflow-providers-cncf-kubernetes 10.5.0rc1__py3-none-any.whl → 10.6.0rc1__py3-none-any.whl

airflow/providers/cncf/kubernetes/__init__.py

@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "10.5.0"
+__version__ = "10.6.0"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.10.0"

airflow/providers/cncf/kubernetes/cli/kubernetes_command.py

@@ -26,16 +26,17 @@ from kubernetes import client
 from kubernetes.client.api_client import ApiClient
 from kubernetes.client.rest import ApiException
 
-from airflow.models import DagRun, TaskInstance
+from airflow.models import DagModel, DagRun, TaskInstance
 from airflow.providers.cncf.kubernetes import pod_generator
 from airflow.providers.cncf.kubernetes.executors.kubernetes_executor import KubeConfig
 from airflow.providers.cncf.kubernetes.kube_client import get_kube_client
 from airflow.providers.cncf.kubernetes.kubernetes_helper_functions import create_unique_id
-from airflow.providers.cncf.kubernetes.pod_generator import PodGenerator
+from airflow.providers.cncf.kubernetes.pod_generator import PodGenerator, generate_pod_command_args
 from airflow.providers.cncf.kubernetes.version_compat import AIRFLOW_V_3_0_PLUS
 from airflow.utils import cli as cli_utils, yaml
 from airflow.utils.cli import get_dag
 from airflow.utils.providers_configuration_loader import providers_configuration_loaded
+from airflow.utils.types import DagRunType
 
 
 @cli_utils.action_cli
@@ -48,14 +49,28 @@ def generate_pod_yaml(args):
     else:
         dag = get_dag(subdir=args.subdir, dag_id=args.dag_id)
     yaml_output_path = args.output_path
+
+    dm = DagModel(dag_id=dag.dag_id)
+
     if AIRFLOW_V_3_0_PLUS:
         dr = DagRun(dag.dag_id, logical_date=logical_date)
+        dr.run_id = DagRun.generate_run_id(
+            run_type=DagRunType.MANUAL, logical_date=logical_date, run_after=logical_date
+        )
+        dm.bundle_name = args.bundle_name if args.bundle_name else "default"
+        dm.relative_fileloc = dag.relative_fileloc
     else:
         dr = DagRun(dag.dag_id, execution_date=logical_date)
+        dr.run_id = DagRun.generate_run_id(run_type=DagRunType.MANUAL, execution_date=logical_date)
+
     kube_config = KubeConfig()
+
     for task in dag.tasks:
-        ti = TaskInstance(task, None)
+        ti = TaskInstance(task, run_id=dr.run_id)
         ti.dag_run = dr
+        ti.dag_model = dm
+
+        command_args = generate_pod_command_args(ti)
         pod = PodGenerator.construct_pod(
             dag_id=args.dag_id,
             task_id=ti.task_id,
@@ -63,7 +78,7 @@ def generate_pod_yaml(args):
             try_number=ti.try_number,
             kube_image=kube_config.kube_image,
             date=ti.logical_date if AIRFLOW_V_3_0_PLUS else ti.execution_date,
-            args=ti.command_as_list(),
+            args=command_args,
             pod_override_object=PodGenerator.from_obj(ti.executor_config),
             scheduler_job_id="worker-config",
             namespace=kube_config.executor_namespace,

airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py

@@ -76,13 +76,13 @@ from airflow.utils.state import TaskInstanceState
 
 if TYPE_CHECKING:
     import argparse
+    from collections.abc import Sequence
 
     from kubernetes import client
     from kubernetes.client import models as k8s
     from sqlalchemy.orm import Session
 
     from airflow.executors import workloads
-    from airflow.executors.base_executor import CommandType
     from airflow.models.taskinstance import TaskInstance
     from airflow.models.taskinstancekey import TaskInstanceKey
     from airflow.providers.cncf.kubernetes.executors.kubernetes_executor_types import (
@@ -254,7 +254,7 @@ class KubernetesExecutor(BaseExecutor):
     def execute_async(
         self,
         key: TaskInstanceKey,
-        command: CommandType,
+        command: Any,
         queue: str | None = None,
         executor_config: Any | None = None,
     ) -> None:
@@ -292,7 +292,7 @@ class KubernetesExecutor(BaseExecutor):
         ti = workload.ti
         self.queued_tasks[ti.key] = workload
 
-    def _process_workloads(self, workloads: list[workloads.All]) -> None:
+    def _process_workloads(self, workloads: Sequence[workloads.All]) -> None:
         from airflow.executors.workloads import ExecuteTask
 
         # Airflow V3 version

airflow/providers/cncf/kubernetes/executors/kubernetes_executor_types.py

@@ -20,10 +20,14 @@ from typing import TYPE_CHECKING, Any, Optional, Union
 
 ADOPTED = "adopted"
 if TYPE_CHECKING:
-    from airflow.executors.base_executor import CommandType
+    from collections.abc import Sequence
+
     from airflow.models.taskinstance import TaskInstanceKey
     from airflow.utils.state import TaskInstanceState
 
+    # TODO: Remove after Airflow 2 support is removed
+    CommandType = Sequence[str]
+
     # TaskInstance key, command, configuration, pod_template_file
     KubernetesJobType = tuple[TaskInstanceKey, CommandType, Any, Optional[str]]
 

airflow/providers/cncf/kubernetes/executors/kubernetes_executor_utils.py

@@ -41,7 +41,7 @@ from airflow.providers.cncf.kubernetes.kubernetes_helper_functions import (
     annotations_to_key,
     create_unique_id,
 )
-from airflow.providers.cncf.kubernetes.pod_generator import PodGenerator
+from airflow.providers.cncf.kubernetes.pod_generator import PodGenerator, workload_to_command_args
 from airflow.utils.log.logging_mixin import LoggingMixin
 from airflow.utils.singleton import Singleton
 from airflow.utils.state import TaskInstanceState
@@ -387,20 +387,12 @@ class AirflowKubernetesScheduler(LoggingMixin):
         key, command, kube_executor_config, pod_template_file = next_job
 
         dag_id, task_id, run_id, try_number, map_index = key
-        ser_input = ""
         if len(command) == 1:
             from airflow.executors.workloads import ExecuteTask
 
             if isinstance(command[0], ExecuteTask):
                 workload = command[0]
-                ser_input = workload.model_dump_json()
-                command = [
-                    "python",
-                    "-m",
-                    "airflow.sdk.execution_time.execute_workload",
-                    "--json-path",
-                    "/tmp/execute/input.json",
-                ]
+                command = workload_to_command_args(workload)
             else:
                 raise ValueError(
                     f"KubernetesExecutor doesn't know how to handle workload of type: {type(command[0])}"
@@ -427,7 +419,6 @@ class AirflowKubernetesScheduler(LoggingMixin):
             date=None,
             run_id=run_id,
             args=list(command),
-            content_json_for_volume=ser_input,
             pod_override_object=kube_executor_config,
             base_worker_pod=base_worker_pod,
             with_mutation_hook=True,

airflow/providers/cncf/kubernetes/executors/local_kubernetes_executor.py

@@ -18,7 +18,7 @@
 from __future__ import annotations
 
 from collections.abc import Sequence
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 from deprecated import deprecated
 
@@ -26,18 +26,17 @@ from airflow.configuration import conf
 from airflow.exceptions import AirflowProviderDeprecationWarning
 from airflow.executors.base_executor import BaseExecutor
 from airflow.providers.cncf.kubernetes.executors.kubernetes_executor import KubernetesExecutor
+from airflow.providers.cncf.kubernetes.version_compat import AIRFLOW_V_3_0_PLUS
 
 if TYPE_CHECKING:
     from airflow.callbacks.base_callback_sink import BaseCallbackSink
     from airflow.callbacks.callback_requests import CallbackRequest
-    from airflow.executors.base_executor import (
-        CommandType,
-        EventBufferValueType,
-        QueuedTaskInstanceType,
-    )
+    from airflow.executors.base_executor import EventBufferValueType
     from airflow.executors.local_executor import LocalExecutor
     from airflow.models.taskinstance import SimpleTaskInstance, TaskInstance, TaskInstanceKey
 
+    CommandType = Sequence[str]
+
 
 class LocalKubernetesExecutor(BaseExecutor):
     """
@@ -81,7 +80,7 @@ class LocalKubernetesExecutor(BaseExecutor):
         """Not implemented for hybrid executors."""
 
     @property
-    def queued_tasks(self) -> dict[TaskInstanceKey, QueuedTaskInstanceType]:
+    def queued_tasks(self) -> dict[TaskInstanceKey, Any]:
         """Return queued tasks from local and kubernetes executor."""
         queued_tasks = self.local_executor.queued_tasks.copy()
         # TODO: fix this, there is misalignment between the types of queued_tasks so it is likely wrong
@@ -121,6 +120,13 @@ class LocalKubernetesExecutor(BaseExecutor):
 
     def start(self) -> None:
         """Start local and kubernetes executor."""
+        if AIRFLOW_V_3_0_PLUS:
+            raise RuntimeError(
+                f"{self.__class__.__name__} does not support Airflow 3.0+. See "
+                "https://airflow.apache.org/docs/apache-airflow/stable/core-concepts/executor/index.html#using-multiple-executors-concurrently"
+                " how to use multiple executors concurrently."
+            )
+
         self.log.info("Starting local and Kubernetes Executor")
         self.local_executor.start()
         self.kubernetes_executor.start()
@@ -145,7 +151,7 @@ class LocalKubernetesExecutor(BaseExecutor):
         """Queues command via local or kubernetes executor."""
         executor = self._router(task_instance)
         self.log.debug("Using executor: %s for %s", executor.__class__.__name__, task_instance.key)
-        executor.queue_command(task_instance, command, priority, queue)
+        executor.queue_command(task_instance, command, priority, queue)  # type: ignore[union-attr]
 
     def queue_task_instance(
         self,
@@ -171,7 +177,7 @@ class LocalKubernetesExecutor(BaseExecutor):
         if not hasattr(task_instance, "pickle_id"):
             del kwargs["pickle_id"]
 
-        executor.queue_task_instance(
+        executor.queue_task_instance(  # type: ignore[union-attr]
             task_instance=task_instance,
             mark_success=mark_success,
             ignore_all_deps=ignore_all_deps,

airflow/providers/cncf/kubernetes/get_provider_info.py

@@ -169,7 +169,7 @@ def get_provider_info():
                         "version_added": "8.1.0",
                         "type": "string",
                         "example": None,
-                        "default": "CreateContainerConfigError,ErrImagePull,CreateContainerError,ImageInspectError, InvalidImageName",
+                        "default": "CreateContainerConfigError,ErrImagePull,CreateContainerError,ImageInspectError,InvalidImageName",
                     },
                     "worker_pods_creation_batch_size": {
                         "description": 'Number of Kubernetes Worker Pod creation calls per scheduler loop.\nNote that the current default of "1" will only launch a single pod\nper-heartbeat. It is HIGHLY recommended that users increase this\nnumber to match the tolerance of their kubernetes cluster for\nbetter performance.\n',

airflow/providers/cncf/kubernetes/kube_config.py

@@ -42,9 +42,12 @@ class KubeConfig:
         )
         self.worker_pod_pending_fatal_container_state_reasons = []
         if conf.get(self.kubernetes_section, "worker_pod_pending_fatal_container_state_reasons", fallback=""):
-            self.worker_pod_pending_fatal_container_state_reasons = conf.get(
-                self.kubernetes_section, "worker_pod_pending_fatal_container_state_reasons"
-            ).split(",")
+            self.worker_pod_pending_fatal_container_state_reasons = [
+                r.strip()
+                for r in conf.get(
+                    self.kubernetes_section, "worker_pod_pending_fatal_container_state_reasons"
+                ).split(",")
+            ]
 
         self.worker_pods_creation_batch_size = conf.getint(
             self.kubernetes_section, "worker_pods_creation_batch_size"

airflow/providers/cncf/kubernetes/operators/pod.py

@@ -18,6 +18,7 @@
 
 from __future__ import annotations
 
+import asyncio
 import datetime
 import json
 import logging
@@ -83,6 +84,7 @@ from airflow.providers.cncf.kubernetes.utils.pod_manager import (
 from airflow.settings import pod_mutation_hook
 from airflow.utils import yaml
 from airflow.utils.helpers import prune_dict, validate_key
+from airflow.utils.xcom import XCOM_RETURN_KEY
 from airflow.version import version as airflow_version
 
 if TYPE_CHECKING:
@@ -233,6 +235,7 @@ class KubernetesPodOperator(BaseOperator):
     :param logging_interval: max time in seconds that task should be in deferred state before
         resuming to fetch the latest logs. If ``None``, then the task will remain in deferred state until pod
         is done, and no logs will be visible until that time.
+    :param trigger_kwargs: additional keyword parameters passed to the trigger
     """
 
     # !!! Changes in KubernetesPodOperator's arguments should be also reflected in !!!
@@ -266,6 +269,7 @@ class KubernetesPodOperator(BaseOperator):
         "node_selector",
         "kubernetes_conn_id",
         "base_container_name",
+        "trigger_kwargs",
     )
     template_fields_renderers = {"env_vars": "py"}
 
@@ -305,7 +309,7 @@ class KubernetesPodOperator(BaseOperator):
         node_selector: dict | None = None,
         image_pull_secrets: list[k8s.V1LocalObjectReference] | None = None,
         service_account_name: str | None = None,
-        automount_service_account_token: bool = True,
+        automount_service_account_token: bool | None = None,
         hostnetwork: bool = False,
         host_aliases: list[k8s.V1HostAlias] | None = None,
         tolerations: list[k8s.V1Toleration] | None = None,
@@ -339,6 +343,7 @@ class KubernetesPodOperator(BaseOperator):
         ) = None,
         progress_callback: Callable[[str], None] | None = None,
         logging_interval: int | None = None,
+        trigger_kwargs: dict | None = None,
         **kwargs,
     ) -> None:
         super().__init__(**kwargs)
@@ -428,6 +433,7 @@ class KubernetesPodOperator(BaseOperator):
         self.termination_message_policy = termination_message_policy
         self.active_deadline_seconds = active_deadline_seconds
         self.logging_interval = logging_interval
+        self.trigger_kwargs = trigger_kwargs
 
         self._config_dict: dict | None = None  # TODO: remove it when removing convert_config_file_to_dict
         self._progress_callback = progress_callback
@@ -572,19 +578,46 @@ class KubernetesPodOperator(BaseOperator):
         if self.reattach_on_restart:
             pod = self.find_pod(pod_request_obj.metadata.namespace, context=context)
             if pod:
-                return pod
+                # If pod is terminated then delete the pod an create a new as not possible to get xcom
+                pod_phase = (
+                    pod.status.phase if hasattr(pod, "status") and hasattr(pod.status, "phase") else None
+                )
+                if pod_phase and pod_phase not in (PodPhase.SUCCEEDED, PodPhase.FAILED):
+                    return pod
+
+                self.log.info(
+                    "Found terminated old matching pod %s with labels %s",
+                    pod.metadata.name,
+                    pod.metadata.labels,
+                )
+
+                # if not required to delete the pod then keep old logic and not automatically create new pod
+                deleted_pod = self.process_pod_deletion(pod)
+                if not deleted_pod:
+                    return pod
+
+                self.log.info("Deleted pod to handle rerun and create new pod!")
+
         self.log.debug("Starting pod:\n%s", yaml.safe_dump(pod_request_obj.to_dict()))
         self.pod_manager.create_pod(pod=pod_request_obj)
         return pod_request_obj
 
     def await_pod_start(self, pod: k8s.V1Pod) -> None:
         try:
-            self.pod_manager.await_pod_start(
-                pod=pod,
-                schedule_timeout=self.schedule_timeout_seconds,
-                startup_timeout=self.startup_timeout_seconds,
-                check_interval=self.startup_check_interval_seconds,
+            loop = asyncio.get_event_loop()
+            events_task = asyncio.ensure_future(
+                self.pod_manager.watch_pod_events(pod, self.startup_check_interval_seconds)
             )
+            loop.run_until_complete(
+                self.pod_manager.await_pod_start(
+                    pod=pod,
+                    schedule_timeout=self.schedule_timeout_seconds,
+                    startup_timeout=self.startup_timeout_seconds,
+                    check_interval=self.startup_check_interval_seconds,
+                )
+            )
+            loop.run_until_complete(events_task)
+            loop.close()
         except PodLaunchFailedException:
             if self.log_events_on_failure:
                 self._read_pod_events(pod, reraise=False)
@@ -688,6 +721,8 @@ class KubernetesPodOperator(BaseOperator):
             self.cleanup(
                 pod=pod_to_clean,
                 remote_pod=self.remote_pod,
+                xcom_result=result,
+                context=context,
             )
             for callback in self.callbacks:
                 callback.on_pod_cleanup(
@@ -812,6 +847,7 @@ class KubernetesPodOperator(BaseOperator):
                 on_finish_action=self.on_finish_action.value,
                 last_log_time=last_log_time,
                 logging_interval=self.logging_interval,
+                trigger_kwargs=self.trigger_kwargs,
             ),
             method_name="trigger_reentry",
         )
@@ -958,7 +994,13 @@ class KubernetesPodOperator(BaseOperator):
                 pod=pod, client=self.client, mode=ExecutionMode.SYNC, operator=self, context=context
             )
 
-    def cleanup(self, pod: k8s.V1Pod, remote_pod: k8s.V1Pod):
+    def cleanup(
+        self,
+        pod: k8s.V1Pod,
+        remote_pod: k8s.V1Pod,
+        xcom_result: dict | None = None,
+        context: Context | None = None,
+    ) -> None:
         # Skip cleaning the pod in the following scenarios.
         # 1. If a task got marked as failed, "on_kill" method would be called and the pod will be cleaned up
         # there. Cleaning it up again will raise an exception (which might cause retry).
@@ -978,6 +1020,10 @@ class KubernetesPodOperator(BaseOperator):
         )
 
         if failed:
+            if self.do_xcom_push and xcom_result and context:
+                # Ensure that existing XCom is pushed even in case of failure
+                context["ti"].xcom_push(XCOM_RETURN_KEY, xcom_result)
+
             if self.log_events_on_failure:
                 self._read_pod_events(pod, reraise=False)
 
@@ -1062,7 +1108,7 @@ class KubernetesPodOperator(BaseOperator):
         if self.KILL_ISTIO_PROXY_SUCCESS_MSG not in output_str:
             raise AirflowException("Error while deleting istio-proxy sidecar: %s", output_str)
 
-    def process_pod_deletion(self, pod: k8s.V1Pod, *, reraise=True):
+    def process_pod_deletion(self, pod: k8s.V1Pod, *, reraise=True) -> bool:
         with _optionally_suppress(reraise=reraise):
             if pod is not None:
                 should_delete_pod = (self.on_finish_action == OnFinishAction.DELETE_POD) or (
@@ -1075,8 +1121,10 @@ class KubernetesPodOperator(BaseOperator):
                 if should_delete_pod:
                     self.log.info("Deleting pod: %s", pod.metadata.name)
                     self.pod_manager.delete_pod(pod)
-                else:
-                    self.log.info("Skipping deleting pod: %s", pod.metadata.name)
+                    return True
+                self.log.info("Skipping deleting pod: %s", pod.metadata.name)
+
+        return False
 
     def _build_find_pod_label_selector(self, context: Context | None = None, *, exclude_checked=True) -> str:
         labels = {

airflow/providers/cncf/kubernetes/pod_generator.py

@@ -34,7 +34,7 @@ from functools import reduce
 from typing import TYPE_CHECKING
 
 from dateutil import parser
-from kubernetes.client import V1EmptyDirVolumeSource, V1Volume, V1VolumeMount, models as k8s
+from kubernetes.client import models as k8s
 from kubernetes.client.api_client import ApiClient
 
 from airflow.exceptions import (
@@ -46,6 +46,7 @@ from airflow.providers.cncf.kubernetes.kubernetes_helper_functions import (
     POD_NAME_MAX_LENGTH,
     add_unique_suffix,
 )
+from airflow.providers.cncf.kubernetes.version_compat import AIRFLOW_V_3_0_PLUS
 from airflow.utils import yaml
 from airflow.utils.hashlib_wrapper import md5
 from airflow.version import version as airflow_version
@@ -53,11 +54,49 @@ from airflow.version import version as airflow_version
 if TYPE_CHECKING:
     import datetime
 
+    from airflow.executors import workloads
+    from airflow.models.taskinstance import TaskInstance
+
 log = logging.getLogger(__name__)
 
 MAX_LABEL_LEN = 63
 
 
+def workload_to_command_args(workload: workloads.ExecuteTask) -> list[str]:
+    """
+    Convert a workload object to Task SDK command arguments.
+
+    :param workload: The ExecuteTask workload to convert
+    :return: List of command arguments for the Task SDK
+    """
+    ser_input = workload.model_dump_json()
+    return [
+        "python",
+        "-m",
+        "airflow.sdk.execution_time.execute_workload",
+        "--json-string",
+        ser_input,
+    ]
+
+
+def generate_pod_command_args(task_instance: TaskInstance) -> list[str]:
+    """
+    Generate command arguments for a ``TaskInstance`` to be used in a Kubernetes pod.
+
+    This function handles backwards compatibility between Airflow 2.x and 3.x:
+    - In Airflow 2.x: Uses the existing ``command_as_list()`` method
+    - In Airflow 3.x: Uses the Task SDK workload approach with serialized workload
+    """
+    if AIRFLOW_V_3_0_PLUS:
+        # In Airflow 3+, use the Task SDK workload approach
+        from airflow.executors import workloads
+
+        workload = workloads.ExecuteTask.make(task_instance)
+        return workload_to_command_args(workload)
+    # In Airflow 2.x, use the existing method
+    return task_instance.command_as_list()
+
+
 def make_safe_label_value(string: str) -> str:
     """
     Normalize a provided label to be of valid length and characters.
@@ -287,7 +326,6 @@ class PodGenerator:
         scheduler_job_id: str,
         run_id: str | None = None,
         map_index: int = -1,
-        content_json_for_volume: str = "",
         *,
         with_mutation_hook: bool = False,
     ) -> k8s.V1Pod:
@@ -355,39 +393,6 @@ class PodGenerator:
             containers=[main_container],
         )
 
-        if content_json_for_volume:
-            import shlex
-
-            input_file_path = "/tmp/execute/input.json"
-            execute_volume = V1Volume(
-                name="execute-volume",
-                empty_dir=V1EmptyDirVolumeSource(),
-            )
-
-            execute_volume_mount = V1VolumeMount(
-                name="execute-volume",
-                mount_path="/tmp/execute",
-                read_only=False,
-            )
-
-            escaped_json = shlex.quote(content_json_for_volume)
-            init_container = k8s.V1Container(
-                name="init-container",
-                image="busybox",
-                command=["/bin/sh", "-c", f"echo {escaped_json} > {input_file_path}"],
-                volume_mounts=[execute_volume_mount],
-            )
-
-            main_container.volume_mounts = [execute_volume_mount]
-            main_container.command = args[:-1]
-            main_container.args = args[-1:]
-
-            podspec = k8s.V1PodSpec(
-                containers=[main_container],
-                volumes=[execute_volume],
-                init_containers=[init_container],
-            )
-
         dynamic_pod.spec = podspec
 
         # Reconcile the pods starting with the first chronologically,
@@ -546,7 +551,7 @@ def merge_objects(base_obj, client_obj):
 
     for base_key in base_obj.to_dict():
         base_val = getattr(base_obj, base_key, None)
-        if not getattr(client_obj, base_key, None) and base_val:
+        if not getattr(client_obj, base_key, None) and base_val is not None:
             if not isinstance(client_obj_cp, dict):
                 setattr(client_obj_cp, base_key, base_val)
             else:

airflow/providers/cncf/kubernetes/template_rendering.py

@@ -19,11 +19,14 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING
 
+from jinja2 import TemplateAssertionError, UndefinedError
 from kubernetes.client.api_client import ApiClient
 
+from airflow.exceptions import AirflowException
 from airflow.providers.cncf.kubernetes.kube_config import KubeConfig
 from airflow.providers.cncf.kubernetes.kubernetes_helper_functions import create_unique_id
-from airflow.providers.cncf.kubernetes.pod_generator import PodGenerator
+from airflow.providers.cncf.kubernetes.pod_generator import PodGenerator, generate_pod_command_args
+from airflow.utils.session import NEW_SESSION, provide_session
 
 if TYPE_CHECKING:
     from airflow.models.taskinstance import TaskInstance
@@ -40,6 +43,10 @@ def render_k8s_pod_yaml(task_instance: TaskInstance) -> dict | None:
         # If no such pod_template_file override was passed, we can simply render
         # The pod spec using the default template.
         pod_template_file = kube_config.pod_template_file
+
+    # Generate command args using shared utility function
+    command_args = generate_pod_command_args(task_instance)
+
     pod = PodGenerator.construct_pod(
         dag_id=task_instance.dag_id,
         run_id=task_instance.run_id,
@@ -49,7 +56,7 @@ def render_k8s_pod_yaml(task_instance: TaskInstance) -> dict | None:
         pod_id=create_unique_id(task_instance.dag_id, task_instance.task_id),
         try_number=task_instance.try_number,
         kube_image=kube_config.kube_image,
-        args=task_instance.command_as_list(),
+        args=command_args,
         pod_override_object=PodGenerator.from_obj(task_instance.executor_config),
         scheduler_job_id="0",
         namespace=kube_config.executor_namespace,
@@ -58,3 +65,17 @@ def render_k8s_pod_yaml(task_instance: TaskInstance) -> dict | None:
     )
     sanitized_pod = ApiClient().sanitize_for_serialization(pod)
     return sanitized_pod
+
+
+@provide_session
+def get_rendered_k8s_spec(task_instance: TaskInstance, session=NEW_SESSION) -> dict | None:
+    """Fetch rendered template fields from DB."""
+    from airflow.models.renderedtifields import RenderedTaskInstanceFields
+
+    rendered_k8s_spec = RenderedTaskInstanceFields.get_k8s_pod_yaml(task_instance, session=session)
+    if not rendered_k8s_spec:
+        try:
+            rendered_k8s_spec = render_k8s_pod_yaml(task_instance)
+        except (TemplateAssertionError, UndefinedError) as e:
+            raise AirflowException(f"Unable to render a k8s spec for this taskinstance: {e}") from e
+    return rendered_k8s_spec

airflow/providers/cncf/kubernetes/triggers/pod.py

@@ -75,6 +75,7 @@ class KubernetesPodTrigger(BaseTrigger):
     :param logging_interval: number of seconds to wait before kicking it back to
         the operator to print latest logs. If ``None`` will wait until container done.
     :param last_log_time: where to resume logs from
+    :param trigger_kwargs: additional keyword parameters to send in the event
     """
 
     def __init__(
@@ -94,6 +95,7 @@ class KubernetesPodTrigger(BaseTrigger):
         on_finish_action: str = "delete_pod",
         last_log_time: DateTime | None = None,
         logging_interval: int | None = None,
+        trigger_kwargs: dict | None = None,
     ):
         super().__init__()
         self.pod_name = pod_name
@@ -111,6 +113,7 @@ class KubernetesPodTrigger(BaseTrigger):
         self.last_log_time = last_log_time
         self.logging_interval = logging_interval
         self.on_finish_action = OnFinishAction(on_finish_action)
+        self.trigger_kwargs = trigger_kwargs or {}
 
         self._since_time = None
 
@@ -134,6 +137,7 @@ class KubernetesPodTrigger(BaseTrigger):
                 "on_finish_action": self.on_finish_action.value,
                 "last_log_time": self.last_log_time,
                 "logging_interval": self.logging_interval,
+                "trigger_kwargs": self.trigger_kwargs,
             },
         )
 
@@ -149,6 +153,7 @@ class KubernetesPodTrigger(BaseTrigger):
                         "namespace": self.pod_namespace,
                         "name": self.pod_name,
                         "message": "All containers inside pod have started successfully.",
+                        **self.trigger_kwargs,
                     }
                 )
             elif state == ContainerState.FAILED:
@@ -158,6 +163,7 @@ class KubernetesPodTrigger(BaseTrigger):
                         "namespace": self.pod_namespace,
                         "name": self.pod_name,
                         "message": "pod failed",
+                        **self.trigger_kwargs,
                     }
                 )
             else:
@@ -172,6 +178,7 @@ class KubernetesPodTrigger(BaseTrigger):
                     "namespace": self.pod_namespace,
                     "status": "timeout",
                     "message": message,
+                    **self.trigger_kwargs,
                 }
             )
             return
@@ -183,6 +190,7 @@ class KubernetesPodTrigger(BaseTrigger):
                     "status": "error",
                     "message": str(e),
                     "stack_trace": traceback.format_exc(),
+                    **self.trigger_kwargs,
                 }
             )
             return
@@ -234,6 +242,7 @@ class KubernetesPodTrigger(BaseTrigger):
                         "namespace": self.pod_namespace,
                         "name": self.pod_name,
                         "last_log_time": self.last_log_time,
+                        **self.trigger_kwargs,
                     }
                 )
             if container_state == ContainerState.FAILED:
@@ -244,6 +253,7 @@ class KubernetesPodTrigger(BaseTrigger):
                         "name": self.pod_name,
                         "message": "Container state failed",
                         "last_log_time": self.last_log_time,
+                        **self.trigger_kwargs,
                     }
                 )
             self.log.debug("Container is not completed and still working.")
@@ -254,6 +264,7 @@ class KubernetesPodTrigger(BaseTrigger):
                         "last_log_time": self.last_log_time,
                         "namespace": self.pod_namespace,
                         "name": self.pod_name,
+                        **self.trigger_kwargs,
                     }
                 )
             self.log.debug("Sleeping for %s seconds.", self.poll_interval)

airflow/providers/cncf/kubernetes/utils/pod_manager.py

@@ -18,6 +18,7 @@
 
 from __future__ import annotations
 
+import asyncio
 import enum
 import json
 import math
@@ -49,6 +50,7 @@ if TYPE_CHECKING:
     from kubernetes.client.models.v1_container_state import V1ContainerState
     from kubernetes.client.models.v1_container_state_waiting import V1ContainerStateWaiting
     from kubernetes.client.models.v1_container_status import V1ContainerStatus
+    from kubernetes.client.models.v1_object_reference import V1ObjectReference
     from kubernetes.client.models.v1_pod import V1Pod
     from kubernetes.client.models.v1_pod_condition import V1PodCondition
     from urllib3.response import HTTPResponse
@@ -377,7 +379,21 @@ class PodManager(LoggingMixin):
         """Launch the pod asynchronously."""
         return self.run_pod_async(pod)
 
-    def await_pod_start(
+    async def watch_pod_events(self, pod: V1Pod, check_interval: int = 1) -> None:
+        """Read pod events and writes into log."""
+        self.keep_watching_for_events = True
+        num_events = 0
+        while self.keep_watching_for_events:
+            events = self.read_pod_events(pod)
+            for new_event in events.items[num_events:]:
+                involved_object: V1ObjectReference = new_event.involved_object
+                self.log.info(
+                    "The Pod has an Event: %s from %s", new_event.message, involved_object.field_path
+                )
+            num_events = len(events.items)
+            await asyncio.sleep(check_interval)
+
+    async def await_pod_start(
         self, pod: V1Pod, schedule_timeout: int = 120, startup_timeout: int = 120, check_interval: int = 1
     ) -> None:
         """
@@ -439,7 +455,7 @@ class PodManager(LoggingMixin):
                                 f"\n{container_waiting.message}"
                             )
 
-            time.sleep(check_interval)
+            await asyncio.sleep(check_interval)
 
     def fetch_container_logs(
         self,
@@ -822,6 +838,10 @@ class PodManager(LoggingMixin):
             if self.container_is_running(pod, PodDefaults.SIDECAR_CONTAINER_NAME):
                 self.log.info("The xcom sidecar container has started.")
                 break
+            if self.container_is_terminated(pod, PodDefaults.SIDECAR_CONTAINER_NAME):
+                raise AirflowException(
+                    "Xcom sidecar container is already terminated! Not possible to read xcom output of task."
+                )
             if (time.time() - last_log_time) >= log_interval:
                 self.log.warning(
                     "Still waiting for the xcom sidecar container to start. Elapsed time: %d seconds.",

{apache_airflow_providers_cncf_kubernetes-10.5.0rc1.dist-info → apache_airflow_providers_cncf_kubernetes-10.6.0rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apache-airflow-providers-cncf-kubernetes
-Version: 10.5.0rc1
+Version: 10.6.0rc1
 Summary: Provider package apache-airflow-providers-cncf-kubernetes for Apache Airflow
 Keywords: airflow-provider,cncf.kubernetes,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -24,11 +24,11 @@ Requires-Dist: aiofiles>=23.2.0
 Requires-Dist: apache-airflow>=2.10.0rc1
 Requires-Dist: asgiref>=3.5.2
 Requires-Dist: cryptography>=41.0.0
-Requires-Dist: kubernetes>=29.0.0,<=31.0.0
-Requires-Dist: kubernetes_asyncio>=29.0.0,<=31.0.0
+Requires-Dist: kubernetes>=32.0.0,<33.0.0
+Requires-Dist: kubernetes_asyncio>=32.0.0,<33.0.0
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.staged.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.5.0/changelog.html
-Project-URL: Documentation, https://airflow.staged.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.5.0
+Project-URL: Changelog, https://airflow.staged.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.6.0/changelog.html
+Project-URL: Documentation, https://airflow.staged.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.6.0
 Project-URL: Mastodon, https://fosstodon.org/@airflow
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
@@ -59,7 +59,7 @@ Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
 
 Package ``apache-airflow-providers-cncf-kubernetes``
 
-Release: ``10.5.0``
+Release: ``10.6.0``
 
 
 `Kubernetes <https://kubernetes.io/>`__
@@ -72,7 +72,7 @@ This is a provider package for ``cncf.kubernetes`` provider. All classes for thi
 are in ``airflow.providers.cncf.kubernetes`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.5.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.6.0/>`_.
 
 Installation
 ------------
@@ -86,17 +86,17 @@ The package supports the following python versions: 3.9,3.10,3.11,3.12
 Requirements
 ------------
 
-======================  =====================
+======================  ====================
 PIP package             Version required
-======================  =====================
+======================  ====================
 ``aiofiles``            ``>=23.2.0``
 ``apache-airflow``      ``>=2.10.0``
 ``asgiref``             ``>=3.5.2``
 ``cryptography``        ``>=41.0.0``
-``kubernetes``          ``>=29.0.0,<=31.0.0``
-``kubernetes_asyncio``  ``>=29.0.0,<=31.0.0``
-======================  =====================
+``kubernetes``          ``>=32.0.0,<33.0.0``
+``kubernetes_asyncio``  ``>=32.0.0,<33.0.0``
+======================  ====================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.5.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.6.0/changelog.html>`_.
 

{apache_airflow_providers_cncf_kubernetes-10.5.0rc1.dist-info → apache_airflow_providers_cncf_kubernetes-10.6.0rc1.dist-info}/RECORD

@@ -1,30 +1,30 @@
 airflow/providers/cncf/kubernetes/LICENSE,sha256=gXPVwptPlW1TJ4HSuG5OMPg-a3h43OGMkZRR1rpwfJA,10850
-airflow/providers/cncf/kubernetes/__init__.py,sha256=JbKTwSfn5nKWBEgMs8EXthKBZupv6f6sucVlGir2TKI,1505
+airflow/providers/cncf/kubernetes/__init__.py,sha256=3zCSE5o2NsRNwIL9r5ir0zlepR293afTmqi8EUI-9Yo,1505
 airflow/providers/cncf/kubernetes/callbacks.py,sha256=5zGmQthojdT9iBEV3LIyBq-oKzjv2D4dOYCjYRbb61c,6076
 airflow/providers/cncf/kubernetes/exceptions.py,sha256=3cNEZTnrltBsqwzHiLfckwYYc_IWY1g4PcRs6zuMWWA,1137
-airflow/providers/cncf/kubernetes/get_provider_info.py,sha256=m4eN2HKhaAI2kD-mOvbMcbv5zxU78iP4pea_LltJ2v0,16042
+airflow/providers/cncf/kubernetes/get_provider_info.py,sha256=Git4HycOcHrb4zD9W7ZYsqNDkQSQ4uipSJO_GaPiroE,16041
 airflow/providers/cncf/kubernetes/k8s_model.py,sha256=xmdFhX29DjegoZ-cq8-KDL9soVYXf4OpU6fAGr3cPTU,2101
 airflow/providers/cncf/kubernetes/kube_client.py,sha256=yflZxLousXA9d7t67KrEy55qzb1cUhEyy6yCPkEem28,5329
-airflow/providers/cncf/kubernetes/kube_config.py,sha256=3qWdCp2z4g8gX_sIOProgwp52UxM5kAIYabkxaX297g,5079
+airflow/providers/cncf/kubernetes/kube_config.py,sha256=RX-ilybMXAcXg4gGyKfh6IYRWEf7llPycp9LqbYEe6s,5154
 airflow/providers/cncf/kubernetes/kubernetes_helper_functions.py,sha256=opxof6wxHEAHwa_zRB47QJBBrV5St4rIZzAiptA9Rek,5510
-airflow/providers/cncf/kubernetes/pod_generator.py,sha256=f24Qdg4QA9d8gaHA0X78jDojtm9swEgLLqcG6yEx2rc,20857
+airflow/providers/cncf/kubernetes/pod_generator.py,sha256=0VEcAtT2SzAFwSDsQWe2QdrY2mDV8s4hBw0qLcmIMGw,21038
 airflow/providers/cncf/kubernetes/python_kubernetes_script.jinja2,sha256=I0EHRGwLHjSiX85e51HBIoddRDnC8TJPFrDBqQq_NJg,1776
 airflow/providers/cncf/kubernetes/python_kubernetes_script.py,sha256=KnTlZSWCZhwvj89fSc2kgIRTaI4iLNKPquHc2wXnluo,3460
 airflow/providers/cncf/kubernetes/secret.py,sha256=wj-T9gouqau_X14slAstGmnSxqXJQzdLwUdURzHna0I,5209
-airflow/providers/cncf/kubernetes/template_rendering.py,sha256=NyrAc2rsZ0oyXxtRHiY8qkaH4tftHlexTpr7YE4UQY0,2682
+airflow/providers/cncf/kubernetes/template_rendering.py,sha256=WSUBhjGSDhjNtA4IFlbYyX50rvYN6UA4dMk0cPqgOjo,3618
 airflow/providers/cncf/kubernetes/version_compat.py,sha256=j5PCtXvZ71aBjixu-EFTNtVDPsngzzs7os0ZQDgFVDk,1536
 airflow/providers/cncf/kubernetes/backcompat/__init__.py,sha256=KXF76f3v1jIFUBNz8kwxVMvm7i4mNo35LbIG9IijBNc,1299
 airflow/providers/cncf/kubernetes/backcompat/backwards_compat_converters.py,sha256=FkRRtIEucp2hYrecGVYVgyPI6-b7hE7X7L17Z3r459Y,4303
 airflow/providers/cncf/kubernetes/cli/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
-airflow/providers/cncf/kubernetes/cli/kubernetes_command.py,sha256=FRR8p50FgHaVPS8x1rbXFSd47NtBtidqL4JyTyfxqnQ,7366
+airflow/providers/cncf/kubernetes/cli/kubernetes_command.py,sha256=lvs9RJ-fjrCdisE2rnRJkxUxpn6rdtFX4j7D94ASO9c,7943
 airflow/providers/cncf/kubernetes/decorators/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
 airflow/providers/cncf/kubernetes/decorators/kubernetes.py,sha256=_OnebMazgYTJoCnkaMsRxwF6f2slKU_-ucrml8z8cq0,6449
 airflow/providers/cncf/kubernetes/decorators/kubernetes_cmd.py,sha256=Rsak_cPl_-yNv7SKh0PI7r2UgqZF-Li7zIxwX5cedc0,4951
 airflow/providers/cncf/kubernetes/executors/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
-airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py,sha256=TomXdMCdtnnsKOWZiNR2lrh0ZmHghfbzXjpjeRDJTFA,31976
-airflow/providers/cncf/kubernetes/executors/kubernetes_executor_types.py,sha256=L8_8HOHd_4O8WW6xT2tp49-yOj0EMKCYK5YqMOOx_bI,1973
-airflow/providers/cncf/kubernetes/executors/kubernetes_executor_utils.py,sha256=2xqLS8DQhnwVwxnv4an8cjv4HPqP_TSiVFjjADPMeHM,24750
-airflow/providers/cncf/kubernetes/executors/local_kubernetes_executor.py,sha256=TuFRbs1zqKajJoZmo25kT4AGd-_-iD-UbhfOY30EOck,11591
+airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py,sha256=dpy1d0BU-pID_t9wPmi7h9PULbk9sQBIWqKxy9PNSB8,31953
+airflow/providers/cncf/kubernetes/executors/kubernetes_executor_types.py,sha256=drjK3ZGVJ8pETX0uwT4s6zqWPMegXumMQDgwBipx_xc,2042
+airflow/providers/cncf/kubernetes/executors/kubernetes_executor_utils.py,sha256=1aoIOoL9-ufnvZxHOau-ZJtt14Xl2zxlah2SuSmNnp0,24461
+airflow/providers/cncf/kubernetes/executors/local_kubernetes_executor.py,sha256=icUEyhfHZYI4GvacTrYdS_nSohLk5tLzR5ZNUAYlhQY,12042
 airflow/providers/cncf/kubernetes/hooks/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
 airflow/providers/cncf/kubernetes/hooks/kubernetes.py,sha256=KPv5t7Xpvx9Nw9uampf1VCMdnq_ySqRzFbt07NKIjZY,37107
 airflow/providers/cncf/kubernetes/kubernetes_executor_templates/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
@@ -33,7 +33,7 @@ airflow/providers/cncf/kubernetes/operators/__init__.py,sha256=mlJxuZLkd5x-iq2SB
 airflow/providers/cncf/kubernetes/operators/custom_object_launcher.py,sha256=jTVHQt1vp5gELrLNyM-DrZ1ywgmTy3Hh1i6wyl7AGS0,15314
 airflow/providers/cncf/kubernetes/operators/job.py,sha256=aK2MogooZ6K7uVC0hWRYhCGgzwHOERrgPyknWu5mp7c,23771
 airflow/providers/cncf/kubernetes/operators/kueue.py,sha256=eEfl__06k15-21Y86qHOLAsY2zR1OWM4QgQhDteDBP0,4563
-airflow/providers/cncf/kubernetes/operators/pod.py,sha256=kTWxSQQvxHI3rzVDx-Z1jEQa4g4ZN7o_7smkMRyOw8U,58632
+airflow/providers/cncf/kubernetes/operators/pod.py,sha256=hF3M93zB-gEvdKaGV0gMkMzDJF0URnpXSbcoKWR99_g,60607
 airflow/providers/cncf/kubernetes/operators/resource.py,sha256=Q5WssuDyjtzo1Op1pzUmYG4JZZdzCKTe-vTZEy8HSNA,7579
 airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py,sha256=0TO86G-gbWAkQvaWBfhCp6ZJwoQzciH-UGR5kgw9fmg,13847
 airflow/providers/cncf/kubernetes/pod_template_file_examples/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
@@ -48,13 +48,13 @@ airflow/providers/cncf/kubernetes/sensors/__init__.py,sha256=9hdXHABrVpkbpjZgUft
 airflow/providers/cncf/kubernetes/sensors/spark_kubernetes.py,sha256=sMSuE4bziqPYzBNIZ2y1ab00kGO2tlS3Z7AmePBFA3w,5356
 airflow/providers/cncf/kubernetes/triggers/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
 airflow/providers/cncf/kubernetes/triggers/job.py,sha256=DGbC1FZktBF-00Lb0pU9iIKQnmdW8HWklp5Wwq54OEY,6754
-airflow/providers/cncf/kubernetes/triggers/pod.py,sha256=IeoMrPECgsr2Sswfvd7Fl6p3uCAvnZLvAco3tO2wu7Q,12865
+airflow/providers/cncf/kubernetes/triggers/pod.py,sha256=YEyYTQONZu66uHGp_38N_A56txZNamA0Kc4UA0I3sJk,13414
 airflow/providers/cncf/kubernetes/utils/__init__.py,sha256=ClZN0VPjWySdVwS_ktH7rrgL9VLAcs3OSJSB9s3zaYw,863
 airflow/providers/cncf/kubernetes/utils/delete_from.py,sha256=poObZSoEJwQyaYWilEURs8f4CDY2sn_pfwS31Lf579A,5195
 airflow/providers/cncf/kubernetes/utils/k8s_resource_iterator.py,sha256=DLypjkD_3YDixRTcsxEjgvHZNbbG9qamlz05eBqaWzU,1955
-airflow/providers/cncf/kubernetes/utils/pod_manager.py,sha256=8edX-K9t2YgGHYsEeqUmKodHv4jsEHdPjii89KlE0dw,39436
+airflow/providers/cncf/kubernetes/utils/pod_manager.py,sha256=Vpww3vAHX48QtApI-j1cI1jNiozQ6YMA7wfKFV1ELI0,40471
 airflow/providers/cncf/kubernetes/utils/xcom_sidecar.py,sha256=k6bdmVJ21OrAwGmWwledRrAmaty9ZrmbuM-IbaI4mqo,2519
-apache_airflow_providers_cncf_kubernetes-10.5.0rc1.dist-info/entry_points.txt,sha256=ByD3QJJyP9CfmTYtpNI1953akD38RUDgpGXLaq9vpOw,111
-apache_airflow_providers_cncf_kubernetes-10.5.0rc1.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
-apache_airflow_providers_cncf_kubernetes-10.5.0rc1.dist-info/METADATA,sha256=N5m9VD9yH_AjVp57oab1u-GkxOae0YlvC5FxJyRItHk,4328
-apache_airflow_providers_cncf_kubernetes-10.5.0rc1.dist-info/RECORD,,
+apache_airflow_providers_cncf_kubernetes-10.6.0rc1.dist-info/entry_points.txt,sha256=ByD3QJJyP9CfmTYtpNI1953akD38RUDgpGXLaq9vpOw,111
+apache_airflow_providers_cncf_kubernetes-10.6.0rc1.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
+apache_airflow_providers_cncf_kubernetes-10.6.0rc1.dist-info/METADATA,sha256=ZGRMBZlrTVFhLBekuYd9NxcSjhnXYLNtOOG2E6PT0xU,4321
+apache_airflow_providers_cncf_kubernetes-10.6.0rc1.dist-info/RECORD,,