apache-airflow-providers-cncf-kubernetes 10.6.1__py3-none-any.whl → 10.6.2rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
29
29
 
30
30
  __all__ = ["__version__"]
31
31
 
32
- __version__ = "10.6.1"
32
+ __version__ = "10.6.2"
33
33
 
34
34
  if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
35
35
  "2.10.0"
@@ -66,7 +66,12 @@ def generate_pod_yaml(args):
66
66
  kube_config = KubeConfig()
67
67
 
68
68
  for task in dag.tasks:
69
- ti = TaskInstance(task, run_id=dr.run_id)
69
+ if AIRFLOW_V_3_0_PLUS:
70
+ from uuid6 import uuid7
71
+
72
+ ti = TaskInstance(task, run_id=dr.run_id, dag_version_id=uuid7())
73
+ else:
74
+ ti = TaskInstance(task, run_id=dr.run_id)
70
75
  ti.dag_run = dr
71
76
  ti.dag_model = dm
72
77
 
@@ -307,7 +307,7 @@ class KubernetesExecutor(BaseExecutor):
307
307
  executor_config = w.ti.executor_config or {}
308
308
 
309
309
  del self.queued_tasks[key]
310
- self.execute_async(key=key, command=command, queue=queue, executor_config=executor_config) # type: ignore[arg-type]
310
+ self.execute_async(key=key, command=command, queue=queue, executor_config=executor_config)
311
311
  self.running.add(key)
312
312
 
313
313
  def sync(self) -> None:
@@ -44,11 +44,7 @@ from airflow.providers.cncf.kubernetes.utils.pod_manager import (
44
44
  container_is_completed,
45
45
  container_is_running,
46
46
  )
47
-
48
- try:
49
- from airflow.sdk import BaseHook
50
- except ImportError:
51
- from airflow.hooks.base import BaseHook # type: ignore[attr-defined,no-redef]
47
+ from airflow.providers.cncf.kubernetes.version_compat import BaseHook
52
48
  from airflow.utils import yaml
53
49
 
54
50
  if TYPE_CHECKING:
@@ -775,7 +771,7 @@ class AsyncKubernetesHook(KubernetesHook):
775
771
  if self.config_dict:
776
772
  self.log.debug(LOADING_KUBE_CONFIG_FILE_RESOURCE.format("config dictionary"))
777
773
  self._is_in_cluster = False
778
- await async_config.load_kube_config_from_dict(self.config_dict)
774
+ await async_config.load_kube_config_from_dict(self.config_dict, context=cluster_context)
779
775
  return async_client.ApiClient()
780
776
 
781
777
  if kubeconfig_path is not None:
@@ -22,16 +22,17 @@ import copy
22
22
  import json
23
23
  import logging
24
24
  import os
25
+ import warnings
25
26
  from collections.abc import Sequence
26
27
  from functools import cached_property
27
- from typing import TYPE_CHECKING
28
+ from typing import TYPE_CHECKING, Any, Literal
28
29
 
29
30
  from kubernetes.client import BatchV1Api, models as k8s
30
31
  from kubernetes.client.api_client import ApiClient
31
32
  from kubernetes.client.rest import ApiException
32
33
 
33
34
  from airflow.configuration import conf
34
- from airflow.exceptions import AirflowException
35
+ from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
35
36
  from airflow.providers.cncf.kubernetes.hooks.kubernetes import KubernetesHook
36
37
  from airflow.providers.cncf.kubernetes.kubernetes_helper_functions import (
37
38
  add_unique_suffix,
@@ -81,6 +82,17 @@ class KubernetesJobOperator(KubernetesPodOperator):
81
82
  Used if the parameter `wait_until_job_complete` set True.
82
83
  :param deferrable: Run operator in the deferrable mode. Note that the parameter
83
84
  `wait_until_job_complete` must be set True.
85
+ :param on_kill_propagation_policy: Whether and how garbage collection will be performed. Default is 'Foreground'.
86
+ Acceptable values are:
87
+ 'Orphan' - orphan the dependents;
88
+ 'Background' - allow the garbage collector to delete the dependents in the background;
89
+ 'Foreground' - a cascading policy that deletes all dependents in the foreground.
90
+ Default value is 'Foreground'.
91
+ :param discover_pods_retry_number: Number of time list_namespaced_pod will be performed to discover
92
+ already running pods.
93
+ :param unwrap_single: Unwrap single result from the pod. For example, when set to `True` - if the XCom
94
+ result should be `['res']`, the final result would be `'res'`. Default is True to support backward
95
+ compatibility.
84
96
  """
85
97
 
86
98
  template_fields: Sequence[str] = tuple({"job_template_file"} | set(KubernetesPodOperator.template_fields))
@@ -101,8 +113,12 @@ class KubernetesJobOperator(KubernetesPodOperator):
101
113
  wait_until_job_complete: bool = False,
102
114
  job_poll_interval: float = 10,
103
115
  deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
116
+ on_kill_propagation_policy: Literal["Foreground", "Background", "Orphan"] = "Foreground",
117
+ discover_pods_retry_number: int = 3,
118
+ unwrap_single: bool = True,
104
119
  **kwargs,
105
120
  ) -> None:
121
+ self._pod = None
106
122
  super().__init__(**kwargs)
107
123
  self.job_template_file = job_template_file
108
124
  self.full_job_spec = full_job_spec
@@ -119,6 +135,22 @@ class KubernetesJobOperator(KubernetesPodOperator):
119
135
  self.wait_until_job_complete = wait_until_job_complete
120
136
  self.job_poll_interval = job_poll_interval
121
137
  self.deferrable = deferrable
138
+ self.on_kill_propagation_policy = on_kill_propagation_policy
139
+ self.discover_pods_retry_number = discover_pods_retry_number
140
+ self.unwrap_single = unwrap_single
141
+
142
+ @property
143
+ def pod(self):
144
+ warnings.warn(
145
+ "`pod` parameter is deprecated, please use `pods`",
146
+ AirflowProviderDeprecationWarning,
147
+ stacklevel=2,
148
+ )
149
+ return self.pods[0] if self.pods else None
150
+
151
+ @pod.setter
152
+ def pod(self, value):
153
+ self._pod = value
122
154
 
123
155
  @cached_property
124
156
  def _incluster_namespace(self):
@@ -167,12 +199,16 @@ class KubernetesJobOperator(KubernetesPodOperator):
167
199
  ti.xcom_push(key="job_name", value=self.job.metadata.name)
168
200
  ti.xcom_push(key="job_namespace", value=self.job.metadata.namespace)
169
201
 
170
- self.pod: k8s.V1Pod | None
171
- if self.pod is None:
172
- self.pod = self.get_or_create_pod( # must set `self.pod` for `on_kill`
173
- pod_request_obj=self.pod_request_obj,
174
- context=context,
175
- )
202
+ self.pods: Sequence[k8s.V1Pod] | None = None
203
+ if self.parallelism is None and self.pod is None:
204
+ self.pods = [
205
+ self.get_or_create_pod(
206
+ pod_request_obj=self.pod_request_obj,
207
+ context=context,
208
+ )
209
+ ]
210
+ else:
211
+ self.pods = self.get_pods(pod_request_obj=self.pod_request_obj, context=context)
176
212
 
177
213
  if self.wait_until_job_complete and self.deferrable:
178
214
  self.execute_deferrable()
@@ -180,22 +216,25 @@ class KubernetesJobOperator(KubernetesPodOperator):
180
216
 
181
217
  if self.wait_until_job_complete:
182
218
  if self.do_xcom_push:
183
- self.pod_manager.await_container_completion(
184
- pod=self.pod, container_name=self.base_container_name
185
- )
186
- self.pod_manager.await_xcom_sidecar_container_start(pod=self.pod)
187
- xcom_result = self.extract_xcom(pod=self.pod)
219
+ xcom_result = []
220
+ for pod in self.pods:
221
+ self.pod_manager.await_container_completion(
222
+ pod=pod, container_name=self.base_container_name
223
+ )
224
+ self.pod_manager.await_xcom_sidecar_container_start(pod=pod)
225
+ xcom_result.append(self.extract_xcom(pod=pod))
188
226
  self.job = self.hook.wait_until_job_complete(
189
227
  job_name=self.job.metadata.name,
190
228
  namespace=self.job.metadata.namespace,
191
229
  job_poll_interval=self.job_poll_interval,
192
230
  )
193
231
  if self.get_logs:
194
- self.pod_manager.fetch_requested_container_logs(
195
- pod=self.pod,
196
- containers=self.container_logs,
197
- follow_logs=True,
198
- )
232
+ for pod in self.pods:
233
+ self.pod_manager.fetch_requested_container_logs(
234
+ pod=pod,
235
+ containers=self.container_logs,
236
+ follow_logs=True,
237
+ )
199
238
 
200
239
  ti.xcom_push(key="job", value=self.job.to_dict())
201
240
  if self.wait_until_job_complete:
@@ -209,10 +248,10 @@ class KubernetesJobOperator(KubernetesPodOperator):
209
248
  def execute_deferrable(self):
210
249
  self.defer(
211
250
  trigger=KubernetesJobTrigger(
212
- job_name=self.job.metadata.name, # type: ignore[union-attr]
213
- job_namespace=self.job.metadata.namespace, # type: ignore[union-attr]
214
- pod_name=self.pod.metadata.name, # type: ignore[union-attr]
215
- pod_namespace=self.pod.metadata.namespace, # type: ignore[union-attr]
251
+ job_name=self.job.metadata.name,
252
+ job_namespace=self.job.metadata.namespace,
253
+ pod_names=[pod.metadata.name for pod in self.pods],
254
+ pod_namespace=self.pods[0].metadata.namespace,
216
255
  base_container_name=self.base_container_name,
217
256
  kubernetes_conn_id=self.kubernetes_conn_id,
218
257
  cluster_context=self.cluster_context,
@@ -232,20 +271,23 @@ class KubernetesJobOperator(KubernetesPodOperator):
232
271
  raise AirflowException(event["message"])
233
272
 
234
273
  if self.get_logs:
235
- pod_name = event["pod_name"]
236
- pod_namespace = event["pod_namespace"]
237
- self.pod = self.hook.get_pod(pod_name, pod_namespace)
238
- if not self.pod:
239
- raise PodNotFoundException("Could not find pod after resuming from deferral")
240
- self._write_logs(self.pod)
274
+ for pod_name in event["pod_names"]:
275
+ pod_namespace = event["pod_namespace"]
276
+ pod = self.hook.get_pod(pod_name, pod_namespace)
277
+ if not pod:
278
+ raise PodNotFoundException("Could not find pod after resuming from deferral")
279
+ self._write_logs(pod)
241
280
 
242
281
  if self.do_xcom_push:
243
- xcom_result = event["xcom_result"]
244
- if isinstance(xcom_result, str) and xcom_result.rstrip() == EMPTY_XCOM_RESULT:
245
- self.log.info("xcom result file is empty.")
246
- return None
247
- self.log.info("xcom result: \n%s", xcom_result)
248
- return json.loads(xcom_result)
282
+ xcom_results: list[Any | None] = []
283
+ for xcom_result in event["xcom_result"]:
284
+ if isinstance(xcom_result, str) and xcom_result.rstrip() == EMPTY_XCOM_RESULT:
285
+ self.log.info("xcom result file is empty.")
286
+ xcom_results.append(None)
287
+ continue
288
+ self.log.info("xcom result: \n%s", xcom_result)
289
+ xcom_results.append(json.loads(xcom_result))
290
+ return xcom_results[0] if self.unwrap_single and len(xcom_results) == 1 else xcom_results
249
291
 
250
292
  @staticmethod
251
293
  def deserialize_job_template_file(path: str) -> k8s.V1Job:
@@ -275,12 +317,11 @@ class KubernetesJobOperator(KubernetesPodOperator):
275
317
  kwargs = {
276
318
  "name": job.metadata.name,
277
319
  "namespace": job.metadata.namespace,
320
+ "propagation_policy": self.on_kill_propagation_policy,
278
321
  }
279
322
  if self.termination_grace_period is not None:
280
323
  kwargs.update(grace_period_seconds=self.termination_grace_period)
281
324
  self.job_client.delete_namespaced_job(**kwargs)
282
- if self.pod:
283
- super().on_kill()
284
325
 
285
326
  def build_job_request_obj(self, context: Context | None = None) -> k8s.V1Job:
286
327
  """
@@ -400,6 +441,29 @@ class KubernetesJobOperator(KubernetesPodOperator):
400
441
 
401
442
  return None
402
443
 
444
+ def get_pods(
445
+ self, pod_request_obj: k8s.V1Pod, context: Context, *, exclude_checked: bool = True
446
+ ) -> Sequence[k8s.V1Pod]:
447
+ """Return an already-running pods if exists."""
448
+ label_selector = self._build_find_pod_label_selector(context, exclude_checked=exclude_checked)
449
+ pod_list: Sequence[k8s.V1Pod] = []
450
+ retry_number: int = 0
451
+
452
+ while len(pod_list) != self.parallelism or retry_number <= self.discover_pods_retry_number:
453
+ pod_list = self.client.list_namespaced_pod(
454
+ namespace=pod_request_obj.metadata.namespace,
455
+ label_selector=label_selector,
456
+ ).items
457
+ retry_number += 1
458
+
459
+ if len(pod_list) == 0:
460
+ raise AirflowException(f"No pods running with labels {label_selector}")
461
+
462
+ for pod_instance in pod_list:
463
+ self.log_matching_pod(pod=pod_instance, context=context)
464
+
465
+ return pod_list
466
+
403
467
 
404
468
  class KubernetesDeleteJobOperator(BaseOperator):
405
469
  """
@@ -68,7 +68,7 @@ from airflow.providers.cncf.kubernetes.kubernetes_helper_functions import (
68
68
  )
69
69
  from airflow.providers.cncf.kubernetes.pod_generator import PodGenerator
70
70
  from airflow.providers.cncf.kubernetes.triggers.pod import KubernetesPodTrigger
71
- from airflow.providers.cncf.kubernetes.utils import xcom_sidecar # type: ignore[attr-defined]
71
+ from airflow.providers.cncf.kubernetes.utils import xcom_sidecar
72
72
  from airflow.providers.cncf.kubernetes.utils.pod_manager import (
73
73
  EMPTY_XCOM_RESULT,
74
74
  OnFinishAction,
@@ -80,11 +80,10 @@ from airflow.providers.cncf.kubernetes.utils.pod_manager import (
80
80
  container_is_succeeded,
81
81
  get_container_termination_message,
82
82
  )
83
- from airflow.providers.cncf.kubernetes.version_compat import BaseOperator
83
+ from airflow.providers.cncf.kubernetes.version_compat import XCOM_RETURN_KEY, BaseOperator
84
84
  from airflow.settings import pod_mutation_hook
85
85
  from airflow.utils import yaml
86
86
  from airflow.utils.helpers import prune_dict, validate_key
87
- from airflow.utils.xcom import XCOM_RETURN_KEY
88
87
  from airflow.version import version as airflow_version
89
88
 
90
89
  if TYPE_CHECKING:
@@ -579,10 +578,15 @@ class KubernetesPodOperator(BaseOperator):
579
578
  pod = self.find_pod(pod_request_obj.metadata.namespace, context=context)
580
579
  if pod:
581
580
  # If pod is terminated then delete the pod an create a new as not possible to get xcom
582
- pod_phase = (
583
- pod.status.phase if hasattr(pod, "status") and hasattr(pod.status, "phase") else None
584
- )
585
- if pod_phase and pod_phase not in (PodPhase.SUCCEEDED, PodPhase.FAILED):
581
+ pod_phase = pod.status.phase if pod.status and pod.status.phase else None
582
+ pod_reason = pod.status.reason.lower() if pod.status and pod.status.reason else ""
583
+ if pod_phase not in (PodPhase.SUCCEEDED, PodPhase.FAILED) and pod_reason != "evicted":
584
+ self.log.info(
585
+ "Reusing existing pod '%s' (phase=%s, reason=%s) since it is not terminated or evicted.",
586
+ pod.metadata.name,
587
+ pod_phase,
588
+ pod_reason,
589
+ )
586
590
  return pod
587
591
 
588
592
  self.log.info(
@@ -604,20 +608,18 @@ class KubernetesPodOperator(BaseOperator):
604
608
 
605
609
  def await_pod_start(self, pod: k8s.V1Pod) -> None:
606
610
  try:
607
- loop = asyncio.get_event_loop()
608
- events_task = asyncio.ensure_future(
609
- self.pod_manager.watch_pod_events(pod, self.startup_check_interval_seconds)
610
- )
611
- loop.run_until_complete(
612
- self.pod_manager.await_pod_start(
611
+
612
+ async def _await_pod_start():
613
+ events_task = self.pod_manager.watch_pod_events(pod, self.startup_check_interval_seconds)
614
+ pod_start_task = self.pod_manager.await_pod_start(
613
615
  pod=pod,
614
616
  schedule_timeout=self.schedule_timeout_seconds,
615
617
  startup_timeout=self.startup_timeout_seconds,
616
618
  check_interval=self.startup_check_interval_seconds,
617
619
  )
618
- )
619
- loop.run_until_complete(events_task)
620
- loop.close()
620
+ await asyncio.gather(pod_start_task, events_task)
621
+
622
+ asyncio.run(_await_pod_start())
621
623
  except PodLaunchFailedException:
622
624
  if self.log_events_on_failure:
623
625
  self._read_pod_events(pod, reraise=False)
@@ -106,10 +106,9 @@ class KubernetesResourceBaseOperator(BaseOperator):
106
106
  group = api_version[0 : api_version.find("/")]
107
107
  version = api_version[api_version.find("/") + 1 :]
108
108
 
109
- namespace = None
110
- if body.get("metadata"):
111
- metadata: dict = body.get("metadata", None)
112
- namespace = metadata.get("namespace", None)
109
+ metadata = body.get("metadata", {}) if body else None
110
+ namespace = metadata.get("namespace") if metadata else None
111
+
113
112
  if namespace is None:
114
113
  namespace = self.get_namespace()
115
114
 
@@ -17,10 +17,12 @@
17
17
  from __future__ import annotations
18
18
 
19
19
  import asyncio
20
+ import warnings
20
21
  from collections.abc import AsyncIterator
21
22
  from functools import cached_property
22
23
  from typing import TYPE_CHECKING, Any
23
24
 
25
+ from airflow.exceptions import AirflowProviderDeprecationWarning
24
26
  from airflow.providers.cncf.kubernetes.hooks.kubernetes import AsyncKubernetesHook, KubernetesHook
25
27
  from airflow.providers.cncf.kubernetes.utils.pod_manager import PodManager
26
28
  from airflow.providers.cncf.kubernetes.utils.xcom_sidecar import PodDefaults
@@ -36,7 +38,8 @@ class KubernetesJobTrigger(BaseTrigger):
36
38
 
37
39
  :param job_name: The name of the job.
38
40
  :param job_namespace: The namespace of the job.
39
- :param pod_name: The name of the Pod.
41
+ :param pod_name: The name of the Pod. Parameter is deprecated, please use pod_names instead.
42
+ :param pod_names: The name of the Pods.
40
43
  :param pod_namespace: The namespace of the Pod.
41
44
  :param base_container_name: The name of the base container in the pod.
42
45
  :param kubernetes_conn_id: The :ref:`kubernetes connection id <howto/connection:kubernetes>`
@@ -55,9 +58,10 @@ class KubernetesJobTrigger(BaseTrigger):
55
58
  self,
56
59
  job_name: str,
57
60
  job_namespace: str,
58
- pod_name: str,
61
+ pod_names: list[str],
59
62
  pod_namespace: str,
60
63
  base_container_name: str,
64
+ pod_name: str | None = None,
61
65
  kubernetes_conn_id: str | None = None,
62
66
  poll_interval: float = 10.0,
63
67
  cluster_context: str | None = None,
@@ -69,7 +73,13 @@ class KubernetesJobTrigger(BaseTrigger):
69
73
  super().__init__()
70
74
  self.job_name = job_name
71
75
  self.job_namespace = job_namespace
72
- self.pod_name = pod_name
76
+ if pod_name is not None:
77
+ self._pod_name = pod_name
78
+ self.pod_names = [
79
+ self.pod_name,
80
+ ]
81
+ else:
82
+ self.pod_names = pod_names
73
83
  self.pod_namespace = pod_namespace
74
84
  self.base_container_name = base_container_name
75
85
  self.kubernetes_conn_id = kubernetes_conn_id
@@ -80,6 +90,15 @@ class KubernetesJobTrigger(BaseTrigger):
80
90
  self.get_logs = get_logs
81
91
  self.do_xcom_push = do_xcom_push
82
92
 
93
+ @property
94
+ def pod_name(self):
95
+ warnings.warn(
96
+ "`pod_name` parameter is deprecated, please use `pod_names`",
97
+ AirflowProviderDeprecationWarning,
98
+ stacklevel=2,
99
+ )
100
+ return self._pod_name
101
+
83
102
  def serialize(self) -> tuple[str, dict[str, Any]]:
84
103
  """Serialize KubernetesCreateJobTrigger arguments and classpath."""
85
104
  return (
@@ -87,7 +106,7 @@ class KubernetesJobTrigger(BaseTrigger):
87
106
  {
88
107
  "job_name": self.job_name,
89
108
  "job_namespace": self.job_namespace,
90
- "pod_name": self.pod_name,
109
+ "pod_names": self.pod_names,
91
110
  "pod_namespace": self.pod_namespace,
92
111
  "base_container_name": self.base_container_name,
93
112
  "kubernetes_conn_id": self.kubernetes_conn_id,
@@ -100,23 +119,25 @@ class KubernetesJobTrigger(BaseTrigger):
100
119
  },
101
120
  )
102
121
 
103
- async def run(self) -> AsyncIterator[TriggerEvent]: # type: ignore[override]
122
+ async def run(self) -> AsyncIterator[TriggerEvent]:
104
123
  """Get current job status and yield a TriggerEvent."""
105
- if self.get_logs or self.do_xcom_push:
106
- pod = await self.hook.get_pod(name=self.pod_name, namespace=self.pod_namespace)
107
124
  if self.do_xcom_push:
108
- await self.hook.wait_until_container_complete(
109
- name=self.pod_name, namespace=self.pod_namespace, container_name=self.base_container_name
110
- )
111
- self.log.info("Checking if xcom sidecar container is started.")
112
- await self.hook.wait_until_container_started(
113
- name=self.pod_name,
114
- namespace=self.pod_namespace,
115
- container_name=PodDefaults.SIDECAR_CONTAINER_NAME,
116
- )
117
- self.log.info("Extracting result from xcom sidecar container.")
118
- loop = asyncio.get_running_loop()
119
- xcom_result = await loop.run_in_executor(None, self.pod_manager.extract_xcom, pod)
125
+ xcom_results = []
126
+ for pod_name in self.pod_names:
127
+ pod = await self.hook.get_pod(name=pod_name, namespace=self.pod_namespace)
128
+ await self.hook.wait_until_container_complete(
129
+ name=pod_name, namespace=self.pod_namespace, container_name=self.base_container_name
130
+ )
131
+ self.log.info("Checking if xcom sidecar container is started.")
132
+ await self.hook.wait_until_container_started(
133
+ name=pod_name,
134
+ namespace=self.pod_namespace,
135
+ container_name=PodDefaults.SIDECAR_CONTAINER_NAME,
136
+ )
137
+ self.log.info("Extracting result from xcom sidecar container.")
138
+ loop = asyncio.get_running_loop()
139
+ xcom_result = await loop.run_in_executor(None, self.pod_manager.extract_xcom, pod)
140
+ xcom_results.append(xcom_result)
120
141
  job: V1Job = await self.hook.wait_until_job_complete(name=self.job_name, namespace=self.job_namespace)
121
142
  job_dict = job.to_dict()
122
143
  error_message = self.hook.is_job_failed(job=job)
@@ -124,14 +145,14 @@ class KubernetesJobTrigger(BaseTrigger):
124
145
  {
125
146
  "name": job.metadata.name,
126
147
  "namespace": job.metadata.namespace,
127
- "pod_name": pod.metadata.name if self.get_logs else None,
128
- "pod_namespace": pod.metadata.namespace if self.get_logs else None,
148
+ "pod_names": [pod_name for pod_name in self.pod_names] if self.get_logs else None,
149
+ "pod_namespace": self.pod_namespace if self.get_logs else None,
129
150
  "status": "error" if error_message else "success",
130
151
  "message": f"Job failed with error: {error_message}"
131
152
  if error_message
132
153
  else "Job completed successfully",
133
154
  "job": job_dict,
134
- "xcom_result": xcom_result if self.do_xcom_push else None,
155
+ "xcom_result": xcom_results if self.do_xcom_push else None,
135
156
  }
136
157
  )
137
158
 
@@ -141,7 +141,7 @@ class KubernetesPodTrigger(BaseTrigger):
141
141
  },
142
142
  )
143
143
 
144
- async def run(self) -> AsyncIterator[TriggerEvent]: # type: ignore[override]
144
+ async def run(self) -> AsyncIterator[TriggerEvent]:
145
145
  """Get current pod status and yield a TriggerEvent."""
146
146
  self.log.info("Checking pod %r in namespace %r.", self.pod_name, self.pod_namespace)
147
147
  try:
@@ -338,6 +338,7 @@ class PodManager(LoggingMixin):
338
338
  self._client = kube_client
339
339
  self._watch = watch.Watch()
340
340
  self._callbacks = callbacks or []
341
+ self.stop_watching_events = False
341
342
 
342
343
  def run_pod_async(self, pod: V1Pod, **kwargs) -> V1Pod:
343
344
  """Run POD asynchronously."""
@@ -380,9 +381,8 @@ class PodManager(LoggingMixin):
380
381
 
381
382
  async def watch_pod_events(self, pod: V1Pod, check_interval: int = 1) -> None:
382
383
  """Read pod events and writes into log."""
383
- self.keep_watching_for_events = True
384
384
  num_events = 0
385
- while self.keep_watching_for_events:
385
+ while not self.stop_watching_events:
386
386
  events = self.read_pod_events(pod)
387
387
  for new_event in events.items[num_events:]:
388
388
  involved_object: V1ObjectReference = new_event.involved_object
@@ -413,7 +413,7 @@ class PodManager(LoggingMixin):
413
413
  remote_pod = self.read_pod(pod)
414
414
  pod_status = remote_pod.status
415
415
  if pod_status.phase != PodPhase.PENDING:
416
- self.keep_watching_for_events = False
416
+ self.stop_watching_events = True
417
417
  self.log.info("::endgroup::")
418
418
  break
419
419
 
@@ -36,9 +36,12 @@ AIRFLOW_V_3_0_PLUS = get_base_airflow_version_tuple() >= (3, 0, 0)
36
36
  AIRFLOW_V_3_1_PLUS = get_base_airflow_version_tuple() >= (3, 1, 0)
37
37
 
38
38
  if AIRFLOW_V_3_1_PLUS:
39
- from airflow.sdk import BaseOperator
39
+ from airflow.models.xcom import XCOM_RETURN_KEY
40
+ from airflow.sdk import BaseHook, BaseOperator
40
41
  else:
41
- from airflow.models import BaseOperator # type: ignore[no-redef]
42
+ from airflow.hooks.base import BaseHook # type: ignore[attr-defined,no-redef]
43
+ from airflow.models import BaseOperator
44
+ from airflow.utils.xcom import XCOM_RETURN_KEY # type: ignore[no-redef]
42
45
 
43
46
  if AIRFLOW_V_3_0_PLUS:
44
47
  from airflow.sdk import BaseSensorOperator
@@ -54,9 +57,11 @@ else:
54
57
  __all__ = [
55
58
  "AIRFLOW_V_3_0_PLUS",
56
59
  "AIRFLOW_V_3_1_PLUS",
60
+ "BaseHook",
57
61
  "BaseOperator",
58
62
  "BaseSensorOperator",
59
63
  "DecoratedOperator",
60
64
  "TaskDecorator",
61
65
  "task_decorator_factory",
66
+ "XCOM_RETURN_KEY",
62
67
  ]
@@ -1,11 +1,11 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: apache-airflow-providers-cncf-kubernetes
3
- Version: 10.6.1
3
+ Version: 10.6.2rc1
4
4
  Summary: Provider package apache-airflow-providers-cncf-kubernetes for Apache Airflow
5
5
  Keywords: airflow-provider,cncf.kubernetes,airflow,integration
6
6
  Author-email: Apache Software Foundation <dev@airflow.apache.org>
7
7
  Maintainer-email: Apache Software Foundation <dev@airflow.apache.org>
8
- Requires-Python: ~=3.10
8
+ Requires-Python: >=3.10
9
9
  Description-Content-Type: text/x-rst
10
10
  Classifier: Development Status :: 5 - Production/Stable
11
11
  Classifier: Environment :: Console
@@ -18,16 +18,17 @@ Classifier: License :: OSI Approved :: Apache Software License
18
18
  Classifier: Programming Language :: Python :: 3.10
19
19
  Classifier: Programming Language :: Python :: 3.11
20
20
  Classifier: Programming Language :: Python :: 3.12
21
+ Classifier: Programming Language :: Python :: 3.13
21
22
  Classifier: Topic :: System :: Monitoring
22
23
  Requires-Dist: aiofiles>=23.2.0
23
- Requires-Dist: apache-airflow>=2.10.0
24
+ Requires-Dist: apache-airflow>=2.10.0rc1
24
25
  Requires-Dist: asgiref>=3.5.2
25
26
  Requires-Dist: cryptography>=41.0.0
26
27
  Requires-Dist: kubernetes>=32.0.0,<33.0.0
27
28
  Requires-Dist: kubernetes_asyncio>=32.0.0,<33.0.0
28
29
  Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
29
- Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.6.1/changelog.html
30
- Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.6.1
30
+ Project-URL: Changelog, https://airflow.staged.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.6.2/changelog.html
31
+ Project-URL: Documentation, https://airflow.staged.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.6.2
31
32
  Project-URL: Mastodon, https://fosstodon.org/@airflow
32
33
  Project-URL: Slack Chat, https://s.apache.org/airflow-slack
33
34
  Project-URL: Source Code, https://github.com/apache/airflow
@@ -58,8 +59,9 @@ Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
58
59
 
59
60
  Package ``apache-airflow-providers-cncf-kubernetes``
60
61
 
61
- Release: ``10.6.1``
62
+ Release: ``10.6.2``
62
63
 
64
+ Release Date: ``|PypiReleaseDate|``
63
65
 
64
66
  `Kubernetes <https://kubernetes.io/>`__
65
67
 
@@ -71,7 +73,7 @@ This is a provider package for ``cncf.kubernetes`` provider. All classes for thi
71
73
  are in ``airflow.providers.cncf.kubernetes`` python package.
72
74
 
73
75
  You can find package information and changelog for the provider
74
- in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.6.1/>`_.
76
+ in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.6.2/>`_.
75
77
 
76
78
  Installation
77
79
  ------------
@@ -80,7 +82,7 @@ You can install this package on top of an existing Airflow 2 installation (see `
80
82
  for the minimum Airflow version supported) via
81
83
  ``pip install apache-airflow-providers-cncf-kubernetes``
82
84
 
83
- The package supports the following python versions: 3.10,3.11,3.12
85
+ The package supports the following python versions: 3.10,3.11,3.12,3.13
84
86
 
85
87
  Requirements
86
88
  ------------
@@ -97,5 +99,5 @@ PIP package Version required
97
99
  ====================== ====================
98
100
 
99
101
  The changelog for the provider package can be found in the
100
- `changelog <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.6.1/changelog.html>`_.
102
+ `changelog <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.6.2/changelog.html>`_.
101
103
 
@@ -1,5 +1,5 @@
1
1
  airflow/providers/cncf/kubernetes/LICENSE,sha256=gXPVwptPlW1TJ4HSuG5OMPg-a3h43OGMkZRR1rpwfJA,10850
2
- airflow/providers/cncf/kubernetes/__init__.py,sha256=AQ6Q8rLaXzQMcoW9BfUmJ16gyXwo5AfHF6jIzoMta7U,1505
2
+ airflow/providers/cncf/kubernetes/__init__.py,sha256=SbcfhJ9x-WDZ4lgAQmdWbnkHLZdEu8JX1WNoXkWIgiQ,1505
3
3
  airflow/providers/cncf/kubernetes/callbacks.py,sha256=1nCLXFJKtr5FM9ApB8Drw5VAGSC3TDFsPSTMtRnAR3Q,6085
4
4
  airflow/providers/cncf/kubernetes/exceptions.py,sha256=3cNEZTnrltBsqwzHiLfckwYYc_IWY1g4PcRs6zuMWWA,1137
5
5
  airflow/providers/cncf/kubernetes/get_provider_info.py,sha256=Git4HycOcHrb4zD9W7ZYsqNDkQSQ4uipSJO_GaPiroE,16041
@@ -12,29 +12,29 @@ airflow/providers/cncf/kubernetes/python_kubernetes_script.jinja2,sha256=I0EHRGw
12
12
  airflow/providers/cncf/kubernetes/python_kubernetes_script.py,sha256=KnTlZSWCZhwvj89fSc2kgIRTaI4iLNKPquHc2wXnluo,3460
13
13
  airflow/providers/cncf/kubernetes/secret.py,sha256=wj-T9gouqau_X14slAstGmnSxqXJQzdLwUdURzHna0I,5209
14
14
  airflow/providers/cncf/kubernetes/template_rendering.py,sha256=WSUBhjGSDhjNtA4IFlbYyX50rvYN6UA4dMk0cPqgOjo,3618
15
- airflow/providers/cncf/kubernetes/version_compat.py,sha256=UYay_aaghzMUY2canTQ26qMfsv2vvzb4nPAfhX8mXm4,2349
15
+ airflow/providers/cncf/kubernetes/version_compat.py,sha256=X10pKX3Re8KsThOJVvVsvDeDj33-RL0D2rl_-PPI9TM,2584
16
16
  airflow/providers/cncf/kubernetes/backcompat/__init__.py,sha256=KXF76f3v1jIFUBNz8kwxVMvm7i4mNo35LbIG9IijBNc,1299
17
17
  airflow/providers/cncf/kubernetes/backcompat/backwards_compat_converters.py,sha256=FkRRtIEucp2hYrecGVYVgyPI6-b7hE7X7L17Z3r459Y,4303
18
18
  airflow/providers/cncf/kubernetes/cli/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
19
- airflow/providers/cncf/kubernetes/cli/kubernetes_command.py,sha256=lvs9RJ-fjrCdisE2rnRJkxUxpn6rdtFX4j7D94ASO9c,7943
19
+ airflow/providers/cncf/kubernetes/cli/kubernetes_command.py,sha256=o6ABRTKrG4GExMEZ1ebVVTrJ2l3yPx270BNOP2kLfMI,8107
20
20
  airflow/providers/cncf/kubernetes/decorators/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
21
21
  airflow/providers/cncf/kubernetes/decorators/kubernetes.py,sha256=d27TR2k-NbpwQSwHd7L265ZZYXiRBlPg7na7RsrH1Ik,6216
22
22
  airflow/providers/cncf/kubernetes/decorators/kubernetes_cmd.py,sha256=C-G6_Ye42mfRTegOE0riO8NuWh2AaBuXCF3j6LPRKOI,4719
23
23
  airflow/providers/cncf/kubernetes/executors/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
24
- airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py,sha256=dpy1d0BU-pID_t9wPmi7h9PULbk9sQBIWqKxy9PNSB8,31953
24
+ airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py,sha256=W_TdvmD1pToHr6JQpUJr9YJNaQxjFDl6cDiWlrY3-DI,31927
25
25
  airflow/providers/cncf/kubernetes/executors/kubernetes_executor_types.py,sha256=9EpTm3u0R6FgX7L41PFLUT2FRunpt7AFfbHvcA2pXmA,2004
26
26
  airflow/providers/cncf/kubernetes/executors/kubernetes_executor_utils.py,sha256=1aoIOoL9-ufnvZxHOau-ZJtt14Xl2zxlah2SuSmNnp0,24461
27
27
  airflow/providers/cncf/kubernetes/executors/local_kubernetes_executor.py,sha256=CWCN4b6Ircs-3tCxJjBsrjl4Q0ABBJIwqlZr7a5lW6k,12243
28
28
  airflow/providers/cncf/kubernetes/hooks/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
29
- airflow/providers/cncf/kubernetes/hooks/kubernetes.py,sha256=33RZ1wlcuxDaVEu8z59cggbnKGluoTyTzAdkwBtMUZI,37243
29
+ airflow/providers/cncf/kubernetes/hooks/kubernetes.py,sha256=lLNGbLizQu-gRU6GDa3MTobqvbrWqFKIqbqK1SVEgr8,37192
30
30
  airflow/providers/cncf/kubernetes/kubernetes_executor_templates/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
31
31
  airflow/providers/cncf/kubernetes/kubernetes_executor_templates/basic_template.yaml,sha256=yzJmXN4ZyB4aDwI_GIugpL9-f1YMVy__X-LQSbeU95A,2567
32
32
  airflow/providers/cncf/kubernetes/operators/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
33
33
  airflow/providers/cncf/kubernetes/operators/custom_object_launcher.py,sha256=jTVHQt1vp5gELrLNyM-DrZ1ywgmTy3Hh1i6wyl7AGS0,15314
34
- airflow/providers/cncf/kubernetes/operators/job.py,sha256=pJUBp831gJmo4ppJS1Fc85uzTa4dKh-5r4LSn0crVGU,23805
34
+ airflow/providers/cncf/kubernetes/operators/job.py,sha256=lYUO_HAC_NyiyBZvNFZ_6Itk5bRCsI3BIyj3KZmHCpw,26775
35
35
  airflow/providers/cncf/kubernetes/operators/kueue.py,sha256=Xr-G4uvgSzmqm72LmUsRlTbuZ4hFG6GTI8lvdLoya9Y,4597
36
- airflow/providers/cncf/kubernetes/operators/pod.py,sha256=PDYTONjck2hQnWsK6P3pU4NgobCQ_gDpbXxqxivIn6Y,60732
37
- airflow/providers/cncf/kubernetes/operators/resource.py,sha256=9tfoxEyOTZ3d0PqejKW3aRxEyWTOjbjDnwQgMK3Ln6g,7613
36
+ airflow/providers/cncf/kubernetes/operators/pod.py,sha256=vzwZ6xeJY3goqND8Hop7veqZDNKri-igFjJhhpBrnY8,60959
37
+ airflow/providers/cncf/kubernetes/operators/resource.py,sha256=XQrlbLbk-tlN_CQnETa9hgFzxxL82hh-Fs0XM5SDhyg,7574
38
38
  airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py,sha256=lqZqehu-YUMDxFTwSlQm0e10pdoU5njus5TtMLOdEUI,13898
39
39
  airflow/providers/cncf/kubernetes/pod_template_file_examples/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
40
40
  airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_image_template.yaml,sha256=7JdppZ-XDBpv2Bnde2SthhcME8w3b8xQdPAK1fJGW60,2256
@@ -47,14 +47,14 @@ airflow/providers/cncf/kubernetes/resource_convert/secret.py,sha256=ElZCMbTWeTKo
47
47
  airflow/providers/cncf/kubernetes/sensors/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
48
48
  airflow/providers/cncf/kubernetes/sensors/spark_kubernetes.py,sha256=NrrUUXK1ctldF9j-pjEjJEvPQafH6b4rpLtSrJJVNEU,5384
49
49
  airflow/providers/cncf/kubernetes/triggers/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
50
- airflow/providers/cncf/kubernetes/triggers/job.py,sha256=DGbC1FZktBF-00Lb0pU9iIKQnmdW8HWklp5Wwq54OEY,6754
51
- airflow/providers/cncf/kubernetes/triggers/pod.py,sha256=YEyYTQONZu66uHGp_38N_A56txZNamA0Kc4UA0I3sJk,13414
50
+ airflow/providers/cncf/kubernetes/triggers/job.py,sha256=_lLP6ZYRV4kdwb7U0w5QFnlY1E9deZ5wtg-nrlfl6-8,7505
51
+ airflow/providers/cncf/kubernetes/triggers/pod.py,sha256=AVk0-dJN_wjMeZtImMxan4JZ7sSl-allC8ga1o3WhKM,13388
52
52
  airflow/providers/cncf/kubernetes/utils/__init__.py,sha256=ClZN0VPjWySdVwS_ktH7rrgL9VLAcs3OSJSB9s3zaYw,863
53
53
  airflow/providers/cncf/kubernetes/utils/delete_from.py,sha256=poObZSoEJwQyaYWilEURs8f4CDY2sn_pfwS31Lf579A,5195
54
54
  airflow/providers/cncf/kubernetes/utils/k8s_resource_iterator.py,sha256=pl-G-2WhZVbewKkwmL9AxPo1hAQWHHEPK43b-ruF4-w,1937
55
- airflow/providers/cncf/kubernetes/utils/pod_manager.py,sha256=TljcFQBXNHraXfoZ0UDT3iMSv7VlAJZwW-f6prJUmxA,40442
55
+ airflow/providers/cncf/kubernetes/utils/pod_manager.py,sha256=DTIwT1b-MGpiIAao5SuKD8Qgu3vlGZ-NYeR_Mf5W7rE,40434
56
56
  airflow/providers/cncf/kubernetes/utils/xcom_sidecar.py,sha256=k6bdmVJ21OrAwGmWwledRrAmaty9ZrmbuM-IbaI4mqo,2519
57
- apache_airflow_providers_cncf_kubernetes-10.6.1.dist-info/entry_points.txt,sha256=ByD3QJJyP9CfmTYtpNI1953akD38RUDgpGXLaq9vpOw,111
58
- apache_airflow_providers_cncf_kubernetes-10.6.1.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
59
- apache_airflow_providers_cncf_kubernetes-10.6.1.dist-info/METADATA,sha256=66ZN-e0KZ38i8TlI78jb0r3CDy1qjRRtEVs-VqN_71E,4248
60
- apache_airflow_providers_cncf_kubernetes-10.6.1.dist-info/RECORD,,
57
+ apache_airflow_providers_cncf_kubernetes-10.6.2rc1.dist-info/entry_points.txt,sha256=ByD3QJJyP9CfmTYtpNI1953akD38RUDgpGXLaq9vpOw,111
58
+ apache_airflow_providers_cncf_kubernetes-10.6.2rc1.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
59
+ apache_airflow_providers_cncf_kubernetes-10.6.2rc1.dist-info/METADATA,sha256=dni-tpQAUVmw38LhfONN7-hwe0LiDKBp9gTtocirMQM,4360
60
+ apache_airflow_providers_cncf_kubernetes-10.6.2rc1.dist-info/RECORD,,