apache-airflow-providers-cncf-kubernetes 10.6.1__py3-none-any.whl → 10.6.2rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/cncf/kubernetes/__init__.py +1 -1
- airflow/providers/cncf/kubernetes/cli/kubernetes_command.py +6 -1
- airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py +1 -1
- airflow/providers/cncf/kubernetes/hooks/kubernetes.py +2 -6
- airflow/providers/cncf/kubernetes/operators/job.py +100 -36
- airflow/providers/cncf/kubernetes/operators/pod.py +18 -16
- airflow/providers/cncf/kubernetes/operators/resource.py +3 -4
- airflow/providers/cncf/kubernetes/triggers/job.py +43 -22
- airflow/providers/cncf/kubernetes/triggers/pod.py +1 -1
- airflow/providers/cncf/kubernetes/utils/pod_manager.py +3 -3
- airflow/providers/cncf/kubernetes/version_compat.py +7 -2
- {apache_airflow_providers_cncf_kubernetes-10.6.1.dist-info → apache_airflow_providers_cncf_kubernetes-10.6.2rc1.dist-info}/METADATA +11 -9
- {apache_airflow_providers_cncf_kubernetes-10.6.1.dist-info → apache_airflow_providers_cncf_kubernetes-10.6.2rc1.dist-info}/RECORD +15 -15
- {apache_airflow_providers_cncf_kubernetes-10.6.1.dist-info → apache_airflow_providers_cncf_kubernetes-10.6.2rc1.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_cncf_kubernetes-10.6.1.dist-info → apache_airflow_providers_cncf_kubernetes-10.6.2rc1.dist-info}/entry_points.txt +0 -0
@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
|
|
29
29
|
|
30
30
|
__all__ = ["__version__"]
|
31
31
|
|
32
|
-
__version__ = "10.6.
|
32
|
+
__version__ = "10.6.2"
|
33
33
|
|
34
34
|
if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
|
35
35
|
"2.10.0"
|
@@ -66,7 +66,12 @@ def generate_pod_yaml(args):
|
|
66
66
|
kube_config = KubeConfig()
|
67
67
|
|
68
68
|
for task in dag.tasks:
|
69
|
-
|
69
|
+
if AIRFLOW_V_3_0_PLUS:
|
70
|
+
from uuid6 import uuid7
|
71
|
+
|
72
|
+
ti = TaskInstance(task, run_id=dr.run_id, dag_version_id=uuid7())
|
73
|
+
else:
|
74
|
+
ti = TaskInstance(task, run_id=dr.run_id)
|
70
75
|
ti.dag_run = dr
|
71
76
|
ti.dag_model = dm
|
72
77
|
|
@@ -307,7 +307,7 @@ class KubernetesExecutor(BaseExecutor):
|
|
307
307
|
executor_config = w.ti.executor_config or {}
|
308
308
|
|
309
309
|
del self.queued_tasks[key]
|
310
|
-
self.execute_async(key=key, command=command, queue=queue, executor_config=executor_config)
|
310
|
+
self.execute_async(key=key, command=command, queue=queue, executor_config=executor_config)
|
311
311
|
self.running.add(key)
|
312
312
|
|
313
313
|
def sync(self) -> None:
|
@@ -44,11 +44,7 @@ from airflow.providers.cncf.kubernetes.utils.pod_manager import (
|
|
44
44
|
container_is_completed,
|
45
45
|
container_is_running,
|
46
46
|
)
|
47
|
-
|
48
|
-
try:
|
49
|
-
from airflow.sdk import BaseHook
|
50
|
-
except ImportError:
|
51
|
-
from airflow.hooks.base import BaseHook # type: ignore[attr-defined,no-redef]
|
47
|
+
from airflow.providers.cncf.kubernetes.version_compat import BaseHook
|
52
48
|
from airflow.utils import yaml
|
53
49
|
|
54
50
|
if TYPE_CHECKING:
|
@@ -775,7 +771,7 @@ class AsyncKubernetesHook(KubernetesHook):
|
|
775
771
|
if self.config_dict:
|
776
772
|
self.log.debug(LOADING_KUBE_CONFIG_FILE_RESOURCE.format("config dictionary"))
|
777
773
|
self._is_in_cluster = False
|
778
|
-
await async_config.load_kube_config_from_dict(self.config_dict)
|
774
|
+
await async_config.load_kube_config_from_dict(self.config_dict, context=cluster_context)
|
779
775
|
return async_client.ApiClient()
|
780
776
|
|
781
777
|
if kubeconfig_path is not None:
|
@@ -22,16 +22,17 @@ import copy
|
|
22
22
|
import json
|
23
23
|
import logging
|
24
24
|
import os
|
25
|
+
import warnings
|
25
26
|
from collections.abc import Sequence
|
26
27
|
from functools import cached_property
|
27
|
-
from typing import TYPE_CHECKING
|
28
|
+
from typing import TYPE_CHECKING, Any, Literal
|
28
29
|
|
29
30
|
from kubernetes.client import BatchV1Api, models as k8s
|
30
31
|
from kubernetes.client.api_client import ApiClient
|
31
32
|
from kubernetes.client.rest import ApiException
|
32
33
|
|
33
34
|
from airflow.configuration import conf
|
34
|
-
from airflow.exceptions import AirflowException
|
35
|
+
from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
|
35
36
|
from airflow.providers.cncf.kubernetes.hooks.kubernetes import KubernetesHook
|
36
37
|
from airflow.providers.cncf.kubernetes.kubernetes_helper_functions import (
|
37
38
|
add_unique_suffix,
|
@@ -81,6 +82,17 @@ class KubernetesJobOperator(KubernetesPodOperator):
|
|
81
82
|
Used if the parameter `wait_until_job_complete` set True.
|
82
83
|
:param deferrable: Run operator in the deferrable mode. Note that the parameter
|
83
84
|
`wait_until_job_complete` must be set True.
|
85
|
+
:param on_kill_propagation_policy: Whether and how garbage collection will be performed. Default is 'Foreground'.
|
86
|
+
Acceptable values are:
|
87
|
+
'Orphan' - orphan the dependents;
|
88
|
+
'Background' - allow the garbage collector to delete the dependents in the background;
|
89
|
+
'Foreground' - a cascading policy that deletes all dependents in the foreground.
|
90
|
+
Default value is 'Foreground'.
|
91
|
+
:param discover_pods_retry_number: Number of time list_namespaced_pod will be performed to discover
|
92
|
+
already running pods.
|
93
|
+
:param unwrap_single: Unwrap single result from the pod. For example, when set to `True` - if the XCom
|
94
|
+
result should be `['res']`, the final result would be `'res'`. Default is True to support backward
|
95
|
+
compatibility.
|
84
96
|
"""
|
85
97
|
|
86
98
|
template_fields: Sequence[str] = tuple({"job_template_file"} | set(KubernetesPodOperator.template_fields))
|
@@ -101,8 +113,12 @@ class KubernetesJobOperator(KubernetesPodOperator):
|
|
101
113
|
wait_until_job_complete: bool = False,
|
102
114
|
job_poll_interval: float = 10,
|
103
115
|
deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
|
116
|
+
on_kill_propagation_policy: Literal["Foreground", "Background", "Orphan"] = "Foreground",
|
117
|
+
discover_pods_retry_number: int = 3,
|
118
|
+
unwrap_single: bool = True,
|
104
119
|
**kwargs,
|
105
120
|
) -> None:
|
121
|
+
self._pod = None
|
106
122
|
super().__init__(**kwargs)
|
107
123
|
self.job_template_file = job_template_file
|
108
124
|
self.full_job_spec = full_job_spec
|
@@ -119,6 +135,22 @@ class KubernetesJobOperator(KubernetesPodOperator):
|
|
119
135
|
self.wait_until_job_complete = wait_until_job_complete
|
120
136
|
self.job_poll_interval = job_poll_interval
|
121
137
|
self.deferrable = deferrable
|
138
|
+
self.on_kill_propagation_policy = on_kill_propagation_policy
|
139
|
+
self.discover_pods_retry_number = discover_pods_retry_number
|
140
|
+
self.unwrap_single = unwrap_single
|
141
|
+
|
142
|
+
@property
|
143
|
+
def pod(self):
|
144
|
+
warnings.warn(
|
145
|
+
"`pod` parameter is deprecated, please use `pods`",
|
146
|
+
AirflowProviderDeprecationWarning,
|
147
|
+
stacklevel=2,
|
148
|
+
)
|
149
|
+
return self.pods[0] if self.pods else None
|
150
|
+
|
151
|
+
@pod.setter
|
152
|
+
def pod(self, value):
|
153
|
+
self._pod = value
|
122
154
|
|
123
155
|
@cached_property
|
124
156
|
def _incluster_namespace(self):
|
@@ -167,12 +199,16 @@ class KubernetesJobOperator(KubernetesPodOperator):
|
|
167
199
|
ti.xcom_push(key="job_name", value=self.job.metadata.name)
|
168
200
|
ti.xcom_push(key="job_namespace", value=self.job.metadata.namespace)
|
169
201
|
|
170
|
-
self.
|
171
|
-
if self.pod is None:
|
172
|
-
self.
|
173
|
-
|
174
|
-
|
175
|
-
|
202
|
+
self.pods: Sequence[k8s.V1Pod] | None = None
|
203
|
+
if self.parallelism is None and self.pod is None:
|
204
|
+
self.pods = [
|
205
|
+
self.get_or_create_pod(
|
206
|
+
pod_request_obj=self.pod_request_obj,
|
207
|
+
context=context,
|
208
|
+
)
|
209
|
+
]
|
210
|
+
else:
|
211
|
+
self.pods = self.get_pods(pod_request_obj=self.pod_request_obj, context=context)
|
176
212
|
|
177
213
|
if self.wait_until_job_complete and self.deferrable:
|
178
214
|
self.execute_deferrable()
|
@@ -180,22 +216,25 @@ class KubernetesJobOperator(KubernetesPodOperator):
|
|
180
216
|
|
181
217
|
if self.wait_until_job_complete:
|
182
218
|
if self.do_xcom_push:
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
219
|
+
xcom_result = []
|
220
|
+
for pod in self.pods:
|
221
|
+
self.pod_manager.await_container_completion(
|
222
|
+
pod=pod, container_name=self.base_container_name
|
223
|
+
)
|
224
|
+
self.pod_manager.await_xcom_sidecar_container_start(pod=pod)
|
225
|
+
xcom_result.append(self.extract_xcom(pod=pod))
|
188
226
|
self.job = self.hook.wait_until_job_complete(
|
189
227
|
job_name=self.job.metadata.name,
|
190
228
|
namespace=self.job.metadata.namespace,
|
191
229
|
job_poll_interval=self.job_poll_interval,
|
192
230
|
)
|
193
231
|
if self.get_logs:
|
194
|
-
self.
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
232
|
+
for pod in self.pods:
|
233
|
+
self.pod_manager.fetch_requested_container_logs(
|
234
|
+
pod=pod,
|
235
|
+
containers=self.container_logs,
|
236
|
+
follow_logs=True,
|
237
|
+
)
|
199
238
|
|
200
239
|
ti.xcom_push(key="job", value=self.job.to_dict())
|
201
240
|
if self.wait_until_job_complete:
|
@@ -209,10 +248,10 @@ class KubernetesJobOperator(KubernetesPodOperator):
|
|
209
248
|
def execute_deferrable(self):
|
210
249
|
self.defer(
|
211
250
|
trigger=KubernetesJobTrigger(
|
212
|
-
job_name=self.job.metadata.name,
|
213
|
-
job_namespace=self.job.metadata.namespace,
|
214
|
-
|
215
|
-
pod_namespace=self.
|
251
|
+
job_name=self.job.metadata.name,
|
252
|
+
job_namespace=self.job.metadata.namespace,
|
253
|
+
pod_names=[pod.metadata.name for pod in self.pods],
|
254
|
+
pod_namespace=self.pods[0].metadata.namespace,
|
216
255
|
base_container_name=self.base_container_name,
|
217
256
|
kubernetes_conn_id=self.kubernetes_conn_id,
|
218
257
|
cluster_context=self.cluster_context,
|
@@ -232,20 +271,23 @@ class KubernetesJobOperator(KubernetesPodOperator):
|
|
232
271
|
raise AirflowException(event["message"])
|
233
272
|
|
234
273
|
if self.get_logs:
|
235
|
-
pod_name
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
274
|
+
for pod_name in event["pod_names"]:
|
275
|
+
pod_namespace = event["pod_namespace"]
|
276
|
+
pod = self.hook.get_pod(pod_name, pod_namespace)
|
277
|
+
if not pod:
|
278
|
+
raise PodNotFoundException("Could not find pod after resuming from deferral")
|
279
|
+
self._write_logs(pod)
|
241
280
|
|
242
281
|
if self.do_xcom_push:
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
282
|
+
xcom_results: list[Any | None] = []
|
283
|
+
for xcom_result in event["xcom_result"]:
|
284
|
+
if isinstance(xcom_result, str) and xcom_result.rstrip() == EMPTY_XCOM_RESULT:
|
285
|
+
self.log.info("xcom result file is empty.")
|
286
|
+
xcom_results.append(None)
|
287
|
+
continue
|
288
|
+
self.log.info("xcom result: \n%s", xcom_result)
|
289
|
+
xcom_results.append(json.loads(xcom_result))
|
290
|
+
return xcom_results[0] if self.unwrap_single and len(xcom_results) == 1 else xcom_results
|
249
291
|
|
250
292
|
@staticmethod
|
251
293
|
def deserialize_job_template_file(path: str) -> k8s.V1Job:
|
@@ -275,12 +317,11 @@ class KubernetesJobOperator(KubernetesPodOperator):
|
|
275
317
|
kwargs = {
|
276
318
|
"name": job.metadata.name,
|
277
319
|
"namespace": job.metadata.namespace,
|
320
|
+
"propagation_policy": self.on_kill_propagation_policy,
|
278
321
|
}
|
279
322
|
if self.termination_grace_period is not None:
|
280
323
|
kwargs.update(grace_period_seconds=self.termination_grace_period)
|
281
324
|
self.job_client.delete_namespaced_job(**kwargs)
|
282
|
-
if self.pod:
|
283
|
-
super().on_kill()
|
284
325
|
|
285
326
|
def build_job_request_obj(self, context: Context | None = None) -> k8s.V1Job:
|
286
327
|
"""
|
@@ -400,6 +441,29 @@ class KubernetesJobOperator(KubernetesPodOperator):
|
|
400
441
|
|
401
442
|
return None
|
402
443
|
|
444
|
+
def get_pods(
|
445
|
+
self, pod_request_obj: k8s.V1Pod, context: Context, *, exclude_checked: bool = True
|
446
|
+
) -> Sequence[k8s.V1Pod]:
|
447
|
+
"""Return an already-running pods if exists."""
|
448
|
+
label_selector = self._build_find_pod_label_selector(context, exclude_checked=exclude_checked)
|
449
|
+
pod_list: Sequence[k8s.V1Pod] = []
|
450
|
+
retry_number: int = 0
|
451
|
+
|
452
|
+
while len(pod_list) != self.parallelism or retry_number <= self.discover_pods_retry_number:
|
453
|
+
pod_list = self.client.list_namespaced_pod(
|
454
|
+
namespace=pod_request_obj.metadata.namespace,
|
455
|
+
label_selector=label_selector,
|
456
|
+
).items
|
457
|
+
retry_number += 1
|
458
|
+
|
459
|
+
if len(pod_list) == 0:
|
460
|
+
raise AirflowException(f"No pods running with labels {label_selector}")
|
461
|
+
|
462
|
+
for pod_instance in pod_list:
|
463
|
+
self.log_matching_pod(pod=pod_instance, context=context)
|
464
|
+
|
465
|
+
return pod_list
|
466
|
+
|
403
467
|
|
404
468
|
class KubernetesDeleteJobOperator(BaseOperator):
|
405
469
|
"""
|
@@ -68,7 +68,7 @@ from airflow.providers.cncf.kubernetes.kubernetes_helper_functions import (
|
|
68
68
|
)
|
69
69
|
from airflow.providers.cncf.kubernetes.pod_generator import PodGenerator
|
70
70
|
from airflow.providers.cncf.kubernetes.triggers.pod import KubernetesPodTrigger
|
71
|
-
from airflow.providers.cncf.kubernetes.utils import xcom_sidecar
|
71
|
+
from airflow.providers.cncf.kubernetes.utils import xcom_sidecar
|
72
72
|
from airflow.providers.cncf.kubernetes.utils.pod_manager import (
|
73
73
|
EMPTY_XCOM_RESULT,
|
74
74
|
OnFinishAction,
|
@@ -80,11 +80,10 @@ from airflow.providers.cncf.kubernetes.utils.pod_manager import (
|
|
80
80
|
container_is_succeeded,
|
81
81
|
get_container_termination_message,
|
82
82
|
)
|
83
|
-
from airflow.providers.cncf.kubernetes.version_compat import BaseOperator
|
83
|
+
from airflow.providers.cncf.kubernetes.version_compat import XCOM_RETURN_KEY, BaseOperator
|
84
84
|
from airflow.settings import pod_mutation_hook
|
85
85
|
from airflow.utils import yaml
|
86
86
|
from airflow.utils.helpers import prune_dict, validate_key
|
87
|
-
from airflow.utils.xcom import XCOM_RETURN_KEY
|
88
87
|
from airflow.version import version as airflow_version
|
89
88
|
|
90
89
|
if TYPE_CHECKING:
|
@@ -579,10 +578,15 @@ class KubernetesPodOperator(BaseOperator):
|
|
579
578
|
pod = self.find_pod(pod_request_obj.metadata.namespace, context=context)
|
580
579
|
if pod:
|
581
580
|
# If pod is terminated then delete the pod an create a new as not possible to get xcom
|
582
|
-
pod_phase =
|
583
|
-
|
584
|
-
)
|
585
|
-
|
581
|
+
pod_phase = pod.status.phase if pod.status and pod.status.phase else None
|
582
|
+
pod_reason = pod.status.reason.lower() if pod.status and pod.status.reason else ""
|
583
|
+
if pod_phase not in (PodPhase.SUCCEEDED, PodPhase.FAILED) and pod_reason != "evicted":
|
584
|
+
self.log.info(
|
585
|
+
"Reusing existing pod '%s' (phase=%s, reason=%s) since it is not terminated or evicted.",
|
586
|
+
pod.metadata.name,
|
587
|
+
pod_phase,
|
588
|
+
pod_reason,
|
589
|
+
)
|
586
590
|
return pod
|
587
591
|
|
588
592
|
self.log.info(
|
@@ -604,20 +608,18 @@ class KubernetesPodOperator(BaseOperator):
|
|
604
608
|
|
605
609
|
def await_pod_start(self, pod: k8s.V1Pod) -> None:
|
606
610
|
try:
|
607
|
-
|
608
|
-
|
609
|
-
self.pod_manager.watch_pod_events(pod, self.startup_check_interval_seconds)
|
610
|
-
|
611
|
-
loop.run_until_complete(
|
612
|
-
self.pod_manager.await_pod_start(
|
611
|
+
|
612
|
+
async def _await_pod_start():
|
613
|
+
events_task = self.pod_manager.watch_pod_events(pod, self.startup_check_interval_seconds)
|
614
|
+
pod_start_task = self.pod_manager.await_pod_start(
|
613
615
|
pod=pod,
|
614
616
|
schedule_timeout=self.schedule_timeout_seconds,
|
615
617
|
startup_timeout=self.startup_timeout_seconds,
|
616
618
|
check_interval=self.startup_check_interval_seconds,
|
617
619
|
)
|
618
|
-
|
619
|
-
|
620
|
-
|
620
|
+
await asyncio.gather(pod_start_task, events_task)
|
621
|
+
|
622
|
+
asyncio.run(_await_pod_start())
|
621
623
|
except PodLaunchFailedException:
|
622
624
|
if self.log_events_on_failure:
|
623
625
|
self._read_pod_events(pod, reraise=False)
|
@@ -106,10 +106,9 @@ class KubernetesResourceBaseOperator(BaseOperator):
|
|
106
106
|
group = api_version[0 : api_version.find("/")]
|
107
107
|
version = api_version[api_version.find("/") + 1 :]
|
108
108
|
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
namespace = metadata.get("namespace", None)
|
109
|
+
metadata = body.get("metadata", {}) if body else None
|
110
|
+
namespace = metadata.get("namespace") if metadata else None
|
111
|
+
|
113
112
|
if namespace is None:
|
114
113
|
namespace = self.get_namespace()
|
115
114
|
|
@@ -17,10 +17,12 @@
|
|
17
17
|
from __future__ import annotations
|
18
18
|
|
19
19
|
import asyncio
|
20
|
+
import warnings
|
20
21
|
from collections.abc import AsyncIterator
|
21
22
|
from functools import cached_property
|
22
23
|
from typing import TYPE_CHECKING, Any
|
23
24
|
|
25
|
+
from airflow.exceptions import AirflowProviderDeprecationWarning
|
24
26
|
from airflow.providers.cncf.kubernetes.hooks.kubernetes import AsyncKubernetesHook, KubernetesHook
|
25
27
|
from airflow.providers.cncf.kubernetes.utils.pod_manager import PodManager
|
26
28
|
from airflow.providers.cncf.kubernetes.utils.xcom_sidecar import PodDefaults
|
@@ -36,7 +38,8 @@ class KubernetesJobTrigger(BaseTrigger):
|
|
36
38
|
|
37
39
|
:param job_name: The name of the job.
|
38
40
|
:param job_namespace: The namespace of the job.
|
39
|
-
:param pod_name: The name of the Pod.
|
41
|
+
:param pod_name: The name of the Pod. Parameter is deprecated, please use pod_names instead.
|
42
|
+
:param pod_names: The name of the Pods.
|
40
43
|
:param pod_namespace: The namespace of the Pod.
|
41
44
|
:param base_container_name: The name of the base container in the pod.
|
42
45
|
:param kubernetes_conn_id: The :ref:`kubernetes connection id <howto/connection:kubernetes>`
|
@@ -55,9 +58,10 @@ class KubernetesJobTrigger(BaseTrigger):
|
|
55
58
|
self,
|
56
59
|
job_name: str,
|
57
60
|
job_namespace: str,
|
58
|
-
|
61
|
+
pod_names: list[str],
|
59
62
|
pod_namespace: str,
|
60
63
|
base_container_name: str,
|
64
|
+
pod_name: str | None = None,
|
61
65
|
kubernetes_conn_id: str | None = None,
|
62
66
|
poll_interval: float = 10.0,
|
63
67
|
cluster_context: str | None = None,
|
@@ -69,7 +73,13 @@ class KubernetesJobTrigger(BaseTrigger):
|
|
69
73
|
super().__init__()
|
70
74
|
self.job_name = job_name
|
71
75
|
self.job_namespace = job_namespace
|
72
|
-
|
76
|
+
if pod_name is not None:
|
77
|
+
self._pod_name = pod_name
|
78
|
+
self.pod_names = [
|
79
|
+
self.pod_name,
|
80
|
+
]
|
81
|
+
else:
|
82
|
+
self.pod_names = pod_names
|
73
83
|
self.pod_namespace = pod_namespace
|
74
84
|
self.base_container_name = base_container_name
|
75
85
|
self.kubernetes_conn_id = kubernetes_conn_id
|
@@ -80,6 +90,15 @@ class KubernetesJobTrigger(BaseTrigger):
|
|
80
90
|
self.get_logs = get_logs
|
81
91
|
self.do_xcom_push = do_xcom_push
|
82
92
|
|
93
|
+
@property
|
94
|
+
def pod_name(self):
|
95
|
+
warnings.warn(
|
96
|
+
"`pod_name` parameter is deprecated, please use `pod_names`",
|
97
|
+
AirflowProviderDeprecationWarning,
|
98
|
+
stacklevel=2,
|
99
|
+
)
|
100
|
+
return self._pod_name
|
101
|
+
|
83
102
|
def serialize(self) -> tuple[str, dict[str, Any]]:
|
84
103
|
"""Serialize KubernetesCreateJobTrigger arguments and classpath."""
|
85
104
|
return (
|
@@ -87,7 +106,7 @@ class KubernetesJobTrigger(BaseTrigger):
|
|
87
106
|
{
|
88
107
|
"job_name": self.job_name,
|
89
108
|
"job_namespace": self.job_namespace,
|
90
|
-
"
|
109
|
+
"pod_names": self.pod_names,
|
91
110
|
"pod_namespace": self.pod_namespace,
|
92
111
|
"base_container_name": self.base_container_name,
|
93
112
|
"kubernetes_conn_id": self.kubernetes_conn_id,
|
@@ -100,23 +119,25 @@ class KubernetesJobTrigger(BaseTrigger):
|
|
100
119
|
},
|
101
120
|
)
|
102
121
|
|
103
|
-
async def run(self) -> AsyncIterator[TriggerEvent]:
|
122
|
+
async def run(self) -> AsyncIterator[TriggerEvent]:
|
104
123
|
"""Get current job status and yield a TriggerEvent."""
|
105
|
-
if self.get_logs or self.do_xcom_push:
|
106
|
-
pod = await self.hook.get_pod(name=self.pod_name, namespace=self.pod_namespace)
|
107
124
|
if self.do_xcom_push:
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
125
|
+
xcom_results = []
|
126
|
+
for pod_name in self.pod_names:
|
127
|
+
pod = await self.hook.get_pod(name=pod_name, namespace=self.pod_namespace)
|
128
|
+
await self.hook.wait_until_container_complete(
|
129
|
+
name=pod_name, namespace=self.pod_namespace, container_name=self.base_container_name
|
130
|
+
)
|
131
|
+
self.log.info("Checking if xcom sidecar container is started.")
|
132
|
+
await self.hook.wait_until_container_started(
|
133
|
+
name=pod_name,
|
134
|
+
namespace=self.pod_namespace,
|
135
|
+
container_name=PodDefaults.SIDECAR_CONTAINER_NAME,
|
136
|
+
)
|
137
|
+
self.log.info("Extracting result from xcom sidecar container.")
|
138
|
+
loop = asyncio.get_running_loop()
|
139
|
+
xcom_result = await loop.run_in_executor(None, self.pod_manager.extract_xcom, pod)
|
140
|
+
xcom_results.append(xcom_result)
|
120
141
|
job: V1Job = await self.hook.wait_until_job_complete(name=self.job_name, namespace=self.job_namespace)
|
121
142
|
job_dict = job.to_dict()
|
122
143
|
error_message = self.hook.is_job_failed(job=job)
|
@@ -124,14 +145,14 @@ class KubernetesJobTrigger(BaseTrigger):
|
|
124
145
|
{
|
125
146
|
"name": job.metadata.name,
|
126
147
|
"namespace": job.metadata.namespace,
|
127
|
-
"
|
128
|
-
"pod_namespace":
|
148
|
+
"pod_names": [pod_name for pod_name in self.pod_names] if self.get_logs else None,
|
149
|
+
"pod_namespace": self.pod_namespace if self.get_logs else None,
|
129
150
|
"status": "error" if error_message else "success",
|
130
151
|
"message": f"Job failed with error: {error_message}"
|
131
152
|
if error_message
|
132
153
|
else "Job completed successfully",
|
133
154
|
"job": job_dict,
|
134
|
-
"xcom_result":
|
155
|
+
"xcom_result": xcom_results if self.do_xcom_push else None,
|
135
156
|
}
|
136
157
|
)
|
137
158
|
|
@@ -141,7 +141,7 @@ class KubernetesPodTrigger(BaseTrigger):
|
|
141
141
|
},
|
142
142
|
)
|
143
143
|
|
144
|
-
async def run(self) -> AsyncIterator[TriggerEvent]:
|
144
|
+
async def run(self) -> AsyncIterator[TriggerEvent]:
|
145
145
|
"""Get current pod status and yield a TriggerEvent."""
|
146
146
|
self.log.info("Checking pod %r in namespace %r.", self.pod_name, self.pod_namespace)
|
147
147
|
try:
|
@@ -338,6 +338,7 @@ class PodManager(LoggingMixin):
|
|
338
338
|
self._client = kube_client
|
339
339
|
self._watch = watch.Watch()
|
340
340
|
self._callbacks = callbacks or []
|
341
|
+
self.stop_watching_events = False
|
341
342
|
|
342
343
|
def run_pod_async(self, pod: V1Pod, **kwargs) -> V1Pod:
|
343
344
|
"""Run POD asynchronously."""
|
@@ -380,9 +381,8 @@ class PodManager(LoggingMixin):
|
|
380
381
|
|
381
382
|
async def watch_pod_events(self, pod: V1Pod, check_interval: int = 1) -> None:
|
382
383
|
"""Read pod events and writes into log."""
|
383
|
-
self.keep_watching_for_events = True
|
384
384
|
num_events = 0
|
385
|
-
while self.
|
385
|
+
while not self.stop_watching_events:
|
386
386
|
events = self.read_pod_events(pod)
|
387
387
|
for new_event in events.items[num_events:]:
|
388
388
|
involved_object: V1ObjectReference = new_event.involved_object
|
@@ -413,7 +413,7 @@ class PodManager(LoggingMixin):
|
|
413
413
|
remote_pod = self.read_pod(pod)
|
414
414
|
pod_status = remote_pod.status
|
415
415
|
if pod_status.phase != PodPhase.PENDING:
|
416
|
-
self.
|
416
|
+
self.stop_watching_events = True
|
417
417
|
self.log.info("::endgroup::")
|
418
418
|
break
|
419
419
|
|
@@ -36,9 +36,12 @@ AIRFLOW_V_3_0_PLUS = get_base_airflow_version_tuple() >= (3, 0, 0)
|
|
36
36
|
AIRFLOW_V_3_1_PLUS = get_base_airflow_version_tuple() >= (3, 1, 0)
|
37
37
|
|
38
38
|
if AIRFLOW_V_3_1_PLUS:
|
39
|
-
from airflow.
|
39
|
+
from airflow.models.xcom import XCOM_RETURN_KEY
|
40
|
+
from airflow.sdk import BaseHook, BaseOperator
|
40
41
|
else:
|
41
|
-
from airflow.
|
42
|
+
from airflow.hooks.base import BaseHook # type: ignore[attr-defined,no-redef]
|
43
|
+
from airflow.models import BaseOperator
|
44
|
+
from airflow.utils.xcom import XCOM_RETURN_KEY # type: ignore[no-redef]
|
42
45
|
|
43
46
|
if AIRFLOW_V_3_0_PLUS:
|
44
47
|
from airflow.sdk import BaseSensorOperator
|
@@ -54,9 +57,11 @@ else:
|
|
54
57
|
__all__ = [
|
55
58
|
"AIRFLOW_V_3_0_PLUS",
|
56
59
|
"AIRFLOW_V_3_1_PLUS",
|
60
|
+
"BaseHook",
|
57
61
|
"BaseOperator",
|
58
62
|
"BaseSensorOperator",
|
59
63
|
"DecoratedOperator",
|
60
64
|
"TaskDecorator",
|
61
65
|
"task_decorator_factory",
|
66
|
+
"XCOM_RETURN_KEY",
|
62
67
|
]
|
@@ -1,11 +1,11 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: apache-airflow-providers-cncf-kubernetes
|
3
|
-
Version: 10.6.
|
3
|
+
Version: 10.6.2rc1
|
4
4
|
Summary: Provider package apache-airflow-providers-cncf-kubernetes for Apache Airflow
|
5
5
|
Keywords: airflow-provider,cncf.kubernetes,airflow,integration
|
6
6
|
Author-email: Apache Software Foundation <dev@airflow.apache.org>
|
7
7
|
Maintainer-email: Apache Software Foundation <dev@airflow.apache.org>
|
8
|
-
Requires-Python:
|
8
|
+
Requires-Python: >=3.10
|
9
9
|
Description-Content-Type: text/x-rst
|
10
10
|
Classifier: Development Status :: 5 - Production/Stable
|
11
11
|
Classifier: Environment :: Console
|
@@ -18,16 +18,17 @@ Classifier: License :: OSI Approved :: Apache Software License
|
|
18
18
|
Classifier: Programming Language :: Python :: 3.10
|
19
19
|
Classifier: Programming Language :: Python :: 3.11
|
20
20
|
Classifier: Programming Language :: Python :: 3.12
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
21
22
|
Classifier: Topic :: System :: Monitoring
|
22
23
|
Requires-Dist: aiofiles>=23.2.0
|
23
|
-
Requires-Dist: apache-airflow>=2.10.
|
24
|
+
Requires-Dist: apache-airflow>=2.10.0rc1
|
24
25
|
Requires-Dist: asgiref>=3.5.2
|
25
26
|
Requires-Dist: cryptography>=41.0.0
|
26
27
|
Requires-Dist: kubernetes>=32.0.0,<33.0.0
|
27
28
|
Requires-Dist: kubernetes_asyncio>=32.0.0,<33.0.0
|
28
29
|
Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
|
29
|
-
Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.6.
|
30
|
-
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.6.
|
30
|
+
Project-URL: Changelog, https://airflow.staged.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.6.2/changelog.html
|
31
|
+
Project-URL: Documentation, https://airflow.staged.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.6.2
|
31
32
|
Project-URL: Mastodon, https://fosstodon.org/@airflow
|
32
33
|
Project-URL: Slack Chat, https://s.apache.org/airflow-slack
|
33
34
|
Project-URL: Source Code, https://github.com/apache/airflow
|
@@ -58,8 +59,9 @@ Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
|
|
58
59
|
|
59
60
|
Package ``apache-airflow-providers-cncf-kubernetes``
|
60
61
|
|
61
|
-
Release: ``10.6.
|
62
|
+
Release: ``10.6.2``
|
62
63
|
|
64
|
+
Release Date: ``|PypiReleaseDate|``
|
63
65
|
|
64
66
|
`Kubernetes <https://kubernetes.io/>`__
|
65
67
|
|
@@ -71,7 +73,7 @@ This is a provider package for ``cncf.kubernetes`` provider. All classes for thi
|
|
71
73
|
are in ``airflow.providers.cncf.kubernetes`` python package.
|
72
74
|
|
73
75
|
You can find package information and changelog for the provider
|
74
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.6.
|
76
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.6.2/>`_.
|
75
77
|
|
76
78
|
Installation
|
77
79
|
------------
|
@@ -80,7 +82,7 @@ You can install this package on top of an existing Airflow 2 installation (see `
|
|
80
82
|
for the minimum Airflow version supported) via
|
81
83
|
``pip install apache-airflow-providers-cncf-kubernetes``
|
82
84
|
|
83
|
-
The package supports the following python versions: 3.10,3.11,3.12
|
85
|
+
The package supports the following python versions: 3.10,3.11,3.12,3.13
|
84
86
|
|
85
87
|
Requirements
|
86
88
|
------------
|
@@ -97,5 +99,5 @@ PIP package Version required
|
|
97
99
|
====================== ====================
|
98
100
|
|
99
101
|
The changelog for the provider package can be found in the
|
100
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.6.
|
102
|
+
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.6.2/changelog.html>`_.
|
101
103
|
|
@@ -1,5 +1,5 @@
|
|
1
1
|
airflow/providers/cncf/kubernetes/LICENSE,sha256=gXPVwptPlW1TJ4HSuG5OMPg-a3h43OGMkZRR1rpwfJA,10850
|
2
|
-
airflow/providers/cncf/kubernetes/__init__.py,sha256=
|
2
|
+
airflow/providers/cncf/kubernetes/__init__.py,sha256=SbcfhJ9x-WDZ4lgAQmdWbnkHLZdEu8JX1WNoXkWIgiQ,1505
|
3
3
|
airflow/providers/cncf/kubernetes/callbacks.py,sha256=1nCLXFJKtr5FM9ApB8Drw5VAGSC3TDFsPSTMtRnAR3Q,6085
|
4
4
|
airflow/providers/cncf/kubernetes/exceptions.py,sha256=3cNEZTnrltBsqwzHiLfckwYYc_IWY1g4PcRs6zuMWWA,1137
|
5
5
|
airflow/providers/cncf/kubernetes/get_provider_info.py,sha256=Git4HycOcHrb4zD9W7ZYsqNDkQSQ4uipSJO_GaPiroE,16041
|
@@ -12,29 +12,29 @@ airflow/providers/cncf/kubernetes/python_kubernetes_script.jinja2,sha256=I0EHRGw
|
|
12
12
|
airflow/providers/cncf/kubernetes/python_kubernetes_script.py,sha256=KnTlZSWCZhwvj89fSc2kgIRTaI4iLNKPquHc2wXnluo,3460
|
13
13
|
airflow/providers/cncf/kubernetes/secret.py,sha256=wj-T9gouqau_X14slAstGmnSxqXJQzdLwUdURzHna0I,5209
|
14
14
|
airflow/providers/cncf/kubernetes/template_rendering.py,sha256=WSUBhjGSDhjNtA4IFlbYyX50rvYN6UA4dMk0cPqgOjo,3618
|
15
|
-
airflow/providers/cncf/kubernetes/version_compat.py,sha256=
|
15
|
+
airflow/providers/cncf/kubernetes/version_compat.py,sha256=X10pKX3Re8KsThOJVvVsvDeDj33-RL0D2rl_-PPI9TM,2584
|
16
16
|
airflow/providers/cncf/kubernetes/backcompat/__init__.py,sha256=KXF76f3v1jIFUBNz8kwxVMvm7i4mNo35LbIG9IijBNc,1299
|
17
17
|
airflow/providers/cncf/kubernetes/backcompat/backwards_compat_converters.py,sha256=FkRRtIEucp2hYrecGVYVgyPI6-b7hE7X7L17Z3r459Y,4303
|
18
18
|
airflow/providers/cncf/kubernetes/cli/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
19
|
-
airflow/providers/cncf/kubernetes/cli/kubernetes_command.py,sha256=
|
19
|
+
airflow/providers/cncf/kubernetes/cli/kubernetes_command.py,sha256=o6ABRTKrG4GExMEZ1ebVVTrJ2l3yPx270BNOP2kLfMI,8107
|
20
20
|
airflow/providers/cncf/kubernetes/decorators/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
|
21
21
|
airflow/providers/cncf/kubernetes/decorators/kubernetes.py,sha256=d27TR2k-NbpwQSwHd7L265ZZYXiRBlPg7na7RsrH1Ik,6216
|
22
22
|
airflow/providers/cncf/kubernetes/decorators/kubernetes_cmd.py,sha256=C-G6_Ye42mfRTegOE0riO8NuWh2AaBuXCF3j6LPRKOI,4719
|
23
23
|
airflow/providers/cncf/kubernetes/executors/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
|
24
|
-
airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py,sha256=
|
24
|
+
airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py,sha256=W_TdvmD1pToHr6JQpUJr9YJNaQxjFDl6cDiWlrY3-DI,31927
|
25
25
|
airflow/providers/cncf/kubernetes/executors/kubernetes_executor_types.py,sha256=9EpTm3u0R6FgX7L41PFLUT2FRunpt7AFfbHvcA2pXmA,2004
|
26
26
|
airflow/providers/cncf/kubernetes/executors/kubernetes_executor_utils.py,sha256=1aoIOoL9-ufnvZxHOau-ZJtt14Xl2zxlah2SuSmNnp0,24461
|
27
27
|
airflow/providers/cncf/kubernetes/executors/local_kubernetes_executor.py,sha256=CWCN4b6Ircs-3tCxJjBsrjl4Q0ABBJIwqlZr7a5lW6k,12243
|
28
28
|
airflow/providers/cncf/kubernetes/hooks/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
29
|
-
airflow/providers/cncf/kubernetes/hooks/kubernetes.py,sha256=
|
29
|
+
airflow/providers/cncf/kubernetes/hooks/kubernetes.py,sha256=lLNGbLizQu-gRU6GDa3MTobqvbrWqFKIqbqK1SVEgr8,37192
|
30
30
|
airflow/providers/cncf/kubernetes/kubernetes_executor_templates/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
31
31
|
airflow/providers/cncf/kubernetes/kubernetes_executor_templates/basic_template.yaml,sha256=yzJmXN4ZyB4aDwI_GIugpL9-f1YMVy__X-LQSbeU95A,2567
|
32
32
|
airflow/providers/cncf/kubernetes/operators/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
|
33
33
|
airflow/providers/cncf/kubernetes/operators/custom_object_launcher.py,sha256=jTVHQt1vp5gELrLNyM-DrZ1ywgmTy3Hh1i6wyl7AGS0,15314
|
34
|
-
airflow/providers/cncf/kubernetes/operators/job.py,sha256=
|
34
|
+
airflow/providers/cncf/kubernetes/operators/job.py,sha256=lYUO_HAC_NyiyBZvNFZ_6Itk5bRCsI3BIyj3KZmHCpw,26775
|
35
35
|
airflow/providers/cncf/kubernetes/operators/kueue.py,sha256=Xr-G4uvgSzmqm72LmUsRlTbuZ4hFG6GTI8lvdLoya9Y,4597
|
36
|
-
airflow/providers/cncf/kubernetes/operators/pod.py,sha256=
|
37
|
-
airflow/providers/cncf/kubernetes/operators/resource.py,sha256=
|
36
|
+
airflow/providers/cncf/kubernetes/operators/pod.py,sha256=vzwZ6xeJY3goqND8Hop7veqZDNKri-igFjJhhpBrnY8,60959
|
37
|
+
airflow/providers/cncf/kubernetes/operators/resource.py,sha256=XQrlbLbk-tlN_CQnETa9hgFzxxL82hh-Fs0XM5SDhyg,7574
|
38
38
|
airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py,sha256=lqZqehu-YUMDxFTwSlQm0e10pdoU5njus5TtMLOdEUI,13898
|
39
39
|
airflow/providers/cncf/kubernetes/pod_template_file_examples/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
40
40
|
airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_image_template.yaml,sha256=7JdppZ-XDBpv2Bnde2SthhcME8w3b8xQdPAK1fJGW60,2256
|
@@ -47,14 +47,14 @@ airflow/providers/cncf/kubernetes/resource_convert/secret.py,sha256=ElZCMbTWeTKo
|
|
47
47
|
airflow/providers/cncf/kubernetes/sensors/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
48
48
|
airflow/providers/cncf/kubernetes/sensors/spark_kubernetes.py,sha256=NrrUUXK1ctldF9j-pjEjJEvPQafH6b4rpLtSrJJVNEU,5384
|
49
49
|
airflow/providers/cncf/kubernetes/triggers/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
50
|
-
airflow/providers/cncf/kubernetes/triggers/job.py,sha256=
|
51
|
-
airflow/providers/cncf/kubernetes/triggers/pod.py,sha256=
|
50
|
+
airflow/providers/cncf/kubernetes/triggers/job.py,sha256=_lLP6ZYRV4kdwb7U0w5QFnlY1E9deZ5wtg-nrlfl6-8,7505
|
51
|
+
airflow/providers/cncf/kubernetes/triggers/pod.py,sha256=AVk0-dJN_wjMeZtImMxan4JZ7sSl-allC8ga1o3WhKM,13388
|
52
52
|
airflow/providers/cncf/kubernetes/utils/__init__.py,sha256=ClZN0VPjWySdVwS_ktH7rrgL9VLAcs3OSJSB9s3zaYw,863
|
53
53
|
airflow/providers/cncf/kubernetes/utils/delete_from.py,sha256=poObZSoEJwQyaYWilEURs8f4CDY2sn_pfwS31Lf579A,5195
|
54
54
|
airflow/providers/cncf/kubernetes/utils/k8s_resource_iterator.py,sha256=pl-G-2WhZVbewKkwmL9AxPo1hAQWHHEPK43b-ruF4-w,1937
|
55
|
-
airflow/providers/cncf/kubernetes/utils/pod_manager.py,sha256=
|
55
|
+
airflow/providers/cncf/kubernetes/utils/pod_manager.py,sha256=DTIwT1b-MGpiIAao5SuKD8Qgu3vlGZ-NYeR_Mf5W7rE,40434
|
56
56
|
airflow/providers/cncf/kubernetes/utils/xcom_sidecar.py,sha256=k6bdmVJ21OrAwGmWwledRrAmaty9ZrmbuM-IbaI4mqo,2519
|
57
|
-
apache_airflow_providers_cncf_kubernetes-10.6.
|
58
|
-
apache_airflow_providers_cncf_kubernetes-10.6.
|
59
|
-
apache_airflow_providers_cncf_kubernetes-10.6.
|
60
|
-
apache_airflow_providers_cncf_kubernetes-10.6.
|
57
|
+
apache_airflow_providers_cncf_kubernetes-10.6.2rc1.dist-info/entry_points.txt,sha256=ByD3QJJyP9CfmTYtpNI1953akD38RUDgpGXLaq9vpOw,111
|
58
|
+
apache_airflow_providers_cncf_kubernetes-10.6.2rc1.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
|
59
|
+
apache_airflow_providers_cncf_kubernetes-10.6.2rc1.dist-info/METADATA,sha256=dni-tpQAUVmw38LhfONN7-hwe0LiDKBp9gTtocirMQM,4360
|
60
|
+
apache_airflow_providers_cncf_kubernetes-10.6.2rc1.dist-info/RECORD,,
|
File without changes
|