apache-airflow-providers-cncf-kubernetes 9.0.1rc1__py3-none-any.whl → 10.0.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of apache-airflow-providers-cncf-kubernetes might be problematic. Click here for more details.
- airflow/providers/cncf/kubernetes/__init__.py +1 -1
- airflow/providers/cncf/kubernetes/decorators/kubernetes.py +1 -1
- airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py +6 -1
- airflow/providers/cncf/kubernetes/executors/local_kubernetes_executor.py +7 -2
- airflow/providers/cncf/kubernetes/get_provider_info.py +1 -0
- airflow/providers/cncf/kubernetes/kubernetes_helper_functions.py +0 -41
- airflow/providers/cncf/kubernetes/operators/pod.py +6 -32
- airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py +10 -4
- airflow/providers/cncf/kubernetes/pod_generator.py +0 -118
- airflow/providers/cncf/kubernetes/triggers/pod.py +1 -22
- airflow/providers/cncf/kubernetes/utils/pod_manager.py +2 -21
- {apache_airflow_providers_cncf_kubernetes-9.0.1rc1.dist-info → apache_airflow_providers_cncf_kubernetes-10.0.0rc1.dist-info}/METADATA +7 -7
- {apache_airflow_providers_cncf_kubernetes-9.0.1rc1.dist-info → apache_airflow_providers_cncf_kubernetes-10.0.0rc1.dist-info}/RECORD +15 -18
- {apache_airflow_providers_cncf_kubernetes-9.0.1rc1.dist-info → apache_airflow_providers_cncf_kubernetes-10.0.0rc1.dist-info}/WHEEL +1 -1
- airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py +0 -31
- airflow/providers/cncf/kubernetes/pod_launcher_deprecated.py +0 -320
- airflow/providers/cncf/kubernetes/triggers/kubernetes_pod.py +0 -31
- {apache_airflow_providers_cncf_kubernetes-9.0.1rc1.dist-info → apache_airflow_providers_cncf_kubernetes-10.0.0rc1.dist-info}/entry_points.txt +0 -0
|
@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
|
|
|
29
29
|
|
|
30
30
|
__all__ = ["__version__"]
|
|
31
31
|
|
|
32
|
-
__version__ = "
|
|
32
|
+
__version__ = "10.0.0"
|
|
33
33
|
|
|
34
34
|
if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
|
|
35
35
|
"2.8.0"
|
|
@@ -65,7 +65,7 @@ class _KubernetesDecoratedOperator(DecoratedOperator, KubernetesPodOperator):
|
|
|
65
65
|
# there are some cases we can't deepcopy the objects (e.g protobuf).
|
|
66
66
|
shallow_copy_attrs: Sequence[str] = ("python_callable",)
|
|
67
67
|
|
|
68
|
-
def __init__(self, namespace: str =
|
|
68
|
+
def __init__(self, namespace: str | None = None, use_dill: bool = False, **kwargs) -> None:
|
|
69
69
|
self.use_dill = use_dill
|
|
70
70
|
super().__init__(
|
|
71
71
|
namespace=namespace,
|
|
@@ -768,8 +768,13 @@ class KubernetesExecutor(BaseExecutor):
|
|
|
768
768
|
self.result_queue.join()
|
|
769
769
|
except ConnectionResetError:
|
|
770
770
|
self.log.exception("Connection Reset error while flushing task_queue and result_queue.")
|
|
771
|
+
except Exception:
|
|
772
|
+
self.log.exception("Unknown error while flushing task queue and result queue.")
|
|
771
773
|
if self.kube_scheduler:
|
|
772
|
-
|
|
774
|
+
try:
|
|
775
|
+
self.kube_scheduler.terminate()
|
|
776
|
+
except Exception:
|
|
777
|
+
self.log.exception("Unknown error while flushing task queue and result queue.")
|
|
773
778
|
self._manager.shutdown()
|
|
774
779
|
|
|
775
780
|
def terminate(self):
|
|
@@ -45,6 +45,7 @@ class LocalKubernetesExecutor(BaseExecutor):
|
|
|
45
45
|
"""
|
|
46
46
|
|
|
47
47
|
supports_ad_hoc_ti_run: bool = True
|
|
48
|
+
# TODO: Remove this attribute once providers rely on Airflow >=3.0.0
|
|
48
49
|
supports_pickling: bool = False
|
|
49
50
|
supports_sentry: bool = False
|
|
50
51
|
|
|
@@ -146,7 +147,6 @@ class LocalKubernetesExecutor(BaseExecutor):
|
|
|
146
147
|
self,
|
|
147
148
|
task_instance: TaskInstance,
|
|
148
149
|
mark_success: bool = False,
|
|
149
|
-
pickle_id: int | None = None,
|
|
150
150
|
ignore_all_deps: bool = False,
|
|
151
151
|
ignore_depends_on_past: bool = False,
|
|
152
152
|
wait_for_past_depends_before_skipping: bool = False,
|
|
@@ -154,6 +154,7 @@ class LocalKubernetesExecutor(BaseExecutor):
|
|
|
154
154
|
ignore_ti_state: bool = False,
|
|
155
155
|
pool: str | None = None,
|
|
156
156
|
cfg_path: str | None = None,
|
|
157
|
+
**kwargs,
|
|
157
158
|
) -> None:
|
|
158
159
|
"""Queues task instance via local or kubernetes executor."""
|
|
159
160
|
from airflow.models.taskinstance import SimpleTaskInstance
|
|
@@ -162,10 +163,13 @@ class LocalKubernetesExecutor(BaseExecutor):
|
|
|
162
163
|
self.log.debug(
|
|
163
164
|
"Using executor: %s to queue_task_instance for %s", executor.__class__.__name__, task_instance.key
|
|
164
165
|
)
|
|
166
|
+
|
|
167
|
+
if not hasattr(task_instance, "pickle_id"):
|
|
168
|
+
del kwargs["pickle_id"]
|
|
169
|
+
|
|
165
170
|
executor.queue_task_instance(
|
|
166
171
|
task_instance=task_instance,
|
|
167
172
|
mark_success=mark_success,
|
|
168
|
-
pickle_id=pickle_id,
|
|
169
173
|
ignore_all_deps=ignore_all_deps,
|
|
170
174
|
ignore_depends_on_past=ignore_depends_on_past,
|
|
171
175
|
wait_for_past_depends_before_skipping=wait_for_past_depends_before_skipping,
|
|
@@ -173,6 +177,7 @@ class LocalKubernetesExecutor(BaseExecutor):
|
|
|
173
177
|
ignore_ti_state=ignore_ti_state,
|
|
174
178
|
pool=pool,
|
|
175
179
|
cfg_path=cfg_path,
|
|
180
|
+
**kwargs,
|
|
176
181
|
)
|
|
177
182
|
|
|
178
183
|
def get_task_log(self, ti: TaskInstance, try_number: int) -> tuple[list[str], list[str]]:
|
|
@@ -23,12 +23,10 @@ from functools import cache
|
|
|
23
23
|
from typing import TYPE_CHECKING
|
|
24
24
|
|
|
25
25
|
import pendulum
|
|
26
|
-
from deprecated import deprecated
|
|
27
26
|
from kubernetes.client.rest import ApiException
|
|
28
27
|
from slugify import slugify
|
|
29
28
|
|
|
30
29
|
from airflow.configuration import conf
|
|
31
|
-
from airflow.exceptions import AirflowProviderDeprecationWarning
|
|
32
30
|
|
|
33
31
|
if TYPE_CHECKING:
|
|
34
32
|
from airflow.models.taskinstancekey import TaskInstanceKey
|
|
@@ -62,22 +60,6 @@ def add_unique_suffix(*, name: str, rand_len: int = 8, max_len: int = POD_NAME_M
|
|
|
62
60
|
return name[: max_len - len(suffix)].strip("-.") + suffix
|
|
63
61
|
|
|
64
62
|
|
|
65
|
-
@deprecated(
|
|
66
|
-
reason="This function is deprecated. Please use `add_unique_suffix`",
|
|
67
|
-
category=AirflowProviderDeprecationWarning,
|
|
68
|
-
)
|
|
69
|
-
def add_pod_suffix(*, pod_name: str, rand_len: int = 8, max_len: int = POD_NAME_MAX_LENGTH) -> str:
|
|
70
|
-
"""
|
|
71
|
-
Add random string to pod name while staying under max length.
|
|
72
|
-
|
|
73
|
-
:param pod_name: name of the pod
|
|
74
|
-
:param rand_len: length of the random string to append
|
|
75
|
-
:param max_len: maximum length of the pod name
|
|
76
|
-
:meta private:
|
|
77
|
-
"""
|
|
78
|
-
return add_unique_suffix(name=pod_name, rand_len=rand_len, max_len=max_len)
|
|
79
|
-
|
|
80
|
-
|
|
81
63
|
def create_unique_id(
|
|
82
64
|
dag_id: str | None = None,
|
|
83
65
|
task_id: str | None = None,
|
|
@@ -110,29 +92,6 @@ def create_unique_id(
|
|
|
110
92
|
return base_name
|
|
111
93
|
|
|
112
94
|
|
|
113
|
-
@deprecated(
|
|
114
|
-
reason="This function is deprecated. Please use `create_unique_id`.",
|
|
115
|
-
category=AirflowProviderDeprecationWarning,
|
|
116
|
-
)
|
|
117
|
-
def create_pod_id(
|
|
118
|
-
dag_id: str | None = None,
|
|
119
|
-
task_id: str | None = None,
|
|
120
|
-
*,
|
|
121
|
-
max_length: int = POD_NAME_MAX_LENGTH,
|
|
122
|
-
unique: bool = True,
|
|
123
|
-
) -> str:
|
|
124
|
-
"""
|
|
125
|
-
Generate unique pod ID given a dag_id and / or task_id.
|
|
126
|
-
|
|
127
|
-
:param dag_id: DAG ID
|
|
128
|
-
:param task_id: Task ID
|
|
129
|
-
:param max_length: max number of characters
|
|
130
|
-
:param unique: whether a random string suffix should be added
|
|
131
|
-
:return: A valid identifier for a kubernetes pod name
|
|
132
|
-
"""
|
|
133
|
-
return create_unique_id(dag_id=dag_id, task_id=task_id, max_length=max_length, unique=unique)
|
|
134
|
-
|
|
135
|
-
|
|
136
95
|
def annotations_to_key(annotations: dict[str, str]) -> TaskInstanceKey:
|
|
137
96
|
"""Build a TaskInstanceKey based on pod annotations."""
|
|
138
97
|
log.debug("Creating task key for annotations %s", annotations)
|
|
@@ -26,8 +26,7 @@ import os
|
|
|
26
26
|
import re
|
|
27
27
|
import shlex
|
|
28
28
|
import string
|
|
29
|
-
import
|
|
30
|
-
from collections.abc import Container
|
|
29
|
+
from collections.abc import Container, Mapping
|
|
31
30
|
from contextlib import AbstractContextManager
|
|
32
31
|
from enum import Enum
|
|
33
32
|
from functools import cached_property
|
|
@@ -35,7 +34,6 @@ from typing import TYPE_CHECKING, Any, Callable, Iterable, Literal, Sequence
|
|
|
35
34
|
|
|
36
35
|
import kubernetes
|
|
37
36
|
import tenacity
|
|
38
|
-
from deprecated import deprecated
|
|
39
37
|
from kubernetes.client import CoreV1Api, V1Pod, models as k8s
|
|
40
38
|
from kubernetes.client.exceptions import ApiException
|
|
41
39
|
from kubernetes.stream import stream
|
|
@@ -44,7 +42,6 @@ from urllib3.exceptions import HTTPError
|
|
|
44
42
|
from airflow.configuration import conf
|
|
45
43
|
from airflow.exceptions import (
|
|
46
44
|
AirflowException,
|
|
47
|
-
AirflowProviderDeprecationWarning,
|
|
48
45
|
AirflowSkipException,
|
|
49
46
|
TaskDeferred,
|
|
50
47
|
)
|
|
@@ -215,18 +212,12 @@ class KubernetesPodOperator(BaseOperator):
|
|
|
215
212
|
:param on_finish_action: What to do when the pod reaches its final state, or the execution is interrupted.
|
|
216
213
|
If "delete_pod", the pod will be deleted regardless its state; if "delete_succeeded_pod",
|
|
217
214
|
only succeeded pod will be deleted. You can set to "keep_pod" to keep the pod.
|
|
218
|
-
:param is_delete_operator_pod: What to do when the pod reaches its final
|
|
219
|
-
state, or the execution is interrupted. If True (default), delete the
|
|
220
|
-
pod; if False, leave the pod.
|
|
221
|
-
Deprecated - use `on_finish_action` instead.
|
|
222
215
|
:param termination_message_policy: The termination message policy of the base container.
|
|
223
216
|
Default value is "File"
|
|
224
217
|
:param active_deadline_seconds: The active_deadline_seconds which translates to active_deadline_seconds
|
|
225
218
|
in V1PodSpec.
|
|
226
219
|
:param callbacks: KubernetesPodOperatorCallback instance contains the callbacks methods on different step
|
|
227
220
|
of KubernetesPodOperator.
|
|
228
|
-
:param progress_callback: Callback function for receiving k8s container logs.
|
|
229
|
-
`progress_callback` is deprecated, please use :param `callbacks` instead.
|
|
230
221
|
:param logging_interval: max time in seconds that task should be in deferred state before
|
|
231
222
|
resuming to fetch the latest logs. If ``None``, then the task will remain in deferred state until pod
|
|
232
223
|
is done, and no logs will be visible until that time.
|
|
@@ -404,19 +395,8 @@ class KubernetesPodOperator(BaseOperator):
|
|
|
404
395
|
self.poll_interval = poll_interval
|
|
405
396
|
self.remote_pod: k8s.V1Pod | None = None
|
|
406
397
|
self.log_pod_spec_on_failure = log_pod_spec_on_failure
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
"`is_delete_operator_pod` parameter is deprecated, please use `on_finish_action`",
|
|
410
|
-
AirflowProviderDeprecationWarning,
|
|
411
|
-
stacklevel=2,
|
|
412
|
-
)
|
|
413
|
-
self.on_finish_action = (
|
|
414
|
-
OnFinishAction.DELETE_POD if is_delete_operator_pod else OnFinishAction.KEEP_POD
|
|
415
|
-
)
|
|
416
|
-
self.is_delete_operator_pod = is_delete_operator_pod
|
|
417
|
-
else:
|
|
418
|
-
self.on_finish_action = OnFinishAction(on_finish_action)
|
|
419
|
-
self.is_delete_operator_pod = self.on_finish_action == OnFinishAction.DELETE_POD
|
|
398
|
+
self.on_finish_action = OnFinishAction(on_finish_action)
|
|
399
|
+
self.is_delete_operator_pod = self.on_finish_action == OnFinishAction.DELETE_POD
|
|
420
400
|
self.termination_message_policy = termination_message_policy
|
|
421
401
|
self.active_deadline_seconds = active_deadline_seconds
|
|
422
402
|
self.logging_interval = logging_interval
|
|
@@ -436,7 +416,7 @@ class KubernetesPodOperator(BaseOperator):
|
|
|
436
416
|
def _render_nested_template_fields(
|
|
437
417
|
self,
|
|
438
418
|
content: Any,
|
|
439
|
-
context:
|
|
419
|
+
context: Mapping[str, Any],
|
|
440
420
|
jinja_env: jinja2.Environment,
|
|
441
421
|
seen_oids: set,
|
|
442
422
|
) -> None:
|
|
@@ -512,9 +492,7 @@ class KubernetesPodOperator(BaseOperator):
|
|
|
512
492
|
|
|
513
493
|
@cached_property
|
|
514
494
|
def pod_manager(self) -> PodManager:
|
|
515
|
-
return PodManager(
|
|
516
|
-
kube_client=self.client, callbacks=self.callbacks, progress_callback=self._progress_callback
|
|
517
|
-
)
|
|
495
|
+
return PodManager(kube_client=self.client, callbacks=self.callbacks)
|
|
518
496
|
|
|
519
497
|
@cached_property
|
|
520
498
|
def hook(self) -> PodOperatorHookProtocol:
|
|
@@ -563,7 +541,7 @@ class KubernetesPodOperator(BaseOperator):
|
|
|
563
541
|
|
|
564
542
|
def get_or_create_pod(self, pod_request_obj: k8s.V1Pod, context: Context) -> k8s.V1Pod:
|
|
565
543
|
if self.reattach_on_restart:
|
|
566
|
-
pod = self.find_pod(
|
|
544
|
+
pod = self.find_pod(pod_request_obj.metadata.namespace, context=context)
|
|
567
545
|
if pod:
|
|
568
546
|
return pod
|
|
569
547
|
self.log.debug("Starting pod:\n%s", yaml.safe_dump(pod_request_obj.to_dict()))
|
|
@@ -1161,10 +1139,6 @@ class KubernetesPodOperator(BaseOperator):
|
|
|
1161
1139
|
pod = self.build_pod_request_obj()
|
|
1162
1140
|
print(yaml.dump(prune_dict(pod.to_dict(), mode="strict")))
|
|
1163
1141
|
|
|
1164
|
-
@deprecated(reason="use `trigger_reentry` instead.", category=AirflowProviderDeprecationWarning)
|
|
1165
|
-
def execute_complete(self, context: Context, event: dict, **kwargs):
|
|
1166
|
-
return self.trigger_reentry(context=context, event=event)
|
|
1167
|
-
|
|
1168
1142
|
def process_duplicate_label_pods(self, pod_list: list[k8s.V1Pod]) -> k8s.V1Pod:
|
|
1169
1143
|
"""
|
|
1170
1144
|
Patch or delete the existing pod with duplicate labels.
|
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
# under the License.
|
|
18
18
|
from __future__ import annotations
|
|
19
19
|
|
|
20
|
+
from collections.abc import Mapping
|
|
20
21
|
from functools import cached_property
|
|
21
22
|
from pathlib import Path
|
|
22
23
|
from typing import TYPE_CHECKING, Any
|
|
@@ -67,6 +68,7 @@ class SparkKubernetesOperator(KubernetesPodOperator):
|
|
|
67
68
|
state, or the execution is interrupted. If True (default), delete the
|
|
68
69
|
pod; if False, leave the pod.
|
|
69
70
|
:param kubernetes_conn_id: the connection to Kubernetes cluster
|
|
71
|
+
:param random_name_suffix: If True, adds a random suffix to the pod name
|
|
70
72
|
"""
|
|
71
73
|
|
|
72
74
|
template_fields = ["application_file", "namespace", "template_spec", "kubernetes_conn_id"]
|
|
@@ -93,10 +95,9 @@ class SparkKubernetesOperator(KubernetesPodOperator):
|
|
|
93
95
|
reattach_on_restart: bool = True,
|
|
94
96
|
delete_on_termination: bool = True,
|
|
95
97
|
kubernetes_conn_id: str = "kubernetes_default",
|
|
98
|
+
random_name_suffix: bool = True,
|
|
96
99
|
**kwargs,
|
|
97
100
|
) -> None:
|
|
98
|
-
if kwargs.get("xcom_push") is not None:
|
|
99
|
-
raise AirflowException("'xcom_push' was deprecated, use 'do_xcom_push' instead")
|
|
100
101
|
super().__init__(name=name, **kwargs)
|
|
101
102
|
self.image = image
|
|
102
103
|
self.code_path = code_path
|
|
@@ -111,6 +112,7 @@ class SparkKubernetesOperator(KubernetesPodOperator):
|
|
|
111
112
|
self.get_logs = get_logs
|
|
112
113
|
self.log_events_on_failure = log_events_on_failure
|
|
113
114
|
self.success_run_history_limit = success_run_history_limit
|
|
115
|
+
self.random_name_suffix = random_name_suffix
|
|
114
116
|
|
|
115
117
|
if self.base_container_name != self.BASE_CONTAINER_NAME:
|
|
116
118
|
self.log.warning(
|
|
@@ -127,7 +129,7 @@ class SparkKubernetesOperator(KubernetesPodOperator):
|
|
|
127
129
|
def _render_nested_template_fields(
|
|
128
130
|
self,
|
|
129
131
|
content: Any,
|
|
130
|
-
context:
|
|
132
|
+
context: Mapping[str, Any],
|
|
131
133
|
jinja_env: jinja2.Environment,
|
|
132
134
|
seen_oids: set,
|
|
133
135
|
) -> None:
|
|
@@ -163,7 +165,11 @@ class SparkKubernetesOperator(KubernetesPodOperator):
|
|
|
163
165
|
self.name or self.template_body.get("spark", {}).get("metadata", {}).get("name") or self.task_id
|
|
164
166
|
)
|
|
165
167
|
|
|
166
|
-
|
|
168
|
+
if self.random_name_suffix:
|
|
169
|
+
updated_name = add_unique_suffix(name=name, max_len=MAX_LABEL_LEN)
|
|
170
|
+
else:
|
|
171
|
+
# truncation is required to maintain the same behavior as before
|
|
172
|
+
updated_name = name[:MAX_LABEL_LEN]
|
|
167
173
|
|
|
168
174
|
return self._set_name(updated_name)
|
|
169
175
|
|
|
@@ -34,23 +34,16 @@ from typing import TYPE_CHECKING
|
|
|
34
34
|
|
|
35
35
|
import re2
|
|
36
36
|
from dateutil import parser
|
|
37
|
-
from deprecated import deprecated
|
|
38
37
|
from kubernetes.client import models as k8s
|
|
39
38
|
from kubernetes.client.api_client import ApiClient
|
|
40
39
|
|
|
41
40
|
from airflow.exceptions import (
|
|
42
41
|
AirflowConfigException,
|
|
43
42
|
AirflowException,
|
|
44
|
-
AirflowProviderDeprecationWarning,
|
|
45
43
|
)
|
|
46
44
|
from airflow.providers.cncf.kubernetes.kubernetes_helper_functions import (
|
|
47
45
|
POD_NAME_MAX_LENGTH,
|
|
48
46
|
add_unique_suffix,
|
|
49
|
-
rand_str,
|
|
50
|
-
)
|
|
51
|
-
from airflow.providers.cncf.kubernetes.pod_generator_deprecated import (
|
|
52
|
-
PodDefaults as PodDefaultsDeprecated,
|
|
53
|
-
PodGenerator as PodGeneratorDeprecated,
|
|
54
47
|
)
|
|
55
48
|
from airflow.utils import yaml
|
|
56
49
|
from airflow.utils.hashlib_wrapper import md5
|
|
@@ -155,40 +148,6 @@ class PodGenerator:
|
|
|
155
148
|
# Attach sidecar
|
|
156
149
|
self.extract_xcom = extract_xcom
|
|
157
150
|
|
|
158
|
-
@deprecated(
|
|
159
|
-
reason="This method is deprecated and will be removed in the future releases",
|
|
160
|
-
category=AirflowProviderDeprecationWarning,
|
|
161
|
-
)
|
|
162
|
-
def gen_pod(self) -> k8s.V1Pod:
|
|
163
|
-
"""Generate pod."""
|
|
164
|
-
result = self.ud_pod
|
|
165
|
-
|
|
166
|
-
result.metadata.name = add_unique_suffix(name=result.metadata.name)
|
|
167
|
-
|
|
168
|
-
if self.extract_xcom:
|
|
169
|
-
result = self.add_xcom_sidecar(result)
|
|
170
|
-
|
|
171
|
-
return result
|
|
172
|
-
|
|
173
|
-
@staticmethod
|
|
174
|
-
@deprecated(
|
|
175
|
-
reason=(
|
|
176
|
-
"This function is deprecated. "
|
|
177
|
-
"Please use airflow.providers.cncf.kubernetes.utils.xcom_sidecar.add_xcom_sidecar instead"
|
|
178
|
-
),
|
|
179
|
-
category=AirflowProviderDeprecationWarning,
|
|
180
|
-
)
|
|
181
|
-
def add_xcom_sidecar(pod: k8s.V1Pod) -> k8s.V1Pod:
|
|
182
|
-
"""Add sidecar."""
|
|
183
|
-
pod_cp = copy.deepcopy(pod)
|
|
184
|
-
pod_cp.spec.volumes = pod.spec.volumes or []
|
|
185
|
-
pod_cp.spec.volumes.insert(0, PodDefaultsDeprecated.VOLUME)
|
|
186
|
-
pod_cp.spec.containers[0].volume_mounts = pod_cp.spec.containers[0].volume_mounts or []
|
|
187
|
-
pod_cp.spec.containers[0].volume_mounts.insert(0, PodDefaultsDeprecated.VOLUME_MOUNT)
|
|
188
|
-
pod_cp.spec.containers.append(PodDefaultsDeprecated.SIDECAR_CONTAINER)
|
|
189
|
-
|
|
190
|
-
return pod_cp
|
|
191
|
-
|
|
192
151
|
@staticmethod
|
|
193
152
|
def from_obj(obj) -> dict | k8s.V1Pod | None:
|
|
194
153
|
"""Convert to pod from obj."""
|
|
@@ -210,57 +169,11 @@ class PodGenerator:
|
|
|
210
169
|
|
|
211
170
|
if isinstance(k8s_object, k8s.V1Pod):
|
|
212
171
|
return k8s_object
|
|
213
|
-
elif isinstance(k8s_legacy_object, dict):
|
|
214
|
-
warnings.warn(
|
|
215
|
-
"Using a dictionary for the executor_config is deprecated and will soon be removed. "
|
|
216
|
-
'Please use a `kubernetes.client.models.V1Pod` class with a "pod_override" key'
|
|
217
|
-
" instead. ",
|
|
218
|
-
category=AirflowProviderDeprecationWarning,
|
|
219
|
-
stacklevel=2,
|
|
220
|
-
)
|
|
221
|
-
return PodGenerator.from_legacy_obj(obj)
|
|
222
172
|
else:
|
|
223
173
|
raise TypeError(
|
|
224
174
|
"Cannot convert a non-kubernetes.client.models.V1Pod object into a KubernetesExecutorConfig"
|
|
225
175
|
)
|
|
226
176
|
|
|
227
|
-
@staticmethod
|
|
228
|
-
def from_legacy_obj(obj) -> k8s.V1Pod | None:
|
|
229
|
-
"""Convert to pod from obj."""
|
|
230
|
-
if obj is None:
|
|
231
|
-
return None
|
|
232
|
-
|
|
233
|
-
# We do not want to extract constant here from ExecutorLoader because it is just
|
|
234
|
-
# A name in dictionary rather than executor selection mechanism and it causes cyclic import
|
|
235
|
-
namespaced = obj.get("KubernetesExecutor", {})
|
|
236
|
-
|
|
237
|
-
if not namespaced:
|
|
238
|
-
return None
|
|
239
|
-
|
|
240
|
-
resources = namespaced.get("resources")
|
|
241
|
-
|
|
242
|
-
if resources is None:
|
|
243
|
-
requests = {
|
|
244
|
-
"cpu": namespaced.pop("request_cpu", None),
|
|
245
|
-
"memory": namespaced.pop("request_memory", None),
|
|
246
|
-
"ephemeral-storage": namespaced.get("ephemeral-storage"), # We pop this one in limits
|
|
247
|
-
}
|
|
248
|
-
limits = {
|
|
249
|
-
"cpu": namespaced.pop("limit_cpu", None),
|
|
250
|
-
"memory": namespaced.pop("limit_memory", None),
|
|
251
|
-
"ephemeral-storage": namespaced.pop("ephemeral-storage", None),
|
|
252
|
-
}
|
|
253
|
-
all_resources = list(requests.values()) + list(limits.values())
|
|
254
|
-
if all(r is None for r in all_resources):
|
|
255
|
-
resources = None
|
|
256
|
-
else:
|
|
257
|
-
# remove None's so they don't become 0's
|
|
258
|
-
requests = {k: v for k, v in requests.items() if v is not None}
|
|
259
|
-
limits = {k: v for k, v in limits.items() if v is not None}
|
|
260
|
-
resources = k8s.V1ResourceRequirements(requests=requests, limits=limits)
|
|
261
|
-
namespaced["resources"] = resources
|
|
262
|
-
return PodGeneratorDeprecated(**namespaced).gen_pod()
|
|
263
|
-
|
|
264
177
|
@staticmethod
|
|
265
178
|
def reconcile_pods(base_pod: k8s.V1Pod, client_pod: k8s.V1Pod | None) -> k8s.V1Pod:
|
|
266
179
|
"""
|
|
@@ -579,37 +492,6 @@ class PodGenerator:
|
|
|
579
492
|
api_client = ApiClient()
|
|
580
493
|
return api_client._ApiClient__deserialize_model(pod_dict, k8s.V1Pod)
|
|
581
494
|
|
|
582
|
-
@staticmethod
|
|
583
|
-
@deprecated(
|
|
584
|
-
reason="This method is deprecated. Use `add_pod_suffix` in `kubernetes_helper_functions`.",
|
|
585
|
-
category=AirflowProviderDeprecationWarning,
|
|
586
|
-
)
|
|
587
|
-
def make_unique_pod_id(pod_id: str) -> str | None:
|
|
588
|
-
r"""
|
|
589
|
-
Generate a unique Pod name.
|
|
590
|
-
|
|
591
|
-
Kubernetes pod names must consist of one or more lowercase
|
|
592
|
-
rfc1035/rfc1123 labels separated by '.' with a maximum length of 253
|
|
593
|
-
characters.
|
|
594
|
-
|
|
595
|
-
Name must pass the following regex for validation
|
|
596
|
-
``^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$``
|
|
597
|
-
|
|
598
|
-
For more details, see:
|
|
599
|
-
https://github.com/kubernetes/kubernetes/blob/release-1.1/docs/design/identifiers.md
|
|
600
|
-
|
|
601
|
-
:param pod_id: requested pod name
|
|
602
|
-
:return: ``str`` valid Pod name of appropriate length
|
|
603
|
-
"""
|
|
604
|
-
if not pod_id:
|
|
605
|
-
return None
|
|
606
|
-
|
|
607
|
-
max_pod_id_len = 100 # arbitrarily chosen
|
|
608
|
-
suffix = rand_str(8) # 8 seems good enough
|
|
609
|
-
base_pod_id_len = max_pod_id_len - len(suffix) - 1 # -1 for separator
|
|
610
|
-
trimmed_pod_id = pod_id[:base_pod_id_len].rstrip("-.")
|
|
611
|
-
return f"{trimmed_pod_id}-{suffix}"
|
|
612
|
-
|
|
613
495
|
|
|
614
496
|
def merge_objects(base_obj, client_obj):
|
|
615
497
|
"""
|
|
@@ -19,12 +19,10 @@ from __future__ import annotations
|
|
|
19
19
|
import asyncio
|
|
20
20
|
import datetime
|
|
21
21
|
import traceback
|
|
22
|
-
import warnings
|
|
23
22
|
from enum import Enum
|
|
24
23
|
from functools import cached_property
|
|
25
24
|
from typing import TYPE_CHECKING, Any, AsyncIterator
|
|
26
25
|
|
|
27
|
-
from airflow.exceptions import AirflowProviderDeprecationWarning
|
|
28
26
|
from airflow.providers.cncf.kubernetes.hooks.kubernetes import AsyncKubernetesHook
|
|
29
27
|
from airflow.providers.cncf.kubernetes.utils.pod_manager import (
|
|
30
28
|
OnFinishAction,
|
|
@@ -71,10 +69,6 @@ class KubernetesPodTrigger(BaseTrigger):
|
|
|
71
69
|
:param on_finish_action: What to do when the pod reaches its final state, or the execution is interrupted.
|
|
72
70
|
If "delete_pod", the pod will be deleted regardless its state; if "delete_succeeded_pod",
|
|
73
71
|
only succeeded pod will be deleted. You can set to "keep_pod" to keep the pod.
|
|
74
|
-
:param should_delete_pod: What to do when the pod reaches its final
|
|
75
|
-
state, or the execution is interrupted. If True (default), delete the
|
|
76
|
-
pod; if False, leave the pod.
|
|
77
|
-
Deprecated - use `on_finish_action` instead.
|
|
78
72
|
:param logging_interval: number of seconds to wait before kicking it back to
|
|
79
73
|
the operator to print latest logs. If ``None`` will wait until container done.
|
|
80
74
|
:param last_log_time: where to resume logs from
|
|
@@ -95,7 +89,6 @@ class KubernetesPodTrigger(BaseTrigger):
|
|
|
95
89
|
startup_timeout: int = 120,
|
|
96
90
|
startup_check_interval: int = 5,
|
|
97
91
|
on_finish_action: str = "delete_pod",
|
|
98
|
-
should_delete_pod: bool | None = None,
|
|
99
92
|
last_log_time: DateTime | None = None,
|
|
100
93
|
logging_interval: int | None = None,
|
|
101
94
|
):
|
|
@@ -114,20 +107,7 @@ class KubernetesPodTrigger(BaseTrigger):
|
|
|
114
107
|
self.startup_check_interval = startup_check_interval
|
|
115
108
|
self.last_log_time = last_log_time
|
|
116
109
|
self.logging_interval = logging_interval
|
|
117
|
-
|
|
118
|
-
if should_delete_pod is not None:
|
|
119
|
-
warnings.warn(
|
|
120
|
-
"`should_delete_pod` parameter is deprecated, please use `on_finish_action`",
|
|
121
|
-
category=AirflowProviderDeprecationWarning,
|
|
122
|
-
stacklevel=2,
|
|
123
|
-
)
|
|
124
|
-
self.on_finish_action = (
|
|
125
|
-
OnFinishAction.DELETE_POD if should_delete_pod else OnFinishAction.KEEP_POD
|
|
126
|
-
)
|
|
127
|
-
self.should_delete_pod = should_delete_pod
|
|
128
|
-
else:
|
|
129
|
-
self.on_finish_action = OnFinishAction(on_finish_action)
|
|
130
|
-
self.should_delete_pod = self.on_finish_action == OnFinishAction.DELETE_POD
|
|
110
|
+
self.on_finish_action = OnFinishAction(on_finish_action)
|
|
131
111
|
|
|
132
112
|
self._since_time = None
|
|
133
113
|
|
|
@@ -148,7 +128,6 @@ class KubernetesPodTrigger(BaseTrigger):
|
|
|
148
128
|
"startup_timeout": self.startup_timeout,
|
|
149
129
|
"startup_check_interval": self.startup_check_interval,
|
|
150
130
|
"trigger_start_time": self.trigger_start_time,
|
|
151
|
-
"should_delete_pod": self.should_delete_pod,
|
|
152
131
|
"on_finish_action": self.on_finish_action.value,
|
|
153
132
|
"last_log_time": self.last_log_time,
|
|
154
133
|
"logging_interval": self.logging_interval,
|
|
@@ -26,11 +26,10 @@ from collections.abc import Iterable
|
|
|
26
26
|
from contextlib import closing, suppress
|
|
27
27
|
from dataclasses import dataclass
|
|
28
28
|
from datetime import timedelta
|
|
29
|
-
from typing import TYPE_CHECKING,
|
|
29
|
+
from typing import TYPE_CHECKING, Generator, Protocol, cast
|
|
30
30
|
|
|
31
31
|
import pendulum
|
|
32
32
|
import tenacity
|
|
33
|
-
from deprecated import deprecated
|
|
34
33
|
from kubernetes import client, watch
|
|
35
34
|
from kubernetes.client.rest import ApiException
|
|
36
35
|
from kubernetes.stream import stream as kubernetes_stream
|
|
@@ -39,7 +38,7 @@ from pendulum.parsing.exceptions import ParserError
|
|
|
39
38
|
from typing_extensions import Literal
|
|
40
39
|
from urllib3.exceptions import HTTPError, TimeoutError
|
|
41
40
|
|
|
42
|
-
from airflow.exceptions import AirflowException
|
|
41
|
+
from airflow.exceptions import AirflowException
|
|
43
42
|
from airflow.providers.cncf.kubernetes.callbacks import ExecutionMode, KubernetesPodOperatorCallback
|
|
44
43
|
from airflow.providers.cncf.kubernetes.utils.xcom_sidecar import PodDefaults
|
|
45
44
|
from airflow.utils.log.logging_mixin import LoggingMixin
|
|
@@ -302,19 +301,15 @@ class PodManager(LoggingMixin):
|
|
|
302
301
|
self,
|
|
303
302
|
kube_client: client.CoreV1Api,
|
|
304
303
|
callbacks: type[KubernetesPodOperatorCallback] | None = None,
|
|
305
|
-
progress_callback: Callable[[str], None] | None = None,
|
|
306
304
|
):
|
|
307
305
|
"""
|
|
308
306
|
Create the launcher.
|
|
309
307
|
|
|
310
308
|
:param kube_client: kubernetes client
|
|
311
309
|
:param callbacks:
|
|
312
|
-
:param progress_callback: Callback function invoked when fetching container log.
|
|
313
|
-
This parameter is deprecated, please use ````
|
|
314
310
|
"""
|
|
315
311
|
super().__init__()
|
|
316
312
|
self._client = kube_client
|
|
317
|
-
self._progress_callback = progress_callback
|
|
318
313
|
self._watch = watch.Watch()
|
|
319
314
|
self._callbacks = callbacks
|
|
320
315
|
|
|
@@ -383,16 +378,6 @@ class PodManager(LoggingMixin):
|
|
|
383
378
|
raise PodLaunchFailedException(msg)
|
|
384
379
|
time.sleep(startup_check_interval)
|
|
385
380
|
|
|
386
|
-
@deprecated(
|
|
387
|
-
reason=(
|
|
388
|
-
"Method `follow_container_logs` is deprecated. Use `fetch_container_logs` instead "
|
|
389
|
-
"with option `follow=True`."
|
|
390
|
-
),
|
|
391
|
-
category=AirflowProviderDeprecationWarning,
|
|
392
|
-
)
|
|
393
|
-
def follow_container_logs(self, pod: V1Pod, container_name: str) -> PodLoggingStatus:
|
|
394
|
-
return self.fetch_container_logs(pod=pod, container_name=container_name, follow=True)
|
|
395
|
-
|
|
396
381
|
def fetch_container_logs(
|
|
397
382
|
self,
|
|
398
383
|
pod: V1Pod,
|
|
@@ -461,8 +446,6 @@ class PodManager(LoggingMixin):
|
|
|
461
446
|
progress_callback_lines.append(line)
|
|
462
447
|
else: # previous log line is complete
|
|
463
448
|
for line in progress_callback_lines:
|
|
464
|
-
if self._progress_callback:
|
|
465
|
-
self._progress_callback(line)
|
|
466
449
|
if self._callbacks:
|
|
467
450
|
self._callbacks.progress_callback(
|
|
468
451
|
line=line, client=self._client, mode=ExecutionMode.SYNC
|
|
@@ -479,8 +462,6 @@ class PodManager(LoggingMixin):
|
|
|
479
462
|
finally:
|
|
480
463
|
# log the last line and update the last_captured_timestamp
|
|
481
464
|
for line in progress_callback_lines:
|
|
482
|
-
if self._progress_callback:
|
|
483
|
-
self._progress_callback(line)
|
|
484
465
|
if self._callbacks:
|
|
485
466
|
self._callbacks.progress_callback(
|
|
486
467
|
line=line, client=self._client, mode=ExecutionMode.SYNC
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
2
|
Name: apache-airflow-providers-cncf-kubernetes
|
|
3
|
-
Version:
|
|
3
|
+
Version: 10.0.0rc1
|
|
4
4
|
Summary: Provider package apache-airflow-providers-cncf-kubernetes for Apache Airflow
|
|
5
5
|
Keywords: airflow-provider,cncf.kubernetes,airflow,integration
|
|
6
6
|
Author-email: Apache Software Foundation <dev@airflow.apache.org>
|
|
@@ -28,8 +28,8 @@ Requires-Dist: google-re2>=1.0
|
|
|
28
28
|
Requires-Dist: kubernetes>=29.0.0,<=31.0.0
|
|
29
29
|
Requires-Dist: kubernetes_asyncio>=29.0.0,<=31.0.0
|
|
30
30
|
Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
|
|
31
|
-
Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/
|
|
32
|
-
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/
|
|
31
|
+
Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.0.0/changelog.html
|
|
32
|
+
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.0.0
|
|
33
33
|
Project-URL: Slack Chat, https://s.apache.org/airflow-slack
|
|
34
34
|
Project-URL: Source Code, https://github.com/apache/airflow
|
|
35
35
|
Project-URL: Twitter, https://twitter.com/ApacheAirflow
|
|
@@ -79,7 +79,7 @@ Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
|
|
|
79
79
|
|
|
80
80
|
Package ``apache-airflow-providers-cncf-kubernetes``
|
|
81
81
|
|
|
82
|
-
Release: ``
|
|
82
|
+
Release: ``10.0.0.rc1``
|
|
83
83
|
|
|
84
84
|
|
|
85
85
|
`Kubernetes <https://kubernetes.io/>`__
|
|
@@ -92,7 +92,7 @@ This is a provider package for ``cncf.kubernetes`` provider. All classes for thi
|
|
|
92
92
|
are in ``airflow.providers.cncf.kubernetes`` python package.
|
|
93
93
|
|
|
94
94
|
You can find package information and changelog for the provider
|
|
95
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/
|
|
95
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.0.0/>`_.
|
|
96
96
|
|
|
97
97
|
Installation
|
|
98
98
|
------------
|
|
@@ -119,4 +119,4 @@ PIP package Version required
|
|
|
119
119
|
====================== =====================
|
|
120
120
|
|
|
121
121
|
The changelog for the provider package can be found in the
|
|
122
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/
|
|
122
|
+
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.0.0/changelog.html>`_.
|
|
@@ -1,14 +1,13 @@
|
|
|
1
1
|
airflow/providers/cncf/kubernetes/LICENSE,sha256=FFb4jd2AXnOOf7XLP04pQW6jbdhG49TxlGY6fFpCV1Y,13609
|
|
2
|
-
airflow/providers/cncf/kubernetes/__init__.py,sha256=
|
|
2
|
+
airflow/providers/cncf/kubernetes/__init__.py,sha256=nxLYWteyTzCKa1NCHauBZsE1vOl0TSVqpqd4IR6taqg,1503
|
|
3
3
|
airflow/providers/cncf/kubernetes/callbacks.py,sha256=SK_gKvGWuU-nxHfsqsYMlNQ8HZbHfpvyItOqieel2lc,4162
|
|
4
|
-
airflow/providers/cncf/kubernetes/get_provider_info.py,sha256=
|
|
4
|
+
airflow/providers/cncf/kubernetes/get_provider_info.py,sha256=UdYm7n5XGH9N6hcCKJr5hMC5ekk-2lNP5wgj0plC5JY,18008
|
|
5
5
|
airflow/providers/cncf/kubernetes/k8s_model.py,sha256=xmdFhX29DjegoZ-cq8-KDL9soVYXf4OpU6fAGr3cPTU,2101
|
|
6
6
|
airflow/providers/cncf/kubernetes/kube_client.py,sha256=yflZxLousXA9d7t67KrEy55qzb1cUhEyy6yCPkEem28,5329
|
|
7
7
|
airflow/providers/cncf/kubernetes/kube_config.py,sha256=FAmhZZ_Z2JtoVzL6wENSjcwrlwAenHttTX_Ild9aEms,5225
|
|
8
|
-
airflow/providers/cncf/kubernetes/kubernetes_helper_functions.py,sha256=
|
|
9
|
-
airflow/providers/cncf/kubernetes/pod_generator.py,sha256=
|
|
8
|
+
airflow/providers/cncf/kubernetes/kubernetes_helper_functions.py,sha256=tb6HykGOUKZIXXZssJyQLKBlWv_KoE5mjjxwOkeJHW0,5391
|
|
9
|
+
airflow/providers/cncf/kubernetes/pod_generator.py,sha256=9Yd4FY0MbXc61JuOZwQu5WlFV6MlLBy72YMHF3lD8ao,19765
|
|
10
10
|
airflow/providers/cncf/kubernetes/pod_generator_deprecated.py,sha256=aukRdia0Wtv0FVZXgvaIWQxdWfnT9cxzH0UwOVph8dE,11998
|
|
11
|
-
airflow/providers/cncf/kubernetes/pod_launcher_deprecated.py,sha256=v1dpXnlpGI6nTmwO-h15rfs42NYcg9lrEpxrgA0ynE8,12039
|
|
12
11
|
airflow/providers/cncf/kubernetes/python_kubernetes_script.jinja2,sha256=I0EHRGwLHjSiX85e51HBIoddRDnC8TJPFrDBqQq_NJg,1776
|
|
13
12
|
airflow/providers/cncf/kubernetes/python_kubernetes_script.py,sha256=KnTlZSWCZhwvj89fSc2kgIRTaI4iLNKPquHc2wXnluo,3460
|
|
14
13
|
airflow/providers/cncf/kubernetes/secret.py,sha256=wj-T9gouqau_X14slAstGmnSxqXJQzdLwUdURzHna0I,5209
|
|
@@ -18,12 +17,12 @@ airflow/providers/cncf/kubernetes/backcompat/backwards_compat_converters.py,sha2
|
|
|
18
17
|
airflow/providers/cncf/kubernetes/cli/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
|
19
18
|
airflow/providers/cncf/kubernetes/cli/kubernetes_command.py,sha256=W49hsWZ3XWUJgFLte56tMNGqmOvcFitl7iFjmFR8Ezo,6990
|
|
20
19
|
airflow/providers/cncf/kubernetes/decorators/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
|
|
21
|
-
airflow/providers/cncf/kubernetes/decorators/kubernetes.py,sha256=
|
|
20
|
+
airflow/providers/cncf/kubernetes/decorators/kubernetes.py,sha256=zVDHw_zwgRbobTOouJtCEqDrjrVwINGcB1dA-wG-ME0,5787
|
|
22
21
|
airflow/providers/cncf/kubernetes/executors/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
|
|
23
|
-
airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py,sha256=
|
|
22
|
+
airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py,sha256=mVNIA5SSJj3hRWJy05Rp6DVhzNgrxW-pyENeIAv9Jdg,34271
|
|
24
23
|
airflow/providers/cncf/kubernetes/executors/kubernetes_executor_types.py,sha256=9rRhfRuujGtSE73Ax0kC12whZCgWF2m6j5w9G9e0F_I,1673
|
|
25
24
|
airflow/providers/cncf/kubernetes/executors/kubernetes_executor_utils.py,sha256=f2FHTyQyfFR1jjQmZV9bfX7U4Vxk5h61KPJO0yMcGbM,23356
|
|
26
|
-
airflow/providers/cncf/kubernetes/executors/local_kubernetes_executor.py,sha256=
|
|
25
|
+
airflow/providers/cncf/kubernetes/executors/local_kubernetes_executor.py,sha256=6cEGS83tGVUm6tSsjVenaJXSstViBnwgrvu-XIW4XQg,10976
|
|
27
26
|
airflow/providers/cncf/kubernetes/hooks/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
|
28
27
|
airflow/providers/cncf/kubernetes/hooks/kubernetes.py,sha256=WSWQZrAzyrjN9y5TCS796X1L7oB-Sq_2BUzhT9xAE2k,33632
|
|
29
28
|
airflow/providers/cncf/kubernetes/kubernetes_executor_templates/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
|
@@ -31,10 +30,9 @@ airflow/providers/cncf/kubernetes/kubernetes_executor_templates/basic_template.y
|
|
|
31
30
|
airflow/providers/cncf/kubernetes/operators/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
|
|
32
31
|
airflow/providers/cncf/kubernetes/operators/custom_object_launcher.py,sha256=ZEXw_PqGItO47AEgGKqAqwFHFo-gb9-7jgEMRJgOfNU,15311
|
|
33
32
|
airflow/providers/cncf/kubernetes/operators/job.py,sha256=RyPuT0yP7-tRrrS7uqDUBGI_xA9_YEg-JLUDJ5QcTA8,23809
|
|
34
|
-
airflow/providers/cncf/kubernetes/operators/
|
|
35
|
-
airflow/providers/cncf/kubernetes/operators/pod.py,sha256=MAHaLxy5ScgbY0LO5i3ejH3MHqYyXG3a9harRg-8guE,54803
|
|
33
|
+
airflow/providers/cncf/kubernetes/operators/pod.py,sha256=f8JoEsgZbjSNlClTfw8XeCm-_vSkwpUglXzOiDyhBUg,53495
|
|
36
34
|
airflow/providers/cncf/kubernetes/operators/resource.py,sha256=ccbZQKB1B5N4Y-ruItacB5Q105Tc0uNSoEmUAaSCrGQ,7570
|
|
37
|
-
airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py,sha256=
|
|
35
|
+
airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py,sha256=rsUfqJ_TqjqZSIeUvc6J7pTJiaiX1Ba2WI5Tn9qtfbw,13923
|
|
38
36
|
airflow/providers/cncf/kubernetes/pod_template_file_examples/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
|
39
37
|
airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_image_template.yaml,sha256=7JdppZ-XDBpv2Bnde2SthhcME8w3b8xQdPAK1fJGW60,2256
|
|
40
38
|
airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_volume_template.yaml,sha256=-Pk_EwKpyWRYZKOnumUxVrDeAfFJ0nr3WZ7JNnvppzg,2442
|
|
@@ -47,14 +45,13 @@ airflow/providers/cncf/kubernetes/sensors/__init__.py,sha256=9hdXHABrVpkbpjZgUft
|
|
|
47
45
|
airflow/providers/cncf/kubernetes/sensors/spark_kubernetes.py,sha256=ogD8m8baqPN_hGEXjN1NGp1dvslG8ZPBjy_LPr6Zo7I,5353
|
|
48
46
|
airflow/providers/cncf/kubernetes/triggers/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
|
49
47
|
airflow/providers/cncf/kubernetes/triggers/job.py,sha256=fe0kaQos_m_1rVE-1YJBZKKFVLxrlHOB8oI5ntVzSN8,6727
|
|
50
|
-
airflow/providers/cncf/kubernetes/triggers/
|
|
51
|
-
airflow/providers/cncf/kubernetes/triggers/pod.py,sha256=4L5ZXswM11kAwAietMcfwPuelA8CJ12Jgo40Lndsquc,13629
|
|
48
|
+
airflow/providers/cncf/kubernetes/triggers/pod.py,sha256=Q-1GPILDgGJHoxtkaH78edhNeR6BekRG8f0qpFLGcSI,12626
|
|
52
49
|
airflow/providers/cncf/kubernetes/utils/__init__.py,sha256=ClZN0VPjWySdVwS_ktH7rrgL9VLAcs3OSJSB9s3zaYw,863
|
|
53
50
|
airflow/providers/cncf/kubernetes/utils/delete_from.py,sha256=poObZSoEJwQyaYWilEURs8f4CDY2sn_pfwS31Lf579A,5195
|
|
54
51
|
airflow/providers/cncf/kubernetes/utils/k8s_resource_iterator.py,sha256=-Pgc5i2WEDl7ZBvtJZ4eWDqqlSj8WdULqwUyOWmsRp8,1928
|
|
55
|
-
airflow/providers/cncf/kubernetes/utils/pod_manager.py,sha256=
|
|
52
|
+
airflow/providers/cncf/kubernetes/utils/pod_manager.py,sha256=Xp_K8nay4WYKbG_cJ9mrhc_sFkdmu40Klq859sq9vNs,33286
|
|
56
53
|
airflow/providers/cncf/kubernetes/utils/xcom_sidecar.py,sha256=k6bdmVJ21OrAwGmWwledRrAmaty9ZrmbuM-IbaI4mqo,2519
|
|
57
|
-
apache_airflow_providers_cncf_kubernetes-
|
|
58
|
-
apache_airflow_providers_cncf_kubernetes-
|
|
59
|
-
apache_airflow_providers_cncf_kubernetes-
|
|
60
|
-
apache_airflow_providers_cncf_kubernetes-
|
|
54
|
+
apache_airflow_providers_cncf_kubernetes-10.0.0rc1.dist-info/entry_points.txt,sha256=ByD3QJJyP9CfmTYtpNI1953akD38RUDgpGXLaq9vpOw,111
|
|
55
|
+
apache_airflow_providers_cncf_kubernetes-10.0.0rc1.dist-info/WHEEL,sha256=CpUCUxeHQbRN5UGRQHYRJorO5Af-Qy_fHMctcQ8DSGI,82
|
|
56
|
+
apache_airflow_providers_cncf_kubernetes-10.0.0rc1.dist-info/METADATA,sha256=6twDWiNVfo2cHd4z4ctS_lWWD84d6PPAGU_cmkIBN7Q,5222
|
|
57
|
+
apache_airflow_providers_cncf_kubernetes-10.0.0rc1.dist-info/RECORD,,
|
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
#
|
|
2
|
-
# Licensed to the Apache Software Foundation (ASF) under one
|
|
3
|
-
# or more contributor license agreements. See the NOTICE file
|
|
4
|
-
# distributed with this work for additional information
|
|
5
|
-
# regarding copyright ownership. The ASF licenses this file
|
|
6
|
-
# to you under the Apache License, Version 2.0 (the
|
|
7
|
-
# "License"); you may not use this file except in compliance
|
|
8
|
-
# with the License. You may obtain a copy of the License at
|
|
9
|
-
#
|
|
10
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
11
|
-
#
|
|
12
|
-
# Unless required by applicable law or agreed to in writing,
|
|
13
|
-
# software distributed under the License is distributed on an
|
|
14
|
-
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
15
|
-
# KIND, either express or implied. See the License for the
|
|
16
|
-
# specific language governing permissions and limitations
|
|
17
|
-
# under the License.
|
|
18
|
-
"""This module is deprecated. Please use :mod:`airflow.providers.cncf.kubernetes.operators.pod` instead."""
|
|
19
|
-
|
|
20
|
-
from __future__ import annotations
|
|
21
|
-
|
|
22
|
-
import warnings
|
|
23
|
-
|
|
24
|
-
from airflow.exceptions import AirflowProviderDeprecationWarning
|
|
25
|
-
from airflow.providers.cncf.kubernetes.operators.pod import * # noqa: F403
|
|
26
|
-
|
|
27
|
-
warnings.warn(
|
|
28
|
-
"This module is deprecated. Please use `airflow.providers.cncf.kubernetes.operators.pod` instead.",
|
|
29
|
-
AirflowProviderDeprecationWarning,
|
|
30
|
-
stacklevel=2,
|
|
31
|
-
)
|
|
@@ -1,320 +0,0 @@
|
|
|
1
|
-
# Licensed to the Apache Software Foundation (ASF) under one
|
|
2
|
-
# or more contributor license agreements. See the NOTICE file
|
|
3
|
-
# distributed with this work for additional information
|
|
4
|
-
# regarding copyright ownership. The ASF licenses this file
|
|
5
|
-
# to you under the Apache License, Version 2.0 (the
|
|
6
|
-
# "License"); you may not use this file except in compliance
|
|
7
|
-
# with the License. You may obtain a copy of the License at
|
|
8
|
-
#
|
|
9
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
-
#
|
|
11
|
-
# Unless required by applicable law or agreed to in writing,
|
|
12
|
-
# software distributed under the License is distributed on an
|
|
13
|
-
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
14
|
-
# KIND, either express or implied. See the License for the
|
|
15
|
-
# specific language governing permissions and limitations
|
|
16
|
-
# under the License.
|
|
17
|
-
"""Launches pods."""
|
|
18
|
-
|
|
19
|
-
from __future__ import annotations
|
|
20
|
-
|
|
21
|
-
import json
|
|
22
|
-
import math
|
|
23
|
-
import time
|
|
24
|
-
import warnings
|
|
25
|
-
from typing import TYPE_CHECKING, cast
|
|
26
|
-
|
|
27
|
-
import pendulum
|
|
28
|
-
import tenacity
|
|
29
|
-
from kubernetes import client, watch
|
|
30
|
-
from kubernetes.client.rest import ApiException
|
|
31
|
-
from kubernetes.stream import stream as kubernetes_stream
|
|
32
|
-
from requests.exceptions import HTTPError
|
|
33
|
-
|
|
34
|
-
from airflow.exceptions import AirflowException, RemovedInAirflow3Warning
|
|
35
|
-
from airflow.providers.cncf.kubernetes.kube_client import get_kube_client
|
|
36
|
-
from airflow.providers.cncf.kubernetes.pod_generator import PodDefaultsDeprecated
|
|
37
|
-
from airflow.settings import pod_mutation_hook
|
|
38
|
-
from airflow.utils.log.logging_mixin import LoggingMixin
|
|
39
|
-
from airflow.utils.state import State
|
|
40
|
-
|
|
41
|
-
if TYPE_CHECKING:
|
|
42
|
-
from kubernetes.client.models.v1_pod import V1Pod
|
|
43
|
-
|
|
44
|
-
warnings.warn(
|
|
45
|
-
"""
|
|
46
|
-
Please use :mod: Please use `airflow.providers.cncf.kubernetes.utils.pod_manager`
|
|
47
|
-
|
|
48
|
-
To use this module install the provider package by installing this pip package:
|
|
49
|
-
|
|
50
|
-
https://pypi.org/project/apache-airflow-providers-cncf-kubernetes/
|
|
51
|
-
|
|
52
|
-
""",
|
|
53
|
-
RemovedInAirflow3Warning,
|
|
54
|
-
stacklevel=2,
|
|
55
|
-
)
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
class PodStatus:
|
|
59
|
-
"""Status of the pods."""
|
|
60
|
-
|
|
61
|
-
PENDING = "pending"
|
|
62
|
-
RUNNING = "running"
|
|
63
|
-
FAILED = "failed"
|
|
64
|
-
SUCCEEDED = "succeeded"
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
class PodLauncher(LoggingMixin):
|
|
68
|
-
"""
|
|
69
|
-
Deprecated class for launching pods.
|
|
70
|
-
|
|
71
|
-
Please use airflow.providers.cncf.kubernetes.utils.pod_manager.PodManager instead.
|
|
72
|
-
"""
|
|
73
|
-
|
|
74
|
-
def __init__(
|
|
75
|
-
self,
|
|
76
|
-
kube_client: client.CoreV1Api = None,
|
|
77
|
-
in_cluster: bool = True,
|
|
78
|
-
cluster_context: str | None = None,
|
|
79
|
-
extract_xcom: bool = False,
|
|
80
|
-
):
|
|
81
|
-
"""
|
|
82
|
-
Launch pods; DEPRECATED.
|
|
83
|
-
|
|
84
|
-
Please use airflow.providers.cncf.kubernetes.utils.pod_manager.PodManager
|
|
85
|
-
instead to create the launcher.
|
|
86
|
-
|
|
87
|
-
:param kube_client: kubernetes client
|
|
88
|
-
:param in_cluster: whether we are in cluster
|
|
89
|
-
:param cluster_context: context of the cluster
|
|
90
|
-
:param extract_xcom: whether we should extract xcom
|
|
91
|
-
"""
|
|
92
|
-
super().__init__()
|
|
93
|
-
self._client = kube_client or get_kube_client(in_cluster=in_cluster, cluster_context=cluster_context)
|
|
94
|
-
self._watch = watch.Watch()
|
|
95
|
-
self.extract_xcom = extract_xcom
|
|
96
|
-
|
|
97
|
-
def run_pod_async(self, pod: V1Pod, **kwargs):
|
|
98
|
-
"""Run pod asynchronously."""
|
|
99
|
-
pod_mutation_hook(pod)
|
|
100
|
-
|
|
101
|
-
sanitized_pod = self._client.api_client.sanitize_for_serialization(pod)
|
|
102
|
-
json_pod = json.dumps(sanitized_pod, indent=2)
|
|
103
|
-
|
|
104
|
-
self.log.debug("Pod Creation Request: \n%s", json_pod)
|
|
105
|
-
try:
|
|
106
|
-
resp = self._client.create_namespaced_pod(
|
|
107
|
-
body=sanitized_pod, namespace=pod.metadata.namespace, **kwargs
|
|
108
|
-
)
|
|
109
|
-
self.log.debug("Pod Creation Response: %s", resp)
|
|
110
|
-
except Exception as e:
|
|
111
|
-
self.log.exception("Exception when attempting to create Namespaced Pod: %s", json_pod)
|
|
112
|
-
raise e
|
|
113
|
-
return resp
|
|
114
|
-
|
|
115
|
-
def delete_pod(self, pod: V1Pod):
|
|
116
|
-
"""Delete pod."""
|
|
117
|
-
try:
|
|
118
|
-
self._client.delete_namespaced_pod(
|
|
119
|
-
pod.metadata.name, pod.metadata.namespace, body=client.V1DeleteOptions()
|
|
120
|
-
)
|
|
121
|
-
except ApiException as e:
|
|
122
|
-
# If the pod is already deleted
|
|
123
|
-
if str(e.status) != "404":
|
|
124
|
-
raise
|
|
125
|
-
|
|
126
|
-
def start_pod(self, pod: V1Pod, startup_timeout: int = 120):
|
|
127
|
-
"""
|
|
128
|
-
Launch the pod synchronously and wait for completion.
|
|
129
|
-
|
|
130
|
-
:param pod:
|
|
131
|
-
:param startup_timeout: Timeout for startup of the pod (if pod is pending for too long, fails task)
|
|
132
|
-
:return:
|
|
133
|
-
"""
|
|
134
|
-
resp = self.run_pod_async(pod)
|
|
135
|
-
start_time = time.monotonic()
|
|
136
|
-
if resp.status.start_time is None:
|
|
137
|
-
while self.pod_not_started(pod):
|
|
138
|
-
self.log.warning("Pod not yet started: %s", pod.metadata.name)
|
|
139
|
-
if time.monotonic() >= start_time + startup_timeout:
|
|
140
|
-
raise AirflowException("Pod took too long to start")
|
|
141
|
-
time.sleep(1)
|
|
142
|
-
|
|
143
|
-
def monitor_pod(self, pod: V1Pod, get_logs: bool) -> tuple[State, str | None]:
|
|
144
|
-
"""
|
|
145
|
-
Monitor a pod and return the final state.
|
|
146
|
-
|
|
147
|
-
:param pod: pod spec that will be monitored
|
|
148
|
-
:param get_logs: whether to read the logs locally
|
|
149
|
-
"""
|
|
150
|
-
if get_logs:
|
|
151
|
-
read_logs_since_sec = None
|
|
152
|
-
last_log_time: pendulum.DateTime | None = None
|
|
153
|
-
while True:
|
|
154
|
-
logs = self.read_pod_logs(pod, timestamps=True, since_seconds=read_logs_since_sec)
|
|
155
|
-
for line in logs:
|
|
156
|
-
timestamp, message = self.parse_log_line(line.decode("utf-8"))
|
|
157
|
-
if timestamp:
|
|
158
|
-
last_log_time = cast(pendulum.DateTime, pendulum.parse(timestamp))
|
|
159
|
-
self.log.info(message)
|
|
160
|
-
time.sleep(1)
|
|
161
|
-
|
|
162
|
-
if not self.base_container_is_running(pod):
|
|
163
|
-
break
|
|
164
|
-
|
|
165
|
-
self.log.warning("Pod %s log read interrupted", pod.metadata.name)
|
|
166
|
-
if last_log_time:
|
|
167
|
-
delta = pendulum.now() - last_log_time
|
|
168
|
-
# Prefer logs duplication rather than loss
|
|
169
|
-
read_logs_since_sec = math.ceil(delta.total_seconds())
|
|
170
|
-
result = None
|
|
171
|
-
if self.extract_xcom:
|
|
172
|
-
while self.base_container_is_running(pod):
|
|
173
|
-
self.log.info("Container %s has state %s", pod.metadata.name, State.RUNNING)
|
|
174
|
-
time.sleep(2)
|
|
175
|
-
result = self._extract_xcom(pod)
|
|
176
|
-
self.log.info(result)
|
|
177
|
-
result = json.loads(result)
|
|
178
|
-
while self.pod_is_running(pod):
|
|
179
|
-
self.log.info("Pod %s has state %s", pod.metadata.name, State.RUNNING)
|
|
180
|
-
time.sleep(2)
|
|
181
|
-
return self._task_status(self.read_pod(pod)), result
|
|
182
|
-
|
|
183
|
-
def parse_log_line(self, line: str) -> tuple[str | None, str]:
|
|
184
|
-
"""
|
|
185
|
-
Parse K8s log line and returns the final state.
|
|
186
|
-
|
|
187
|
-
:param line: k8s log line
|
|
188
|
-
:return: timestamp and log message
|
|
189
|
-
"""
|
|
190
|
-
timestamp, sep, message = line.strip().partition(" ")
|
|
191
|
-
if not sep:
|
|
192
|
-
self.log.error(
|
|
193
|
-
"Error parsing timestamp (no timestamp in message: %r). "
|
|
194
|
-
"Will continue execution but won't update timestamp",
|
|
195
|
-
line,
|
|
196
|
-
)
|
|
197
|
-
return None, line
|
|
198
|
-
return timestamp, message
|
|
199
|
-
|
|
200
|
-
def _task_status(self, event):
|
|
201
|
-
self.log.info("Event: %s had an event of type %s", event.metadata.name, event.status.phase)
|
|
202
|
-
status = self.process_status(event.metadata.name, event.status.phase)
|
|
203
|
-
return status
|
|
204
|
-
|
|
205
|
-
def pod_not_started(self, pod: V1Pod):
|
|
206
|
-
"""Test if pod has not started."""
|
|
207
|
-
state = self._task_status(self.read_pod(pod))
|
|
208
|
-
return state == State.QUEUED
|
|
209
|
-
|
|
210
|
-
def pod_is_running(self, pod: V1Pod):
|
|
211
|
-
"""Test if pod is running."""
|
|
212
|
-
state = self._task_status(self.read_pod(pod))
|
|
213
|
-
return state not in (State.SUCCESS, State.FAILED)
|
|
214
|
-
|
|
215
|
-
def base_container_is_running(self, pod: V1Pod):
|
|
216
|
-
"""Test if base container is running."""
|
|
217
|
-
event = self.read_pod(pod)
|
|
218
|
-
status = next((s for s in event.status.container_statuses if s.name == "base"), None)
|
|
219
|
-
if not status:
|
|
220
|
-
return False
|
|
221
|
-
return status.state.running is not None
|
|
222
|
-
|
|
223
|
-
@tenacity.retry(stop=tenacity.stop_after_attempt(3), wait=tenacity.wait_exponential(), reraise=True)
|
|
224
|
-
def read_pod_logs(
|
|
225
|
-
self,
|
|
226
|
-
pod: V1Pod,
|
|
227
|
-
tail_lines: int | None = None,
|
|
228
|
-
timestamps: bool = False,
|
|
229
|
-
since_seconds: int | None = None,
|
|
230
|
-
):
|
|
231
|
-
"""Read log from the pod."""
|
|
232
|
-
additional_kwargs = {}
|
|
233
|
-
if since_seconds:
|
|
234
|
-
additional_kwargs["since_seconds"] = since_seconds
|
|
235
|
-
|
|
236
|
-
if tail_lines:
|
|
237
|
-
additional_kwargs["tail_lines"] = tail_lines
|
|
238
|
-
|
|
239
|
-
try:
|
|
240
|
-
return self._client.read_namespaced_pod_log(
|
|
241
|
-
name=pod.metadata.name,
|
|
242
|
-
namespace=pod.metadata.namespace,
|
|
243
|
-
container="base",
|
|
244
|
-
follow=True,
|
|
245
|
-
timestamps=timestamps,
|
|
246
|
-
_preload_content=False,
|
|
247
|
-
**additional_kwargs,
|
|
248
|
-
)
|
|
249
|
-
except HTTPError as e:
|
|
250
|
-
raise AirflowException(f"There was an error reading the kubernetes API: {e}")
|
|
251
|
-
|
|
252
|
-
@tenacity.retry(stop=tenacity.stop_after_attempt(3), wait=tenacity.wait_exponential(), reraise=True)
|
|
253
|
-
def read_pod_events(self, pod):
|
|
254
|
-
"""Read events from the pod."""
|
|
255
|
-
try:
|
|
256
|
-
return self._client.list_namespaced_event(
|
|
257
|
-
namespace=pod.metadata.namespace, field_selector=f"involvedObject.name={pod.metadata.name}"
|
|
258
|
-
)
|
|
259
|
-
except HTTPError as e:
|
|
260
|
-
raise AirflowException(f"There was an error reading the kubernetes API: {e}")
|
|
261
|
-
|
|
262
|
-
@tenacity.retry(stop=tenacity.stop_after_attempt(3), wait=tenacity.wait_exponential(), reraise=True)
|
|
263
|
-
def read_pod(self, pod: V1Pod):
|
|
264
|
-
"""Read pod information."""
|
|
265
|
-
try:
|
|
266
|
-
return self._client.read_namespaced_pod(pod.metadata.name, pod.metadata.namespace)
|
|
267
|
-
except HTTPError as e:
|
|
268
|
-
raise AirflowException(f"There was an error reading the kubernetes API: {e}")
|
|
269
|
-
|
|
270
|
-
def _extract_xcom(self, pod: V1Pod):
|
|
271
|
-
resp = kubernetes_stream(
|
|
272
|
-
self._client.connect_get_namespaced_pod_exec,
|
|
273
|
-
pod.metadata.name,
|
|
274
|
-
pod.metadata.namespace,
|
|
275
|
-
container=PodDefaultsDeprecated.SIDECAR_CONTAINER_NAME,
|
|
276
|
-
command=["/bin/sh"],
|
|
277
|
-
stdin=True,
|
|
278
|
-
stdout=True,
|
|
279
|
-
stderr=True,
|
|
280
|
-
tty=False,
|
|
281
|
-
_preload_content=False,
|
|
282
|
-
)
|
|
283
|
-
try:
|
|
284
|
-
result = self._exec_pod_command(resp, f"cat {PodDefaultsDeprecated.XCOM_MOUNT_PATH}/return.json")
|
|
285
|
-
self._exec_pod_command(resp, "kill -s SIGINT 1")
|
|
286
|
-
finally:
|
|
287
|
-
resp.close()
|
|
288
|
-
if result is None:
|
|
289
|
-
raise AirflowException(f"Failed to extract xcom from pod: {pod.metadata.name}")
|
|
290
|
-
return result
|
|
291
|
-
|
|
292
|
-
def _exec_pod_command(self, resp, command):
|
|
293
|
-
if resp.is_open():
|
|
294
|
-
self.log.info("Running command... %s\n", command)
|
|
295
|
-
resp.write_stdin(command + "\n")
|
|
296
|
-
while resp.is_open():
|
|
297
|
-
resp.update(timeout=1)
|
|
298
|
-
if resp.peek_stdout():
|
|
299
|
-
return resp.read_stdout()
|
|
300
|
-
if resp.peek_stderr():
|
|
301
|
-
self.log.info(resp.read_stderr())
|
|
302
|
-
break
|
|
303
|
-
return None
|
|
304
|
-
|
|
305
|
-
def process_status(self, job_id, status):
|
|
306
|
-
"""Process status information for the job."""
|
|
307
|
-
status = status.lower()
|
|
308
|
-
if status == PodStatus.PENDING:
|
|
309
|
-
return State.QUEUED
|
|
310
|
-
elif status == PodStatus.FAILED:
|
|
311
|
-
self.log.error("Event with job id %s Failed", job_id)
|
|
312
|
-
return State.FAILED
|
|
313
|
-
elif status == PodStatus.SUCCEEDED:
|
|
314
|
-
self.log.info("Event with job id %s Succeeded", job_id)
|
|
315
|
-
return State.SUCCESS
|
|
316
|
-
elif status == PodStatus.RUNNING:
|
|
317
|
-
return State.RUNNING
|
|
318
|
-
else:
|
|
319
|
-
self.log.error("Event: Invalid state %s on job %s", status, job_id)
|
|
320
|
-
return State.FAILED
|
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
#
|
|
2
|
-
# Licensed to the Apache Software Foundation (ASF) under one
|
|
3
|
-
# or more contributor license agreements. See the NOTICE file
|
|
4
|
-
# distributed with this work for additional information
|
|
5
|
-
# regarding copyright ownership. The ASF licenses this file
|
|
6
|
-
# to you under the Apache License, Version 2.0 (the
|
|
7
|
-
# "License"); you may not use this file except in compliance
|
|
8
|
-
# with the License. You may obtain a copy of the License at
|
|
9
|
-
#
|
|
10
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
11
|
-
#
|
|
12
|
-
# Unless required by applicable law or agreed to in writing,
|
|
13
|
-
# software distributed under the License is distributed on an
|
|
14
|
-
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
15
|
-
# KIND, either express or implied. See the License for the
|
|
16
|
-
# specific language governing permissions and limitations
|
|
17
|
-
# under the License.
|
|
18
|
-
"""This module is deprecated. Please use :mod:`airflow.providers.cncf.kubernetes.triggers.pod` instead."""
|
|
19
|
-
|
|
20
|
-
from __future__ import annotations
|
|
21
|
-
|
|
22
|
-
import warnings
|
|
23
|
-
|
|
24
|
-
from airflow.exceptions import AirflowProviderDeprecationWarning
|
|
25
|
-
from airflow.providers.cncf.kubernetes.triggers.pod import * # noqa: F403
|
|
26
|
-
|
|
27
|
-
warnings.warn(
|
|
28
|
-
"This module is deprecated. Please use `airflow.providers.cncf.kubernetes.triggers.pod` instead.",
|
|
29
|
-
AirflowProviderDeprecationWarning,
|
|
30
|
-
stacklevel=2,
|
|
31
|
-
)
|