apache-airflow-providers-cncf-kubernetes 7.0.0__py3-none-any.whl → 7.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/cncf/kubernetes/__init__.py +1 -1
- airflow/providers/cncf/kubernetes/backcompat/backwards_compat_converters.py +10 -10
- airflow/providers/cncf/kubernetes/get_provider_info.py +3 -1
- airflow/providers/cncf/kubernetes/hooks/kubernetes.py +14 -12
- airflow/providers/cncf/kubernetes/operators/pod.py +22 -12
- airflow/providers/cncf/kubernetes/operators/resource.py +103 -0
- airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py +1 -1
- airflow/providers/cncf/kubernetes/python_kubernetes_script.py +2 -2
- airflow/providers/cncf/kubernetes/sensors/spark_kubernetes.py +1 -1
- airflow/providers/cncf/kubernetes/triggers/pod.py +1 -1
- airflow/providers/cncf/kubernetes/utils/delete_from.py +157 -0
- airflow/providers/cncf/kubernetes/utils/pod_manager.py +35 -21
- airflow/providers/cncf/kubernetes/utils/xcom_sidecar.py +2 -2
- {apache_airflow_providers_cncf_kubernetes-7.0.0.dist-info → apache_airflow_providers_cncf_kubernetes-7.1.0.dist-info}/METADATA +37 -7
- {apache_airflow_providers_cncf_kubernetes-7.0.0.dist-info → apache_airflow_providers_cncf_kubernetes-7.1.0.dist-info}/RECORD +20 -18
- {apache_airflow_providers_cncf_kubernetes-7.0.0.dist-info → apache_airflow_providers_cncf_kubernetes-7.1.0.dist-info}/LICENSE +0 -0
- {apache_airflow_providers_cncf_kubernetes-7.0.0.dist-info → apache_airflow_providers_cncf_kubernetes-7.1.0.dist-info}/NOTICE +0 -0
- {apache_airflow_providers_cncf_kubernetes-7.0.0.dist-info → apache_airflow_providers_cncf_kubernetes-7.1.0.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_cncf_kubernetes-7.0.0.dist-info → apache_airflow_providers_cncf_kubernetes-7.1.0.dist-info}/entry_points.txt +0 -0
- {apache_airflow_providers_cncf_kubernetes-7.0.0.dist-info → apache_airflow_providers_cncf_kubernetes-7.1.0.dist-info}/top_level.txt +0 -0
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
# KIND, either express or implied. See the License for the
|
|
15
15
|
# specific language governing permissions and limitations
|
|
16
16
|
# under the License.
|
|
17
|
-
"""Executes task in a Kubernetes POD"""
|
|
17
|
+
"""Executes task in a Kubernetes POD."""
|
|
18
18
|
from __future__ import annotations
|
|
19
19
|
|
|
20
20
|
from kubernetes.client import ApiClient, models as k8s
|
|
@@ -43,7 +43,7 @@ def _convert_from_dict(obj, new_class):
|
|
|
43
43
|
|
|
44
44
|
|
|
45
45
|
def convert_volume(volume) -> k8s.V1Volume:
|
|
46
|
-
"""Converts an airflow Volume object into a k8s.V1Volume
|
|
46
|
+
"""Converts an airflow Volume object into a k8s.V1Volume.
|
|
47
47
|
|
|
48
48
|
:param volume:
|
|
49
49
|
"""
|
|
@@ -51,7 +51,7 @@ def convert_volume(volume) -> k8s.V1Volume:
|
|
|
51
51
|
|
|
52
52
|
|
|
53
53
|
def convert_volume_mount(volume_mount) -> k8s.V1VolumeMount:
|
|
54
|
-
"""Converts an airflow VolumeMount object into a k8s.V1VolumeMount
|
|
54
|
+
"""Converts an airflow VolumeMount object into a k8s.V1VolumeMount.
|
|
55
55
|
|
|
56
56
|
:param volume_mount:
|
|
57
57
|
"""
|
|
@@ -59,7 +59,7 @@ def convert_volume_mount(volume_mount) -> k8s.V1VolumeMount:
|
|
|
59
59
|
|
|
60
60
|
|
|
61
61
|
def convert_port(port) -> k8s.V1ContainerPort:
|
|
62
|
-
"""Converts an airflow Port object into a k8s.V1ContainerPort
|
|
62
|
+
"""Converts an airflow Port object into a k8s.V1ContainerPort.
|
|
63
63
|
|
|
64
64
|
:param port:
|
|
65
65
|
"""
|
|
@@ -67,7 +67,7 @@ def convert_port(port) -> k8s.V1ContainerPort:
|
|
|
67
67
|
|
|
68
68
|
|
|
69
69
|
def convert_env_vars(env_vars) -> list[k8s.V1EnvVar]:
|
|
70
|
-
"""Converts a dictionary into a list of env_vars
|
|
70
|
+
"""Converts a dictionary into a list of env_vars.
|
|
71
71
|
|
|
72
72
|
:param env_vars:
|
|
73
73
|
"""
|
|
@@ -83,7 +83,7 @@ def convert_env_vars(env_vars) -> list[k8s.V1EnvVar]:
|
|
|
83
83
|
|
|
84
84
|
|
|
85
85
|
def convert_pod_runtime_info_env(pod_runtime_info_envs) -> k8s.V1EnvVar:
|
|
86
|
-
"""Converts a PodRuntimeInfoEnv into an k8s.V1EnvVar
|
|
86
|
+
"""Converts a PodRuntimeInfoEnv into an k8s.V1EnvVar.
|
|
87
87
|
|
|
88
88
|
:param pod_runtime_info_envs:
|
|
89
89
|
"""
|
|
@@ -91,7 +91,7 @@ def convert_pod_runtime_info_env(pod_runtime_info_envs) -> k8s.V1EnvVar:
|
|
|
91
91
|
|
|
92
92
|
|
|
93
93
|
def convert_image_pull_secrets(image_pull_secrets) -> list[k8s.V1LocalObjectReference]:
|
|
94
|
-
"""Converts a PodRuntimeInfoEnv into an k8s.V1EnvVar
|
|
94
|
+
"""Converts a PodRuntimeInfoEnv into an k8s.V1EnvVar.
|
|
95
95
|
|
|
96
96
|
:param image_pull_secrets:
|
|
97
97
|
"""
|
|
@@ -103,7 +103,7 @@ def convert_image_pull_secrets(image_pull_secrets) -> list[k8s.V1LocalObjectRefe
|
|
|
103
103
|
|
|
104
104
|
|
|
105
105
|
def convert_configmap(configmaps) -> k8s.V1EnvFromSource:
|
|
106
|
-
"""Converts a str into an k8s.V1EnvFromSource
|
|
106
|
+
"""Converts a str into an k8s.V1EnvFromSource.
|
|
107
107
|
|
|
108
108
|
:param configmaps:
|
|
109
109
|
"""
|
|
@@ -111,10 +111,10 @@ def convert_configmap(configmaps) -> k8s.V1EnvFromSource:
|
|
|
111
111
|
|
|
112
112
|
|
|
113
113
|
def convert_affinity(affinity) -> k8s.V1Affinity:
|
|
114
|
-
"""Converts a dict into an k8s.V1Affinity"""
|
|
114
|
+
"""Converts a dict into an k8s.V1Affinity."""
|
|
115
115
|
return _convert_from_dict(affinity, k8s.V1Affinity)
|
|
116
116
|
|
|
117
117
|
|
|
118
118
|
def convert_toleration(toleration) -> k8s.V1Toleration:
|
|
119
|
-
"""Converts a dict into an k8s.V1Toleration"""
|
|
119
|
+
"""Converts a dict into an k8s.V1Toleration."""
|
|
120
120
|
return _convert_from_dict(toleration, k8s.V1Toleration)
|
|
@@ -29,6 +29,7 @@ def get_provider_info():
|
|
|
29
29
|
"description": "`Kubernetes <https://kubernetes.io/>`__\n",
|
|
30
30
|
"suspended": False,
|
|
31
31
|
"versions": [
|
|
32
|
+
"7.1.0",
|
|
32
33
|
"7.0.0",
|
|
33
34
|
"6.1.0",
|
|
34
35
|
"6.0.0",
|
|
@@ -93,6 +94,7 @@ def get_provider_info():
|
|
|
93
94
|
"airflow.providers.cncf.kubernetes.operators.kubernetes_pod",
|
|
94
95
|
"airflow.providers.cncf.kubernetes.operators.pod",
|
|
95
96
|
"airflow.providers.cncf.kubernetes.operators.spark_kubernetes",
|
|
97
|
+
"airflow.providers.cncf.kubernetes.operators.resource",
|
|
96
98
|
],
|
|
97
99
|
}
|
|
98
100
|
],
|
|
@@ -111,7 +113,7 @@ def get_provider_info():
|
|
|
111
113
|
"triggers": [
|
|
112
114
|
{
|
|
113
115
|
"integration-name": "Kubernetes",
|
|
114
|
-
"
|
|
116
|
+
"python-modules": ["airflow.providers.cncf.kubernetes.triggers.pod"],
|
|
115
117
|
}
|
|
116
118
|
],
|
|
117
119
|
"connection-types": [
|
|
@@ -18,6 +18,7 @@ from __future__ import annotations
|
|
|
18
18
|
|
|
19
19
|
import contextlib
|
|
20
20
|
import tempfile
|
|
21
|
+
from functools import cached_property
|
|
21
22
|
from typing import TYPE_CHECKING, Any, Generator
|
|
22
23
|
|
|
23
24
|
from asgiref.sync import sync_to_async
|
|
@@ -27,7 +28,6 @@ from kubernetes.config import ConfigException
|
|
|
27
28
|
from kubernetes_asyncio import client as async_client, config as async_config
|
|
28
29
|
from urllib3.exceptions import HTTPError
|
|
29
30
|
|
|
30
|
-
from airflow.compat.functools import cached_property
|
|
31
31
|
from airflow.exceptions import AirflowException, AirflowNotFoundException
|
|
32
32
|
from airflow.hooks.base import BaseHook
|
|
33
33
|
from airflow.kubernetes.kube_client import _disable_verify_ssl, _enable_tcp_keepalive
|
|
@@ -84,7 +84,7 @@ class KubernetesHook(BaseHook, PodOperatorHookProtocol):
|
|
|
84
84
|
|
|
85
85
|
@staticmethod
|
|
86
86
|
def get_connection_form_widgets() -> dict[str, Any]:
|
|
87
|
-
"""Returns connection widgets to add to connection form"""
|
|
87
|
+
"""Returns connection widgets to add to connection form."""
|
|
88
88
|
from flask_appbuilder.fieldwidgets import BS3TextFieldWidget
|
|
89
89
|
from flask_babel import lazy_gettext
|
|
90
90
|
from wtforms import BooleanField, StringField
|
|
@@ -103,7 +103,7 @@ class KubernetesHook(BaseHook, PodOperatorHookProtocol):
|
|
|
103
103
|
|
|
104
104
|
@staticmethod
|
|
105
105
|
def get_ui_field_behaviour() -> dict[str, Any]:
|
|
106
|
-
"""Returns custom field behaviour"""
|
|
106
|
+
"""Returns custom field behaviour."""
|
|
107
107
|
return {
|
|
108
108
|
"hidden_fields": ["host", "schema", "login", "password", "port", "extra"],
|
|
109
109
|
"relabeling": {},
|
|
@@ -177,7 +177,7 @@ class KubernetesHook(BaseHook, PodOperatorHookProtocol):
|
|
|
177
177
|
return self.conn_extras.get(prefixed_name) or None
|
|
178
178
|
|
|
179
179
|
def get_conn(self) -> client.ApiClient:
|
|
180
|
-
"""Returns kubernetes api session for use with requests"""
|
|
180
|
+
"""Returns kubernetes api session for use with requests."""
|
|
181
181
|
in_cluster = self._coalesce_param(self.in_cluster, self._get_field("in_cluster"))
|
|
182
182
|
cluster_context = self._coalesce_param(self.cluster_context, self._get_field("cluster_context"))
|
|
183
183
|
kubeconfig_path = self._coalesce_param(self.config_file, self._get_field("kube_config_path"))
|
|
@@ -253,7 +253,7 @@ class KubernetesHook(BaseHook, PodOperatorHookProtocol):
|
|
|
253
253
|
|
|
254
254
|
@property
|
|
255
255
|
def is_in_cluster(self) -> bool:
|
|
256
|
-
"""Expose whether the hook is configured with ``load_incluster_config`` or not"""
|
|
256
|
+
"""Expose whether the hook is configured with ``load_incluster_config`` or not."""
|
|
257
257
|
if self._is_in_cluster is not None:
|
|
258
258
|
return self._is_in_cluster
|
|
259
259
|
self.api_client # so we can determine if we are in_cluster or not
|
|
@@ -263,7 +263,7 @@ class KubernetesHook(BaseHook, PodOperatorHookProtocol):
|
|
|
263
263
|
|
|
264
264
|
@cached_property
|
|
265
265
|
def api_client(self) -> client.ApiClient:
|
|
266
|
-
"""Cached Kubernetes API client"""
|
|
266
|
+
"""Cached Kubernetes API client."""
|
|
267
267
|
return self.get_conn()
|
|
268
268
|
|
|
269
269
|
@cached_property
|
|
@@ -278,7 +278,8 @@ class KubernetesHook(BaseHook, PodOperatorHookProtocol):
|
|
|
278
278
|
self, group: str, version: str, plural: str, body: str | dict, namespace: str | None = None
|
|
279
279
|
):
|
|
280
280
|
"""
|
|
281
|
-
Creates custom resource definition object in Kubernetes
|
|
281
|
+
Creates custom resource definition object in Kubernetes.
|
|
282
|
+
|
|
282
283
|
:param group: api group
|
|
283
284
|
:param version: api version
|
|
284
285
|
:param plural: api plural
|
|
@@ -307,7 +308,7 @@ class KubernetesHook(BaseHook, PodOperatorHookProtocol):
|
|
|
307
308
|
self, group: str, version: str, plural: str, name: str, namespace: str | None = None
|
|
308
309
|
):
|
|
309
310
|
"""
|
|
310
|
-
Get custom resource definition object from Kubernetes
|
|
311
|
+
Get custom resource definition object from Kubernetes.
|
|
311
312
|
|
|
312
313
|
:param group: api group
|
|
313
314
|
:param version: api version
|
|
@@ -329,7 +330,7 @@ class KubernetesHook(BaseHook, PodOperatorHookProtocol):
|
|
|
329
330
|
self, group: str, version: str, plural: str, name: str, namespace: str | None = None, **kwargs
|
|
330
331
|
):
|
|
331
332
|
"""
|
|
332
|
-
Delete custom resource definition object from Kubernetes
|
|
333
|
+
Delete custom resource definition object from Kubernetes.
|
|
333
334
|
|
|
334
335
|
:param group: api group
|
|
335
336
|
:param version: api version
|
|
@@ -348,7 +349,7 @@ class KubernetesHook(BaseHook, PodOperatorHookProtocol):
|
|
|
348
349
|
)
|
|
349
350
|
|
|
350
351
|
def get_namespace(self) -> str | None:
|
|
351
|
-
"""Returns the namespace that defined in the connection"""
|
|
352
|
+
"""Returns the namespace that defined in the connection."""
|
|
352
353
|
if self.conn_id:
|
|
353
354
|
return self._get_field("namespace")
|
|
354
355
|
return None
|
|
@@ -412,7 +413,8 @@ class KubernetesHook(BaseHook, PodOperatorHookProtocol):
|
|
|
412
413
|
**kwargs,
|
|
413
414
|
):
|
|
414
415
|
"""
|
|
415
|
-
Retrieves a list of Kind pod which belong default kubernetes namespace
|
|
416
|
+
Retrieves a list of Kind pod which belong default kubernetes namespace.
|
|
417
|
+
|
|
416
418
|
:param label_selector: A selector to restrict the list of returned objects by their labels
|
|
417
419
|
:param namespace: kubernetes namespace
|
|
418
420
|
:param watch: Watch for changes to the described resources and return them as a stream
|
|
@@ -449,7 +451,7 @@ class AsyncKubernetesHook(KubernetesHook):
|
|
|
449
451
|
self._extras: dict | None = None
|
|
450
452
|
|
|
451
453
|
async def _load_config(self):
|
|
452
|
-
"""Returns Kubernetes API session for use with requests"""
|
|
454
|
+
"""Returns Kubernetes API session for use with requests."""
|
|
453
455
|
in_cluster = self._coalesce_param(self.in_cluster, await self._get_field("in_cluster"))
|
|
454
456
|
cluster_context = self._coalesce_param(self.cluster_context, await self._get_field("cluster_context"))
|
|
455
457
|
kubeconfig_path = self._coalesce_param(self.config_file, await self._get_field("kube_config_path"))
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
# KIND, either express or implied. See the License for the
|
|
15
15
|
# specific language governing permissions and limitations
|
|
16
16
|
# under the License.
|
|
17
|
-
"""Executes task in a Kubernetes POD"""
|
|
17
|
+
"""Executes task in a Kubernetes POD."""
|
|
18
18
|
|
|
19
19
|
from __future__ import annotations
|
|
20
20
|
|
|
@@ -25,13 +25,13 @@ import secrets
|
|
|
25
25
|
import string
|
|
26
26
|
from collections.abc import Container
|
|
27
27
|
from contextlib import AbstractContextManager
|
|
28
|
+
from functools import cached_property
|
|
28
29
|
from typing import TYPE_CHECKING, Any, Sequence
|
|
29
30
|
|
|
30
31
|
from kubernetes.client import CoreV1Api, models as k8s
|
|
31
32
|
from slugify import slugify
|
|
32
33
|
from urllib3.exceptions import HTTPError
|
|
33
34
|
|
|
34
|
-
from airflow.compat.functools import cached_property
|
|
35
35
|
from airflow.exceptions import AirflowException, AirflowSkipException
|
|
36
36
|
from airflow.kubernetes import pod_generator
|
|
37
37
|
from airflow.kubernetes.pod_generator import PodGenerator
|
|
@@ -85,7 +85,7 @@ def _rand_str(num):
|
|
|
85
85
|
|
|
86
86
|
|
|
87
87
|
def _add_pod_suffix(*, pod_name, rand_len=8, max_len=253):
|
|
88
|
-
"""Add random string to pod name while staying under max len
|
|
88
|
+
"""Add random string to pod name while staying under max len.
|
|
89
89
|
|
|
90
90
|
TODO: when min airflow version >= 2.5, delete this function and import from kubernetes_helper_functions.
|
|
91
91
|
|
|
@@ -135,7 +135,7 @@ class PodReattachFailure(AirflowException):
|
|
|
135
135
|
|
|
136
136
|
class KubernetesPodOperator(BaseOperator):
|
|
137
137
|
"""
|
|
138
|
-
Execute a task in a Kubernetes Pod
|
|
138
|
+
Execute a task in a Kubernetes Pod.
|
|
139
139
|
|
|
140
140
|
.. seealso::
|
|
141
141
|
For more information on how to use this operator, take a look at the guide:
|
|
@@ -225,6 +225,7 @@ class KubernetesPodOperator(BaseOperator):
|
|
|
225
225
|
container name to use.
|
|
226
226
|
:param deferrable: Run operator in the deferrable mode.
|
|
227
227
|
:param poll_interval: Polling period in seconds to check for the status. Used only in deferrable mode.
|
|
228
|
+
:param log_pod_spec_on_failure: Log the pod's specification if a failure occurs
|
|
228
229
|
"""
|
|
229
230
|
|
|
230
231
|
# This field can be overloaded at the instance level via base_container_name
|
|
@@ -301,6 +302,7 @@ class KubernetesPodOperator(BaseOperator):
|
|
|
301
302
|
base_container_name: str | None = None,
|
|
302
303
|
deferrable: bool = False,
|
|
303
304
|
poll_interval: float = 2,
|
|
305
|
+
log_pod_spec_on_failure: bool = True,
|
|
304
306
|
**kwargs,
|
|
305
307
|
) -> None:
|
|
306
308
|
# TODO: remove in provider 6.0.0 release. This is a mitigate step to advise users to switch to the
|
|
@@ -381,6 +383,7 @@ class KubernetesPodOperator(BaseOperator):
|
|
|
381
383
|
self.deferrable = deferrable
|
|
382
384
|
self.poll_interval = poll_interval
|
|
383
385
|
self.remote_pod: k8s.V1Pod | None = None
|
|
386
|
+
self.log_pod_spec_on_failure = log_pod_spec_on_failure
|
|
384
387
|
self._config_dict: dict | None = None # TODO: remove it when removing convert_config_file_to_dict
|
|
385
388
|
|
|
386
389
|
@cached_property
|
|
@@ -423,7 +426,7 @@ class KubernetesPodOperator(BaseOperator):
|
|
|
423
426
|
@staticmethod
|
|
424
427
|
def _get_ti_pod_labels(context: Context | None = None, include_try_number: bool = True) -> dict[str, str]:
|
|
425
428
|
"""
|
|
426
|
-
Generate labels for the pod to track the pod in case of Operator crash
|
|
429
|
+
Generate labels for the pod to track the pod in case of Operator crash.
|
|
427
430
|
|
|
428
431
|
:param context: task context provided by airflow DAG
|
|
429
432
|
:return: dict
|
|
@@ -515,7 +518,7 @@ class KubernetesPodOperator(BaseOperator):
|
|
|
515
518
|
raise
|
|
516
519
|
|
|
517
520
|
def extract_xcom(self, pod: k8s.V1Pod):
|
|
518
|
-
"""Retrieves xcom value and kills xcom sidecar container"""
|
|
521
|
+
"""Retrieves xcom value and kills xcom sidecar container."""
|
|
519
522
|
result = self.pod_manager.extract_xcom(pod)
|
|
520
523
|
if isinstance(result, str) and result.rstrip() == "__airflow_xcom_result_empty__":
|
|
521
524
|
self.log.info("xcom result file is empty.")
|
|
@@ -525,7 +528,7 @@ class KubernetesPodOperator(BaseOperator):
|
|
|
525
528
|
return json.loads(result)
|
|
526
529
|
|
|
527
530
|
def execute(self, context: Context):
|
|
528
|
-
"""Based on the deferrable parameter runs the pod asynchronously or synchronously"""
|
|
531
|
+
"""Based on the deferrable parameter runs the pod asynchronously or synchronously."""
|
|
529
532
|
if self.deferrable:
|
|
530
533
|
self.execute_async(context)
|
|
531
534
|
else:
|
|
@@ -676,7 +679,6 @@ class KubernetesPodOperator(BaseOperator):
|
|
|
676
679
|
self.process_pod_deletion(remote_pod, reraise=False)
|
|
677
680
|
|
|
678
681
|
error_message = get_container_termination_message(remote_pod, self.base_container_name)
|
|
679
|
-
error_message = "\n" + error_message if error_message else ""
|
|
680
682
|
if self.skip_on_exit_code is not None:
|
|
681
683
|
container_statuses = (
|
|
682
684
|
remote_pod.status.container_statuses if remote_pod and remote_pod.status else None
|
|
@@ -697,14 +699,22 @@ class KubernetesPodOperator(BaseOperator):
|
|
|
697
699
|
f"{self.skip_on_exit_code}. Skipping."
|
|
698
700
|
)
|
|
699
701
|
raise AirflowException(
|
|
700
|
-
|
|
701
|
-
|
|
702
|
+
"\n".join(
|
|
703
|
+
filter(
|
|
704
|
+
None,
|
|
705
|
+
[
|
|
706
|
+
f"Pod {pod and pod.metadata.name} returned a failure.",
|
|
707
|
+
error_message if isinstance(error_message, str) else None,
|
|
708
|
+
f"remote_pod: {remote_pod}" if self.log_pod_spec_on_failure else None,
|
|
709
|
+
],
|
|
710
|
+
)
|
|
711
|
+
)
|
|
702
712
|
)
|
|
703
713
|
else:
|
|
704
714
|
self.process_pod_deletion(remote_pod, reraise=False)
|
|
705
715
|
|
|
706
716
|
def _read_pod_events(self, pod, *, reraise=True):
|
|
707
|
-
"""Will fetch and emit events from pod"""
|
|
717
|
+
"""Will fetch and emit events from pod."""
|
|
708
718
|
with _optionally_suppress(reraise=reraise):
|
|
709
719
|
for event in self.pod_manager.read_pod_events(pod).items:
|
|
710
720
|
self.log.error("Pod Event: %s - %s", event.reason, event.message)
|
|
@@ -735,7 +745,7 @@ class KubernetesPodOperator(BaseOperator):
|
|
|
735
745
|
return None
|
|
736
746
|
|
|
737
747
|
def patch_already_checked(self, pod: k8s.V1Pod, *, reraise=True):
|
|
738
|
-
"""Add an "already checked" annotation to ensure we don't reattach on retries"""
|
|
748
|
+
"""Add an "already checked" annotation to ensure we don't reattach on retries."""
|
|
739
749
|
with _optionally_suppress(reraise=reraise):
|
|
740
750
|
self.client.patch_namespaced_pod(
|
|
741
751
|
name=pod.metadata.name,
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
|
2
|
+
# or more contributor license agreements. See the NOTICE file
|
|
3
|
+
# distributed with this work for additional information
|
|
4
|
+
# regarding copyright ownership. The ASF licenses this file
|
|
5
|
+
# to you under the Apache License, Version 2.0 (the
|
|
6
|
+
# "License"); you may not use this file except in compliance
|
|
7
|
+
# with the License. You may obtain a copy of the License at
|
|
8
|
+
#
|
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
#
|
|
11
|
+
# Unless required by applicable law or agreed to in writing,
|
|
12
|
+
# software distributed under the License is distributed on an
|
|
13
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
14
|
+
# KIND, either express or implied. See the License for the
|
|
15
|
+
# specific language governing permissions and limitations
|
|
16
|
+
# under the License.
|
|
17
|
+
"""Manage a Kubernetes Resource."""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
from functools import cached_property
|
|
22
|
+
|
|
23
|
+
import yaml
|
|
24
|
+
from kubernetes.client import ApiClient
|
|
25
|
+
from kubernetes.utils import create_from_yaml
|
|
26
|
+
|
|
27
|
+
from airflow.models import BaseOperator
|
|
28
|
+
from airflow.providers.cncf.kubernetes.hooks.kubernetes import KubernetesHook
|
|
29
|
+
from airflow.providers.cncf.kubernetes.utils.delete_from import delete_from_yaml
|
|
30
|
+
|
|
31
|
+
__all__ = ["KubernetesCreateResourceOperator", "KubernetesDeleteResourceOperator"]
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class KubernetesResourceBaseOperator(BaseOperator):
|
|
35
|
+
"""
|
|
36
|
+
Abstract base class for all Kubernetes Resource operators.
|
|
37
|
+
|
|
38
|
+
:param yaml_conf: string. Contains the kubernetes resources to Create or Delete
|
|
39
|
+
:param namespace: string. Contains the namespace to create all resources inside.
|
|
40
|
+
The namespace must preexist otherwise the resource creation will fail.
|
|
41
|
+
If the API object in the yaml file already contains a namespace definition then
|
|
42
|
+
this parameter has no effect.
|
|
43
|
+
:param kubernetes_conn_id: The :ref:`kubernetes connection id <howto/connection:kubernetes>`
|
|
44
|
+
for the Kubernetes cluster.
|
|
45
|
+
:param in_cluster: run kubernetes client with in_cluster configuration.
|
|
46
|
+
:param cluster_context: context that points to kubernetes cluster.
|
|
47
|
+
Ignored when in_cluster is True. If None, current-context is used.
|
|
48
|
+
:param config_file: The path to the Kubernetes config file. (templated)
|
|
49
|
+
If not specified, default value is ``~/.kube/config``
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
template_fields = ("yaml_conf",)
|
|
53
|
+
template_fields_renderers = {"yaml_conf": "yaml"}
|
|
54
|
+
|
|
55
|
+
def __init__(
|
|
56
|
+
self,
|
|
57
|
+
*,
|
|
58
|
+
yaml_conf: str,
|
|
59
|
+
namespace: str | None = None,
|
|
60
|
+
kubernetes_conn_id: str | None = KubernetesHook.default_conn_name,
|
|
61
|
+
**kwargs,
|
|
62
|
+
) -> None:
|
|
63
|
+
super().__init__(**kwargs)
|
|
64
|
+
self._namespace = namespace
|
|
65
|
+
self.kubernetes_conn_id = kubernetes_conn_id
|
|
66
|
+
self.yaml_conf = yaml_conf
|
|
67
|
+
|
|
68
|
+
@cached_property
|
|
69
|
+
def client(self) -> ApiClient:
|
|
70
|
+
return self.hook.api_client
|
|
71
|
+
|
|
72
|
+
@cached_property
|
|
73
|
+
def hook(self) -> KubernetesHook:
|
|
74
|
+
hook = KubernetesHook(conn_id=self.kubernetes_conn_id)
|
|
75
|
+
return hook
|
|
76
|
+
|
|
77
|
+
def get_namespace(self) -> str:
|
|
78
|
+
if self._namespace:
|
|
79
|
+
return self._namespace
|
|
80
|
+
else:
|
|
81
|
+
return self.hook.get_namespace() or "default"
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
class KubernetesCreateResourceOperator(KubernetesResourceBaseOperator):
|
|
85
|
+
"""Create a resource in a kubernetes."""
|
|
86
|
+
|
|
87
|
+
def execute(self, context) -> None:
|
|
88
|
+
create_from_yaml(
|
|
89
|
+
k8s_client=self.client,
|
|
90
|
+
yaml_objects=yaml.safe_load_all(self.yaml_conf),
|
|
91
|
+
namespace=self.get_namespace(),
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
class KubernetesDeleteResourceOperator(KubernetesResourceBaseOperator):
|
|
96
|
+
"""Delete a resource in a kubernetes."""
|
|
97
|
+
|
|
98
|
+
def execute(self, context) -> None:
|
|
99
|
+
delete_from_yaml(
|
|
100
|
+
k8s_client=self.client,
|
|
101
|
+
yaml_objects=yaml.safe_load_all(self.yaml_conf),
|
|
102
|
+
namespace=self.get_namespace(),
|
|
103
|
+
)
|
|
@@ -30,7 +30,7 @@ if TYPE_CHECKING:
|
|
|
30
30
|
|
|
31
31
|
class SparkKubernetesOperator(BaseOperator):
|
|
32
32
|
"""
|
|
33
|
-
Creates sparkApplication object in kubernetes cluster
|
|
33
|
+
Creates sparkApplication object in kubernetes cluster.
|
|
34
34
|
|
|
35
35
|
.. seealso::
|
|
36
36
|
For more detail about Spark Application Object have a look at the reference:
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
# KIND, either express or implied. See the License for the
|
|
16
16
|
# specific language governing permissions and limitations
|
|
17
17
|
# under the License.
|
|
18
|
-
"""Utilities for using the kubernetes decorator"""
|
|
18
|
+
"""Utilities for using the kubernetes decorator."""
|
|
19
19
|
from __future__ import annotations
|
|
20
20
|
|
|
21
21
|
import os
|
|
@@ -39,7 +39,7 @@ def _balance_parens(after_decorator):
|
|
|
39
39
|
|
|
40
40
|
def remove_task_decorator(python_source: str, task_decorator_name: str) -> str:
|
|
41
41
|
"""
|
|
42
|
-
Removes @task.kubernetes or similar as well as @setup and @teardown
|
|
42
|
+
Removes @task.kubernetes or similar as well as @setup and @teardown.
|
|
43
43
|
|
|
44
44
|
:param python_source: python source code
|
|
45
45
|
:param task_decorator_name: the task decorator name
|
|
@@ -31,7 +31,7 @@ if TYPE_CHECKING:
|
|
|
31
31
|
|
|
32
32
|
class SparkKubernetesSensor(BaseSensorOperator):
|
|
33
33
|
"""
|
|
34
|
-
Checks sparkApplication object in kubernetes cluster
|
|
34
|
+
Checks sparkApplication object in kubernetes cluster.
|
|
35
35
|
|
|
36
36
|
.. seealso::
|
|
37
37
|
For more detail about Spark Application Object have a look at the reference:
|
|
@@ -116,7 +116,7 @@ class KubernetesPodTrigger(BaseTrigger):
|
|
|
116
116
|
)
|
|
117
117
|
|
|
118
118
|
async def run(self) -> AsyncIterator[TriggerEvent]: # type: ignore[override]
|
|
119
|
-
"""Gets current pod status and yields a TriggerEvent"""
|
|
119
|
+
"""Gets current pod status and yields a TriggerEvent."""
|
|
120
120
|
hook = self._get_async_hook()
|
|
121
121
|
self.log.info("Checking pod %r in namespace %r.", self.pod_name, self.pod_namespace)
|
|
122
122
|
while True:
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
|
2
|
+
# or more contributor license agreements. See the NOTICE file
|
|
3
|
+
# distributed with this work for additional information
|
|
4
|
+
# regarding copyright ownership. The ASF licenses this file
|
|
5
|
+
# to you under the Apache License, Version 2.0 (the
|
|
6
|
+
# "License"); you may not use this file except in compliance
|
|
7
|
+
# with the License. You may obtain a copy of the License at
|
|
8
|
+
#
|
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
#
|
|
11
|
+
# Unless required by applicable law or agreed to in writing,
|
|
12
|
+
# software distributed under the License is distributed on an
|
|
13
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
14
|
+
# KIND, either express or implied. See the License for the
|
|
15
|
+
# specific language governing permissions and limitations
|
|
16
|
+
# under the License.
|
|
17
|
+
|
|
18
|
+
# from https://github.com/tomplus/kubernetes_asyncio/pull/239/files
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import re
|
|
23
|
+
|
|
24
|
+
from kubernetes import client
|
|
25
|
+
from kubernetes.client import ApiClient
|
|
26
|
+
|
|
27
|
+
DEFAULT_DELETION_BODY = client.V1DeleteOptions(
|
|
28
|
+
propagation_policy="Background",
|
|
29
|
+
grace_period_seconds=5,
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def delete_from_dict(k8s_client, data, body, namespace, verbose=False, **kwargs):
|
|
34
|
+
api_exceptions = []
|
|
35
|
+
|
|
36
|
+
if "List" in data["kind"]:
|
|
37
|
+
kind = data["kind"].replace("List", "")
|
|
38
|
+
for yml_doc in data["items"]:
|
|
39
|
+
if kind != "":
|
|
40
|
+
yml_doc["apiVersion"] = data["apiVersion"]
|
|
41
|
+
yml_doc["kind"] = kind
|
|
42
|
+
try:
|
|
43
|
+
_delete_from_yaml_single_item(
|
|
44
|
+
k8s_client=k8s_client,
|
|
45
|
+
yml_document=yml_doc,
|
|
46
|
+
verbose=verbose,
|
|
47
|
+
namespace=namespace,
|
|
48
|
+
body=body,
|
|
49
|
+
**kwargs,
|
|
50
|
+
)
|
|
51
|
+
except client.rest.ApiException as api_exception:
|
|
52
|
+
api_exceptions.append(api_exception)
|
|
53
|
+
else:
|
|
54
|
+
|
|
55
|
+
try:
|
|
56
|
+
_delete_from_yaml_single_item(
|
|
57
|
+
k8s_client=k8s_client,
|
|
58
|
+
yml_document=data,
|
|
59
|
+
verbose=verbose,
|
|
60
|
+
namespace=namespace,
|
|
61
|
+
body=body,
|
|
62
|
+
**kwargs,
|
|
63
|
+
)
|
|
64
|
+
except client.rest.ApiException as api_exception:
|
|
65
|
+
api_exceptions.append(api_exception)
|
|
66
|
+
|
|
67
|
+
if api_exceptions:
|
|
68
|
+
raise FailToDeleteError(api_exceptions)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def delete_from_yaml(
|
|
72
|
+
*,
|
|
73
|
+
k8s_client: ApiClient,
|
|
74
|
+
yaml_objects=None,
|
|
75
|
+
verbose: bool = False,
|
|
76
|
+
namespace: str = "default",
|
|
77
|
+
body: dict | None = None,
|
|
78
|
+
**kwargs,
|
|
79
|
+
):
|
|
80
|
+
for yml_document in yaml_objects:
|
|
81
|
+
if yml_document is None:
|
|
82
|
+
continue
|
|
83
|
+
else:
|
|
84
|
+
delete_from_dict(
|
|
85
|
+
k8s_client=k8s_client,
|
|
86
|
+
data=yml_document,
|
|
87
|
+
body=body,
|
|
88
|
+
namespace=namespace,
|
|
89
|
+
verbose=verbose,
|
|
90
|
+
**kwargs,
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _delete_from_yaml_single_item(
|
|
95
|
+
*,
|
|
96
|
+
k8s_client: ApiClient,
|
|
97
|
+
yml_document: dict,
|
|
98
|
+
verbose: bool = False,
|
|
99
|
+
namespace: str = "default",
|
|
100
|
+
body: dict | None = None,
|
|
101
|
+
**kwargs,
|
|
102
|
+
):
|
|
103
|
+
if body is None:
|
|
104
|
+
body = DEFAULT_DELETION_BODY
|
|
105
|
+
|
|
106
|
+
# get group and version from apiVersion
|
|
107
|
+
group, _, version = yml_document["apiVersion"].partition("/")
|
|
108
|
+
if version == "":
|
|
109
|
+
version = group
|
|
110
|
+
group = "core"
|
|
111
|
+
# Take care for the case e.g. api_type is "apiextensions.k8s.io"
|
|
112
|
+
# Only replace the last instance
|
|
113
|
+
group = "".join(group.rsplit(".k8s.io", 1))
|
|
114
|
+
# convert group name from DNS subdomain format to
|
|
115
|
+
# python class name convention
|
|
116
|
+
group = "".join(word.capitalize() for word in group.split("."))
|
|
117
|
+
fcn_to_call = f"{group}{version.capitalize()}Api"
|
|
118
|
+
k8s_api = getattr(client, fcn_to_call)(k8s_client)
|
|
119
|
+
# Replace CamelCased action_type into snake_case
|
|
120
|
+
kind = yml_document["kind"]
|
|
121
|
+
kind = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", kind)
|
|
122
|
+
kind = re.sub("([a-z0-9])([A-Z])", r"\1_\2", kind).lower()
|
|
123
|
+
|
|
124
|
+
# Decide which namespace we are going to use for deleting the object
|
|
125
|
+
# IMPORTANT: the docs namespace takes precedence over the namespace in args
|
|
126
|
+
# create_from_yaml_single_item have same behaviour
|
|
127
|
+
if "namespace" in yml_document["metadata"]:
|
|
128
|
+
namespace = yml_document["metadata"]["namespace"]
|
|
129
|
+
name = yml_document["metadata"]["name"]
|
|
130
|
+
|
|
131
|
+
# Expect the user to delete namespaced objects more often
|
|
132
|
+
resp: client.V1Status
|
|
133
|
+
if hasattr(k8s_api, f"delete_namespaced_{kind}"):
|
|
134
|
+
resp = getattr(k8s_api, f"delete_namespaced_{kind}")(
|
|
135
|
+
name=name, namespace=namespace, body=body, **kwargs
|
|
136
|
+
)
|
|
137
|
+
else:
|
|
138
|
+
resp = getattr(k8s_api, f"delete_{kind}")(name=name, body=body, **kwargs)
|
|
139
|
+
if verbose:
|
|
140
|
+
print(f"{kind} deleted. status='{str(resp.status)}'")
|
|
141
|
+
return resp
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
class FailToDeleteError(Exception):
|
|
145
|
+
"""
|
|
146
|
+
An exception class for handling error if an error occurred when
|
|
147
|
+
handling a yaml file during deletion of the resource.
|
|
148
|
+
"""
|
|
149
|
+
|
|
150
|
+
def __init__(self, api_exceptions: list):
|
|
151
|
+
self.api_exceptions = api_exceptions
|
|
152
|
+
|
|
153
|
+
def __str__(self):
|
|
154
|
+
msg = ""
|
|
155
|
+
for api_exception in self.api_exceptions:
|
|
156
|
+
msg += f"Error from server ({api_exception.reason}):{api_exception.body}\n"
|
|
157
|
+
return msg
|
|
@@ -14,10 +14,11 @@
|
|
|
14
14
|
# KIND, either express or implied. See the License for the
|
|
15
15
|
# specific language governing permissions and limitations
|
|
16
16
|
# under the License.
|
|
17
|
-
"""Launches PODs"""
|
|
17
|
+
"""Launches PODs."""
|
|
18
18
|
from __future__ import annotations
|
|
19
19
|
|
|
20
20
|
import json
|
|
21
|
+
import logging
|
|
21
22
|
import math
|
|
22
23
|
import time
|
|
23
24
|
import warnings
|
|
@@ -35,6 +36,7 @@ from kubernetes.client.rest import ApiException
|
|
|
35
36
|
from kubernetes.stream import stream as kubernetes_stream
|
|
36
37
|
from pendulum import DateTime
|
|
37
38
|
from pendulum.parsing.exceptions import ParserError
|
|
39
|
+
from tenacity import before_log
|
|
38
40
|
from urllib3.exceptions import HTTPError as BaseHTTPError
|
|
39
41
|
from urllib3.response import HTTPResponse
|
|
40
42
|
|
|
@@ -53,7 +55,7 @@ class PodLaunchFailedException(AirflowException):
|
|
|
53
55
|
|
|
54
56
|
|
|
55
57
|
def should_retry_start_pod(exception: BaseException) -> bool:
|
|
56
|
-
"""Check if an Exception indicates a transient error and warrants retrying"""
|
|
58
|
+
"""Check if an Exception indicates a transient error and warrants retrying."""
|
|
57
59
|
if isinstance(exception, ApiException):
|
|
58
60
|
return exception.status == 409
|
|
59
61
|
return False
|
|
@@ -75,7 +77,7 @@ class PodPhase:
|
|
|
75
77
|
|
|
76
78
|
class PodOperatorHookProtocol(Protocol):
|
|
77
79
|
"""
|
|
78
|
-
Protocol to define methods relied upon by KubernetesPodOperator
|
|
80
|
+
Protocol to define methods relied upon by KubernetesPodOperator.
|
|
79
81
|
|
|
80
82
|
Subclasses of KubernetesPodOperator, such as GKEStartPodOperator, may use
|
|
81
83
|
hooks that don't extend KubernetesHook. We use this protocol to document the
|
|
@@ -88,17 +90,17 @@ class PodOperatorHookProtocol(Protocol):
|
|
|
88
90
|
|
|
89
91
|
@property
|
|
90
92
|
def is_in_cluster(self) -> bool:
|
|
91
|
-
"""Expose whether the hook is configured with ``load_incluster_config`` or not"""
|
|
93
|
+
"""Expose whether the hook is configured with ``load_incluster_config`` or not."""
|
|
92
94
|
|
|
93
95
|
def get_pod(self, name: str, namespace: str) -> V1Pod:
|
|
94
96
|
"""Read pod object from kubernetes API."""
|
|
95
97
|
|
|
96
98
|
def get_namespace(self) -> str | None:
|
|
97
|
-
"""Returns the namespace that defined in the connection"""
|
|
99
|
+
"""Returns the namespace that defined in the connection."""
|
|
98
100
|
|
|
99
101
|
|
|
100
102
|
def get_container_status(pod: V1Pod, container_name: str) -> V1ContainerStatus | None:
|
|
101
|
-
"""Retrieves container status"""
|
|
103
|
+
"""Retrieves container status."""
|
|
102
104
|
container_statuses = pod.status.container_statuses if pod and pod.status else None
|
|
103
105
|
if container_statuses:
|
|
104
106
|
# In general the variable container_statuses can store multiple items matching different containers.
|
|
@@ -145,7 +147,8 @@ class PodLogsConsumer:
|
|
|
145
147
|
"""
|
|
146
148
|
PodLogsConsumer is responsible for pulling pod logs from a stream with checking a container status before
|
|
147
149
|
reading data.
|
|
148
|
-
|
|
150
|
+
|
|
151
|
+
This class is a workaround for the issue https://github.com/apache/airflow/issues/23497.
|
|
149
152
|
|
|
150
153
|
:param response: HTTP response with logs
|
|
151
154
|
:param pod: Pod instance from Kubernetes client
|
|
@@ -229,7 +232,7 @@ class PodLogsConsumer:
|
|
|
229
232
|
|
|
230
233
|
@dataclass
|
|
231
234
|
class PodLoggingStatus:
|
|
232
|
-
"""Used for returning the status of the pod and last log time when exiting from `fetch_container_logs
|
|
235
|
+
"""Used for returning the status of the pod and last log time when exiting from `fetch_container_logs`."""
|
|
233
236
|
|
|
234
237
|
running: bool
|
|
235
238
|
last_log_time: DateTime | None
|
|
@@ -238,7 +241,7 @@ class PodLoggingStatus:
|
|
|
238
241
|
class PodManager(LoggingMixin):
|
|
239
242
|
"""
|
|
240
243
|
Helper class for creating, monitoring, and otherwise interacting with Kubernetes pods
|
|
241
|
-
for use with the KubernetesPodOperator
|
|
244
|
+
for use with the KubernetesPodOperator.
|
|
242
245
|
"""
|
|
243
246
|
|
|
244
247
|
def __init__(
|
|
@@ -255,7 +258,7 @@ class PodManager(LoggingMixin):
|
|
|
255
258
|
self._watch = watch.Watch()
|
|
256
259
|
|
|
257
260
|
def run_pod_async(self, pod: V1Pod, **kwargs) -> V1Pod:
|
|
258
|
-
"""Runs POD asynchronously"""
|
|
261
|
+
"""Runs POD asynchronously."""
|
|
259
262
|
sanitized_pod = self._client.api_client.sanitize_for_serialization(pod)
|
|
260
263
|
json_pod = json.dumps(sanitized_pod, indent=2)
|
|
261
264
|
|
|
@@ -273,7 +276,7 @@ class PodManager(LoggingMixin):
|
|
|
273
276
|
return resp
|
|
274
277
|
|
|
275
278
|
def delete_pod(self, pod: V1Pod) -> None:
|
|
276
|
-
"""Deletes POD"""
|
|
279
|
+
"""Deletes POD."""
|
|
277
280
|
try:
|
|
278
281
|
self._client.delete_namespaced_pod(
|
|
279
282
|
pod.metadata.name, pod.metadata.namespace, body=client.V1DeleteOptions()
|
|
@@ -295,7 +298,7 @@ class PodManager(LoggingMixin):
|
|
|
295
298
|
|
|
296
299
|
def await_pod_start(self, pod: V1Pod, startup_timeout: int = 120) -> None:
|
|
297
300
|
"""
|
|
298
|
-
Waits for the pod to reach phase other than ``Pending
|
|
301
|
+
Waits for the pod to reach phase other than ``Pending``.
|
|
299
302
|
|
|
300
303
|
:param pod:
|
|
301
304
|
:param startup_timeout: Timeout (in seconds) for startup of the pod
|
|
@@ -336,9 +339,20 @@ class PodManager(LoggingMixin):
|
|
|
336
339
|
) -> PodLoggingStatus:
|
|
337
340
|
"""
|
|
338
341
|
Follows the logs of container and streams to airflow logging.
|
|
342
|
+
|
|
339
343
|
Returns when container exits.
|
|
344
|
+
|
|
345
|
+
Between when the pod starts and logs being available, there might be a delay due to CSR not approved
|
|
346
|
+
and signed yet. In such situation, ApiException is thrown. This is why we are retrying on this
|
|
347
|
+
specific exception.
|
|
340
348
|
"""
|
|
341
349
|
|
|
350
|
+
@tenacity.retry(
|
|
351
|
+
retry=tenacity.retry_if_exception_type(ApiException),
|
|
352
|
+
stop=tenacity.stop_after_attempt(10),
|
|
353
|
+
wait=tenacity.wait_fixed(1),
|
|
354
|
+
before=before_log(self.log, logging.INFO),
|
|
355
|
+
)
|
|
342
356
|
def consume_logs(
|
|
343
357
|
*, since_time: DateTime | None = None, follow: bool = True, termination_timeout: int = 120
|
|
344
358
|
) -> DateTime | None:
|
|
@@ -400,7 +414,7 @@ class PodManager(LoggingMixin):
|
|
|
400
414
|
|
|
401
415
|
def await_container_completion(self, pod: V1Pod, container_name: str) -> None:
|
|
402
416
|
"""
|
|
403
|
-
Waits for the given container in the given pod to be completed
|
|
417
|
+
Waits for the given container in the given pod to be completed.
|
|
404
418
|
|
|
405
419
|
:param pod: pod spec that will be monitored
|
|
406
420
|
:param container_name: name of the container within the pod to monitor
|
|
@@ -410,7 +424,7 @@ class PodManager(LoggingMixin):
|
|
|
410
424
|
|
|
411
425
|
def await_pod_completion(self, pod: V1Pod) -> V1Pod:
|
|
412
426
|
"""
|
|
413
|
-
Monitors a pod and returns the final state
|
|
427
|
+
Monitors a pod and returns the final state.
|
|
414
428
|
|
|
415
429
|
:param pod: pod spec that will be monitored
|
|
416
430
|
:return: tuple[State, str | None]
|
|
@@ -425,7 +439,7 @@ class PodManager(LoggingMixin):
|
|
|
425
439
|
|
|
426
440
|
def parse_log_line(self, line: str) -> tuple[DateTime | None, str]:
|
|
427
441
|
"""
|
|
428
|
-
Parse K8s log line and returns the final state
|
|
442
|
+
Parse K8s log line and returns the final state.
|
|
429
443
|
|
|
430
444
|
:param line: k8s log line
|
|
431
445
|
:return: timestamp and log message
|
|
@@ -448,12 +462,12 @@ class PodManager(LoggingMixin):
|
|
|
448
462
|
return last_log_time, message
|
|
449
463
|
|
|
450
464
|
def container_is_running(self, pod: V1Pod, container_name: str) -> bool:
|
|
451
|
-
"""Reads pod and checks if container is running"""
|
|
465
|
+
"""Reads pod and checks if container is running."""
|
|
452
466
|
remote_pod = self.read_pod(pod)
|
|
453
467
|
return container_is_running(pod=remote_pod, container_name=container_name)
|
|
454
468
|
|
|
455
469
|
def container_is_terminated(self, pod: V1Pod, container_name: str) -> bool:
|
|
456
|
-
"""Reads pod and checks if container is terminated"""
|
|
470
|
+
"""Reads pod and checks if container is terminated."""
|
|
457
471
|
remote_pod = self.read_pod(pod)
|
|
458
472
|
return container_is_terminated(pod=remote_pod, container_name=container_name)
|
|
459
473
|
|
|
@@ -468,7 +482,7 @@ class PodManager(LoggingMixin):
|
|
|
468
482
|
follow=True,
|
|
469
483
|
post_termination_timeout: int = 120,
|
|
470
484
|
) -> PodLogsConsumer:
|
|
471
|
-
"""Reads log from the POD"""
|
|
485
|
+
"""Reads log from the POD."""
|
|
472
486
|
additional_kwargs = {}
|
|
473
487
|
if since_seconds:
|
|
474
488
|
additional_kwargs["since_seconds"] = since_seconds
|
|
@@ -500,7 +514,7 @@ class PodManager(LoggingMixin):
|
|
|
500
514
|
|
|
501
515
|
@tenacity.retry(stop=tenacity.stop_after_attempt(3), wait=tenacity.wait_exponential(), reraise=True)
|
|
502
516
|
def read_pod_events(self, pod: V1Pod) -> CoreV1EventList:
|
|
503
|
-
"""Reads events from the POD"""
|
|
517
|
+
"""Reads events from the POD."""
|
|
504
518
|
try:
|
|
505
519
|
return self._client.list_namespaced_event(
|
|
506
520
|
namespace=pod.metadata.namespace, field_selector=f"involvedObject.name={pod.metadata.name}"
|
|
@@ -510,7 +524,7 @@ class PodManager(LoggingMixin):
|
|
|
510
524
|
|
|
511
525
|
@tenacity.retry(stop=tenacity.stop_after_attempt(3), wait=tenacity.wait_exponential(), reraise=True)
|
|
512
526
|
def read_pod(self, pod: V1Pod) -> V1Pod:
|
|
513
|
-
"""Read POD information"""
|
|
527
|
+
"""Read POD information."""
|
|
514
528
|
try:
|
|
515
529
|
return self._client.read_namespaced_pod(pod.metadata.name, pod.metadata.namespace)
|
|
516
530
|
except BaseHTTPError as e:
|
|
@@ -529,7 +543,7 @@ class PodManager(LoggingMixin):
|
|
|
529
543
|
time.sleep(1)
|
|
530
544
|
|
|
531
545
|
def extract_xcom(self, pod: V1Pod) -> str:
|
|
532
|
-
"""Retrieves XCom value and kills xcom sidecar container"""
|
|
546
|
+
"""Retrieves XCom value and kills xcom sidecar container."""
|
|
533
547
|
with closing(
|
|
534
548
|
kubernetes_stream(
|
|
535
549
|
self._client.connect_get_namespaced_pod_exec,
|
|
@@ -27,7 +27,7 @@ from kubernetes.client import models as k8s
|
|
|
27
27
|
|
|
28
28
|
|
|
29
29
|
class PodDefaults:
|
|
30
|
-
"""Static defaults for Pods"""
|
|
30
|
+
"""Static defaults for Pods."""
|
|
31
31
|
|
|
32
32
|
XCOM_MOUNT_PATH = "/airflow/xcom"
|
|
33
33
|
SIDECAR_CONTAINER_NAME = "airflow-xcom-sidecar"
|
|
@@ -54,7 +54,7 @@ def add_xcom_sidecar(
|
|
|
54
54
|
sidecar_container_image: str | None = None,
|
|
55
55
|
sidecar_container_resources: k8s.V1ResourceRequirements | dict | None = None,
|
|
56
56
|
) -> k8s.V1Pod:
|
|
57
|
-
"""Adds sidecar"""
|
|
57
|
+
"""Adds sidecar."""
|
|
58
58
|
pod_cp = copy.deepcopy(pod)
|
|
59
59
|
pod_cp.spec.volumes = pod.spec.volumes or []
|
|
60
60
|
pod_cp.spec.volumes.insert(0, PodDefaults.VOLUME)
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: apache-airflow-providers-cncf-kubernetes
|
|
3
|
-
Version: 7.
|
|
3
|
+
Version: 7.1.0
|
|
4
4
|
Summary: Provider for Apache Airflow. Implements apache-airflow-providers-cncf-kubernetes package
|
|
5
5
|
Home-page: https://airflow.apache.org/
|
|
6
6
|
Download-URL: https://archive.apache.org/dist/airflow/providers
|
|
7
7
|
Author: Apache Software Foundation
|
|
8
8
|
Author-email: dev@airflow.apache.org
|
|
9
9
|
License: Apache License 2.0
|
|
10
|
-
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/7.
|
|
10
|
+
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/7.1.0/
|
|
11
11
|
Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
|
|
12
12
|
Project-URL: Source Code, https://github.com/apache/airflow
|
|
13
13
|
Project-URL: Slack Chat, https://s.apache.org/airflow-slack
|
|
@@ -21,12 +21,12 @@ Classifier: Intended Audience :: System Administrators
|
|
|
21
21
|
Classifier: Framework :: Apache Airflow
|
|
22
22
|
Classifier: Framework :: Apache Airflow :: Provider
|
|
23
23
|
Classifier: License :: OSI Approved :: Apache Software License
|
|
24
|
-
Classifier: Programming Language :: Python :: 3.7
|
|
25
24
|
Classifier: Programming Language :: Python :: 3.8
|
|
26
25
|
Classifier: Programming Language :: Python :: 3.9
|
|
27
26
|
Classifier: Programming Language :: Python :: 3.10
|
|
27
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
28
28
|
Classifier: Topic :: System :: Monitoring
|
|
29
|
-
Requires-Python: ~=3.
|
|
29
|
+
Requires-Python: ~=3.8
|
|
30
30
|
Description-Content-Type: text/x-rst
|
|
31
31
|
License-File: LICENSE
|
|
32
32
|
License-File: NOTICE
|
|
@@ -57,7 +57,7 @@ Requires-Dist: kubernetes-asyncio (<25,>=18.20.1)
|
|
|
57
57
|
|
|
58
58
|
Package ``apache-airflow-providers-cncf-kubernetes``
|
|
59
59
|
|
|
60
|
-
Release: ``7.
|
|
60
|
+
Release: ``7.1.0``
|
|
61
61
|
|
|
62
62
|
|
|
63
63
|
`Kubernetes <https://kubernetes.io/>`__
|
|
@@ -70,7 +70,7 @@ This is a provider package for ``cncf.kubernetes`` provider. All classes for thi
|
|
|
70
70
|
are in ``airflow.providers.cncf.kubernetes`` python package.
|
|
71
71
|
|
|
72
72
|
You can find package information and changelog for the provider
|
|
73
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/7.
|
|
73
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/7.1.0/>`_.
|
|
74
74
|
|
|
75
75
|
|
|
76
76
|
Installation
|
|
@@ -80,7 +80,7 @@ You can install this package on top of an existing Airflow 2 installation (see `
|
|
|
80
80
|
for the minimum Airflow version supported) via
|
|
81
81
|
``pip install apache-airflow-providers-cncf-kubernetes``
|
|
82
82
|
|
|
83
|
-
The package supports the following python versions: 3.
|
|
83
|
+
The package supports the following python versions: 3.8,3.9,3.10,3.11
|
|
84
84
|
|
|
85
85
|
Requirements
|
|
86
86
|
------------
|
|
@@ -121,7 +121,37 @@ PIP package Version required
|
|
|
121
121
|
Changelog
|
|
122
122
|
---------
|
|
123
123
|
|
|
124
|
+
7.1.0
|
|
125
|
+
.....
|
|
126
|
+
|
|
127
|
+
.. note::
|
|
128
|
+
This release dropped support for Python 3.7
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
Features
|
|
132
|
+
~~~~~~~~
|
|
133
|
+
* ``KubernetesResourceOperator - KubernetesDeleteResourceOperator & KubernetesCreateResourceOperator (#29930)``
|
|
134
|
+
* ``add a return when the event is yielded in a loop to stop the execution (#31985)``
|
|
135
|
+
* ``Add possibility to disable logging the pod template in a case when task fails (#31595)``
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
Bug Fixes
|
|
139
|
+
~~~~~~~~~
|
|
140
|
+
|
|
141
|
+
* ``Remove return statement after yield from triggers class (#31703)``
|
|
142
|
+
* ``Fix Fargate logging for AWS system tests (#31622)``
|
|
124
143
|
|
|
144
|
+
Misc
|
|
145
|
+
~~~~
|
|
146
|
+
|
|
147
|
+
* ``Remove Python 3.7 support (#30963)``
|
|
148
|
+
|
|
149
|
+
.. Below changes are excluded from the changelog. Move them to
|
|
150
|
+
appropriate section above if needed. Do not delete the lines(!):
|
|
151
|
+
* ``Add D400 pydocstyle check (#31742)``
|
|
152
|
+
* ``Add discoverability for triggers in provider.yaml (#31576)``
|
|
153
|
+
* ``Add D400 pydocstyle check - Providers (#31427)``
|
|
154
|
+
* ``Add note about dropping Python 3.7 for providers (#32015)``
|
|
125
155
|
|
|
126
156
|
7.0.0
|
|
127
157
|
.....
|
|
@@ -1,29 +1,31 @@
|
|
|
1
|
-
airflow/providers/cncf/kubernetes/__init__.py,sha256=
|
|
2
|
-
airflow/providers/cncf/kubernetes/get_provider_info.py,sha256=
|
|
1
|
+
airflow/providers/cncf/kubernetes/__init__.py,sha256=yDhNEIzXXCroQ9yTMoRYvxJf_hD8E_kmQ85DId-n_F0,1540
|
|
2
|
+
airflow/providers/cncf/kubernetes/get_provider_info.py,sha256=ygvrMWmTxo7qSbF7gbU-2Je4i4_DIZ_DoRqYNHdUwgk,4462
|
|
3
3
|
airflow/providers/cncf/kubernetes/python_kubernetes_script.jinja2,sha256=gUGBhBTFWIXjnjxTAjawWIacDwk2EDxoGEzVQYnlUT8,1741
|
|
4
|
-
airflow/providers/cncf/kubernetes/python_kubernetes_script.py,sha256=
|
|
4
|
+
airflow/providers/cncf/kubernetes/python_kubernetes_script.py,sha256=wGiq25lEycY2Rp3VrtigJ9TrECfKdHmBNuChSJM-Kms,3345
|
|
5
5
|
airflow/providers/cncf/kubernetes/backcompat/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
|
6
|
-
airflow/providers/cncf/kubernetes/backcompat/backwards_compat_converters.py,sha256=
|
|
6
|
+
airflow/providers/cncf/kubernetes/backcompat/backwards_compat_converters.py,sha256=wlJGIb282YI-O5LVYBNXuiaQ1a6L5woZWZidyllpmFI,3914
|
|
7
7
|
airflow/providers/cncf/kubernetes/decorators/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
|
|
8
8
|
airflow/providers/cncf/kubernetes/decorators/kubernetes.py,sha256=1BtTiZS0W1w_X-GixB1mXftgz2utPFT2ci2GX83sEY4,6178
|
|
9
9
|
airflow/providers/cncf/kubernetes/hooks/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
|
10
|
-
airflow/providers/cncf/kubernetes/hooks/kubernetes.py,sha256=
|
|
10
|
+
airflow/providers/cncf/kubernetes/hooks/kubernetes.py,sha256=YmgGY36NNOBLne57x1iyyQb2T09AXMnCHGwJeN2UWcI,23076
|
|
11
11
|
airflow/providers/cncf/kubernetes/operators/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
|
|
12
12
|
airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py,sha256=yxsHr9tZKlREYtfq41g16miGvVk2fZ1Xo7dm6SxbLXw,1154
|
|
13
|
-
airflow/providers/cncf/kubernetes/operators/pod.py,sha256=
|
|
14
|
-
airflow/providers/cncf/kubernetes/operators/
|
|
13
|
+
airflow/providers/cncf/kubernetes/operators/pod.py,sha256=ereGCOO6rGhHD1fl7wOzTr7dn_iKimT_qAl0oiiNFOM,39267
|
|
14
|
+
airflow/providers/cncf/kubernetes/operators/resource.py,sha256=cWpygD5-d8O3ja2U3oOd3kkSzUs5ya6uGVEQA-8eq9g,3827
|
|
15
|
+
airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py,sha256=80JCB6GP42FNr-UAox-8rv44Jp74z80a_JBKNdpJaF4,4860
|
|
15
16
|
airflow/providers/cncf/kubernetes/sensors/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
|
16
|
-
airflow/providers/cncf/kubernetes/sensors/spark_kubernetes.py,sha256=
|
|
17
|
+
airflow/providers/cncf/kubernetes/sensors/spark_kubernetes.py,sha256=8GTjy9uB6R8ium_o4eItB4PqSUdgvCGAj9oPzQjIf2k,5226
|
|
17
18
|
airflow/providers/cncf/kubernetes/triggers/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
|
18
19
|
airflow/providers/cncf/kubernetes/triggers/kubernetes_pod.py,sha256=oEpmUYtEDVN7VV5mMXod5ZoNREoHXTuxj-saWliMOBM,1152
|
|
19
|
-
airflow/providers/cncf/kubernetes/triggers/pod.py,sha256=
|
|
20
|
+
airflow/providers/cncf/kubernetes/triggers/pod.py,sha256=EVIj9WrP1tHjseaxwWmvDiiPIYyZV0XGCuDehkdtJBI,10368
|
|
20
21
|
airflow/providers/cncf/kubernetes/utils/__init__.py,sha256=ClZN0VPjWySdVwS_ktH7rrgL9VLAcs3OSJSB9s3zaYw,863
|
|
21
|
-
airflow/providers/cncf/kubernetes/utils/
|
|
22
|
-
airflow/providers/cncf/kubernetes/utils/
|
|
23
|
-
|
|
24
|
-
apache_airflow_providers_cncf_kubernetes-7.
|
|
25
|
-
apache_airflow_providers_cncf_kubernetes-7.
|
|
26
|
-
apache_airflow_providers_cncf_kubernetes-7.
|
|
27
|
-
apache_airflow_providers_cncf_kubernetes-7.
|
|
28
|
-
apache_airflow_providers_cncf_kubernetes-7.
|
|
29
|
-
apache_airflow_providers_cncf_kubernetes-7.
|
|
22
|
+
airflow/providers/cncf/kubernetes/utils/delete_from.py,sha256=uCsWKiOfe9x9HOTojo_iU14AMN8hNC7k7tYiM77g61U,5209
|
|
23
|
+
airflow/providers/cncf/kubernetes/utils/pod_manager.py,sha256=HJUfuo5hlhQnOrtuPa3Rej-eUyiHPxfo_6FpSDnZErY,23192
|
|
24
|
+
airflow/providers/cncf/kubernetes/utils/xcom_sidecar.py,sha256=NbRy1RV9QnrNOjxpVzhx8MDiQcOJNWwnC2_y-3ATFOU,2644
|
|
25
|
+
apache_airflow_providers_cncf_kubernetes-7.1.0.dist-info/LICENSE,sha256=gXPVwptPlW1TJ4HSuG5OMPg-a3h43OGMkZRR1rpwfJA,10850
|
|
26
|
+
apache_airflow_providers_cncf_kubernetes-7.1.0.dist-info/METADATA,sha256=XCyg-9gZEJoivRWK7pEtq_xan5bBRk3bb8J11ZOabpA,35959
|
|
27
|
+
apache_airflow_providers_cncf_kubernetes-7.1.0.dist-info/NOTICE,sha256=m-6s2XynUxVSUIxO4rVablAZCvFq-wmLrqV91DotRBw,240
|
|
28
|
+
apache_airflow_providers_cncf_kubernetes-7.1.0.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
|
|
29
|
+
apache_airflow_providers_cncf_kubernetes-7.1.0.dist-info/entry_points.txt,sha256=GZl6SYJuUg-3koITGRd9PU1lBmqhecrKTeCQ6-wyHpM,112
|
|
30
|
+
apache_airflow_providers_cncf_kubernetes-7.1.0.dist-info/top_level.txt,sha256=OeMVH5md7fr2QQWpnZoOWWxWO-0WH1IP70lpTVwopPg,8
|
|
31
|
+
apache_airflow_providers_cncf_kubernetes-7.1.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|