apache-airflow-providers-cncf-kubernetes 3.1.0__py3-none-any.whl → 10.10.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/cncf/kubernetes/__init__.py +18 -23
- airflow/providers/cncf/kubernetes/backcompat/__init__.py +17 -0
- airflow/providers/cncf/kubernetes/backcompat/backwards_compat_converters.py +31 -49
- airflow/providers/cncf/kubernetes/callbacks.py +200 -0
- airflow/providers/cncf/kubernetes/cli/__init__.py +16 -0
- airflow/providers/cncf/kubernetes/cli/kubernetes_command.py +195 -0
- airflow/providers/cncf/kubernetes/decorators/kubernetes.py +163 -0
- airflow/providers/cncf/kubernetes/decorators/kubernetes_cmd.py +118 -0
- airflow/providers/cncf/kubernetes/exceptions.py +37 -0
- airflow/providers/cncf/kubernetes/executors/__init__.py +17 -0
- airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py +831 -0
- airflow/providers/cncf/kubernetes/executors/kubernetes_executor_types.py +91 -0
- airflow/providers/cncf/kubernetes/executors/kubernetes_executor_utils.py +736 -0
- airflow/providers/cncf/kubernetes/executors/local_kubernetes_executor.py +306 -0
- airflow/providers/cncf/kubernetes/get_provider_info.py +249 -50
- airflow/providers/cncf/kubernetes/hooks/kubernetes.py +846 -112
- airflow/providers/cncf/kubernetes/k8s_model.py +62 -0
- airflow/providers/cncf/kubernetes/kube_client.py +156 -0
- airflow/providers/cncf/kubernetes/kube_config.py +125 -0
- airflow/providers/cncf/kubernetes/kubernetes_executor_templates/__init__.py +16 -0
- airflow/providers/cncf/kubernetes/kubernetes_executor_templates/basic_template.yaml +79 -0
- airflow/providers/cncf/kubernetes/kubernetes_helper_functions.py +165 -0
- airflow/providers/cncf/kubernetes/operators/custom_object_launcher.py +368 -0
- airflow/providers/cncf/kubernetes/operators/job.py +646 -0
- airflow/providers/cncf/kubernetes/operators/kueue.py +132 -0
- airflow/providers/cncf/kubernetes/operators/pod.py +1417 -0
- airflow/providers/cncf/kubernetes/operators/resource.py +191 -0
- airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py +336 -35
- airflow/providers/cncf/kubernetes/pod_generator.py +592 -0
- airflow/providers/cncf/kubernetes/pod_template_file_examples/__init__.py +16 -0
- airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_image_template.yaml +68 -0
- airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_volume_template.yaml +74 -0
- airflow/providers/cncf/kubernetes/pod_template_file_examples/git_sync_template.yaml +95 -0
- airflow/providers/cncf/kubernetes/python_kubernetes_script.jinja2 +51 -0
- airflow/providers/cncf/kubernetes/python_kubernetes_script.py +92 -0
- airflow/providers/cncf/kubernetes/resource_convert/__init__.py +16 -0
- airflow/providers/cncf/kubernetes/resource_convert/configmap.py +52 -0
- airflow/providers/cncf/kubernetes/resource_convert/env_variable.py +39 -0
- airflow/providers/cncf/kubernetes/resource_convert/secret.py +40 -0
- airflow/providers/cncf/kubernetes/secret.py +128 -0
- airflow/providers/cncf/kubernetes/sensors/spark_kubernetes.py +30 -14
- airflow/providers/cncf/kubernetes/template_rendering.py +81 -0
- airflow/providers/cncf/kubernetes/triggers/__init__.py +16 -0
- airflow/providers/cncf/kubernetes/triggers/job.py +176 -0
- airflow/providers/cncf/kubernetes/triggers/pod.py +344 -0
- airflow/providers/cncf/kubernetes/utils/__init__.py +3 -0
- airflow/providers/cncf/kubernetes/utils/container.py +118 -0
- airflow/providers/cncf/kubernetes/utils/delete_from.py +154 -0
- airflow/providers/cncf/kubernetes/utils/k8s_resource_iterator.py +46 -0
- airflow/providers/cncf/kubernetes/utils/pod_manager.py +887 -152
- airflow/providers/cncf/kubernetes/utils/xcom_sidecar.py +25 -16
- airflow/providers/cncf/kubernetes/version_compat.py +38 -0
- apache_airflow_providers_cncf_kubernetes-10.10.0rc1.dist-info/METADATA +125 -0
- apache_airflow_providers_cncf_kubernetes-10.10.0rc1.dist-info/RECORD +62 -0
- {apache_airflow_providers_cncf_kubernetes-3.1.0.dist-info → apache_airflow_providers_cncf_kubernetes-10.10.0rc1.dist-info}/WHEEL +1 -2
- apache_airflow_providers_cncf_kubernetes-10.10.0rc1.dist-info/entry_points.txt +3 -0
- apache_airflow_providers_cncf_kubernetes-10.10.0rc1.dist-info/licenses/NOTICE +5 -0
- airflow/providers/cncf/kubernetes/backcompat/pod.py +0 -119
- airflow/providers/cncf/kubernetes/backcompat/pod_runtime_info_env.py +0 -56
- airflow/providers/cncf/kubernetes/backcompat/volume.py +0 -62
- airflow/providers/cncf/kubernetes/backcompat/volume_mount.py +0 -58
- airflow/providers/cncf/kubernetes/example_dags/example_kubernetes.py +0 -163
- airflow/providers/cncf/kubernetes/example_dags/example_spark_kubernetes.py +0 -66
- airflow/providers/cncf/kubernetes/example_dags/example_spark_kubernetes_spark_pi.yaml +0 -57
- airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py +0 -622
- apache_airflow_providers_cncf_kubernetes-3.1.0.dist-info/METADATA +0 -452
- apache_airflow_providers_cncf_kubernetes-3.1.0.dist-info/NOTICE +0 -6
- apache_airflow_providers_cncf_kubernetes-3.1.0.dist-info/RECORD +0 -29
- apache_airflow_providers_cncf_kubernetes-3.1.0.dist-info/entry_points.txt +0 -3
- apache_airflow_providers_cncf_kubernetes-3.1.0.dist-info/top_level.txt +0 -1
- /airflow/providers/cncf/kubernetes/{example_dags → decorators}/__init__.py +0 -0
- {apache_airflow_providers_cncf_kubernetes-3.1.0.dist-info → apache_airflow_providers_cncf_kubernetes-10.10.0rc1.dist-info/licenses}/LICENSE +0 -0
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
#
|
|
2
1
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
3
2
|
# or more contributor license agreements. See the NOTICE file
|
|
4
3
|
# distributed with this work for additional information
|
|
@@ -15,30 +14,26 @@
|
|
|
15
14
|
# KIND, either express or implied. See the License for the
|
|
16
15
|
# specific language governing permissions and limitations
|
|
17
16
|
# under the License.
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
17
|
+
#
|
|
18
|
+
# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
|
|
19
|
+
# OVERWRITTEN WHEN PREPARING DOCUMENTATION FOR THE PACKAGES.
|
|
20
|
+
#
|
|
21
|
+
# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
|
|
22
|
+
# `PROVIDER__INIT__PY_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
|
|
23
|
+
#
|
|
24
|
+
from __future__ import annotations
|
|
26
25
|
|
|
27
|
-
|
|
28
|
-
# that we can update the version used in this provider and have it work for older versions
|
|
29
|
-
import copyreg
|
|
30
|
-
import logging
|
|
26
|
+
import packaging.version
|
|
31
27
|
|
|
32
|
-
|
|
33
|
-
if logging.getLogger(logger.name) is not logger:
|
|
34
|
-
import pickle
|
|
28
|
+
from airflow import __version__ as airflow_version
|
|
35
29
|
|
|
36
|
-
|
|
37
|
-
return logging.getLogger, (logger.name,)
|
|
30
|
+
__all__ = ["__version__"]
|
|
38
31
|
|
|
39
|
-
|
|
40
|
-
return logging.getLogger, ()
|
|
32
|
+
__version__ = "10.10.0"
|
|
41
33
|
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
34
|
+
if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
|
|
35
|
+
"2.10.0"
|
|
36
|
+
):
|
|
37
|
+
raise RuntimeError(
|
|
38
|
+
f"The package `apache-airflow-providers-cncf-kubernetes:{__version__}` needs Apache Airflow 2.10.0+"
|
|
39
|
+
)
|
|
@@ -14,3 +14,20 @@
|
|
|
14
14
|
# KIND, either express or implied. See the License for the
|
|
15
15
|
# specific language governing permissions and limitations
|
|
16
16
|
# under the License.
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def get_logical_date_key() -> str:
|
|
22
|
+
"""
|
|
23
|
+
Get the key for execution/logical date for the appropriate Airflow version.
|
|
24
|
+
|
|
25
|
+
This is done by detecting the CLI argument name. There are various ways to
|
|
26
|
+
do this, but the CLI key name has a very small import footprint (especially
|
|
27
|
+
compared to importing ORM models).
|
|
28
|
+
"""
|
|
29
|
+
from airflow.cli import cli_config
|
|
30
|
+
|
|
31
|
+
if hasattr(cli_config, "ARG_LOGICAL_DATE"):
|
|
32
|
+
return "logical_date"
|
|
33
|
+
return "execution_date"
|
|
@@ -14,9 +14,9 @@
|
|
|
14
14
|
# KIND, either express or implied. See the License for the
|
|
15
15
|
# specific language governing permissions and limitations
|
|
16
16
|
# under the License.
|
|
17
|
-
"""Executes task in a Kubernetes POD"""
|
|
17
|
+
"""Executes task in a Kubernetes POD."""
|
|
18
18
|
|
|
19
|
-
from
|
|
19
|
+
from __future__ import annotations
|
|
20
20
|
|
|
21
21
|
from kubernetes.client import ApiClient, models as k8s
|
|
22
22
|
|
|
@@ -27,123 +27,105 @@ def _convert_kube_model_object(obj, new_class):
|
|
|
27
27
|
convert_op = getattr(obj, "to_k8s_client_obj", None)
|
|
28
28
|
if callable(convert_op):
|
|
29
29
|
return obj.to_k8s_client_obj()
|
|
30
|
-
|
|
30
|
+
if isinstance(obj, new_class):
|
|
31
31
|
return obj
|
|
32
|
-
|
|
33
|
-
raise AirflowException(f"Expected {new_class}, got {type(obj)}")
|
|
32
|
+
raise AirflowException(f"Expected {new_class}, got {type(obj)}")
|
|
34
33
|
|
|
35
34
|
|
|
36
35
|
def _convert_from_dict(obj, new_class):
|
|
37
36
|
if isinstance(obj, new_class):
|
|
38
37
|
return obj
|
|
39
|
-
|
|
38
|
+
if isinstance(obj, dict):
|
|
40
39
|
api_client = ApiClient()
|
|
41
40
|
return api_client._ApiClient__deserialize_model(obj, new_class)
|
|
42
|
-
|
|
43
|
-
raise AirflowException(f"Expected dict or {new_class}, got {type(obj)}")
|
|
41
|
+
raise AirflowException(f"Expected dict or {new_class}, got {type(obj)}")
|
|
44
42
|
|
|
45
43
|
|
|
46
44
|
def convert_volume(volume) -> k8s.V1Volume:
|
|
47
45
|
"""
|
|
48
|
-
|
|
46
|
+
Convert an airflow Volume object into a k8s.V1Volume.
|
|
49
47
|
|
|
50
48
|
:param volume:
|
|
51
|
-
:return: k8s.V1Volume
|
|
52
49
|
"""
|
|
53
50
|
return _convert_kube_model_object(volume, k8s.V1Volume)
|
|
54
51
|
|
|
55
52
|
|
|
56
53
|
def convert_volume_mount(volume_mount) -> k8s.V1VolumeMount:
|
|
57
54
|
"""
|
|
58
|
-
|
|
55
|
+
Convert an airflow VolumeMount object into a k8s.V1VolumeMount.
|
|
59
56
|
|
|
60
57
|
:param volume_mount:
|
|
61
|
-
:return: k8s.V1VolumeMount
|
|
62
58
|
"""
|
|
63
59
|
return _convert_kube_model_object(volume_mount, k8s.V1VolumeMount)
|
|
64
60
|
|
|
65
61
|
|
|
66
|
-
def
|
|
62
|
+
def convert_port(port) -> k8s.V1ContainerPort:
|
|
67
63
|
"""
|
|
68
|
-
|
|
64
|
+
Convert an airflow Port object into a k8s.V1ContainerPort.
|
|
69
65
|
|
|
70
|
-
:param
|
|
71
|
-
:return: k8s.V1ResourceRequirements
|
|
66
|
+
:param port:
|
|
72
67
|
"""
|
|
73
|
-
|
|
74
|
-
from airflow.providers.cncf.kubernetes.backcompat.pod import Resources
|
|
75
|
-
|
|
76
|
-
resources = Resources(**resources)
|
|
77
|
-
return _convert_kube_model_object(resources, k8s.V1ResourceRequirements)
|
|
68
|
+
return _convert_kube_model_object(port, k8s.V1ContainerPort)
|
|
78
69
|
|
|
79
70
|
|
|
80
|
-
def
|
|
71
|
+
def convert_env_vars(env_vars: list[k8s.V1EnvVar] | dict[str, str]) -> list[k8s.V1EnvVar]:
|
|
81
72
|
"""
|
|
82
|
-
|
|
73
|
+
Coerce env var collection for kubernetes.
|
|
83
74
|
|
|
84
|
-
|
|
85
|
-
:return: k8s.V1ContainerPort
|
|
75
|
+
If the collection is a str-str dict, convert it into a list of ``V1EnvVar`` variables.
|
|
86
76
|
"""
|
|
87
|
-
|
|
77
|
+
if isinstance(env_vars, dict):
|
|
78
|
+
return [k8s.V1EnvVar(name=k, value=v) for k, v in env_vars.items()]
|
|
79
|
+
return env_vars
|
|
88
80
|
|
|
89
81
|
|
|
90
|
-
def
|
|
82
|
+
def convert_env_vars_or_raise_error(env_vars: list[k8s.V1EnvVar] | dict[str, str]) -> list[k8s.V1EnvVar]:
|
|
91
83
|
"""
|
|
92
|
-
|
|
84
|
+
Separate function to convert env var collection for kubernetes and then raise an error if it is still the wrong type.
|
|
93
85
|
|
|
94
|
-
|
|
95
|
-
:return:
|
|
86
|
+
This is used after the template strings have been rendered.
|
|
96
87
|
"""
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
for k, v in env_vars.items():
|
|
100
|
-
res.append(k8s.V1EnvVar(name=k, value=v))
|
|
101
|
-
return res
|
|
102
|
-
elif isinstance(env_vars, list):
|
|
88
|
+
env_vars = convert_env_vars(env_vars)
|
|
89
|
+
if isinstance(env_vars, list):
|
|
103
90
|
return env_vars
|
|
104
|
-
|
|
105
|
-
raise AirflowException(f"Expected dict or list, got {type(env_vars)}")
|
|
91
|
+
raise AirflowException(f"Expected dict or list, got {type(env_vars)}")
|
|
106
92
|
|
|
107
93
|
|
|
108
94
|
def convert_pod_runtime_info_env(pod_runtime_info_envs) -> k8s.V1EnvVar:
|
|
109
95
|
"""
|
|
110
|
-
|
|
96
|
+
Convert a PodRuntimeInfoEnv into an k8s.V1EnvVar.
|
|
111
97
|
|
|
112
98
|
:param pod_runtime_info_envs:
|
|
113
|
-
:return:
|
|
114
99
|
"""
|
|
115
100
|
return _convert_kube_model_object(pod_runtime_info_envs, k8s.V1EnvVar)
|
|
116
101
|
|
|
117
102
|
|
|
118
|
-
def convert_image_pull_secrets(image_pull_secrets) ->
|
|
103
|
+
def convert_image_pull_secrets(image_pull_secrets) -> list[k8s.V1LocalObjectReference]:
|
|
119
104
|
"""
|
|
120
|
-
|
|
105
|
+
Convert a PodRuntimeInfoEnv into an k8s.V1EnvVar.
|
|
121
106
|
|
|
122
107
|
:param image_pull_secrets:
|
|
123
|
-
:return:
|
|
124
108
|
"""
|
|
125
109
|
if isinstance(image_pull_secrets, str):
|
|
126
110
|
secrets = image_pull_secrets.split(",")
|
|
127
111
|
return [k8s.V1LocalObjectReference(name=secret) for secret in secrets]
|
|
128
|
-
|
|
129
|
-
return image_pull_secrets
|
|
112
|
+
return image_pull_secrets
|
|
130
113
|
|
|
131
114
|
|
|
132
115
|
def convert_configmap(configmaps) -> k8s.V1EnvFromSource:
|
|
133
116
|
"""
|
|
134
|
-
|
|
117
|
+
Convert a str into an k8s.V1EnvFromSource.
|
|
135
118
|
|
|
136
119
|
:param configmaps:
|
|
137
|
-
:return:
|
|
138
120
|
"""
|
|
139
121
|
return k8s.V1EnvFromSource(config_map_ref=k8s.V1ConfigMapEnvSource(name=configmaps))
|
|
140
122
|
|
|
141
123
|
|
|
142
124
|
def convert_affinity(affinity) -> k8s.V1Affinity:
|
|
143
|
-
"""
|
|
125
|
+
"""Convert a dict into an k8s.V1Affinity."""
|
|
144
126
|
return _convert_from_dict(affinity, k8s.V1Affinity)
|
|
145
127
|
|
|
146
128
|
|
|
147
129
|
def convert_toleration(toleration) -> k8s.V1Toleration:
|
|
148
|
-
"""
|
|
130
|
+
"""Convert a dict into an k8s.V1Toleration."""
|
|
149
131
|
return _convert_from_dict(toleration, k8s.V1Toleration)
|
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
|
2
|
+
# or more contributor license agreements. See the NOTICE file
|
|
3
|
+
# distributed with this work for additional information
|
|
4
|
+
# regarding copyright ownership. The ASF licenses this file
|
|
5
|
+
# to you under the Apache License, Version 2.0 (the
|
|
6
|
+
# "License"); you may not use this file except in compliance
|
|
7
|
+
# with the License. You may obtain a copy of the License at
|
|
8
|
+
#
|
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
#
|
|
11
|
+
# Unless required by applicable law or agreed to in writing,
|
|
12
|
+
# software distributed under the License is distributed on an
|
|
13
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
14
|
+
# KIND, either express or implied. See the License for the
|
|
15
|
+
# specific language governing permissions and limitations
|
|
16
|
+
# under the License.
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
from enum import Enum
|
|
20
|
+
from typing import TYPE_CHECKING, TypeAlias
|
|
21
|
+
|
|
22
|
+
import kubernetes.client as k8s
|
|
23
|
+
import kubernetes_asyncio.client as async_k8s
|
|
24
|
+
|
|
25
|
+
if TYPE_CHECKING:
|
|
26
|
+
from airflow.providers.cncf.kubernetes.operators.pod import KubernetesPodOperator
|
|
27
|
+
from airflow.utils.context import Context
|
|
28
|
+
|
|
29
|
+
client_type: TypeAlias = k8s.CoreV1Api | async_k8s.CoreV1Api
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class ExecutionMode(str, Enum):
|
|
33
|
+
"""Enum class for execution mode."""
|
|
34
|
+
|
|
35
|
+
SYNC = "sync"
|
|
36
|
+
ASYNC = "async"
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class KubernetesPodOperatorCallback:
|
|
40
|
+
"""
|
|
41
|
+
`KubernetesPodOperator` callbacks methods.
|
|
42
|
+
|
|
43
|
+
Currently, the callbacks methods are not called in the async mode, this support will be added
|
|
44
|
+
in the future.
|
|
45
|
+
"""
|
|
46
|
+
|
|
47
|
+
@staticmethod
|
|
48
|
+
def on_sync_client_creation(*, client: k8s.CoreV1Api, operator: KubernetesPodOperator, **kwargs) -> None:
|
|
49
|
+
"""
|
|
50
|
+
Invoke this callback after creating the sync client.
|
|
51
|
+
|
|
52
|
+
:param client: the created `kubernetes.client.CoreV1Api` client.
|
|
53
|
+
"""
|
|
54
|
+
pass
|
|
55
|
+
|
|
56
|
+
@staticmethod
|
|
57
|
+
def on_pod_manifest_created(
|
|
58
|
+
*,
|
|
59
|
+
pod_request: k8s.V1Pod,
|
|
60
|
+
client: client_type,
|
|
61
|
+
mode: str,
|
|
62
|
+
operator: KubernetesPodOperator,
|
|
63
|
+
context: Context,
|
|
64
|
+
**kwargs,
|
|
65
|
+
) -> None:
|
|
66
|
+
"""
|
|
67
|
+
Invoke this callback after KPO creates the V1Pod manifest but before the pod is created.
|
|
68
|
+
|
|
69
|
+
:param pod_request: the kubernetes pod manifest
|
|
70
|
+
:param client: the Kubernetes client that can be used in the callback.
|
|
71
|
+
:param mode: the current execution mode, it's one of (`sync`, `async`).
|
|
72
|
+
"""
|
|
73
|
+
pass
|
|
74
|
+
|
|
75
|
+
@staticmethod
|
|
76
|
+
def on_pod_creation(
|
|
77
|
+
*,
|
|
78
|
+
pod: k8s.V1Pod,
|
|
79
|
+
client: client_type,
|
|
80
|
+
mode: str,
|
|
81
|
+
operator: KubernetesPodOperator,
|
|
82
|
+
context: Context,
|
|
83
|
+
**kwargs,
|
|
84
|
+
) -> None:
|
|
85
|
+
"""
|
|
86
|
+
Invoke this callback after creating the pod.
|
|
87
|
+
|
|
88
|
+
:param pod: the created pod.
|
|
89
|
+
:param client: the Kubernetes client that can be used in the callback.
|
|
90
|
+
:param mode: the current execution mode, it's one of (`sync`, `async`).
|
|
91
|
+
"""
|
|
92
|
+
pass
|
|
93
|
+
|
|
94
|
+
@staticmethod
|
|
95
|
+
def on_pod_starting(
|
|
96
|
+
*,
|
|
97
|
+
pod: k8s.V1Pod,
|
|
98
|
+
client: client_type,
|
|
99
|
+
mode: str,
|
|
100
|
+
operator: KubernetesPodOperator,
|
|
101
|
+
context: Context,
|
|
102
|
+
**kwargs,
|
|
103
|
+
) -> None:
|
|
104
|
+
"""
|
|
105
|
+
Invoke this callback when the pod starts.
|
|
106
|
+
|
|
107
|
+
:param pod: the started pod.
|
|
108
|
+
:param client: the Kubernetes client that can be used in the callback.
|
|
109
|
+
:param mode: the current execution mode, it's one of (`sync`, `async`).
|
|
110
|
+
"""
|
|
111
|
+
pass
|
|
112
|
+
|
|
113
|
+
@staticmethod
|
|
114
|
+
def on_pod_completion(
|
|
115
|
+
*,
|
|
116
|
+
pod: k8s.V1Pod,
|
|
117
|
+
client: client_type,
|
|
118
|
+
mode: str,
|
|
119
|
+
operator: KubernetesPodOperator,
|
|
120
|
+
context: Context,
|
|
121
|
+
**kwargs,
|
|
122
|
+
) -> None:
|
|
123
|
+
"""
|
|
124
|
+
Invoke this callback when the pod completes.
|
|
125
|
+
|
|
126
|
+
:param pod: the completed pod.
|
|
127
|
+
:param client: the Kubernetes client that can be used in the callback.
|
|
128
|
+
:param mode: the current execution mode, it's one of (`sync`, `async`).
|
|
129
|
+
"""
|
|
130
|
+
pass
|
|
131
|
+
|
|
132
|
+
@staticmethod
|
|
133
|
+
def on_pod_teardown(
|
|
134
|
+
*,
|
|
135
|
+
pod: k8s.V1Pod,
|
|
136
|
+
client: client_type,
|
|
137
|
+
mode: str,
|
|
138
|
+
operator: KubernetesPodOperator,
|
|
139
|
+
context: Context,
|
|
140
|
+
**kwargs,
|
|
141
|
+
) -> None:
|
|
142
|
+
"""
|
|
143
|
+
Invoke this callback after all pod completion callbacks but before the pod is deleted.
|
|
144
|
+
|
|
145
|
+
:param pod: the completed pod.
|
|
146
|
+
:param client: the Kubernetes client that can be used in the callback.
|
|
147
|
+
:param mode: the current execution mode, it's one of (`sync`, `async`).
|
|
148
|
+
"""
|
|
149
|
+
pass
|
|
150
|
+
|
|
151
|
+
@staticmethod
|
|
152
|
+
def on_pod_cleanup(
|
|
153
|
+
*,
|
|
154
|
+
pod: k8s.V1Pod,
|
|
155
|
+
client: client_type,
|
|
156
|
+
mode: str,
|
|
157
|
+
operator: KubernetesPodOperator,
|
|
158
|
+
context: Context,
|
|
159
|
+
**kwargs,
|
|
160
|
+
):
|
|
161
|
+
"""
|
|
162
|
+
Invoke this callback after cleaning/deleting the pod.
|
|
163
|
+
|
|
164
|
+
:param pod: the completed pod.
|
|
165
|
+
:param client: the Kubernetes client that can be used in the callback.
|
|
166
|
+
:param mode: the current execution mode, it's one of (`sync`, `async`).
|
|
167
|
+
"""
|
|
168
|
+
pass
|
|
169
|
+
|
|
170
|
+
@staticmethod
|
|
171
|
+
def on_operator_resuming(
|
|
172
|
+
*,
|
|
173
|
+
pod: k8s.V1Pod,
|
|
174
|
+
event: dict,
|
|
175
|
+
client: client_type,
|
|
176
|
+
mode: str,
|
|
177
|
+
operator: KubernetesPodOperator,
|
|
178
|
+
context: Context,
|
|
179
|
+
**kwargs,
|
|
180
|
+
) -> None:
|
|
181
|
+
"""
|
|
182
|
+
Invoke this callback when resuming the `KubernetesPodOperator` from deferred state.
|
|
183
|
+
|
|
184
|
+
:param pod: the current state of the pod.
|
|
185
|
+
:param event: the returned event from the Trigger.
|
|
186
|
+
:param client: the Kubernetes client that can be used in the callback.
|
|
187
|
+
:param mode: the current execution mode, it's one of (`sync`, `async`).
|
|
188
|
+
"""
|
|
189
|
+
pass
|
|
190
|
+
|
|
191
|
+
@staticmethod
|
|
192
|
+
def progress_callback(*, line: str, client: client_type, mode: str, **kwargs) -> None:
|
|
193
|
+
"""
|
|
194
|
+
Invoke this callback to process pod container logs.
|
|
195
|
+
|
|
196
|
+
:param line: the read line of log.
|
|
197
|
+
:param client: the Kubernetes client that can be used in the callback.
|
|
198
|
+
:param mode: the current execution mode, it's one of (`sync`, `async`).
|
|
199
|
+
"""
|
|
200
|
+
pass
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
|
2
|
+
# or more contributor license agreements. See the NOTICE file
|
|
3
|
+
# distributed with this work for additional information
|
|
4
|
+
# regarding copyright ownership. The ASF licenses this file
|
|
5
|
+
# to you under the Apache License, Version 2.0 (the
|
|
6
|
+
# "License"); you may not use this file except in compliance
|
|
7
|
+
# with the License. You may obtain a copy of the License at
|
|
8
|
+
#
|
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
#
|
|
11
|
+
# Unless required by applicable law or agreed to in writing,
|
|
12
|
+
# software distributed under the License is distributed on an
|
|
13
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
14
|
+
# KIND, either express or implied. See the License for the
|
|
15
|
+
# specific language governing permissions and limitations
|
|
16
|
+
# under the License.
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
|
2
|
+
# or more contributor license agreements. See the NOTICE file
|
|
3
|
+
# distributed with this work for additional information
|
|
4
|
+
# regarding copyright ownership. The ASF licenses this file
|
|
5
|
+
# to you under the Apache License, Version 2.0 (the
|
|
6
|
+
# "License"); you may not use this file except in compliance
|
|
7
|
+
# with the License. You may obtain a copy of the License at
|
|
8
|
+
#
|
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
#
|
|
11
|
+
# Unless required by applicable law or agreed to in writing,
|
|
12
|
+
# software distributed under the License is distributed on an
|
|
13
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
14
|
+
# KIND, either express or implied. See the License for the
|
|
15
|
+
# specific language governing permissions and limitations
|
|
16
|
+
# under the License.
|
|
17
|
+
"""Kubernetes sub-commands."""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import os
|
|
22
|
+
import sys
|
|
23
|
+
from datetime import datetime, timedelta
|
|
24
|
+
|
|
25
|
+
from kubernetes import client
|
|
26
|
+
from kubernetes.client.api_client import ApiClient
|
|
27
|
+
from kubernetes.client.rest import ApiException
|
|
28
|
+
|
|
29
|
+
from airflow.models import DagModel, DagRun, TaskInstance
|
|
30
|
+
from airflow.providers.cncf.kubernetes import pod_generator
|
|
31
|
+
from airflow.providers.cncf.kubernetes.executors.kubernetes_executor import KubeConfig
|
|
32
|
+
from airflow.providers.cncf.kubernetes.kube_client import get_kube_client
|
|
33
|
+
from airflow.providers.cncf.kubernetes.kubernetes_helper_functions import create_unique_id
|
|
34
|
+
from airflow.providers.cncf.kubernetes.pod_generator import PodGenerator, generate_pod_command_args
|
|
35
|
+
from airflow.providers.cncf.kubernetes.version_compat import AIRFLOW_V_3_0_PLUS, AIRFLOW_V_3_1_PLUS
|
|
36
|
+
from airflow.utils import cli as cli_utils, yaml
|
|
37
|
+
from airflow.utils.providers_configuration_loader import providers_configuration_loaded
|
|
38
|
+
from airflow.utils.types import DagRunType
|
|
39
|
+
|
|
40
|
+
if AIRFLOW_V_3_1_PLUS:
|
|
41
|
+
from airflow.utils.cli import get_bagged_dag
|
|
42
|
+
else:
|
|
43
|
+
from airflow.utils.cli import get_dag as get_bagged_dag # type: ignore[attr-defined,no-redef]
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@cli_utils.action_cli
|
|
47
|
+
@providers_configuration_loaded
|
|
48
|
+
def generate_pod_yaml(args):
|
|
49
|
+
"""Generate yaml files for each task in the DAG. Used for testing output of KubernetesExecutor."""
|
|
50
|
+
logical_date = args.logical_date if AIRFLOW_V_3_0_PLUS else args.execution_date
|
|
51
|
+
if AIRFLOW_V_3_0_PLUS:
|
|
52
|
+
dag = get_bagged_dag(bundle_names=args.bundle_name, dag_id=args.dag_id)
|
|
53
|
+
else:
|
|
54
|
+
dag = get_bagged_dag(subdir=args.subdir, dag_id=args.dag_id)
|
|
55
|
+
yaml_output_path = args.output_path
|
|
56
|
+
|
|
57
|
+
dm = DagModel(dag_id=dag.dag_id)
|
|
58
|
+
|
|
59
|
+
if AIRFLOW_V_3_0_PLUS:
|
|
60
|
+
dr = DagRun(dag.dag_id, logical_date=logical_date)
|
|
61
|
+
dr.run_id = DagRun.generate_run_id(
|
|
62
|
+
run_type=DagRunType.MANUAL, logical_date=logical_date, run_after=logical_date
|
|
63
|
+
)
|
|
64
|
+
dm.bundle_name = args.bundle_name if args.bundle_name else "default"
|
|
65
|
+
dm.relative_fileloc = dag.relative_fileloc
|
|
66
|
+
else:
|
|
67
|
+
dr = DagRun(dag.dag_id, execution_date=logical_date)
|
|
68
|
+
dr.run_id = DagRun.generate_run_id(run_type=DagRunType.MANUAL, execution_date=logical_date)
|
|
69
|
+
|
|
70
|
+
kube_config = KubeConfig()
|
|
71
|
+
|
|
72
|
+
for task in dag.tasks:
|
|
73
|
+
if AIRFLOW_V_3_0_PLUS:
|
|
74
|
+
from uuid6 import uuid7
|
|
75
|
+
|
|
76
|
+
ti = TaskInstance(task, run_id=dr.run_id, dag_version_id=uuid7())
|
|
77
|
+
else:
|
|
78
|
+
ti = TaskInstance(task, run_id=dr.run_id)
|
|
79
|
+
ti.dag_run = dr
|
|
80
|
+
ti.dag_model = dm
|
|
81
|
+
|
|
82
|
+
command_args = generate_pod_command_args(ti)
|
|
83
|
+
pod = PodGenerator.construct_pod(
|
|
84
|
+
dag_id=args.dag_id,
|
|
85
|
+
task_id=ti.task_id,
|
|
86
|
+
pod_id=create_unique_id(args.dag_id, ti.task_id),
|
|
87
|
+
try_number=ti.try_number,
|
|
88
|
+
kube_image=kube_config.kube_image,
|
|
89
|
+
date=ti.logical_date if AIRFLOW_V_3_0_PLUS else ti.execution_date,
|
|
90
|
+
args=command_args,
|
|
91
|
+
pod_override_object=PodGenerator.from_obj(ti.executor_config),
|
|
92
|
+
scheduler_job_id="worker-config",
|
|
93
|
+
namespace=kube_config.executor_namespace,
|
|
94
|
+
base_worker_pod=PodGenerator.deserialize_model_file(kube_config.pod_template_file),
|
|
95
|
+
with_mutation_hook=True,
|
|
96
|
+
)
|
|
97
|
+
api_client = ApiClient()
|
|
98
|
+
date_string = pod_generator.datetime_to_label_safe_datestring(logical_date)
|
|
99
|
+
yaml_file_name = f"{args.dag_id}_{ti.task_id}_{date_string}.yml"
|
|
100
|
+
os.makedirs(os.path.dirname(yaml_output_path + "/airflow_yaml_output/"), exist_ok=True)
|
|
101
|
+
with open(yaml_output_path + "/airflow_yaml_output/" + yaml_file_name, "w") as output:
|
|
102
|
+
sanitized_pod = api_client.sanitize_for_serialization(pod)
|
|
103
|
+
output.write(yaml.dump(sanitized_pod))
|
|
104
|
+
print(f"YAML output can be found at {yaml_output_path}/airflow_yaml_output/")
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
@cli_utils.action_cli(check_db=False)
|
|
108
|
+
@providers_configuration_loaded
|
|
109
|
+
def cleanup_pods(args):
|
|
110
|
+
"""Clean up k8s pods in evicted/failed/succeeded/pending states."""
|
|
111
|
+
namespace = args.namespace
|
|
112
|
+
|
|
113
|
+
min_pending_minutes = args.min_pending_minutes
|
|
114
|
+
# protect newly created pods from deletion
|
|
115
|
+
if min_pending_minutes < 5:
|
|
116
|
+
min_pending_minutes = 5
|
|
117
|
+
|
|
118
|
+
# https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/
|
|
119
|
+
# All Containers in the Pod have terminated in success, and will not be restarted.
|
|
120
|
+
pod_succeeded = "succeeded"
|
|
121
|
+
|
|
122
|
+
# The Pod has been accepted by the Kubernetes cluster,
|
|
123
|
+
# but one or more of the containers has not been set up and made ready to run.
|
|
124
|
+
pod_pending = "pending"
|
|
125
|
+
|
|
126
|
+
# All Containers in the Pod have terminated, and at least one Container has terminated in failure.
|
|
127
|
+
# That is, the Container either exited with non-zero status or was terminated by the system.
|
|
128
|
+
pod_failed = "failed"
|
|
129
|
+
|
|
130
|
+
# https://kubernetes.io/docs/tasks/administer-cluster/out-of-resource/
|
|
131
|
+
pod_reason_evicted = "evicted"
|
|
132
|
+
# If pod is failed and restartPolicy is:
|
|
133
|
+
# * Always: Restart Container; Pod phase stays Running.
|
|
134
|
+
# * OnFailure: Restart Container; Pod phase stays Running.
|
|
135
|
+
# * Never: Pod phase becomes Failed.
|
|
136
|
+
pod_restart_policy_never = "never"
|
|
137
|
+
|
|
138
|
+
print("Loading Kubernetes configuration")
|
|
139
|
+
kube_client = get_kube_client()
|
|
140
|
+
print(f"Listing pods in namespace {namespace}")
|
|
141
|
+
airflow_pod_labels = [
|
|
142
|
+
"dag_id",
|
|
143
|
+
"task_id",
|
|
144
|
+
"try_number",
|
|
145
|
+
"airflow_version",
|
|
146
|
+
]
|
|
147
|
+
list_kwargs = {"namespace": namespace, "limit": 500, "label_selector": ",".join(airflow_pod_labels)}
|
|
148
|
+
|
|
149
|
+
while True:
|
|
150
|
+
pod_list = kube_client.list_namespaced_pod(**list_kwargs)
|
|
151
|
+
for pod in pod_list.items:
|
|
152
|
+
pod_name = pod.metadata.name
|
|
153
|
+
print(f"Inspecting pod {pod_name}")
|
|
154
|
+
pod_phase = pod.status.phase.lower()
|
|
155
|
+
pod_reason = pod.status.reason.lower() if pod.status.reason else ""
|
|
156
|
+
pod_restart_policy = pod.spec.restart_policy.lower()
|
|
157
|
+
current_time = datetime.now(pod.metadata.creation_timestamp.tzinfo)
|
|
158
|
+
|
|
159
|
+
if (
|
|
160
|
+
pod_phase == pod_succeeded
|
|
161
|
+
or (pod_phase == pod_failed and pod_restart_policy == pod_restart_policy_never)
|
|
162
|
+
or (pod_reason == pod_reason_evicted)
|
|
163
|
+
or (
|
|
164
|
+
pod_phase == pod_pending
|
|
165
|
+
and current_time - pod.metadata.creation_timestamp
|
|
166
|
+
> timedelta(minutes=min_pending_minutes)
|
|
167
|
+
)
|
|
168
|
+
):
|
|
169
|
+
print(
|
|
170
|
+
f'Deleting pod "{pod_name}" phase "{pod_phase}" and reason "{pod_reason}", '
|
|
171
|
+
f'restart policy "{pod_restart_policy}"'
|
|
172
|
+
)
|
|
173
|
+
try:
|
|
174
|
+
_delete_pod(pod.metadata.name, namespace)
|
|
175
|
+
except ApiException as e:
|
|
176
|
+
print(f"Can't remove POD: {e}", file=sys.stderr)
|
|
177
|
+
else:
|
|
178
|
+
print(f"No action taken on pod {pod_name}")
|
|
179
|
+
continue_token = pod_list.metadata._continue
|
|
180
|
+
if not continue_token:
|
|
181
|
+
break
|
|
182
|
+
list_kwargs["_continue"] = continue_token
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _delete_pod(name, namespace):
|
|
186
|
+
"""
|
|
187
|
+
Delete a namespaced pod.
|
|
188
|
+
|
|
189
|
+
Helper Function for cleanup_pods.
|
|
190
|
+
"""
|
|
191
|
+
kube_client = get_kube_client()
|
|
192
|
+
delete_options = client.V1DeleteOptions()
|
|
193
|
+
print(f'Deleting POD "{name}" from "{namespace}" namespace')
|
|
194
|
+
api_response = kube_client.delete_namespaced_pod(name=name, namespace=namespace, body=delete_options)
|
|
195
|
+
print(api_response)
|