apache-airflow-providers-cncf-kubernetes 3.1.0__py3-none-any.whl → 10.10.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. airflow/providers/cncf/kubernetes/__init__.py +18 -23
  2. airflow/providers/cncf/kubernetes/backcompat/__init__.py +17 -0
  3. airflow/providers/cncf/kubernetes/backcompat/backwards_compat_converters.py +31 -49
  4. airflow/providers/cncf/kubernetes/callbacks.py +200 -0
  5. airflow/providers/cncf/kubernetes/cli/__init__.py +16 -0
  6. airflow/providers/cncf/kubernetes/cli/kubernetes_command.py +195 -0
  7. airflow/providers/cncf/kubernetes/decorators/kubernetes.py +163 -0
  8. airflow/providers/cncf/kubernetes/decorators/kubernetes_cmd.py +118 -0
  9. airflow/providers/cncf/kubernetes/exceptions.py +37 -0
  10. airflow/providers/cncf/kubernetes/executors/__init__.py +17 -0
  11. airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py +831 -0
  12. airflow/providers/cncf/kubernetes/executors/kubernetes_executor_types.py +91 -0
  13. airflow/providers/cncf/kubernetes/executors/kubernetes_executor_utils.py +736 -0
  14. airflow/providers/cncf/kubernetes/executors/local_kubernetes_executor.py +306 -0
  15. airflow/providers/cncf/kubernetes/get_provider_info.py +249 -50
  16. airflow/providers/cncf/kubernetes/hooks/kubernetes.py +846 -112
  17. airflow/providers/cncf/kubernetes/k8s_model.py +62 -0
  18. airflow/providers/cncf/kubernetes/kube_client.py +156 -0
  19. airflow/providers/cncf/kubernetes/kube_config.py +125 -0
  20. airflow/providers/cncf/kubernetes/kubernetes_executor_templates/__init__.py +16 -0
  21. airflow/providers/cncf/kubernetes/kubernetes_executor_templates/basic_template.yaml +79 -0
  22. airflow/providers/cncf/kubernetes/kubernetes_helper_functions.py +165 -0
  23. airflow/providers/cncf/kubernetes/operators/custom_object_launcher.py +368 -0
  24. airflow/providers/cncf/kubernetes/operators/job.py +646 -0
  25. airflow/providers/cncf/kubernetes/operators/kueue.py +132 -0
  26. airflow/providers/cncf/kubernetes/operators/pod.py +1417 -0
  27. airflow/providers/cncf/kubernetes/operators/resource.py +191 -0
  28. airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py +336 -35
  29. airflow/providers/cncf/kubernetes/pod_generator.py +592 -0
  30. airflow/providers/cncf/kubernetes/pod_template_file_examples/__init__.py +16 -0
  31. airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_image_template.yaml +68 -0
  32. airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_volume_template.yaml +74 -0
  33. airflow/providers/cncf/kubernetes/pod_template_file_examples/git_sync_template.yaml +95 -0
  34. airflow/providers/cncf/kubernetes/python_kubernetes_script.jinja2 +51 -0
  35. airflow/providers/cncf/kubernetes/python_kubernetes_script.py +92 -0
  36. airflow/providers/cncf/kubernetes/resource_convert/__init__.py +16 -0
  37. airflow/providers/cncf/kubernetes/resource_convert/configmap.py +52 -0
  38. airflow/providers/cncf/kubernetes/resource_convert/env_variable.py +39 -0
  39. airflow/providers/cncf/kubernetes/resource_convert/secret.py +40 -0
  40. airflow/providers/cncf/kubernetes/secret.py +128 -0
  41. airflow/providers/cncf/kubernetes/sensors/spark_kubernetes.py +30 -14
  42. airflow/providers/cncf/kubernetes/template_rendering.py +81 -0
  43. airflow/providers/cncf/kubernetes/triggers/__init__.py +16 -0
  44. airflow/providers/cncf/kubernetes/triggers/job.py +176 -0
  45. airflow/providers/cncf/kubernetes/triggers/pod.py +344 -0
  46. airflow/providers/cncf/kubernetes/utils/__init__.py +3 -0
  47. airflow/providers/cncf/kubernetes/utils/container.py +118 -0
  48. airflow/providers/cncf/kubernetes/utils/delete_from.py +154 -0
  49. airflow/providers/cncf/kubernetes/utils/k8s_resource_iterator.py +46 -0
  50. airflow/providers/cncf/kubernetes/utils/pod_manager.py +887 -152
  51. airflow/providers/cncf/kubernetes/utils/xcom_sidecar.py +25 -16
  52. airflow/providers/cncf/kubernetes/version_compat.py +38 -0
  53. apache_airflow_providers_cncf_kubernetes-10.10.0rc1.dist-info/METADATA +125 -0
  54. apache_airflow_providers_cncf_kubernetes-10.10.0rc1.dist-info/RECORD +62 -0
  55. {apache_airflow_providers_cncf_kubernetes-3.1.0.dist-info → apache_airflow_providers_cncf_kubernetes-10.10.0rc1.dist-info}/WHEEL +1 -2
  56. apache_airflow_providers_cncf_kubernetes-10.10.0rc1.dist-info/entry_points.txt +3 -0
  57. apache_airflow_providers_cncf_kubernetes-10.10.0rc1.dist-info/licenses/NOTICE +5 -0
  58. airflow/providers/cncf/kubernetes/backcompat/pod.py +0 -119
  59. airflow/providers/cncf/kubernetes/backcompat/pod_runtime_info_env.py +0 -56
  60. airflow/providers/cncf/kubernetes/backcompat/volume.py +0 -62
  61. airflow/providers/cncf/kubernetes/backcompat/volume_mount.py +0 -58
  62. airflow/providers/cncf/kubernetes/example_dags/example_kubernetes.py +0 -163
  63. airflow/providers/cncf/kubernetes/example_dags/example_spark_kubernetes.py +0 -66
  64. airflow/providers/cncf/kubernetes/example_dags/example_spark_kubernetes_spark_pi.yaml +0 -57
  65. airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py +0 -622
  66. apache_airflow_providers_cncf_kubernetes-3.1.0.dist-info/METADATA +0 -452
  67. apache_airflow_providers_cncf_kubernetes-3.1.0.dist-info/NOTICE +0 -6
  68. apache_airflow_providers_cncf_kubernetes-3.1.0.dist-info/RECORD +0 -29
  69. apache_airflow_providers_cncf_kubernetes-3.1.0.dist-info/entry_points.txt +0 -3
  70. apache_airflow_providers_cncf_kubernetes-3.1.0.dist-info/top_level.txt +0 -1
  71. /airflow/providers/cncf/kubernetes/{example_dags → decorators}/__init__.py +0 -0
  72. {apache_airflow_providers_cncf_kubernetes-3.1.0.dist-info → apache_airflow_providers_cncf_kubernetes-10.10.0rc1.dist-info/licenses}/LICENSE +0 -0
@@ -1,4 +1,3 @@
1
- #
2
1
  # Licensed to the Apache Software Foundation (ASF) under one
3
2
  # or more contributor license agreements. See the NOTICE file
4
3
  # distributed with this work for additional information
@@ -15,30 +14,26 @@
15
14
  # KIND, either express or implied. See the License for the
16
15
  # specific language governing permissions and limitations
17
16
  # under the License.
18
- import sys
19
-
20
- if sys.version_info < (3, 7):
21
- # This is needed because the Python Kubernetes client >= 12.0 contains a logging object, meaning that
22
- # v1.Pod et al. are not pickleable on Python 3.6.
23
-
24
- # Python 3.7 added this via https://bugs.python.org/issue30520 in 2017 -- but Python 3.6 doesn't have this
25
- # method.
17
+ #
18
+ # NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
19
+ # OVERWRITTEN WHEN PREPARING DOCUMENTATION FOR THE PACKAGES.
20
+ #
21
+ # IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
22
+ # `PROVIDER__INIT__PY_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
23
+ #
24
+ from __future__ import annotations
26
25
 
27
- # This is duplicated/backported from airflow.logging_config in 2.2, but by having it here as well it means
28
- # that we can update the version used in this provider and have it work for older versions
29
- import copyreg
30
- import logging
26
+ import packaging.version
31
27
 
32
- def _reduce_Logger(logger):
33
- if logging.getLogger(logger.name) is not logger:
34
- import pickle
28
+ from airflow import __version__ as airflow_version
35
29
 
36
- raise pickle.PicklingError('logger cannot be pickled')
37
- return logging.getLogger, (logger.name,)
30
+ __all__ = ["__version__"]
38
31
 
39
- def _reduce_RootLogger(logger):
40
- return logging.getLogger, ()
32
+ __version__ = "10.10.0"
41
33
 
42
- if logging.Logger not in copyreg.dispatch_table:
43
- copyreg.pickle(logging.Logger, _reduce_Logger)
44
- copyreg.pickle(logging.RootLogger, _reduce_RootLogger)
34
+ if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
35
+ "2.10.0"
36
+ ):
37
+ raise RuntimeError(
38
+ f"The package `apache-airflow-providers-cncf-kubernetes:{__version__}` needs Apache Airflow 2.10.0+"
39
+ )
@@ -14,3 +14,20 @@
14
14
  # KIND, either express or implied. See the License for the
15
15
  # specific language governing permissions and limitations
16
16
  # under the License.
17
+
18
+ from __future__ import annotations
19
+
20
+
21
+ def get_logical_date_key() -> str:
22
+ """
23
+ Get the key for execution/logical date for the appropriate Airflow version.
24
+
25
+ This is done by detecting the CLI argument name. There are various ways to
26
+ do this, but the CLI key name has a very small import footprint (especially
27
+ compared to importing ORM models).
28
+ """
29
+ from airflow.cli import cli_config
30
+
31
+ if hasattr(cli_config, "ARG_LOGICAL_DATE"):
32
+ return "logical_date"
33
+ return "execution_date"
@@ -14,9 +14,9 @@
14
14
  # KIND, either express or implied. See the License for the
15
15
  # specific language governing permissions and limitations
16
16
  # under the License.
17
- """Executes task in a Kubernetes POD"""
17
+ """Executes task in a Kubernetes POD."""
18
18
 
19
- from typing import List
19
+ from __future__ import annotations
20
20
 
21
21
  from kubernetes.client import ApiClient, models as k8s
22
22
 
@@ -27,123 +27,105 @@ def _convert_kube_model_object(obj, new_class):
27
27
  convert_op = getattr(obj, "to_k8s_client_obj", None)
28
28
  if callable(convert_op):
29
29
  return obj.to_k8s_client_obj()
30
- elif isinstance(obj, new_class):
30
+ if isinstance(obj, new_class):
31
31
  return obj
32
- else:
33
- raise AirflowException(f"Expected {new_class}, got {type(obj)}")
32
+ raise AirflowException(f"Expected {new_class}, got {type(obj)}")
34
33
 
35
34
 
36
35
  def _convert_from_dict(obj, new_class):
37
36
  if isinstance(obj, new_class):
38
37
  return obj
39
- elif isinstance(obj, dict):
38
+ if isinstance(obj, dict):
40
39
  api_client = ApiClient()
41
40
  return api_client._ApiClient__deserialize_model(obj, new_class)
42
- else:
43
- raise AirflowException(f"Expected dict or {new_class}, got {type(obj)}")
41
+ raise AirflowException(f"Expected dict or {new_class}, got {type(obj)}")
44
42
 
45
43
 
46
44
  def convert_volume(volume) -> k8s.V1Volume:
47
45
  """
48
- Converts an airflow Volume object into a k8s.V1Volume
46
+ Convert an airflow Volume object into a k8s.V1Volume.
49
47
 
50
48
  :param volume:
51
- :return: k8s.V1Volume
52
49
  """
53
50
  return _convert_kube_model_object(volume, k8s.V1Volume)
54
51
 
55
52
 
56
53
  def convert_volume_mount(volume_mount) -> k8s.V1VolumeMount:
57
54
  """
58
- Converts an airflow VolumeMount object into a k8s.V1VolumeMount
55
+ Convert an airflow VolumeMount object into a k8s.V1VolumeMount.
59
56
 
60
57
  :param volume_mount:
61
- :return: k8s.V1VolumeMount
62
58
  """
63
59
  return _convert_kube_model_object(volume_mount, k8s.V1VolumeMount)
64
60
 
65
61
 
66
- def convert_resources(resources) -> k8s.V1ResourceRequirements:
62
+ def convert_port(port) -> k8s.V1ContainerPort:
67
63
  """
68
- Converts an airflow Resources object into a k8s.V1ResourceRequirements
64
+ Convert an airflow Port object into a k8s.V1ContainerPort.
69
65
 
70
- :param resources:
71
- :return: k8s.V1ResourceRequirements
66
+ :param port:
72
67
  """
73
- if isinstance(resources, dict):
74
- from airflow.providers.cncf.kubernetes.backcompat.pod import Resources
75
-
76
- resources = Resources(**resources)
77
- return _convert_kube_model_object(resources, k8s.V1ResourceRequirements)
68
+ return _convert_kube_model_object(port, k8s.V1ContainerPort)
78
69
 
79
70
 
80
- def convert_port(port) -> k8s.V1ContainerPort:
71
+ def convert_env_vars(env_vars: list[k8s.V1EnvVar] | dict[str, str]) -> list[k8s.V1EnvVar]:
81
72
  """
82
- Converts an airflow Port object into a k8s.V1ContainerPort
73
+ Coerce env var collection for kubernetes.
83
74
 
84
- :param port:
85
- :return: k8s.V1ContainerPort
75
+ If the collection is a str-str dict, convert it into a list of ``V1EnvVar`` variables.
86
76
  """
87
- return _convert_kube_model_object(port, k8s.V1ContainerPort)
77
+ if isinstance(env_vars, dict):
78
+ return [k8s.V1EnvVar(name=k, value=v) for k, v in env_vars.items()]
79
+ return env_vars
88
80
 
89
81
 
90
- def convert_env_vars(env_vars) -> List[k8s.V1EnvVar]:
82
+ def convert_env_vars_or_raise_error(env_vars: list[k8s.V1EnvVar] | dict[str, str]) -> list[k8s.V1EnvVar]:
91
83
  """
92
- Converts a dictionary into a list of env_vars
84
+ Separate function to convert env var collection for kubernetes and then raise an error if it is still the wrong type.
93
85
 
94
- :param env_vars:
95
- :return:
86
+ This is used after the template strings have been rendered.
96
87
  """
97
- if isinstance(env_vars, dict):
98
- res = []
99
- for k, v in env_vars.items():
100
- res.append(k8s.V1EnvVar(name=k, value=v))
101
- return res
102
- elif isinstance(env_vars, list):
88
+ env_vars = convert_env_vars(env_vars)
89
+ if isinstance(env_vars, list):
103
90
  return env_vars
104
- else:
105
- raise AirflowException(f"Expected dict or list, got {type(env_vars)}")
91
+ raise AirflowException(f"Expected dict or list, got {type(env_vars)}")
106
92
 
107
93
 
108
94
  def convert_pod_runtime_info_env(pod_runtime_info_envs) -> k8s.V1EnvVar:
109
95
  """
110
- Converts a PodRuntimeInfoEnv into an k8s.V1EnvVar
96
+ Convert a PodRuntimeInfoEnv into an k8s.V1EnvVar.
111
97
 
112
98
  :param pod_runtime_info_envs:
113
- :return:
114
99
  """
115
100
  return _convert_kube_model_object(pod_runtime_info_envs, k8s.V1EnvVar)
116
101
 
117
102
 
118
- def convert_image_pull_secrets(image_pull_secrets) -> List[k8s.V1LocalObjectReference]:
103
+ def convert_image_pull_secrets(image_pull_secrets) -> list[k8s.V1LocalObjectReference]:
119
104
  """
120
- Converts a PodRuntimeInfoEnv into an k8s.V1EnvVar
105
+ Convert a PodRuntimeInfoEnv into an k8s.V1EnvVar.
121
106
 
122
107
  :param image_pull_secrets:
123
- :return:
124
108
  """
125
109
  if isinstance(image_pull_secrets, str):
126
110
  secrets = image_pull_secrets.split(",")
127
111
  return [k8s.V1LocalObjectReference(name=secret) for secret in secrets]
128
- else:
129
- return image_pull_secrets
112
+ return image_pull_secrets
130
113
 
131
114
 
132
115
  def convert_configmap(configmaps) -> k8s.V1EnvFromSource:
133
116
  """
134
- Converts a str into an k8s.V1EnvFromSource
117
+ Convert a str into an k8s.V1EnvFromSource.
135
118
 
136
119
  :param configmaps:
137
- :return:
138
120
  """
139
121
  return k8s.V1EnvFromSource(config_map_ref=k8s.V1ConfigMapEnvSource(name=configmaps))
140
122
 
141
123
 
142
124
  def convert_affinity(affinity) -> k8s.V1Affinity:
143
- """Converts a dict into an k8s.V1Affinity"""
125
+ """Convert a dict into an k8s.V1Affinity."""
144
126
  return _convert_from_dict(affinity, k8s.V1Affinity)
145
127
 
146
128
 
147
129
  def convert_toleration(toleration) -> k8s.V1Toleration:
148
- """Converts a dict into an k8s.V1Toleration"""
130
+ """Convert a dict into an k8s.V1Toleration."""
149
131
  return _convert_from_dict(toleration, k8s.V1Toleration)
@@ -0,0 +1,200 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+ from __future__ import annotations
18
+
19
+ from enum import Enum
20
+ from typing import TYPE_CHECKING, TypeAlias
21
+
22
+ import kubernetes.client as k8s
23
+ import kubernetes_asyncio.client as async_k8s
24
+
25
+ if TYPE_CHECKING:
26
+ from airflow.providers.cncf.kubernetes.operators.pod import KubernetesPodOperator
27
+ from airflow.utils.context import Context
28
+
29
+ client_type: TypeAlias = k8s.CoreV1Api | async_k8s.CoreV1Api
30
+
31
+
32
+ class ExecutionMode(str, Enum):
33
+ """Enum class for execution mode."""
34
+
35
+ SYNC = "sync"
36
+ ASYNC = "async"
37
+
38
+
39
+ class KubernetesPodOperatorCallback:
40
+ """
41
+ `KubernetesPodOperator` callbacks methods.
42
+
43
+ Currently, the callbacks methods are not called in the async mode, this support will be added
44
+ in the future.
45
+ """
46
+
47
+ @staticmethod
48
+ def on_sync_client_creation(*, client: k8s.CoreV1Api, operator: KubernetesPodOperator, **kwargs) -> None:
49
+ """
50
+ Invoke this callback after creating the sync client.
51
+
52
+ :param client: the created `kubernetes.client.CoreV1Api` client.
53
+ """
54
+ pass
55
+
56
+ @staticmethod
57
+ def on_pod_manifest_created(
58
+ *,
59
+ pod_request: k8s.V1Pod,
60
+ client: client_type,
61
+ mode: str,
62
+ operator: KubernetesPodOperator,
63
+ context: Context,
64
+ **kwargs,
65
+ ) -> None:
66
+ """
67
+ Invoke this callback after KPO creates the V1Pod manifest but before the pod is created.
68
+
69
+ :param pod_request: the kubernetes pod manifest
70
+ :param client: the Kubernetes client that can be used in the callback.
71
+ :param mode: the current execution mode, it's one of (`sync`, `async`).
72
+ """
73
+ pass
74
+
75
+ @staticmethod
76
+ def on_pod_creation(
77
+ *,
78
+ pod: k8s.V1Pod,
79
+ client: client_type,
80
+ mode: str,
81
+ operator: KubernetesPodOperator,
82
+ context: Context,
83
+ **kwargs,
84
+ ) -> None:
85
+ """
86
+ Invoke this callback after creating the pod.
87
+
88
+ :param pod: the created pod.
89
+ :param client: the Kubernetes client that can be used in the callback.
90
+ :param mode: the current execution mode, it's one of (`sync`, `async`).
91
+ """
92
+ pass
93
+
94
+ @staticmethod
95
+ def on_pod_starting(
96
+ *,
97
+ pod: k8s.V1Pod,
98
+ client: client_type,
99
+ mode: str,
100
+ operator: KubernetesPodOperator,
101
+ context: Context,
102
+ **kwargs,
103
+ ) -> None:
104
+ """
105
+ Invoke this callback when the pod starts.
106
+
107
+ :param pod: the started pod.
108
+ :param client: the Kubernetes client that can be used in the callback.
109
+ :param mode: the current execution mode, it's one of (`sync`, `async`).
110
+ """
111
+ pass
112
+
113
+ @staticmethod
114
+ def on_pod_completion(
115
+ *,
116
+ pod: k8s.V1Pod,
117
+ client: client_type,
118
+ mode: str,
119
+ operator: KubernetesPodOperator,
120
+ context: Context,
121
+ **kwargs,
122
+ ) -> None:
123
+ """
124
+ Invoke this callback when the pod completes.
125
+
126
+ :param pod: the completed pod.
127
+ :param client: the Kubernetes client that can be used in the callback.
128
+ :param mode: the current execution mode, it's one of (`sync`, `async`).
129
+ """
130
+ pass
131
+
132
+ @staticmethod
133
+ def on_pod_teardown(
134
+ *,
135
+ pod: k8s.V1Pod,
136
+ client: client_type,
137
+ mode: str,
138
+ operator: KubernetesPodOperator,
139
+ context: Context,
140
+ **kwargs,
141
+ ) -> None:
142
+ """
143
+ Invoke this callback after all pod completion callbacks but before the pod is deleted.
144
+
145
+ :param pod: the completed pod.
146
+ :param client: the Kubernetes client that can be used in the callback.
147
+ :param mode: the current execution mode, it's one of (`sync`, `async`).
148
+ """
149
+ pass
150
+
151
+ @staticmethod
152
+ def on_pod_cleanup(
153
+ *,
154
+ pod: k8s.V1Pod,
155
+ client: client_type,
156
+ mode: str,
157
+ operator: KubernetesPodOperator,
158
+ context: Context,
159
+ **kwargs,
160
+ ):
161
+ """
162
+ Invoke this callback after cleaning/deleting the pod.
163
+
164
+ :param pod: the completed pod.
165
+ :param client: the Kubernetes client that can be used in the callback.
166
+ :param mode: the current execution mode, it's one of (`sync`, `async`).
167
+ """
168
+ pass
169
+
170
+ @staticmethod
171
+ def on_operator_resuming(
172
+ *,
173
+ pod: k8s.V1Pod,
174
+ event: dict,
175
+ client: client_type,
176
+ mode: str,
177
+ operator: KubernetesPodOperator,
178
+ context: Context,
179
+ **kwargs,
180
+ ) -> None:
181
+ """
182
+ Invoke this callback when resuming the `KubernetesPodOperator` from deferred state.
183
+
184
+ :param pod: the current state of the pod.
185
+ :param event: the returned event from the Trigger.
186
+ :param client: the Kubernetes client that can be used in the callback.
187
+ :param mode: the current execution mode, it's one of (`sync`, `async`).
188
+ """
189
+ pass
190
+
191
+ @staticmethod
192
+ def progress_callback(*, line: str, client: client_type, mode: str, **kwargs) -> None:
193
+ """
194
+ Invoke this callback to process pod container logs.
195
+
196
+ :param line: the read line of log.
197
+ :param client: the Kubernetes client that can be used in the callback.
198
+ :param mode: the current execution mode, it's one of (`sync`, `async`).
199
+ """
200
+ pass
@@ -0,0 +1,16 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
@@ -0,0 +1,195 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+ """Kubernetes sub-commands."""
18
+
19
+ from __future__ import annotations
20
+
21
+ import os
22
+ import sys
23
+ from datetime import datetime, timedelta
24
+
25
+ from kubernetes import client
26
+ from kubernetes.client.api_client import ApiClient
27
+ from kubernetes.client.rest import ApiException
28
+
29
+ from airflow.models import DagModel, DagRun, TaskInstance
30
+ from airflow.providers.cncf.kubernetes import pod_generator
31
+ from airflow.providers.cncf.kubernetes.executors.kubernetes_executor import KubeConfig
32
+ from airflow.providers.cncf.kubernetes.kube_client import get_kube_client
33
+ from airflow.providers.cncf.kubernetes.kubernetes_helper_functions import create_unique_id
34
+ from airflow.providers.cncf.kubernetes.pod_generator import PodGenerator, generate_pod_command_args
35
+ from airflow.providers.cncf.kubernetes.version_compat import AIRFLOW_V_3_0_PLUS, AIRFLOW_V_3_1_PLUS
36
+ from airflow.utils import cli as cli_utils, yaml
37
+ from airflow.utils.providers_configuration_loader import providers_configuration_loaded
38
+ from airflow.utils.types import DagRunType
39
+
40
+ if AIRFLOW_V_3_1_PLUS:
41
+ from airflow.utils.cli import get_bagged_dag
42
+ else:
43
+ from airflow.utils.cli import get_dag as get_bagged_dag # type: ignore[attr-defined,no-redef]
44
+
45
+
46
+ @cli_utils.action_cli
47
+ @providers_configuration_loaded
48
+ def generate_pod_yaml(args):
49
+ """Generate yaml files for each task in the DAG. Used for testing output of KubernetesExecutor."""
50
+ logical_date = args.logical_date if AIRFLOW_V_3_0_PLUS else args.execution_date
51
+ if AIRFLOW_V_3_0_PLUS:
52
+ dag = get_bagged_dag(bundle_names=args.bundle_name, dag_id=args.dag_id)
53
+ else:
54
+ dag = get_bagged_dag(subdir=args.subdir, dag_id=args.dag_id)
55
+ yaml_output_path = args.output_path
56
+
57
+ dm = DagModel(dag_id=dag.dag_id)
58
+
59
+ if AIRFLOW_V_3_0_PLUS:
60
+ dr = DagRun(dag.dag_id, logical_date=logical_date)
61
+ dr.run_id = DagRun.generate_run_id(
62
+ run_type=DagRunType.MANUAL, logical_date=logical_date, run_after=logical_date
63
+ )
64
+ dm.bundle_name = args.bundle_name if args.bundle_name else "default"
65
+ dm.relative_fileloc = dag.relative_fileloc
66
+ else:
67
+ dr = DagRun(dag.dag_id, execution_date=logical_date)
68
+ dr.run_id = DagRun.generate_run_id(run_type=DagRunType.MANUAL, execution_date=logical_date)
69
+
70
+ kube_config = KubeConfig()
71
+
72
+ for task in dag.tasks:
73
+ if AIRFLOW_V_3_0_PLUS:
74
+ from uuid6 import uuid7
75
+
76
+ ti = TaskInstance(task, run_id=dr.run_id, dag_version_id=uuid7())
77
+ else:
78
+ ti = TaskInstance(task, run_id=dr.run_id)
79
+ ti.dag_run = dr
80
+ ti.dag_model = dm
81
+
82
+ command_args = generate_pod_command_args(ti)
83
+ pod = PodGenerator.construct_pod(
84
+ dag_id=args.dag_id,
85
+ task_id=ti.task_id,
86
+ pod_id=create_unique_id(args.dag_id, ti.task_id),
87
+ try_number=ti.try_number,
88
+ kube_image=kube_config.kube_image,
89
+ date=ti.logical_date if AIRFLOW_V_3_0_PLUS else ti.execution_date,
90
+ args=command_args,
91
+ pod_override_object=PodGenerator.from_obj(ti.executor_config),
92
+ scheduler_job_id="worker-config",
93
+ namespace=kube_config.executor_namespace,
94
+ base_worker_pod=PodGenerator.deserialize_model_file(kube_config.pod_template_file),
95
+ with_mutation_hook=True,
96
+ )
97
+ api_client = ApiClient()
98
+ date_string = pod_generator.datetime_to_label_safe_datestring(logical_date)
99
+ yaml_file_name = f"{args.dag_id}_{ti.task_id}_{date_string}.yml"
100
+ os.makedirs(os.path.dirname(yaml_output_path + "/airflow_yaml_output/"), exist_ok=True)
101
+ with open(yaml_output_path + "/airflow_yaml_output/" + yaml_file_name, "w") as output:
102
+ sanitized_pod = api_client.sanitize_for_serialization(pod)
103
+ output.write(yaml.dump(sanitized_pod))
104
+ print(f"YAML output can be found at {yaml_output_path}/airflow_yaml_output/")
105
+
106
+
107
+ @cli_utils.action_cli(check_db=False)
108
+ @providers_configuration_loaded
109
+ def cleanup_pods(args):
110
+ """Clean up k8s pods in evicted/failed/succeeded/pending states."""
111
+ namespace = args.namespace
112
+
113
+ min_pending_minutes = args.min_pending_minutes
114
+ # protect newly created pods from deletion
115
+ if min_pending_minutes < 5:
116
+ min_pending_minutes = 5
117
+
118
+ # https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/
119
+ # All Containers in the Pod have terminated in success, and will not be restarted.
120
+ pod_succeeded = "succeeded"
121
+
122
+ # The Pod has been accepted by the Kubernetes cluster,
123
+ # but one or more of the containers has not been set up and made ready to run.
124
+ pod_pending = "pending"
125
+
126
+ # All Containers in the Pod have terminated, and at least one Container has terminated in failure.
127
+ # That is, the Container either exited with non-zero status or was terminated by the system.
128
+ pod_failed = "failed"
129
+
130
+ # https://kubernetes.io/docs/tasks/administer-cluster/out-of-resource/
131
+ pod_reason_evicted = "evicted"
132
+ # If pod is failed and restartPolicy is:
133
+ # * Always: Restart Container; Pod phase stays Running.
134
+ # * OnFailure: Restart Container; Pod phase stays Running.
135
+ # * Never: Pod phase becomes Failed.
136
+ pod_restart_policy_never = "never"
137
+
138
+ print("Loading Kubernetes configuration")
139
+ kube_client = get_kube_client()
140
+ print(f"Listing pods in namespace {namespace}")
141
+ airflow_pod_labels = [
142
+ "dag_id",
143
+ "task_id",
144
+ "try_number",
145
+ "airflow_version",
146
+ ]
147
+ list_kwargs = {"namespace": namespace, "limit": 500, "label_selector": ",".join(airflow_pod_labels)}
148
+
149
+ while True:
150
+ pod_list = kube_client.list_namespaced_pod(**list_kwargs)
151
+ for pod in pod_list.items:
152
+ pod_name = pod.metadata.name
153
+ print(f"Inspecting pod {pod_name}")
154
+ pod_phase = pod.status.phase.lower()
155
+ pod_reason = pod.status.reason.lower() if pod.status.reason else ""
156
+ pod_restart_policy = pod.spec.restart_policy.lower()
157
+ current_time = datetime.now(pod.metadata.creation_timestamp.tzinfo)
158
+
159
+ if (
160
+ pod_phase == pod_succeeded
161
+ or (pod_phase == pod_failed and pod_restart_policy == pod_restart_policy_never)
162
+ or (pod_reason == pod_reason_evicted)
163
+ or (
164
+ pod_phase == pod_pending
165
+ and current_time - pod.metadata.creation_timestamp
166
+ > timedelta(minutes=min_pending_minutes)
167
+ )
168
+ ):
169
+ print(
170
+ f'Deleting pod "{pod_name}" phase "{pod_phase}" and reason "{pod_reason}", '
171
+ f'restart policy "{pod_restart_policy}"'
172
+ )
173
+ try:
174
+ _delete_pod(pod.metadata.name, namespace)
175
+ except ApiException as e:
176
+ print(f"Can't remove POD: {e}", file=sys.stderr)
177
+ else:
178
+ print(f"No action taken on pod {pod_name}")
179
+ continue_token = pod_list.metadata._continue
180
+ if not continue_token:
181
+ break
182
+ list_kwargs["_continue"] = continue_token
183
+
184
+
185
+ def _delete_pod(name, namespace):
186
+ """
187
+ Delete a namespaced pod.
188
+
189
+ Helper Function for cleanup_pods.
190
+ """
191
+ kube_client = get_kube_client()
192
+ delete_options = client.V1DeleteOptions()
193
+ print(f'Deleting POD "{name}" from "{namespace}" namespace')
194
+ api_response = kube_client.delete_namespaced_pod(name=name, namespace=namespace, body=delete_options)
195
+ print(api_response)