apache-airflow-providers-cncf-kubernetes 7.9.0__tar.gz → 7.10.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {apache-airflow-providers-cncf-kubernetes-7.9.0/apache_airflow_providers_cncf_kubernetes.egg-info → apache_airflow_providers_cncf_kubernetes-7.10.0}/PKG-INFO +32 -22
- {apache-airflow-providers-cncf-kubernetes-7.9.0 → apache_airflow_providers_cncf_kubernetes-7.10.0}/README.rst +11 -4
- {apache-airflow-providers-cncf-kubernetes-7.9.0 → apache_airflow_providers_cncf_kubernetes-7.10.0/airflow/providers/cncf/kubernetes}/LICENSE +52 -0
- {apache-airflow-providers-cncf-kubernetes-7.9.0 → apache_airflow_providers_cncf_kubernetes-7.10.0}/airflow/providers/cncf/kubernetes/__init__.py +4 -5
- {apache-airflow-providers-cncf-kubernetes-7.9.0 → apache_airflow_providers_cncf_kubernetes-7.10.0}/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py +109 -106
- {apache-airflow-providers-cncf-kubernetes-7.9.0 → apache_airflow_providers_cncf_kubernetes-7.10.0}/airflow/providers/cncf/kubernetes/get_provider_info.py +4 -2
- apache_airflow_providers_cncf_kubernetes-7.10.0/airflow/providers/cncf/kubernetes/kubernetes_executor_templates/basic_template.yaml +79 -0
- {apache-airflow-providers-cncf-kubernetes-7.9.0 → apache_airflow_providers_cncf_kubernetes-7.10.0}/airflow/providers/cncf/kubernetes/operators/pod.py +2 -1
- {apache-airflow-providers-cncf-kubernetes-7.9.0 → apache_airflow_providers_cncf_kubernetes-7.10.0}/airflow/providers/cncf/kubernetes/operators/resource.py +51 -16
- {apache-airflow-providers-cncf-kubernetes-7.9.0 → apache_airflow_providers_cncf_kubernetes-7.10.0}/airflow/providers/cncf/kubernetes/pod_generator.py +2 -2
- apache_airflow_providers_cncf_kubernetes-7.10.0/airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_image_template.yaml +68 -0
- apache_airflow_providers_cncf_kubernetes-7.10.0/airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_volume_template.yaml +74 -0
- apache_airflow_providers_cncf_kubernetes-7.10.0/airflow/providers/cncf/kubernetes/pod_template_file_examples/git_sync_template.yaml +95 -0
- {apache-airflow-providers-cncf-kubernetes-7.9.0 → apache_airflow_providers_cncf_kubernetes-7.10.0}/airflow/providers/cncf/kubernetes/python_kubernetes_script.py +6 -1
- apache_airflow_providers_cncf_kubernetes-7.10.0/airflow/providers/cncf/kubernetes/utils/k8s_resource_iterator.py +46 -0
- {apache-airflow-providers-cncf-kubernetes-7.9.0 → apache_airflow_providers_cncf_kubernetes-7.10.0}/airflow/providers/cncf/kubernetes/utils/pod_manager.py +27 -25
- apache_airflow_providers_cncf_kubernetes-7.10.0/pyproject.toml +80 -0
- apache-airflow-providers-cncf-kubernetes-7.9.0/MANIFEST.in +0 -33
- apache-airflow-providers-cncf-kubernetes-7.9.0/NOTICE +0 -6
- apache-airflow-providers-cncf-kubernetes-7.9.0/PKG-INFO +0 -112
- apache-airflow-providers-cncf-kubernetes-7.9.0/apache_airflow_providers_cncf_kubernetes.egg-info/SOURCES.txt +0 -55
- apache-airflow-providers-cncf-kubernetes-7.9.0/apache_airflow_providers_cncf_kubernetes.egg-info/dependency_links.txt +0 -1
- apache-airflow-providers-cncf-kubernetes-7.9.0/apache_airflow_providers_cncf_kubernetes.egg-info/entry_points.txt +0 -2
- apache-airflow-providers-cncf-kubernetes-7.9.0/apache_airflow_providers_cncf_kubernetes.egg-info/not-zip-safe +0 -1
- apache-airflow-providers-cncf-kubernetes-7.9.0/apache_airflow_providers_cncf_kubernetes.egg-info/requires.txt +0 -7
- apache-airflow-providers-cncf-kubernetes-7.9.0/apache_airflow_providers_cncf_kubernetes.egg-info/top_level.txt +0 -1
- apache-airflow-providers-cncf-kubernetes-7.9.0/pyproject.toml +0 -185
- apache-airflow-providers-cncf-kubernetes-7.9.0/setup.cfg +0 -68
- apache-airflow-providers-cncf-kubernetes-7.9.0/setup.py +0 -49
- {apache-airflow-providers-cncf-kubernetes-7.9.0 → apache_airflow_providers_cncf_kubernetes-7.10.0}/airflow/providers/cncf/kubernetes/backcompat/__init__.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.9.0 → apache_airflow_providers_cncf_kubernetes-7.10.0}/airflow/providers/cncf/kubernetes/backcompat/backwards_compat_converters.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.9.0 → apache_airflow_providers_cncf_kubernetes-7.10.0}/airflow/providers/cncf/kubernetes/decorators/__init__.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.9.0 → apache_airflow_providers_cncf_kubernetes-7.10.0}/airflow/providers/cncf/kubernetes/decorators/kubernetes.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.9.0 → apache_airflow_providers_cncf_kubernetes-7.10.0}/airflow/providers/cncf/kubernetes/executors/__init__.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.9.0 → apache_airflow_providers_cncf_kubernetes-7.10.0}/airflow/providers/cncf/kubernetes/executors/kubernetes_executor_types.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.9.0 → apache_airflow_providers_cncf_kubernetes-7.10.0}/airflow/providers/cncf/kubernetes/executors/kubernetes_executor_utils.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.9.0 → apache_airflow_providers_cncf_kubernetes-7.10.0}/airflow/providers/cncf/kubernetes/executors/local_kubernetes_executor.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.9.0 → apache_airflow_providers_cncf_kubernetes-7.10.0}/airflow/providers/cncf/kubernetes/hooks/__init__.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.9.0 → apache_airflow_providers_cncf_kubernetes-7.10.0}/airflow/providers/cncf/kubernetes/hooks/kubernetes.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.9.0 → apache_airflow_providers_cncf_kubernetes-7.10.0}/airflow/providers/cncf/kubernetes/k8s_model.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.9.0 → apache_airflow_providers_cncf_kubernetes-7.10.0}/airflow/providers/cncf/kubernetes/kube_client.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.9.0 → apache_airflow_providers_cncf_kubernetes-7.10.0}/airflow/providers/cncf/kubernetes/kube_config.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.9.0 → apache_airflow_providers_cncf_kubernetes-7.10.0}/airflow/providers/cncf/kubernetes/kubernetes_executor_templates/__init__.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.9.0 → apache_airflow_providers_cncf_kubernetes-7.10.0}/airflow/providers/cncf/kubernetes/kubernetes_helper_functions.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.9.0 → apache_airflow_providers_cncf_kubernetes-7.10.0}/airflow/providers/cncf/kubernetes/operators/__init__.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.9.0 → apache_airflow_providers_cncf_kubernetes-7.10.0}/airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.9.0 → apache_airflow_providers_cncf_kubernetes-7.10.0}/airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.9.0 → apache_airflow_providers_cncf_kubernetes-7.10.0}/airflow/providers/cncf/kubernetes/pod_generator_deprecated.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.9.0 → apache_airflow_providers_cncf_kubernetes-7.10.0}/airflow/providers/cncf/kubernetes/pod_launcher_deprecated.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.9.0 → apache_airflow_providers_cncf_kubernetes-7.10.0}/airflow/providers/cncf/kubernetes/pod_template_file_examples/__init__.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.9.0 → apache_airflow_providers_cncf_kubernetes-7.10.0}/airflow/providers/cncf/kubernetes/python_kubernetes_script.jinja2 +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.9.0 → apache_airflow_providers_cncf_kubernetes-7.10.0}/airflow/providers/cncf/kubernetes/secret.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.9.0 → apache_airflow_providers_cncf_kubernetes-7.10.0}/airflow/providers/cncf/kubernetes/sensors/__init__.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.9.0 → apache_airflow_providers_cncf_kubernetes-7.10.0}/airflow/providers/cncf/kubernetes/sensors/spark_kubernetes.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.9.0 → apache_airflow_providers_cncf_kubernetes-7.10.0}/airflow/providers/cncf/kubernetes/template_rendering.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.9.0 → apache_airflow_providers_cncf_kubernetes-7.10.0}/airflow/providers/cncf/kubernetes/triggers/__init__.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.9.0 → apache_airflow_providers_cncf_kubernetes-7.10.0}/airflow/providers/cncf/kubernetes/triggers/kubernetes_pod.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.9.0 → apache_airflow_providers_cncf_kubernetes-7.10.0}/airflow/providers/cncf/kubernetes/triggers/pod.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.9.0 → apache_airflow_providers_cncf_kubernetes-7.10.0}/airflow/providers/cncf/kubernetes/utils/__init__.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.9.0 → apache_airflow_providers_cncf_kubernetes-7.10.0}/airflow/providers/cncf/kubernetes/utils/delete_from.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.9.0 → apache_airflow_providers_cncf_kubernetes-7.10.0}/airflow/providers/cncf/kubernetes/utils/k8s_hashlib_wrapper.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.9.0 → apache_airflow_providers_cncf_kubernetes-7.10.0}/airflow/providers/cncf/kubernetes/utils/xcom_sidecar.py +0 -0
|
@@ -1,19 +1,12 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: apache-airflow-providers-cncf-kubernetes
|
|
3
|
-
Version: 7.
|
|
4
|
-
Summary: Provider
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/7.9.0/
|
|
11
|
-
Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/7.9.0/changelog.html
|
|
12
|
-
Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
|
|
13
|
-
Project-URL: Source Code, https://github.com/apache/airflow
|
|
14
|
-
Project-URL: Slack Chat, https://s.apache.org/airflow-slack
|
|
15
|
-
Project-URL: Twitter, https://twitter.com/ApacheAirflow
|
|
16
|
-
Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
|
|
3
|
+
Version: 7.10.0
|
|
4
|
+
Summary: Provider package apache-airflow-providers-cncf-kubernetes for Apache Airflow
|
|
5
|
+
Keywords: airflow-provider,cncf.kubernetes,airflow,integration
|
|
6
|
+
Author-email: Apache Software Foundation <dev@airflow.apache.org>
|
|
7
|
+
Maintainer-email: Apache Software Foundation <dev@airflow.apache.org>
|
|
8
|
+
Requires-Python: ~=3.8
|
|
9
|
+
Description-Content-Type: text/x-rst
|
|
17
10
|
Classifier: Development Status :: 5 - Production/Stable
|
|
18
11
|
Classifier: Environment :: Console
|
|
19
12
|
Classifier: Environment :: Web Environment
|
|
@@ -27,10 +20,20 @@ Classifier: Programming Language :: Python :: 3.9
|
|
|
27
20
|
Classifier: Programming Language :: Python :: 3.10
|
|
28
21
|
Classifier: Programming Language :: Python :: 3.11
|
|
29
22
|
Classifier: Topic :: System :: Monitoring
|
|
30
|
-
Requires-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
23
|
+
Requires-Dist: aiofiles>=23.2.0
|
|
24
|
+
Requires-Dist: apache-airflow>=2.5.0
|
|
25
|
+
Requires-Dist: asgiref>=3.5.2
|
|
26
|
+
Requires-Dist: cryptography>=2.0.0
|
|
27
|
+
Requires-Dist: google-re2>=1.0
|
|
28
|
+
Requires-Dist: kubernetes>=21.7.0,<24
|
|
29
|
+
Requires-Dist: kubernetes_asyncio>=18.20.1,<25
|
|
30
|
+
Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
|
|
31
|
+
Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/7.10.0/changelog.html
|
|
32
|
+
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/7.10.0
|
|
33
|
+
Project-URL: Slack Chat, https://s.apache.org/airflow-slack
|
|
34
|
+
Project-URL: Source Code, https://github.com/apache/airflow
|
|
35
|
+
Project-URL: Twitter, https://twitter.com/ApacheAirflow
|
|
36
|
+
Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
|
|
34
37
|
|
|
35
38
|
|
|
36
39
|
.. Licensed to the Apache Software Foundation (ASF) under one
|
|
@@ -49,7 +52,8 @@ License-File: NOTICE
|
|
|
49
52
|
KIND, either express or implied. See the License for the
|
|
50
53
|
specific language governing permissions and limitations
|
|
51
54
|
under the License.
|
|
52
|
-
|
|
55
|
+
|
|
56
|
+
.. Licensed to the Apache Software Foundation (ASF) under one
|
|
53
57
|
or more contributor license agreements. See the NOTICE file
|
|
54
58
|
distributed with this work for additional information
|
|
55
59
|
regarding copyright ownership. The ASF licenses this file
|
|
@@ -66,10 +70,16 @@ License-File: NOTICE
|
|
|
66
70
|
specific language governing permissions and limitations
|
|
67
71
|
under the License.
|
|
68
72
|
|
|
73
|
+
.. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
|
|
74
|
+
OVERWRITTEN WHEN PREPARING PACKAGES.
|
|
75
|
+
|
|
76
|
+
.. IF YOU WANT TO MODIFY TEMPLATE FOR THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
|
|
77
|
+
`PROVIDER_README_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
|
|
78
|
+
|
|
69
79
|
|
|
70
80
|
Package ``apache-airflow-providers-cncf-kubernetes``
|
|
71
81
|
|
|
72
|
-
Release: ``7.
|
|
82
|
+
Release: ``7.10.0``
|
|
73
83
|
|
|
74
84
|
|
|
75
85
|
`Kubernetes <https://kubernetes.io/>`__
|
|
@@ -82,7 +92,7 @@ This is a provider package for ``cncf.kubernetes`` provider. All classes for thi
|
|
|
82
92
|
are in ``airflow.providers.cncf.kubernetes`` python package.
|
|
83
93
|
|
|
84
94
|
You can find package information and changelog for the provider
|
|
85
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/7.
|
|
95
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/7.10.0/>`_.
|
|
86
96
|
|
|
87
97
|
Installation
|
|
88
98
|
------------
|
|
@@ -109,4 +119,4 @@ PIP package Version required
|
|
|
109
119
|
====================== ==================
|
|
110
120
|
|
|
111
121
|
The changelog for the provider package can be found in the
|
|
112
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/7.
|
|
122
|
+
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/7.10.0/changelog.html>`_.
|
|
@@ -15,7 +15,8 @@
|
|
|
15
15
|
KIND, either express or implied. See the License for the
|
|
16
16
|
specific language governing permissions and limitations
|
|
17
17
|
under the License.
|
|
18
|
-
|
|
18
|
+
|
|
19
|
+
.. Licensed to the Apache Software Foundation (ASF) under one
|
|
19
20
|
or more contributor license agreements. See the NOTICE file
|
|
20
21
|
distributed with this work for additional information
|
|
21
22
|
regarding copyright ownership. The ASF licenses this file
|
|
@@ -32,10 +33,16 @@
|
|
|
32
33
|
specific language governing permissions and limitations
|
|
33
34
|
under the License.
|
|
34
35
|
|
|
36
|
+
.. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
|
|
37
|
+
OVERWRITTEN WHEN PREPARING PACKAGES.
|
|
38
|
+
|
|
39
|
+
.. IF YOU WANT TO MODIFY TEMPLATE FOR THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
|
|
40
|
+
`PROVIDER_README_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
|
|
41
|
+
|
|
35
42
|
|
|
36
43
|
Package ``apache-airflow-providers-cncf-kubernetes``
|
|
37
44
|
|
|
38
|
-
Release: ``7.
|
|
45
|
+
Release: ``7.10.0``
|
|
39
46
|
|
|
40
47
|
|
|
41
48
|
`Kubernetes <https://kubernetes.io/>`__
|
|
@@ -48,7 +55,7 @@ This is a provider package for ``cncf.kubernetes`` provider. All classes for thi
|
|
|
48
55
|
are in ``airflow.providers.cncf.kubernetes`` python package.
|
|
49
56
|
|
|
50
57
|
You can find package information and changelog for the provider
|
|
51
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/7.
|
|
58
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/7.10.0/>`_.
|
|
52
59
|
|
|
53
60
|
Installation
|
|
54
61
|
------------
|
|
@@ -75,4 +82,4 @@ PIP package Version required
|
|
|
75
82
|
====================== ==================
|
|
76
83
|
|
|
77
84
|
The changelog for the provider package can be found in the
|
|
78
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/7.
|
|
85
|
+
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/7.10.0/changelog.html>`_.
|
|
@@ -199,3 +199,55 @@ distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
199
199
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
200
200
|
See the License for the specific language governing permissions and
|
|
201
201
|
limitations under the License.
|
|
202
|
+
|
|
203
|
+
============================================================================
|
|
204
|
+
APACHE AIRFLOW SUBCOMPONENTS:
|
|
205
|
+
|
|
206
|
+
The Apache Airflow project contains subcomponents with separate copyright
|
|
207
|
+
notices and license terms. Your use of the source code for the these
|
|
208
|
+
subcomponents is subject to the terms and conditions of the following
|
|
209
|
+
licenses.
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
========================================================================
|
|
213
|
+
Third party Apache 2.0 licenses
|
|
214
|
+
========================================================================
|
|
215
|
+
|
|
216
|
+
The following components are provided under the Apache 2.0 License.
|
|
217
|
+
See project link for details. The text of each license is also included
|
|
218
|
+
at licenses/LICENSE-[project].txt.
|
|
219
|
+
|
|
220
|
+
(ALv2 License) hue v4.3.0 (https://github.com/cloudera/hue/)
|
|
221
|
+
(ALv2 License) jqclock v2.3.0 (https://github.com/JohnRDOrazio/jQuery-Clock-Plugin)
|
|
222
|
+
(ALv2 License) bootstrap3-typeahead v4.0.2 (https://github.com/bassjobsen/Bootstrap-3-Typeahead)
|
|
223
|
+
(ALv2 License) connexion v2.7.0 (https://github.com/zalando/connexion)
|
|
224
|
+
|
|
225
|
+
========================================================================
|
|
226
|
+
MIT licenses
|
|
227
|
+
========================================================================
|
|
228
|
+
|
|
229
|
+
The following components are provided under the MIT License. See project link for details.
|
|
230
|
+
The text of each license is also included at licenses/LICENSE-[project].txt.
|
|
231
|
+
|
|
232
|
+
(MIT License) jquery v3.5.1 (https://jquery.org/license/)
|
|
233
|
+
(MIT License) dagre-d3 v0.6.4 (https://github.com/cpettitt/dagre-d3)
|
|
234
|
+
(MIT License) bootstrap v3.4.1 (https://github.com/twbs/bootstrap/)
|
|
235
|
+
(MIT License) d3-tip v0.9.1 (https://github.com/Caged/d3-tip)
|
|
236
|
+
(MIT License) dataTables v1.10.25 (https://datatables.net)
|
|
237
|
+
(MIT License) normalize.css v3.0.2 (http://necolas.github.io/normalize.css/)
|
|
238
|
+
(MIT License) ElasticMock v1.3.2 (https://github.com/vrcmarcos/elasticmock)
|
|
239
|
+
(MIT License) MomentJS v2.24.0 (http://momentjs.com/)
|
|
240
|
+
(MIT License) eonasdan-bootstrap-datetimepicker v4.17.49 (https://github.com/eonasdan/bootstrap-datetimepicker/)
|
|
241
|
+
|
|
242
|
+
========================================================================
|
|
243
|
+
BSD 3-Clause licenses
|
|
244
|
+
========================================================================
|
|
245
|
+
The following components are provided under the BSD 3-Clause license. See project links for details.
|
|
246
|
+
The text of each license is also included at licenses/LICENSE-[project].txt.
|
|
247
|
+
|
|
248
|
+
(BSD 3 License) d3 v5.16.0 (https://d3js.org)
|
|
249
|
+
(BSD 3 License) d3-shape v2.1.0 (https://github.com/d3/d3-shape)
|
|
250
|
+
(BSD 3 License) cgroupspy 0.2.1 (https://github.com/cloudsigma/cgroupspy)
|
|
251
|
+
|
|
252
|
+
========================================================================
|
|
253
|
+
See licenses/LICENSES-ui.txt for packages used in `/airflow/www`
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
#
|
|
2
1
|
# Licensed to the Apache Software Foundation (ASF) under one
|
|
3
2
|
# or more contributor license agreements. See the NOTICE file
|
|
4
3
|
# distributed with this work for additional information
|
|
@@ -19,8 +18,8 @@
|
|
|
19
18
|
# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
|
|
20
19
|
# OVERWRITTEN WHEN PREPARING DOCUMENTATION FOR THE PACKAGES.
|
|
21
20
|
#
|
|
22
|
-
# IF YOU WANT TO MODIFY
|
|
23
|
-
# `PROVIDER__INIT__PY_TEMPLATE.py.jinja2` IN the `dev/
|
|
21
|
+
# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
|
|
22
|
+
# `PROVIDER__INIT__PY_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
|
|
24
23
|
#
|
|
25
24
|
from __future__ import annotations
|
|
26
25
|
|
|
@@ -28,7 +27,7 @@ import packaging.version
|
|
|
28
27
|
|
|
29
28
|
__all__ = ["__version__"]
|
|
30
29
|
|
|
31
|
-
__version__ = "7.
|
|
30
|
+
__version__ = "7.10.0"
|
|
32
31
|
|
|
33
32
|
try:
|
|
34
33
|
from airflow import __version__ as airflow_version
|
|
@@ -39,5 +38,5 @@ if packaging.version.parse(packaging.version.parse(airflow_version).base_version
|
|
|
39
38
|
"2.5.0"
|
|
40
39
|
):
|
|
41
40
|
raise RuntimeError(
|
|
42
|
-
f"The package `apache-airflow-providers-cncf-kubernetes:{__version__}`
|
|
41
|
+
f"The package `apache-airflow-providers-cncf-kubernetes:{__version__}` needs Apache Airflow 2.5.0+"
|
|
43
42
|
)
|
|
@@ -37,6 +37,7 @@ from typing import TYPE_CHECKING, Any, Sequence
|
|
|
37
37
|
from sqlalchemy import select, update
|
|
38
38
|
|
|
39
39
|
from airflow.providers.cncf.kubernetes.pod_generator import PodMutationHookException, PodReconciliationError
|
|
40
|
+
from airflow.stats import Stats
|
|
40
41
|
|
|
41
42
|
try:
|
|
42
43
|
from airflow.cli.cli_config import (
|
|
@@ -209,89 +210,90 @@ class KubernetesExecutor(BaseExecutor):
|
|
|
209
210
|
assert self.kube_client
|
|
210
211
|
from airflow.models.taskinstance import TaskInstance
|
|
211
212
|
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
if self.kubernetes_queue:
|
|
217
|
-
query = query.where(TaskInstance.queue == self.kubernetes_queue)
|
|
218
|
-
queued_tis: list[TaskInstance] = session.scalars(query).all()
|
|
219
|
-
self.log.info("Found %s queued task instances", len(queued_tis))
|
|
220
|
-
|
|
221
|
-
# Go through the "last seen" dictionary and clean out old entries
|
|
222
|
-
allowed_age = self.kube_config.worker_pods_queued_check_interval * 3
|
|
223
|
-
for key, timestamp in list(self.last_handled.items()):
|
|
224
|
-
if time.time() - timestamp > allowed_age:
|
|
225
|
-
del self.last_handled[key]
|
|
226
|
-
|
|
227
|
-
if not queued_tis:
|
|
228
|
-
return
|
|
229
|
-
|
|
230
|
-
# airflow worker label selector batch call
|
|
231
|
-
kwargs = {"label_selector": f"airflow-worker={self._make_safe_label_value(str(self.job_id))}"}
|
|
232
|
-
if self.kube_config.kube_client_request_args:
|
|
233
|
-
kwargs.update(self.kube_config.kube_client_request_args)
|
|
234
|
-
pod_list = self._list_pods(kwargs)
|
|
235
|
-
|
|
236
|
-
# create a set against pod query label fields
|
|
237
|
-
label_search_set = set()
|
|
238
|
-
for pod in pod_list:
|
|
239
|
-
dag_id = pod.metadata.labels.get("dag_id", None)
|
|
240
|
-
task_id = pod.metadata.labels.get("task_id", None)
|
|
241
|
-
airflow_worker = pod.metadata.labels.get("airflow-worker", None)
|
|
242
|
-
map_index = pod.metadata.labels.get("map_index", None)
|
|
243
|
-
run_id = pod.metadata.labels.get("run_id", None)
|
|
244
|
-
execution_date = pod.metadata.labels.get("execution_date", None)
|
|
245
|
-
if dag_id is None or task_id is None or airflow_worker is None:
|
|
246
|
-
continue
|
|
247
|
-
label_search_base_str = f"dag_id={dag_id},task_id={task_id},airflow-worker={airflow_worker}"
|
|
248
|
-
if map_index is not None:
|
|
249
|
-
label_search_base_str += f",map_index={map_index}"
|
|
250
|
-
if run_id is not None:
|
|
251
|
-
label_search_str = f"{label_search_base_str},run_id={run_id}"
|
|
252
|
-
label_search_set.add(label_search_str)
|
|
253
|
-
if execution_date is not None:
|
|
254
|
-
label_search_str = f"{label_search_base_str},execution_date={execution_date}"
|
|
255
|
-
label_search_set.add(label_search_str)
|
|
256
|
-
|
|
257
|
-
for ti in queued_tis:
|
|
258
|
-
self.log.debug("Checking task instance %s", ti)
|
|
259
|
-
|
|
260
|
-
# Check to see if we've handled it ourselves recently
|
|
261
|
-
if ti.key in self.last_handled:
|
|
262
|
-
continue
|
|
263
|
-
|
|
264
|
-
# Build the pod selector
|
|
265
|
-
base_label_selector = (
|
|
266
|
-
f"dag_id={self._make_safe_label_value(ti.dag_id)},"
|
|
267
|
-
f"task_id={self._make_safe_label_value(ti.task_id)},"
|
|
268
|
-
f"airflow-worker={self._make_safe_label_value(str(ti.queued_by_job_id))}"
|
|
269
|
-
)
|
|
270
|
-
if ti.map_index >= 0:
|
|
271
|
-
# Old tasks _couldn't_ be mapped, so we don't have to worry about compat
|
|
272
|
-
base_label_selector += f",map_index={ti.map_index}"
|
|
273
|
-
|
|
274
|
-
# Try run_id first
|
|
275
|
-
label_search_str = f"{base_label_selector},run_id={self._make_safe_label_value(ti.run_id)}"
|
|
276
|
-
if label_search_str in label_search_set:
|
|
277
|
-
continue
|
|
278
|
-
# Fallback to old style of using execution_date
|
|
279
|
-
label_search_str = (
|
|
280
|
-
f"{base_label_selector},execution_date={self._make_safe_label_value(ti.execution_date)}"
|
|
213
|
+
with Stats.timer("kubernetes_executor.clear_not_launched_queued_tasks.duration"):
|
|
214
|
+
self.log.debug("Clearing tasks that have not been launched")
|
|
215
|
+
query = select(TaskInstance).where(
|
|
216
|
+
TaskInstance.state == TaskInstanceState.QUEUED, TaskInstance.queued_by_job_id == self.job_id
|
|
281
217
|
)
|
|
282
|
-
if
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
218
|
+
if self.kubernetes_queue:
|
|
219
|
+
query = query.where(TaskInstance.queue == self.kubernetes_queue)
|
|
220
|
+
queued_tis: list[TaskInstance] = session.scalars(query).all()
|
|
221
|
+
self.log.info("Found %s queued task instances", len(queued_tis))
|
|
222
|
+
|
|
223
|
+
# Go through the "last seen" dictionary and clean out old entries
|
|
224
|
+
allowed_age = self.kube_config.worker_pods_queued_check_interval * 3
|
|
225
|
+
for key, timestamp in list(self.last_handled.items()):
|
|
226
|
+
if time.time() - timestamp > allowed_age:
|
|
227
|
+
del self.last_handled[key]
|
|
228
|
+
|
|
229
|
+
if not queued_tis:
|
|
230
|
+
return
|
|
231
|
+
|
|
232
|
+
# airflow worker label selector batch call
|
|
233
|
+
kwargs = {"label_selector": f"airflow-worker={self._make_safe_label_value(str(self.job_id))}"}
|
|
234
|
+
if self.kube_config.kube_client_request_args:
|
|
235
|
+
kwargs.update(self.kube_config.kube_client_request_args)
|
|
236
|
+
pod_list = self._list_pods(kwargs)
|
|
237
|
+
|
|
238
|
+
# create a set against pod query label fields
|
|
239
|
+
label_search_set = set()
|
|
240
|
+
for pod in pod_list:
|
|
241
|
+
dag_id = pod.metadata.labels.get("dag_id", None)
|
|
242
|
+
task_id = pod.metadata.labels.get("task_id", None)
|
|
243
|
+
airflow_worker = pod.metadata.labels.get("airflow-worker", None)
|
|
244
|
+
map_index = pod.metadata.labels.get("map_index", None)
|
|
245
|
+
run_id = pod.metadata.labels.get("run_id", None)
|
|
246
|
+
execution_date = pod.metadata.labels.get("execution_date", None)
|
|
247
|
+
if dag_id is None or task_id is None or airflow_worker is None:
|
|
248
|
+
continue
|
|
249
|
+
label_search_base_str = f"dag_id={dag_id},task_id={task_id},airflow-worker={airflow_worker}"
|
|
250
|
+
if map_index is not None:
|
|
251
|
+
label_search_base_str += f",map_index={map_index}"
|
|
252
|
+
if run_id is not None:
|
|
253
|
+
label_search_str = f"{label_search_base_str},run_id={run_id}"
|
|
254
|
+
label_search_set.add(label_search_str)
|
|
255
|
+
if execution_date is not None:
|
|
256
|
+
label_search_str = f"{label_search_base_str},execution_date={execution_date}"
|
|
257
|
+
label_search_set.add(label_search_str)
|
|
258
|
+
|
|
259
|
+
for ti in queued_tis:
|
|
260
|
+
self.log.debug("Checking task instance %s", ti)
|
|
261
|
+
|
|
262
|
+
# Check to see if we've handled it ourselves recently
|
|
263
|
+
if ti.key in self.last_handled:
|
|
264
|
+
continue
|
|
265
|
+
|
|
266
|
+
# Build the pod selector
|
|
267
|
+
base_label_selector = (
|
|
268
|
+
f"dag_id={self._make_safe_label_value(ti.dag_id)},"
|
|
269
|
+
f"task_id={self._make_safe_label_value(ti.task_id)},"
|
|
270
|
+
f"airflow-worker={self._make_safe_label_value(str(ti.queued_by_job_id))}"
|
|
271
|
+
)
|
|
272
|
+
if ti.map_index >= 0:
|
|
273
|
+
# Old tasks _couldn't_ be mapped, so we don't have to worry about compat
|
|
274
|
+
base_label_selector += f",map_index={ti.map_index}"
|
|
275
|
+
|
|
276
|
+
# Try run_id first
|
|
277
|
+
label_search_str = f"{base_label_selector},run_id={self._make_safe_label_value(ti.run_id)}"
|
|
278
|
+
if label_search_str in label_search_set:
|
|
279
|
+
continue
|
|
280
|
+
# Fallback to old style of using execution_date
|
|
281
|
+
label_search_str = (
|
|
282
|
+
f"{base_label_selector},execution_date={self._make_safe_label_value(ti.execution_date)}"
|
|
283
|
+
)
|
|
284
|
+
if label_search_str in label_search_set:
|
|
285
|
+
continue
|
|
286
|
+
self.log.info("TaskInstance: %s found in queued state but was not launched, rescheduling", ti)
|
|
287
|
+
session.execute(
|
|
288
|
+
update(TaskInstance)
|
|
289
|
+
.where(
|
|
290
|
+
TaskInstance.dag_id == ti.dag_id,
|
|
291
|
+
TaskInstance.task_id == ti.task_id,
|
|
292
|
+
TaskInstance.run_id == ti.run_id,
|
|
293
|
+
TaskInstance.map_index == ti.map_index,
|
|
294
|
+
)
|
|
295
|
+
.values(state=TaskInstanceState.SCHEDULED)
|
|
292
296
|
)
|
|
293
|
-
.values(state=TaskInstanceState.SCHEDULED)
|
|
294
|
-
)
|
|
295
297
|
|
|
296
298
|
def start(self) -> None:
|
|
297
299
|
"""Start the executor."""
|
|
@@ -534,31 +536,32 @@ class KubernetesExecutor(BaseExecutor):
|
|
|
534
536
|
return messages, ["\n".join(log)]
|
|
535
537
|
|
|
536
538
|
def try_adopt_task_instances(self, tis: Sequence[TaskInstance]) -> Sequence[TaskInstance]:
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
scheduler_job_id
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
"
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
539
|
+
with Stats.timer("kubernetes_executor.adopt_task_instances.duration"):
|
|
540
|
+
# Always flush TIs without queued_by_job_id
|
|
541
|
+
tis_to_flush = [ti for ti in tis if not ti.queued_by_job_id]
|
|
542
|
+
scheduler_job_ids = {ti.queued_by_job_id for ti in tis}
|
|
543
|
+
tis_to_flush_by_key = {ti.key: ti for ti in tis if ti.queued_by_job_id}
|
|
544
|
+
kube_client: client.CoreV1Api = self.kube_client
|
|
545
|
+
for scheduler_job_id in scheduler_job_ids:
|
|
546
|
+
scheduler_job_id = self._make_safe_label_value(str(scheduler_job_id))
|
|
547
|
+
# We will look for any pods owned by the no-longer-running scheduler,
|
|
548
|
+
# but will exclude only successful pods, as those TIs will have a terminal state
|
|
549
|
+
# and not be up for adoption!
|
|
550
|
+
# Those workers that failed, however, are okay to adopt here as their TI will
|
|
551
|
+
# still be in queued.
|
|
552
|
+
query_kwargs = {
|
|
553
|
+
"field_selector": "status.phase!=Succeeded",
|
|
554
|
+
"label_selector": (
|
|
555
|
+
"kubernetes_executor=True,"
|
|
556
|
+
f"airflow-worker={scheduler_job_id},{POD_EXECUTOR_DONE_KEY}!=True"
|
|
557
|
+
),
|
|
558
|
+
}
|
|
559
|
+
pod_list = self._list_pods(query_kwargs)
|
|
560
|
+
for pod in pod_list:
|
|
561
|
+
self.adopt_launched_task(kube_client, pod, tis_to_flush_by_key)
|
|
562
|
+
self._adopt_completed_pods(kube_client)
|
|
563
|
+
tis_to_flush.extend(tis_to_flush_by_key.values())
|
|
564
|
+
return tis_to_flush
|
|
562
565
|
|
|
563
566
|
def cleanup_stuck_queued_tasks(self, tis: list[TaskInstance]) -> list[str]:
|
|
564
567
|
"""
|
|
@@ -18,8 +18,8 @@
|
|
|
18
18
|
# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE
|
|
19
19
|
# OVERWRITTEN WHEN PREPARING PACKAGES.
|
|
20
20
|
#
|
|
21
|
-
# IF YOU WANT TO MODIFY
|
|
22
|
-
# `get_provider_info_TEMPLATE.py.jinja2` IN the `
|
|
21
|
+
# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
|
|
22
|
+
# `get_provider_info_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
|
|
23
23
|
|
|
24
24
|
|
|
25
25
|
def get_provider_info():
|
|
@@ -28,7 +28,9 @@ def get_provider_info():
|
|
|
28
28
|
"name": "Kubernetes",
|
|
29
29
|
"description": "`Kubernetes <https://kubernetes.io/>`__\n",
|
|
30
30
|
"suspended": False,
|
|
31
|
+
"source-date-epoch": 1700827450,
|
|
31
32
|
"versions": [
|
|
33
|
+
"7.10.0",
|
|
32
34
|
"7.9.0",
|
|
33
35
|
"7.8.0",
|
|
34
36
|
"7.7.0",
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
|
2
|
+
# or more contributor license agreements. See the NOTICE file
|
|
3
|
+
# distributed with this work for additional information
|
|
4
|
+
# regarding copyright ownership. The ASF licenses this file
|
|
5
|
+
# to you under the Apache License, Version 2.0 (the
|
|
6
|
+
# "License"); you may not use this file except in compliance
|
|
7
|
+
# with the License. You may obtain a copy of the License at
|
|
8
|
+
#
|
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
#
|
|
11
|
+
# Unless required by applicable law or agreed to in writing,
|
|
12
|
+
# software distributed under the License is distributed on an
|
|
13
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
14
|
+
# KIND, either express or implied. See the License for the
|
|
15
|
+
# specific language governing permissions and limitations
|
|
16
|
+
# under the License.
|
|
17
|
+
---
|
|
18
|
+
kind: Pod
|
|
19
|
+
apiVersion: v1
|
|
20
|
+
metadata:
|
|
21
|
+
name: placeholder-name-dont-delete
|
|
22
|
+
namespace: placeholder-name-dont-delete
|
|
23
|
+
labels:
|
|
24
|
+
mylabel: foo
|
|
25
|
+
spec:
|
|
26
|
+
containers:
|
|
27
|
+
- name: base
|
|
28
|
+
image: placeholder-name-dont-delete
|
|
29
|
+
env:
|
|
30
|
+
- name: AIRFLOW__CORE__EXECUTOR
|
|
31
|
+
value: LocalExecutor
|
|
32
|
+
- name: AIRFLOW_HOME
|
|
33
|
+
value: /opt/airflow
|
|
34
|
+
- name: AIRFLOW__CORE__DAGS_FOLDER
|
|
35
|
+
value: /opt/airflow/dags
|
|
36
|
+
- name: AIRFLOW__CORE__FERNET_KEY
|
|
37
|
+
valueFrom:
|
|
38
|
+
secretKeyRef:
|
|
39
|
+
name: airflow-fernet-key
|
|
40
|
+
key: fernet-key
|
|
41
|
+
- name: AIRFLOW__DATABASE__SQL_ALCHEMY_CONN
|
|
42
|
+
valueFrom:
|
|
43
|
+
secretKeyRef:
|
|
44
|
+
name: airflow-airflow-metadata
|
|
45
|
+
key: connection
|
|
46
|
+
- name: foo
|
|
47
|
+
value: bar
|
|
48
|
+
resources: {}
|
|
49
|
+
volumeMounts:
|
|
50
|
+
- name: airflow-logs
|
|
51
|
+
mountPath: /opt/airflow/logs
|
|
52
|
+
- name: airflow-config
|
|
53
|
+
readOnly: true
|
|
54
|
+
mountPath: /opt/airflow/airflow.cfg
|
|
55
|
+
subPath: airflow.cfg
|
|
56
|
+
- name: airflow-config
|
|
57
|
+
readOnly: true
|
|
58
|
+
mountPath: /opt/airflow/config/airflow_local_settings.py
|
|
59
|
+
subPath: airflow_local_settings.py
|
|
60
|
+
terminationMessagePath: /dev/termination-log
|
|
61
|
+
terminationMessagePolicy: File
|
|
62
|
+
imagePullPolicy: IfNotPresent
|
|
63
|
+
volumes:
|
|
64
|
+
- name: airflow-logs
|
|
65
|
+
emptyDir: {}
|
|
66
|
+
- name: airflow-config
|
|
67
|
+
configMap:
|
|
68
|
+
name: airflow-airflow-config
|
|
69
|
+
defaultMode: 420
|
|
70
|
+
restartPolicy: Never
|
|
71
|
+
terminationGracePeriodSeconds: 30
|
|
72
|
+
serviceAccountName: airflow-worker
|
|
73
|
+
serviceAccount: airflow-worker
|
|
74
|
+
securityContext:
|
|
75
|
+
runAsUser: 50000
|
|
76
|
+
fsGroup: 50000
|
|
77
|
+
imagePullSecrets:
|
|
78
|
+
- name: airflow-registry
|
|
79
|
+
schedulerName: default-scheduler
|
|
@@ -191,7 +191,7 @@ class KubernetesPodOperator(BaseOperator):
|
|
|
191
191
|
:param image_pull_policy: Specify a policy to cache or always pull an image.
|
|
192
192
|
:param annotations: non-identifying metadata you can attach to the Pod.
|
|
193
193
|
Can be a large range of data, and can include characters
|
|
194
|
-
that are not permitted by labels.
|
|
194
|
+
that are not permitted by labels. (templated)
|
|
195
195
|
:param container_resources: resources for the launched pod. (templated)
|
|
196
196
|
:param affinity: affinity scheduling rules for the launched pod.
|
|
197
197
|
:param config_file: The path to the Kubernetes config file. (templated)
|
|
@@ -261,6 +261,7 @@ class KubernetesPodOperator(BaseOperator):
|
|
|
261
261
|
template_fields: Sequence[str] = (
|
|
262
262
|
"image",
|
|
263
263
|
"cmds",
|
|
264
|
+
"annotations",
|
|
264
265
|
"arguments",
|
|
265
266
|
"env_vars",
|
|
266
267
|
"labels",
|