apache-airflow-providers-cncf-kubernetes 7.6.0rc1__tar.gz → 7.7.0rc1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of apache-airflow-providers-cncf-kubernetes might be problematic. Click here for more details.
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1/apache_airflow_providers_cncf_kubernetes.egg-info → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/PKG-INFO +7 -7
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/README.rst +4 -4
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/airflow/providers/cncf/kubernetes/__init__.py +3 -3
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py +2 -2
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/airflow/providers/cncf/kubernetes/get_provider_info.py +2 -1
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/airflow/providers/cncf/kubernetes/operators/pod.py +6 -7
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py +1 -1
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/airflow/providers/cncf/kubernetes/python_kubernetes_script.py +2 -2
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/airflow/providers/cncf/kubernetes/template_rendering.py +1 -1
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/airflow/providers/cncf/kubernetes/triggers/pod.py +1 -1
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/airflow/providers/cncf/kubernetes/utils/delete_from.py +1 -1
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/airflow/providers/cncf/kubernetes/utils/pod_manager.py +64 -37
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1/apache_airflow_providers_cncf_kubernetes.egg-info}/PKG-INFO +7 -7
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/apache_airflow_providers_cncf_kubernetes.egg-info/requires.txt +1 -1
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/pyproject.toml +7 -1
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/setup.cfg +3 -3
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/setup.py +1 -1
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/LICENSE +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/MANIFEST.in +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/NOTICE +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/airflow/providers/cncf/kubernetes/backcompat/__init__.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/airflow/providers/cncf/kubernetes/backcompat/backwards_compat_converters.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/airflow/providers/cncf/kubernetes/decorators/__init__.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/airflow/providers/cncf/kubernetes/decorators/kubernetes.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/airflow/providers/cncf/kubernetes/executors/__init__.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/airflow/providers/cncf/kubernetes/executors/kubernetes_executor_types.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/airflow/providers/cncf/kubernetes/executors/kubernetes_executor_utils.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/airflow/providers/cncf/kubernetes/executors/local_kubernetes_executor.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/airflow/providers/cncf/kubernetes/hooks/__init__.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/airflow/providers/cncf/kubernetes/hooks/kubernetes.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/airflow/providers/cncf/kubernetes/k8s_model.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/airflow/providers/cncf/kubernetes/kube_client.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/airflow/providers/cncf/kubernetes/kube_config.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/airflow/providers/cncf/kubernetes/kubernetes_executor_templates/__init__.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/airflow/providers/cncf/kubernetes/kubernetes_helper_functions.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/airflow/providers/cncf/kubernetes/operators/__init__.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/airflow/providers/cncf/kubernetes/operators/resource.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/airflow/providers/cncf/kubernetes/pod_generator.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/airflow/providers/cncf/kubernetes/pod_generator_deprecated.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/airflow/providers/cncf/kubernetes/pod_launcher_deprecated.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/airflow/providers/cncf/kubernetes/pod_template_file_examples/__init__.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/airflow/providers/cncf/kubernetes/python_kubernetes_script.jinja2 +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/airflow/providers/cncf/kubernetes/secret.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/airflow/providers/cncf/kubernetes/sensors/__init__.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/airflow/providers/cncf/kubernetes/sensors/spark_kubernetes.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/airflow/providers/cncf/kubernetes/triggers/__init__.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/airflow/providers/cncf/kubernetes/triggers/kubernetes_pod.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/airflow/providers/cncf/kubernetes/utils/__init__.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/airflow/providers/cncf/kubernetes/utils/k8s_hashlib_wrapper.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/airflow/providers/cncf/kubernetes/utils/xcom_sidecar.py +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/apache_airflow_providers_cncf_kubernetes.egg-info/SOURCES.txt +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/apache_airflow_providers_cncf_kubernetes.egg-info/dependency_links.txt +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/apache_airflow_providers_cncf_kubernetes.egg-info/entry_points.txt +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/apache_airflow_providers_cncf_kubernetes.egg-info/not-zip-safe +0 -0
- {apache-airflow-providers-cncf-kubernetes-7.6.0rc1 → apache-airflow-providers-cncf-kubernetes-7.7.0rc1}/apache_airflow_providers_cncf_kubernetes.egg-info/top_level.txt +0 -0
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: apache-airflow-providers-cncf-kubernetes
|
|
3
|
-
Version: 7.
|
|
3
|
+
Version: 7.7.0rc1
|
|
4
4
|
Summary: Provider for Apache Airflow. Implements apache-airflow-providers-cncf-kubernetes package
|
|
5
5
|
Home-page: https://airflow.apache.org/
|
|
6
6
|
Download-URL: https://archive.apache.org/dist/airflow/providers
|
|
7
7
|
Author: Apache Software Foundation
|
|
8
8
|
Author-email: dev@airflow.apache.org
|
|
9
9
|
License: Apache License 2.0
|
|
10
|
-
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/7.
|
|
11
|
-
Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/7.
|
|
10
|
+
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/7.7.0/
|
|
11
|
+
Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/7.7.0/changelog.html
|
|
12
12
|
Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
|
|
13
13
|
Project-URL: Source Code, https://github.com/apache/airflow
|
|
14
14
|
Project-URL: Slack Chat, https://s.apache.org/airflow-slack
|
|
@@ -70,7 +70,7 @@ License-File: NOTICE
|
|
|
70
70
|
|
|
71
71
|
Package ``apache-airflow-providers-cncf-kubernetes``
|
|
72
72
|
|
|
73
|
-
Release: ``7.
|
|
73
|
+
Release: ``7.7.0rc1``
|
|
74
74
|
|
|
75
75
|
|
|
76
76
|
`Kubernetes <https://kubernetes.io/>`__
|
|
@@ -83,7 +83,7 @@ This is a provider package for ``cncf.kubernetes`` provider. All classes for thi
|
|
|
83
83
|
are in ``airflow.providers.cncf.kubernetes`` python package.
|
|
84
84
|
|
|
85
85
|
You can find package information and changelog for the provider
|
|
86
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/7.
|
|
86
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/7.7.0/>`_.
|
|
87
87
|
|
|
88
88
|
|
|
89
89
|
Installation
|
|
@@ -101,7 +101,7 @@ Requirements
|
|
|
101
101
|
====================== ==================
|
|
102
102
|
PIP package Version required
|
|
103
103
|
====================== ==================
|
|
104
|
-
``apache-airflow`` ``>=2.
|
|
104
|
+
``apache-airflow`` ``>=2.5.0``
|
|
105
105
|
``asgiref`` ``>=3.5.2``
|
|
106
106
|
``cryptography`` ``>=2.0.0``
|
|
107
107
|
``kubernetes`` ``>=21.7.0,<24``
|
|
@@ -110,4 +110,4 @@ PIP package Version required
|
|
|
110
110
|
====================== ==================
|
|
111
111
|
|
|
112
112
|
The changelog for the provider package can be found in the
|
|
113
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/7.
|
|
113
|
+
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/7.7.0/changelog.html>`_.
|
|
@@ -36,7 +36,7 @@
|
|
|
36
36
|
|
|
37
37
|
Package ``apache-airflow-providers-cncf-kubernetes``
|
|
38
38
|
|
|
39
|
-
Release: ``7.
|
|
39
|
+
Release: ``7.7.0rc1``
|
|
40
40
|
|
|
41
41
|
|
|
42
42
|
`Kubernetes <https://kubernetes.io/>`__
|
|
@@ -49,7 +49,7 @@ This is a provider package for ``cncf.kubernetes`` provider. All classes for thi
|
|
|
49
49
|
are in ``airflow.providers.cncf.kubernetes`` python package.
|
|
50
50
|
|
|
51
51
|
You can find package information and changelog for the provider
|
|
52
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/7.
|
|
52
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/7.7.0/>`_.
|
|
53
53
|
|
|
54
54
|
|
|
55
55
|
Installation
|
|
@@ -67,7 +67,7 @@ Requirements
|
|
|
67
67
|
====================== ==================
|
|
68
68
|
PIP package Version required
|
|
69
69
|
====================== ==================
|
|
70
|
-
``apache-airflow`` ``>=2.
|
|
70
|
+
``apache-airflow`` ``>=2.5.0``
|
|
71
71
|
``asgiref`` ``>=3.5.2``
|
|
72
72
|
``cryptography`` ``>=2.0.0``
|
|
73
73
|
``kubernetes`` ``>=21.7.0,<24``
|
|
@@ -76,4 +76,4 @@ PIP package Version required
|
|
|
76
76
|
====================== ==================
|
|
77
77
|
|
|
78
78
|
The changelog for the provider package can be found in the
|
|
79
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/7.
|
|
79
|
+
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/7.7.0/changelog.html>`_.
|
|
@@ -28,7 +28,7 @@ import packaging.version
|
|
|
28
28
|
|
|
29
29
|
__all__ = ["__version__"]
|
|
30
30
|
|
|
31
|
-
__version__ = "7.
|
|
31
|
+
__version__ = "7.7.0"
|
|
32
32
|
|
|
33
33
|
try:
|
|
34
34
|
from airflow import __version__ as airflow_version
|
|
@@ -36,8 +36,8 @@ except ImportError:
|
|
|
36
36
|
from airflow.version import version as airflow_version
|
|
37
37
|
|
|
38
38
|
if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
|
|
39
|
-
"2.
|
|
39
|
+
"2.5.0"
|
|
40
40
|
):
|
|
41
41
|
raise RuntimeError(
|
|
42
|
-
f"The package `apache-airflow-providers-cncf-kubernetes:{__version__}` requires Apache Airflow 2.
|
|
42
|
+
f"The package `apache-airflow-providers-cncf-kubernetes:{__version__}` requires Apache Airflow 2.5.0+" # NOQA: E501
|
|
43
43
|
)
|
|
@@ -438,10 +438,10 @@ class KubernetesExecutor(BaseExecutor):
|
|
|
438
438
|
if self.kube_config.delete_worker_pods:
|
|
439
439
|
if state != TaskInstanceState.FAILED or self.kube_config.delete_worker_pods_on_failure:
|
|
440
440
|
self.kube_scheduler.delete_pod(pod_name=pod_name, namespace=namespace)
|
|
441
|
-
self.log.info("Deleted pod: %s in namespace %s",
|
|
441
|
+
self.log.info("Deleted pod: %s in namespace %s", key, namespace)
|
|
442
442
|
else:
|
|
443
443
|
self.kube_scheduler.patch_pod_executor_done(pod_name=pod_name, namespace=namespace)
|
|
444
|
-
self.log.info("Patched pod %s in namespace %s to mark it as done",
|
|
444
|
+
self.log.info("Patched pod %s in namespace %s to mark it as done", key, namespace)
|
|
445
445
|
|
|
446
446
|
try:
|
|
447
447
|
self.running.remove(key)
|
|
@@ -29,6 +29,7 @@ def get_provider_info():
|
|
|
29
29
|
"description": "`Kubernetes <https://kubernetes.io/>`__\n",
|
|
30
30
|
"suspended": False,
|
|
31
31
|
"versions": [
|
|
32
|
+
"7.7.0",
|
|
32
33
|
"7.6.0",
|
|
33
34
|
"7.5.1",
|
|
34
35
|
"7.5.0",
|
|
@@ -74,7 +75,7 @@ def get_provider_info():
|
|
|
74
75
|
"1.0.0",
|
|
75
76
|
],
|
|
76
77
|
"dependencies": [
|
|
77
|
-
"apache-airflow>=2.
|
|
78
|
+
"apache-airflow>=2.5.0",
|
|
78
79
|
"asgiref>=3.5.2",
|
|
79
80
|
"cryptography>=2.0.0",
|
|
80
81
|
"kubernetes>=21.7.0,<24",
|
|
@@ -371,7 +371,7 @@ class KubernetesPodOperator(BaseOperator):
|
|
|
371
371
|
self.get_logs = get_logs
|
|
372
372
|
self.container_logs = container_logs
|
|
373
373
|
if self.container_logs == KubernetesPodOperator.BASE_CONTAINER_NAME:
|
|
374
|
-
self.container_logs = base_container_name
|
|
374
|
+
self.container_logs = base_container_name or self.BASE_CONTAINER_NAME
|
|
375
375
|
self.image_pull_policy = image_pull_policy
|
|
376
376
|
self.node_selector = node_selector or {}
|
|
377
377
|
self.annotations = annotations or {}
|
|
@@ -679,7 +679,10 @@ class KubernetesPodOperator(BaseOperator):
|
|
|
679
679
|
return xcom_sidecar_output
|
|
680
680
|
finally:
|
|
681
681
|
istio_enabled = self.is_istio_enabled(pod)
|
|
682
|
-
|
|
682
|
+
# Skip await_pod_completion when the event is 'timeout' due to the pod can hang
|
|
683
|
+
# on the ErrImagePull or ContainerCreating step and it will never complete
|
|
684
|
+
if event["status"] != "timeout":
|
|
685
|
+
pod = self.pod_manager.await_pod_completion(pod, istio_enabled, self.base_container_name)
|
|
683
686
|
if pod is not None:
|
|
684
687
|
self.post_complete_action(
|
|
685
688
|
pod=pod,
|
|
@@ -804,7 +807,6 @@ class KubernetesPodOperator(BaseOperator):
|
|
|
804
807
|
raise Exception("Error while deleting istio-proxy sidecar: %s", output_str)
|
|
805
808
|
|
|
806
809
|
def process_pod_deletion(self, pod: k8s.V1Pod, *, reraise=True):
|
|
807
|
-
istio_enabled = self.is_istio_enabled(pod)
|
|
808
810
|
with _optionally_suppress(reraise=reraise):
|
|
809
811
|
if pod is not None:
|
|
810
812
|
should_delete_pod = (
|
|
@@ -818,12 +820,9 @@ class KubernetesPodOperator(BaseOperator):
|
|
|
818
820
|
and container_is_succeeded(pod, self.base_container_name)
|
|
819
821
|
)
|
|
820
822
|
)
|
|
821
|
-
if should_delete_pod
|
|
823
|
+
if should_delete_pod:
|
|
822
824
|
self.log.info("Deleting pod: %s", pod.metadata.name)
|
|
823
825
|
self.pod_manager.delete_pod(pod)
|
|
824
|
-
elif should_delete_pod and istio_enabled:
|
|
825
|
-
self.log.info("Deleting istio-proxy sidecar inside %s: ", pod.metadata.name)
|
|
826
|
-
self.kill_istio_sidecar(pod)
|
|
827
826
|
else:
|
|
828
827
|
self.log.info("Skipping deleting pod: %s", pod.metadata.name)
|
|
829
828
|
|
|
@@ -24,7 +24,7 @@ from typing import TYPE_CHECKING, Sequence
|
|
|
24
24
|
from kubernetes.client import ApiException
|
|
25
25
|
from kubernetes.watch import Watch
|
|
26
26
|
|
|
27
|
-
from airflow import AirflowException
|
|
27
|
+
from airflow.exceptions import AirflowException
|
|
28
28
|
from airflow.models import BaseOperator
|
|
29
29
|
from airflow.providers.cncf.kubernetes.hooks.kubernetes import KubernetesHook, _load_body_to_dict
|
|
30
30
|
|
|
@@ -31,9 +31,9 @@ def _balance_parens(after_decorator):
|
|
|
31
31
|
while num_paren:
|
|
32
32
|
current = after_decorator.popleft()
|
|
33
33
|
if current == "(":
|
|
34
|
-
num_paren
|
|
34
|
+
num_paren += 1
|
|
35
35
|
elif current == ")":
|
|
36
|
-
num_paren
|
|
36
|
+
num_paren -= 1
|
|
37
37
|
return "".join(after_decorator)
|
|
38
38
|
|
|
39
39
|
|
|
@@ -22,7 +22,7 @@ from typing import TYPE_CHECKING
|
|
|
22
22
|
from jinja2 import TemplateAssertionError, UndefinedError
|
|
23
23
|
from kubernetes.client.api_client import ApiClient
|
|
24
24
|
|
|
25
|
-
from airflow import AirflowException
|
|
25
|
+
from airflow.exceptions import AirflowException
|
|
26
26
|
from airflow.providers.cncf.kubernetes.kube_config import KubeConfig
|
|
27
27
|
from airflow.providers.cncf.kubernetes.kubernetes_helper_functions import (
|
|
28
28
|
create_pod_id,
|
|
@@ -166,7 +166,7 @@ class KubernetesPodTrigger(BaseTrigger):
|
|
|
166
166
|
elif self.should_wait(pod_phase=pod_status, container_state=container_state):
|
|
167
167
|
self.log.info("Container is not completed and still working.")
|
|
168
168
|
|
|
169
|
-
if pod_status == PodPhase.PENDING and container_state
|
|
169
|
+
if pod_status == PodPhase.PENDING and container_state != ContainerState.RUNNING:
|
|
170
170
|
delta = datetime.datetime.now(tz=datetime.timezone.utc) - self.trigger_start_time
|
|
171
171
|
if delta.total_seconds() >= self.startup_timeout:
|
|
172
172
|
message = (
|
|
@@ -138,7 +138,7 @@ def _delete_from_yaml_single_item(
|
|
|
138
138
|
else:
|
|
139
139
|
resp = getattr(k8s_api, f"delete_{kind}")(name=name, body=body, **kwargs)
|
|
140
140
|
if verbose:
|
|
141
|
-
print(f"{kind} deleted. status='{
|
|
141
|
+
print(f"{kind} deleted. status='{resp.status}'")
|
|
142
142
|
return resp
|
|
143
143
|
|
|
144
144
|
|
|
@@ -361,7 +361,8 @@ class PodManager(LoggingMixin):
|
|
|
361
361
|
warnings.warn(
|
|
362
362
|
"Method `follow_container_logs` is deprecated. Use `fetch_container_logs` instead"
|
|
363
363
|
"with option `follow=True`.",
|
|
364
|
-
AirflowProviderDeprecationWarning,
|
|
364
|
+
category=AirflowProviderDeprecationWarning,
|
|
365
|
+
stacklevel=2,
|
|
365
366
|
)
|
|
366
367
|
return self.fetch_container_logs(pod=pod, container_name=container_name, follow=True)
|
|
367
368
|
|
|
@@ -391,8 +392,12 @@ class PodManager(LoggingMixin):
|
|
|
391
392
|
before=before_log(self.log, logging.INFO),
|
|
392
393
|
)
|
|
393
394
|
def consume_logs(
|
|
394
|
-
*,
|
|
395
|
-
|
|
395
|
+
*,
|
|
396
|
+
since_time: DateTime | None = None,
|
|
397
|
+
follow: bool = True,
|
|
398
|
+
termination_timeout: int = 120,
|
|
399
|
+
logs: PodLogsConsumer | None,
|
|
400
|
+
) -> tuple[DateTime | None, PodLogsConsumer | None]:
|
|
396
401
|
"""
|
|
397
402
|
Tries to follow container logs until container completes.
|
|
398
403
|
|
|
@@ -411,16 +416,39 @@ class PodManager(LoggingMixin):
|
|
|
411
416
|
math.ceil((pendulum.now() - since_time).total_seconds()) if since_time else None
|
|
412
417
|
),
|
|
413
418
|
follow=follow,
|
|
414
|
-
post_termination_timeout=
|
|
419
|
+
post_termination_timeout=post_termination_timeout,
|
|
415
420
|
)
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
421
|
+
message_to_log = None
|
|
422
|
+
message_timestamp = None
|
|
423
|
+
progress_callback_lines = []
|
|
424
|
+
try:
|
|
425
|
+
for raw_line in logs:
|
|
426
|
+
line = raw_line.decode("utf-8", errors="backslashreplace")
|
|
427
|
+
line_timestamp, message = self.parse_log_line(line)
|
|
428
|
+
if line_timestamp: # detect new log line
|
|
429
|
+
if message_to_log is None: # first line in the log
|
|
430
|
+
message_to_log = message
|
|
431
|
+
message_timestamp = line_timestamp
|
|
432
|
+
progress_callback_lines.append(line)
|
|
433
|
+
else: # previous log line is complete
|
|
434
|
+
if self._progress_callback:
|
|
435
|
+
for line in progress_callback_lines:
|
|
436
|
+
self._progress_callback(line)
|
|
437
|
+
self.log.info("[%s] %s", container_name, message_to_log)
|
|
438
|
+
last_captured_timestamp = message_timestamp
|
|
439
|
+
message_to_log = message
|
|
440
|
+
message_timestamp = line_timestamp
|
|
441
|
+
progress_callback_lines = [line]
|
|
442
|
+
else: # continuation of the previous log line
|
|
443
|
+
message_to_log = f"{message_to_log}\n{message}"
|
|
444
|
+
progress_callback_lines.append(line)
|
|
445
|
+
finally:
|
|
446
|
+
# log the last line and update the last_captured_timestamp
|
|
419
447
|
if self._progress_callback:
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
448
|
+
for line in progress_callback_lines:
|
|
449
|
+
self._progress_callback(line)
|
|
450
|
+
self.log.info("[%s] %s", container_name, message_to_log)
|
|
451
|
+
last_captured_timestamp = message_timestamp
|
|
424
452
|
except BaseHTTPError as e:
|
|
425
453
|
self.log.warning(
|
|
426
454
|
"Reading of logs interrupted for container %r with error %r; will retry. "
|
|
@@ -433,15 +461,19 @@ class PodManager(LoggingMixin):
|
|
|
433
461
|
pod.metadata.name,
|
|
434
462
|
exc_info=True,
|
|
435
463
|
)
|
|
436
|
-
return last_captured_timestamp or since_time
|
|
464
|
+
return last_captured_timestamp or since_time, logs
|
|
437
465
|
|
|
438
466
|
# note: `read_pod_logs` follows the logs, so we shouldn't necessarily *need* to
|
|
439
467
|
# loop as we do here. But in a long-running process we might temporarily lose connectivity.
|
|
440
468
|
# So the looping logic is there to let us resume following the logs.
|
|
469
|
+
logs = None
|
|
441
470
|
last_log_time = since_time
|
|
442
471
|
while True:
|
|
443
|
-
last_log_time = consume_logs(
|
|
444
|
-
since_time=last_log_time,
|
|
472
|
+
last_log_time, logs = consume_logs(
|
|
473
|
+
since_time=last_log_time,
|
|
474
|
+
follow=follow,
|
|
475
|
+
termination_timeout=post_termination_timeout,
|
|
476
|
+
logs=logs,
|
|
445
477
|
)
|
|
446
478
|
if not self.container_is_running(pod, container_name=container_name):
|
|
447
479
|
return PodLoggingStatus(running=False, last_log_time=last_log_time)
|
|
@@ -560,16 +592,10 @@ class PodManager(LoggingMixin):
|
|
|
560
592
|
"""
|
|
561
593
|
timestamp, sep, message = line.strip().partition(" ")
|
|
562
594
|
if not sep:
|
|
563
|
-
self.log.error(
|
|
564
|
-
"Error parsing timestamp (no timestamp in message %r). "
|
|
565
|
-
"Will continue execution but won't update timestamp",
|
|
566
|
-
line,
|
|
567
|
-
)
|
|
568
595
|
return None, line
|
|
569
596
|
try:
|
|
570
597
|
last_log_time = cast(DateTime, pendulum.parse(timestamp))
|
|
571
598
|
except ParserError:
|
|
572
|
-
self.log.error("Error parsing timestamp. Will continue execution but won't update timestamp")
|
|
573
599
|
return None, line
|
|
574
600
|
return last_log_time, message
|
|
575
601
|
|
|
@@ -728,23 +754,24 @@ class PodManager(LoggingMixin):
|
|
|
728
754
|
self._exec_pod_command(resp, "kill -s SIGINT 1")
|
|
729
755
|
|
|
730
756
|
def _exec_pod_command(self, resp, command: str) -> str | None:
|
|
731
|
-
res =
|
|
732
|
-
if resp.is_open():
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
757
|
+
res = ""
|
|
758
|
+
if not resp.is_open():
|
|
759
|
+
return None
|
|
760
|
+
self.log.info("Running command... %s", command)
|
|
761
|
+
resp.write_stdin(f"{command}\n")
|
|
762
|
+
while resp.is_open():
|
|
763
|
+
resp.update(timeout=1)
|
|
764
|
+
while resp.peek_stdout():
|
|
765
|
+
res += resp.read_stdout()
|
|
766
|
+
error_res = ""
|
|
767
|
+
while resp.peek_stderr():
|
|
768
|
+
error_res += resp.read_stderr()
|
|
769
|
+
if error_res:
|
|
770
|
+
self.log.info("stderr from command: %s", error_res)
|
|
771
|
+
break
|
|
772
|
+
if res:
|
|
773
|
+
return res
|
|
774
|
+
return None
|
|
748
775
|
|
|
749
776
|
|
|
750
777
|
class OnFinishAction(str, enum.Enum):
|
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: apache-airflow-providers-cncf-kubernetes
|
|
3
|
-
Version: 7.
|
|
3
|
+
Version: 7.7.0rc1
|
|
4
4
|
Summary: Provider for Apache Airflow. Implements apache-airflow-providers-cncf-kubernetes package
|
|
5
5
|
Home-page: https://airflow.apache.org/
|
|
6
6
|
Download-URL: https://archive.apache.org/dist/airflow/providers
|
|
7
7
|
Author: Apache Software Foundation
|
|
8
8
|
Author-email: dev@airflow.apache.org
|
|
9
9
|
License: Apache License 2.0
|
|
10
|
-
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/7.
|
|
11
|
-
Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/7.
|
|
10
|
+
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/7.7.0/
|
|
11
|
+
Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/7.7.0/changelog.html
|
|
12
12
|
Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
|
|
13
13
|
Project-URL: Source Code, https://github.com/apache/airflow
|
|
14
14
|
Project-URL: Slack Chat, https://s.apache.org/airflow-slack
|
|
@@ -70,7 +70,7 @@ License-File: NOTICE
|
|
|
70
70
|
|
|
71
71
|
Package ``apache-airflow-providers-cncf-kubernetes``
|
|
72
72
|
|
|
73
|
-
Release: ``7.
|
|
73
|
+
Release: ``7.7.0rc1``
|
|
74
74
|
|
|
75
75
|
|
|
76
76
|
`Kubernetes <https://kubernetes.io/>`__
|
|
@@ -83,7 +83,7 @@ This is a provider package for ``cncf.kubernetes`` provider. All classes for thi
|
|
|
83
83
|
are in ``airflow.providers.cncf.kubernetes`` python package.
|
|
84
84
|
|
|
85
85
|
You can find package information and changelog for the provider
|
|
86
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/7.
|
|
86
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/7.7.0/>`_.
|
|
87
87
|
|
|
88
88
|
|
|
89
89
|
Installation
|
|
@@ -101,7 +101,7 @@ Requirements
|
|
|
101
101
|
====================== ==================
|
|
102
102
|
PIP package Version required
|
|
103
103
|
====================== ==================
|
|
104
|
-
``apache-airflow`` ``>=2.
|
|
104
|
+
``apache-airflow`` ``>=2.5.0``
|
|
105
105
|
``asgiref`` ``>=3.5.2``
|
|
106
106
|
``cryptography`` ``>=2.0.0``
|
|
107
107
|
``kubernetes`` ``>=21.7.0,<24``
|
|
@@ -110,4 +110,4 @@ PIP package Version required
|
|
|
110
110
|
====================== ==================
|
|
111
111
|
|
|
112
112
|
The changelog for the provider package can be found in the
|
|
113
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/7.
|
|
113
|
+
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/7.7.0/changelog.html>`_.
|
|
@@ -63,6 +63,7 @@ extend-select = [
|
|
|
63
63
|
"D419",
|
|
64
64
|
"TCH001", # typing-only-first-party-import
|
|
65
65
|
"TCH002", # typing-only-third-party-import
|
|
66
|
+
"TID251",
|
|
66
67
|
]
|
|
67
68
|
extend-ignore = [
|
|
68
69
|
"D203",
|
|
@@ -122,7 +123,8 @@ combine-as-imports = true
|
|
|
122
123
|
|
|
123
124
|
# The Pydantic representations of SqlAlchemy Models are not parsed well with Pydantic
|
|
124
125
|
# when __future__.annotations is used so we need to skip them from upgrading
|
|
125
|
-
|
|
126
|
+
# Pydantic also require models to be imported during execution
|
|
127
|
+
"airflow/serialization/pydantic/*.py" = ["I002", "UP007", "TCH001"]
|
|
126
128
|
|
|
127
129
|
# Ignore pydoc style from these
|
|
128
130
|
"*.pyi" = ["D"]
|
|
@@ -144,6 +146,10 @@ combine-as-imports = true
|
|
|
144
146
|
"tests/providers/elasticsearch/log/elasticmock/__init__.py" = ["E402"]
|
|
145
147
|
"tests/providers/elasticsearch/log/elasticmock/utilities/__init__.py" = ["E402"]
|
|
146
148
|
|
|
149
|
+
[tool.ruff.flake8-tidy-imports.banned-api]
|
|
150
|
+
"airflow.AirflowException".msg = "Use airflow.exceptions.AirflowException instead."
|
|
151
|
+
"airflow.Dataset".msg = "Use airflow.datasets.Dataset instead."
|
|
152
|
+
|
|
147
153
|
[tool.coverage.run]
|
|
148
154
|
branch = true
|
|
149
155
|
relative_files = true
|
|
@@ -27,8 +27,8 @@ classifiers =
|
|
|
27
27
|
Programming Language :: Python :: 3.11
|
|
28
28
|
Topic :: System :: Monitoring
|
|
29
29
|
project_urls =
|
|
30
|
-
Documentation=https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/7.
|
|
31
|
-
Changelog=https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/7.
|
|
30
|
+
Documentation=https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/7.7.0/
|
|
31
|
+
Changelog=https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/7.7.0/changelog.html
|
|
32
32
|
Bug Tracker=https://github.com/apache/airflow/issues
|
|
33
33
|
Source Code=https://github.com/apache/airflow
|
|
34
34
|
Slack Chat=https://s.apache.org/airflow-slack
|
|
@@ -47,7 +47,7 @@ setup_requires =
|
|
|
47
47
|
setuptools
|
|
48
48
|
wheel
|
|
49
49
|
install_requires =
|
|
50
|
-
apache-airflow>=2.
|
|
50
|
+
apache-airflow>=2.5.0.dev0
|
|
51
51
|
asgiref>=3.5.2
|
|
52
52
|
cryptography>=2.0.0
|
|
53
53
|
google-re2>=1.0
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|