apache-airflow-providers-cncf-kubernetes 9.0.1__py3-none-any.whl → 10.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
29
29
 
30
30
  __all__ = ["__version__"]
31
31
 
32
- __version__ = "9.0.1"
32
+ __version__ = "10.0.0"
33
33
 
34
34
  if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
35
35
  "2.8.0"
@@ -65,7 +65,7 @@ class _KubernetesDecoratedOperator(DecoratedOperator, KubernetesPodOperator):
65
65
  # there are some cases we can't deepcopy the objects (e.g protobuf).
66
66
  shallow_copy_attrs: Sequence[str] = ("python_callable",)
67
67
 
68
- def __init__(self, namespace: str = "default", use_dill: bool = False, **kwargs) -> None:
68
+ def __init__(self, namespace: str | None = None, use_dill: bool = False, **kwargs) -> None:
69
69
  self.use_dill = use_dill
70
70
  super().__init__(
71
71
  namespace=namespace,
@@ -768,8 +768,13 @@ class KubernetesExecutor(BaseExecutor):
768
768
  self.result_queue.join()
769
769
  except ConnectionResetError:
770
770
  self.log.exception("Connection Reset error while flushing task_queue and result_queue.")
771
+ except Exception:
772
+ self.log.exception("Unknown error while flushing task queue and result queue.")
771
773
  if self.kube_scheduler:
772
- self.kube_scheduler.terminate()
774
+ try:
775
+ self.kube_scheduler.terminate()
776
+ except Exception:
777
+ self.log.exception("Unknown error while flushing task queue and result queue.")
773
778
  self._manager.shutdown()
774
779
 
775
780
  def terminate(self):
@@ -45,6 +45,7 @@ class LocalKubernetesExecutor(BaseExecutor):
45
45
  """
46
46
 
47
47
  supports_ad_hoc_ti_run: bool = True
48
+ # TODO: Remove this attribute once providers rely on Airflow >=3.0.0
48
49
  supports_pickling: bool = False
49
50
  supports_sentry: bool = False
50
51
 
@@ -146,7 +147,6 @@ class LocalKubernetesExecutor(BaseExecutor):
146
147
  self,
147
148
  task_instance: TaskInstance,
148
149
  mark_success: bool = False,
149
- pickle_id: int | None = None,
150
150
  ignore_all_deps: bool = False,
151
151
  ignore_depends_on_past: bool = False,
152
152
  wait_for_past_depends_before_skipping: bool = False,
@@ -154,6 +154,7 @@ class LocalKubernetesExecutor(BaseExecutor):
154
154
  ignore_ti_state: bool = False,
155
155
  pool: str | None = None,
156
156
  cfg_path: str | None = None,
157
+ **kwargs,
157
158
  ) -> None:
158
159
  """Queues task instance via local or kubernetes executor."""
159
160
  from airflow.models.taskinstance import SimpleTaskInstance
@@ -162,10 +163,13 @@ class LocalKubernetesExecutor(BaseExecutor):
162
163
  self.log.debug(
163
164
  "Using executor: %s to queue_task_instance for %s", executor.__class__.__name__, task_instance.key
164
165
  )
166
+
167
+ if not hasattr(task_instance, "pickle_id"):
168
+ del kwargs["pickle_id"]
169
+
165
170
  executor.queue_task_instance(
166
171
  task_instance=task_instance,
167
172
  mark_success=mark_success,
168
- pickle_id=pickle_id,
169
173
  ignore_all_deps=ignore_all_deps,
170
174
  ignore_depends_on_past=ignore_depends_on_past,
171
175
  wait_for_past_depends_before_skipping=wait_for_past_depends_before_skipping,
@@ -173,6 +177,7 @@ class LocalKubernetesExecutor(BaseExecutor):
173
177
  ignore_ti_state=ignore_ti_state,
174
178
  pool=pool,
175
179
  cfg_path=cfg_path,
180
+ **kwargs,
176
181
  )
177
182
 
178
183
  def get_task_log(self, ti: TaskInstance, try_number: int) -> tuple[list[str], list[str]]:
@@ -30,6 +30,7 @@ def get_provider_info():
30
30
  "state": "ready",
31
31
  "source-date-epoch": 1730012271,
32
32
  "versions": [
33
+ "10.0.0",
33
34
  "9.0.1",
34
35
  "9.0.0",
35
36
  "8.4.2",
@@ -23,12 +23,10 @@ from functools import cache
23
23
  from typing import TYPE_CHECKING
24
24
 
25
25
  import pendulum
26
- from deprecated import deprecated
27
26
  from kubernetes.client.rest import ApiException
28
27
  from slugify import slugify
29
28
 
30
29
  from airflow.configuration import conf
31
- from airflow.exceptions import AirflowProviderDeprecationWarning
32
30
 
33
31
  if TYPE_CHECKING:
34
32
  from airflow.models.taskinstancekey import TaskInstanceKey
@@ -62,22 +60,6 @@ def add_unique_suffix(*, name: str, rand_len: int = 8, max_len: int = POD_NAME_M
62
60
  return name[: max_len - len(suffix)].strip("-.") + suffix
63
61
 
64
62
 
65
- @deprecated(
66
- reason="This function is deprecated. Please use `add_unique_suffix`",
67
- category=AirflowProviderDeprecationWarning,
68
- )
69
- def add_pod_suffix(*, pod_name: str, rand_len: int = 8, max_len: int = POD_NAME_MAX_LENGTH) -> str:
70
- """
71
- Add random string to pod name while staying under max length.
72
-
73
- :param pod_name: name of the pod
74
- :param rand_len: length of the random string to append
75
- :param max_len: maximum length of the pod name
76
- :meta private:
77
- """
78
- return add_unique_suffix(name=pod_name, rand_len=rand_len, max_len=max_len)
79
-
80
-
81
63
  def create_unique_id(
82
64
  dag_id: str | None = None,
83
65
  task_id: str | None = None,
@@ -110,29 +92,6 @@ def create_unique_id(
110
92
  return base_name
111
93
 
112
94
 
113
- @deprecated(
114
- reason="This function is deprecated. Please use `create_unique_id`.",
115
- category=AirflowProviderDeprecationWarning,
116
- )
117
- def create_pod_id(
118
- dag_id: str | None = None,
119
- task_id: str | None = None,
120
- *,
121
- max_length: int = POD_NAME_MAX_LENGTH,
122
- unique: bool = True,
123
- ) -> str:
124
- """
125
- Generate unique pod ID given a dag_id and / or task_id.
126
-
127
- :param dag_id: DAG ID
128
- :param task_id: Task ID
129
- :param max_length: max number of characters
130
- :param unique: whether a random string suffix should be added
131
- :return: A valid identifier for a kubernetes pod name
132
- """
133
- return create_unique_id(dag_id=dag_id, task_id=task_id, max_length=max_length, unique=unique)
134
-
135
-
136
95
  def annotations_to_key(annotations: dict[str, str]) -> TaskInstanceKey:
137
96
  """Build a TaskInstanceKey based on pod annotations."""
138
97
  log.debug("Creating task key for annotations %s", annotations)
@@ -26,8 +26,7 @@ import os
26
26
  import re
27
27
  import shlex
28
28
  import string
29
- import warnings
30
- from collections.abc import Container
29
+ from collections.abc import Container, Mapping
31
30
  from contextlib import AbstractContextManager
32
31
  from enum import Enum
33
32
  from functools import cached_property
@@ -35,7 +34,6 @@ from typing import TYPE_CHECKING, Any, Callable, Iterable, Literal, Sequence
35
34
 
36
35
  import kubernetes
37
36
  import tenacity
38
- from deprecated import deprecated
39
37
  from kubernetes.client import CoreV1Api, V1Pod, models as k8s
40
38
  from kubernetes.client.exceptions import ApiException
41
39
  from kubernetes.stream import stream
@@ -44,7 +42,6 @@ from urllib3.exceptions import HTTPError
44
42
  from airflow.configuration import conf
45
43
  from airflow.exceptions import (
46
44
  AirflowException,
47
- AirflowProviderDeprecationWarning,
48
45
  AirflowSkipException,
49
46
  TaskDeferred,
50
47
  )
@@ -215,18 +212,12 @@ class KubernetesPodOperator(BaseOperator):
215
212
  :param on_finish_action: What to do when the pod reaches its final state, or the execution is interrupted.
216
213
  If "delete_pod", the pod will be deleted regardless its state; if "delete_succeeded_pod",
217
214
  only succeeded pod will be deleted. You can set to "keep_pod" to keep the pod.
218
- :param is_delete_operator_pod: What to do when the pod reaches its final
219
- state, or the execution is interrupted. If True (default), delete the
220
- pod; if False, leave the pod.
221
- Deprecated - use `on_finish_action` instead.
222
215
  :param termination_message_policy: The termination message policy of the base container.
223
216
  Default value is "File"
224
217
  :param active_deadline_seconds: The active_deadline_seconds which translates to active_deadline_seconds
225
218
  in V1PodSpec.
226
219
  :param callbacks: KubernetesPodOperatorCallback instance contains the callbacks methods on different step
227
220
  of KubernetesPodOperator.
228
- :param progress_callback: Callback function for receiving k8s container logs.
229
- `progress_callback` is deprecated, please use :param `callbacks` instead.
230
221
  :param logging_interval: max time in seconds that task should be in deferred state before
231
222
  resuming to fetch the latest logs. If ``None``, then the task will remain in deferred state until pod
232
223
  is done, and no logs will be visible until that time.
@@ -404,19 +395,8 @@ class KubernetesPodOperator(BaseOperator):
404
395
  self.poll_interval = poll_interval
405
396
  self.remote_pod: k8s.V1Pod | None = None
406
397
  self.log_pod_spec_on_failure = log_pod_spec_on_failure
407
- if is_delete_operator_pod is not None:
408
- warnings.warn(
409
- "`is_delete_operator_pod` parameter is deprecated, please use `on_finish_action`",
410
- AirflowProviderDeprecationWarning,
411
- stacklevel=2,
412
- )
413
- self.on_finish_action = (
414
- OnFinishAction.DELETE_POD if is_delete_operator_pod else OnFinishAction.KEEP_POD
415
- )
416
- self.is_delete_operator_pod = is_delete_operator_pod
417
- else:
418
- self.on_finish_action = OnFinishAction(on_finish_action)
419
- self.is_delete_operator_pod = self.on_finish_action == OnFinishAction.DELETE_POD
398
+ self.on_finish_action = OnFinishAction(on_finish_action)
399
+ self.is_delete_operator_pod = self.on_finish_action == OnFinishAction.DELETE_POD
420
400
  self.termination_message_policy = termination_message_policy
421
401
  self.active_deadline_seconds = active_deadline_seconds
422
402
  self.logging_interval = logging_interval
@@ -436,7 +416,7 @@ class KubernetesPodOperator(BaseOperator):
436
416
  def _render_nested_template_fields(
437
417
  self,
438
418
  content: Any,
439
- context: Context,
419
+ context: Mapping[str, Any],
440
420
  jinja_env: jinja2.Environment,
441
421
  seen_oids: set,
442
422
  ) -> None:
@@ -512,9 +492,7 @@ class KubernetesPodOperator(BaseOperator):
512
492
 
513
493
  @cached_property
514
494
  def pod_manager(self) -> PodManager:
515
- return PodManager(
516
- kube_client=self.client, callbacks=self.callbacks, progress_callback=self._progress_callback
517
- )
495
+ return PodManager(kube_client=self.client, callbacks=self.callbacks)
518
496
 
519
497
  @cached_property
520
498
  def hook(self) -> PodOperatorHookProtocol:
@@ -563,7 +541,7 @@ class KubernetesPodOperator(BaseOperator):
563
541
 
564
542
  def get_or_create_pod(self, pod_request_obj: k8s.V1Pod, context: Context) -> k8s.V1Pod:
565
543
  if self.reattach_on_restart:
566
- pod = self.find_pod(self.namespace or pod_request_obj.metadata.namespace, context=context)
544
+ pod = self.find_pod(pod_request_obj.metadata.namespace, context=context)
567
545
  if pod:
568
546
  return pod
569
547
  self.log.debug("Starting pod:\n%s", yaml.safe_dump(pod_request_obj.to_dict()))
@@ -1161,10 +1139,6 @@ class KubernetesPodOperator(BaseOperator):
1161
1139
  pod = self.build_pod_request_obj()
1162
1140
  print(yaml.dump(prune_dict(pod.to_dict(), mode="strict")))
1163
1141
 
1164
- @deprecated(reason="use `trigger_reentry` instead.", category=AirflowProviderDeprecationWarning)
1165
- def execute_complete(self, context: Context, event: dict, **kwargs):
1166
- return self.trigger_reentry(context=context, event=event)
1167
-
1168
1142
  def process_duplicate_label_pods(self, pod_list: list[k8s.V1Pod]) -> k8s.V1Pod:
1169
1143
  """
1170
1144
  Patch or delete the existing pod with duplicate labels.
@@ -17,6 +17,7 @@
17
17
  # under the License.
18
18
  from __future__ import annotations
19
19
 
20
+ from collections.abc import Mapping
20
21
  from functools import cached_property
21
22
  from pathlib import Path
22
23
  from typing import TYPE_CHECKING, Any
@@ -67,6 +68,7 @@ class SparkKubernetesOperator(KubernetesPodOperator):
67
68
  state, or the execution is interrupted. If True (default), delete the
68
69
  pod; if False, leave the pod.
69
70
  :param kubernetes_conn_id: the connection to Kubernetes cluster
71
+ :param random_name_suffix: If True, adds a random suffix to the pod name
70
72
  """
71
73
 
72
74
  template_fields = ["application_file", "namespace", "template_spec", "kubernetes_conn_id"]
@@ -93,10 +95,9 @@ class SparkKubernetesOperator(KubernetesPodOperator):
93
95
  reattach_on_restart: bool = True,
94
96
  delete_on_termination: bool = True,
95
97
  kubernetes_conn_id: str = "kubernetes_default",
98
+ random_name_suffix: bool = True,
96
99
  **kwargs,
97
100
  ) -> None:
98
- if kwargs.get("xcom_push") is not None:
99
- raise AirflowException("'xcom_push' was deprecated, use 'do_xcom_push' instead")
100
101
  super().__init__(name=name, **kwargs)
101
102
  self.image = image
102
103
  self.code_path = code_path
@@ -111,6 +112,7 @@ class SparkKubernetesOperator(KubernetesPodOperator):
111
112
  self.get_logs = get_logs
112
113
  self.log_events_on_failure = log_events_on_failure
113
114
  self.success_run_history_limit = success_run_history_limit
115
+ self.random_name_suffix = random_name_suffix
114
116
 
115
117
  if self.base_container_name != self.BASE_CONTAINER_NAME:
116
118
  self.log.warning(
@@ -127,7 +129,7 @@ class SparkKubernetesOperator(KubernetesPodOperator):
127
129
  def _render_nested_template_fields(
128
130
  self,
129
131
  content: Any,
130
- context: Context,
132
+ context: Mapping[str, Any],
131
133
  jinja_env: jinja2.Environment,
132
134
  seen_oids: set,
133
135
  ) -> None:
@@ -163,7 +165,11 @@ class SparkKubernetesOperator(KubernetesPodOperator):
163
165
  self.name or self.template_body.get("spark", {}).get("metadata", {}).get("name") or self.task_id
164
166
  )
165
167
 
166
- updated_name = add_unique_suffix(name=name, max_len=MAX_LABEL_LEN)
168
+ if self.random_name_suffix:
169
+ updated_name = add_unique_suffix(name=name, max_len=MAX_LABEL_LEN)
170
+ else:
171
+ # truncation is required to maintain the same behavior as before
172
+ updated_name = name[:MAX_LABEL_LEN]
167
173
 
168
174
  return self._set_name(updated_name)
169
175
 
@@ -34,23 +34,16 @@ from typing import TYPE_CHECKING
34
34
 
35
35
  import re2
36
36
  from dateutil import parser
37
- from deprecated import deprecated
38
37
  from kubernetes.client import models as k8s
39
38
  from kubernetes.client.api_client import ApiClient
40
39
 
41
40
  from airflow.exceptions import (
42
41
  AirflowConfigException,
43
42
  AirflowException,
44
- AirflowProviderDeprecationWarning,
45
43
  )
46
44
  from airflow.providers.cncf.kubernetes.kubernetes_helper_functions import (
47
45
  POD_NAME_MAX_LENGTH,
48
46
  add_unique_suffix,
49
- rand_str,
50
- )
51
- from airflow.providers.cncf.kubernetes.pod_generator_deprecated import (
52
- PodDefaults as PodDefaultsDeprecated,
53
- PodGenerator as PodGeneratorDeprecated,
54
47
  )
55
48
  from airflow.utils import yaml
56
49
  from airflow.utils.hashlib_wrapper import md5
@@ -155,40 +148,6 @@ class PodGenerator:
155
148
  # Attach sidecar
156
149
  self.extract_xcom = extract_xcom
157
150
 
158
- @deprecated(
159
- reason="This method is deprecated and will be removed in the future releases",
160
- category=AirflowProviderDeprecationWarning,
161
- )
162
- def gen_pod(self) -> k8s.V1Pod:
163
- """Generate pod."""
164
- result = self.ud_pod
165
-
166
- result.metadata.name = add_unique_suffix(name=result.metadata.name)
167
-
168
- if self.extract_xcom:
169
- result = self.add_xcom_sidecar(result)
170
-
171
- return result
172
-
173
- @staticmethod
174
- @deprecated(
175
- reason=(
176
- "This function is deprecated. "
177
- "Please use airflow.providers.cncf.kubernetes.utils.xcom_sidecar.add_xcom_sidecar instead"
178
- ),
179
- category=AirflowProviderDeprecationWarning,
180
- )
181
- def add_xcom_sidecar(pod: k8s.V1Pod) -> k8s.V1Pod:
182
- """Add sidecar."""
183
- pod_cp = copy.deepcopy(pod)
184
- pod_cp.spec.volumes = pod.spec.volumes or []
185
- pod_cp.spec.volumes.insert(0, PodDefaultsDeprecated.VOLUME)
186
- pod_cp.spec.containers[0].volume_mounts = pod_cp.spec.containers[0].volume_mounts or []
187
- pod_cp.spec.containers[0].volume_mounts.insert(0, PodDefaultsDeprecated.VOLUME_MOUNT)
188
- pod_cp.spec.containers.append(PodDefaultsDeprecated.SIDECAR_CONTAINER)
189
-
190
- return pod_cp
191
-
192
151
  @staticmethod
193
152
  def from_obj(obj) -> dict | k8s.V1Pod | None:
194
153
  """Convert to pod from obj."""
@@ -210,57 +169,11 @@ class PodGenerator:
210
169
 
211
170
  if isinstance(k8s_object, k8s.V1Pod):
212
171
  return k8s_object
213
- elif isinstance(k8s_legacy_object, dict):
214
- warnings.warn(
215
- "Using a dictionary for the executor_config is deprecated and will soon be removed. "
216
- 'Please use a `kubernetes.client.models.V1Pod` class with a "pod_override" key'
217
- " instead. ",
218
- category=AirflowProviderDeprecationWarning,
219
- stacklevel=2,
220
- )
221
- return PodGenerator.from_legacy_obj(obj)
222
172
  else:
223
173
  raise TypeError(
224
174
  "Cannot convert a non-kubernetes.client.models.V1Pod object into a KubernetesExecutorConfig"
225
175
  )
226
176
 
227
- @staticmethod
228
- def from_legacy_obj(obj) -> k8s.V1Pod | None:
229
- """Convert to pod from obj."""
230
- if obj is None:
231
- return None
232
-
233
- # We do not want to extract constant here from ExecutorLoader because it is just
234
- # A name in dictionary rather than executor selection mechanism and it causes cyclic import
235
- namespaced = obj.get("KubernetesExecutor", {})
236
-
237
- if not namespaced:
238
- return None
239
-
240
- resources = namespaced.get("resources")
241
-
242
- if resources is None:
243
- requests = {
244
- "cpu": namespaced.pop("request_cpu", None),
245
- "memory": namespaced.pop("request_memory", None),
246
- "ephemeral-storage": namespaced.get("ephemeral-storage"), # We pop this one in limits
247
- }
248
- limits = {
249
- "cpu": namespaced.pop("limit_cpu", None),
250
- "memory": namespaced.pop("limit_memory", None),
251
- "ephemeral-storage": namespaced.pop("ephemeral-storage", None),
252
- }
253
- all_resources = list(requests.values()) + list(limits.values())
254
- if all(r is None for r in all_resources):
255
- resources = None
256
- else:
257
- # remove None's so they don't become 0's
258
- requests = {k: v for k, v in requests.items() if v is not None}
259
- limits = {k: v for k, v in limits.items() if v is not None}
260
- resources = k8s.V1ResourceRequirements(requests=requests, limits=limits)
261
- namespaced["resources"] = resources
262
- return PodGeneratorDeprecated(**namespaced).gen_pod()
263
-
264
177
  @staticmethod
265
178
  def reconcile_pods(base_pod: k8s.V1Pod, client_pod: k8s.V1Pod | None) -> k8s.V1Pod:
266
179
  """
@@ -579,37 +492,6 @@ class PodGenerator:
579
492
  api_client = ApiClient()
580
493
  return api_client._ApiClient__deserialize_model(pod_dict, k8s.V1Pod)
581
494
 
582
- @staticmethod
583
- @deprecated(
584
- reason="This method is deprecated. Use `add_pod_suffix` in `kubernetes_helper_functions`.",
585
- category=AirflowProviderDeprecationWarning,
586
- )
587
- def make_unique_pod_id(pod_id: str) -> str | None:
588
- r"""
589
- Generate a unique Pod name.
590
-
591
- Kubernetes pod names must consist of one or more lowercase
592
- rfc1035/rfc1123 labels separated by '.' with a maximum length of 253
593
- characters.
594
-
595
- Name must pass the following regex for validation
596
- ``^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$``
597
-
598
- For more details, see:
599
- https://github.com/kubernetes/kubernetes/blob/release-1.1/docs/design/identifiers.md
600
-
601
- :param pod_id: requested pod name
602
- :return: ``str`` valid Pod name of appropriate length
603
- """
604
- if not pod_id:
605
- return None
606
-
607
- max_pod_id_len = 100 # arbitrarily chosen
608
- suffix = rand_str(8) # 8 seems good enough
609
- base_pod_id_len = max_pod_id_len - len(suffix) - 1 # -1 for separator
610
- trimmed_pod_id = pod_id[:base_pod_id_len].rstrip("-.")
611
- return f"{trimmed_pod_id}-{suffix}"
612
-
613
495
 
614
496
  def merge_objects(base_obj, client_obj):
615
497
  """
@@ -19,12 +19,10 @@ from __future__ import annotations
19
19
  import asyncio
20
20
  import datetime
21
21
  import traceback
22
- import warnings
23
22
  from enum import Enum
24
23
  from functools import cached_property
25
24
  from typing import TYPE_CHECKING, Any, AsyncIterator
26
25
 
27
- from airflow.exceptions import AirflowProviderDeprecationWarning
28
26
  from airflow.providers.cncf.kubernetes.hooks.kubernetes import AsyncKubernetesHook
29
27
  from airflow.providers.cncf.kubernetes.utils.pod_manager import (
30
28
  OnFinishAction,
@@ -71,10 +69,6 @@ class KubernetesPodTrigger(BaseTrigger):
71
69
  :param on_finish_action: What to do when the pod reaches its final state, or the execution is interrupted.
72
70
  If "delete_pod", the pod will be deleted regardless its state; if "delete_succeeded_pod",
73
71
  only succeeded pod will be deleted. You can set to "keep_pod" to keep the pod.
74
- :param should_delete_pod: What to do when the pod reaches its final
75
- state, or the execution is interrupted. If True (default), delete the
76
- pod; if False, leave the pod.
77
- Deprecated - use `on_finish_action` instead.
78
72
  :param logging_interval: number of seconds to wait before kicking it back to
79
73
  the operator to print latest logs. If ``None`` will wait until container done.
80
74
  :param last_log_time: where to resume logs from
@@ -95,7 +89,6 @@ class KubernetesPodTrigger(BaseTrigger):
95
89
  startup_timeout: int = 120,
96
90
  startup_check_interval: int = 5,
97
91
  on_finish_action: str = "delete_pod",
98
- should_delete_pod: bool | None = None,
99
92
  last_log_time: DateTime | None = None,
100
93
  logging_interval: int | None = None,
101
94
  ):
@@ -114,20 +107,7 @@ class KubernetesPodTrigger(BaseTrigger):
114
107
  self.startup_check_interval = startup_check_interval
115
108
  self.last_log_time = last_log_time
116
109
  self.logging_interval = logging_interval
117
-
118
- if should_delete_pod is not None:
119
- warnings.warn(
120
- "`should_delete_pod` parameter is deprecated, please use `on_finish_action`",
121
- category=AirflowProviderDeprecationWarning,
122
- stacklevel=2,
123
- )
124
- self.on_finish_action = (
125
- OnFinishAction.DELETE_POD if should_delete_pod else OnFinishAction.KEEP_POD
126
- )
127
- self.should_delete_pod = should_delete_pod
128
- else:
129
- self.on_finish_action = OnFinishAction(on_finish_action)
130
- self.should_delete_pod = self.on_finish_action == OnFinishAction.DELETE_POD
110
+ self.on_finish_action = OnFinishAction(on_finish_action)
131
111
 
132
112
  self._since_time = None
133
113
 
@@ -148,7 +128,6 @@ class KubernetesPodTrigger(BaseTrigger):
148
128
  "startup_timeout": self.startup_timeout,
149
129
  "startup_check_interval": self.startup_check_interval,
150
130
  "trigger_start_time": self.trigger_start_time,
151
- "should_delete_pod": self.should_delete_pod,
152
131
  "on_finish_action": self.on_finish_action.value,
153
132
  "last_log_time": self.last_log_time,
154
133
  "logging_interval": self.logging_interval,
@@ -26,11 +26,10 @@ from collections.abc import Iterable
26
26
  from contextlib import closing, suppress
27
27
  from dataclasses import dataclass
28
28
  from datetime import timedelta
29
- from typing import TYPE_CHECKING, Callable, Generator, Protocol, cast
29
+ from typing import TYPE_CHECKING, Generator, Protocol, cast
30
30
 
31
31
  import pendulum
32
32
  import tenacity
33
- from deprecated import deprecated
34
33
  from kubernetes import client, watch
35
34
  from kubernetes.client.rest import ApiException
36
35
  from kubernetes.stream import stream as kubernetes_stream
@@ -39,7 +38,7 @@ from pendulum.parsing.exceptions import ParserError
39
38
  from typing_extensions import Literal
40
39
  from urllib3.exceptions import HTTPError, TimeoutError
41
40
 
42
- from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
41
+ from airflow.exceptions import AirflowException
43
42
  from airflow.providers.cncf.kubernetes.callbacks import ExecutionMode, KubernetesPodOperatorCallback
44
43
  from airflow.providers.cncf.kubernetes.utils.xcom_sidecar import PodDefaults
45
44
  from airflow.utils.log.logging_mixin import LoggingMixin
@@ -302,19 +301,15 @@ class PodManager(LoggingMixin):
302
301
  self,
303
302
  kube_client: client.CoreV1Api,
304
303
  callbacks: type[KubernetesPodOperatorCallback] | None = None,
305
- progress_callback: Callable[[str], None] | None = None,
306
304
  ):
307
305
  """
308
306
  Create the launcher.
309
307
 
310
308
  :param kube_client: kubernetes client
311
309
  :param callbacks:
312
- :param progress_callback: Callback function invoked when fetching container log.
313
- This parameter is deprecated, please use ````
314
310
  """
315
311
  super().__init__()
316
312
  self._client = kube_client
317
- self._progress_callback = progress_callback
318
313
  self._watch = watch.Watch()
319
314
  self._callbacks = callbacks
320
315
 
@@ -383,16 +378,6 @@ class PodManager(LoggingMixin):
383
378
  raise PodLaunchFailedException(msg)
384
379
  time.sleep(startup_check_interval)
385
380
 
386
- @deprecated(
387
- reason=(
388
- "Method `follow_container_logs` is deprecated. Use `fetch_container_logs` instead "
389
- "with option `follow=True`."
390
- ),
391
- category=AirflowProviderDeprecationWarning,
392
- )
393
- def follow_container_logs(self, pod: V1Pod, container_name: str) -> PodLoggingStatus:
394
- return self.fetch_container_logs(pod=pod, container_name=container_name, follow=True)
395
-
396
381
  def fetch_container_logs(
397
382
  self,
398
383
  pod: V1Pod,
@@ -461,8 +446,6 @@ class PodManager(LoggingMixin):
461
446
  progress_callback_lines.append(line)
462
447
  else: # previous log line is complete
463
448
  for line in progress_callback_lines:
464
- if self._progress_callback:
465
- self._progress_callback(line)
466
449
  if self._callbacks:
467
450
  self._callbacks.progress_callback(
468
451
  line=line, client=self._client, mode=ExecutionMode.SYNC
@@ -479,8 +462,6 @@ class PodManager(LoggingMixin):
479
462
  finally:
480
463
  # log the last line and update the last_captured_timestamp
481
464
  for line in progress_callback_lines:
482
- if self._progress_callback:
483
- self._progress_callback(line)
484
465
  if self._callbacks:
485
466
  self._callbacks.progress_callback(
486
467
  line=line, client=self._client, mode=ExecutionMode.SYNC
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.3
2
2
  Name: apache-airflow-providers-cncf-kubernetes
3
- Version: 9.0.1
3
+ Version: 10.0.0
4
4
  Summary: Provider package apache-airflow-providers-cncf-kubernetes for Apache Airflow
5
5
  Keywords: airflow-provider,cncf.kubernetes,airflow,integration
6
6
  Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -28,8 +28,8 @@ Requires-Dist: google-re2>=1.0
28
28
  Requires-Dist: kubernetes>=29.0.0,<=31.0.0
29
29
  Requires-Dist: kubernetes_asyncio>=29.0.0,<=31.0.0
30
30
  Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
31
- Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/9.0.1/changelog.html
32
- Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/9.0.1
31
+ Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.0.0/changelog.html
32
+ Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.0.0
33
33
  Project-URL: Slack Chat, https://s.apache.org/airflow-slack
34
34
  Project-URL: Source Code, https://github.com/apache/airflow
35
35
  Project-URL: Twitter, https://twitter.com/ApacheAirflow
@@ -79,7 +79,7 @@ Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
79
79
 
80
80
  Package ``apache-airflow-providers-cncf-kubernetes``
81
81
 
82
- Release: ``9.0.1``
82
+ Release: ``10.0.0``
83
83
 
84
84
 
85
85
  `Kubernetes <https://kubernetes.io/>`__
@@ -92,7 +92,7 @@ This is a provider package for ``cncf.kubernetes`` provider. All classes for thi
92
92
  are in ``airflow.providers.cncf.kubernetes`` python package.
93
93
 
94
94
  You can find package information and changelog for the provider
95
- in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/9.0.1/>`_.
95
+ in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.0.0/>`_.
96
96
 
97
97
  Installation
98
98
  ------------
@@ -119,4 +119,4 @@ PIP package Version required
119
119
  ====================== =====================
120
120
 
121
121
  The changelog for the provider package can be found in the
122
- `changelog <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/9.0.1/changelog.html>`_.
122
+ `changelog <https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.0.0/changelog.html>`_.
@@ -1,14 +1,13 @@
1
1
  airflow/providers/cncf/kubernetes/LICENSE,sha256=FFb4jd2AXnOOf7XLP04pQW6jbdhG49TxlGY6fFpCV1Y,13609
2
- airflow/providers/cncf/kubernetes/__init__.py,sha256=p5-ifXMiybbsAM0J0xLXPmubE5pHXdZmpYD5CriC2Ek,1502
2
+ airflow/providers/cncf/kubernetes/__init__.py,sha256=nxLYWteyTzCKa1NCHauBZsE1vOl0TSVqpqd4IR6taqg,1503
3
3
  airflow/providers/cncf/kubernetes/callbacks.py,sha256=SK_gKvGWuU-nxHfsqsYMlNQ8HZbHfpvyItOqieel2lc,4162
4
- airflow/providers/cncf/kubernetes/get_provider_info.py,sha256=B0FUhOfC6Xkj7ynRLYH2CJCEHMVNRQMStARc89kfEC4,17986
4
+ airflow/providers/cncf/kubernetes/get_provider_info.py,sha256=UdYm7n5XGH9N6hcCKJr5hMC5ekk-2lNP5wgj0plC5JY,18008
5
5
  airflow/providers/cncf/kubernetes/k8s_model.py,sha256=xmdFhX29DjegoZ-cq8-KDL9soVYXf4OpU6fAGr3cPTU,2101
6
6
  airflow/providers/cncf/kubernetes/kube_client.py,sha256=yflZxLousXA9d7t67KrEy55qzb1cUhEyy6yCPkEem28,5329
7
7
  airflow/providers/cncf/kubernetes/kube_config.py,sha256=FAmhZZ_Z2JtoVzL6wENSjcwrlwAenHttTX_Ild9aEms,5225
8
- airflow/providers/cncf/kubernetes/kubernetes_helper_functions.py,sha256=fAwcXq2vkxbQP0Sc1yOjVX8IvVBlt5ThwzwtRV1DmTQ,6765
9
- airflow/providers/cncf/kubernetes/pod_generator.py,sha256=ihmz3-R6anQ1bg_gQ42B5SqpJdlEjZHXro3isfwiUU4,24522
8
+ airflow/providers/cncf/kubernetes/kubernetes_helper_functions.py,sha256=tb6HykGOUKZIXXZssJyQLKBlWv_KoE5mjjxwOkeJHW0,5391
9
+ airflow/providers/cncf/kubernetes/pod_generator.py,sha256=9Yd4FY0MbXc61JuOZwQu5WlFV6MlLBy72YMHF3lD8ao,19765
10
10
  airflow/providers/cncf/kubernetes/pod_generator_deprecated.py,sha256=aukRdia0Wtv0FVZXgvaIWQxdWfnT9cxzH0UwOVph8dE,11998
11
- airflow/providers/cncf/kubernetes/pod_launcher_deprecated.py,sha256=v1dpXnlpGI6nTmwO-h15rfs42NYcg9lrEpxrgA0ynE8,12039
12
11
  airflow/providers/cncf/kubernetes/python_kubernetes_script.jinja2,sha256=I0EHRGwLHjSiX85e51HBIoddRDnC8TJPFrDBqQq_NJg,1776
13
12
  airflow/providers/cncf/kubernetes/python_kubernetes_script.py,sha256=KnTlZSWCZhwvj89fSc2kgIRTaI4iLNKPquHc2wXnluo,3460
14
13
  airflow/providers/cncf/kubernetes/secret.py,sha256=wj-T9gouqau_X14slAstGmnSxqXJQzdLwUdURzHna0I,5209
@@ -18,12 +17,12 @@ airflow/providers/cncf/kubernetes/backcompat/backwards_compat_converters.py,sha2
18
17
  airflow/providers/cncf/kubernetes/cli/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
19
18
  airflow/providers/cncf/kubernetes/cli/kubernetes_command.py,sha256=W49hsWZ3XWUJgFLte56tMNGqmOvcFitl7iFjmFR8Ezo,6990
20
19
  airflow/providers/cncf/kubernetes/decorators/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
21
- airflow/providers/cncf/kubernetes/decorators/kubernetes.py,sha256=qfU_MDOhBsHc2leu1RW3YE5steDFOnWxcVswhjC6GXU,5785
20
+ airflow/providers/cncf/kubernetes/decorators/kubernetes.py,sha256=zVDHw_zwgRbobTOouJtCEqDrjrVwINGcB1dA-wG-ME0,5787
22
21
  airflow/providers/cncf/kubernetes/executors/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
23
- airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py,sha256=tco4nuKFjKmVA_pIsjli-NlB9DA31cq_845hD54N6mM,34006
22
+ airflow/providers/cncf/kubernetes/executors/kubernetes_executor.py,sha256=mVNIA5SSJj3hRWJy05Rp6DVhzNgrxW-pyENeIAv9Jdg,34271
24
23
  airflow/providers/cncf/kubernetes/executors/kubernetes_executor_types.py,sha256=9rRhfRuujGtSE73Ax0kC12whZCgWF2m6j5w9G9e0F_I,1673
25
24
  airflow/providers/cncf/kubernetes/executors/kubernetes_executor_utils.py,sha256=f2FHTyQyfFR1jjQmZV9bfX7U4Vxk5h61KPJO0yMcGbM,23356
26
- airflow/providers/cncf/kubernetes/executors/local_kubernetes_executor.py,sha256=tlUMdInlQjw5pJTxtyQ1lBzq8HCVttNLvZsCtNtb1Io,10844
25
+ airflow/providers/cncf/kubernetes/executors/local_kubernetes_executor.py,sha256=6cEGS83tGVUm6tSsjVenaJXSstViBnwgrvu-XIW4XQg,10976
27
26
  airflow/providers/cncf/kubernetes/hooks/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
28
27
  airflow/providers/cncf/kubernetes/hooks/kubernetes.py,sha256=WSWQZrAzyrjN9y5TCS796X1L7oB-Sq_2BUzhT9xAE2k,33632
29
28
  airflow/providers/cncf/kubernetes/kubernetes_executor_templates/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
@@ -31,10 +30,9 @@ airflow/providers/cncf/kubernetes/kubernetes_executor_templates/basic_template.y
31
30
  airflow/providers/cncf/kubernetes/operators/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
32
31
  airflow/providers/cncf/kubernetes/operators/custom_object_launcher.py,sha256=ZEXw_PqGItO47AEgGKqAqwFHFo-gb9-7jgEMRJgOfNU,15311
33
32
  airflow/providers/cncf/kubernetes/operators/job.py,sha256=RyPuT0yP7-tRrrS7uqDUBGI_xA9_YEg-JLUDJ5QcTA8,23809
34
- airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py,sha256=EFwHEdEqq4I18BeV9hgp_GFxrLI78ihZAccbyGLSjtY,1269
35
- airflow/providers/cncf/kubernetes/operators/pod.py,sha256=MAHaLxy5ScgbY0LO5i3ejH3MHqYyXG3a9harRg-8guE,54803
33
+ airflow/providers/cncf/kubernetes/operators/pod.py,sha256=f8JoEsgZbjSNlClTfw8XeCm-_vSkwpUglXzOiDyhBUg,53495
36
34
  airflow/providers/cncf/kubernetes/operators/resource.py,sha256=ccbZQKB1B5N4Y-ruItacB5Q105Tc0uNSoEmUAaSCrGQ,7570
37
- airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py,sha256=oi1JUCoRo7AzAOYhKgbCB60Hmg_slRdj3FsQKWQRhcI,13668
35
+ airflow/providers/cncf/kubernetes/operators/spark_kubernetes.py,sha256=rsUfqJ_TqjqZSIeUvc6J7pTJiaiX1Ba2WI5Tn9qtfbw,13923
38
36
  airflow/providers/cncf/kubernetes/pod_template_file_examples/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
39
37
  airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_image_template.yaml,sha256=7JdppZ-XDBpv2Bnde2SthhcME8w3b8xQdPAK1fJGW60,2256
40
38
  airflow/providers/cncf/kubernetes/pod_template_file_examples/dags_in_volume_template.yaml,sha256=-Pk_EwKpyWRYZKOnumUxVrDeAfFJ0nr3WZ7JNnvppzg,2442
@@ -47,14 +45,13 @@ airflow/providers/cncf/kubernetes/sensors/__init__.py,sha256=9hdXHABrVpkbpjZgUft
47
45
  airflow/providers/cncf/kubernetes/sensors/spark_kubernetes.py,sha256=ogD8m8baqPN_hGEXjN1NGp1dvslG8ZPBjy_LPr6Zo7I,5353
48
46
  airflow/providers/cncf/kubernetes/triggers/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
49
47
  airflow/providers/cncf/kubernetes/triggers/job.py,sha256=fe0kaQos_m_1rVE-1YJBZKKFVLxrlHOB8oI5ntVzSN8,6727
50
- airflow/providers/cncf/kubernetes/triggers/kubernetes_pod.py,sha256=Cfq-gmdXsW2QGFiX-EYAGsCpN_MDvofbfr3bK7QfQho,1266
51
- airflow/providers/cncf/kubernetes/triggers/pod.py,sha256=4L5ZXswM11kAwAietMcfwPuelA8CJ12Jgo40Lndsquc,13629
48
+ airflow/providers/cncf/kubernetes/triggers/pod.py,sha256=Q-1GPILDgGJHoxtkaH78edhNeR6BekRG8f0qpFLGcSI,12626
52
49
  airflow/providers/cncf/kubernetes/utils/__init__.py,sha256=ClZN0VPjWySdVwS_ktH7rrgL9VLAcs3OSJSB9s3zaYw,863
53
50
  airflow/providers/cncf/kubernetes/utils/delete_from.py,sha256=poObZSoEJwQyaYWilEURs8f4CDY2sn_pfwS31Lf579A,5195
54
51
  airflow/providers/cncf/kubernetes/utils/k8s_resource_iterator.py,sha256=-Pgc5i2WEDl7ZBvtJZ4eWDqqlSj8WdULqwUyOWmsRp8,1928
55
- airflow/providers/cncf/kubernetes/utils/pod_manager.py,sha256=eITiFHU8lsCy3UwRUm-3uOQ3J1BgkshaXd1FJOJ3Q4A,34298
52
+ airflow/providers/cncf/kubernetes/utils/pod_manager.py,sha256=Xp_K8nay4WYKbG_cJ9mrhc_sFkdmu40Klq859sq9vNs,33286
56
53
  airflow/providers/cncf/kubernetes/utils/xcom_sidecar.py,sha256=k6bdmVJ21OrAwGmWwledRrAmaty9ZrmbuM-IbaI4mqo,2519
57
- apache_airflow_providers_cncf_kubernetes-9.0.1.dist-info/entry_points.txt,sha256=ByD3QJJyP9CfmTYtpNI1953akD38RUDgpGXLaq9vpOw,111
58
- apache_airflow_providers_cncf_kubernetes-9.0.1.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
59
- apache_airflow_providers_cncf_kubernetes-9.0.1.dist-info/METADATA,sha256=TcqzgCgCkgrxxFlTSpaaOjhIiN747VjmE1fVa4hlTQY,5206
60
- apache_airflow_providers_cncf_kubernetes-9.0.1.dist-info/RECORD,,
54
+ apache_airflow_providers_cncf_kubernetes-10.0.0.dist-info/entry_points.txt,sha256=ByD3QJJyP9CfmTYtpNI1953akD38RUDgpGXLaq9vpOw,111
55
+ apache_airflow_providers_cncf_kubernetes-10.0.0.dist-info/WHEEL,sha256=CpUCUxeHQbRN5UGRQHYRJorO5Af-Qy_fHMctcQ8DSGI,82
56
+ apache_airflow_providers_cncf_kubernetes-10.0.0.dist-info/METADATA,sha256=GQI7La3mlDH3aGgNxvWhkZSNNkJs4jEBkGLozAKa1WM,5212
57
+ apache_airflow_providers_cncf_kubernetes-10.0.0.dist-info/RECORD,,
@@ -1,4 +1,4 @@
1
1
  Wheel-Version: 1.0
2
- Generator: flit 3.9.0
2
+ Generator: flit 3.10.1
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
@@ -1,31 +0,0 @@
1
- #
2
- # Licensed to the Apache Software Foundation (ASF) under one
3
- # or more contributor license agreements. See the NOTICE file
4
- # distributed with this work for additional information
5
- # regarding copyright ownership. The ASF licenses this file
6
- # to you under the Apache License, Version 2.0 (the
7
- # "License"); you may not use this file except in compliance
8
- # with the License. You may obtain a copy of the License at
9
- #
10
- # http://www.apache.org/licenses/LICENSE-2.0
11
- #
12
- # Unless required by applicable law or agreed to in writing,
13
- # software distributed under the License is distributed on an
14
- # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15
- # KIND, either express or implied. See the License for the
16
- # specific language governing permissions and limitations
17
- # under the License.
18
- """This module is deprecated. Please use :mod:`airflow.providers.cncf.kubernetes.operators.pod` instead."""
19
-
20
- from __future__ import annotations
21
-
22
- import warnings
23
-
24
- from airflow.exceptions import AirflowProviderDeprecationWarning
25
- from airflow.providers.cncf.kubernetes.operators.pod import * # noqa: F403
26
-
27
- warnings.warn(
28
- "This module is deprecated. Please use `airflow.providers.cncf.kubernetes.operators.pod` instead.",
29
- AirflowProviderDeprecationWarning,
30
- stacklevel=2,
31
- )
@@ -1,320 +0,0 @@
1
- # Licensed to the Apache Software Foundation (ASF) under one
2
- # or more contributor license agreements. See the NOTICE file
3
- # distributed with this work for additional information
4
- # regarding copyright ownership. The ASF licenses this file
5
- # to you under the Apache License, Version 2.0 (the
6
- # "License"); you may not use this file except in compliance
7
- # with the License. You may obtain a copy of the License at
8
- #
9
- # http://www.apache.org/licenses/LICENSE-2.0
10
- #
11
- # Unless required by applicable law or agreed to in writing,
12
- # software distributed under the License is distributed on an
13
- # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
- # KIND, either express or implied. See the License for the
15
- # specific language governing permissions and limitations
16
- # under the License.
17
- """Launches pods."""
18
-
19
- from __future__ import annotations
20
-
21
- import json
22
- import math
23
- import time
24
- import warnings
25
- from typing import TYPE_CHECKING, cast
26
-
27
- import pendulum
28
- import tenacity
29
- from kubernetes import client, watch
30
- from kubernetes.client.rest import ApiException
31
- from kubernetes.stream import stream as kubernetes_stream
32
- from requests.exceptions import HTTPError
33
-
34
- from airflow.exceptions import AirflowException, RemovedInAirflow3Warning
35
- from airflow.providers.cncf.kubernetes.kube_client import get_kube_client
36
- from airflow.providers.cncf.kubernetes.pod_generator import PodDefaultsDeprecated
37
- from airflow.settings import pod_mutation_hook
38
- from airflow.utils.log.logging_mixin import LoggingMixin
39
- from airflow.utils.state import State
40
-
41
- if TYPE_CHECKING:
42
- from kubernetes.client.models.v1_pod import V1Pod
43
-
44
- warnings.warn(
45
- """
46
- Please use :mod: Please use `airflow.providers.cncf.kubernetes.utils.pod_manager`
47
-
48
- To use this module install the provider package by installing this pip package:
49
-
50
- https://pypi.org/project/apache-airflow-providers-cncf-kubernetes/
51
-
52
- """,
53
- RemovedInAirflow3Warning,
54
- stacklevel=2,
55
- )
56
-
57
-
58
- class PodStatus:
59
- """Status of the pods."""
60
-
61
- PENDING = "pending"
62
- RUNNING = "running"
63
- FAILED = "failed"
64
- SUCCEEDED = "succeeded"
65
-
66
-
67
- class PodLauncher(LoggingMixin):
68
- """
69
- Deprecated class for launching pods.
70
-
71
- Please use airflow.providers.cncf.kubernetes.utils.pod_manager.PodManager instead.
72
- """
73
-
74
- def __init__(
75
- self,
76
- kube_client: client.CoreV1Api = None,
77
- in_cluster: bool = True,
78
- cluster_context: str | None = None,
79
- extract_xcom: bool = False,
80
- ):
81
- """
82
- Launch pods; DEPRECATED.
83
-
84
- Please use airflow.providers.cncf.kubernetes.utils.pod_manager.PodManager
85
- instead to create the launcher.
86
-
87
- :param kube_client: kubernetes client
88
- :param in_cluster: whether we are in cluster
89
- :param cluster_context: context of the cluster
90
- :param extract_xcom: whether we should extract xcom
91
- """
92
- super().__init__()
93
- self._client = kube_client or get_kube_client(in_cluster=in_cluster, cluster_context=cluster_context)
94
- self._watch = watch.Watch()
95
- self.extract_xcom = extract_xcom
96
-
97
- def run_pod_async(self, pod: V1Pod, **kwargs):
98
- """Run pod asynchronously."""
99
- pod_mutation_hook(pod)
100
-
101
- sanitized_pod = self._client.api_client.sanitize_for_serialization(pod)
102
- json_pod = json.dumps(sanitized_pod, indent=2)
103
-
104
- self.log.debug("Pod Creation Request: \n%s", json_pod)
105
- try:
106
- resp = self._client.create_namespaced_pod(
107
- body=sanitized_pod, namespace=pod.metadata.namespace, **kwargs
108
- )
109
- self.log.debug("Pod Creation Response: %s", resp)
110
- except Exception as e:
111
- self.log.exception("Exception when attempting to create Namespaced Pod: %s", json_pod)
112
- raise e
113
- return resp
114
-
115
- def delete_pod(self, pod: V1Pod):
116
- """Delete pod."""
117
- try:
118
- self._client.delete_namespaced_pod(
119
- pod.metadata.name, pod.metadata.namespace, body=client.V1DeleteOptions()
120
- )
121
- except ApiException as e:
122
- # If the pod is already deleted
123
- if str(e.status) != "404":
124
- raise
125
-
126
- def start_pod(self, pod: V1Pod, startup_timeout: int = 120):
127
- """
128
- Launch the pod synchronously and wait for completion.
129
-
130
- :param pod:
131
- :param startup_timeout: Timeout for startup of the pod (if pod is pending for too long, fails task)
132
- :return:
133
- """
134
- resp = self.run_pod_async(pod)
135
- start_time = time.monotonic()
136
- if resp.status.start_time is None:
137
- while self.pod_not_started(pod):
138
- self.log.warning("Pod not yet started: %s", pod.metadata.name)
139
- if time.monotonic() >= start_time + startup_timeout:
140
- raise AirflowException("Pod took too long to start")
141
- time.sleep(1)
142
-
143
- def monitor_pod(self, pod: V1Pod, get_logs: bool) -> tuple[State, str | None]:
144
- """
145
- Monitor a pod and return the final state.
146
-
147
- :param pod: pod spec that will be monitored
148
- :param get_logs: whether to read the logs locally
149
- """
150
- if get_logs:
151
- read_logs_since_sec = None
152
- last_log_time: pendulum.DateTime | None = None
153
- while True:
154
- logs = self.read_pod_logs(pod, timestamps=True, since_seconds=read_logs_since_sec)
155
- for line in logs:
156
- timestamp, message = self.parse_log_line(line.decode("utf-8"))
157
- if timestamp:
158
- last_log_time = cast(pendulum.DateTime, pendulum.parse(timestamp))
159
- self.log.info(message)
160
- time.sleep(1)
161
-
162
- if not self.base_container_is_running(pod):
163
- break
164
-
165
- self.log.warning("Pod %s log read interrupted", pod.metadata.name)
166
- if last_log_time:
167
- delta = pendulum.now() - last_log_time
168
- # Prefer logs duplication rather than loss
169
- read_logs_since_sec = math.ceil(delta.total_seconds())
170
- result = None
171
- if self.extract_xcom:
172
- while self.base_container_is_running(pod):
173
- self.log.info("Container %s has state %s", pod.metadata.name, State.RUNNING)
174
- time.sleep(2)
175
- result = self._extract_xcom(pod)
176
- self.log.info(result)
177
- result = json.loads(result)
178
- while self.pod_is_running(pod):
179
- self.log.info("Pod %s has state %s", pod.metadata.name, State.RUNNING)
180
- time.sleep(2)
181
- return self._task_status(self.read_pod(pod)), result
182
-
183
- def parse_log_line(self, line: str) -> tuple[str | None, str]:
184
- """
185
- Parse K8s log line and returns the final state.
186
-
187
- :param line: k8s log line
188
- :return: timestamp and log message
189
- """
190
- timestamp, sep, message = line.strip().partition(" ")
191
- if not sep:
192
- self.log.error(
193
- "Error parsing timestamp (no timestamp in message: %r). "
194
- "Will continue execution but won't update timestamp",
195
- line,
196
- )
197
- return None, line
198
- return timestamp, message
199
-
200
- def _task_status(self, event):
201
- self.log.info("Event: %s had an event of type %s", event.metadata.name, event.status.phase)
202
- status = self.process_status(event.metadata.name, event.status.phase)
203
- return status
204
-
205
- def pod_not_started(self, pod: V1Pod):
206
- """Test if pod has not started."""
207
- state = self._task_status(self.read_pod(pod))
208
- return state == State.QUEUED
209
-
210
- def pod_is_running(self, pod: V1Pod):
211
- """Test if pod is running."""
212
- state = self._task_status(self.read_pod(pod))
213
- return state not in (State.SUCCESS, State.FAILED)
214
-
215
- def base_container_is_running(self, pod: V1Pod):
216
- """Test if base container is running."""
217
- event = self.read_pod(pod)
218
- status = next((s for s in event.status.container_statuses if s.name == "base"), None)
219
- if not status:
220
- return False
221
- return status.state.running is not None
222
-
223
- @tenacity.retry(stop=tenacity.stop_after_attempt(3), wait=tenacity.wait_exponential(), reraise=True)
224
- def read_pod_logs(
225
- self,
226
- pod: V1Pod,
227
- tail_lines: int | None = None,
228
- timestamps: bool = False,
229
- since_seconds: int | None = None,
230
- ):
231
- """Read log from the pod."""
232
- additional_kwargs = {}
233
- if since_seconds:
234
- additional_kwargs["since_seconds"] = since_seconds
235
-
236
- if tail_lines:
237
- additional_kwargs["tail_lines"] = tail_lines
238
-
239
- try:
240
- return self._client.read_namespaced_pod_log(
241
- name=pod.metadata.name,
242
- namespace=pod.metadata.namespace,
243
- container="base",
244
- follow=True,
245
- timestamps=timestamps,
246
- _preload_content=False,
247
- **additional_kwargs,
248
- )
249
- except HTTPError as e:
250
- raise AirflowException(f"There was an error reading the kubernetes API: {e}")
251
-
252
- @tenacity.retry(stop=tenacity.stop_after_attempt(3), wait=tenacity.wait_exponential(), reraise=True)
253
- def read_pod_events(self, pod):
254
- """Read events from the pod."""
255
- try:
256
- return self._client.list_namespaced_event(
257
- namespace=pod.metadata.namespace, field_selector=f"involvedObject.name={pod.metadata.name}"
258
- )
259
- except HTTPError as e:
260
- raise AirflowException(f"There was an error reading the kubernetes API: {e}")
261
-
262
- @tenacity.retry(stop=tenacity.stop_after_attempt(3), wait=tenacity.wait_exponential(), reraise=True)
263
- def read_pod(self, pod: V1Pod):
264
- """Read pod information."""
265
- try:
266
- return self._client.read_namespaced_pod(pod.metadata.name, pod.metadata.namespace)
267
- except HTTPError as e:
268
- raise AirflowException(f"There was an error reading the kubernetes API: {e}")
269
-
270
- def _extract_xcom(self, pod: V1Pod):
271
- resp = kubernetes_stream(
272
- self._client.connect_get_namespaced_pod_exec,
273
- pod.metadata.name,
274
- pod.metadata.namespace,
275
- container=PodDefaultsDeprecated.SIDECAR_CONTAINER_NAME,
276
- command=["/bin/sh"],
277
- stdin=True,
278
- stdout=True,
279
- stderr=True,
280
- tty=False,
281
- _preload_content=False,
282
- )
283
- try:
284
- result = self._exec_pod_command(resp, f"cat {PodDefaultsDeprecated.XCOM_MOUNT_PATH}/return.json")
285
- self._exec_pod_command(resp, "kill -s SIGINT 1")
286
- finally:
287
- resp.close()
288
- if result is None:
289
- raise AirflowException(f"Failed to extract xcom from pod: {pod.metadata.name}")
290
- return result
291
-
292
- def _exec_pod_command(self, resp, command):
293
- if resp.is_open():
294
- self.log.info("Running command... %s\n", command)
295
- resp.write_stdin(command + "\n")
296
- while resp.is_open():
297
- resp.update(timeout=1)
298
- if resp.peek_stdout():
299
- return resp.read_stdout()
300
- if resp.peek_stderr():
301
- self.log.info(resp.read_stderr())
302
- break
303
- return None
304
-
305
- def process_status(self, job_id, status):
306
- """Process status information for the job."""
307
- status = status.lower()
308
- if status == PodStatus.PENDING:
309
- return State.QUEUED
310
- elif status == PodStatus.FAILED:
311
- self.log.error("Event with job id %s Failed", job_id)
312
- return State.FAILED
313
- elif status == PodStatus.SUCCEEDED:
314
- self.log.info("Event with job id %s Succeeded", job_id)
315
- return State.SUCCESS
316
- elif status == PodStatus.RUNNING:
317
- return State.RUNNING
318
- else:
319
- self.log.error("Event: Invalid state %s on job %s", status, job_id)
320
- return State.FAILED
@@ -1,31 +0,0 @@
1
- #
2
- # Licensed to the Apache Software Foundation (ASF) under one
3
- # or more contributor license agreements. See the NOTICE file
4
- # distributed with this work for additional information
5
- # regarding copyright ownership. The ASF licenses this file
6
- # to you under the Apache License, Version 2.0 (the
7
- # "License"); you may not use this file except in compliance
8
- # with the License. You may obtain a copy of the License at
9
- #
10
- # http://www.apache.org/licenses/LICENSE-2.0
11
- #
12
- # Unless required by applicable law or agreed to in writing,
13
- # software distributed under the License is distributed on an
14
- # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15
- # KIND, either express or implied. See the License for the
16
- # specific language governing permissions and limitations
17
- # under the License.
18
- """This module is deprecated. Please use :mod:`airflow.providers.cncf.kubernetes.triggers.pod` instead."""
19
-
20
- from __future__ import annotations
21
-
22
- import warnings
23
-
24
- from airflow.exceptions import AirflowProviderDeprecationWarning
25
- from airflow.providers.cncf.kubernetes.triggers.pod import * # noqa: F403
26
-
27
- warnings.warn(
28
- "This module is deprecated. Please use `airflow.providers.cncf.kubernetes.triggers.pod` instead.",
29
- AirflowProviderDeprecationWarning,
30
- stacklevel=2,
31
- )