apache-airflow-providers-amazon 9.6.0__py3-none-any.whl → 9.6.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/amazon/__init__.py +1 -1
- airflow/providers/amazon/aws/auth_manager/cli/avp_commands.py +15 -18
- airflow/providers/amazon/aws/auth_manager/router/login.py +1 -1
- airflow/providers/amazon/aws/executors/ecs/ecs_executor.py +3 -4
- airflow/providers/amazon/aws/executors/ecs/ecs_executor_config.py +1 -1
- airflow/providers/amazon/aws/executors/ecs/utils.py +1 -1
- airflow/providers/amazon/aws/hooks/athena.py +1 -1
- airflow/providers/amazon/aws/hooks/base_aws.py +12 -15
- airflow/providers/amazon/aws/hooks/batch_client.py +11 -0
- airflow/providers/amazon/aws/hooks/cloud_formation.py +1 -2
- airflow/providers/amazon/aws/hooks/datasync.py +2 -2
- airflow/providers/amazon/aws/hooks/dms.py +2 -3
- airflow/providers/amazon/aws/hooks/dynamodb.py +1 -2
- airflow/providers/amazon/aws/hooks/emr.py +14 -17
- airflow/providers/amazon/aws/hooks/glue.py +9 -13
- airflow/providers/amazon/aws/hooks/mwaa.py +6 -7
- airflow/providers/amazon/aws/hooks/redshift_data.py +1 -1
- airflow/providers/amazon/aws/hooks/redshift_sql.py +5 -6
- airflow/providers/amazon/aws/hooks/s3.py +3 -6
- airflow/providers/amazon/aws/hooks/sagemaker.py +6 -9
- airflow/providers/amazon/aws/hooks/sagemaker_unified_studio.py +5 -6
- airflow/providers/amazon/aws/links/base_aws.py +2 -2
- airflow/providers/amazon/aws/links/emr.py +2 -4
- airflow/providers/amazon/aws/log/cloudwatch_task_handler.py +3 -5
- airflow/providers/amazon/aws/log/s3_task_handler.py +1 -2
- airflow/providers/amazon/aws/operators/athena.py +1 -1
- airflow/providers/amazon/aws/operators/batch.py +37 -42
- airflow/providers/amazon/aws/operators/bedrock.py +1 -1
- airflow/providers/amazon/aws/operators/ecs.py +4 -6
- airflow/providers/amazon/aws/operators/eks.py +146 -139
- airflow/providers/amazon/aws/operators/emr.py +4 -5
- airflow/providers/amazon/aws/operators/mwaa.py +1 -1
- airflow/providers/amazon/aws/operators/neptune.py +2 -2
- airflow/providers/amazon/aws/operators/redshift_data.py +1 -2
- airflow/providers/amazon/aws/operators/s3.py +9 -13
- airflow/providers/amazon/aws/operators/sagemaker.py +11 -19
- airflow/providers/amazon/aws/secrets/secrets_manager.py +1 -2
- airflow/providers/amazon/aws/sensors/batch.py +33 -55
- airflow/providers/amazon/aws/sensors/eks.py +64 -54
- airflow/providers/amazon/aws/sensors/glacier.py +4 -5
- airflow/providers/amazon/aws/sensors/glue.py +6 -9
- airflow/providers/amazon/aws/sensors/glue_crawler.py +2 -4
- airflow/providers/amazon/aws/sensors/redshift_cluster.py +1 -1
- airflow/providers/amazon/aws/sensors/s3.py +1 -2
- airflow/providers/amazon/aws/sensors/sagemaker_unified_studio.py +4 -5
- airflow/providers/amazon/aws/sensors/sqs.py +1 -2
- airflow/providers/amazon/aws/utils/connection_wrapper.py +1 -1
- airflow/providers/amazon/aws/utils/sqs.py +1 -2
- airflow/providers/amazon/aws/utils/tags.py +2 -3
- airflow/providers/amazon/aws/waiters/mwaa.json +1 -1
- {apache_airflow_providers_amazon-9.6.0.dist-info → apache_airflow_providers_amazon-9.6.1.dist-info}/METADATA +8 -7
- {apache_airflow_providers_amazon-9.6.0.dist-info → apache_airflow_providers_amazon-9.6.1.dist-info}/RECORD +54 -54
- {apache_airflow_providers_amazon-9.6.0.dist-info → apache_airflow_providers_amazon-9.6.1.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_amazon-9.6.0.dist-info → apache_airflow_providers_amazon-9.6.1.dist-info}/entry_points.txt +0 -0
@@ -19,18 +19,18 @@
|
|
19
19
|
from __future__ import annotations
|
20
20
|
|
21
21
|
import logging
|
22
|
+
import warnings
|
22
23
|
from ast import literal_eval
|
23
24
|
from collections.abc import Sequence
|
24
25
|
from datetime import timedelta
|
25
|
-
from functools import cached_property
|
26
26
|
from typing import TYPE_CHECKING, Any, cast
|
27
27
|
|
28
28
|
from botocore.exceptions import ClientError, WaiterError
|
29
29
|
|
30
30
|
from airflow.configuration import conf
|
31
|
-
from airflow.exceptions import AirflowException
|
32
|
-
from airflow.models import BaseOperator
|
31
|
+
from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
|
33
32
|
from airflow.providers.amazon.aws.hooks.eks import EksHook
|
33
|
+
from airflow.providers.amazon.aws.operators.base_aws import AwsBaseOperator
|
34
34
|
from airflow.providers.amazon.aws.triggers.eks import (
|
35
35
|
EksCreateClusterTrigger,
|
36
36
|
EksCreateFargateProfileTrigger,
|
@@ -40,6 +40,7 @@ from airflow.providers.amazon.aws.triggers.eks import (
|
|
40
40
|
EksDeleteNodegroupTrigger,
|
41
41
|
)
|
42
42
|
from airflow.providers.amazon.aws.utils import validate_execute_complete_event
|
43
|
+
from airflow.providers.amazon.aws.utils.mixins import aws_template_fields
|
43
44
|
from airflow.providers.amazon.aws.utils.waiter_with_logging import wait
|
44
45
|
from airflow.providers.cncf.kubernetes.utils.pod_manager import OnFinishAction
|
45
46
|
|
@@ -139,7 +140,7 @@ def _create_compute(
|
|
139
140
|
)
|
140
141
|
|
141
142
|
|
142
|
-
class EksCreateClusterOperator(
|
143
|
+
class EksCreateClusterOperator(AwsBaseOperator[EksHook]):
|
143
144
|
"""
|
144
145
|
Creates an Amazon EKS Cluster control plane.
|
145
146
|
|
@@ -168,13 +169,14 @@ class EksCreateClusterOperator(BaseOperator):
|
|
168
169
|
Defaults to 'nodegroup' to generate an EKS Managed Nodegroup.
|
169
170
|
:param create_cluster_kwargs: Optional parameters to pass to the CreateCluster API (templated)
|
170
171
|
:param wait_for_completion: If True, waits for operator to complete. (default: False) (templated)
|
171
|
-
:param aws_conn_id: The Airflow connection used for AWS credentials.
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
:param
|
177
|
-
|
172
|
+
:param aws_conn_id: The Airflow connection used for AWS credentials.
|
173
|
+
If this is ``None`` or empty then the default boto3 behaviour is used. If
|
174
|
+
running Airflow in a distributed manner and aws_conn_id is None or
|
175
|
+
empty, then default boto3 configuration would be used (and must be
|
176
|
+
maintained on each worker node).
|
177
|
+
:param region_name: AWS region_name. If not specified then the default boto3 behaviour is used.
|
178
|
+
:param verify: Whether or not to verify SSL certificates. See:
|
179
|
+
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html
|
178
180
|
|
179
181
|
If compute is assigned the value of 'nodegroup':
|
180
182
|
|
@@ -200,7 +202,8 @@ class EksCreateClusterOperator(BaseOperator):
|
|
200
202
|
|
201
203
|
"""
|
202
204
|
|
203
|
-
|
205
|
+
aws_hook_class = EksHook
|
206
|
+
template_fields: Sequence[str] = aws_template_fields(
|
204
207
|
"cluster_name",
|
205
208
|
"cluster_role_arn",
|
206
209
|
"resources_vpc_config",
|
@@ -214,8 +217,6 @@ class EksCreateClusterOperator(BaseOperator):
|
|
214
217
|
"fargate_selectors",
|
215
218
|
"create_fargate_profile_kwargs",
|
216
219
|
"wait_for_completion",
|
217
|
-
"aws_conn_id",
|
218
|
-
"region",
|
219
220
|
)
|
220
221
|
|
221
222
|
def __init__(
|
@@ -233,7 +234,6 @@ class EksCreateClusterOperator(BaseOperator):
|
|
233
234
|
fargate_selectors: list | None = None,
|
234
235
|
create_fargate_profile_kwargs: dict | None = None,
|
235
236
|
wait_for_completion: bool = False,
|
236
|
-
aws_conn_id: str | None = DEFAULT_CONN_ID,
|
237
237
|
region: str | None = None,
|
238
238
|
deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
|
239
239
|
waiter_delay: int = 30,
|
@@ -253,30 +253,31 @@ class EksCreateClusterOperator(BaseOperator):
|
|
253
253
|
self.wait_for_completion = wait_for_completion
|
254
254
|
self.waiter_delay = waiter_delay
|
255
255
|
self.waiter_max_attempts = waiter_max_attempts
|
256
|
-
self.aws_conn_id = aws_conn_id
|
257
|
-
self.region = region
|
258
256
|
self.nodegroup_name = nodegroup_name
|
259
257
|
self.create_nodegroup_kwargs = create_nodegroup_kwargs or {}
|
260
258
|
self.fargate_selectors = fargate_selectors or [{"namespace": DEFAULT_NAMESPACE_NAME}]
|
261
259
|
self.fargate_profile_name = fargate_profile_name
|
262
260
|
self.deferrable = deferrable
|
263
|
-
super().__init__(
|
264
|
-
**kwargs,
|
265
|
-
)
|
266
261
|
|
267
|
-
|
268
|
-
|
269
|
-
|
262
|
+
if region is not None:
|
263
|
+
warnings.warn(
|
264
|
+
message="Parameter `region` is deprecated. Use the parameter `region_name` instead",
|
265
|
+
category=AirflowProviderDeprecationWarning,
|
266
|
+
stacklevel=2,
|
267
|
+
)
|
268
|
+
kwargs["region_name"] = region
|
269
|
+
|
270
|
+
super().__init__(**kwargs)
|
270
271
|
|
271
272
|
def execute(self, context: Context):
|
272
273
|
if self.compute:
|
273
274
|
if self.compute not in SUPPORTED_COMPUTE_VALUES:
|
274
275
|
raise ValueError("Provided compute type is not supported.")
|
275
|
-
|
276
|
+
if (self.compute == "nodegroup") and not self.nodegroup_role_arn:
|
276
277
|
raise ValueError(
|
277
278
|
MISSING_ARN_MSG.format(compute=NODEGROUP_FULL_NAME, requirement="nodegroup_role_arn")
|
278
279
|
)
|
279
|
-
|
280
|
+
if (self.compute == "fargate") and not self.fargate_pod_execution_role_arn:
|
280
281
|
raise ValueError(
|
281
282
|
MISSING_ARN_MSG.format(
|
282
283
|
compute=FARGATE_FULL_NAME, requirement="fargate_pod_execution_role_arn"
|
@@ -302,7 +303,7 @@ class EksCreateClusterOperator(BaseOperator):
|
|
302
303
|
trigger=EksCreateClusterTrigger(
|
303
304
|
cluster_name=self.cluster_name,
|
304
305
|
aws_conn_id=self.aws_conn_id,
|
305
|
-
region_name=self.
|
306
|
+
region_name=self.region_name,
|
306
307
|
waiter_delay=self.waiter_delay,
|
307
308
|
waiter_max_attempts=self.waiter_max_attempts,
|
308
309
|
),
|
@@ -327,7 +328,7 @@ class EksCreateClusterOperator(BaseOperator):
|
|
327
328
|
compute=self.compute,
|
328
329
|
cluster_name=self.cluster_name,
|
329
330
|
aws_conn_id=self.aws_conn_id,
|
330
|
-
region=self.
|
331
|
+
region=self.region_name,
|
331
332
|
wait_for_completion=self.wait_for_completion,
|
332
333
|
waiter_delay=self.waiter_delay,
|
333
334
|
waiter_max_attempts=self.waiter_max_attempts,
|
@@ -345,7 +346,7 @@ class EksCreateClusterOperator(BaseOperator):
|
|
345
346
|
if event is None:
|
346
347
|
self.log.error("Trigger error: event is None")
|
347
348
|
raise AirflowException("Trigger error: event is None")
|
348
|
-
|
349
|
+
if event["status"] == "failed":
|
349
350
|
self.log.error("Cluster failed to start and will be torn down.")
|
350
351
|
self.hook.delete_cluster(name=self.cluster_name)
|
351
352
|
self.defer(
|
@@ -354,7 +355,7 @@ class EksCreateClusterOperator(BaseOperator):
|
|
354
355
|
waiter_delay=self.waiter_delay,
|
355
356
|
waiter_max_attempts=self.waiter_max_attempts,
|
356
357
|
aws_conn_id=self.aws_conn_id,
|
357
|
-
region_name=self.
|
358
|
+
region_name=self.region_name,
|
358
359
|
force_delete_compute=False,
|
359
360
|
),
|
360
361
|
method_name="execute_failed",
|
@@ -366,7 +367,7 @@ class EksCreateClusterOperator(BaseOperator):
|
|
366
367
|
compute=self.compute,
|
367
368
|
cluster_name=self.cluster_name,
|
368
369
|
aws_conn_id=self.aws_conn_id,
|
369
|
-
region=self.
|
370
|
+
region=self.region_name,
|
370
371
|
wait_for_completion=self.wait_for_completion,
|
371
372
|
waiter_delay=self.waiter_delay,
|
372
373
|
waiter_max_attempts=self.waiter_max_attempts,
|
@@ -387,7 +388,7 @@ class EksCreateClusterOperator(BaseOperator):
|
|
387
388
|
waiter_delay=self.waiter_delay,
|
388
389
|
waiter_max_attempts=self.waiter_max_attempts,
|
389
390
|
aws_conn_id=self.aws_conn_id,
|
390
|
-
region_name=self.
|
391
|
+
region_name=self.region_name,
|
391
392
|
),
|
392
393
|
method_name="execute_complete",
|
393
394
|
timeout=timedelta(seconds=self.waiter_max_attempts * self.waiter_delay),
|
@@ -398,7 +399,7 @@ class EksCreateClusterOperator(BaseOperator):
|
|
398
399
|
nodegroup_name=self.nodegroup_name,
|
399
400
|
cluster_name=self.cluster_name,
|
400
401
|
aws_conn_id=self.aws_conn_id,
|
401
|
-
region_name=self.
|
402
|
+
region_name=self.region_name,
|
402
403
|
waiter_delay=self.waiter_delay,
|
403
404
|
waiter_max_attempts=self.waiter_max_attempts,
|
404
405
|
),
|
@@ -410,7 +411,7 @@ class EksCreateClusterOperator(BaseOperator):
|
|
410
411
|
if event is None:
|
411
412
|
self.log.info("Trigger error: event is None")
|
412
413
|
raise AirflowException("Trigger error: event is None")
|
413
|
-
|
414
|
+
if event["status"] == "deleted":
|
414
415
|
self.log.info("Cluster deleted")
|
415
416
|
raise AirflowException("Error creating cluster")
|
416
417
|
|
@@ -424,7 +425,7 @@ class EksCreateClusterOperator(BaseOperator):
|
|
424
425
|
self.log.info("%s created successfully", resource)
|
425
426
|
|
426
427
|
|
427
|
-
class EksCreateNodegroupOperator(
|
428
|
+
class EksCreateNodegroupOperator(AwsBaseOperator[EksHook]):
|
428
429
|
"""
|
429
430
|
Creates an Amazon EKS managed node group for an existing Amazon EKS Cluster.
|
430
431
|
|
@@ -440,13 +441,14 @@ class EksCreateNodegroupOperator(BaseOperator):
|
|
440
441
|
The Amazon Resource Name (ARN) of the IAM role to associate with the managed nodegroup. (templated)
|
441
442
|
:param create_nodegroup_kwargs: Optional parameters to pass to the Create Nodegroup API (templated)
|
442
443
|
:param wait_for_completion: If True, waits for operator to complete. (default: False) (templated)
|
443
|
-
:param aws_conn_id: The Airflow connection used for AWS credentials.
|
444
|
-
|
445
|
-
|
446
|
-
|
447
|
-
|
448
|
-
:param
|
449
|
-
|
444
|
+
:param aws_conn_id: The Airflow connection used for AWS credentials.
|
445
|
+
If this is ``None`` or empty then the default boto3 behaviour is used. If
|
446
|
+
running Airflow in a distributed manner and aws_conn_id is None or
|
447
|
+
empty, then default boto3 configuration would be used (and must be
|
448
|
+
maintained on each worker node).
|
449
|
+
:param region_name: AWS region_name. If not specified then the default boto3 behaviour is used.
|
450
|
+
:param verify: Whether or not to verify SSL certificates. See:
|
451
|
+
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html
|
450
452
|
:param waiter_delay: Time (in seconds) to wait between two consecutive calls to check nodegroup state
|
451
453
|
:param waiter_max_attempts: The maximum number of attempts to check nodegroup state
|
452
454
|
:param deferrable: If True, the operator will wait asynchronously for the nodegroup to be created.
|
@@ -455,15 +457,14 @@ class EksCreateNodegroupOperator(BaseOperator):
|
|
455
457
|
|
456
458
|
"""
|
457
459
|
|
458
|
-
|
460
|
+
aws_hook_class = EksHook
|
461
|
+
template_fields: Sequence[str] = aws_template_fields(
|
459
462
|
"cluster_name",
|
460
463
|
"nodegroup_subnets",
|
461
464
|
"nodegroup_role_arn",
|
462
465
|
"nodegroup_name",
|
463
466
|
"create_nodegroup_kwargs",
|
464
467
|
"wait_for_completion",
|
465
|
-
"aws_conn_id",
|
466
|
-
"region",
|
467
468
|
)
|
468
469
|
|
469
470
|
def __init__(
|
@@ -474,7 +475,6 @@ class EksCreateNodegroupOperator(BaseOperator):
|
|
474
475
|
nodegroup_name: str = DEFAULT_NODEGROUP_NAME,
|
475
476
|
create_nodegroup_kwargs: dict | None = None,
|
476
477
|
wait_for_completion: bool = False,
|
477
|
-
aws_conn_id: str | None = DEFAULT_CONN_ID,
|
478
478
|
region: str | None = None,
|
479
479
|
waiter_delay: int = 30,
|
480
480
|
waiter_max_attempts: int = 80,
|
@@ -490,11 +490,18 @@ class EksCreateNodegroupOperator(BaseOperator):
|
|
490
490
|
if deferrable:
|
491
491
|
wait_for_completion = False
|
492
492
|
self.wait_for_completion = wait_for_completion
|
493
|
-
self.aws_conn_id = aws_conn_id
|
494
|
-
self.region = region
|
495
493
|
self.waiter_delay = waiter_delay
|
496
494
|
self.waiter_max_attempts = waiter_max_attempts
|
497
495
|
self.deferrable = deferrable
|
496
|
+
|
497
|
+
if region is not None:
|
498
|
+
warnings.warn(
|
499
|
+
message="Parameter `region` is deprecated. Use the parameter `region_name` instead",
|
500
|
+
category=AirflowProviderDeprecationWarning,
|
501
|
+
stacklevel=2,
|
502
|
+
)
|
503
|
+
kwargs["region_name"] = region
|
504
|
+
|
498
505
|
super().__init__(**kwargs)
|
499
506
|
|
500
507
|
def execute(self, context: Context):
|
@@ -516,7 +523,7 @@ class EksCreateNodegroupOperator(BaseOperator):
|
|
516
523
|
compute=self.compute,
|
517
524
|
cluster_name=self.cluster_name,
|
518
525
|
aws_conn_id=self.aws_conn_id,
|
519
|
-
region=self.
|
526
|
+
region=self.region_name,
|
520
527
|
wait_for_completion=self.wait_for_completion,
|
521
528
|
waiter_delay=self.waiter_delay,
|
522
529
|
waiter_max_attempts=self.waiter_max_attempts,
|
@@ -532,7 +539,7 @@ class EksCreateNodegroupOperator(BaseOperator):
|
|
532
539
|
cluster_name=self.cluster_name,
|
533
540
|
nodegroup_name=self.nodegroup_name,
|
534
541
|
aws_conn_id=self.aws_conn_id,
|
535
|
-
region_name=self.
|
542
|
+
region_name=self.region_name,
|
536
543
|
waiter_delay=self.waiter_delay,
|
537
544
|
waiter_max_attempts=self.waiter_max_attempts,
|
538
545
|
),
|
@@ -549,7 +556,7 @@ class EksCreateNodegroupOperator(BaseOperator):
|
|
549
556
|
raise AirflowException(f"Error creating nodegroup: {validated_event}")
|
550
557
|
|
551
558
|
|
552
|
-
class EksCreateFargateProfileOperator(
|
559
|
+
class EksCreateFargateProfileOperator(AwsBaseOperator[EksHook]):
|
553
560
|
"""
|
554
561
|
Creates an AWS Fargate profile for an Amazon EKS cluster.
|
555
562
|
|
@@ -565,14 +572,14 @@ class EksCreateFargateProfileOperator(BaseOperator):
|
|
565
572
|
:param create_fargate_profile_kwargs: Optional parameters to pass to the CreateFargate Profile API
|
566
573
|
(templated)
|
567
574
|
:param wait_for_completion: If True, waits for operator to complete. (default: False) (templated)
|
568
|
-
|
569
|
-
|
570
|
-
|
571
|
-
|
572
|
-
|
573
|
-
|
574
|
-
:param
|
575
|
-
|
575
|
+
:param aws_conn_id: The Airflow connection used for AWS credentials.
|
576
|
+
If this is ``None`` or empty then the default boto3 behaviour is used. If
|
577
|
+
running Airflow in a distributed manner and aws_conn_id is None or
|
578
|
+
empty, then default boto3 configuration would be used (and must be
|
579
|
+
maintained on each worker node).
|
580
|
+
:param region_name: AWS region_name. If not specified then the default boto3 behaviour is used.
|
581
|
+
:param verify: Whether or not to verify SSL certificates. See:
|
582
|
+
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html
|
576
583
|
:param waiter_delay: Time (in seconds) to wait between two consecutive calls to check profile status
|
577
584
|
:param waiter_max_attempts: The maximum number of attempts to check the status of the profile.
|
578
585
|
:param deferrable: If True, the operator will wait asynchronously for the profile to be created.
|
@@ -580,15 +587,14 @@ class EksCreateFargateProfileOperator(BaseOperator):
|
|
580
587
|
(default: False)
|
581
588
|
"""
|
582
589
|
|
583
|
-
|
590
|
+
aws_hook_class = EksHook
|
591
|
+
template_fields: Sequence[str] = aws_template_fields(
|
584
592
|
"cluster_name",
|
585
593
|
"pod_execution_role_arn",
|
586
594
|
"selectors",
|
587
595
|
"fargate_profile_name",
|
588
596
|
"create_fargate_profile_kwargs",
|
589
597
|
"wait_for_completion",
|
590
|
-
"aws_conn_id",
|
591
|
-
"region",
|
592
598
|
)
|
593
599
|
|
594
600
|
def __init__(
|
@@ -598,9 +604,8 @@ class EksCreateFargateProfileOperator(BaseOperator):
|
|
598
604
|
selectors: list,
|
599
605
|
fargate_profile_name: str = DEFAULT_FARGATE_PROFILE_NAME,
|
600
606
|
create_fargate_profile_kwargs: dict | None = None,
|
601
|
-
wait_for_completion: bool = False,
|
602
|
-
aws_conn_id: str | None = DEFAULT_CONN_ID,
|
603
607
|
region: str | None = None,
|
608
|
+
wait_for_completion: bool = False,
|
604
609
|
waiter_delay: int = 10,
|
605
610
|
waiter_max_attempts: int = 60,
|
606
611
|
deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
|
@@ -614,22 +619,26 @@ class EksCreateFargateProfileOperator(BaseOperator):
|
|
614
619
|
if deferrable:
|
615
620
|
wait_for_completion = False
|
616
621
|
self.wait_for_completion = wait_for_completion
|
617
|
-
self.aws_conn_id = aws_conn_id
|
618
|
-
self.region = region
|
619
622
|
self.waiter_delay = waiter_delay
|
620
623
|
self.waiter_max_attempts = waiter_max_attempts
|
621
624
|
self.deferrable = deferrable
|
622
625
|
self.compute = "fargate"
|
623
|
-
|
624
|
-
|
625
|
-
|
626
|
+
|
627
|
+
if region is not None:
|
628
|
+
warnings.warn(
|
629
|
+
message="Parameter `region` is deprecated. Use the parameter `region_name` instead",
|
630
|
+
category=AirflowProviderDeprecationWarning,
|
631
|
+
stacklevel=2,
|
632
|
+
)
|
633
|
+
kwargs["region_name"] = region
|
634
|
+
super().__init__(**kwargs)
|
626
635
|
|
627
636
|
def execute(self, context: Context):
|
628
637
|
_create_compute(
|
629
638
|
compute=self.compute,
|
630
639
|
cluster_name=self.cluster_name,
|
631
640
|
aws_conn_id=self.aws_conn_id,
|
632
|
-
region=self.
|
641
|
+
region=self.region_name,
|
633
642
|
wait_for_completion=self.wait_for_completion,
|
634
643
|
waiter_delay=self.waiter_delay,
|
635
644
|
waiter_max_attempts=self.waiter_max_attempts,
|
@@ -646,7 +655,7 @@ class EksCreateFargateProfileOperator(BaseOperator):
|
|
646
655
|
aws_conn_id=self.aws_conn_id,
|
647
656
|
waiter_delay=self.waiter_delay,
|
648
657
|
waiter_max_attempts=self.waiter_max_attempts,
|
649
|
-
region_name=self.
|
658
|
+
region_name=self.region_name,
|
650
659
|
),
|
651
660
|
method_name="execute_complete",
|
652
661
|
# timeout is set to ensure that if a trigger dies, the timeout does not restart
|
@@ -663,7 +672,7 @@ class EksCreateFargateProfileOperator(BaseOperator):
|
|
663
672
|
self.log.info("Fargate profile created successfully")
|
664
673
|
|
665
674
|
|
666
|
-
class EksDeleteClusterOperator(
|
675
|
+
class EksDeleteClusterOperator(AwsBaseOperator[EksHook]):
|
667
676
|
"""
|
668
677
|
Deletes the Amazon EKS Cluster control plane and all nodegroups attached to it.
|
669
678
|
|
@@ -682,6 +691,8 @@ class EksDeleteClusterOperator(BaseOperator):
|
|
682
691
|
maintained on each worker node).
|
683
692
|
:param region: Which AWS region the connection should use. (templated)
|
684
693
|
If this is None or empty then the default boto3 behaviour is used.
|
694
|
+
:param verify: Whether or not to verify SSL certificates. See:
|
695
|
+
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html
|
685
696
|
:param waiter_delay: Time (in seconds) to wait between two consecutive calls to check cluster state
|
686
697
|
:param waiter_max_attempts: The maximum number of attempts to check cluster state
|
687
698
|
:param deferrable: If True, the operator will wait asynchronously for the cluster to be deleted.
|
@@ -690,21 +701,17 @@ class EksDeleteClusterOperator(BaseOperator):
|
|
690
701
|
|
691
702
|
"""
|
692
703
|
|
693
|
-
|
694
|
-
|
695
|
-
"force_delete_compute",
|
696
|
-
"wait_for_completion",
|
697
|
-
"aws_conn_id",
|
698
|
-
"region",
|
704
|
+
aws_hook_class = EksHook
|
705
|
+
template_fields: Sequence[str] = aws_template_fields(
|
706
|
+
"cluster_name", "force_delete_compute", "wait_for_completion"
|
699
707
|
)
|
700
708
|
|
701
709
|
def __init__(
|
702
710
|
self,
|
703
711
|
cluster_name: str,
|
704
712
|
force_delete_compute: bool = False,
|
705
|
-
wait_for_completion: bool = False,
|
706
|
-
aws_conn_id: str | None = DEFAULT_CONN_ID,
|
707
713
|
region: str | None = None,
|
714
|
+
wait_for_completion: bool = False,
|
708
715
|
deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
|
709
716
|
waiter_delay: int = 30,
|
710
717
|
waiter_max_attempts: int = 40,
|
@@ -715,18 +722,21 @@ class EksDeleteClusterOperator(BaseOperator):
|
|
715
722
|
if deferrable:
|
716
723
|
wait_for_completion = False
|
717
724
|
self.wait_for_completion = wait_for_completion
|
718
|
-
self.aws_conn_id = aws_conn_id
|
719
|
-
self.region = region
|
720
725
|
self.deferrable = deferrable
|
721
726
|
self.waiter_delay = waiter_delay
|
722
727
|
self.waiter_max_attempts = waiter_max_attempts
|
728
|
+
|
729
|
+
if region is not None:
|
730
|
+
warnings.warn(
|
731
|
+
message="Parameter `region` is deprecated. Use the parameter `region_name` instead",
|
732
|
+
category=AirflowProviderDeprecationWarning,
|
733
|
+
stacklevel=2,
|
734
|
+
)
|
735
|
+
kwargs["region_name"] = region
|
736
|
+
|
723
737
|
super().__init__(**kwargs)
|
724
738
|
|
725
739
|
def execute(self, context: Context):
|
726
|
-
eks_hook = EksHook(
|
727
|
-
aws_conn_id=self.aws_conn_id,
|
728
|
-
region_name=self.region,
|
729
|
-
)
|
730
740
|
if self.deferrable:
|
731
741
|
self.defer(
|
732
742
|
trigger=EksDeleteClusterTrigger(
|
@@ -734,47 +744,47 @@ class EksDeleteClusterOperator(BaseOperator):
|
|
734
744
|
waiter_delay=self.waiter_delay,
|
735
745
|
waiter_max_attempts=self.waiter_max_attempts,
|
736
746
|
aws_conn_id=self.aws_conn_id,
|
737
|
-
region_name=self.
|
747
|
+
region_name=self.region_name,
|
738
748
|
force_delete_compute=self.force_delete_compute,
|
739
749
|
),
|
740
750
|
method_name="execute_complete",
|
741
751
|
timeout=timedelta(seconds=self.waiter_delay * self.waiter_max_attempts),
|
742
752
|
)
|
743
753
|
elif self.force_delete_compute:
|
744
|
-
self.delete_any_nodegroups(
|
745
|
-
self.delete_any_fargate_profiles(
|
754
|
+
self.delete_any_nodegroups()
|
755
|
+
self.delete_any_fargate_profiles()
|
746
756
|
|
747
|
-
|
757
|
+
self.hook.delete_cluster(name=self.cluster_name)
|
748
758
|
|
749
759
|
if self.wait_for_completion:
|
750
760
|
self.log.info("Waiting for cluster to delete. This will take some time.")
|
751
|
-
|
761
|
+
self.hook.conn.get_waiter("cluster_deleted").wait(name=self.cluster_name)
|
752
762
|
|
753
|
-
def delete_any_nodegroups(self
|
763
|
+
def delete_any_nodegroups(self) -> None:
|
754
764
|
"""
|
755
765
|
Delete all Amazon EKS managed node groups for a provided Amazon EKS Cluster.
|
756
766
|
|
757
767
|
Amazon EKS managed node groups can be deleted in parallel, so we can send all
|
758
768
|
delete commands in bulk and move on once the count of nodegroups is zero.
|
759
769
|
"""
|
760
|
-
nodegroups =
|
770
|
+
nodegroups = self.hook.list_nodegroups(clusterName=self.cluster_name)
|
761
771
|
if nodegroups:
|
762
772
|
self.log.info(CAN_NOT_DELETE_MSG.format(compute=NODEGROUP_FULL_NAME, count=len(nodegroups)))
|
763
773
|
for group in nodegroups:
|
764
|
-
|
774
|
+
self.hook.delete_nodegroup(clusterName=self.cluster_name, nodegroupName=group)
|
765
775
|
# Note this is a custom waiter so we're using hook.get_waiter(), not hook.conn.get_waiter().
|
766
776
|
self.log.info("Waiting for all nodegroups to delete. This will take some time.")
|
767
|
-
|
777
|
+
self.hook.get_waiter("all_nodegroups_deleted").wait(clusterName=self.cluster_name)
|
768
778
|
self.log.info(SUCCESS_MSG.format(compute=NODEGROUP_FULL_NAME))
|
769
779
|
|
770
|
-
def delete_any_fargate_profiles(self
|
780
|
+
def delete_any_fargate_profiles(self) -> None:
|
771
781
|
"""
|
772
782
|
Delete all EKS Fargate profiles for a provided Amazon EKS Cluster.
|
773
783
|
|
774
784
|
EKS Fargate profiles must be deleted one at a time, so we must wait
|
775
785
|
for one to be deleted before sending the next delete command.
|
776
786
|
"""
|
777
|
-
fargate_profiles =
|
787
|
+
fargate_profiles = self.hook.list_fargate_profiles(clusterName=self.cluster_name)
|
778
788
|
if fargate_profiles:
|
779
789
|
self.log.info(CAN_NOT_DELETE_MSG.format(compute=FARGATE_FULL_NAME, count=len(fargate_profiles)))
|
780
790
|
self.log.info("Waiting for Fargate profiles to delete. This will take some time.")
|
@@ -782,8 +792,8 @@ class EksDeleteClusterOperator(BaseOperator):
|
|
782
792
|
# The API will return a (cluster) ResourceInUseException if you try
|
783
793
|
# to delete Fargate profiles in parallel the way we can with nodegroups,
|
784
794
|
# so each must be deleted sequentially
|
785
|
-
|
786
|
-
|
795
|
+
self.hook.delete_fargate_profile(clusterName=self.cluster_name, fargateProfileName=profile)
|
796
|
+
self.hook.conn.get_waiter("fargate_profile_deleted").wait(
|
787
797
|
clusterName=self.cluster_name, fargateProfileName=profile
|
788
798
|
)
|
789
799
|
self.log.info(SUCCESS_MSG.format(compute=FARGATE_FULL_NAME))
|
@@ -795,7 +805,7 @@ class EksDeleteClusterOperator(BaseOperator):
|
|
795
805
|
self.log.info("Cluster deleted successfully.")
|
796
806
|
|
797
807
|
|
798
|
-
class EksDeleteNodegroupOperator(
|
808
|
+
class EksDeleteNodegroupOperator(AwsBaseOperator[EksHook]):
|
799
809
|
"""
|
800
810
|
Deletes an Amazon EKS managed node group from an Amazon EKS Cluster.
|
801
811
|
|
@@ -811,8 +821,10 @@ class EksDeleteNodegroupOperator(BaseOperator):
|
|
811
821
|
running Airflow in a distributed manner and aws_conn_id is None or
|
812
822
|
empty, then the default boto3 configuration would be used (and must be
|
813
823
|
maintained on each worker node).
|
814
|
-
:param
|
824
|
+
:param region_name: Which AWS region the connection should use. (templated)
|
815
825
|
If this is None or empty then the default boto3 behaviour is used.
|
826
|
+
:param verify: Whether or not to verify SSL certificates. See:
|
827
|
+
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html
|
816
828
|
:param waiter_delay: Time (in seconds) to wait between two consecutive calls to check nodegroup state
|
817
829
|
:param waiter_max_attempts: The maximum number of attempts to check nodegroup state
|
818
830
|
:param deferrable: If True, the operator will wait asynchronously for the nodegroup to be deleted.
|
@@ -821,21 +833,17 @@ class EksDeleteNodegroupOperator(BaseOperator):
|
|
821
833
|
|
822
834
|
"""
|
823
835
|
|
824
|
-
|
825
|
-
|
826
|
-
"nodegroup_name",
|
827
|
-
"wait_for_completion",
|
828
|
-
"aws_conn_id",
|
829
|
-
"region",
|
836
|
+
aws_hook_class = EksHook
|
837
|
+
template_fields: Sequence[str] = aws_template_fields(
|
838
|
+
"cluster_name", "nodegroup_name", "wait_for_completion"
|
830
839
|
)
|
831
840
|
|
832
841
|
def __init__(
|
833
842
|
self,
|
834
843
|
cluster_name: str,
|
835
844
|
nodegroup_name: str,
|
836
|
-
wait_for_completion: bool = False,
|
837
|
-
aws_conn_id: str | None = DEFAULT_CONN_ID,
|
838
845
|
region: str | None = None,
|
846
|
+
wait_for_completion: bool = False,
|
839
847
|
waiter_delay: int = 30,
|
840
848
|
waiter_max_attempts: int = 40,
|
841
849
|
deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
|
@@ -844,27 +852,28 @@ class EksDeleteNodegroupOperator(BaseOperator):
|
|
844
852
|
self.cluster_name = cluster_name
|
845
853
|
self.nodegroup_name = nodegroup_name
|
846
854
|
self.wait_for_completion = wait_for_completion
|
847
|
-
self.aws_conn_id = aws_conn_id
|
848
|
-
self.region = region
|
849
855
|
self.waiter_delay = waiter_delay
|
850
856
|
self.waiter_max_attempts = waiter_max_attempts
|
851
857
|
self.deferrable = deferrable
|
858
|
+
|
859
|
+
if region is not None:
|
860
|
+
warnings.warn(
|
861
|
+
message="Parameter `region` is deprecated. Use the parameter `region_name` instead",
|
862
|
+
category=AirflowProviderDeprecationWarning,
|
863
|
+
stacklevel=2,
|
864
|
+
)
|
865
|
+
kwargs["region_name"] = region
|
852
866
|
super().__init__(**kwargs)
|
853
867
|
|
854
868
|
def execute(self, context: Context):
|
855
|
-
|
856
|
-
aws_conn_id=self.aws_conn_id,
|
857
|
-
region_name=self.region,
|
858
|
-
)
|
859
|
-
|
860
|
-
eks_hook.delete_nodegroup(clusterName=self.cluster_name, nodegroupName=self.nodegroup_name)
|
869
|
+
self.hook.delete_nodegroup(clusterName=self.cluster_name, nodegroupName=self.nodegroup_name)
|
861
870
|
if self.deferrable:
|
862
871
|
self.defer(
|
863
872
|
trigger=EksDeleteNodegroupTrigger(
|
864
873
|
cluster_name=self.cluster_name,
|
865
874
|
nodegroup_name=self.nodegroup_name,
|
866
875
|
aws_conn_id=self.aws_conn_id,
|
867
|
-
region_name=self.
|
876
|
+
region_name=self.region_name,
|
868
877
|
waiter_delay=self.waiter_delay,
|
869
878
|
waiter_max_attempts=self.waiter_max_attempts,
|
870
879
|
),
|
@@ -875,7 +884,7 @@ class EksDeleteNodegroupOperator(BaseOperator):
|
|
875
884
|
)
|
876
885
|
elif self.wait_for_completion:
|
877
886
|
self.log.info("Waiting for nodegroup to delete. This will take some time.")
|
878
|
-
|
887
|
+
self.hook.conn.get_waiter("nodegroup_deleted").wait(
|
879
888
|
clusterName=self.cluster_name, nodegroupName=self.nodegroup_name
|
880
889
|
)
|
881
890
|
|
@@ -886,7 +895,7 @@ class EksDeleteNodegroupOperator(BaseOperator):
|
|
886
895
|
raise AirflowException(f"Error deleting nodegroup: {validated_event}")
|
887
896
|
|
888
897
|
|
889
|
-
class EksDeleteFargateProfileOperator(
|
898
|
+
class EksDeleteFargateProfileOperator(AwsBaseOperator[EksHook]):
|
890
899
|
"""
|
891
900
|
Deletes an AWS Fargate profile from an Amazon EKS Cluster.
|
892
901
|
|
@@ -902,8 +911,10 @@ class EksDeleteFargateProfileOperator(BaseOperator):
|
|
902
911
|
running Airflow in a distributed manner and aws_conn_id is None or
|
903
912
|
empty, then the default boto3 configuration would be used (and must be
|
904
913
|
maintained on each worker node).
|
905
|
-
:param
|
914
|
+
:param region_name: Which AWS region the connection should use. (templated)
|
906
915
|
If this is None or empty then the default boto3 behaviour is used.
|
916
|
+
:param verify: Whether or not to verify SSL certificates. See:
|
917
|
+
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html
|
907
918
|
:param waiter_delay: Time (in seconds) to wait between two consecutive calls to check profile status
|
908
919
|
:param waiter_max_attempts: The maximum number of attempts to check the status of the profile.
|
909
920
|
:param deferrable: If True, the operator will wait asynchronously for the profile to be deleted.
|
@@ -911,43 +922,39 @@ class EksDeleteFargateProfileOperator(BaseOperator):
|
|
911
922
|
(default: False)
|
912
923
|
"""
|
913
924
|
|
914
|
-
|
915
|
-
|
916
|
-
"fargate_profile_name",
|
917
|
-
"wait_for_completion",
|
918
|
-
"aws_conn_id",
|
919
|
-
"region",
|
925
|
+
aws_hook_class = EksHook
|
926
|
+
template_fields: Sequence[str] = aws_template_fields(
|
927
|
+
"cluster_name", "fargate_profile_name", "wait_for_completion"
|
920
928
|
)
|
921
929
|
|
922
930
|
def __init__(
|
923
931
|
self,
|
924
932
|
cluster_name: str,
|
925
933
|
fargate_profile_name: str,
|
926
|
-
wait_for_completion: bool = False,
|
927
|
-
aws_conn_id: str | None = DEFAULT_CONN_ID,
|
928
934
|
region: str | None = None,
|
935
|
+
wait_for_completion: bool = False,
|
929
936
|
waiter_delay: int = 30,
|
930
937
|
waiter_max_attempts: int = 60,
|
931
938
|
deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
|
932
939
|
**kwargs,
|
933
940
|
) -> None:
|
934
|
-
super().__init__(**kwargs)
|
935
941
|
self.cluster_name = cluster_name
|
936
942
|
self.fargate_profile_name = fargate_profile_name
|
937
943
|
self.wait_for_completion = wait_for_completion
|
938
|
-
self.aws_conn_id = aws_conn_id
|
939
|
-
self.region = region
|
940
944
|
self.waiter_delay = waiter_delay
|
941
945
|
self.waiter_max_attempts = waiter_max_attempts
|
942
946
|
self.deferrable = deferrable
|
947
|
+
if region is not None:
|
948
|
+
warnings.warn(
|
949
|
+
message="Parameter `region` is deprecated. Use the parameter `region_name` instead",
|
950
|
+
category=AirflowProviderDeprecationWarning,
|
951
|
+
stacklevel=2,
|
952
|
+
)
|
953
|
+
kwargs["region_name"] = region
|
954
|
+
super().__init__(**kwargs)
|
943
955
|
|
944
956
|
def execute(self, context: Context):
|
945
|
-
|
946
|
-
aws_conn_id=self.aws_conn_id,
|
947
|
-
region_name=self.region,
|
948
|
-
)
|
949
|
-
|
950
|
-
eks_hook.delete_fargate_profile(
|
957
|
+
self.hook.delete_fargate_profile(
|
951
958
|
clusterName=self.cluster_name, fargateProfileName=self.fargate_profile_name
|
952
959
|
)
|
953
960
|
if self.deferrable:
|
@@ -958,7 +965,7 @@ class EksDeleteFargateProfileOperator(BaseOperator):
|
|
958
965
|
aws_conn_id=self.aws_conn_id,
|
959
966
|
waiter_delay=self.waiter_delay,
|
960
967
|
waiter_max_attempts=self.waiter_max_attempts,
|
961
|
-
region_name=self.
|
968
|
+
region_name=self.region_name,
|
962
969
|
),
|
963
970
|
method_name="execute_complete",
|
964
971
|
# timeout is set to ensure that if a trigger dies, the timeout does not restart
|
@@ -967,7 +974,7 @@ class EksDeleteFargateProfileOperator(BaseOperator):
|
|
967
974
|
)
|
968
975
|
elif self.wait_for_completion:
|
969
976
|
self.log.info("Waiting for Fargate profile to delete. This will take some time.")
|
970
|
-
|
977
|
+
self.hook.conn.get_waiter("fargate_profile_deleted").wait(
|
971
978
|
clusterName=self.cluster_name,
|
972
979
|
fargateProfileName=self.fargate_profile_name,
|
973
980
|
WaiterConfig={"Delay": self.waiter_delay, "MaxAttempts": self.waiter_max_attempts},
|