apache-airflow-providers-amazon 8.29.0rc1__py3-none-any.whl → 9.0.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/amazon/__init__.py +1 -1
- airflow/providers/amazon/aws/{datasets → assets}/s3.py +10 -6
- airflow/providers/amazon/aws/auth_manager/avp/entities.py +1 -1
- airflow/providers/amazon/aws/auth_manager/aws_auth_manager.py +5 -11
- airflow/providers/amazon/aws/auth_manager/cli/avp_commands.py +2 -5
- airflow/providers/amazon/aws/auth_manager/cli/definition.py +0 -6
- airflow/providers/amazon/aws/auth_manager/views/auth.py +1 -1
- airflow/providers/amazon/aws/hooks/athena.py +3 -17
- airflow/providers/amazon/aws/hooks/base_aws.py +4 -162
- airflow/providers/amazon/aws/hooks/logs.py +1 -20
- airflow/providers/amazon/aws/hooks/quicksight.py +1 -17
- airflow/providers/amazon/aws/hooks/redshift_cluster.py +6 -120
- airflow/providers/amazon/aws/hooks/redshift_data.py +52 -14
- airflow/providers/amazon/aws/hooks/s3.py +24 -27
- airflow/providers/amazon/aws/hooks/sagemaker.py +4 -48
- airflow/providers/amazon/aws/log/s3_task_handler.py +1 -6
- airflow/providers/amazon/aws/operators/appflow.py +1 -10
- airflow/providers/amazon/aws/operators/batch.py +1 -29
- airflow/providers/amazon/aws/operators/datasync.py +1 -8
- airflow/providers/amazon/aws/operators/ecs.py +1 -25
- airflow/providers/amazon/aws/operators/eks.py +7 -46
- airflow/providers/amazon/aws/operators/emr.py +16 -232
- airflow/providers/amazon/aws/operators/glue_databrew.py +1 -10
- airflow/providers/amazon/aws/operators/rds.py +3 -17
- airflow/providers/amazon/aws/operators/redshift_data.py +18 -3
- airflow/providers/amazon/aws/operators/s3.py +12 -2
- airflow/providers/amazon/aws/operators/sagemaker.py +10 -32
- airflow/providers/amazon/aws/secrets/secrets_manager.py +1 -40
- airflow/providers/amazon/aws/sensors/batch.py +1 -8
- airflow/providers/amazon/aws/sensors/dms.py +1 -8
- airflow/providers/amazon/aws/sensors/dynamodb.py +22 -8
- airflow/providers/amazon/aws/sensors/emr.py +0 -7
- airflow/providers/amazon/aws/sensors/glue_catalog_partition.py +1 -8
- airflow/providers/amazon/aws/sensors/glue_crawler.py +1 -8
- airflow/providers/amazon/aws/sensors/quicksight.py +1 -29
- airflow/providers/amazon/aws/sensors/redshift_cluster.py +1 -8
- airflow/providers/amazon/aws/sensors/s3.py +1 -8
- airflow/providers/amazon/aws/sensors/sagemaker.py +2 -9
- airflow/providers/amazon/aws/sensors/sqs.py +1 -8
- airflow/providers/amazon/aws/sensors/step_function.py +1 -8
- airflow/providers/amazon/aws/transfers/base.py +1 -14
- airflow/providers/amazon/aws/transfers/gcs_to_s3.py +5 -33
- airflow/providers/amazon/aws/transfers/redshift_to_s3.py +15 -10
- airflow/providers/amazon/aws/transfers/s3_to_redshift.py +6 -6
- airflow/providers/amazon/aws/transfers/sql_to_s3.py +3 -6
- airflow/providers/amazon/aws/triggers/batch.py +1 -168
- airflow/providers/amazon/aws/triggers/eks.py +1 -20
- airflow/providers/amazon/aws/triggers/emr.py +0 -32
- airflow/providers/amazon/aws/triggers/glue_crawler.py +0 -11
- airflow/providers/amazon/aws/triggers/glue_databrew.py +0 -21
- airflow/providers/amazon/aws/triggers/rds.py +0 -79
- airflow/providers/amazon/aws/triggers/redshift_cluster.py +5 -64
- airflow/providers/amazon/aws/triggers/sagemaker.py +2 -93
- airflow/providers/amazon/aws/utils/asset_compat_lineage_collector.py +106 -0
- airflow/providers/amazon/aws/utils/connection_wrapper.py +4 -164
- airflow/providers/amazon/aws/utils/mixins.py +1 -23
- airflow/providers/amazon/aws/utils/openlineage.py +3 -1
- airflow/providers/amazon/aws/utils/task_log_fetcher.py +1 -1
- airflow/providers/amazon/get_provider_info.py +13 -4
- {apache_airflow_providers_amazon-8.29.0rc1.dist-info → apache_airflow_providers_amazon-9.0.0rc1.dist-info}/METADATA +8 -9
- {apache_airflow_providers_amazon-8.29.0rc1.dist-info → apache_airflow_providers_amazon-9.0.0rc1.dist-info}/RECORD +64 -64
- airflow/providers/amazon/aws/auth_manager/cli/idc_commands.py +0 -149
- /airflow/providers/amazon/aws/{datasets → assets}/__init__.py +0 -0
- {apache_airflow_providers_amazon-8.29.0rc1.dist-info → apache_airflow_providers_amazon-9.0.0rc1.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_amazon-8.29.0rc1.dist-info → apache_airflow_providers_amazon-9.0.0rc1.dist-info}/entry_points.txt +0 -0
@@ -19,17 +19,15 @@
|
|
19
19
|
from __future__ import annotations
|
20
20
|
|
21
21
|
import logging
|
22
|
-
import warnings
|
23
22
|
from ast import literal_eval
|
24
23
|
from datetime import timedelta
|
25
24
|
from functools import cached_property
|
26
25
|
from typing import TYPE_CHECKING, Any, List, Sequence, cast
|
27
26
|
|
28
27
|
from botocore.exceptions import ClientError, WaiterError
|
29
|
-
from deprecated import deprecated
|
30
28
|
|
31
29
|
from airflow.configuration import conf
|
32
|
-
from airflow.exceptions import AirflowException
|
30
|
+
from airflow.exceptions import AirflowException
|
33
31
|
from airflow.models import BaseOperator
|
34
32
|
from airflow.providers.amazon.aws.hooks.eks import EksHook
|
35
33
|
from airflow.providers.amazon.aws.triggers.eks import (
|
@@ -267,17 +265,6 @@ class EksCreateClusterOperator(BaseOperator):
|
|
267
265
|
def hook(self) -> EksHook:
|
268
266
|
return EksHook(aws_conn_id=self.aws_conn_id, region_name=self.region)
|
269
267
|
|
270
|
-
@property
|
271
|
-
@deprecated(
|
272
|
-
reason=(
|
273
|
-
"`eks_hook` property is deprecated and will be removed in the future. "
|
274
|
-
"Please use `hook` property instead."
|
275
|
-
),
|
276
|
-
category=AirflowProviderDeprecationWarning,
|
277
|
-
)
|
278
|
-
def eks_hook(self):
|
279
|
-
return self.hook
|
280
|
-
|
281
268
|
def execute(self, context: Context):
|
282
269
|
if self.compute:
|
283
270
|
if self.compute not in SUPPORTED_COMPUTE_VALUES:
|
@@ -397,7 +384,7 @@ class EksCreateClusterOperator(BaseOperator):
|
|
397
384
|
waiter_delay=self.waiter_delay,
|
398
385
|
waiter_max_attempts=self.waiter_max_attempts,
|
399
386
|
aws_conn_id=self.aws_conn_id,
|
400
|
-
|
387
|
+
region_name=self.region,
|
401
388
|
),
|
402
389
|
method_name="execute_complete",
|
403
390
|
timeout=timedelta(seconds=self.waiter_max_attempts * self.waiter_delay),
|
@@ -656,7 +643,7 @@ class EksCreateFargateProfileOperator(BaseOperator):
|
|
656
643
|
aws_conn_id=self.aws_conn_id,
|
657
644
|
waiter_delay=self.waiter_delay,
|
658
645
|
waiter_max_attempts=self.waiter_max_attempts,
|
659
|
-
|
646
|
+
region_name=self.region,
|
660
647
|
),
|
661
648
|
method_name="execute_complete",
|
662
649
|
# timeout is set to ensure that if a trigger dies, the timeout does not restart
|
@@ -968,7 +955,7 @@ class EksDeleteFargateProfileOperator(BaseOperator):
|
|
968
955
|
aws_conn_id=self.aws_conn_id,
|
969
956
|
waiter_delay=self.waiter_delay,
|
970
957
|
waiter_max_attempts=self.waiter_max_attempts,
|
971
|
-
|
958
|
+
region_name=self.region,
|
972
959
|
),
|
973
960
|
method_name="execute_complete",
|
974
961
|
# timeout is set to ensure that if a trigger dies, the timeout does not restart
|
@@ -1016,11 +1003,6 @@ class EksPodOperator(KubernetesPodOperator):
|
|
1016
1003
|
If "delete_pod", the pod will be deleted regardless its state; if "delete_succeeded_pod",
|
1017
1004
|
only succeeded pod will be deleted. You can set to "keep_pod" to keep the pod.
|
1018
1005
|
Current default is `keep_pod`, but this will be changed in the next major release of this provider.
|
1019
|
-
:param is_delete_operator_pod: What to do when the pod reaches its final
|
1020
|
-
state, or the execution is interrupted. If True, delete the
|
1021
|
-
pod; if False, leave the pod. Current default is False, but this will be
|
1022
|
-
changed in the next major release of this provider.
|
1023
|
-
Deprecated - use `on_finish_action` instead.
|
1024
1006
|
|
1025
1007
|
"""
|
1026
1008
|
|
@@ -1043,37 +1025,16 @@ class EksPodOperator(KubernetesPodOperator):
|
|
1043
1025
|
# file is stored locally in the worker and not in the cluster.
|
1044
1026
|
in_cluster: bool = False,
|
1045
1027
|
namespace: str = DEFAULT_NAMESPACE_NAME,
|
1046
|
-
pod_context: str | None = None,
|
1047
1028
|
pod_name: str | None = None,
|
1048
|
-
pod_username: str | None = None,
|
1049
1029
|
aws_conn_id: str | None = DEFAULT_CONN_ID,
|
1050
1030
|
region: str | None = None,
|
1051
1031
|
on_finish_action: str | None = None,
|
1052
|
-
is_delete_operator_pod: bool | None = None,
|
1053
1032
|
**kwargs,
|
1054
1033
|
) -> None:
|
1055
|
-
if
|
1056
|
-
|
1057
|
-
"`is_delete_operator_pod` parameter is deprecated, please use `on_finish_action`",
|
1058
|
-
AirflowProviderDeprecationWarning,
|
1059
|
-
stacklevel=2,
|
1060
|
-
)
|
1061
|
-
kwargs["on_finish_action"] = (
|
1062
|
-
OnFinishAction.DELETE_POD if is_delete_operator_pod else OnFinishAction.KEEP_POD
|
1063
|
-
)
|
1034
|
+
if on_finish_action is not None:
|
1035
|
+
kwargs["on_finish_action"] = OnFinishAction(on_finish_action)
|
1064
1036
|
else:
|
1065
|
-
|
1066
|
-
kwargs["on_finish_action"] = OnFinishAction(on_finish_action)
|
1067
|
-
else:
|
1068
|
-
warnings.warn(
|
1069
|
-
f"You have not set parameter `on_finish_action` in class {self.__class__.__name__}. "
|
1070
|
-
"Currently the default for this parameter is `keep_pod` but in a future release"
|
1071
|
-
" the default will be changed to `delete_pod`. To ensure pods are not deleted in"
|
1072
|
-
" the future you will need to set `on_finish_action=keep_pod` explicitly.",
|
1073
|
-
AirflowProviderDeprecationWarning,
|
1074
|
-
stacklevel=2,
|
1075
|
-
)
|
1076
|
-
kwargs["on_finish_action"] = OnFinishAction.KEEP_POD
|
1037
|
+
kwargs["on_finish_action"] = OnFinishAction.DELETE_POD
|
1077
1038
|
|
1078
1039
|
self.cluster_name = cluster_name
|
1079
1040
|
self.in_cluster = in_cluster
|
@@ -18,14 +18,13 @@
|
|
18
18
|
from __future__ import annotations
|
19
19
|
|
20
20
|
import ast
|
21
|
-
import warnings
|
22
21
|
from datetime import timedelta
|
23
22
|
from functools import cached_property
|
24
23
|
from typing import TYPE_CHECKING, Any, Sequence
|
25
24
|
from uuid import uuid4
|
26
25
|
|
27
26
|
from airflow.configuration import conf
|
28
|
-
from airflow.exceptions import AirflowException
|
27
|
+
from airflow.exceptions import AirflowException
|
29
28
|
from airflow.models import BaseOperator
|
30
29
|
from airflow.providers.amazon.aws.hooks.emr import EmrContainerHook, EmrHook, EmrServerlessHook
|
31
30
|
from airflow.providers.amazon.aws.links.emr import (
|
@@ -227,11 +226,6 @@ class EmrStartNotebookExecutionOperator(BaseOperator):
|
|
227
226
|
:param tags: Optional list of key value pair to associate with the notebook execution.
|
228
227
|
:param waiter_max_attempts: Maximum number of tries before failing.
|
229
228
|
:param waiter_delay: Number of seconds between polling the state of the notebook.
|
230
|
-
|
231
|
-
:param waiter_countdown: Total amount of time the operator will wait for the notebook to stop.
|
232
|
-
Defaults to 25 * 60 seconds. (Deprecated. Please use waiter_max_attempts.)
|
233
|
-
:param waiter_check_interval_seconds: Number of seconds between polling the state of the notebook.
|
234
|
-
Defaults to 60 seconds. (Deprecated. Please use waiter_delay.)
|
235
229
|
"""
|
236
230
|
|
237
231
|
template_fields: Sequence[str] = (
|
@@ -261,35 +255,10 @@ class EmrStartNotebookExecutionOperator(BaseOperator):
|
|
261
255
|
tags: list | None = None,
|
262
256
|
wait_for_completion: bool = False,
|
263
257
|
aws_conn_id: str | None = "aws_default",
|
264
|
-
# TODO: waiter_max_attempts and waiter_delay should default to None when the other two are deprecated.
|
265
258
|
waiter_max_attempts: int | None = None,
|
266
259
|
waiter_delay: int | None = None,
|
267
|
-
waiter_countdown: int | None = None,
|
268
|
-
waiter_check_interval_seconds: int | None = None,
|
269
260
|
**kwargs: Any,
|
270
261
|
):
|
271
|
-
if waiter_check_interval_seconds:
|
272
|
-
warnings.warn(
|
273
|
-
"The parameter `waiter_check_interval_seconds` has been deprecated to "
|
274
|
-
"standardize naming conventions. Please `use waiter_delay instead`. In the "
|
275
|
-
"future this will default to None and defer to the waiter's default value.",
|
276
|
-
AirflowProviderDeprecationWarning,
|
277
|
-
stacklevel=2,
|
278
|
-
)
|
279
|
-
else:
|
280
|
-
waiter_check_interval_seconds = 60
|
281
|
-
if waiter_countdown:
|
282
|
-
warnings.warn(
|
283
|
-
"The parameter waiter_countdown has been deprecated to standardize "
|
284
|
-
"naming conventions. Please use waiter_max_attempts instead. In the "
|
285
|
-
"future this will default to None and defer to the waiter's default value.",
|
286
|
-
AirflowProviderDeprecationWarning,
|
287
|
-
stacklevel=2,
|
288
|
-
)
|
289
|
-
# waiter_countdown defaults to never timing out, which is not supported
|
290
|
-
# by boto waiters, so we will set it here to "a very long time" for now.
|
291
|
-
waiter_max_attempts = (waiter_countdown or 999) // waiter_check_interval_seconds
|
292
|
-
|
293
262
|
super().__init__(**kwargs)
|
294
263
|
self.editor_id = editor_id
|
295
264
|
self.relative_path = relative_path
|
@@ -302,7 +271,7 @@ class EmrStartNotebookExecutionOperator(BaseOperator):
|
|
302
271
|
self.cluster_id = cluster_id
|
303
272
|
self.aws_conn_id = aws_conn_id
|
304
273
|
self.waiter_max_attempts = waiter_max_attempts or 25
|
305
|
-
self.waiter_delay = waiter_delay or
|
274
|
+
self.waiter_delay = waiter_delay or 60
|
306
275
|
self.master_instance_security_group_id = master_instance_security_group_id
|
307
276
|
|
308
277
|
def execute(self, context: Context):
|
@@ -371,11 +340,6 @@ class EmrStopNotebookExecutionOperator(BaseOperator):
|
|
371
340
|
maintained on each worker node).
|
372
341
|
:param waiter_max_attempts: Maximum number of tries before failing.
|
373
342
|
:param waiter_delay: Number of seconds between polling the state of the notebook.
|
374
|
-
|
375
|
-
:param waiter_countdown: Total amount of time the operator will wait for the notebook to stop.
|
376
|
-
Defaults to 25 * 60 seconds. (Deprecated. Please use waiter_max_attempts.)
|
377
|
-
:param waiter_check_interval_seconds: Number of seconds between polling the state of the notebook.
|
378
|
-
Defaults to 60 seconds. (Deprecated. Please use waiter_delay.)
|
379
343
|
"""
|
380
344
|
|
381
345
|
template_fields: Sequence[str] = (
|
@@ -389,41 +353,16 @@ class EmrStopNotebookExecutionOperator(BaseOperator):
|
|
389
353
|
notebook_execution_id: str,
|
390
354
|
wait_for_completion: bool = False,
|
391
355
|
aws_conn_id: str | None = "aws_default",
|
392
|
-
# TODO: waiter_max_attempts and waiter_delay should default to None when the other two are deprecated.
|
393
356
|
waiter_max_attempts: int | None = None,
|
394
357
|
waiter_delay: int | None = None,
|
395
|
-
waiter_countdown: int | None = None,
|
396
|
-
waiter_check_interval_seconds: int | None = None,
|
397
358
|
**kwargs: Any,
|
398
359
|
):
|
399
|
-
if waiter_check_interval_seconds:
|
400
|
-
warnings.warn(
|
401
|
-
"The parameter `waiter_check_interval_seconds` has been deprecated to "
|
402
|
-
"standardize naming conventions. Please `use waiter_delay instead`. In the "
|
403
|
-
"future this will default to None and defer to the waiter's default value.",
|
404
|
-
AirflowProviderDeprecationWarning,
|
405
|
-
stacklevel=2,
|
406
|
-
)
|
407
|
-
else:
|
408
|
-
waiter_check_interval_seconds = 60
|
409
|
-
if waiter_countdown:
|
410
|
-
warnings.warn(
|
411
|
-
"The parameter waiter_countdown has been deprecated to standardize "
|
412
|
-
"naming conventions. Please use waiter_max_attempts instead. In the "
|
413
|
-
"future this will default to None and defer to the waiter's default value.",
|
414
|
-
AirflowProviderDeprecationWarning,
|
415
|
-
stacklevel=2,
|
416
|
-
)
|
417
|
-
# waiter_countdown defaults to never timing out, which is not supported
|
418
|
-
# by boto waiters, so we will set it here to "a very long time" for now.
|
419
|
-
waiter_max_attempts = (waiter_countdown or 999) // waiter_check_interval_seconds
|
420
|
-
|
421
360
|
super().__init__(**kwargs)
|
422
361
|
self.notebook_execution_id = notebook_execution_id
|
423
362
|
self.wait_for_completion = wait_for_completion
|
424
363
|
self.aws_conn_id = aws_conn_id
|
425
364
|
self.waiter_max_attempts = waiter_max_attempts or 25
|
426
|
-
self.waiter_delay = waiter_delay or
|
365
|
+
self.waiter_delay = waiter_delay or 60
|
427
366
|
|
428
367
|
def execute(self, context: Context) -> None:
|
429
368
|
emr_hook = EmrHook(aws_conn_id=self.aws_conn_id)
|
@@ -518,7 +457,6 @@ class EmrContainerOperator(BaseOperator):
|
|
518
457
|
:param aws_conn_id: The Airflow connection used for AWS credentials.
|
519
458
|
:param wait_for_completion: Whether or not to wait in the operator for the job to complete.
|
520
459
|
:param poll_interval: Time (in seconds) to wait between two consecutive calls to check query status on EMR
|
521
|
-
:param max_tries: Deprecated - use max_polling_attempts instead.
|
522
460
|
:param max_polling_attempts: Maximum number of times to wait for the job run to finish.
|
523
461
|
Defaults to None, which will poll until the job is *not* in a pending, submitted, or running state.
|
524
462
|
:param job_retry_max_attempts: Maximum number of times to retry when the EMR job fails.
|
@@ -551,7 +489,6 @@ class EmrContainerOperator(BaseOperator):
|
|
551
489
|
aws_conn_id: str | None = "aws_default",
|
552
490
|
wait_for_completion: bool = True,
|
553
491
|
poll_interval: int = 30,
|
554
|
-
max_tries: int | None = None,
|
555
492
|
tags: dict | None = None,
|
556
493
|
max_polling_attempts: int | None = None,
|
557
494
|
job_retry_max_attempts: int | None = None,
|
@@ -575,18 +512,6 @@ class EmrContainerOperator(BaseOperator):
|
|
575
512
|
self.job_id: str | None = None
|
576
513
|
self.deferrable = deferrable
|
577
514
|
|
578
|
-
if max_tries:
|
579
|
-
warnings.warn(
|
580
|
-
f"Parameter `{self.__class__.__name__}.max_tries` is deprecated and will be removed "
|
581
|
-
"in a future release. Please use method `max_polling_attempts` instead.",
|
582
|
-
AirflowProviderDeprecationWarning,
|
583
|
-
stacklevel=2,
|
584
|
-
)
|
585
|
-
if max_polling_attempts and max_polling_attempts != max_tries:
|
586
|
-
raise ValueError("max_polling_attempts must be the same value as max_tries")
|
587
|
-
else:
|
588
|
-
self.max_polling_attempts = max_tries
|
589
|
-
|
590
515
|
@cached_property
|
591
516
|
def hook(self) -> EmrContainerHook:
|
592
517
|
"""Create and return an EmrContainerHook."""
|
@@ -715,11 +640,6 @@ class EmrCreateJobFlowOperator(BaseOperator):
|
|
715
640
|
completion (True)
|
716
641
|
:param waiter_max_attempts: Maximum number of tries before failing.
|
717
642
|
:param waiter_delay: Number of seconds between polling the state of the notebook.
|
718
|
-
|
719
|
-
:param waiter_countdown: Max. seconds to wait for jobflow completion (only in combination with
|
720
|
-
wait_for_completion=True, None = no limit) (Deprecated. Please use waiter_max_attempts.)
|
721
|
-
:param waiter_check_interval_seconds: Number of seconds between polling the jobflow state. Defaults to 60
|
722
|
-
seconds. (Deprecated. Please use waiter_delay.)
|
723
643
|
:param deferrable: If True, the operator will wait asynchronously for the crawl to complete.
|
724
644
|
This implies waiting for completion. This mode requires aiobotocore module to be installed.
|
725
645
|
(default: False)
|
@@ -748,33 +668,9 @@ class EmrCreateJobFlowOperator(BaseOperator):
|
|
748
668
|
wait_for_completion: bool = False,
|
749
669
|
waiter_max_attempts: int | None = None,
|
750
670
|
waiter_delay: int | None = None,
|
751
|
-
waiter_countdown: int | None = None,
|
752
|
-
waiter_check_interval_seconds: int | None = None,
|
753
671
|
deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
|
754
672
|
**kwargs: Any,
|
755
673
|
):
|
756
|
-
if waiter_check_interval_seconds:
|
757
|
-
warnings.warn(
|
758
|
-
"The parameter `waiter_check_interval_seconds` has been deprecated to "
|
759
|
-
"standardize naming conventions. Please `use waiter_delay instead`. In the "
|
760
|
-
"future this will default to None and defer to the waiter's default value.",
|
761
|
-
AirflowProviderDeprecationWarning,
|
762
|
-
stacklevel=2,
|
763
|
-
)
|
764
|
-
else:
|
765
|
-
waiter_check_interval_seconds = 60
|
766
|
-
if waiter_countdown:
|
767
|
-
warnings.warn(
|
768
|
-
"The parameter waiter_countdown has been deprecated to standardize "
|
769
|
-
"naming conventions. Please use waiter_max_attempts instead. In the "
|
770
|
-
"future this will default to None and defer to the waiter's default value.",
|
771
|
-
AirflowProviderDeprecationWarning,
|
772
|
-
stacklevel=2,
|
773
|
-
)
|
774
|
-
# waiter_countdown defaults to never timing out, which is not supported
|
775
|
-
# by boto waiters, so we will set it here to "a very long time" for now.
|
776
|
-
waiter_max_attempts = (waiter_countdown or 999) // waiter_check_interval_seconds
|
777
|
-
|
778
674
|
super().__init__(**kwargs)
|
779
675
|
self.aws_conn_id = aws_conn_id
|
780
676
|
self.emr_conn_id = emr_conn_id
|
@@ -782,7 +678,7 @@ class EmrCreateJobFlowOperator(BaseOperator):
|
|
782
678
|
self.region_name = region_name
|
783
679
|
self.wait_for_completion = wait_for_completion
|
784
680
|
self.waiter_max_attempts = waiter_max_attempts or 60
|
785
|
-
self.waiter_delay = waiter_delay or
|
681
|
+
self.waiter_delay = waiter_delay or 60
|
786
682
|
self.deferrable = deferrable
|
787
683
|
|
788
684
|
@cached_property
|
@@ -1054,10 +950,6 @@ class EmrServerlessCreateApplicationOperator(BaseOperator):
|
|
1054
950
|
running Airflow in a distributed manner and aws_conn_id is None or
|
1055
951
|
empty, then default boto3 configuration would be used (and must be
|
1056
952
|
maintained on each worker node).
|
1057
|
-
:param waiter_countdown: (deprecated) Total amount of time, in seconds, the operator will wait for
|
1058
|
-
the application to start. Defaults to 25 minutes.
|
1059
|
-
:param waiter_check_interval_seconds: (deprecated) Number of seconds between polling the state
|
1060
|
-
of the application. Defaults to 60 seconds.
|
1061
953
|
:waiter_max_attempts: Number of times the waiter should poll the application to check the state.
|
1062
954
|
If not set, the waiter will use its default value.
|
1063
955
|
:param waiter_delay: Number of seconds between polling the state of the application.
|
@@ -1074,38 +966,14 @@ class EmrServerlessCreateApplicationOperator(BaseOperator):
|
|
1074
966
|
config: dict | None = None,
|
1075
967
|
wait_for_completion: bool = True,
|
1076
968
|
aws_conn_id: str | None = "aws_default",
|
1077
|
-
waiter_countdown: int | ArgNotSet = NOTSET,
|
1078
|
-
waiter_check_interval_seconds: int | ArgNotSet = NOTSET,
|
1079
969
|
waiter_max_attempts: int | ArgNotSet = NOTSET,
|
1080
970
|
waiter_delay: int | ArgNotSet = NOTSET,
|
1081
971
|
deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
|
1082
972
|
**kwargs,
|
1083
973
|
):
|
1084
|
-
if
|
1085
|
-
|
1086
|
-
|
1087
|
-
waiter_delay = waiter_check_interval_seconds if waiter_delay is NOTSET else waiter_delay
|
1088
|
-
warnings.warn(
|
1089
|
-
"The parameter waiter_check_interval_seconds has been deprecated to standardize "
|
1090
|
-
"naming conventions. Please use waiter_delay instead. In the "
|
1091
|
-
"future this will default to None and defer to the waiter's default value.",
|
1092
|
-
AirflowProviderDeprecationWarning,
|
1093
|
-
stacklevel=2,
|
1094
|
-
)
|
1095
|
-
if waiter_countdown is NOTSET:
|
1096
|
-
waiter_max_attempts = 25 if waiter_max_attempts is NOTSET else waiter_max_attempts
|
1097
|
-
else:
|
1098
|
-
if waiter_max_attempts is NOTSET:
|
1099
|
-
# ignoring mypy because it doesn't like ArgNotSet as an operand, but neither variables
|
1100
|
-
# are of type ArgNotSet at this point.
|
1101
|
-
waiter_max_attempts = waiter_countdown // waiter_delay # type: ignore[operator]
|
1102
|
-
warnings.warn(
|
1103
|
-
"The parameter waiter_countdown has been deprecated to standardize "
|
1104
|
-
"naming conventions. Please use waiter_max_attempts instead. In the "
|
1105
|
-
"future this will default to None and defer to the waiter's default value.",
|
1106
|
-
AirflowProviderDeprecationWarning,
|
1107
|
-
stacklevel=2,
|
1108
|
-
)
|
974
|
+
waiter_delay = 60 if waiter_delay is NOTSET else waiter_delay
|
975
|
+
waiter_max_attempts = 25 if waiter_max_attempts is NOTSET else waiter_max_attempts
|
976
|
+
|
1109
977
|
self.aws_conn_id = aws_conn_id
|
1110
978
|
self.release_label = release_label
|
1111
979
|
self.job_type = job_type
|
@@ -1228,10 +1096,6 @@ class EmrServerlessStartJobOperator(BaseOperator):
|
|
1228
1096
|
empty, then default boto3 configuration would be used (and must be
|
1229
1097
|
maintained on each worker node).
|
1230
1098
|
:param name: Name for the EMR Serverless job. If not provided, a default name will be assigned.
|
1231
|
-
:param waiter_countdown: (deprecated) Total amount of time, in seconds, the operator will wait for
|
1232
|
-
the job finish. Defaults to 25 minutes.
|
1233
|
-
:param waiter_check_interval_seconds: (deprecated) Number of seconds between polling the state of the job.
|
1234
|
-
Defaults to 60 seconds.
|
1235
1099
|
:waiter_max_attempts: Number of times the waiter should poll the application to check the state.
|
1236
1100
|
If not set, the waiter will use its default value.
|
1237
1101
|
:param waiter_delay: Number of seconds between polling the state of the job run.
|
@@ -1276,39 +1140,15 @@ class EmrServerlessStartJobOperator(BaseOperator):
|
|
1276
1140
|
wait_for_completion: bool = True,
|
1277
1141
|
aws_conn_id: str | None = "aws_default",
|
1278
1142
|
name: str | None = None,
|
1279
|
-
waiter_countdown: int | ArgNotSet = NOTSET,
|
1280
|
-
waiter_check_interval_seconds: int | ArgNotSet = NOTSET,
|
1281
1143
|
waiter_max_attempts: int | ArgNotSet = NOTSET,
|
1282
1144
|
waiter_delay: int | ArgNotSet = NOTSET,
|
1283
1145
|
deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
|
1284
1146
|
enable_application_ui_links: bool = False,
|
1285
1147
|
**kwargs,
|
1286
1148
|
):
|
1287
|
-
if
|
1288
|
-
|
1289
|
-
|
1290
|
-
waiter_delay = waiter_check_interval_seconds if waiter_delay is NOTSET else waiter_delay
|
1291
|
-
warnings.warn(
|
1292
|
-
"The parameter waiter_check_interval_seconds has been deprecated to standardize "
|
1293
|
-
"naming conventions. Please use waiter_delay instead. In the "
|
1294
|
-
"future this will default to None and defer to the waiter's default value.",
|
1295
|
-
AirflowProviderDeprecationWarning,
|
1296
|
-
stacklevel=2,
|
1297
|
-
)
|
1298
|
-
if waiter_countdown is NOTSET:
|
1299
|
-
waiter_max_attempts = 25 if waiter_max_attempts is NOTSET else waiter_max_attempts
|
1300
|
-
else:
|
1301
|
-
if waiter_max_attempts is NOTSET:
|
1302
|
-
# ignoring mypy because it doesn't like ArgNotSet as an operand, but neither variables
|
1303
|
-
# are of type ArgNotSet at this point.
|
1304
|
-
waiter_max_attempts = waiter_countdown // waiter_delay # type: ignore[operator]
|
1305
|
-
warnings.warn(
|
1306
|
-
"The parameter waiter_countdown has been deprecated to standardize "
|
1307
|
-
"naming conventions. Please use waiter_max_attempts instead. In the "
|
1308
|
-
"future this will default to None and defer to the waiter's default value.",
|
1309
|
-
AirflowProviderDeprecationWarning,
|
1310
|
-
stacklevel=2,
|
1311
|
-
)
|
1149
|
+
waiter_delay = 60 if waiter_delay is NOTSET else waiter_delay
|
1150
|
+
waiter_max_attempts = 25 if waiter_max_attempts is NOTSET else waiter_max_attempts
|
1151
|
+
|
1312
1152
|
self.aws_conn_id = aws_conn_id
|
1313
1153
|
self.application_id = application_id
|
1314
1154
|
self.execution_role_arn = execution_role_arn
|
@@ -1566,10 +1406,6 @@ class EmrServerlessStopApplicationOperator(BaseOperator):
|
|
1566
1406
|
running Airflow in a distributed manner and aws_conn_id is None or
|
1567
1407
|
empty, then default boto3 configuration would be used (and must be
|
1568
1408
|
maintained on each worker node).
|
1569
|
-
:param waiter_countdown: (deprecated) Total amount of time, in seconds, the operator will wait for
|
1570
|
-
the application be stopped. Defaults to 5 minutes.
|
1571
|
-
:param waiter_check_interval_seconds: (deprecated) Number of seconds between polling the state of the
|
1572
|
-
application. Defaults to 60 seconds.
|
1573
1409
|
:param force_stop: If set to True, any job for that app that is not in a terminal state will be cancelled.
|
1574
1410
|
Otherwise, trying to stop an app with running jobs will return an error.
|
1575
1411
|
If you want to wait for the jobs to finish gracefully, use
|
@@ -1590,39 +1426,15 @@ class EmrServerlessStopApplicationOperator(BaseOperator):
|
|
1590
1426
|
application_id: str,
|
1591
1427
|
wait_for_completion: bool = True,
|
1592
1428
|
aws_conn_id: str | None = "aws_default",
|
1593
|
-
waiter_countdown: int | ArgNotSet = NOTSET,
|
1594
|
-
waiter_check_interval_seconds: int | ArgNotSet = NOTSET,
|
1595
1429
|
waiter_max_attempts: int | ArgNotSet = NOTSET,
|
1596
1430
|
waiter_delay: int | ArgNotSet = NOTSET,
|
1597
1431
|
force_stop: bool = False,
|
1598
1432
|
deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
|
1599
1433
|
**kwargs,
|
1600
1434
|
):
|
1601
|
-
if
|
1602
|
-
|
1603
|
-
|
1604
|
-
waiter_delay = waiter_check_interval_seconds if waiter_delay is NOTSET else waiter_delay
|
1605
|
-
warnings.warn(
|
1606
|
-
"The parameter waiter_check_interval_seconds has been deprecated to standardize "
|
1607
|
-
"naming conventions. Please use waiter_delay instead. In the "
|
1608
|
-
"future this will default to None and defer to the waiter's default value.",
|
1609
|
-
AirflowProviderDeprecationWarning,
|
1610
|
-
stacklevel=2,
|
1611
|
-
)
|
1612
|
-
if waiter_countdown is NOTSET:
|
1613
|
-
waiter_max_attempts = 25 if waiter_max_attempts is NOTSET else waiter_max_attempts
|
1614
|
-
else:
|
1615
|
-
if waiter_max_attempts is NOTSET:
|
1616
|
-
# ignoring mypy because it doesn't like ArgNotSet as an operand, but neither variables
|
1617
|
-
# are of type ArgNotSet at this point.
|
1618
|
-
waiter_max_attempts = waiter_countdown // waiter_delay # type: ignore[operator]
|
1619
|
-
warnings.warn(
|
1620
|
-
"The parameter waiter_countdown has been deprecated to standardize "
|
1621
|
-
"naming conventions. Please use waiter_max_attempts instead. In the "
|
1622
|
-
"future this will default to None and defer to the waiter's default value.",
|
1623
|
-
AirflowProviderDeprecationWarning,
|
1624
|
-
stacklevel=2,
|
1625
|
-
)
|
1435
|
+
waiter_delay = 60 if waiter_delay is NOTSET else waiter_delay
|
1436
|
+
waiter_max_attempts = 25 if waiter_max_attempts is NOTSET else waiter_max_attempts
|
1437
|
+
|
1626
1438
|
self.aws_conn_id = aws_conn_id
|
1627
1439
|
self.application_id = application_id
|
1628
1440
|
self.wait_for_completion = False if deferrable else wait_for_completion
|
@@ -1734,10 +1546,6 @@ class EmrServerlessDeleteApplicationOperator(EmrServerlessStopApplicationOperato
|
|
1734
1546
|
running Airflow in a distributed manner and aws_conn_id is None or
|
1735
1547
|
empty, then default boto3 configuration would be used (and must be
|
1736
1548
|
maintained on each worker node).
|
1737
|
-
:param waiter_countdown: (deprecated) Total amount of time, in seconds, the operator will wait for each
|
1738
|
-
step of first,the application to be stopped, and then deleted. Defaults to 25 minutes.
|
1739
|
-
:param waiter_check_interval_seconds: (deprecated) Number of seconds between polling the state
|
1740
|
-
of the application. Defaults to 60 seconds.
|
1741
1549
|
:waiter_max_attempts: Number of times the waiter should poll the application to check the state.
|
1742
1550
|
Defaults to 25.
|
1743
1551
|
:param waiter_delay: Number of seconds between polling the state of the application.
|
@@ -1758,39 +1566,15 @@ class EmrServerlessDeleteApplicationOperator(EmrServerlessStopApplicationOperato
|
|
1758
1566
|
application_id: str,
|
1759
1567
|
wait_for_completion: bool = True,
|
1760
1568
|
aws_conn_id: str | None = "aws_default",
|
1761
|
-
waiter_countdown: int | ArgNotSet = NOTSET,
|
1762
|
-
waiter_check_interval_seconds: int | ArgNotSet = NOTSET,
|
1763
1569
|
waiter_max_attempts: int | ArgNotSet = NOTSET,
|
1764
1570
|
waiter_delay: int | ArgNotSet = NOTSET,
|
1765
1571
|
force_stop: bool = False,
|
1766
1572
|
deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
|
1767
1573
|
**kwargs,
|
1768
1574
|
):
|
1769
|
-
if
|
1770
|
-
|
1771
|
-
|
1772
|
-
waiter_delay = waiter_check_interval_seconds if waiter_delay is NOTSET else waiter_delay
|
1773
|
-
warnings.warn(
|
1774
|
-
"The parameter waiter_check_interval_seconds has been deprecated to standardize "
|
1775
|
-
"naming conventions. Please use waiter_delay instead. In the "
|
1776
|
-
"future this will default to None and defer to the waiter's default value.",
|
1777
|
-
AirflowProviderDeprecationWarning,
|
1778
|
-
stacklevel=2,
|
1779
|
-
)
|
1780
|
-
if waiter_countdown is NOTSET:
|
1781
|
-
waiter_max_attempts = 25 if waiter_max_attempts is NOTSET else waiter_max_attempts
|
1782
|
-
else:
|
1783
|
-
if waiter_max_attempts is NOTSET:
|
1784
|
-
# ignoring mypy because it doesn't like ArgNotSet as an operand, but neither variables
|
1785
|
-
# are of type ArgNotSet at this point.
|
1786
|
-
waiter_max_attempts = waiter_countdown // waiter_delay # type: ignore[operator]
|
1787
|
-
warnings.warn(
|
1788
|
-
"The parameter waiter_countdown has been deprecated to standardize "
|
1789
|
-
"naming conventions. Please use waiter_max_attempts instead. In the "
|
1790
|
-
"future this will default to None and defer to the waiter's default value.",
|
1791
|
-
AirflowProviderDeprecationWarning,
|
1792
|
-
stacklevel=2,
|
1793
|
-
)
|
1575
|
+
waiter_delay = 60 if waiter_delay is NOTSET else waiter_delay
|
1576
|
+
waiter_max_attempts = 25 if waiter_max_attempts is NOTSET else waiter_max_attempts
|
1577
|
+
|
1794
1578
|
self.wait_for_delete_completion = wait_for_completion
|
1795
1579
|
# super stops the app
|
1796
1580
|
super().__init__(
|
@@ -17,11 +17,10 @@
|
|
17
17
|
# under the License.
|
18
18
|
from __future__ import annotations
|
19
19
|
|
20
|
-
import warnings
|
21
20
|
from typing import TYPE_CHECKING, Any, Sequence
|
22
21
|
|
23
22
|
from airflow.configuration import conf
|
24
|
-
from airflow.exceptions import AirflowException
|
23
|
+
from airflow.exceptions import AirflowException
|
25
24
|
from airflow.providers.amazon.aws.hooks.glue_databrew import GlueDataBrewHook
|
26
25
|
from airflow.providers.amazon.aws.operators.base_aws import AwsBaseOperator
|
27
26
|
from airflow.providers.amazon.aws.triggers.glue_databrew import GlueDataBrewJobCompleteTrigger
|
@@ -49,7 +48,6 @@ class GlueDataBrewStartJobOperator(AwsBaseOperator[GlueDataBrewHook]):
|
|
49
48
|
:param deferrable: If True, the operator will wait asynchronously for the job to complete.
|
50
49
|
This implies waiting for completion. This mode requires aiobotocore module to be installed.
|
51
50
|
(default: False)
|
52
|
-
:param delay: Time in seconds to wait between status checks. (Deprecated).
|
53
51
|
:param waiter_delay: Time in seconds to wait between status checks. Default is 30.
|
54
52
|
:param waiter_max_attempts: Maximum number of attempts to check for job completion. (default: 60)
|
55
53
|
:return: dictionary with key run_id and value of the resulting job's run_id.
|
@@ -92,13 +90,6 @@ class GlueDataBrewStartJobOperator(AwsBaseOperator[GlueDataBrewHook]):
|
|
92
90
|
self.waiter_delay = waiter_delay
|
93
91
|
self.waiter_max_attempts = waiter_max_attempts
|
94
92
|
self.deferrable = deferrable
|
95
|
-
if delay is not None:
|
96
|
-
warnings.warn(
|
97
|
-
"please use `waiter_delay` instead of delay.",
|
98
|
-
AirflowProviderDeprecationWarning,
|
99
|
-
stacklevel=2,
|
100
|
-
)
|
101
|
-
self.waiter_delay = delay
|
102
93
|
|
103
94
|
def execute(self, context: Context):
|
104
95
|
job = self.hook.conn.start_job_run(Name=self.job_name)
|
@@ -18,13 +18,12 @@
|
|
18
18
|
from __future__ import annotations
|
19
19
|
|
20
20
|
import json
|
21
|
-
import warnings
|
22
21
|
from datetime import timedelta
|
23
22
|
from functools import cached_property
|
24
23
|
from typing import TYPE_CHECKING, Any, Sequence
|
25
24
|
|
26
25
|
from airflow.configuration import conf
|
27
|
-
from airflow.exceptions import AirflowException
|
26
|
+
from airflow.exceptions import AirflowException
|
28
27
|
from airflow.models import BaseOperator
|
29
28
|
from airflow.providers.amazon.aws.hooks.rds import RdsHook
|
30
29
|
from airflow.providers.amazon.aws.triggers.rds import (
|
@@ -55,30 +54,17 @@ class RdsBaseOperator(BaseOperator):
|
|
55
54
|
*args,
|
56
55
|
aws_conn_id: str | None = "aws_conn_id",
|
57
56
|
region_name: str | None = None,
|
58
|
-
hook_params: dict | None = None,
|
59
57
|
**kwargs,
|
60
58
|
):
|
61
|
-
if hook_params is not None:
|
62
|
-
warnings.warn(
|
63
|
-
"The parameter hook_params is deprecated and will be removed. "
|
64
|
-
"Note that it is also incompatible with deferrable mode. "
|
65
|
-
"You can use the region_name parameter to specify the region. "
|
66
|
-
"If you were using hook_params for other purposes, please get in touch either on "
|
67
|
-
"airflow slack, or by opening a github issue on the project. "
|
68
|
-
"You can mention https://github.com/apache/airflow/pull/32352",
|
69
|
-
AirflowProviderDeprecationWarning,
|
70
|
-
stacklevel=3, # 2 is in the operator's init, 3 is in the user code creating the operator
|
71
|
-
)
|
72
|
-
self.hook_params = hook_params or {}
|
73
59
|
self.aws_conn_id = aws_conn_id
|
74
|
-
self.region_name = region_name
|
60
|
+
self.region_name = region_name
|
75
61
|
super().__init__(*args, **kwargs)
|
76
62
|
|
77
63
|
self._await_interval = 60 # seconds
|
78
64
|
|
79
65
|
@cached_property
|
80
66
|
def hook(self) -> RdsHook:
|
81
|
-
return RdsHook(aws_conn_id=self.aws_conn_id, region_name=self.region_name
|
67
|
+
return RdsHook(aws_conn_id=self.aws_conn_id, region_name=self.region_name)
|
82
68
|
|
83
69
|
def execute(self, context: Context) -> str:
|
84
70
|
"""Different implementations for snapshots, tasks and events."""
|