apache-airflow-providers-amazon 8.16.0rc1__py3-none-any.whl → 8.17.0rc2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/amazon/__init__.py +1 -1
- airflow/providers/amazon/aws/auth_manager/avp/entities.py +1 -0
- airflow/providers/amazon/aws/auth_manager/avp/facade.py +34 -19
- airflow/providers/amazon/aws/auth_manager/aws_auth_manager.py +44 -1
- airflow/providers/amazon/aws/auth_manager/cli/__init__.py +16 -0
- airflow/providers/amazon/aws/auth_manager/cli/avp_commands.py +178 -0
- airflow/providers/amazon/aws/auth_manager/cli/definition.py +62 -0
- airflow/providers/amazon/aws/auth_manager/cli/schema.json +171 -0
- airflow/providers/amazon/aws/auth_manager/constants.py +1 -0
- airflow/providers/amazon/aws/executors/ecs/ecs_executor.py +77 -23
- airflow/providers/amazon/aws/executors/ecs/ecs_executor_config.py +17 -0
- airflow/providers/amazon/aws/executors/ecs/utils.py +1 -1
- airflow/providers/amazon/aws/executors/utils/__init__.py +16 -0
- airflow/providers/amazon/aws/executors/utils/exponential_backoff_retry.py +60 -0
- airflow/providers/amazon/aws/hooks/athena_sql.py +168 -0
- airflow/providers/amazon/aws/hooks/base_aws.py +14 -0
- airflow/providers/amazon/aws/hooks/quicksight.py +33 -18
- airflow/providers/amazon/aws/hooks/redshift_data.py +66 -17
- airflow/providers/amazon/aws/hooks/redshift_sql.py +1 -1
- airflow/providers/amazon/aws/hooks/s3.py +18 -4
- airflow/providers/amazon/aws/log/cloudwatch_task_handler.py +2 -2
- airflow/providers/amazon/aws/operators/batch.py +33 -15
- airflow/providers/amazon/aws/operators/cloud_formation.py +37 -26
- airflow/providers/amazon/aws/operators/datasync.py +19 -18
- airflow/providers/amazon/aws/operators/dms.py +57 -69
- airflow/providers/amazon/aws/operators/ec2.py +19 -5
- airflow/providers/amazon/aws/operators/emr.py +30 -10
- airflow/providers/amazon/aws/operators/eventbridge.py +57 -80
- airflow/providers/amazon/aws/operators/quicksight.py +17 -24
- airflow/providers/amazon/aws/operators/redshift_data.py +68 -19
- airflow/providers/amazon/aws/operators/s3.py +1 -1
- airflow/providers/amazon/aws/operators/sagemaker.py +42 -12
- airflow/providers/amazon/aws/sensors/cloud_formation.py +30 -25
- airflow/providers/amazon/aws/sensors/dms.py +31 -24
- airflow/providers/amazon/aws/sensors/dynamodb.py +15 -15
- airflow/providers/amazon/aws/sensors/quicksight.py +34 -24
- airflow/providers/amazon/aws/sensors/redshift_cluster.py +41 -3
- airflow/providers/amazon/aws/sensors/s3.py +13 -8
- airflow/providers/amazon/aws/triggers/redshift_cluster.py +54 -2
- airflow/providers/amazon/aws/triggers/redshift_data.py +113 -0
- airflow/providers/amazon/aws/triggers/s3.py +9 -4
- airflow/providers/amazon/get_provider_info.py +55 -16
- {apache_airflow_providers_amazon-8.16.0rc1.dist-info → apache_airflow_providers_amazon-8.17.0rc2.dist-info}/METADATA +15 -13
- {apache_airflow_providers_amazon-8.16.0rc1.dist-info → apache_airflow_providers_amazon-8.17.0rc2.dist-info}/RECORD +46 -38
- {apache_airflow_providers_amazon-8.16.0rc1.dist-info → apache_airflow_providers_amazon-8.17.0rc2.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_amazon-8.16.0rc1.dist-info → apache_airflow_providers_amazon-8.17.0rc2.dist-info}/entry_points.txt +0 -0
@@ -18,16 +18,15 @@ from __future__ import annotations
|
|
18
18
|
|
19
19
|
from typing import TYPE_CHECKING, Sequence
|
20
20
|
|
21
|
-
from airflow.models import BaseOperator
|
22
21
|
from airflow.providers.amazon.aws.hooks.quicksight import QuickSightHook
|
22
|
+
from airflow.providers.amazon.aws.operators.base_aws import AwsBaseOperator
|
23
|
+
from airflow.providers.amazon.aws.utils.mixins import aws_template_fields
|
23
24
|
|
24
25
|
if TYPE_CHECKING:
|
25
26
|
from airflow.utils.context import Context
|
26
27
|
|
27
|
-
DEFAULT_CONN_ID = "aws_default"
|
28
28
|
|
29
|
-
|
30
|
-
class QuickSightCreateIngestionOperator(BaseOperator):
|
29
|
+
class QuickSightCreateIngestionOperator(AwsBaseOperator[QuickSightHook]):
|
31
30
|
"""
|
32
31
|
Creates and starts a new SPICE ingestion for a dataset; also helps to Refresh existing SPICE datasets.
|
33
32
|
|
@@ -43,23 +42,25 @@ class QuickSightCreateIngestionOperator(BaseOperator):
|
|
43
42
|
that the operation waits to check the status of the Amazon QuickSight Ingestion.
|
44
43
|
:param check_interval: if wait is set to be true, this is the time interval
|
45
44
|
in seconds which the operator will check the status of the Amazon QuickSight Ingestion
|
46
|
-
:param aws_conn_id: The Airflow connection used for AWS credentials.
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
:param
|
52
|
-
|
45
|
+
:param aws_conn_id: The Airflow connection used for AWS credentials.
|
46
|
+
If this is ``None`` or empty then the default boto3 behaviour is used. If
|
47
|
+
running Airflow in a distributed manner and aws_conn_id is None or
|
48
|
+
empty, then default boto3 configuration would be used (and must be
|
49
|
+
maintained on each worker node).
|
50
|
+
:param region_name: AWS region_name. If not specified then the default boto3 behaviour is used.
|
51
|
+
:param verify: Whether or not to verify SSL certificates. See:
|
52
|
+
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html
|
53
|
+
:param botocore_config: Configuration dictionary (key-values) for botocore client. See:
|
54
|
+
https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html
|
53
55
|
"""
|
54
56
|
|
55
|
-
|
57
|
+
aws_hook_class = QuickSightHook
|
58
|
+
template_fields: Sequence[str] = aws_template_fields(
|
56
59
|
"data_set_id",
|
57
60
|
"ingestion_id",
|
58
61
|
"ingestion_type",
|
59
62
|
"wait_for_completion",
|
60
63
|
"check_interval",
|
61
|
-
"aws_conn_id",
|
62
|
-
"region",
|
63
64
|
)
|
64
65
|
ui_color = "#ffd700"
|
65
66
|
|
@@ -70,26 +71,18 @@ class QuickSightCreateIngestionOperator(BaseOperator):
|
|
70
71
|
ingestion_type: str = "FULL_REFRESH",
|
71
72
|
wait_for_completion: bool = True,
|
72
73
|
check_interval: int = 30,
|
73
|
-
aws_conn_id: str = DEFAULT_CONN_ID,
|
74
|
-
region: str | None = None,
|
75
74
|
**kwargs,
|
76
75
|
):
|
76
|
+
super().__init__(**kwargs)
|
77
77
|
self.data_set_id = data_set_id
|
78
78
|
self.ingestion_id = ingestion_id
|
79
79
|
self.ingestion_type = ingestion_type
|
80
80
|
self.wait_for_completion = wait_for_completion
|
81
81
|
self.check_interval = check_interval
|
82
|
-
self.aws_conn_id = aws_conn_id
|
83
|
-
self.region = region
|
84
|
-
super().__init__(**kwargs)
|
85
82
|
|
86
83
|
def execute(self, context: Context):
|
87
|
-
hook = QuickSightHook(
|
88
|
-
aws_conn_id=self.aws_conn_id,
|
89
|
-
region_name=self.region,
|
90
|
-
)
|
91
84
|
self.log.info("Running the Amazon QuickSight SPICE Ingestion on Dataset ID: %s", self.data_set_id)
|
92
|
-
return hook.create_ingestion(
|
85
|
+
return self.hook.create_ingestion(
|
93
86
|
data_set_id=self.data_set_id,
|
94
87
|
ingestion_id=self.ingestion_id,
|
95
88
|
ingestion_type=self.ingestion_type,
|
@@ -17,11 +17,14 @@
|
|
17
17
|
# under the License.
|
18
18
|
from __future__ import annotations
|
19
19
|
|
20
|
-
from
|
21
|
-
from typing import TYPE_CHECKING
|
20
|
+
from typing import TYPE_CHECKING, Any
|
22
21
|
|
23
|
-
from airflow.
|
22
|
+
from airflow.configuration import conf
|
23
|
+
from airflow.exceptions import AirflowException
|
24
24
|
from airflow.providers.amazon.aws.hooks.redshift_data import RedshiftDataHook
|
25
|
+
from airflow.providers.amazon.aws.operators.base_aws import AwsBaseOperator
|
26
|
+
from airflow.providers.amazon.aws.triggers.redshift_data import RedshiftDataTrigger
|
27
|
+
from airflow.providers.amazon.aws.utils.mixins import aws_template_fields
|
25
28
|
|
26
29
|
if TYPE_CHECKING:
|
27
30
|
from mypy_boto3_redshift_data.type_defs import GetStatementResultResponseTypeDef
|
@@ -29,7 +32,7 @@ if TYPE_CHECKING:
|
|
29
32
|
from airflow.utils.context import Context
|
30
33
|
|
31
34
|
|
32
|
-
class RedshiftDataOperator(
|
35
|
+
class RedshiftDataOperator(AwsBaseOperator[RedshiftDataHook]):
|
33
36
|
"""
|
34
37
|
Executes SQL Statements against an Amazon Redshift cluster using Redshift Data.
|
35
38
|
|
@@ -49,22 +52,29 @@ class RedshiftDataOperator(BaseOperator):
|
|
49
52
|
:param poll_interval: how often in seconds to check the query status
|
50
53
|
:param return_sql_result: if True will return the result of an SQL statement,
|
51
54
|
if False (default) will return statement ID
|
52
|
-
:param aws_conn_id: aws connection to use
|
53
|
-
:param region: aws region to use
|
54
55
|
:param workgroup_name: name of the Redshift Serverless workgroup. Mutually exclusive with
|
55
56
|
`cluster_identifier`. Specify this parameter to query Redshift Serverless. More info
|
56
57
|
https://docs.aws.amazon.com/redshift/latest/mgmt/working-with-serverless.html
|
58
|
+
:param aws_conn_id: The Airflow connection used for AWS credentials.
|
59
|
+
If this is ``None`` or empty then the default boto3 behaviour is used. If
|
60
|
+
running Airflow in a distributed manner and aws_conn_id is None or
|
61
|
+
empty, then default boto3 configuration would be used (and must be
|
62
|
+
maintained on each worker node).
|
63
|
+
:param region_name: AWS region_name. If not specified then the default boto3 behaviour is used.
|
64
|
+
:param verify: Whether or not to verify SSL certificates. See:
|
65
|
+
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html
|
66
|
+
:param botocore_config: Configuration dictionary (key-values) for botocore client. See:
|
67
|
+
https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html
|
57
68
|
"""
|
58
69
|
|
59
|
-
|
70
|
+
aws_hook_class = RedshiftDataHook
|
71
|
+
template_fields = aws_template_fields(
|
60
72
|
"cluster_identifier",
|
61
73
|
"database",
|
62
74
|
"sql",
|
63
75
|
"db_user",
|
64
76
|
"parameters",
|
65
77
|
"statement_name",
|
66
|
-
"aws_conn_id",
|
67
|
-
"region",
|
68
78
|
"workgroup_name",
|
69
79
|
)
|
70
80
|
template_ext = (".sql",)
|
@@ -84,9 +94,8 @@ class RedshiftDataOperator(BaseOperator):
|
|
84
94
|
wait_for_completion: bool = True,
|
85
95
|
poll_interval: int = 10,
|
86
96
|
return_sql_result: bool = False,
|
87
|
-
aws_conn_id: str = "aws_default",
|
88
|
-
region: str | None = None,
|
89
97
|
workgroup_name: str | None = None,
|
98
|
+
deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
|
90
99
|
**kwargs,
|
91
100
|
) -> None:
|
92
101
|
super().__init__(**kwargs)
|
@@ -108,19 +117,18 @@ class RedshiftDataOperator(BaseOperator):
|
|
108
117
|
poll_interval,
|
109
118
|
)
|
110
119
|
self.return_sql_result = return_sql_result
|
111
|
-
self.aws_conn_id = aws_conn_id
|
112
|
-
self.region = region
|
113
120
|
self.statement_id: str | None = None
|
114
|
-
|
115
|
-
@cached_property
|
116
|
-
def hook(self) -> RedshiftDataHook:
|
117
|
-
"""Create and return an RedshiftDataHook."""
|
118
|
-
return RedshiftDataHook(aws_conn_id=self.aws_conn_id, region_name=self.region)
|
121
|
+
self.deferrable = deferrable
|
119
122
|
|
120
123
|
def execute(self, context: Context) -> GetStatementResultResponseTypeDef | str:
|
121
124
|
"""Execute a statement against Amazon Redshift."""
|
122
125
|
self.log.info("Executing statement: %s", self.sql)
|
123
126
|
|
127
|
+
# Set wait_for_completion to False so that it waits for the status in the deferred task.
|
128
|
+
wait_for_completion = self.wait_for_completion
|
129
|
+
if self.deferrable and self.wait_for_completion:
|
130
|
+
self.wait_for_completion = False
|
131
|
+
|
124
132
|
self.statement_id = self.hook.execute_query(
|
125
133
|
database=self.database,
|
126
134
|
sql=self.sql,
|
@@ -131,10 +139,27 @@ class RedshiftDataOperator(BaseOperator):
|
|
131
139
|
secret_arn=self.secret_arn,
|
132
140
|
statement_name=self.statement_name,
|
133
141
|
with_event=self.with_event,
|
134
|
-
wait_for_completion=
|
142
|
+
wait_for_completion=wait_for_completion,
|
135
143
|
poll_interval=self.poll_interval,
|
136
144
|
)
|
137
145
|
|
146
|
+
if self.deferrable:
|
147
|
+
is_finished = self.hook.check_query_is_finished(self.statement_id)
|
148
|
+
if not is_finished:
|
149
|
+
self.defer(
|
150
|
+
timeout=self.execution_timeout,
|
151
|
+
trigger=RedshiftDataTrigger(
|
152
|
+
statement_id=self.statement_id,
|
153
|
+
task_id=self.task_id,
|
154
|
+
poll_interval=self.poll_interval,
|
155
|
+
aws_conn_id=self.aws_conn_id,
|
156
|
+
region_name=self.region_name,
|
157
|
+
verify=self.verify,
|
158
|
+
botocore_config=self.botocore_config,
|
159
|
+
),
|
160
|
+
method_name="execute_complete",
|
161
|
+
)
|
162
|
+
|
138
163
|
if self.return_sql_result:
|
139
164
|
result = self.hook.conn.get_statement_result(Id=self.statement_id)
|
140
165
|
self.log.debug("Statement result: %s", result)
|
@@ -142,6 +167,30 @@ class RedshiftDataOperator(BaseOperator):
|
|
142
167
|
else:
|
143
168
|
return self.statement_id
|
144
169
|
|
170
|
+
def execute_complete(
|
171
|
+
self, context: Context, event: dict[str, Any] | None = None
|
172
|
+
) -> GetStatementResultResponseTypeDef | str:
|
173
|
+
if event is None:
|
174
|
+
err_msg = "Trigger error: event is None"
|
175
|
+
self.log.info(err_msg)
|
176
|
+
raise AirflowException(err_msg)
|
177
|
+
|
178
|
+
if event["status"] == "error":
|
179
|
+
msg = f"context: {context}, error message: {event['message']}"
|
180
|
+
raise AirflowException(msg)
|
181
|
+
|
182
|
+
statement_id = event["statement_id"]
|
183
|
+
if not statement_id:
|
184
|
+
raise AirflowException("statement_id should not be empty.")
|
185
|
+
|
186
|
+
self.log.info("%s completed successfully.", self.task_id)
|
187
|
+
if self.return_sql_result:
|
188
|
+
result = self.hook.conn.get_statement_result(Id=statement_id)
|
189
|
+
self.log.debug("Statement result: %s", result)
|
190
|
+
return result
|
191
|
+
|
192
|
+
return statement_id
|
193
|
+
|
145
194
|
def on_kill(self) -> None:
|
146
195
|
"""Cancel the submitted redshift query."""
|
147
196
|
if self.statement_id:
|
@@ -744,7 +744,6 @@ class S3ListOperator(BaseOperator):
|
|
744
744
|
:param delimiter: the delimiter marks key hierarchy. (templated)
|
745
745
|
:param aws_conn_id: The connection ID to use when connecting to S3 storage.
|
746
746
|
:param verify: Whether or not to verify SSL certificates for S3 connection.
|
747
|
-
:param apply_wildcard: whether to treat '*' as a wildcard or a plain symbol in the prefix.
|
748
747
|
By default SSL certificates are verified.
|
749
748
|
You can provide the following values:
|
750
749
|
|
@@ -754,6 +753,7 @@ class S3ListOperator(BaseOperator):
|
|
754
753
|
- ``path/to/cert/bundle.pem``: A filename of the CA cert bundle to uses.
|
755
754
|
You can specify this argument if you want to use a different
|
756
755
|
CA cert bundle than the one used by botocore.
|
756
|
+
:param apply_wildcard: whether to treat '*' as a wildcard or a plain symbol in the prefix.
|
757
757
|
|
758
758
|
|
759
759
|
**Example**:
|
@@ -283,8 +283,20 @@ class SageMakerProcessingOperator(SageMakerBaseOperator):
|
|
283
283
|
raise AirflowException(f"Sagemaker Processing Job creation failed: {response}")
|
284
284
|
|
285
285
|
if self.deferrable and self.wait_for_completion:
|
286
|
+
response = self.hook.describe_processing_job(self.config["ProcessingJobName"])
|
287
|
+
status = response["ProcessingJobStatus"]
|
288
|
+
if status in self.hook.failed_states:
|
289
|
+
raise AirflowException(f"SageMaker job failed because {response['FailureReason']}")
|
290
|
+
elif status == "Completed":
|
291
|
+
self.log.info("%s completed successfully.", self.task_id)
|
292
|
+
return {"Processing": serialize(response)}
|
293
|
+
|
294
|
+
timeout = self.execution_timeout
|
295
|
+
if self.max_ingestion_time:
|
296
|
+
timeout = datetime.timedelta(seconds=self.max_ingestion_time)
|
297
|
+
|
286
298
|
self.defer(
|
287
|
-
timeout=
|
299
|
+
timeout=timeout,
|
288
300
|
trigger=SageMakerTrigger(
|
289
301
|
job_name=self.config["ProcessingJobName"],
|
290
302
|
job_type="Processing",
|
@@ -304,6 +316,7 @@ class SageMakerProcessingOperator(SageMakerBaseOperator):
|
|
304
316
|
else:
|
305
317
|
self.log.info(event["message"])
|
306
318
|
self.serialized_job = serialize(self.hook.describe_processing_job(self.config["ProcessingJobName"]))
|
319
|
+
self.log.info("%s completed successfully.", self.task_id)
|
307
320
|
return {"Processing": self.serialized_job}
|
308
321
|
|
309
322
|
def get_openlineage_facets_on_complete(self, task_instance) -> OperatorLineage:
|
@@ -700,8 +713,24 @@ class SageMakerTransformOperator(SageMakerBaseOperator):
|
|
700
713
|
raise AirflowException(f"Sagemaker transform Job creation failed: {response}")
|
701
714
|
|
702
715
|
if self.deferrable and self.wait_for_completion:
|
716
|
+
response = self.hook.describe_transform_job(transform_config["TransformJobName"])
|
717
|
+
status = response["TransformJobStatus"]
|
718
|
+
if status in self.hook.failed_states:
|
719
|
+
raise AirflowException(f"SageMaker job failed because {response['FailureReason']}")
|
720
|
+
|
721
|
+
if status == "Completed":
|
722
|
+
self.log.info("%s completed successfully.", self.task_id)
|
723
|
+
return {
|
724
|
+
"Model": serialize(self.hook.describe_model(transform_config["ModelName"])),
|
725
|
+
"Transform": serialize(response),
|
726
|
+
}
|
727
|
+
|
728
|
+
timeout = self.execution_timeout
|
729
|
+
if self.max_ingestion_time:
|
730
|
+
timeout = datetime.timedelta(seconds=self.max_ingestion_time)
|
731
|
+
|
703
732
|
self.defer(
|
704
|
-
timeout=
|
733
|
+
timeout=timeout,
|
705
734
|
trigger=SageMakerTrigger(
|
706
735
|
job_name=transform_config["TransformJobName"],
|
707
736
|
job_type="Transform",
|
@@ -712,17 +741,18 @@ class SageMakerTransformOperator(SageMakerBaseOperator):
|
|
712
741
|
method_name="execute_complete",
|
713
742
|
)
|
714
743
|
|
715
|
-
|
716
|
-
self.serialized_transform = serialize(
|
717
|
-
self.hook.describe_transform_job(transform_config["TransformJobName"])
|
718
|
-
)
|
719
|
-
return {"Model": self.serialized_model, "Transform": self.serialized_transform}
|
744
|
+
return self.serialize_result()
|
720
745
|
|
721
|
-
def execute_complete(self, context, event=None):
|
722
|
-
if event
|
723
|
-
|
724
|
-
|
725
|
-
|
746
|
+
def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> dict[str, dict]:
|
747
|
+
if event is None:
|
748
|
+
err_msg = "Trigger error: event is None"
|
749
|
+
self.log.error(err_msg)
|
750
|
+
raise AirflowException(err_msg)
|
751
|
+
|
752
|
+
self.log.info(event["message"])
|
753
|
+
return self.serialize_result()
|
754
|
+
|
755
|
+
def serialize_result(self) -> dict[str, dict]:
|
726
756
|
transform_config = self.config.get("Transform", self.config)
|
727
757
|
self.serialized_model = serialize(self.hook.describe_model(transform_config["ModelName"]))
|
728
758
|
self.serialized_transform = serialize(
|
@@ -18,18 +18,19 @@
|
|
18
18
|
"""This module contains sensors for AWS CloudFormation."""
|
19
19
|
from __future__ import annotations
|
20
20
|
|
21
|
-
from functools import cached_property
|
22
21
|
from typing import TYPE_CHECKING, Sequence
|
23
22
|
|
23
|
+
from airflow.providers.amazon.aws.sensors.base_aws import AwsBaseSensor
|
24
|
+
from airflow.providers.amazon.aws.utils.mixins import aws_template_fields
|
25
|
+
|
24
26
|
if TYPE_CHECKING:
|
25
27
|
from airflow.utils.context import Context
|
26
28
|
|
27
29
|
from airflow.exceptions import AirflowSkipException
|
28
30
|
from airflow.providers.amazon.aws.hooks.cloud_formation import CloudFormationHook
|
29
|
-
from airflow.sensors.base import BaseSensorOperator
|
30
31
|
|
31
32
|
|
32
|
-
class CloudFormationCreateStackSensor(
|
33
|
+
class CloudFormationCreateStackSensor(AwsBaseSensor[CloudFormationHook]):
|
33
34
|
"""
|
34
35
|
Waits for a stack to be created successfully on AWS CloudFormation.
|
35
36
|
|
@@ -38,19 +39,25 @@ class CloudFormationCreateStackSensor(BaseSensorOperator):
|
|
38
39
|
:ref:`howto/sensor:CloudFormationCreateStackSensor`
|
39
40
|
|
40
41
|
:param stack_name: The name of the stack to wait for (templated)
|
41
|
-
:param aws_conn_id:
|
42
|
-
|
43
|
-
|
42
|
+
:param aws_conn_id: The Airflow connection used for AWS credentials.
|
43
|
+
If this is ``None`` or empty then the default boto3 behaviour is used. If
|
44
|
+
running Airflow in a distributed manner and aws_conn_id is None or
|
45
|
+
empty, then default boto3 configuration would be used (and must be
|
46
|
+
maintained on each worker node).
|
47
|
+
:param region_name: AWS region_name. If not specified then the default boto3 behaviour is used.
|
48
|
+
:param verify: Whether or not to verify SSL certificates. See:
|
49
|
+
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html
|
50
|
+
:param botocore_config: Configuration dictionary (key-values) for botocore client. See:
|
51
|
+
https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html
|
44
52
|
"""
|
45
53
|
|
46
|
-
|
54
|
+
aws_hook_class = CloudFormationHook
|
55
|
+
template_fields: Sequence[str] = aws_template_fields("stack_name")
|
47
56
|
ui_color = "#C5CAE9"
|
48
57
|
|
49
|
-
def __init__(self, *, stack_name,
|
58
|
+
def __init__(self, *, stack_name, **kwargs):
|
50
59
|
super().__init__(**kwargs)
|
51
60
|
self.stack_name = stack_name
|
52
|
-
self.aws_conn_id = aws_conn_id
|
53
|
-
self.region_name = region_name
|
54
61
|
|
55
62
|
def poke(self, context: Context):
|
56
63
|
stack_status = self.hook.get_stack_status(self.stack_name)
|
@@ -65,13 +72,8 @@ class CloudFormationCreateStackSensor(BaseSensorOperator):
|
|
65
72
|
raise AirflowSkipException(message)
|
66
73
|
raise ValueError(message)
|
67
74
|
|
68
|
-
@cached_property
|
69
|
-
def hook(self) -> CloudFormationHook:
|
70
|
-
"""Create and return a CloudFormationHook."""
|
71
|
-
return CloudFormationHook(aws_conn_id=self.aws_conn_id, region_name=self.region_name)
|
72
75
|
|
73
|
-
|
74
|
-
class CloudFormationDeleteStackSensor(BaseSensorOperator):
|
76
|
+
class CloudFormationDeleteStackSensor(AwsBaseSensor[CloudFormationHook]):
|
75
77
|
"""
|
76
78
|
Waits for a stack to be deleted successfully on AWS CloudFormation.
|
77
79
|
|
@@ -80,12 +82,20 @@ class CloudFormationDeleteStackSensor(BaseSensorOperator):
|
|
80
82
|
:ref:`howto/sensor:CloudFormationDeleteStackSensor`
|
81
83
|
|
82
84
|
:param stack_name: The name of the stack to wait for (templated)
|
83
|
-
:param aws_conn_id:
|
84
|
-
|
85
|
-
|
85
|
+
:param aws_conn_id: The Airflow connection used for AWS credentials.
|
86
|
+
If this is ``None`` or empty then the default boto3 behaviour is used. If
|
87
|
+
running Airflow in a distributed manner and aws_conn_id is None or
|
88
|
+
empty, then default boto3 configuration would be used (and must be
|
89
|
+
maintained on each worker node).
|
90
|
+
:param region_name: AWS region_name. If not specified then the default boto3 behaviour is used.
|
91
|
+
:param verify: Whether or not to verify SSL certificates. See:
|
92
|
+
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html
|
93
|
+
:param botocore_config: Configuration dictionary (key-values) for botocore client. See:
|
94
|
+
https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html
|
86
95
|
"""
|
87
96
|
|
88
|
-
|
97
|
+
aws_hook_class = CloudFormationHook
|
98
|
+
template_fields: Sequence[str] = aws_template_fields("stack_name")
|
89
99
|
ui_color = "#C5CAE9"
|
90
100
|
|
91
101
|
def __init__(
|
@@ -113,8 +123,3 @@ class CloudFormationDeleteStackSensor(BaseSensorOperator):
|
|
113
123
|
if self.soft_fail:
|
114
124
|
raise AirflowSkipException(message)
|
115
125
|
raise ValueError(message)
|
116
|
-
|
117
|
-
@cached_property
|
118
|
-
def hook(self) -> CloudFormationHook:
|
119
|
-
"""Create and return a CloudFormationHook."""
|
120
|
-
return CloudFormationHook(aws_conn_id=self.aws_conn_id, region_name=self.region_name)
|
@@ -17,47 +17,53 @@
|
|
17
17
|
# under the License.
|
18
18
|
from __future__ import annotations
|
19
19
|
|
20
|
-
from functools import cached_property
|
21
20
|
from typing import TYPE_CHECKING, Iterable, Sequence
|
22
21
|
|
23
22
|
from deprecated import deprecated
|
24
23
|
|
25
24
|
from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning, AirflowSkipException
|
26
25
|
from airflow.providers.amazon.aws.hooks.dms import DmsHook
|
27
|
-
from airflow.sensors.
|
26
|
+
from airflow.providers.amazon.aws.sensors.base_aws import AwsBaseSensor
|
27
|
+
from airflow.providers.amazon.aws.utils.mixins import aws_template_fields
|
28
28
|
|
29
29
|
if TYPE_CHECKING:
|
30
30
|
from airflow.utils.context import Context
|
31
31
|
|
32
32
|
|
33
|
-
class DmsTaskBaseSensor(
|
33
|
+
class DmsTaskBaseSensor(AwsBaseSensor[DmsHook]):
|
34
34
|
"""
|
35
35
|
Contains general sensor behavior for DMS task.
|
36
36
|
|
37
37
|
Subclasses should set ``target_statuses`` and ``termination_statuses`` fields.
|
38
38
|
|
39
39
|
:param replication_task_arn: AWS DMS replication task ARN
|
40
|
-
:param aws_conn_id: aws connection to uses
|
41
40
|
:param target_statuses: the target statuses, sensor waits until
|
42
41
|
the task reaches any of these states
|
43
42
|
:param termination_statuses: the termination statuses, sensor fails when
|
44
43
|
the task reaches any of these states
|
44
|
+
:param aws_conn_id: The Airflow connection used for AWS credentials.
|
45
|
+
If this is ``None`` or empty then the default boto3 behaviour is used. If
|
46
|
+
running Airflow in a distributed manner and aws_conn_id is None or
|
47
|
+
empty, then default boto3 configuration would be used (and must be
|
48
|
+
maintained on each worker node).
|
49
|
+
:param region_name: AWS region_name. If not specified then the default boto3 behaviour is used.
|
50
|
+
:param verify: Whether or not to verify SSL certificates. See:
|
51
|
+
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html
|
52
|
+
:param botocore_config: Configuration dictionary (key-values) for botocore client. See:
|
53
|
+
https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html
|
45
54
|
"""
|
46
55
|
|
47
|
-
|
48
|
-
|
56
|
+
aws_hook_class = DmsHook
|
57
|
+
template_fields: Sequence[str] = aws_template_fields("replication_task_arn")
|
49
58
|
|
50
59
|
def __init__(
|
51
60
|
self,
|
52
61
|
replication_task_arn: str,
|
53
|
-
aws_conn_id="aws_default",
|
54
62
|
target_statuses: Iterable[str] | None = None,
|
55
63
|
termination_statuses: Iterable[str] | None = None,
|
56
|
-
*args,
|
57
64
|
**kwargs,
|
58
65
|
):
|
59
|
-
super().__init__(
|
60
|
-
self.aws_conn_id = aws_conn_id
|
66
|
+
super().__init__(**kwargs)
|
61
67
|
self.replication_task_arn = replication_task_arn
|
62
68
|
self.target_statuses: Iterable[str] = target_statuses or []
|
63
69
|
self.termination_statuses: Iterable[str] = termination_statuses or []
|
@@ -67,14 +73,8 @@ class DmsTaskBaseSensor(BaseSensorOperator):
|
|
67
73
|
"""Get DmsHook."""
|
68
74
|
return self.hook
|
69
75
|
|
70
|
-
@cached_property
|
71
|
-
def hook(self) -> DmsHook:
|
72
|
-
return DmsHook(self.aws_conn_id)
|
73
|
-
|
74
76
|
def poke(self, context: Context):
|
75
|
-
|
76
|
-
|
77
|
-
if not status:
|
77
|
+
if not (status := self.hook.get_task_status(self.replication_task_arn)):
|
78
78
|
# TODO: remove this if check when min_airflow_version is set to higher than 2.7.1
|
79
79
|
message = f"Failed to read task status, task with ARN {self.replication_task_arn} not found"
|
80
80
|
if self.soft_fail:
|
@@ -105,15 +105,21 @@ class DmsTaskCompletedSensor(DmsTaskBaseSensor):
|
|
105
105
|
:ref:`howto/sensor:DmsTaskCompletedSensor`
|
106
106
|
|
107
107
|
:param replication_task_arn: AWS DMS replication task ARN
|
108
|
+
:param aws_conn_id: The Airflow connection used for AWS credentials.
|
109
|
+
If this is ``None`` or empty then the default boto3 behaviour is used. If
|
110
|
+
running Airflow in a distributed manner and aws_conn_id is None or
|
111
|
+
empty, then default boto3 configuration would be used (and must be
|
112
|
+
maintained on each worker node).
|
113
|
+
:param region_name: AWS region_name. If not specified then the default boto3 behaviour is used.
|
114
|
+
:param verify: Whether or not to verify SSL certificates. See:
|
115
|
+
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html
|
116
|
+
:param botocore_config: Configuration dictionary (key-values) for botocore client. See:
|
117
|
+
https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html
|
108
118
|
"""
|
109
119
|
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
def __init__(self, *args, **kwargs):
|
114
|
-
super().__init__(*args, **kwargs)
|
115
|
-
self.target_statuses = ["stopped"]
|
116
|
-
self.termination_statuses = [
|
120
|
+
def __init__(self, **kwargs):
|
121
|
+
kwargs["target_statuses"] = ["stopped"]
|
122
|
+
kwargs["termination_statuses"] = [
|
117
123
|
"creating",
|
118
124
|
"deleting",
|
119
125
|
"failed",
|
@@ -123,3 +129,4 @@ class DmsTaskCompletedSensor(DmsTaskBaseSensor):
|
|
123
129
|
"ready",
|
124
130
|
"testing",
|
125
131
|
]
|
132
|
+
super().__init__(**kwargs)
|
@@ -16,17 +16,17 @@
|
|
16
16
|
# under the License.
|
17
17
|
from __future__ import annotations
|
18
18
|
|
19
|
-
from functools import cached_property
|
20
19
|
from typing import TYPE_CHECKING, Any, Iterable, Sequence
|
21
20
|
|
22
21
|
from airflow.providers.amazon.aws.hooks.dynamodb import DynamoDBHook
|
23
|
-
from airflow.sensors.
|
22
|
+
from airflow.providers.amazon.aws.sensors.base_aws import AwsBaseSensor
|
23
|
+
from airflow.providers.amazon.aws.utils.mixins import aws_template_fields
|
24
24
|
|
25
25
|
if TYPE_CHECKING:
|
26
26
|
from airflow.utils.context import Context
|
27
27
|
|
28
28
|
|
29
|
-
class DynamoDBValueSensor(
|
29
|
+
class DynamoDBValueSensor(AwsBaseSensor[DynamoDBHook]):
|
30
30
|
"""
|
31
31
|
Waits for an attribute value to be present for an item in a DynamoDB table.
|
32
32
|
|
@@ -41,11 +41,20 @@ class DynamoDBValueSensor(BaseSensorOperator):
|
|
41
41
|
:param attribute_value: DynamoDB attribute value
|
42
42
|
:param sort_key_name: (optional) DynamoDB sort key name
|
43
43
|
:param sort_key_value: (optional) DynamoDB sort key value
|
44
|
-
:param aws_conn_id:
|
45
|
-
|
44
|
+
:param aws_conn_id: The Airflow connection used for AWS credentials.
|
45
|
+
If this is ``None`` or empty then the default boto3 behaviour is used. If
|
46
|
+
running Airflow in a distributed manner and aws_conn_id is None or
|
47
|
+
empty, then default boto3 configuration would be used (and must be
|
48
|
+
maintained on each worker node).
|
49
|
+
:param region_name: AWS region_name. If not specified then the default boto3 behaviour is used.
|
50
|
+
:param verify: Whether or not to verify SSL certificates. See:
|
51
|
+
https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html
|
52
|
+
:param botocore_config: Configuration dictionary (key-values) for botocore client. See:
|
53
|
+
https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html
|
46
54
|
"""
|
47
55
|
|
48
|
-
|
56
|
+
aws_hook_class = DynamoDBHook
|
57
|
+
template_fields: Sequence[str] = aws_template_fields(
|
49
58
|
"table_name",
|
50
59
|
"partition_key_name",
|
51
60
|
"partition_key_value",
|
@@ -64,8 +73,6 @@ class DynamoDBValueSensor(BaseSensorOperator):
|
|
64
73
|
attribute_value: str | Iterable[str],
|
65
74
|
sort_key_name: str | None = None,
|
66
75
|
sort_key_value: str | None = None,
|
67
|
-
aws_conn_id: str | None = DynamoDBHook.default_conn_name,
|
68
|
-
region_name: str | None = None,
|
69
76
|
**kwargs: Any,
|
70
77
|
):
|
71
78
|
super().__init__(**kwargs)
|
@@ -76,8 +83,6 @@ class DynamoDBValueSensor(BaseSensorOperator):
|
|
76
83
|
self.attribute_value = attribute_value
|
77
84
|
self.sort_key_name = sort_key_name
|
78
85
|
self.sort_key_value = sort_key_value
|
79
|
-
self.aws_conn_id = aws_conn_id
|
80
|
-
self.region_name = region_name
|
81
86
|
|
82
87
|
def poke(self, context: Context) -> bool:
|
83
88
|
"""Test DynamoDB item for matching attribute value."""
|
@@ -108,8 +113,3 @@ class DynamoDBValueSensor(BaseSensorOperator):
|
|
108
113
|
)
|
109
114
|
except KeyError:
|
110
115
|
return False
|
111
|
-
|
112
|
-
@cached_property
|
113
|
-
def hook(self) -> DynamoDBHook:
|
114
|
-
"""Create and return a DynamoDBHook."""
|
115
|
-
return DynamoDBHook(self.aws_conn_id, region_name=self.region_name)
|