apache-airflow-providers-amazon 8.16.0__py3-none-any.whl → 8.17.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. airflow/providers/amazon/__init__.py +1 -1
  2. airflow/providers/amazon/aws/auth_manager/avp/entities.py +1 -0
  3. airflow/providers/amazon/aws/auth_manager/avp/facade.py +34 -19
  4. airflow/providers/amazon/aws/auth_manager/aws_auth_manager.py +44 -1
  5. airflow/providers/amazon/aws/auth_manager/cli/__init__.py +16 -0
  6. airflow/providers/amazon/aws/auth_manager/cli/avp_commands.py +178 -0
  7. airflow/providers/amazon/aws/auth_manager/cli/definition.py +62 -0
  8. airflow/providers/amazon/aws/auth_manager/cli/schema.json +171 -0
  9. airflow/providers/amazon/aws/auth_manager/constants.py +1 -0
  10. airflow/providers/amazon/aws/executors/ecs/ecs_executor.py +77 -23
  11. airflow/providers/amazon/aws/executors/ecs/ecs_executor_config.py +17 -0
  12. airflow/providers/amazon/aws/executors/ecs/utils.py +1 -1
  13. airflow/providers/amazon/aws/executors/utils/__init__.py +16 -0
  14. airflow/providers/amazon/aws/executors/utils/exponential_backoff_retry.py +60 -0
  15. airflow/providers/amazon/aws/hooks/athena_sql.py +168 -0
  16. airflow/providers/amazon/aws/hooks/base_aws.py +14 -0
  17. airflow/providers/amazon/aws/hooks/quicksight.py +33 -18
  18. airflow/providers/amazon/aws/hooks/redshift_data.py +66 -17
  19. airflow/providers/amazon/aws/hooks/redshift_sql.py +1 -1
  20. airflow/providers/amazon/aws/hooks/s3.py +18 -4
  21. airflow/providers/amazon/aws/log/cloudwatch_task_handler.py +2 -2
  22. airflow/providers/amazon/aws/operators/batch.py +33 -15
  23. airflow/providers/amazon/aws/operators/cloud_formation.py +37 -26
  24. airflow/providers/amazon/aws/operators/datasync.py +19 -18
  25. airflow/providers/amazon/aws/operators/dms.py +57 -69
  26. airflow/providers/amazon/aws/operators/ec2.py +19 -5
  27. airflow/providers/amazon/aws/operators/emr.py +30 -10
  28. airflow/providers/amazon/aws/operators/eventbridge.py +57 -80
  29. airflow/providers/amazon/aws/operators/quicksight.py +17 -24
  30. airflow/providers/amazon/aws/operators/redshift_data.py +68 -19
  31. airflow/providers/amazon/aws/operators/s3.py +1 -1
  32. airflow/providers/amazon/aws/operators/sagemaker.py +42 -12
  33. airflow/providers/amazon/aws/sensors/cloud_formation.py +30 -25
  34. airflow/providers/amazon/aws/sensors/dms.py +31 -24
  35. airflow/providers/amazon/aws/sensors/dynamodb.py +15 -15
  36. airflow/providers/amazon/aws/sensors/quicksight.py +34 -24
  37. airflow/providers/amazon/aws/sensors/redshift_cluster.py +41 -3
  38. airflow/providers/amazon/aws/sensors/s3.py +13 -8
  39. airflow/providers/amazon/aws/triggers/redshift_cluster.py +54 -2
  40. airflow/providers/amazon/aws/triggers/redshift_data.py +113 -0
  41. airflow/providers/amazon/aws/triggers/s3.py +9 -4
  42. airflow/providers/amazon/get_provider_info.py +55 -16
  43. {apache_airflow_providers_amazon-8.16.0.dist-info → apache_airflow_providers_amazon-8.17.0.dist-info}/METADATA +15 -13
  44. {apache_airflow_providers_amazon-8.16.0.dist-info → apache_airflow_providers_amazon-8.17.0.dist-info}/RECORD +46 -38
  45. {apache_airflow_providers_amazon-8.16.0.dist-info → apache_airflow_providers_amazon-8.17.0.dist-info}/WHEEL +0 -0
  46. {apache_airflow_providers_amazon-8.16.0.dist-info → apache_airflow_providers_amazon-8.17.0.dist-info}/entry_points.txt +0 -0
@@ -18,16 +18,15 @@ from __future__ import annotations
18
18
 
19
19
  from typing import TYPE_CHECKING, Sequence
20
20
 
21
- from airflow.models import BaseOperator
22
21
  from airflow.providers.amazon.aws.hooks.quicksight import QuickSightHook
22
+ from airflow.providers.amazon.aws.operators.base_aws import AwsBaseOperator
23
+ from airflow.providers.amazon.aws.utils.mixins import aws_template_fields
23
24
 
24
25
  if TYPE_CHECKING:
25
26
  from airflow.utils.context import Context
26
27
 
27
- DEFAULT_CONN_ID = "aws_default"
28
28
 
29
-
30
- class QuickSightCreateIngestionOperator(BaseOperator):
29
+ class QuickSightCreateIngestionOperator(AwsBaseOperator[QuickSightHook]):
31
30
  """
32
31
  Creates and starts a new SPICE ingestion for a dataset; also helps to Refresh existing SPICE datasets.
33
32
 
@@ -43,23 +42,25 @@ class QuickSightCreateIngestionOperator(BaseOperator):
43
42
  that the operation waits to check the status of the Amazon QuickSight Ingestion.
44
43
  :param check_interval: if wait is set to be true, this is the time interval
45
44
  in seconds which the operator will check the status of the Amazon QuickSight Ingestion
46
- :param aws_conn_id: The Airflow connection used for AWS credentials. (templated)
47
- If this is None or empty then the default boto3 behaviour is used. If
48
- running Airflow in a distributed manner and aws_conn_id is None or
49
- empty, then the default boto3 configuration would be used (and must be
50
- maintained on each worker node).
51
- :param region: Which AWS region the connection should use. (templated)
52
- If this is None or empty then the default boto3 behaviour is used.
45
+ :param aws_conn_id: The Airflow connection used for AWS credentials.
46
+ If this is ``None`` or empty then the default boto3 behaviour is used. If
47
+ running Airflow in a distributed manner and aws_conn_id is None or
48
+ empty, then default boto3 configuration would be used (and must be
49
+ maintained on each worker node).
50
+ :param region_name: AWS region_name. If not specified then the default boto3 behaviour is used.
51
+ :param verify: Whether or not to verify SSL certificates. See:
52
+ https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html
53
+ :param botocore_config: Configuration dictionary (key-values) for botocore client. See:
54
+ https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html
53
55
  """
54
56
 
55
- template_fields: Sequence[str] = (
57
+ aws_hook_class = QuickSightHook
58
+ template_fields: Sequence[str] = aws_template_fields(
56
59
  "data_set_id",
57
60
  "ingestion_id",
58
61
  "ingestion_type",
59
62
  "wait_for_completion",
60
63
  "check_interval",
61
- "aws_conn_id",
62
- "region",
63
64
  )
64
65
  ui_color = "#ffd700"
65
66
 
@@ -70,26 +71,18 @@ class QuickSightCreateIngestionOperator(BaseOperator):
70
71
  ingestion_type: str = "FULL_REFRESH",
71
72
  wait_for_completion: bool = True,
72
73
  check_interval: int = 30,
73
- aws_conn_id: str = DEFAULT_CONN_ID,
74
- region: str | None = None,
75
74
  **kwargs,
76
75
  ):
76
+ super().__init__(**kwargs)
77
77
  self.data_set_id = data_set_id
78
78
  self.ingestion_id = ingestion_id
79
79
  self.ingestion_type = ingestion_type
80
80
  self.wait_for_completion = wait_for_completion
81
81
  self.check_interval = check_interval
82
- self.aws_conn_id = aws_conn_id
83
- self.region = region
84
- super().__init__(**kwargs)
85
82
 
86
83
  def execute(self, context: Context):
87
- hook = QuickSightHook(
88
- aws_conn_id=self.aws_conn_id,
89
- region_name=self.region,
90
- )
91
84
  self.log.info("Running the Amazon QuickSight SPICE Ingestion on Dataset ID: %s", self.data_set_id)
92
- return hook.create_ingestion(
85
+ return self.hook.create_ingestion(
93
86
  data_set_id=self.data_set_id,
94
87
  ingestion_id=self.ingestion_id,
95
88
  ingestion_type=self.ingestion_type,
@@ -17,11 +17,14 @@
17
17
  # under the License.
18
18
  from __future__ import annotations
19
19
 
20
- from functools import cached_property
21
- from typing import TYPE_CHECKING
20
+ from typing import TYPE_CHECKING, Any
22
21
 
23
- from airflow.models import BaseOperator
22
+ from airflow.configuration import conf
23
+ from airflow.exceptions import AirflowException
24
24
  from airflow.providers.amazon.aws.hooks.redshift_data import RedshiftDataHook
25
+ from airflow.providers.amazon.aws.operators.base_aws import AwsBaseOperator
26
+ from airflow.providers.amazon.aws.triggers.redshift_data import RedshiftDataTrigger
27
+ from airflow.providers.amazon.aws.utils.mixins import aws_template_fields
25
28
 
26
29
  if TYPE_CHECKING:
27
30
  from mypy_boto3_redshift_data.type_defs import GetStatementResultResponseTypeDef
@@ -29,7 +32,7 @@ if TYPE_CHECKING:
29
32
  from airflow.utils.context import Context
30
33
 
31
34
 
32
- class RedshiftDataOperator(BaseOperator):
35
+ class RedshiftDataOperator(AwsBaseOperator[RedshiftDataHook]):
33
36
  """
34
37
  Executes SQL Statements against an Amazon Redshift cluster using Redshift Data.
35
38
 
@@ -49,22 +52,29 @@ class RedshiftDataOperator(BaseOperator):
49
52
  :param poll_interval: how often in seconds to check the query status
50
53
  :param return_sql_result: if True will return the result of an SQL statement,
51
54
  if False (default) will return statement ID
52
- :param aws_conn_id: aws connection to use
53
- :param region: aws region to use
54
55
  :param workgroup_name: name of the Redshift Serverless workgroup. Mutually exclusive with
55
56
  `cluster_identifier`. Specify this parameter to query Redshift Serverless. More info
56
57
  https://docs.aws.amazon.com/redshift/latest/mgmt/working-with-serverless.html
58
+ :param aws_conn_id: The Airflow connection used for AWS credentials.
59
+ If this is ``None`` or empty then the default boto3 behaviour is used. If
60
+ running Airflow in a distributed manner and aws_conn_id is None or
61
+ empty, then default boto3 configuration would be used (and must be
62
+ maintained on each worker node).
63
+ :param region_name: AWS region_name. If not specified then the default boto3 behaviour is used.
64
+ :param verify: Whether or not to verify SSL certificates. See:
65
+ https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html
66
+ :param botocore_config: Configuration dictionary (key-values) for botocore client. See:
67
+ https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html
57
68
  """
58
69
 
59
- template_fields = (
70
+ aws_hook_class = RedshiftDataHook
71
+ template_fields = aws_template_fields(
60
72
  "cluster_identifier",
61
73
  "database",
62
74
  "sql",
63
75
  "db_user",
64
76
  "parameters",
65
77
  "statement_name",
66
- "aws_conn_id",
67
- "region",
68
78
  "workgroup_name",
69
79
  )
70
80
  template_ext = (".sql",)
@@ -84,9 +94,8 @@ class RedshiftDataOperator(BaseOperator):
84
94
  wait_for_completion: bool = True,
85
95
  poll_interval: int = 10,
86
96
  return_sql_result: bool = False,
87
- aws_conn_id: str = "aws_default",
88
- region: str | None = None,
89
97
  workgroup_name: str | None = None,
98
+ deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
90
99
  **kwargs,
91
100
  ) -> None:
92
101
  super().__init__(**kwargs)
@@ -108,19 +117,18 @@ class RedshiftDataOperator(BaseOperator):
108
117
  poll_interval,
109
118
  )
110
119
  self.return_sql_result = return_sql_result
111
- self.aws_conn_id = aws_conn_id
112
- self.region = region
113
120
  self.statement_id: str | None = None
114
-
115
- @cached_property
116
- def hook(self) -> RedshiftDataHook:
117
- """Create and return an RedshiftDataHook."""
118
- return RedshiftDataHook(aws_conn_id=self.aws_conn_id, region_name=self.region)
121
+ self.deferrable = deferrable
119
122
 
120
123
  def execute(self, context: Context) -> GetStatementResultResponseTypeDef | str:
121
124
  """Execute a statement against Amazon Redshift."""
122
125
  self.log.info("Executing statement: %s", self.sql)
123
126
 
127
+ # Set wait_for_completion to False so that it waits for the status in the deferred task.
128
+ wait_for_completion = self.wait_for_completion
129
+ if self.deferrable and self.wait_for_completion:
130
+ self.wait_for_completion = False
131
+
124
132
  self.statement_id = self.hook.execute_query(
125
133
  database=self.database,
126
134
  sql=self.sql,
@@ -131,10 +139,27 @@ class RedshiftDataOperator(BaseOperator):
131
139
  secret_arn=self.secret_arn,
132
140
  statement_name=self.statement_name,
133
141
  with_event=self.with_event,
134
- wait_for_completion=self.wait_for_completion,
142
+ wait_for_completion=wait_for_completion,
135
143
  poll_interval=self.poll_interval,
136
144
  )
137
145
 
146
+ if self.deferrable:
147
+ is_finished = self.hook.check_query_is_finished(self.statement_id)
148
+ if not is_finished:
149
+ self.defer(
150
+ timeout=self.execution_timeout,
151
+ trigger=RedshiftDataTrigger(
152
+ statement_id=self.statement_id,
153
+ task_id=self.task_id,
154
+ poll_interval=self.poll_interval,
155
+ aws_conn_id=self.aws_conn_id,
156
+ region_name=self.region_name,
157
+ verify=self.verify,
158
+ botocore_config=self.botocore_config,
159
+ ),
160
+ method_name="execute_complete",
161
+ )
162
+
138
163
  if self.return_sql_result:
139
164
  result = self.hook.conn.get_statement_result(Id=self.statement_id)
140
165
  self.log.debug("Statement result: %s", result)
@@ -142,6 +167,30 @@ class RedshiftDataOperator(BaseOperator):
142
167
  else:
143
168
  return self.statement_id
144
169
 
170
+ def execute_complete(
171
+ self, context: Context, event: dict[str, Any] | None = None
172
+ ) -> GetStatementResultResponseTypeDef | str:
173
+ if event is None:
174
+ err_msg = "Trigger error: event is None"
175
+ self.log.info(err_msg)
176
+ raise AirflowException(err_msg)
177
+
178
+ if event["status"] == "error":
179
+ msg = f"context: {context}, error message: {event['message']}"
180
+ raise AirflowException(msg)
181
+
182
+ statement_id = event["statement_id"]
183
+ if not statement_id:
184
+ raise AirflowException("statement_id should not be empty.")
185
+
186
+ self.log.info("%s completed successfully.", self.task_id)
187
+ if self.return_sql_result:
188
+ result = self.hook.conn.get_statement_result(Id=statement_id)
189
+ self.log.debug("Statement result: %s", result)
190
+ return result
191
+
192
+ return statement_id
193
+
145
194
  def on_kill(self) -> None:
146
195
  """Cancel the submitted redshift query."""
147
196
  if self.statement_id:
@@ -744,7 +744,6 @@ class S3ListOperator(BaseOperator):
744
744
  :param delimiter: the delimiter marks key hierarchy. (templated)
745
745
  :param aws_conn_id: The connection ID to use when connecting to S3 storage.
746
746
  :param verify: Whether or not to verify SSL certificates for S3 connection.
747
- :param apply_wildcard: whether to treat '*' as a wildcard or a plain symbol in the prefix.
748
747
  By default SSL certificates are verified.
749
748
  You can provide the following values:
750
749
 
@@ -754,6 +753,7 @@ class S3ListOperator(BaseOperator):
754
753
  - ``path/to/cert/bundle.pem``: A filename of the CA cert bundle to uses.
755
754
  You can specify this argument if you want to use a different
756
755
  CA cert bundle than the one used by botocore.
756
+ :param apply_wildcard: whether to treat '*' as a wildcard or a plain symbol in the prefix.
757
757
 
758
758
 
759
759
  **Example**:
@@ -283,8 +283,20 @@ class SageMakerProcessingOperator(SageMakerBaseOperator):
283
283
  raise AirflowException(f"Sagemaker Processing Job creation failed: {response}")
284
284
 
285
285
  if self.deferrable and self.wait_for_completion:
286
+ response = self.hook.describe_processing_job(self.config["ProcessingJobName"])
287
+ status = response["ProcessingJobStatus"]
288
+ if status in self.hook.failed_states:
289
+ raise AirflowException(f"SageMaker job failed because {response['FailureReason']}")
290
+ elif status == "Completed":
291
+ self.log.info("%s completed successfully.", self.task_id)
292
+ return {"Processing": serialize(response)}
293
+
294
+ timeout = self.execution_timeout
295
+ if self.max_ingestion_time:
296
+ timeout = datetime.timedelta(seconds=self.max_ingestion_time)
297
+
286
298
  self.defer(
287
- timeout=self.execution_timeout,
299
+ timeout=timeout,
288
300
  trigger=SageMakerTrigger(
289
301
  job_name=self.config["ProcessingJobName"],
290
302
  job_type="Processing",
@@ -304,6 +316,7 @@ class SageMakerProcessingOperator(SageMakerBaseOperator):
304
316
  else:
305
317
  self.log.info(event["message"])
306
318
  self.serialized_job = serialize(self.hook.describe_processing_job(self.config["ProcessingJobName"]))
319
+ self.log.info("%s completed successfully.", self.task_id)
307
320
  return {"Processing": self.serialized_job}
308
321
 
309
322
  def get_openlineage_facets_on_complete(self, task_instance) -> OperatorLineage:
@@ -700,8 +713,24 @@ class SageMakerTransformOperator(SageMakerBaseOperator):
700
713
  raise AirflowException(f"Sagemaker transform Job creation failed: {response}")
701
714
 
702
715
  if self.deferrable and self.wait_for_completion:
716
+ response = self.hook.describe_transform_job(transform_config["TransformJobName"])
717
+ status = response["TransformJobStatus"]
718
+ if status in self.hook.failed_states:
719
+ raise AirflowException(f"SageMaker job failed because {response['FailureReason']}")
720
+
721
+ if status == "Completed":
722
+ self.log.info("%s completed successfully.", self.task_id)
723
+ return {
724
+ "Model": serialize(self.hook.describe_model(transform_config["ModelName"])),
725
+ "Transform": serialize(response),
726
+ }
727
+
728
+ timeout = self.execution_timeout
729
+ if self.max_ingestion_time:
730
+ timeout = datetime.timedelta(seconds=self.max_ingestion_time)
731
+
703
732
  self.defer(
704
- timeout=self.execution_timeout,
733
+ timeout=timeout,
705
734
  trigger=SageMakerTrigger(
706
735
  job_name=transform_config["TransformJobName"],
707
736
  job_type="Transform",
@@ -712,17 +741,18 @@ class SageMakerTransformOperator(SageMakerBaseOperator):
712
741
  method_name="execute_complete",
713
742
  )
714
743
 
715
- self.serialized_model = serialize(self.hook.describe_model(transform_config["ModelName"]))
716
- self.serialized_transform = serialize(
717
- self.hook.describe_transform_job(transform_config["TransformJobName"])
718
- )
719
- return {"Model": self.serialized_model, "Transform": self.serialized_transform}
744
+ return self.serialize_result()
720
745
 
721
- def execute_complete(self, context, event=None):
722
- if event["status"] != "success":
723
- raise AirflowException(f"Error while running job: {event}")
724
- else:
725
- self.log.info(event["message"])
746
+ def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> dict[str, dict]:
747
+ if event is None:
748
+ err_msg = "Trigger error: event is None"
749
+ self.log.error(err_msg)
750
+ raise AirflowException(err_msg)
751
+
752
+ self.log.info(event["message"])
753
+ return self.serialize_result()
754
+
755
+ def serialize_result(self) -> dict[str, dict]:
726
756
  transform_config = self.config.get("Transform", self.config)
727
757
  self.serialized_model = serialize(self.hook.describe_model(transform_config["ModelName"]))
728
758
  self.serialized_transform = serialize(
@@ -18,18 +18,19 @@
18
18
  """This module contains sensors for AWS CloudFormation."""
19
19
  from __future__ import annotations
20
20
 
21
- from functools import cached_property
22
21
  from typing import TYPE_CHECKING, Sequence
23
22
 
23
+ from airflow.providers.amazon.aws.sensors.base_aws import AwsBaseSensor
24
+ from airflow.providers.amazon.aws.utils.mixins import aws_template_fields
25
+
24
26
  if TYPE_CHECKING:
25
27
  from airflow.utils.context import Context
26
28
 
27
29
  from airflow.exceptions import AirflowSkipException
28
30
  from airflow.providers.amazon.aws.hooks.cloud_formation import CloudFormationHook
29
- from airflow.sensors.base import BaseSensorOperator
30
31
 
31
32
 
32
- class CloudFormationCreateStackSensor(BaseSensorOperator):
33
+ class CloudFormationCreateStackSensor(AwsBaseSensor[CloudFormationHook]):
33
34
  """
34
35
  Waits for a stack to be created successfully on AWS CloudFormation.
35
36
 
@@ -38,19 +39,25 @@ class CloudFormationCreateStackSensor(BaseSensorOperator):
38
39
  :ref:`howto/sensor:CloudFormationCreateStackSensor`
39
40
 
40
41
  :param stack_name: The name of the stack to wait for (templated)
41
- :param aws_conn_id: ID of the Airflow connection where credentials and extra configuration are
42
- stored
43
- :param poke_interval: Time in seconds that the job should wait between each try
42
+ :param aws_conn_id: The Airflow connection used for AWS credentials.
43
+ If this is ``None`` or empty then the default boto3 behaviour is used. If
44
+ running Airflow in a distributed manner and aws_conn_id is None or
45
+ empty, then default boto3 configuration would be used (and must be
46
+ maintained on each worker node).
47
+ :param region_name: AWS region_name. If not specified then the default boto3 behaviour is used.
48
+ :param verify: Whether or not to verify SSL certificates. See:
49
+ https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html
50
+ :param botocore_config: Configuration dictionary (key-values) for botocore client. See:
51
+ https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html
44
52
  """
45
53
 
46
- template_fields: Sequence[str] = ("stack_name",)
54
+ aws_hook_class = CloudFormationHook
55
+ template_fields: Sequence[str] = aws_template_fields("stack_name")
47
56
  ui_color = "#C5CAE9"
48
57
 
49
- def __init__(self, *, stack_name, aws_conn_id="aws_default", region_name=None, **kwargs):
58
+ def __init__(self, *, stack_name, **kwargs):
50
59
  super().__init__(**kwargs)
51
60
  self.stack_name = stack_name
52
- self.aws_conn_id = aws_conn_id
53
- self.region_name = region_name
54
61
 
55
62
  def poke(self, context: Context):
56
63
  stack_status = self.hook.get_stack_status(self.stack_name)
@@ -65,13 +72,8 @@ class CloudFormationCreateStackSensor(BaseSensorOperator):
65
72
  raise AirflowSkipException(message)
66
73
  raise ValueError(message)
67
74
 
68
- @cached_property
69
- def hook(self) -> CloudFormationHook:
70
- """Create and return a CloudFormationHook."""
71
- return CloudFormationHook(aws_conn_id=self.aws_conn_id, region_name=self.region_name)
72
75
 
73
-
74
- class CloudFormationDeleteStackSensor(BaseSensorOperator):
76
+ class CloudFormationDeleteStackSensor(AwsBaseSensor[CloudFormationHook]):
75
77
  """
76
78
  Waits for a stack to be deleted successfully on AWS CloudFormation.
77
79
 
@@ -80,12 +82,20 @@ class CloudFormationDeleteStackSensor(BaseSensorOperator):
80
82
  :ref:`howto/sensor:CloudFormationDeleteStackSensor`
81
83
 
82
84
  :param stack_name: The name of the stack to wait for (templated)
83
- :param aws_conn_id: ID of the Airflow connection where credentials and extra configuration are
84
- stored
85
- :param poke_interval: Time in seconds that the job should wait between each try
85
+ :param aws_conn_id: The Airflow connection used for AWS credentials.
86
+ If this is ``None`` or empty then the default boto3 behaviour is used. If
87
+ running Airflow in a distributed manner and aws_conn_id is None or
88
+ empty, then default boto3 configuration would be used (and must be
89
+ maintained on each worker node).
90
+ :param region_name: AWS region_name. If not specified then the default boto3 behaviour is used.
91
+ :param verify: Whether or not to verify SSL certificates. See:
92
+ https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html
93
+ :param botocore_config: Configuration dictionary (key-values) for botocore client. See:
94
+ https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html
86
95
  """
87
96
 
88
- template_fields: Sequence[str] = ("stack_name",)
97
+ aws_hook_class = CloudFormationHook
98
+ template_fields: Sequence[str] = aws_template_fields("stack_name")
89
99
  ui_color = "#C5CAE9"
90
100
 
91
101
  def __init__(
@@ -113,8 +123,3 @@ class CloudFormationDeleteStackSensor(BaseSensorOperator):
113
123
  if self.soft_fail:
114
124
  raise AirflowSkipException(message)
115
125
  raise ValueError(message)
116
-
117
- @cached_property
118
- def hook(self) -> CloudFormationHook:
119
- """Create and return a CloudFormationHook."""
120
- return CloudFormationHook(aws_conn_id=self.aws_conn_id, region_name=self.region_name)
@@ -17,47 +17,53 @@
17
17
  # under the License.
18
18
  from __future__ import annotations
19
19
 
20
- from functools import cached_property
21
20
  from typing import TYPE_CHECKING, Iterable, Sequence
22
21
 
23
22
  from deprecated import deprecated
24
23
 
25
24
  from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning, AirflowSkipException
26
25
  from airflow.providers.amazon.aws.hooks.dms import DmsHook
27
- from airflow.sensors.base import BaseSensorOperator
26
+ from airflow.providers.amazon.aws.sensors.base_aws import AwsBaseSensor
27
+ from airflow.providers.amazon.aws.utils.mixins import aws_template_fields
28
28
 
29
29
  if TYPE_CHECKING:
30
30
  from airflow.utils.context import Context
31
31
 
32
32
 
33
- class DmsTaskBaseSensor(BaseSensorOperator):
33
+ class DmsTaskBaseSensor(AwsBaseSensor[DmsHook]):
34
34
  """
35
35
  Contains general sensor behavior for DMS task.
36
36
 
37
37
  Subclasses should set ``target_statuses`` and ``termination_statuses`` fields.
38
38
 
39
39
  :param replication_task_arn: AWS DMS replication task ARN
40
- :param aws_conn_id: aws connection to uses
41
40
  :param target_statuses: the target statuses, sensor waits until
42
41
  the task reaches any of these states
43
42
  :param termination_statuses: the termination statuses, sensor fails when
44
43
  the task reaches any of these states
44
+ :param aws_conn_id: The Airflow connection used for AWS credentials.
45
+ If this is ``None`` or empty then the default boto3 behaviour is used. If
46
+ running Airflow in a distributed manner and aws_conn_id is None or
47
+ empty, then default boto3 configuration would be used (and must be
48
+ maintained on each worker node).
49
+ :param region_name: AWS region_name. If not specified then the default boto3 behaviour is used.
50
+ :param verify: Whether or not to verify SSL certificates. See:
51
+ https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html
52
+ :param botocore_config: Configuration dictionary (key-values) for botocore client. See:
53
+ https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html
45
54
  """
46
55
 
47
- template_fields: Sequence[str] = ("replication_task_arn",)
48
- template_ext: Sequence[str] = ()
56
+ aws_hook_class = DmsHook
57
+ template_fields: Sequence[str] = aws_template_fields("replication_task_arn")
49
58
 
50
59
  def __init__(
51
60
  self,
52
61
  replication_task_arn: str,
53
- aws_conn_id="aws_default",
54
62
  target_statuses: Iterable[str] | None = None,
55
63
  termination_statuses: Iterable[str] | None = None,
56
- *args,
57
64
  **kwargs,
58
65
  ):
59
- super().__init__(*args, **kwargs)
60
- self.aws_conn_id = aws_conn_id
66
+ super().__init__(**kwargs)
61
67
  self.replication_task_arn = replication_task_arn
62
68
  self.target_statuses: Iterable[str] = target_statuses or []
63
69
  self.termination_statuses: Iterable[str] = termination_statuses or []
@@ -67,14 +73,8 @@ class DmsTaskBaseSensor(BaseSensorOperator):
67
73
  """Get DmsHook."""
68
74
  return self.hook
69
75
 
70
- @cached_property
71
- def hook(self) -> DmsHook:
72
- return DmsHook(self.aws_conn_id)
73
-
74
76
  def poke(self, context: Context):
75
- status: str | None = self.hook.get_task_status(self.replication_task_arn)
76
-
77
- if not status:
77
+ if not (status := self.hook.get_task_status(self.replication_task_arn)):
78
78
  # TODO: remove this if check when min_airflow_version is set to higher than 2.7.1
79
79
  message = f"Failed to read task status, task with ARN {self.replication_task_arn} not found"
80
80
  if self.soft_fail:
@@ -105,15 +105,21 @@ class DmsTaskCompletedSensor(DmsTaskBaseSensor):
105
105
  :ref:`howto/sensor:DmsTaskCompletedSensor`
106
106
 
107
107
  :param replication_task_arn: AWS DMS replication task ARN
108
+ :param aws_conn_id: The Airflow connection used for AWS credentials.
109
+ If this is ``None`` or empty then the default boto3 behaviour is used. If
110
+ running Airflow in a distributed manner and aws_conn_id is None or
111
+ empty, then default boto3 configuration would be used (and must be
112
+ maintained on each worker node).
113
+ :param region_name: AWS region_name. If not specified then the default boto3 behaviour is used.
114
+ :param verify: Whether or not to verify SSL certificates. See:
115
+ https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html
116
+ :param botocore_config: Configuration dictionary (key-values) for botocore client. See:
117
+ https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html
108
118
  """
109
119
 
110
- template_fields: Sequence[str] = ("replication_task_arn",)
111
- template_ext: Sequence[str] = ()
112
-
113
- def __init__(self, *args, **kwargs):
114
- super().__init__(*args, **kwargs)
115
- self.target_statuses = ["stopped"]
116
- self.termination_statuses = [
120
+ def __init__(self, **kwargs):
121
+ kwargs["target_statuses"] = ["stopped"]
122
+ kwargs["termination_statuses"] = [
117
123
  "creating",
118
124
  "deleting",
119
125
  "failed",
@@ -123,3 +129,4 @@ class DmsTaskCompletedSensor(DmsTaskBaseSensor):
123
129
  "ready",
124
130
  "testing",
125
131
  ]
132
+ super().__init__(**kwargs)
@@ -16,17 +16,17 @@
16
16
  # under the License.
17
17
  from __future__ import annotations
18
18
 
19
- from functools import cached_property
20
19
  from typing import TYPE_CHECKING, Any, Iterable, Sequence
21
20
 
22
21
  from airflow.providers.amazon.aws.hooks.dynamodb import DynamoDBHook
23
- from airflow.sensors.base import BaseSensorOperator
22
+ from airflow.providers.amazon.aws.sensors.base_aws import AwsBaseSensor
23
+ from airflow.providers.amazon.aws.utils.mixins import aws_template_fields
24
24
 
25
25
  if TYPE_CHECKING:
26
26
  from airflow.utils.context import Context
27
27
 
28
28
 
29
- class DynamoDBValueSensor(BaseSensorOperator):
29
+ class DynamoDBValueSensor(AwsBaseSensor[DynamoDBHook]):
30
30
  """
31
31
  Waits for an attribute value to be present for an item in a DynamoDB table.
32
32
 
@@ -41,11 +41,20 @@ class DynamoDBValueSensor(BaseSensorOperator):
41
41
  :param attribute_value: DynamoDB attribute value
42
42
  :param sort_key_name: (optional) DynamoDB sort key name
43
43
  :param sort_key_value: (optional) DynamoDB sort key value
44
- :param aws_conn_id: aws connection to use
45
- :param region_name: aws region to use
44
+ :param aws_conn_id: The Airflow connection used for AWS credentials.
45
+ If this is ``None`` or empty then the default boto3 behaviour is used. If
46
+ running Airflow in a distributed manner and aws_conn_id is None or
47
+ empty, then default boto3 configuration would be used (and must be
48
+ maintained on each worker node).
49
+ :param region_name: AWS region_name. If not specified then the default boto3 behaviour is used.
50
+ :param verify: Whether or not to verify SSL certificates. See:
51
+ https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html
52
+ :param botocore_config: Configuration dictionary (key-values) for botocore client. See:
53
+ https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html
46
54
  """
47
55
 
48
- template_fields: Sequence[str] = (
56
+ aws_hook_class = DynamoDBHook
57
+ template_fields: Sequence[str] = aws_template_fields(
49
58
  "table_name",
50
59
  "partition_key_name",
51
60
  "partition_key_value",
@@ -64,8 +73,6 @@ class DynamoDBValueSensor(BaseSensorOperator):
64
73
  attribute_value: str | Iterable[str],
65
74
  sort_key_name: str | None = None,
66
75
  sort_key_value: str | None = None,
67
- aws_conn_id: str | None = DynamoDBHook.default_conn_name,
68
- region_name: str | None = None,
69
76
  **kwargs: Any,
70
77
  ):
71
78
  super().__init__(**kwargs)
@@ -76,8 +83,6 @@ class DynamoDBValueSensor(BaseSensorOperator):
76
83
  self.attribute_value = attribute_value
77
84
  self.sort_key_name = sort_key_name
78
85
  self.sort_key_value = sort_key_value
79
- self.aws_conn_id = aws_conn_id
80
- self.region_name = region_name
81
86
 
82
87
  def poke(self, context: Context) -> bool:
83
88
  """Test DynamoDB item for matching attribute value."""
@@ -108,8 +113,3 @@ class DynamoDBValueSensor(BaseSensorOperator):
108
113
  )
109
114
  except KeyError:
110
115
  return False
111
-
112
- @cached_property
113
- def hook(self) -> DynamoDBHook:
114
- """Create and return a DynamoDBHook."""
115
- return DynamoDBHook(self.aws_conn_id, region_name=self.region_name)