apache-airflow-providers-amazon 8.4.0__py3-none-any.whl → 8.5.0rc1__py3-none-any.whl

airflow/providers/amazon/__init__.py

@@ -28,7 +28,7 @@ import packaging.version
 
 __all__ = ["__version__"]
 
-__version__ = "8.4.0"
+__version__ = "8.5.0"
 
 try:
     from airflow import __version__ as airflow_version

airflow/providers/amazon/aws/hooks/eventbridge.py

@@ -16,12 +16,75 @@
 # under the License.
 from __future__ import annotations
 
+import json
+
 from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
+from airflow.providers.amazon.aws.utils import trim_none_values
+
+
+def _validate_json(pattern: str) -> None:
+    try:
+        json.loads(pattern)
+    except ValueError:
+        raise ValueError("`event_pattern` must be a valid JSON string.")
 
 
 class EventBridgeHook(AwsBaseHook):
     """Amazon EventBridge Hook."""
 
     def __init__(self, *args, **kwargs):
-        """Creating object."""
         super().__init__(client_type="events", *args, **kwargs)
+
+    def put_rule(
+        self,
+        name: str,
+        description: str | None = None,
+        event_bus_name: str | None = None,
+        event_pattern: str | None = None,
+        role_arn: str | None = None,
+        schedule_expression: str | None = None,
+        state: str | None = None,
+        tags: list[dict] | None = None,
+        **kwargs,
+    ):
+        """
+        Create or update an EventBridge rule.
+
+        :param name: name of the rule to create or update (required)
+        :param description: description of the rule
+        :param event_bus_name: name or ARN of the event bus to associate with this rule
+        :param event_pattern: pattern of events to be matched to this rule
+        :param role_arn: the Amazon Resource Name of the IAM role associated with the rule
+        :param schedule_expression: the scheduling expression (for example, a cron or rate expression)
+        :param state: indicates whether rule is set to be "ENABLED" or "DISABLED"
+        :param tags: list of key-value pairs to associate with the rule
+
+        """
+        if not (event_pattern or schedule_expression):
+            raise ValueError(
+                "One of `event_pattern` or `schedule_expression` are required in order to "
+                "put or update your rule."
+            )
+
+        if state and state not in ["ENABLED", "DISABLED"]:
+            raise ValueError("`state` must be specified as ENABLED or DISABLED.")
+
+        if event_pattern:
+            _validate_json(event_pattern)
+
+        put_rule_kwargs: dict[str, str | list] = {
+            **trim_none_values(
+                {
+                    "Name": name,
+                    "Description": description,
+                    "EventBusName": event_bus_name,
+                    "EventPattern": event_pattern,
+                    "RoleArn": role_arn,
+                    "ScheduleExpression": schedule_expression,
+                    "State": state,
+                    "Tags": tags,
+                }
+            )
+        }
+
+        return self.conn.put_rule(**put_rule_kwargs)

airflow/providers/amazon/aws/hooks/s3.py

@@ -57,7 +57,7 @@ from airflow.providers.amazon.aws.utils.tags import format_tags
 from airflow.utils.helpers import chunks
 
 if TYPE_CHECKING:
-    from mypy_boto3_s3.service_resource import Object as S3ResourceObject
+    from mypy_boto3_s3.service_resource import Bucket as S3Bucket, Object as S3ResourceObject
 
 T = TypeVar("T", bound=Callable)
 
@@ -298,7 +298,7 @@ class S3Hook(AwsBaseHook):
             return False
 
     @provide_bucket_name
-    def get_bucket(self, bucket_name: str | None = None) -> object:
+    def get_bucket(self, bucket_name: str | None = None) -> S3Bucket:
         """
         Returns a :py:class:`S3.Bucket` object.
 

airflow/providers/amazon/aws/operators/batch.py

@@ -149,7 +149,7 @@ class BatchOperator(BaseOperator):
         parameters: dict | None = None,
         job_id: str | None = None,
         waiters: Any | None = None,
-        max_retries: int | None = None,
+        max_retries: int = 4200,
         status_retries: int | None = None,
         aws_conn_id: str | None = None,
         region_name: str | None = None,
@@ -223,7 +223,7 @@ class BatchOperator(BaseOperator):
                 timeout=self.execution_timeout,
                 trigger=BatchJobTrigger(
                     job_id=self.job_id,
-                    waiter_max_attempts=self.max_retries or 10,
+                    waiter_max_attempts=self.max_retries,
                     aws_conn_id=self.aws_conn_id,
                     region_name=self.region_name,
                     waiter_delay=self.poll_interval,

airflow/providers/amazon/aws/operators/eventbridge.py

@@ -32,6 +32,10 @@ class EventBridgePutEventsOperator(BaseOperator):
     """
     Put Events onto Amazon EventBridge.
 
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:EventBridgePutEventsOperator`
+
     :param entries: the list of events to be put onto EventBridge, each event is a dict (required)
     :param endpoint_id: the URL subdomain of the endpoint
     :param aws_conn_id: the AWS connection to use
@@ -85,3 +89,84 @@ class EventBridgePutEventsOperator(BaseOperator):
 
         if self.do_xcom_push:
             return [e["EventId"] for e in response["Entries"]]
+
+
+class EventBridgePutRuleOperator(BaseOperator):
+    """
+    Create or update a specified EventBridge rule.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:EventBridgePutRuleOperator`
+
+    :param name: name of the rule to create or update (required)
+    :param description: description of the rule
+    :param event_bus_name: name or ARN of the event bus to associate with this rule
+    :param event_pattern: pattern of events to be matched to this rule
+    :param role_arn: the Amazon Resource Name of the IAM role associated with the rule
+    :param schedule_expression: the scheduling expression (for example, a cron or rate expression)
+    :param state: indicates whether rule is set to be "ENABLED" or "DISABLED"
+    :param tags: list of key-value pairs to associate with the rule
+    :param region: the region where rule is to be created or updated
+
+    """
+
+    template_fields: Sequence[str] = (
+        "aws_conn_id",
+        "name",
+        "description",
+        "event_bus_name",
+        "event_pattern",
+        "role_arn",
+        "schedule_expression",
+        "state",
+        "tags",
+        "region_name",
+    )
+
+    def __init__(
+        self,
+        *,
+        name: str,
+        description: str | None = None,
+        event_bus_name: str | None = None,
+        event_pattern: str | None = None,
+        role_arn: str | None = None,
+        schedule_expression: str | None = None,
+        state: str | None = None,
+        tags: list | None = None,
+        region_name: str | None = None,
+        aws_conn_id: str = "aws_default",
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.name = name
+        self.description = description
+        self.event_bus_name = event_bus_name
+        self.event_pattern = event_pattern
+        self.role_arn = role_arn
+        self.region_name = region_name
+        self.schedule_expression = schedule_expression
+        self.state = state
+        self.tags = tags
+        self.aws_conn_id = aws_conn_id
+
+    @cached_property
+    def hook(self) -> EventBridgeHook:
+        """Create and return an EventBridgeHook."""
+        return EventBridgeHook(aws_conn_id=self.aws_conn_id, region_name=self.region_name)
+
+    def execute(self, context: Context):
+
+        self.log.info('Sending rule "%s" to EventBridge.', self.name)
+
+        return self.hook.put_rule(
+            name=self.name,
+            description=self.description,
+            event_bus_name=self.event_bus_name,
+            event_pattern=self.event_pattern,
+            role_arn=self.role_arn,
+            schedule_expression=self.schedule_expression,
+            state=self.state,
+            tags=self.tags,
+        )

airflow/providers/amazon/aws/operators/rds.py

@@ -392,6 +392,8 @@ class RdsCancelExportTaskOperator(RdsBaseOperator):
 
     :param export_task_identifier: The identifier of the snapshot export task to cancel
     :param wait_for_completion:  If True, waits for DB snapshot export to cancel. (default: True)
+    :param check_interval: The amount of time in seconds to wait between attempts
+    :param max_attempts: The maximum number of attempts to be made
     """
 
     template_fields = ("export_task_identifier",)
@@ -402,6 +404,7 @@ class RdsCancelExportTaskOperator(RdsBaseOperator):
         export_task_identifier: str,
         wait_for_completion: bool = True,
         check_interval: int = 30,
+        max_attempts: int = 40,
         **kwargs,
     ):
         super().__init__(**kwargs)
@@ -409,6 +412,7 @@ class RdsCancelExportTaskOperator(RdsBaseOperator):
         self.export_task_identifier = export_task_identifier
         self.wait_for_completion = wait_for_completion
         self.check_interval = check_interval
+        self.max_attempts = max_attempts
 
     def execute(self, context: Context) -> str:
         self.log.info("Canceling export task %s", self.export_task_identifier)
@@ -419,7 +423,10 @@ class RdsCancelExportTaskOperator(RdsBaseOperator):
 
         if self.wait_for_completion:
             self.hook.wait_for_export_task_state(
-                self.export_task_identifier, target_state="canceled", check_interval=self.check_interval
+                self.export_task_identifier,
+                target_state="canceled",
+                check_interval=self.check_interval,
+                max_attempts=self.max_attempts,
             )
         return json.dumps(cancel_export, default=str)
 

airflow/providers/amazon/aws/operators/sagemaker.py

@@ -40,13 +40,16 @@ from airflow.providers.amazon.aws.utils.tags import format_tags
 from airflow.utils.json import AirflowJsonEncoder
 
 if TYPE_CHECKING:
+    from openlineage.client.run import Dataset
+
+    from airflow.providers.openlineage.extractors.base import OperatorLineage
     from airflow.utils.context import Context
 
 DEFAULT_CONN_ID: str = "aws_default"
 CHECK_INTERVAL_SECOND: int = 30
 
 
-def serialize(result: dict) -> str:
+def serialize(result: dict) -> dict:
     return json.loads(json.dumps(result, cls=AirflowJsonEncoder))
 
 
@@ -158,6 +161,14 @@ class SageMakerBaseOperator(BaseOperator):
         """Return SageMakerHook."""
         return SageMakerHook(aws_conn_id=self.aws_conn_id)
 
+    @staticmethod
+    def path_to_s3_dataset(path) -> Dataset:
+        from openlineage.client.run import Dataset
+
+        path = path.replace("s3://", "")
+        split_path = path.split("/")
+        return Dataset(namespace=f"s3://{split_path[0]}", name="/".join(split_path[1:]), facets={})
+
 
 class SageMakerProcessingOperator(SageMakerBaseOperator):
     """
@@ -225,6 +236,7 @@ class SageMakerProcessingOperator(SageMakerBaseOperator):
         self.max_attempts = max_attempts or 60
         self.max_ingestion_time = max_ingestion_time
         self.deferrable = deferrable
+        self.serialized_job: dict
 
     def _create_integer_fields(self) -> None:
         """Set fields which should be cast to integers."""
@@ -282,14 +294,48 @@ class SageMakerProcessingOperator(SageMakerBaseOperator):
                 method_name="execute_complete",
             )
 
-        return {"Processing": serialize(self.hook.describe_processing_job(self.config["ProcessingJobName"]))}
+        self.serialized_job = serialize(self.hook.describe_processing_job(self.config["ProcessingJobName"]))
+        return {"Processing": self.serialized_job}
 
     def execute_complete(self, context, event=None):
         if event["status"] != "success":
             raise AirflowException(f"Error while running job: {event}")
         else:
             self.log.info(event["message"])
-        return {"Processing": serialize(self.hook.describe_processing_job(self.config["ProcessingJobName"]))}
+        self.serialized_job = serialize(self.hook.describe_processing_job(self.config["ProcessingJobName"]))
+        return {"Processing": self.serialized_job}
+
+    def get_openlineage_facets_on_complete(self, task_instance) -> OperatorLineage:
+        """Returns OpenLineage data gathered from SageMaker's API response saved by processing job."""
+        from airflow.providers.openlineage.extractors.base import OperatorLineage
+
+        inputs = []
+        outputs = []
+        try:
+            inputs, outputs = self._extract_s3_dataset_identifiers(
+                processing_inputs=self.serialized_job["ProcessingInputs"],
+                processing_outputs=self.serialized_job["ProcessingOutputConfig"]["Outputs"],
+            )
+        except KeyError:
+            self.log.exception("Could not find input/output information in Xcom.")
+
+        return OperatorLineage(inputs=inputs, outputs=outputs)
+
+    def _extract_s3_dataset_identifiers(self, processing_inputs, processing_outputs):
+        inputs = []
+        outputs = []
+        try:
+            for processing_input in processing_inputs:
+                inputs.append(self.path_to_s3_dataset(processing_input["S3Input"]["S3Uri"]))
+        except KeyError:
+            self.log.exception("Cannot find S3 input details", exc_info=True)
+
+        try:
+            for processing_output in processing_outputs:
+                outputs.append(self.path_to_s3_dataset(processing_output["S3Output"]["S3Uri"]))
+        except KeyError:
+            self.log.exception("Cannot find S3 output details.", exc_info=True)
+        return inputs, outputs
 
 
 class SageMakerEndpointConfigOperator(SageMakerBaseOperator):
@@ -579,6 +625,8 @@ class SageMakerTransformOperator(SageMakerBaseOperator):
                 Provided value: '{action_if_job_exists}'."
             )
         self.deferrable = deferrable
+        self.serialized_model: dict
+        self.serialized_tranform: dict
 
     def _create_integer_fields(self) -> None:
         """Set fields which should be cast to integers."""
@@ -650,10 +698,11 @@ class SageMakerTransformOperator(SageMakerBaseOperator):
                 method_name="execute_complete",
             )
 
-        return {
-            "Model": serialize(self.hook.describe_model(transform_config["ModelName"])),
-            "Transform": serialize(self.hook.describe_transform_job(transform_config["TransformJobName"])),
-        }
+        self.serialized_model = serialize(self.hook.describe_model(transform_config["ModelName"]))
+        self.serialized_tranform = serialize(
+            self.hook.describe_transform_job(transform_config["TransformJobName"])
+        )
+        return {"Model": self.serialized_model, "Transform": self.serialized_tranform}
 
     def execute_complete(self, context, event=None):
         if event["status"] != "success":
@@ -661,10 +710,62 @@ class SageMakerTransformOperator(SageMakerBaseOperator):
         else:
             self.log.info(event["message"])
         transform_config = self.config.get("Transform", self.config)
-        return {
-            "Model": serialize(self.hook.describe_model(transform_config["ModelName"])),
-            "Transform": serialize(self.hook.describe_transform_job(transform_config["TransformJobName"])),
-        }
+        self.serialized_model = serialize(self.hook.describe_model(transform_config["ModelName"]))
+        self.serialized_tranform = serialize(
+            self.hook.describe_transform_job(transform_config["TransformJobName"])
+        )
+        return {"Model": self.serialized_model, "Transform": self.serialized_tranform}
+
+    def get_openlineage_facets_on_complete(self, task_instance) -> OperatorLineage:
+        """Returns OpenLineage data gathered from SageMaker's API response saved by transform job."""
+        from airflow.providers.openlineage.extractors import OperatorLineage
+
+        model_package_arn = None
+        transform_input = None
+        transform_output = None
+
+        try:
+            model_package_arn = self.serialized_model["PrimaryContainer"]["ModelPackageName"]
+        except KeyError:
+            self.log.error("Cannot find Model Package Name.", exc_info=True)
+
+        try:
+            transform_input = self.serialized_tranform["TransformInput"]["DataSource"]["S3DataSource"][
+                "S3Uri"
+            ]
+            transform_output = self.serialized_tranform["TransformOutput"]["S3OutputPath"]
+        except KeyError:
+            self.log.error("Cannot find some required input/output details.", exc_info=True)
+
+        inputs = []
+
+        if transform_input is not None:
+            inputs.append(self.path_to_s3_dataset(transform_input))
+
+        if model_package_arn is not None:
+            model_data_urls = self._get_model_data_urls(model_package_arn)
+            for model_data_url in model_data_urls:
+                inputs.append(self.path_to_s3_dataset(model_data_url))
+
+        outputs = []
+        if transform_output is not None:
+            outputs.append(self.path_to_s3_dataset(transform_output))
+
+        return OperatorLineage(inputs=inputs, outputs=outputs)
+
+    def _get_model_data_urls(self, model_package_arn) -> list:
+        model_data_urls = []
+        try:
+            model_containers = self.hook.get_conn().describe_model_package(
+                ModelPackageName=model_package_arn
+            )["InferenceSpecification"]["Containers"]
+
+            for container in model_containers:
+                model_data_urls.append(container["ModelDataUrl"])
+        except KeyError:
+            self.log.exception("Cannot retrieve model details.", exc_info=True)
+
+        return model_data_urls
 
 
 class SageMakerTuningOperator(SageMakerBaseOperator):
@@ -891,6 +992,7 @@ class SageMakerTrainingOperator(SageMakerBaseOperator):
                 Provided value: '{action_if_job_exists}'."
             )
         self.deferrable = deferrable
+        self.serialized_training_data: dict
 
     def expand_role(self) -> None:
         """Expands an IAM role name into an ARN."""
@@ -951,16 +1053,40 @@ class SageMakerTrainingOperator(SageMakerBaseOperator):
                 method_name="execute_complete",
             )
 
-        result = {"Training": serialize(self.hook.describe_training_job(self.config["TrainingJobName"]))}
-        return result
+        self.serialized_training_data = serialize(
+            self.hook.describe_training_job(self.config["TrainingJobName"])
+        )
+        return {"Training": self.serialized_training_data}
 
     def execute_complete(self, context, event=None):
         if event["status"] != "success":
             raise AirflowException(f"Error while running job: {event}")
         else:
             self.log.info(event["message"])
-        result = {"Training": serialize(self.hook.describe_training_job(self.config["TrainingJobName"]))}
-        return result
+        self.serialized_training_data = serialize(
+            self.hook.describe_training_job(self.config["TrainingJobName"])
+        )
+        return {"Training": self.serialized_training_data}
+
+    def get_openlineage_facets_on_complete(self, task_instance) -> OperatorLineage:
+        """Returns OpenLineage data gathered from SageMaker's API response saved by training job."""
+        from airflow.providers.openlineage.extractors import OperatorLineage
+
+        inputs = []
+        outputs = []
+        try:
+            for input_data in self.serialized_training_data["InputDataConfig"]:
+                inputs.append(self.path_to_s3_dataset(input_data["DataSource"]["S3DataSource"]["S3Uri"]))
+        except KeyError:
+            self.log.exception("Issues extracting inputs.")
+
+        try:
+            outputs.append(
+                self.path_to_s3_dataset(self.serialized_training_data["ModelArtifacts"]["S3ModelArtifacts"])
+            )
+        except KeyError:
+            self.log.exception("Issues extracting inputs.")
+        return OperatorLineage(inputs=inputs, outputs=outputs)
 
 
 class SageMakerDeleteModelOperator(SageMakerBaseOperator):

airflow/providers/amazon/aws/sensors/sqs.py

@@ -18,20 +18,22 @@
 """Reads and then deletes the message from SQS queue."""
 from __future__ import annotations
 
-import json
 from functools import cached_property
 from typing import TYPE_CHECKING, Any, Collection, Literal, Sequence
 
 from deprecated import deprecated
-from jsonpath_ng import parse
 
+from airflow.configuration import conf
 from airflow.exceptions import AirflowException
 from airflow.providers.amazon.aws.hooks.base_aws import BaseAwsConnection
 from airflow.providers.amazon.aws.hooks.sqs import SqsHook
+from airflow.providers.amazon.aws.triggers.sqs import SqsSensorTrigger
+from airflow.providers.amazon.aws.utils.sqs import process_response
 from airflow.sensors.base import BaseSensorOperator
 
 if TYPE_CHECKING:
     from airflow.utils.context import Context
+from datetime import timedelta
 
 
 class SqsSensor(BaseSensorOperator):
@@ -70,6 +72,9 @@ class SqsSensor(BaseSensorOperator):
     :param delete_message_on_reception: Default to `True`, the messages are deleted from the queue
         as soon as being consumed. Otherwise, the messages remain in the queue after consumption and
         should be deleted manually.
+    :param deferrable: If True, the sensor will operate in deferrable more. This mode requires aiobotocore
+        module to be installed.
+        (default: False, but can be overridden in config file by setting default_deferrable to True)
 
     """
 
@@ -88,6 +93,7 @@ class SqsSensor(BaseSensorOperator):
         message_filtering_match_values: Any = None,
         message_filtering_config: Any = None,
         delete_message_on_reception: bool = True,
+        deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
         **kwargs,
     ):
         super().__init__(**kwargs)
@@ -112,6 +118,34 @@ class SqsSensor(BaseSensorOperator):
                 raise TypeError("message_filtering_match_values must be specified for literal matching")
 
         self.message_filtering_config = message_filtering_config
+        self.deferrable = deferrable
+
+    def execute(self, context: Context) -> Any:
+        if self.deferrable:
+            self.defer(
+                trigger=SqsSensorTrigger(
+                    sqs_queue=self.sqs_queue,
+                    aws_conn_id=self.aws_conn_id,
+                    max_messages=self.max_messages,
+                    num_batches=self.num_batches,
+                    wait_time_seconds=self.wait_time_seconds,
+                    visibility_timeout=self.visibility_timeout,
+                    message_filtering=self.message_filtering,
+                    message_filtering_match_values=self.message_filtering_match_values,
+                    message_filtering_config=self.message_filtering_config,
+                    delete_message_on_reception=self.delete_message_on_reception,
+                    waiter_delay=int(self.poke_interval),
+                ),
+                method_name="execute_complete",
+                timeout=timedelta(seconds=self.timeout),
+            )
+        else:
+            super().execute(context=context)
+
+    def execute_complete(self, context: Context, event: dict | None = None) -> None:
+        if event is None or event["status"] != "success":
+            raise AirflowException(f"Trigger error: event is {event}")
+        context["ti"].xcom_push(key="messages", value=event["message_batch"])
 
     def poll_sqs(self, sqs_conn: BaseAwsConnection) -> Collection:
         """
@@ -131,19 +165,7 @@ class SqsSensor(BaseSensorOperator):
             receive_message_kwargs["VisibilityTimeout"] = self.visibility_timeout
 
         response = sqs_conn.receive_message(**receive_message_kwargs)
-
-        if "Messages" not in response:
-            return []
-
-        messages = response["Messages"]
-        num_messages = len(messages)
-        self.log.info("Received %d messages", num_messages)
-
-        if num_messages and self.message_filtering:
-            messages = self.filter_messages(messages)
-            num_messages = len(messages)
-            self.log.info("There are %d messages left after filtering", num_messages)
-        return messages
+        return response
 
     def poke(self, context: Context):
         """
@@ -156,7 +178,13 @@ class SqsSensor(BaseSensorOperator):
 
         # perform multiple SQS call to retrieve messages in series
         for _ in range(self.num_batches):
-            messages = self.poll_sqs(sqs_conn=self.hook.conn)
+            response = self.poll_sqs(sqs_conn=self.hook.conn)
+            messages = process_response(
+                response,
+                self.message_filtering,
+                self.message_filtering_match_values,
+                self.message_filtering_config,
+            )
 
             if not len(messages):
                 continue
@@ -191,37 +219,3 @@ class SqsSensor(BaseSensorOperator):
     @cached_property
     def hook(self) -> SqsHook:
         return SqsHook(aws_conn_id=self.aws_conn_id)
-
-    def filter_messages(self, messages):
-        if self.message_filtering == "literal":
-            return self.filter_messages_literal(messages)
-        if self.message_filtering == "jsonpath":
-            return self.filter_messages_jsonpath(messages)
-        else:
-            raise NotImplementedError("Override this method to define custom filters")
-
-    def filter_messages_literal(self, messages):
-        filtered_messages = []
-        for message in messages:
-            if message["Body"] in self.message_filtering_match_values:
-                filtered_messages.append(message)
-        return filtered_messages
-
-    def filter_messages_jsonpath(self, messages):
-        jsonpath_expr = parse(self.message_filtering_config)
-        filtered_messages = []
-        for message in messages:
-            body = message["Body"]
-            # Body is a string, deserialize to an object and then parse
-            body = json.loads(body)
-            results = jsonpath_expr.find(body)
-            if not results:
-                continue
-            if self.message_filtering_match_values is None:
-                filtered_messages.append(message)
-                continue
-            for result in results:
-                if result.value in self.message_filtering_match_values:
-                    filtered_messages.append(message)
-                    break
-        return filtered_messages

airflow/providers/amazon/aws/transfers/gcs_to_s3.py

@@ -22,7 +22,9 @@ import os
 import warnings
 from typing import TYPE_CHECKING, Sequence
 
-from airflow.exceptions import AirflowProviderDeprecationWarning
+from packaging.version import Version
+
+from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
 from airflow.models import BaseOperator
 from airflow.providers.amazon.aws.hooks.s3 import S3Hook
 from airflow.providers.google.cloud.hooks.gcs import GCSHook
@@ -80,6 +82,8 @@ class GCSToS3Operator(BaseOperator):
          on the bucket is recreated within path passed in dest_s3_key.
     :param match_glob: (Optional) filters objects based on the glob pattern given by the string
         (e.g, ``'**/*/.json'``)
+    :param gcp_user_project: (Optional) The identifier of the Google Cloud project to bill for this request.
+        Required for Requester Pays buckets.
     """
 
     template_fields: Sequence[str] = (
@@ -88,6 +92,7 @@ class GCSToS3Operator(BaseOperator):
         "delimiter",
         "dest_s3_key",
         "google_impersonation_chain",
+        "gcp_user_project",
     )
     ui_color = "#f0eee4"
 
@@ -107,19 +112,13 @@ class GCSToS3Operator(BaseOperator):
         s3_acl_policy: str | None = None,
         keep_directory_structure: bool = True,
         match_glob: str | None = None,
+        gcp_user_project: str | None = None,
         **kwargs,
     ) -> None:
         super().__init__(**kwargs)
 
         self.bucket = bucket
         self.prefix = prefix
-        if delimiter:
-            warnings.warn(
-                "Usage of 'delimiter' is deprecated, please use 'match_glob' instead",
-                AirflowProviderDeprecationWarning,
-                stacklevel=2,
-            )
-        self.delimiter = delimiter
         self.gcp_conn_id = gcp_conn_id
         self.dest_aws_conn_id = dest_aws_conn_id
         self.dest_s3_key = dest_s3_key
@@ -129,11 +128,33 @@ class GCSToS3Operator(BaseOperator):
         self.dest_s3_extra_args = dest_s3_extra_args or {}
         self.s3_acl_policy = s3_acl_policy
         self.keep_directory_structure = keep_directory_structure
+        try:
+            from airflow.providers.google import __version__
+
+            if Version(__version__) >= Version("10.3.0"):
+                self.__is_match_glob_supported = True
+            else:
+                self.__is_match_glob_supported = False
+        except ImportError:  # __version__ was added in 10.1.0, so this means it's < 10.3.0
+            self.__is_match_glob_supported = False
+        if self.__is_match_glob_supported:
+            if delimiter:
+                warnings.warn(
+                    "Usage of 'delimiter' is deprecated, please use 'match_glob' instead",
+                    AirflowProviderDeprecationWarning,
+                    stacklevel=2,
+                )
+        elif match_glob:
+            raise AirflowException(
+                "The 'match_glob' parameter requires 'apache-airflow-providers-google>=10.3.0'."
+            )
+        self.delimiter = delimiter
         self.match_glob = match_glob
+        self.gcp_user_project = gcp_user_project
 
     def execute(self, context: Context) -> list[str]:
         # list all files in an Google Cloud Storage bucket
-        hook = GCSHook(
+        gcs_hook = GCSHook(
             gcp_conn_id=self.gcp_conn_id,
             impersonation_chain=self.google_impersonation_chain,
         )
@@ -145,9 +166,16 @@ class GCSToS3Operator(BaseOperator):
             self.prefix,
         )
 
-        files = hook.list(
-            bucket_name=self.bucket, prefix=self.prefix, delimiter=self.delimiter, match_glob=self.match_glob
-        )
+        list_kwargs = {
+            "bucket_name": self.bucket,
+            "prefix": self.prefix,
+            "delimiter": self.delimiter,
+            "user_project": self.gcp_user_project,
+        }
+        if self.__is_match_glob_supported:
+            list_kwargs["match_glob"] = self.match_glob
+
+        gcs_files = gcs_hook.list(**list_kwargs)  # type: ignore
 
         s3_hook = S3Hook(
             aws_conn_id=self.dest_aws_conn_id, verify=self.dest_verify, extra_args=self.dest_s3_extra_args
@@ -173,24 +201,23 @@ class GCSToS3Operator(BaseOperator):
             existing_files = existing_files if existing_files is not None else []
             # remove the prefix for the existing files to allow the match
             existing_files = [file.replace(prefix, "", 1) for file in existing_files]
-            files = list(set(files) - set(existing_files))
-
-        if files:
+            gcs_files = list(set(gcs_files) - set(existing_files))
 
-            for file in files:
-                with hook.provide_file(object_name=file, bucket_name=self.bucket) as local_tmp_file:
+        if gcs_files:
+            for file in gcs_files:
+                with gcs_hook.provide_file(
+                    object_name=file, bucket_name=self.bucket, user_project=self.gcp_user_project
+                ) as local_tmp_file:
                     dest_key = os.path.join(self.dest_s3_key, file)
                     self.log.info("Saving file to %s", dest_key)
-
                     s3_hook.load_file(
                         filename=local_tmp_file.name,
                         key=dest_key,
                         replace=self.replace,
                         acl_policy=self.s3_acl_policy,
                     )
-
-            self.log.info("All done, uploaded %d files to S3", len(files))
+            self.log.info("All done, uploaded %d files to S3", len(gcs_files))
         else:
             self.log.info("In sync, no files needed to be uploaded to S3")
 
-        return files
+        return gcs_files

airflow/providers/amazon/aws/triggers/sqs.py

@@ -0,0 +1,168 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import asyncio
+from typing import Any, AsyncIterator, Collection, Literal
+
+from airflow.exceptions import AirflowException
+from airflow.providers.amazon.aws.hooks.base_aws import BaseAwsConnection
+from airflow.providers.amazon.aws.hooks.sqs import SqsHook
+from airflow.providers.amazon.aws.utils.sqs import process_response
+from airflow.triggers.base import BaseTrigger, TriggerEvent
+
+
+class SqsSensorTrigger(BaseTrigger):
+    """
+    Asynchronously get messages from an Amazon SQS queue and then delete the messages from the queue.
+
+    :param sqs_queue: The SQS queue url
+    :param aws_conn_id: AWS connection id
+    :param max_messages: The maximum number of messages to retrieve for each poke (templated)
+    :param num_batches: The number of times the sensor will call the SQS API to receive messages (default: 1)
+    :param wait_time_seconds: The time in seconds to wait for receiving messages (default: 1 second)
+    :param visibility_timeout: Visibility timeout, a period of time during which
+        Amazon SQS prevents other consumers from receiving and processing the message.
+    :param message_filtering: Specified how received messages should be filtered. Supported options are:
+        `None` (no filtering, default), `'literal'` (message Body literal match) or `'jsonpath'`
+        (message Body filtered using a JSONPath expression).
+        You may add further methods by overriding the relevant class methods.
+    :param message_filtering_match_values: Optional value/s for the message filter to match on.
+        For example, with literal matching, if a message body matches any of the specified values
+        then it is included. For JSONPath matching, the result of the JSONPath expression is used
+        and may match any of the specified values.
+    :param message_filtering_config: Additional configuration to pass to the message filter.
+        For example with JSONPath filtering you can pass a JSONPath expression string here,
+        such as `'foo[*].baz'`. Messages with a Body which does not match are ignored.
+    :param delete_message_on_reception: Default to `True`, the messages are deleted from the queue
+        as soon as being consumed. Otherwise, the messages remain in the queue after consumption and
+        should be deleted manually.
+    :param waiter_delay: The time in seconds to wait between calls to the SQS API to receive messages.
+    """
+
+    def __init__(
+        self,
+        sqs_queue: str,
+        aws_conn_id: str = "aws_default",
+        max_messages: int = 5,
+        num_batches: int = 1,
+        wait_time_seconds: int = 1,
+        visibility_timeout: int | None = None,
+        message_filtering: Literal["literal", "jsonpath"] | None = None,
+        message_filtering_match_values: Any = None,
+        message_filtering_config: Any = None,
+        delete_message_on_reception: bool = True,
+        waiter_delay: int = 60,
+    ):
+        self.sqs_queue = sqs_queue
+        self.aws_conn_id = aws_conn_id
+        self.max_messages = max_messages
+        self.num_batches = num_batches
+        self.wait_time_seconds = wait_time_seconds
+        self.visibility_timeout = visibility_timeout
+        self.message_filtering = message_filtering
+        self.delete_message_on_reception = delete_message_on_reception
+        self.message_filtering_match_values = message_filtering_match_values
+        self.message_filtering_config = message_filtering_config
+        self.waiter_delay = waiter_delay
+
+    def serialize(self) -> tuple[str, dict[str, Any]]:
+        return (
+            self.__class__.__module__ + "." + self.__class__.__qualname__,
+            {
+                "sqs_queue": self.sqs_queue,
+                "aws_conn_id": self.aws_conn_id,
+                "max_messages": self.max_messages,
+                "num_batches": self.num_batches,
+                "wait_time_seconds": self.wait_time_seconds,
+                "visibility_timeout": self.visibility_timeout,
+                "message_filtering": self.message_filtering,
+                "delete_message_on_reception": self.delete_message_on_reception,
+                "message_filtering_match_values": self.message_filtering_match_values,
+                "message_filtering_config": self.message_filtering_config,
+                "waiter_delay": self.waiter_delay,
+            },
+        )
+
+    @property
+    def hook(self) -> SqsHook:
+        return SqsHook(aws_conn_id=self.aws_conn_id)
+
+    async def poll_sqs(self, client: BaseAwsConnection) -> Collection:
+        """
+        Asynchronously poll SQS queue to retrieve messages.
+
+        :param client: SQS connection
+        :return: A list of messages retrieved from SQS
+        """
+        self.log.info("SqsSensor checking for message on queue: %s", self.sqs_queue)
+
+        receive_message_kwargs = {
+            "QueueUrl": self.sqs_queue,
+            "MaxNumberOfMessages": self.max_messages,
+            "WaitTimeSeconds": self.wait_time_seconds,
+        }
+        if self.visibility_timeout is not None:
+            receive_message_kwargs["VisibilityTimeout"] = self.visibility_timeout
+
+        response = await client.receive_message(**receive_message_kwargs)
+        return response
+
+    async def poke(self, client: Any):
+        message_batch: list[Any] = []
+        for _ in range(self.num_batches):
+            self.log.info("starting call to poll sqs")
+            response = await self.poll_sqs(client=client)
+            messages = process_response(
+                response,
+                self.message_filtering,
+                self.message_filtering_match_values,
+                self.message_filtering_config,
+            )
+
+            if not messages:
+                continue
+
+            message_batch.extend(messages)
+
+            if self.delete_message_on_reception:
+                self.log.info("Deleting %d messages", len(messages))
+
+                entries = [
+                    {"Id": message["MessageId"], "ReceiptHandle": message["ReceiptHandle"]}
+                    for message in messages
+                ]
+                response = await client.delete_message_batch(QueueUrl=self.sqs_queue, Entries=entries)
+
+                if "Successful" not in response:
+                    raise AirflowException(
+                        f"Delete SQS Messages failed {str(response)} for messages {str(messages)}"
+                    )
+
+        return message_batch
+
+    async def run(self) -> AsyncIterator[TriggerEvent]:
+        while True:
+            # This loop will run indefinitely until the timeout, which is set in the self.defer
+            # method, is reached.
+            async with self.hook.async_conn as client:
+                result = await self.poke(client=client)
+                if result:
+                    yield TriggerEvent({"status": "success", "message_batch": result})
+                    break
+                else:
+                    await asyncio.sleep(self.waiter_delay)

airflow/providers/amazon/aws/utils/sqs.py

@@ -0,0 +1,90 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any, Literal
+
+from jsonpath_ng import parse
+
+log = logging.getLogger(__name__)
+
+
+def process_response(
+    response: Any,
+    message_filtering: Literal["literal", "jsonpath"] | None = None,
+    message_filtering_match_values: Any = None,
+    message_filtering_config: Any = None,
+) -> Any:
+    """
+    Process the response from SQS.
+
+    :param response: The response from SQS
+    :return: The processed response
+    """
+    if not isinstance(response, dict):
+        return []
+    elif "Messages" not in response:
+        return []
+
+    messages = response["Messages"]
+    num_messages = len(messages)
+    log.info("Received %d messages", num_messages)
+
+    if num_messages and message_filtering:
+        messages = filter_messages(
+            messages, message_filtering, message_filtering_match_values, message_filtering_config
+        )
+        num_messages = len(messages)
+        log.info("There are %d messages left after filtering", num_messages)
+    return messages
+
+
+def filter_messages(
+    messages, message_filtering, message_filtering_match_values, message_filtering_config
+) -> list[Any]:
+    if message_filtering == "literal":
+        return filter_messages_literal(messages, message_filtering_match_values)
+    if message_filtering == "jsonpath":
+        return filter_messages_jsonpath(messages, message_filtering_match_values, message_filtering_config)
+    else:
+        raise NotImplementedError("Override this method to define custom filters")
+
+
+def filter_messages_literal(messages, message_filtering_match_values) -> list[Any]:
+    return [message for message in messages if message["Body"] in message_filtering_match_values]
+
+
+def filter_messages_jsonpath(messages, message_filtering_match_values, message_filtering_config) -> list[Any]:
+    jsonpath_expr = parse(message_filtering_config)
+    filtered_messages = []
+    for message in messages:
+        body = message["Body"]
+        # Body is a string, deserialize to an object and then parse
+        body = json.loads(body)
+        results = jsonpath_expr.find(body)
+        if not results:
+            continue
+        if message_filtering_match_values is None:
+            filtered_messages.append(message)
+            continue
+        for result in results:
+            if result.value in message_filtering_match_values:
+                filtered_messages.append(message)
+                break
+    return filtered_messages

airflow/providers/amazon/get_provider_info.py

@@ -29,6 +29,7 @@ def get_provider_info():
         "description": "Amazon integration (including `Amazon Web Services (AWS) <https://aws.amazon.com/>`__).\n",
         "suspended": False,
         "versions": [
+            "8.5.0",
             "8.4.0",
             "8.3.1",
             "8.3.0",
@@ -716,6 +717,10 @@ def get_provider_info():
                 "integration-name": "AWS Step Functions",
                 "python-modules": ["airflow.providers.amazon.aws.triggers.step_function"],
             },
+            {
+                "integration-name": "Amazon Simple Queue Service (SQS)",
+                "python-modules": ["airflow.providers.amazon.aws.triggers.sqs"],
+            },
         ],
         "transfers": [
             {

{apache_airflow_providers_amazon-8.4.0.dist-info → apache_airflow_providers_amazon-8.5.0rc1.dist-info}/METADATA

@@ -1,14 +1,14 @@
 Metadata-Version: 2.1
 Name: apache-airflow-providers-amazon
-Version: 8.4.0
+Version: 8.5.0rc1
 Summary: Provider for Apache Airflow. Implements apache-airflow-providers-amazon package
 Home-page: https://airflow.apache.org/
 Download-URL: https://archive.apache.org/dist/airflow/providers
 Author: Apache Software Foundation
 Author-email: dev@airflow.apache.org
 License: Apache License 2.0
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-amazon/8.4.0/
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-amazon/8.4.0/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-amazon/8.5.0/
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-amazon/8.5.0/changelog.html
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
@@ -31,9 +31,9 @@ Requires-Python: ~=3.8
 Description-Content-Type: text/x-rst
 License-File: LICENSE
 License-File: NOTICE
-Requires-Dist: apache-airflow-providers-common-sql (>=1.3.1)
+Requires-Dist: apache-airflow-providers-common-sql (>=1.3.1.dev0)
 Requires-Dist: apache-airflow-providers-http
-Requires-Dist: apache-airflow (>=2.4.0)
+Requires-Dist: apache-airflow (>=2.4.0.dev0)
 Requires-Dist: asgiref
 Requires-Dist: boto3 (>=1.24.0)
 Requires-Dist: jsonpath-ng (>=1.5.3)
@@ -66,6 +66,8 @@ Provides-Extra: microsoft.azure
 Requires-Dist: apache-airflow-providers-microsoft-azure ; extra == 'microsoft.azure'
 Provides-Extra: mongo
 Requires-Dist: apache-airflow-providers-mongo ; extra == 'mongo'
+Provides-Extra: openlineage
+Requires-Dist: apache-airflow-providers-openlineage ; extra == 'openlineage'
 Provides-Extra: pandas
 Requires-Dist: pandas (>=0.17.1) ; extra == 'pandas'
 Provides-Extra: salesforce
@@ -111,7 +113,7 @@ Requires-Dist: apache-airflow-providers-ssh ; extra == 'ssh'
 
 Package ``apache-airflow-providers-amazon``
 
-Release: ``8.4.0``
+Release: ``8.5.0rc1``
 
 
 Amazon integration (including `Amazon Web Services (AWS) <https://aws.amazon.com/>`__).
@@ -124,7 +126,7 @@ This is a provider package for ``amazon`` provider. All classes for this provide
 are in ``airflow.providers.amazon`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-amazon/8.4.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-amazon/8.5.0/>`_.
 
 
 Installation
@@ -184,9 +186,10 @@ Dependent package
 `apache-airflow-providers-imap <https://airflow.apache.org/docs/apache-airflow-providers-imap>`_                        ``imap``
 `apache-airflow-providers-microsoft-azure <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure>`_  ``microsoft.azure``
 `apache-airflow-providers-mongo <https://airflow.apache.org/docs/apache-airflow-providers-mongo>`_                      ``mongo``
+`apache-airflow-providers-openlineage <https://airflow.apache.org/docs/apache-airflow-providers-openlineage>`_          ``openlineage``
 `apache-airflow-providers-salesforce <https://airflow.apache.org/docs/apache-airflow-providers-salesforce>`_            ``salesforce``
 `apache-airflow-providers-ssh <https://airflow.apache.org/docs/apache-airflow-providers-ssh>`_                          ``ssh``
 ======================================================================================================================  ===================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-amazon/8.4.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-amazon/8.5.0/changelog.html>`_.

{apache_airflow_providers_amazon-8.4.0.dist-info → apache_airflow_providers_amazon-8.5.0rc1.dist-info}/RECORD

@@ -1,5 +1,5 @@
-airflow/providers/amazon/__init__.py,sha256=C58S6nS30Yn4Z04vEMZLHK-zKVn9GSuPZYdRQRh_NOc,1575
-airflow/providers/amazon/get_provider_info.py,sha256=oSlIPHBZgMDh2s3nVx8LtDW8Xj6zsH1SqE6ZXBbDNn0,41325
+airflow/providers/amazon/__init__.py,sha256=60eBmUeqz6lH1y1OFUDLhImPGgbEoFVY11WaSwGDKXA,1575
+airflow/providers/amazon/get_provider_info.py,sha256=-5W_isthQiQmmNCP9Wsn-tO_yigmQ-dDKWpEb9DTamY,41529
 airflow/providers/amazon/aws/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
 airflow/providers/amazon/aws/exceptions.py,sha256=UVoxpfQEdWI1319h0U78Z_r5wRFQL6DN14hJw3G1Rgo,1731
 airflow/providers/amazon/aws/hooks/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
@@ -20,7 +20,7 @@ airflow/providers/amazon/aws/hooks/ecs.py,sha256=3o2UX7XkZwcRhDCxLwqM-1hL1zDNSaP
 airflow/providers/amazon/aws/hooks/eks.py,sha256=7J1TunLNdObG0BzSjmqbEF2FWkPWoL2CKfbPskFZf5s,23958
 airflow/providers/amazon/aws/hooks/elasticache_replication_group.py,sha256=D2xrqRkHTowTEGoxPjV5UnKw9-hSFVIR0rZoWWKPGPI,12119
 airflow/providers/amazon/aws/hooks/emr.py,sha256=l9mYczK7E-lWnBozBvU-2HCQ5Q23-id6o1ikiWf8JII,20679
-airflow/providers/amazon/aws/hooks/eventbridge.py,sha256=ZZjzmEMmLUR6qHaWsYC6YOXbyFLv6xCmH-TZwyjYkPM,1099
+airflow/providers/amazon/aws/hooks/eventbridge.py,sha256=vWHZHnVwLsYmS64zSryj6VuWvjMsyiMQIIbJMFVmdSo,3389
 airflow/providers/amazon/aws/hooks/glacier.py,sha256=z4g3PCktIDjoq8O0SV_HDheGb0BNR4yy6bOlRCBH7_k,3452
 airflow/providers/amazon/aws/hooks/glue.py,sha256=KTEw7Y6RPvvP6xmPHYGfe3KMmrGtaMrPbSdHk_WWx7M,16313
 airflow/providers/amazon/aws/hooks/glue_catalog.py,sha256=4TBL4JSfcU4VILMZWL0mby-W2wha1s9Lo3ZDOeqVjkU,7616
@@ -33,7 +33,7 @@ airflow/providers/amazon/aws/hooks/rds.py,sha256=vB8GADMEyfIsrvNZB_B2aDZZB0LSFFB
 airflow/providers/amazon/aws/hooks/redshift_cluster.py,sha256=2-DjciI4d6u8PsOODxgkB3LfKwWBgt2MUQuQcETgHig,13169
 airflow/providers/amazon/aws/hooks/redshift_data.py,sha256=jro_OQravSSoMfsrnIv7chkn-WKD6Sl4CO5nswUL250,8342
 airflow/providers/amazon/aws/hooks/redshift_sql.py,sha256=II8iY4s1_uEEZmztkYipi5M5kfHmtYcurnJk9mC03oo,7160
-airflow/providers/amazon/aws/hooks/s3.py,sha256=lZBq5HTHGJA_cjbvhG6pzkLi4zdnLsfLNIGyxjNokDE,58255
+airflow/providers/amazon/aws/hooks/s3.py,sha256=BqxL4EsR6A0BszuNxzV9KCbqPzpul7tDPoctv7P2oxM,58277
 airflow/providers/amazon/aws/hooks/sagemaker.py,sha256=_nYt3jL9JyRPI4Mu4Yu5xvZPZbkNNTvkuCxkVYqMZcE,56469
 airflow/providers/amazon/aws/hooks/secrets_manager.py,sha256=BTsEHm3GTcoNnE8jBJO403q6wuGkXZFwglQ7l-P9b5M,2667
 airflow/providers/amazon/aws/hooks/ses.py,sha256=M8rSXic50lmrE4D9a34GBLTavShg4zsFgDznlhUZiNY,4146
@@ -56,7 +56,7 @@ airflow/providers/amazon/aws/notifications/chime.py,sha256=4Rg1A1GJYyMoVOUVwTN9w
 airflow/providers/amazon/aws/operators/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
 airflow/providers/amazon/aws/operators/appflow.py,sha256=r65gqA7l6W1AFX71fpWO-okwq1RadtBXi-0AjVYytg0,19662
 airflow/providers/amazon/aws/operators/athena.py,sha256=HGBIXYf2XnORLsRRsdEXRhMad9f12j3jFco-_vzYDRA,7164
-airflow/providers/amazon/aws/operators/batch.py,sha256=ZrFvFK0L6NEJyWe_mfzuWzUVMK_tPbvAXXv6flF3gO8,20436
+airflow/providers/amazon/aws/operators/batch.py,sha256=3zaieJWSZdzI7hlyygKbN9-7Fsh1mrv6AJkSUXrnNHc,20423
 airflow/providers/amazon/aws/operators/cloud_formation.py,sha256=lvK-2LQwNSKgkyWzcEyMurvaEXsdZl4uj49nyi9GLqs,3633
 airflow/providers/amazon/aws/operators/datasync.py,sha256=IEWIwuWZ5iAVY20Io4l5hV-nE2Ak9M2sxTY-7dCfJus,18421
 airflow/providers/amazon/aws/operators/dms.py,sha256=6yRpzvsJ6SS_IP1m5va7mEdkemp-7I0zIZ6u_Ve2mrs,10720
@@ -64,17 +64,17 @@ airflow/providers/amazon/aws/operators/ec2.py,sha256=a6WLow1jf9DgfO_uhh4SaNIZfXz
 airflow/providers/amazon/aws/operators/ecs.py,sha256=18ut_AmWUPHrjk3FDGIVCQ1P5CqCnfPqUXlX_9wDCf4,33315
 airflow/providers/amazon/aws/operators/eks.py,sha256=PktFILJULJ7zWetKge0pzl7nzBTArVYxg8JG2_tgoXA,50133
 airflow/providers/amazon/aws/operators/emr.py,sha256=tNS-s5lMJ735_gOi-7Sf-mrnr7gLsnjCV2JwFCiaqZQ,74988
-airflow/providers/amazon/aws/operators/eventbridge.py,sha256=_JAFe9nuFsroTQ_Xtpu-ZgfAAhzQMyWrvz4A4VLytsI,3035
+airflow/providers/amazon/aws/operators/eventbridge.py,sha256=Zn1ghxlyHhKChNMUkx5fpAGcFBScqKdZroQBz63EQic,5941
 airflow/providers/amazon/aws/operators/glacier.py,sha256=016tRPPtrvl16ixqCJRPOchp3ZpXHAj1AW-brRK9dTQ,3707
 airflow/providers/amazon/aws/operators/glue.py,sha256=2Vsv37IC-032UslGSIzxhAQIIR3mA0XzhCbhqW0er_0,9663
 airflow/providers/amazon/aws/operators/glue_crawler.py,sha256=Xui7G1DSst2Vx1LDOUfZ7trZoDZYKkaPRoVN1fztQTs,4348
 airflow/providers/amazon/aws/operators/lambda_function.py,sha256=uqV-1kSTflMo_UPBhP7oNI22wEJ4DaNixqH2Y2T1zgk,7749
 airflow/providers/amazon/aws/operators/quicksight.py,sha256=GFbMDKNkAr-87fIxJc3DMoxl30RvqS18vMhnn1nBnHY,3968
-airflow/providers/amazon/aws/operators/rds.py,sha256=GqZwQDzuiaoHNiJCT6f8jCF8SDMjladP-FVF0w3ltTM,38326
+airflow/providers/amazon/aws/operators/rds.py,sha256=RtEVzwOu3KI4IMpfKp1S3zEUnA-kmEHgrqJPhmsa24Q,38629
 airflow/providers/amazon/aws/operators/redshift_cluster.py,sha256=-F2Z0t9LFKHjF5D-RTeaJW3tCxmOqpTs8Se_QoyuC5s,32304
 airflow/providers/amazon/aws/operators/redshift_data.py,sha256=tZzGiF3Oe9J2MCtZiu3VBtFDIHlrBOuPgWTtJE1BpRE,5982
 airflow/providers/amazon/aws/operators/s3.py,sha256=AhQzivpiTRbBfy_-1vj8FfI1YXB-CGTASlg7aT0fgIM,30235
-airflow/providers/amazon/aws/operators/sagemaker.py,sha256=HvdfXUiIrHamjyx4IFLEo9t2sEaIVeeSJI0Bt-b39Ig,61246
+airflow/providers/amazon/aws/operators/sagemaker.py,sha256=SexTzcz2zTo2fkZwBz66NyJk3sak2HOJQ6w54vxegwk,66643
 airflow/providers/amazon/aws/operators/sns.py,sha256=vomOmDKeX177_MDOgulMyQVr4FqMHNhPo3-HnrfO370,2960
 airflow/providers/amazon/aws/operators/sqs.py,sha256=gi7qbdNkPTaDo63jfjHFuV4Rsegbg1_FGVd4-WGQMLU,3565
 airflow/providers/amazon/aws/operators/step_function.py,sha256=Q1z050K-Pzw4a1PglhMT-wxiLSIFNLaCusozA00vU4w,6411
@@ -101,7 +101,7 @@ airflow/providers/amazon/aws/sensors/rds.py,sha256=4mLAFLjY8yI3bExVdGATXIIiiuwZy
 airflow/providers/amazon/aws/sensors/redshift_cluster.py,sha256=B2f4y8aHNtdc6vew-XBFNWDx-YLnBRGXZkWEHZQQoRs,2623
 airflow/providers/amazon/aws/sensors/s3.py,sha256=8-H5mCzGXc1YgFRjDYNg0U37beVVqyHCWB3IiCQCVIQ,15066
 airflow/providers/amazon/aws/sensors/sagemaker.py,sha256=tECTTigli8-C8FYnPcm5U6z3aR5AHXZGx9iW9VXhGKc,12794
-airflow/providers/amazon/aws/sensors/sqs.py,sha256=XYZF64l7HgFR1Zl9TZSUVNbTuaMh6yo09T0gzBfiLF8,9543
+airflow/providers/amazon/aws/sensors/sqs.py,sha256=PD0aWAPMUVHbbwH2IJziGPwWpouUYMENPYZK8sTrE9A,9793
 airflow/providers/amazon/aws/sensors/step_function.py,sha256=P2BItKtNrAeL4wnS3RGp-Ep3crUMOfE7O9SH1vvaRCQ,3333
 airflow/providers/amazon/aws/transfers/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
 airflow/providers/amazon/aws/transfers/azure_blob_to_s3.py,sha256=Q2T0VlFOb9ZpC5b7QgvX_hWafPuUkjEnMJq8x7gcOy0,7019
@@ -109,7 +109,7 @@ airflow/providers/amazon/aws/transfers/base.py,sha256=QGrfp6mZwFO691fN1Ig0cVbJXM
 airflow/providers/amazon/aws/transfers/dynamodb_to_s3.py,sha256=T9AofurdwN9vRXTHQXA7OEkIlU5Ym5pw9553bqXU3XQ,7755
 airflow/providers/amazon/aws/transfers/exasol_to_s3.py,sha256=Hwh684UFRIfhYxHlapNuAiIse770ZmeB8Le_KMd6o04,4410
 airflow/providers/amazon/aws/transfers/ftp_to_s3.py,sha256=ZpcB8idMpcPMYcHXR6BrtgTCZo7MUI3b9HzOBphvkhM,6368
-airflow/providers/amazon/aws/transfers/gcs_to_s3.py,sha256=hd33lzoyGBedzaSfx4KbwVjJtL2G2SW65lJ-RU0GYqI,8544
+airflow/providers/amazon/aws/transfers/gcs_to_s3.py,sha256=mn-DWZBnDaKgxF1bjvIOk6x0Gx_yl0kc5360xUXdSm4,9841
 airflow/providers/amazon/aws/transfers/glacier_to_gcs.py,sha256=J4BZ5kgsl3D6kChbdfjML1DP5cC-oXQyvJNwJKnKmzc,4702
 airflow/providers/amazon/aws/transfers/google_api_to_s3.py,sha256=yYnJEqmHLG7ZSaPYcxFhQ3X5Clsd3yCQkrNO9kpFJAY,9042
 airflow/providers/amazon/aws/transfers/hive_to_dynamodb.py,sha256=uflqwFii7EYapwHyCiueFy34YrWLxmYFA8kHrggqhSc,4173
@@ -138,6 +138,7 @@ airflow/providers/amazon/aws/triggers/rds.py,sha256=oPyOnDc1vkRERtG2KHuq4fmhIWZ-
 airflow/providers/amazon/aws/triggers/redshift_cluster.py,sha256=hkjXixGT8HvejumHvJYQH2loZ4RTsJoLHWq1KcxLiqo,10704
 airflow/providers/amazon/aws/triggers/s3.py,sha256=FSNAFSNUwNsSFWGkQbNkfvRXgYZe9g-Qr6w5TYxM2m8,9085
 airflow/providers/amazon/aws/triggers/sagemaker.py,sha256=uXob6OSboFsrHSxqTe74LxdzMgMF9r5kbTiecS2eCvo,7906
+airflow/providers/amazon/aws/triggers/sqs.py,sha256=L7FvTU_EnCgKVKJUKHCwzORTnSeC4MN7yPLYrk6r5-Y,7739
 airflow/providers/amazon/aws/triggers/step_function.py,sha256=4iK7FlWd2n-PIYO1b-cMZh-gjtXQgS91FJi-dH3LL_s,2481
 airflow/providers/amazon/aws/utils/__init__.py,sha256=rGh89AYKSlXNBkmHfmDr8dYJPQ-504_emAXBHIQH-wY,2968
 airflow/providers/amazon/aws/utils/connection_wrapper.py,sha256=UEf1bH6LYdfK3hbP0kPBHWreuJl8FlDA7HMAUc8fuHY,20649
@@ -146,6 +147,7 @@ airflow/providers/amazon/aws/utils/emailer.py,sha256=fHzzErXFs74KP6VXJrCc-1T0K1s
 airflow/providers/amazon/aws/utils/rds.py,sha256=Qx5NEHGdzdmqNOtmAnSk1xErt9v6f-25a5Huos9bvPY,963
 airflow/providers/amazon/aws/utils/redshift.py,sha256=NSSw6kd-jMtXR9iSxNFet_9UUqEN1ZvNzwdA9Abhi9Y,1897
 airflow/providers/amazon/aws/utils/sagemaker.py,sha256=893W8DBPhsyPINbFph9MKKP4O_zwptse0oUWm3XtGDE,1040
+airflow/providers/amazon/aws/utils/sqs.py,sha256=nhO1NXNNOorwsP73GfJFl_EbrhKO6EqR8-WKWTZb-Bo,3271
 airflow/providers/amazon/aws/utils/tags.py,sha256=-WPb4MpzZxV4MHS6OD09EronbR_jlfuVQeEqu4cVnj0,1762
 airflow/providers/amazon/aws/utils/task_log_fetcher.py,sha256=-6yaD9kHpPfIMakkD0SD8CoZf_TBGGnMQYBuKTpcqhI,4491
 airflow/providers/amazon/aws/utils/waiter.py,sha256=FO1WupdK7Z9AonrC8w_XcRpQE7A-o4VlgaqQxV65dbk,3509
@@ -165,10 +167,10 @@ airflow/providers/amazon/aws/waiters/glue.json,sha256=xUPDsOrZ7m2O_0rSj_Dweq-EAV
 airflow/providers/amazon/aws/waiters/redshift.json,sha256=jOBotCgbkko1b_CHcGEbhhRvusgt0YSzVuFiZrqVP30,1742
 airflow/providers/amazon/aws/waiters/sagemaker.json,sha256=JPHuQtUFZ1B7EMLfVmCRevNZ9jgpB71LM0dva8ZEO9A,5254
 airflow/providers/amazon/aws/waiters/stepfunctions.json,sha256=aBaAZaGv8ZZGdN-2gvYEbq3fL_WHI_7s6SSDL-nWS1A,1034
-apache_airflow_providers_amazon-8.4.0.dist-info/LICENSE,sha256=gXPVwptPlW1TJ4HSuG5OMPg-a3h43OGMkZRR1rpwfJA,10850
-apache_airflow_providers_amazon-8.4.0.dist-info/METADATA,sha256=GOAoCoAvGebDyW4ismsmls5xZuePPqeZ5ifdd1GDxMM,9561
-apache_airflow_providers_amazon-8.4.0.dist-info/NOTICE,sha256=m-6s2XynUxVSUIxO4rVablAZCvFq-wmLrqV91DotRBw,240
-apache_airflow_providers_amazon-8.4.0.dist-info/WHEEL,sha256=AtBG6SXL3KF_v0NxLf0ehyVOh0cold-JbJYXNGorC6Q,92
-apache_airflow_providers_amazon-8.4.0.dist-info/entry_points.txt,sha256=HNciwBqNoIpFPDUUu9_tL6HyozCAxlpCifH810uQTOQ,103
-apache_airflow_providers_amazon-8.4.0.dist-info/top_level.txt,sha256=OeMVH5md7fr2QQWpnZoOWWxWO-0WH1IP70lpTVwopPg,8
-apache_airflow_providers_amazon-8.4.0.dist-info/RECORD,,
+apache_airflow_providers_amazon-8.5.0rc1.dist-info/LICENSE,sha256=gXPVwptPlW1TJ4HSuG5OMPg-a3h43OGMkZRR1rpwfJA,10850
+apache_airflow_providers_amazon-8.5.0rc1.dist-info/METADATA,sha256=RFf-KTJsd-sTzZHAydh0O9zZLV3cZYPF-BKyoM5wY8Y,9818
+apache_airflow_providers_amazon-8.5.0rc1.dist-info/NOTICE,sha256=m-6s2XynUxVSUIxO4rVablAZCvFq-wmLrqV91DotRBw,240
+apache_airflow_providers_amazon-8.5.0rc1.dist-info/WHEEL,sha256=AtBG6SXL3KF_v0NxLf0ehyVOh0cold-JbJYXNGorC6Q,92
+apache_airflow_providers_amazon-8.5.0rc1.dist-info/entry_points.txt,sha256=HNciwBqNoIpFPDUUu9_tL6HyozCAxlpCifH810uQTOQ,103
+apache_airflow_providers_amazon-8.5.0rc1.dist-info/top_level.txt,sha256=OeMVH5md7fr2QQWpnZoOWWxWO-0WH1IP70lpTVwopPg,8
+apache_airflow_providers_amazon-8.5.0rc1.dist-info/RECORD,,