apache-airflow-providers-amazon 9.2.0rc2__py3-none-any.whl → 9.3.0__py3-none-any.whl

airflow/providers/amazon/aws/operators/glue.py

@@ -87,6 +87,7 @@ class GlueJobOperator(BaseOperator):
         "script_location",
         "script_args",
         "create_job_kwargs",
+        "run_job_kwargs",
         "s3_bucket",
         "iam_role_name",
         "iam_role_arn",
@@ -95,6 +96,7 @@ class GlueJobOperator(BaseOperator):
     template_fields_renderers = {
         "script_args": "json",
         "create_job_kwargs": "json",
+        "run_job_kwargs": "json",
     }
     ui_color = "#ededed"
 
@@ -242,11 +244,11 @@ class GlueJobOperator(BaseOperator):
         return self._job_run_id
 
     def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> str:
-        event = validate_execute_complete_event(event)
+        validated_event = validate_execute_complete_event(event)
 
-        if event["status"] != "success":
-            raise AirflowException(f"Error in glue job: {event}")
-        return event["value"]
+        if validated_event["status"] != "success":
+            raise AirflowException(f"Error in glue job: {validated_event}")
+        return validated_event["value"]
 
     def on_kill(self):
         """Cancel the running AWS Glue Job."""
@@ -500,18 +502,18 @@ class GlueDataQualityRuleSetEvaluationRunOperator(AwsBaseOperator[GlueDataQualit
         return evaluation_run_id
 
     def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> str:
-        event = validate_execute_complete_event(event)
+        validated_event = validate_execute_complete_event(event)
 
-        if event["status"] != "success":
-            raise AirflowException(f"Error: AWS Glue data quality ruleset evaluation run: {event}")
+        if validated_event["status"] != "success":
+            raise AirflowException(f"Error: AWS Glue data quality ruleset evaluation run: {validated_event}")
 
         self.hook.validate_evaluation_run_results(
-            evaluation_run_id=event["evaluation_run_id"],
+            evaluation_run_id=validated_event["evaluation_run_id"],
             show_results=self.show_results,
             verify_result_status=self.verify_result_status,
         )
 
-        return event["evaluation_run_id"]
+        return validated_event["evaluation_run_id"]
 
 
 class GlueDataQualityRuleRecommendationRunOperator(AwsBaseOperator[GlueDataQualityHook]):
@@ -648,12 +650,12 @@ class GlueDataQualityRuleRecommendationRunOperator(AwsBaseOperator[GlueDataQuali
         return recommendation_run_id
 
     def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> str:
-        event = validate_execute_complete_event(event)
+        validated_event = validate_execute_complete_event(event)
 
-        if event["status"] != "success":
-            raise AirflowException(f"Error: AWS Glue data quality rule recommendation run: {event}")
+        if validated_event["status"] != "success":
+            raise AirflowException(f"Error: AWS Glue data quality rule recommendation run: {validated_event}")
 
         if self.show_results:
-            self.hook.log_recommendation_results(run_id=event["recommendation_run_id"])
+            self.hook.log_recommendation_results(run_id=validated_event["recommendation_run_id"])
 
-        return event["recommendation_run_id"]
+        return validated_event["recommendation_run_id"]

airflow/providers/amazon/aws/operators/glue_crawler.py

@@ -117,8 +117,8 @@ class GlueCrawlerOperator(AwsBaseOperator[GlueCrawlerHook]):
         return crawler_name
 
     def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> str:
-        event = validate_execute_complete_event(event)
+        validated_event = validate_execute_complete_event(event)
 
-        if event["status"] != "success":
-            raise AirflowException(f"Error in glue crawl: {event}")
+        if validated_event["status"] != "success":
+            raise AirflowException(f"Error in glue crawl: {validated_event}")
         return self.config["Name"]

airflow/providers/amazon/aws/operators/glue_databrew.py

@@ -129,13 +129,13 @@ class GlueDataBrewStartJobOperator(AwsBaseOperator[GlueDataBrewHook]):
         return {"run_id": run_id}
 
     def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> dict[str, str]:
-        event = validate_execute_complete_event(event)
+        validated_event = validate_execute_complete_event(event)
 
-        if event["status"] != "success":
-            raise AirflowException("Error while running AWS Glue DataBrew job: %s", event)
+        if validated_event["status"] != "success":
+            raise AirflowException("Error while running AWS Glue DataBrew job: %s", validated_event)
 
-        run_id = event.get("run_id", "")
-        status = event.get("status", "")
+        run_id = validated_event.get("run_id", "")
+        status = validated_event.get("status", "")
 
         self.log.info("AWS Glue DataBrew runID: %s completed with status: %s", run_id, status)
 

airflow/providers/amazon/aws/operators/kinesis_analytics.py

@@ -215,20 +215,20 @@ class KinesisAnalyticsV2StartApplicationOperator(AwsBaseOperator[KinesisAnalytic
         return {"ApplicationARN": describe_response["ApplicationDetail"]["ApplicationARN"]}
 
     def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> dict[str, Any]:
-        event = validate_execute_complete_event(event)
+        validated_event = validate_execute_complete_event(event)
 
-        if event["status"] != "success":
+        if validated_event["status"] != "success":
             raise AirflowException(
-                "Error while starting AWS Managed Service for Apache Flink application: %s", event
+                "Error while starting AWS Managed Service for Apache Flink application: %s", validated_event
             )
 
         response = self.hook.conn.describe_application(
-            ApplicationName=event["application_name"],
+            ApplicationName=validated_event["application_name"],
         )
 
         self.log.info(
             "AWS Managed Service for Apache Flink application %s started successfully.",
-            event["application_name"],
+            validated_event["application_name"],
         )
 
         return {"ApplicationARN": response["ApplicationDetail"]["ApplicationARN"]}
@@ -332,18 +332,18 @@ class KinesisAnalyticsV2StopApplicationOperator(AwsBaseOperator[KinesisAnalytics
         return {"ApplicationARN": describe_response["ApplicationDetail"]["ApplicationARN"]}
 
     def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> dict[str, Any]:
-        event = validate_execute_complete_event(event)
+        validated_event = validate_execute_complete_event(event)
 
-        if event["status"] != "success":
+        if validated_event["status"] != "success":
             raise AirflowException("Error while stopping AWS Managed Service for Apache Flink application")
 
         response = self.hook.conn.describe_application(
-            ApplicationName=event["application_name"],
+            ApplicationName=validated_event["application_name"],
         )
 
         self.log.info(
             "AWS Managed Service for Apache Flink application %s stopped successfully.",
-            event["application_name"],
+            validated_event["application_name"],
         )
 
         return {"ApplicationARN": response["ApplicationDetail"]["ApplicationARN"]}

airflow/providers/amazon/aws/operators/lambda_function.py

@@ -145,13 +145,13 @@ class LambdaCreateFunctionOperator(AwsBaseOperator[LambdaHook]):
         return response.get("FunctionArn")
 
     def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> str:
-        event = validate_execute_complete_event(event)
+        validated_event = validate_execute_complete_event(event)
 
-        if not event or event["status"] != "success":
-            raise AirflowException(f"Trigger error: event is {event}")
+        if not validated_event or validated_event["status"] != "success":
+            raise AirflowException(f"Trigger error: event is {validated_event}")
 
         self.log.info("Lambda function created successfully")
-        return event["function_arn"]
+        return validated_event["function_arn"]
 
 
 class LambdaInvokeFunctionOperator(AwsBaseOperator[LambdaHook]):

airflow/providers/amazon/aws/operators/mwaa.py

@@ -0,0 +1,109 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""This module contains AWS MWAA operators."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import TYPE_CHECKING
+
+from airflow.providers.amazon.aws.hooks.mwaa import MwaaHook
+from airflow.providers.amazon.aws.operators.base_aws import AwsBaseOperator
+from airflow.providers.amazon.aws.utils.mixins import aws_template_fields
+
+if TYPE_CHECKING:
+    from airflow.utils.context import Context
+
+
+class MwaaTriggerDagRunOperator(AwsBaseOperator[MwaaHook]):
+    """
+    Trigger a Dag Run for a Dag in an Amazon MWAA environment.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:MwaaTriggerDagRunOperator`
+
+    :param env_name: The MWAA environment name (templated)
+    :param trigger_dag_id: The ID of the DAG to be triggered (templated)
+    :param trigger_run_id: The Run ID. This together with trigger_dag_id are a unique key. (templated)
+    :param logical_date: The logical date (previously called execution date). This is the time or interval
+        covered by this DAG run, according to the DAG definition. This together with trigger_dag_id are a
+        unique key. (templated)
+    :param data_interval_start: The beginning of the interval the DAG run covers
+    :param data_interval_end: The end of the interval the DAG run covers
+    :param conf: Additional configuration parameters. The value of this field can be set only when creating
+        the object. (templated)
+    :param note: Contains manually entered notes by the user about the DagRun. (templated)
+    """
+
+    aws_hook_class = MwaaHook
+    template_fields: Sequence[str] = aws_template_fields(
+        "env_name",
+        "trigger_dag_id",
+        "trigger_run_id",
+        "logical_date",
+        "data_interval_start",
+        "data_interval_end",
+        "conf",
+        "note",
+    )
+    template_fields_renderers = {"conf": "json"}
+
+    def __init__(
+        self,
+        *,
+        env_name: str,
+        trigger_dag_id: str,
+        trigger_run_id: str | None = None,
+        logical_date: str | None = None,
+        data_interval_start: str | None = None,
+        data_interval_end: str | None = None,
+        conf: dict | None = None,
+        note: str | None = None,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.env_name = env_name
+        self.trigger_dag_id = trigger_dag_id
+        self.trigger_run_id = trigger_run_id
+        self.logical_date = logical_date
+        self.data_interval_start = data_interval_start
+        self.data_interval_end = data_interval_end
+        self.conf = conf if conf else {}
+        self.note = note
+
+    def execute(self, context: Context) -> dict:
+        """
+        Trigger a Dag Run for the Dag in the Amazon MWAA environment.
+
+        :param context: the Context object
+        :return: dict with information about the Dag run
+            For details of the returned dict, see :py:meth:`botocore.client.MWAA.invoke_rest_api`
+        """
+        return self.hook.invoke_rest_api(
+            env_name=self.env_name,
+            path=f"/dags/{self.trigger_dag_id}/dagRuns",
+            method="POST",
+            body={
+                "dag_run_id": self.trigger_run_id,
+                "logical_date": self.logical_date,
+                "data_interval_start": self.data_interval_start,
+                "data_interval_end": self.data_interval_end,
+                "conf": self.conf,
+                "note": self.note,
+            },
+        )

airflow/providers/amazon/aws/operators/rds.py

@@ -627,12 +627,12 @@ class RdsCreateDbInstanceOperator(RdsBaseOperator):
         return json.dumps(create_db_instance, default=str)
 
     def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> str:
-        event = validate_execute_complete_event(event)
+        validated_event = validate_execute_complete_event(event)
 
-        if event["status"] != "success":
-            raise AirflowException(f"DB instance creation failed: {event}")
+        if validated_event["status"] != "success":
+            raise AirflowException(f"DB instance creation failed: {validated_event}")
 
-        return json.dumps(event["response"], default=str)
+        return json.dumps(validated_event["response"], default=str)
 
 
 class RdsDeleteDbInstanceOperator(RdsBaseOperator):
@@ -712,12 +712,12 @@ class RdsDeleteDbInstanceOperator(RdsBaseOperator):
         return json.dumps(delete_db_instance, default=str)
 
     def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> str:
-        event = validate_execute_complete_event(event)
+        validated_event = validate_execute_complete_event(event)
 
-        if event["status"] != "success":
-            raise AirflowException(f"DB instance deletion failed: {event}")
+        if validated_event["status"] != "success":
+            raise AirflowException(f"DB instance deletion failed: {validated_event}")
 
-        return json.dumps(event["response"], default=str)
+        return json.dumps(validated_event["response"], default=str)
 
 
 class RdsStartDbOperator(RdsBaseOperator):
@@ -779,12 +779,12 @@ class RdsStartDbOperator(RdsBaseOperator):
         return json.dumps(start_db_response, default=str)
 
     def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> str:
-        event = validate_execute_complete_event(event)
+        validated_event = validate_execute_complete_event(event)
 
-        if event["status"] != "success":
-            raise AirflowException(f"Failed to start DB: {event}")
+        if validated_event["status"] != "success":
+            raise AirflowException(f"Failed to start DB: {validated_event}")
 
-        return json.dumps(event["response"], default=str)
+        return json.dumps(validated_event["response"], default=str)
 
     def _start_db(self):
         self.log.info("Starting DB %s '%s'", self.db_type.value, self.db_identifier)
@@ -891,12 +891,12 @@ class RdsStopDbOperator(RdsBaseOperator):
         return json.dumps(stop_db_response, default=str)
 
     def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> str:
-        event = validate_execute_complete_event(event)
+        validated_event = validate_execute_complete_event(event)
 
-        if event["status"] != "success":
-            raise AirflowException(f"Failed to start DB: {event}")
+        if validated_event["status"] != "success":
+            raise AirflowException(f"Failed to start DB: {validated_event}")
 
-        return json.dumps(event["response"], default=str)
+        return json.dumps(validated_event["response"], default=str)
 
     def _stop_db(self):
         self.log.info("Stopping DB %s '%s'", self.db_type.value, self.db_identifier)

airflow/providers/amazon/aws/operators/redshift_cluster.py

@@ -321,10 +321,10 @@ class RedshiftCreateClusterOperator(BaseOperator):
         self.log.info(cluster)
 
     def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> None:
-        event = validate_execute_complete_event(event)
+        validated_event = validate_execute_complete_event(event)
 
-        if event["status"] != "success":
-            raise AirflowException(f"Error creating cluster: {event}")
+        if validated_event["status"] != "success":
+            raise AirflowException(f"Error creating cluster: {validated_event}")
 
 
 class RedshiftCreateClusterSnapshotOperator(BaseOperator):
@@ -417,10 +417,10 @@ class RedshiftCreateClusterSnapshotOperator(BaseOperator):
             )
 
     def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> None:
-        event = validate_execute_complete_event(event)
+        validated_event = validate_execute_complete_event(event)
 
-        if event["status"] != "success":
-            raise AirflowException(f"Error creating snapshot: {event}")
+        if validated_event["status"] != "success":
+            raise AirflowException(f"Error creating snapshot: {validated_event}")
 
         self.log.info("Cluster snapshot created.")
 
@@ -577,10 +577,10 @@ class RedshiftResumeClusterOperator(BaseOperator):
                 )
 
     def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> None:
-        event = validate_execute_complete_event(event)
+        validated_event = validate_execute_complete_event(event)
 
-        if event["status"] != "success":
-            raise AirflowException(f"Error resuming cluster: {event}")
+        if validated_event["status"] != "success":
+            raise AirflowException(f"Error resuming cluster: {validated_event}")
         self.log.info("Resumed cluster successfully")
 
 
@@ -683,10 +683,10 @@ class RedshiftPauseClusterOperator(BaseOperator):
                 )
 
     def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> None:
-        event = validate_execute_complete_event(event)
+        validated_event = validate_execute_complete_event(event)
 
-        if event["status"] != "success":
-            raise AirflowException(f"Error pausing cluster: {event}")
+        if validated_event["status"] != "success":
+            raise AirflowException(f"Error pausing cluster: {validated_event}")
         self.log.info("Paused cluster successfully")
 
 
@@ -792,9 +792,9 @@ class RedshiftDeleteClusterOperator(BaseOperator):
             )
 
     def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> None:
-        event = validate_execute_complete_event(event)
+        validated_event = validate_execute_complete_event(event)
 
-        if event["status"] != "success":
-            raise AirflowException(f"Error deleting cluster: {event}")
+        if validated_event["status"] != "success":
+            raise AirflowException(f"Error deleting cluster: {validated_event}")
 
         self.log.info("Cluster deleted successfully")

airflow/providers/amazon/aws/operators/redshift_data.py

@@ -185,13 +185,13 @@ class RedshiftDataOperator(AwsBaseOperator[RedshiftDataHook]):
     def execute_complete(
         self, context: Context, event: dict[str, Any] | None = None
     ) -> list[GetStatementResultResponseTypeDef] | list[str]:
-        event = validate_execute_complete_event(event)
+        validated_event = validate_execute_complete_event(event)
 
-        if event["status"] == "error":
-            msg = f"context: {context}, error message: {event['message']}"
+        if validated_event["status"] == "error":
+            msg = f"context: {context}, error message: {validated_event['message']}"
             raise AirflowException(msg)
 
-        statement_id = event["statement_id"]
+        statement_id = validated_event["statement_id"]
         if not statement_id:
             raise AirflowException("statement_id should not be empty.")
 

airflow/providers/amazon/aws/operators/sagemaker.py

@@ -19,6 +19,7 @@ from __future__ import annotations
 import datetime
 import json
 import time
+import urllib
 from collections.abc import Sequence
 from functools import cached_property
 from typing import TYPE_CHECKING, Any, Callable, ClassVar
@@ -34,6 +35,7 @@ from airflow.providers.amazon.aws.hooks.sagemaker import (
     SageMakerHook,
     secondary_training_status_message,
 )
+from airflow.providers.amazon.aws.links.sagemaker import SageMakerTransformJobLink
 from airflow.providers.amazon.aws.triggers.sagemaker import (
     SageMakerPipelineTrigger,
     SageMakerTrigger,
@@ -165,7 +167,11 @@ class SageMakerBaseOperator(BaseOperator):
             if fail_if_exists:
                 raise AirflowException(f"A SageMaker {resource_type} with name {name} already exists.")
             else:
-                name = f"{proposed_name}-{time.time_ns()//1000000}"
+                max_name_len = 63
+                timestamp = str(
+                    time.time_ns() // 1000000000
+                )  # only keep the relevant datetime (first 10 digits)
+                name = f"{proposed_name[:max_name_len - len(timestamp) - 1]}-{timestamp}"  # we subtract one to make provision for the dash between the truncated name and timestamp
                 self.log.info("Changed %s name to '%s' to avoid collision.", resource_type, name)
         return name
 
@@ -346,13 +352,13 @@ class SageMakerProcessingOperator(SageMakerBaseOperator):
         return {"Processing": self.serialized_job}
 
     def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> dict[str, dict]:
-        event = validate_execute_complete_event(event)
+        validated_event = validate_execute_complete_event(event)
 
-        if event["status"] != "success":
-            raise AirflowException(f"Error while running job: {event}")
+        if validated_event["status"] != "success":
+            raise AirflowException(f"Error while running job: {validated_event}")
 
-        self.log.info(event["message"])
-        self.serialized_job = serialize(self.hook.describe_processing_job(event["job_name"]))
+        self.log.info(validated_event["message"])
+        self.serialized_job = serialize(self.hook.describe_processing_job(validated_event["job_name"]))
         self.log.info("%s completed successfully.", self.task_id)
         return {"Processing": self.serialized_job}
 
@@ -599,12 +605,12 @@ class SageMakerEndpointOperator(SageMakerBaseOperator):
         }
 
     def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> dict[str, dict]:
-        event = validate_execute_complete_event(event)
+        validated_event = validate_execute_complete_event(event)
 
-        if event["status"] != "success":
-            raise AirflowException(f"Error while running job: {event}")
+        if validated_event["status"] != "success":
+            raise AirflowException(f"Error while running job: {validated_event}")
 
-        response = self.hook.describe_endpoint(event["job_name"])
+        response = self.hook.describe_endpoint(validated_event["job_name"])
         return {
             "EndpointConfig": serialize(self.hook.describe_endpoint_config(response["EndpointConfigName"])),
             "Endpoint": serialize(self.hook.describe_endpoint(response["EndpointName"])),
@@ -655,6 +661,8 @@ class SageMakerTransformOperator(SageMakerBaseOperator):
     :return Dict: Returns The ARN of the model created in Amazon SageMaker.
     """
 
+    operator_extra_links = (SageMakerTransformJobLink(),)
+
     def __init__(
         self,
         *,
@@ -761,6 +769,21 @@ class SageMakerTransformOperator(SageMakerBaseOperator):
         if response["ResponseMetadata"]["HTTPStatusCode"] != 200:
             raise AirflowException(f"Sagemaker transform Job creation failed: {response}")
 
+        transform_job_url = SageMakerTransformJobLink.format_str.format(
+            aws_domain=SageMakerTransformJobLink.get_aws_domain(self.hook.conn_partition),
+            region_name=self.hook.conn_region_name,
+            job_name=urllib.parse.quote(transform_config["TransformJobName"], safe=""),
+        )
+        SageMakerTransformJobLink.persist(
+            context=context,
+            operator=self,
+            region_name=self.hook.conn_region_name,
+            aws_partition=self.hook.conn_partition,
+            job_name=urllib.parse.quote(transform_config["TransformJobName"], safe=""),
+        )
+
+        self.log.info("You can monitor this SageMaker Transform job at %s", transform_job_url)
+
         if self.deferrable and self.wait_for_completion:
             response = self.hook.describe_transform_job(transform_config["TransformJobName"])
             status = response["TransformJobStatus"]
@@ -804,10 +827,10 @@ class SageMakerTransformOperator(SageMakerBaseOperator):
         return self._check_if_resource_exists(model_name, "model", describe_func)
 
     def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> dict[str, dict]:
-        event = validate_execute_complete_event(event)
+        validated_event = validate_execute_complete_event(event)
 
-        self.log.info(event["message"])
-        return self.serialize_result(event["job_name"])
+        self.log.info(validated_event["message"])
+        return self.serialize_result(validated_event["job_name"])
 
     def serialize_result(self, job_name: str) -> dict[str, dict]:
         job_description = self.hook.describe_transform_job(job_name)
@@ -976,11 +999,11 @@ class SageMakerTuningOperator(SageMakerBaseOperator):
         return {"Tuning": serialize(description)}
 
     def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> dict[str, dict]:
-        event = validate_execute_complete_event(event)
+        validated_event = validate_execute_complete_event(event)
 
-        if event["status"] != "success":
-            raise AirflowException(f"Error while running job: {event}")
-        return {"Tuning": serialize(self.hook.describe_tuning_job(event["job_name"]))}
+        if validated_event["status"] != "success":
+            raise AirflowException(f"Error while running job: {validated_event}")
+        return {"Tuning": serialize(self.hook.describe_tuning_job(validated_event["job_name"]))}
 
 
 class SageMakerModelOperator(SageMakerBaseOperator):
@@ -1188,13 +1211,13 @@ class SageMakerTrainingOperator(SageMakerBaseOperator):
         return self.serialize_result(self.config["TrainingJobName"])
 
     def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> dict[str, dict]:
-        event = validate_execute_complete_event(event)
+        validated_event = validate_execute_complete_event(event)
 
-        if event["status"] != "success":
-            raise AirflowException(f"Error while running job: {event}")
+        if validated_event["status"] != "success":
+            raise AirflowException(f"Error while running job: {validated_event}")
 
-        self.log.info(event["message"])
-        return self.serialize_result(event["job_name"])
+        self.log.info(validated_event["message"])
+        return self.serialize_result(validated_event["job_name"])
 
     def serialize_result(self, job_name: str) -> dict[str, dict]:
         self.serialized_training_data = serialize(self.hook.describe_training_job(job_name))
@@ -1330,11 +1353,11 @@ class SageMakerStartPipelineOperator(SageMakerBaseOperator):
         return arn
 
     def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> str:
-        event = validate_execute_complete_event(event)
+        validated_event = validate_execute_complete_event(event)
 
-        if event["status"] != "success":
-            raise AirflowException(f"Failure during pipeline execution: {event}")
-        return event["value"]
+        if validated_event["status"] != "success":
+            raise AirflowException(f"Failure during pipeline execution: {validated_event}")
+        return validated_event["value"]
 
 
 class SageMakerStopPipelineOperator(SageMakerBaseOperator):
@@ -1425,10 +1448,10 @@ class SageMakerStopPipelineOperator(SageMakerBaseOperator):
         return status
 
     def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> str:
-        event = validate_execute_complete_event(event)
+        validated_event = validate_execute_complete_event(event)
 
-        if event["status"] != "success":
-            raise AirflowException(f"Failure during pipeline execution: {event}")
+        if validated_event["status"] != "success":
+            raise AirflowException(f"Failure during pipeline execution: {validated_event}")
 
         # theoretically we should do a `describe` call to know this,
         # but if we reach this point, this is the only possible status

airflow/providers/amazon/aws/operators/sqs.py

@@ -44,6 +44,8 @@ class SqsPublishOperator(AwsBaseOperator[SqsHook]):
     :param delay_seconds: message delay (templated) (default: 1 second)
     :param message_group_id: This parameter applies only to FIFO (first-in-first-out) queues. (default: None)
         For details of the attributes parameter see :py:meth:`botocore.client.SQS.send_message`
+    :param message_deduplication_id: This applies only to FIFO (first-in-first-out) queues.
+        For details of the attributes parameter see :py:meth:`botocore.client.SQS.send_message`
     :param aws_conn_id: The Airflow connection used for AWS credentials.
         If this is ``None`` or empty then the default boto3 behaviour is used. If
         running Airflow in a distributed manner and aws_conn_id is None or
@@ -63,6 +65,7 @@ class SqsPublishOperator(AwsBaseOperator[SqsHook]):
         "delay_seconds",
         "message_attributes",
         "message_group_id",
+        "message_deduplication_id",
     )
     template_fields_renderers = {"message_attributes": "json"}
     ui_color = "#6ad3fa"
@@ -75,6 +78,7 @@ class SqsPublishOperator(AwsBaseOperator[SqsHook]):
         message_attributes: dict | None = None,
         delay_seconds: int = 0,
         message_group_id: str | None = None,
+        message_deduplication_id: str | None = None,
         **kwargs,
     ):
         super().__init__(**kwargs)
@@ -83,6 +87,7 @@ class SqsPublishOperator(AwsBaseOperator[SqsHook]):
         self.delay_seconds = delay_seconds
         self.message_attributes = message_attributes or {}
         self.message_group_id = message_group_id
+        self.message_deduplication_id = message_deduplication_id
 
     def execute(self, context: Context) -> dict:
         """
@@ -98,6 +103,7 @@ class SqsPublishOperator(AwsBaseOperator[SqsHook]):
             delay_seconds=self.delay_seconds,
             message_attributes=self.message_attributes,
             message_group_id=self.message_group_id,
+            message_deduplication_id=self.message_deduplication_id,
         )
 
         self.log.info("send_message result: %s", result)