apache-airflow-providers-amazon 8.27.0rc1__py3-none-any.whl → 8.27.0rc2__py3-none-any.whl

airflow/providers/amazon/aws/operators/emr.py

@@ -1382,30 +1382,30 @@ class EmrServerlessStartJobOperator(BaseOperator):
 
         self.persist_links(context)
 
-        if self.deferrable:
-            self.defer(
-                trigger=EmrServerlessStartJobTrigger(
-                    application_id=self.application_id,
-                    job_id=self.job_id,
-                    waiter_delay=self.waiter_delay,
-                    waiter_max_attempts=self.waiter_max_attempts,
-                    aws_conn_id=self.aws_conn_id,
-                ),
-                method_name="execute_complete",
-                timeout=timedelta(seconds=self.waiter_max_attempts * self.waiter_delay),
-            )
-
         if self.wait_for_completion:
-            waiter = self.hook.get_waiter("serverless_job_completed")
-            wait(
-                waiter=waiter,
-                waiter_max_attempts=self.waiter_max_attempts,
-                waiter_delay=self.waiter_delay,
-                args={"applicationId": self.application_id, "jobRunId": self.job_id},
-                failure_message="Serverless Job failed",
-                status_message="Serverless Job status is",
-                status_args=["jobRun.state", "jobRun.stateDetails"],
-            )
+            if self.deferrable:
+                self.defer(
+                    trigger=EmrServerlessStartJobTrigger(
+                        application_id=self.application_id,
+                        job_id=self.job_id,
+                        waiter_delay=self.waiter_delay,
+                        waiter_max_attempts=self.waiter_max_attempts,
+                        aws_conn_id=self.aws_conn_id,
+                    ),
+                    method_name="execute_complete",
+                    timeout=timedelta(seconds=self.waiter_max_attempts * self.waiter_delay),
+                )
+            else:
+                waiter = self.hook.get_waiter("serverless_job_completed")
+                wait(
+                    waiter=waiter,
+                    waiter_max_attempts=self.waiter_max_attempts,
+                    waiter_delay=self.waiter_delay,
+                    args={"applicationId": self.application_id, "jobRunId": self.job_id},
+                    failure_message="Serverless Job failed",
+                    status_message="Serverless Job status is",
+                    status_args=["jobRun.state", "jobRun.stateDetails"],
+                )
 
         return self.job_id
 

airflow/providers/amazon/aws/operators/redshift_cluster.py

@@ -32,6 +32,7 @@ from airflow.providers.amazon.aws.triggers.redshift_cluster import (
     RedshiftResumeClusterTrigger,
 )
 from airflow.providers.amazon.aws.utils import validate_execute_complete_event
+from airflow.utils.helpers import prune_dict
 
 if TYPE_CHECKING:
     from airflow.utils.context import Context
@@ -507,8 +508,8 @@ class RedshiftResumeClusterOperator(BaseOperator):
         aws_conn_id: str | None = "aws_default",
         wait_for_completion: bool = False,
         deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
-        poll_interval: int = 10,
-        max_attempts: int = 10,
+        poll_interval: int = 30,
+        max_attempts: int = 30,
         **kwargs,
     ):
         super().__init__(**kwargs)
@@ -542,38 +543,38 @@ class RedshiftResumeClusterOperator(BaseOperator):
                 else:
                     raise error
 
-        if self.deferrable:
-            cluster_state = redshift_hook.cluster_status(cluster_identifier=self.cluster_identifier)
-            if cluster_state == "available":
-                self.log.info("Resumed cluster successfully")
-            elif cluster_state == "deleting":
-                raise AirflowException(
-                    "Unable to resume cluster since cluster is currently in status: %s", cluster_state
-                )
+        if self.wait_for_completion:
+            if self.deferrable:
+                cluster_state = redshift_hook.cluster_status(cluster_identifier=self.cluster_identifier)
+                if cluster_state == "available":
+                    self.log.info("Resumed cluster successfully")
+                elif cluster_state == "deleting":
+                    raise AirflowException(
+                        "Unable to resume cluster since cluster is currently in status: %s", cluster_state
+                    )
+                else:
+                    self.defer(
+                        trigger=RedshiftResumeClusterTrigger(
+                            cluster_identifier=self.cluster_identifier,
+                            waiter_delay=self.poll_interval,
+                            waiter_max_attempts=self.max_attempts,
+                            aws_conn_id=self.aws_conn_id,
+                        ),
+                        method_name="execute_complete",
+                        # timeout is set to ensure that if a trigger dies, the timeout does not restart
+                        # 60 seconds is added to allow the trigger to exit gracefully (i.e. yield TriggerEvent)
+                        timeout=timedelta(seconds=self.max_attempts * self.poll_interval + 60),
+                    )
             else:
-                self.defer(
-                    trigger=RedshiftResumeClusterTrigger(
-                        cluster_identifier=self.cluster_identifier,
-                        waiter_delay=self.poll_interval,
-                        waiter_max_attempts=self.max_attempts,
-                        aws_conn_id=self.aws_conn_id,
-                    ),
-                    method_name="execute_complete",
-                    # timeout is set to ensure that if a trigger dies, the timeout does not restart
-                    # 60 seconds is added to allow the trigger to exit gracefully (i.e. yield TriggerEvent)
-                    timeout=timedelta(seconds=self.max_attempts * self.poll_interval + 60),
+                waiter = redshift_hook.get_waiter("cluster_resumed")
+                waiter.wait(
+                    ClusterIdentifier=self.cluster_identifier,
+                    WaiterConfig={
+                        "Delay": self.poll_interval,
+                        "MaxAttempts": self.max_attempts,
+                    },
                 )
 
-        if self.wait_for_completion:
-            waiter = redshift_hook.get_waiter("cluster_resumed")
-            waiter.wait(
-                ClusterIdentifier=self.cluster_identifier,
-                WaiterConfig={
-                    "Delay": self.poll_interval,
-                    "MaxAttempts": self.max_attempts,
-                },
-            )
-
     def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> None:
         event = validate_execute_complete_event(event)
 
@@ -596,6 +597,7 @@ class RedshiftPauseClusterOperator(BaseOperator):
         running Airflow in a distributed manner and aws_conn_id is None or
         empty, then default boto3 configuration would be used (and must be
         maintained on each worker node).
+    :param wait_for_completion: If True, waits for the cluster to be paused. (default: False)
     :param deferrable: Run operator in the deferrable mode
     :param poll_interval: Time (in seconds) to wait between two consecutive calls to check cluster state
     :param max_attempts: Maximum number of attempts to poll the cluster
@@ -610,14 +612,16 @@ class RedshiftPauseClusterOperator(BaseOperator):
         *,
         cluster_identifier: str,
         aws_conn_id: str | None = "aws_default",
+        wait_for_completion: bool = False,
         deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
-        poll_interval: int = 10,
-        max_attempts: int = 15,
+        poll_interval: int = 30,
+        max_attempts: int = 30,
         **kwargs,
     ):
         super().__init__(**kwargs)
         self.cluster_identifier = cluster_identifier
         self.aws_conn_id = aws_conn_id
+        self.wait_for_completion = wait_for_completion
         self.deferrable = deferrable
         self.max_attempts = max_attempts
         self.poll_interval = poll_interval
@@ -643,26 +647,38 @@ class RedshiftPauseClusterOperator(BaseOperator):
                     time.sleep(self._attempt_interval)
                 else:
                     raise error
-        if self.deferrable:
-            cluster_state = redshift_hook.cluster_status(cluster_identifier=self.cluster_identifier)
-            if cluster_state == "paused":
-                self.log.info("Paused cluster successfully")
-            elif cluster_state == "deleting":
-                raise AirflowException(
-                    f"Unable to pause cluster since cluster is currently in status: {cluster_state}"
-                )
+        if self.wait_for_completion:
+            if self.deferrable:
+                cluster_state = redshift_hook.cluster_status(cluster_identifier=self.cluster_identifier)
+                if cluster_state == "paused":
+                    self.log.info("Paused cluster successfully")
+                elif cluster_state == "deleting":
+                    raise AirflowException(
+                        f"Unable to pause cluster since cluster is currently in status: {cluster_state}"
+                    )
+                else:
+                    self.defer(
+                        trigger=RedshiftPauseClusterTrigger(
+                            cluster_identifier=self.cluster_identifier,
+                            waiter_delay=self.poll_interval,
+                            waiter_max_attempts=self.max_attempts,
+                            aws_conn_id=self.aws_conn_id,
+                        ),
+                        method_name="execute_complete",
+                        # timeout is set to ensure that if a trigger dies, the timeout does not restart
+                        # 60 seconds is added to allow the trigger to exit gracefully (i.e. yield TriggerEvent)
+                        timeout=timedelta(seconds=self.max_attempts * self.poll_interval + 60),
+                    )
             else:
-                self.defer(
-                    trigger=RedshiftPauseClusterTrigger(
-                        cluster_identifier=self.cluster_identifier,
-                        waiter_delay=self.poll_interval,
-                        waiter_max_attempts=self.max_attempts,
-                        aws_conn_id=self.aws_conn_id,
+                waiter = redshift_hook.get_waiter("cluster_paused")
+                waiter.wait(
+                    ClusterIdentifier=self.cluster_identifier,
+                    WaiterConfig=prune_dict(
+                        {
+                            "Delay": self.poll_interval,
+                            "MaxAttempts": self.max_attempts,
+                        }
                     ),
-                    method_name="execute_complete",
-                    # timeout is set to ensure that if a trigger dies, the timeout does not restart
-                    # 60 seconds is added to allow the trigger to exit gracefully (i.e. yield TriggerEvent)
-                    timeout=timedelta(seconds=self.max_attempts * self.poll_interval + 60),
                 )
 
     def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> None:

airflow/providers/amazon/aws/operators/sagemaker.py

@@ -36,7 +36,6 @@ from airflow.providers.amazon.aws.hooks.sagemaker import (
 )
 from airflow.providers.amazon.aws.triggers.sagemaker import (
     SageMakerPipelineTrigger,
-    SageMakerTrainingPrintLogTrigger,
     SageMakerTrigger,
 )
 from airflow.providers.amazon.aws.utils import trim_none_values, validate_execute_complete_event
@@ -1195,25 +1194,15 @@ class SageMakerTrainingOperator(SageMakerBaseOperator):
             if self.max_ingestion_time:
                 timeout = datetime.timedelta(seconds=self.max_ingestion_time)
 
-            trigger: SageMakerTrainingPrintLogTrigger | SageMakerTrigger
-            if self.print_log:
-                trigger = SageMakerTrainingPrintLogTrigger(
-                    job_name=self.config["TrainingJobName"],
-                    poke_interval=self.check_interval,
-                    aws_conn_id=self.aws_conn_id,
-                )
-            else:
-                trigger = SageMakerTrigger(
+            self.defer(
+                timeout=timeout,
+                trigger=SageMakerTrigger(
                     job_name=self.config["TrainingJobName"],
                     job_type="Training",
                     poke_interval=self.check_interval,
                     max_attempts=self.max_attempts,
                     aws_conn_id=self.aws_conn_id,
-                )
-
-            self.defer(
-                timeout=timeout,
-                trigger=trigger,
+                ),
                 method_name="execute_complete",
             )
 

airflow/providers/amazon/aws/triggers/sagemaker.py

@@ -25,8 +25,9 @@ from functools import cached_property
 from typing import Any, AsyncIterator
 
 from botocore.exceptions import WaiterError
+from deprecated import deprecated
 
-from airflow.exceptions import AirflowException
+from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
 from airflow.providers.amazon.aws.hooks.sagemaker import LogState, SageMakerHook
 from airflow.providers.amazon.aws.utils.waiter_with_logging import async_wait
 from airflow.triggers.base import BaseTrigger, TriggerEvent
@@ -199,6 +200,13 @@ class SageMakerPipelineTrigger(BaseTrigger):
             raise AirflowException("Waiter error: max attempts reached")
 
 
+@deprecated(
+    reason=(
+        "`airflow.providers.amazon.aws.triggers.sagemaker.SageMakerTrainingPrintLogTrigger` "
+        "has been deprecated and will be removed in future. Please use ``SageMakerTrigger`` instead."
+    ),
+    category=AirflowProviderDeprecationWarning,
+)
 class SageMakerTrainingPrintLogTrigger(BaseTrigger):
     """
     SageMakerTrainingPrintLogTrigger is fired as deferred class with params to run the task in triggerer.

{apache_airflow_providers_amazon-8.27.0rc1.dist-info → apache_airflow_providers_amazon-8.27.0rc2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apache-airflow-providers-amazon
-Version: 8.27.0rc1
+Version: 8.27.0rc2
 Summary: Provider package apache-airflow-providers-amazon for Apache Airflow
 Keywords: airflow-provider,amazon,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -125,7 +125,7 @@ Provides-Extra: ssh
 
 Package ``apache-airflow-providers-amazon``
 
-Release: ``8.27.0.rc1``
+Release: ``8.27.0.rc2``
 
 
 Amazon integration (including `Amazon Web Services (AWS) <https://aws.amazon.com/>`__).

{apache_airflow_providers_amazon-8.27.0rc1.dist-info → apache_airflow_providers_amazon-8.27.0rc2.dist-info}/RECORD

@@ -114,7 +114,7 @@ airflow/providers/amazon/aws/operators/dms.py,sha256=6RhUtbELAjp0LLkUWl73kdcH4MR
 airflow/providers/amazon/aws/operators/ec2.py,sha256=aQj6cL3nZzu0tcn3dq6RBSPsByZe8fNtn6qcpQYtlNI,17051
 airflow/providers/amazon/aws/operators/ecs.py,sha256=pmm05ugkdyUNwkfY1X-mQdT8pqui0qJumwt92tEaEgU,32601
 airflow/providers/amazon/aws/operators/eks.py,sha256=0xHtPzxZx3ymr6-iqGvAoZsnA24PrjYRSAiFzuxrl3U,50761
-airflow/providers/amazon/aws/operators/emr.py,sha256=tygUmL90ebpita9eKg7sMygHWW1BVomGfc_wnaHjOpE,84804
+airflow/providers/amazon/aws/operators/emr.py,sha256=XNafysiROWBcMxAPK_9MQplcY-itOivKYnljffQo6eE,84909
 airflow/providers/amazon/aws/operators/eventbridge.py,sha256=e686XFhVi54DbaCk7oVc0fhvH6GIPU3p8jgyCie1yBU,10394
 airflow/providers/amazon/aws/operators/glacier.py,sha256=zxwC6lLk6sWerjlogXq6HgNOJx4h0hkqpGpqn23hJWk,3654
 airflow/providers/amazon/aws/operators/glue.py,sha256=m8hdF6eTyzsK3onOqt6Td0dGshhgf_XU1f4EtMb42LU,28390
@@ -125,10 +125,10 @@ airflow/providers/amazon/aws/operators/lambda_function.py,sha256=96KtK5KUpMPW2i8
 airflow/providers/amazon/aws/operators/neptune.py,sha256=on5oNX5K4yHfW1POE0eeZujta71vkJdVL07vucGjX-4,14751
 airflow/providers/amazon/aws/operators/quicksight.py,sha256=jc3Eof19UfLt5IqbQswRzaHaK8h0ACLY99i_1Prtq10,4089
 airflow/providers/amazon/aws/operators/rds.py,sha256=zGiUIwpO2EdUByCYkgFwLbFCQhg_TCTTHVuNnee6X_g,39325
-airflow/providers/amazon/aws/operators/redshift_cluster.py,sha256=tYTmkXbITvRMaf2R8m9Cx5f2zYF2w1whNfO-l08wKjY,35755
+airflow/providers/amazon/aws/operators/redshift_cluster.py,sha256=rmBHCssxrYEJ8EnENY-AnzC004lbtHvxXHpy69sHtV0,36681
 airflow/providers/amazon/aws/operators/redshift_data.py,sha256=wK-vTDcn0MqOuF9e-71JYIEkLKihah6oGU-p_8VT2HI,8612
 airflow/providers/amazon/aws/operators/s3.py,sha256=8mzkCBGnAbb5CHf0PFokQwdEmio_D3u99JnNoqTBUW4,35844
-airflow/providers/amazon/aws/operators/sagemaker.py,sha256=MJ-LceYW72Mqtpy5rb_i_oxG9BCUAXrAc7IQOatR85Y,84053
+airflow/providers/amazon/aws/operators/sagemaker.py,sha256=0ZP6vyPHMOVkco80sViyDtwhguawSJFo5l1GLOvunQc,83614
 airflow/providers/amazon/aws/operators/sns.py,sha256=Rttd015UhLo4pCplGybxtLhflyu_26IFzYP7WTmQFk8,3730
 airflow/providers/amazon/aws/operators/sqs.py,sha256=0KkhhIblMggNHLxAyrv5dbWcaXvdSWQA2AOQP2CzOlo,4327
 airflow/providers/amazon/aws/operators/step_function.py,sha256=eXZAxZqG5VNPaFVEchyL4vKmOh54jc83ZjrIZDeld34,9515
@@ -207,7 +207,7 @@ airflow/providers/amazon/aws/triggers/rds.py,sha256=i7FvDkHjg7yr-PQsl2ymn_Lu8B2Z
 airflow/providers/amazon/aws/triggers/redshift_cluster.py,sha256=4VlZgtpfCbe9C2d8IW2qK6QmvFIhs8Guciuq8TZ5GbU,12920
 airflow/providers/amazon/aws/triggers/redshift_data.py,sha256=hRWsyFVt0F6yEMnSb-5pZekUcuWhnpw8tNXzf8Im6cI,4245
 airflow/providers/amazon/aws/triggers/s3.py,sha256=J2MvoiQoQ3GKf8OtFF9awvY6SwVXnhKG_m3ZJ1Z7iwU,9444
-airflow/providers/amazon/aws/triggers/sagemaker.py,sha256=C52lhVsiDEiw-1Kswc0kvNDOyPdDvV5bF7vjbuBN45o,11542
+airflow/providers/amazon/aws/triggers/sagemaker.py,sha256=a8gck8xLEV1vQsp1mu8a6Z08pwidGd500sU2byva5-Q,11890
 airflow/providers/amazon/aws/triggers/sqs.py,sha256=tVA1i8XzV5AqbVQAdWrgrLKoZF8ewqgEwV7ggk1hrQM,8257
 airflow/providers/amazon/aws/triggers/step_function.py,sha256=M1HGdrnxL_T9KSCBNy2t531xMNJaFc-Y792T9cSmLGM,2685
 airflow/providers/amazon/aws/utils/__init__.py,sha256=yUkoHb2LuqSyHvj-HAhc2r2s04Kv_PhdyLMq52KarO8,3878
@@ -249,7 +249,7 @@ airflow/providers/amazon/aws/waiters/rds.json,sha256=HNmNQm5J-VaFHzjWb1pE5P7-Ix-
 airflow/providers/amazon/aws/waiters/redshift.json,sha256=jOBotCgbkko1b_CHcGEbhhRvusgt0YSzVuFiZrqVP30,1742
 airflow/providers/amazon/aws/waiters/sagemaker.json,sha256=JPHuQtUFZ1B7EMLfVmCRevNZ9jgpB71LM0dva8ZEO9A,5254
 airflow/providers/amazon/aws/waiters/stepfunctions.json,sha256=aBaAZaGv8ZZGdN-2gvYEbq3fL_WHI_7s6SSDL-nWS1A,1034
-apache_airflow_providers_amazon-8.27.0rc1.dist-info/entry_points.txt,sha256=vlc0ZzhBkMrav1maTRofgksnAw4SwoQLFX9cmnTgktk,102
-apache_airflow_providers_amazon-8.27.0rc1.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
-apache_airflow_providers_amazon-8.27.0rc1.dist-info/METADATA,sha256=ZRHb0Da4c3DYUezf690tX4_PpC6VoOQiWtivY82doB4,10844
-apache_airflow_providers_amazon-8.27.0rc1.dist-info/RECORD,,
+apache_airflow_providers_amazon-8.27.0rc2.dist-info/entry_points.txt,sha256=vlc0ZzhBkMrav1maTRofgksnAw4SwoQLFX9cmnTgktk,102
+apache_airflow_providers_amazon-8.27.0rc2.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
+apache_airflow_providers_amazon-8.27.0rc2.dist-info/METADATA,sha256=AZjFZHwmOK2Qdc3jrmyBgvDHz-U1KKlKQ1HCDUbOo2A,10844
+apache_airflow_providers_amazon-8.27.0rc2.dist-info/RECORD,,