apache-airflow-providers-amazon 8.27.0__py3-none-any.whl → 8.27.0rc1__py3-none-any.whl

airflow/providers/amazon/aws/operators/emr.py

@@ -1382,30 +1382,30 @@ class EmrServerlessStartJobOperator(BaseOperator):
 
         self.persist_links(context)
 
-        if self.wait_for_completion:
-            if self.deferrable:
-                self.defer(
-                    trigger=EmrServerlessStartJobTrigger(
-                        application_id=self.application_id,
-                        job_id=self.job_id,
-                        waiter_delay=self.waiter_delay,
-                        waiter_max_attempts=self.waiter_max_attempts,
-                        aws_conn_id=self.aws_conn_id,
-                    ),
-                    method_name="execute_complete",
-                    timeout=timedelta(seconds=self.waiter_max_attempts * self.waiter_delay),
-                )
-            else:
-                waiter = self.hook.get_waiter("serverless_job_completed")
-                wait(
-                    waiter=waiter,
-                    waiter_max_attempts=self.waiter_max_attempts,
+        if self.deferrable:
+            self.defer(
+                trigger=EmrServerlessStartJobTrigger(
+                    application_id=self.application_id,
+                    job_id=self.job_id,
                     waiter_delay=self.waiter_delay,
-                    args={"applicationId": self.application_id, "jobRunId": self.job_id},
-                    failure_message="Serverless Job failed",
-                    status_message="Serverless Job status is",
-                    status_args=["jobRun.state", "jobRun.stateDetails"],
-                )
+                    waiter_max_attempts=self.waiter_max_attempts,
+                    aws_conn_id=self.aws_conn_id,
+                ),
+                method_name="execute_complete",
+                timeout=timedelta(seconds=self.waiter_max_attempts * self.waiter_delay),
+            )
+
+        if self.wait_for_completion:
+            waiter = self.hook.get_waiter("serverless_job_completed")
+            wait(
+                waiter=waiter,
+                waiter_max_attempts=self.waiter_max_attempts,
+                waiter_delay=self.waiter_delay,
+                args={"applicationId": self.application_id, "jobRunId": self.job_id},
+                failure_message="Serverless Job failed",
+                status_message="Serverless Job status is",
+                status_args=["jobRun.state", "jobRun.stateDetails"],
+            )
 
         return self.job_id
 

airflow/providers/amazon/aws/operators/redshift_cluster.py

@@ -32,7 +32,6 @@ from airflow.providers.amazon.aws.triggers.redshift_cluster import (
     RedshiftResumeClusterTrigger,
 )
 from airflow.providers.amazon.aws.utils import validate_execute_complete_event
-from airflow.utils.helpers import prune_dict
 
 if TYPE_CHECKING:
     from airflow.utils.context import Context
@@ -508,8 +507,8 @@ class RedshiftResumeClusterOperator(BaseOperator):
         aws_conn_id: str | None = "aws_default",
         wait_for_completion: bool = False,
         deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
-        poll_interval: int = 30,
-        max_attempts: int = 30,
+        poll_interval: int = 10,
+        max_attempts: int = 10,
         **kwargs,
     ):
         super().__init__(**kwargs)
@@ -543,38 +542,38 @@ class RedshiftResumeClusterOperator(BaseOperator):
                 else:
                     raise error
 
-        if self.wait_for_completion:
-            if self.deferrable:
-                cluster_state = redshift_hook.cluster_status(cluster_identifier=self.cluster_identifier)
-                if cluster_state == "available":
-                    self.log.info("Resumed cluster successfully")
-                elif cluster_state == "deleting":
-                    raise AirflowException(
-                        "Unable to resume cluster since cluster is currently in status: %s", cluster_state
-                    )
-                else:
-                    self.defer(
-                        trigger=RedshiftResumeClusterTrigger(
-                            cluster_identifier=self.cluster_identifier,
-                            waiter_delay=self.poll_interval,
-                            waiter_max_attempts=self.max_attempts,
-                            aws_conn_id=self.aws_conn_id,
-                        ),
-                        method_name="execute_complete",
-                        # timeout is set to ensure that if a trigger dies, the timeout does not restart
-                        # 60 seconds is added to allow the trigger to exit gracefully (i.e. yield TriggerEvent)
-                        timeout=timedelta(seconds=self.max_attempts * self.poll_interval + 60),
-                    )
+        if self.deferrable:
+            cluster_state = redshift_hook.cluster_status(cluster_identifier=self.cluster_identifier)
+            if cluster_state == "available":
+                self.log.info("Resumed cluster successfully")
+            elif cluster_state == "deleting":
+                raise AirflowException(
+                    "Unable to resume cluster since cluster is currently in status: %s", cluster_state
+                )
             else:
-                waiter = redshift_hook.get_waiter("cluster_resumed")
-                waiter.wait(
-                    ClusterIdentifier=self.cluster_identifier,
-                    WaiterConfig={
-                        "Delay": self.poll_interval,
-                        "MaxAttempts": self.max_attempts,
-                    },
+                self.defer(
+                    trigger=RedshiftResumeClusterTrigger(
+                        cluster_identifier=self.cluster_identifier,
+                        waiter_delay=self.poll_interval,
+                        waiter_max_attempts=self.max_attempts,
+                        aws_conn_id=self.aws_conn_id,
+                    ),
+                    method_name="execute_complete",
+                    # timeout is set to ensure that if a trigger dies, the timeout does not restart
+                    # 60 seconds is added to allow the trigger to exit gracefully (i.e. yield TriggerEvent)
+                    timeout=timedelta(seconds=self.max_attempts * self.poll_interval + 60),
                 )
 
+        if self.wait_for_completion:
+            waiter = redshift_hook.get_waiter("cluster_resumed")
+            waiter.wait(
+                ClusterIdentifier=self.cluster_identifier,
+                WaiterConfig={
+                    "Delay": self.poll_interval,
+                    "MaxAttempts": self.max_attempts,
+                },
+            )
+
     def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> None:
         event = validate_execute_complete_event(event)
 
@@ -597,7 +596,6 @@ class RedshiftPauseClusterOperator(BaseOperator):
         running Airflow in a distributed manner and aws_conn_id is None or
         empty, then default boto3 configuration would be used (and must be
         maintained on each worker node).
-    :param wait_for_completion: If True, waits for the cluster to be paused. (default: False)
     :param deferrable: Run operator in the deferrable mode
     :param poll_interval: Time (in seconds) to wait between two consecutive calls to check cluster state
     :param max_attempts: Maximum number of attempts to poll the cluster
@@ -612,16 +610,14 @@ class RedshiftPauseClusterOperator(BaseOperator):
         *,
         cluster_identifier: str,
         aws_conn_id: str | None = "aws_default",
-        wait_for_completion: bool = False,
         deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False),
-        poll_interval: int = 30,
-        max_attempts: int = 30,
+        poll_interval: int = 10,
+        max_attempts: int = 15,
         **kwargs,
     ):
         super().__init__(**kwargs)
         self.cluster_identifier = cluster_identifier
         self.aws_conn_id = aws_conn_id
-        self.wait_for_completion = wait_for_completion
         self.deferrable = deferrable
         self.max_attempts = max_attempts
         self.poll_interval = poll_interval
@@ -647,38 +643,26 @@ class RedshiftPauseClusterOperator(BaseOperator):
                     time.sleep(self._attempt_interval)
                 else:
                     raise error
-        if self.wait_for_completion:
-            if self.deferrable:
-                cluster_state = redshift_hook.cluster_status(cluster_identifier=self.cluster_identifier)
-                if cluster_state == "paused":
-                    self.log.info("Paused cluster successfully")
-                elif cluster_state == "deleting":
-                    raise AirflowException(
-                        f"Unable to pause cluster since cluster is currently in status: {cluster_state}"
-                    )
-                else:
-                    self.defer(
-                        trigger=RedshiftPauseClusterTrigger(
-                            cluster_identifier=self.cluster_identifier,
-                            waiter_delay=self.poll_interval,
-                            waiter_max_attempts=self.max_attempts,
-                            aws_conn_id=self.aws_conn_id,
-                        ),
-                        method_name="execute_complete",
-                        # timeout is set to ensure that if a trigger dies, the timeout does not restart
-                        # 60 seconds is added to allow the trigger to exit gracefully (i.e. yield TriggerEvent)
-                        timeout=timedelta(seconds=self.max_attempts * self.poll_interval + 60),
-                    )
+        if self.deferrable:
+            cluster_state = redshift_hook.cluster_status(cluster_identifier=self.cluster_identifier)
+            if cluster_state == "paused":
+                self.log.info("Paused cluster successfully")
+            elif cluster_state == "deleting":
+                raise AirflowException(
+                    f"Unable to pause cluster since cluster is currently in status: {cluster_state}"
+                )
             else:
-                waiter = redshift_hook.get_waiter("cluster_paused")
-                waiter.wait(
-                    ClusterIdentifier=self.cluster_identifier,
-                    WaiterConfig=prune_dict(
-                        {
-                            "Delay": self.poll_interval,
-                            "MaxAttempts": self.max_attempts,
-                        }
+                self.defer(
+                    trigger=RedshiftPauseClusterTrigger(
+                        cluster_identifier=self.cluster_identifier,
+                        waiter_delay=self.poll_interval,
+                        waiter_max_attempts=self.max_attempts,
+                        aws_conn_id=self.aws_conn_id,
                     ),
+                    method_name="execute_complete",
+                    # timeout is set to ensure that if a trigger dies, the timeout does not restart
+                    # 60 seconds is added to allow the trigger to exit gracefully (i.e. yield TriggerEvent)
+                    timeout=timedelta(seconds=self.max_attempts * self.poll_interval + 60),
                 )
 
     def execute_complete(self, context: Context, event: dict[str, Any] | None = None) -> None:

airflow/providers/amazon/aws/operators/sagemaker.py

@@ -36,6 +36,7 @@ from airflow.providers.amazon.aws.hooks.sagemaker import (
 )
 from airflow.providers.amazon.aws.triggers.sagemaker import (
     SageMakerPipelineTrigger,
+    SageMakerTrainingPrintLogTrigger,
     SageMakerTrigger,
 )
 from airflow.providers.amazon.aws.utils import trim_none_values, validate_execute_complete_event
@@ -1194,15 +1195,25 @@ class SageMakerTrainingOperator(SageMakerBaseOperator):
             if self.max_ingestion_time:
                 timeout = datetime.timedelta(seconds=self.max_ingestion_time)
 
-            self.defer(
-                timeout=timeout,
-                trigger=SageMakerTrigger(
+            trigger: SageMakerTrainingPrintLogTrigger | SageMakerTrigger
+            if self.print_log:
+                trigger = SageMakerTrainingPrintLogTrigger(
+                    job_name=self.config["TrainingJobName"],
+                    poke_interval=self.check_interval,
+                    aws_conn_id=self.aws_conn_id,
+                )
+            else:
+                trigger = SageMakerTrigger(
                     job_name=self.config["TrainingJobName"],
                     job_type="Training",
                     poke_interval=self.check_interval,
                     max_attempts=self.max_attempts,
                     aws_conn_id=self.aws_conn_id,
-                ),
+                )
+
+            self.defer(
+                timeout=timeout,
+                trigger=trigger,
                 method_name="execute_complete",
             )
 

airflow/providers/amazon/aws/triggers/sagemaker.py

@@ -25,9 +25,8 @@ from functools import cached_property
 from typing import Any, AsyncIterator
 
 from botocore.exceptions import WaiterError
-from deprecated import deprecated
 
-from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
+from airflow.exceptions import AirflowException
 from airflow.providers.amazon.aws.hooks.sagemaker import LogState, SageMakerHook
 from airflow.providers.amazon.aws.utils.waiter_with_logging import async_wait
 from airflow.triggers.base import BaseTrigger, TriggerEvent
@@ -200,13 +199,6 @@ class SageMakerPipelineTrigger(BaseTrigger):
             raise AirflowException("Waiter error: max attempts reached")
 
 
-@deprecated(
-    reason=(
-        "`airflow.providers.amazon.aws.triggers.sagemaker.SageMakerTrainingPrintLogTrigger` "
-        "has been deprecated and will be removed in future. Please use ``SageMakerTrigger`` instead."
-    ),
-    category=AirflowProviderDeprecationWarning,
-)
 class SageMakerTrainingPrintLogTrigger(BaseTrigger):
     """
     SageMakerTrainingPrintLogTrigger is fired as deferred class with params to run the task in triggerer.

{apache_airflow_providers_amazon-8.27.0.dist-info → apache_airflow_providers_amazon-8.27.0rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apache-airflow-providers-amazon
-Version: 8.27.0
+Version: 8.27.0rc1
 Summary: Provider package apache-airflow-providers-amazon for Apache Airflow
 Keywords: airflow-provider,amazon,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -22,11 +22,11 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: System :: Monitoring
 Requires-Dist: PyAthena>=3.0.10
-Requires-Dist: apache-airflow-providers-common-compat>=1.1.0
-Requires-Dist: apache-airflow-providers-common-compat>=1.1.0
-Requires-Dist: apache-airflow-providers-common-sql>=1.3.1
+Requires-Dist: apache-airflow-providers-common-compat>=1.1.0rc0
+Requires-Dist: apache-airflow-providers-common-compat>=1.1.0rc0
+Requires-Dist: apache-airflow-providers-common-sql>=1.3.1rc0
 Requires-Dist: apache-airflow-providers-http
-Requires-Dist: apache-airflow>=2.7.0
+Requires-Dist: apache-airflow>=2.7.0rc0
 Requires-Dist: asgiref>=2.3.0
 Requires-Dist: boto3>=1.34.90
 Requires-Dist: botocore>=1.34.90
@@ -38,7 +38,7 @@ Requires-Dist: sqlalchemy_redshift>=0.8.6
 Requires-Dist: watchtower>=3.0.0,<4
 Requires-Dist: aiobotocore[boto3]>=2.13.0 ; extra == "aiobotocore"
 Requires-Dist: apache-airflow-providers-apache-hive ; extra == "apache.hive"
-Requires-Dist: apache-airflow-providers-cncf-kubernetes>=7.2.0 ; extra == "cncf.kubernetes"
+Requires-Dist: apache-airflow-providers-cncf-kubernetes>=7.2.0rc0 ; extra == "cncf.kubernetes"
 Requires-Dist: apache-airflow-providers-common-compat ; extra == "common.compat"
 Requires-Dist: apache-airflow-providers-common-sql ; extra == "common.sql"
 Requires-Dist: apache-airflow-providers-exasol ; extra == "exasol"
@@ -125,7 +125,7 @@ Provides-Extra: ssh
 
 Package ``apache-airflow-providers-amazon``
 
-Release: ``8.27.0``
+Release: ``8.27.0.rc1``
 
 
 Amazon integration (including `Amazon Web Services (AWS) <https://aws.amazon.com/>`__).

{apache_airflow_providers_amazon-8.27.0.dist-info → apache_airflow_providers_amazon-8.27.0rc1.dist-info}/RECORD

@@ -114,7 +114,7 @@ airflow/providers/amazon/aws/operators/dms.py,sha256=6RhUtbELAjp0LLkUWl73kdcH4MR
 airflow/providers/amazon/aws/operators/ec2.py,sha256=aQj6cL3nZzu0tcn3dq6RBSPsByZe8fNtn6qcpQYtlNI,17051
 airflow/providers/amazon/aws/operators/ecs.py,sha256=pmm05ugkdyUNwkfY1X-mQdT8pqui0qJumwt92tEaEgU,32601
 airflow/providers/amazon/aws/operators/eks.py,sha256=0xHtPzxZx3ymr6-iqGvAoZsnA24PrjYRSAiFzuxrl3U,50761
-airflow/providers/amazon/aws/operators/emr.py,sha256=XNafysiROWBcMxAPK_9MQplcY-itOivKYnljffQo6eE,84909
+airflow/providers/amazon/aws/operators/emr.py,sha256=tygUmL90ebpita9eKg7sMygHWW1BVomGfc_wnaHjOpE,84804
 airflow/providers/amazon/aws/operators/eventbridge.py,sha256=e686XFhVi54DbaCk7oVc0fhvH6GIPU3p8jgyCie1yBU,10394
 airflow/providers/amazon/aws/operators/glacier.py,sha256=zxwC6lLk6sWerjlogXq6HgNOJx4h0hkqpGpqn23hJWk,3654
 airflow/providers/amazon/aws/operators/glue.py,sha256=m8hdF6eTyzsK3onOqt6Td0dGshhgf_XU1f4EtMb42LU,28390
@@ -125,10 +125,10 @@ airflow/providers/amazon/aws/operators/lambda_function.py,sha256=96KtK5KUpMPW2i8
 airflow/providers/amazon/aws/operators/neptune.py,sha256=on5oNX5K4yHfW1POE0eeZujta71vkJdVL07vucGjX-4,14751
 airflow/providers/amazon/aws/operators/quicksight.py,sha256=jc3Eof19UfLt5IqbQswRzaHaK8h0ACLY99i_1Prtq10,4089
 airflow/providers/amazon/aws/operators/rds.py,sha256=zGiUIwpO2EdUByCYkgFwLbFCQhg_TCTTHVuNnee6X_g,39325
-airflow/providers/amazon/aws/operators/redshift_cluster.py,sha256=rmBHCssxrYEJ8EnENY-AnzC004lbtHvxXHpy69sHtV0,36681
+airflow/providers/amazon/aws/operators/redshift_cluster.py,sha256=tYTmkXbITvRMaf2R8m9Cx5f2zYF2w1whNfO-l08wKjY,35755
 airflow/providers/amazon/aws/operators/redshift_data.py,sha256=wK-vTDcn0MqOuF9e-71JYIEkLKihah6oGU-p_8VT2HI,8612
 airflow/providers/amazon/aws/operators/s3.py,sha256=8mzkCBGnAbb5CHf0PFokQwdEmio_D3u99JnNoqTBUW4,35844
-airflow/providers/amazon/aws/operators/sagemaker.py,sha256=0ZP6vyPHMOVkco80sViyDtwhguawSJFo5l1GLOvunQc,83614
+airflow/providers/amazon/aws/operators/sagemaker.py,sha256=MJ-LceYW72Mqtpy5rb_i_oxG9BCUAXrAc7IQOatR85Y,84053
 airflow/providers/amazon/aws/operators/sns.py,sha256=Rttd015UhLo4pCplGybxtLhflyu_26IFzYP7WTmQFk8,3730
 airflow/providers/amazon/aws/operators/sqs.py,sha256=0KkhhIblMggNHLxAyrv5dbWcaXvdSWQA2AOQP2CzOlo,4327
 airflow/providers/amazon/aws/operators/step_function.py,sha256=eXZAxZqG5VNPaFVEchyL4vKmOh54jc83ZjrIZDeld34,9515
@@ -207,7 +207,7 @@ airflow/providers/amazon/aws/triggers/rds.py,sha256=i7FvDkHjg7yr-PQsl2ymn_Lu8B2Z
 airflow/providers/amazon/aws/triggers/redshift_cluster.py,sha256=4VlZgtpfCbe9C2d8IW2qK6QmvFIhs8Guciuq8TZ5GbU,12920
 airflow/providers/amazon/aws/triggers/redshift_data.py,sha256=hRWsyFVt0F6yEMnSb-5pZekUcuWhnpw8tNXzf8Im6cI,4245
 airflow/providers/amazon/aws/triggers/s3.py,sha256=J2MvoiQoQ3GKf8OtFF9awvY6SwVXnhKG_m3ZJ1Z7iwU,9444
-airflow/providers/amazon/aws/triggers/sagemaker.py,sha256=a8gck8xLEV1vQsp1mu8a6Z08pwidGd500sU2byva5-Q,11890
+airflow/providers/amazon/aws/triggers/sagemaker.py,sha256=C52lhVsiDEiw-1Kswc0kvNDOyPdDvV5bF7vjbuBN45o,11542
 airflow/providers/amazon/aws/triggers/sqs.py,sha256=tVA1i8XzV5AqbVQAdWrgrLKoZF8ewqgEwV7ggk1hrQM,8257
 airflow/providers/amazon/aws/triggers/step_function.py,sha256=M1HGdrnxL_T9KSCBNy2t531xMNJaFc-Y792T9cSmLGM,2685
 airflow/providers/amazon/aws/utils/__init__.py,sha256=yUkoHb2LuqSyHvj-HAhc2r2s04Kv_PhdyLMq52KarO8,3878
@@ -249,7 +249,7 @@ airflow/providers/amazon/aws/waiters/rds.json,sha256=HNmNQm5J-VaFHzjWb1pE5P7-Ix-
 airflow/providers/amazon/aws/waiters/redshift.json,sha256=jOBotCgbkko1b_CHcGEbhhRvusgt0YSzVuFiZrqVP30,1742
 airflow/providers/amazon/aws/waiters/sagemaker.json,sha256=JPHuQtUFZ1B7EMLfVmCRevNZ9jgpB71LM0dva8ZEO9A,5254
 airflow/providers/amazon/aws/waiters/stepfunctions.json,sha256=aBaAZaGv8ZZGdN-2gvYEbq3fL_WHI_7s6SSDL-nWS1A,1034
-apache_airflow_providers_amazon-8.27.0.dist-info/entry_points.txt,sha256=vlc0ZzhBkMrav1maTRofgksnAw4SwoQLFX9cmnTgktk,102
-apache_airflow_providers_amazon-8.27.0.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
-apache_airflow_providers_amazon-8.27.0.dist-info/METADATA,sha256=5X4DNhW0DOdh4MsTBW_fObTzfM2_vGxW1Cm3f-Yfcgw,10822
-apache_airflow_providers_amazon-8.27.0.dist-info/RECORD,,
+apache_airflow_providers_amazon-8.27.0rc1.dist-info/entry_points.txt,sha256=vlc0ZzhBkMrav1maTRofgksnAw4SwoQLFX9cmnTgktk,102
+apache_airflow_providers_amazon-8.27.0rc1.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
+apache_airflow_providers_amazon-8.27.0rc1.dist-info/METADATA,sha256=ZRHb0Da4c3DYUezf690tX4_PpC6VoOQiWtivY82doB4,10844
+apache_airflow_providers_amazon-8.27.0rc1.dist-info/RECORD,,