apache-airflow-providers-amazon 9.1.0rc3__py3-none-any.whl → 9.2.0__py3-none-any.whl

airflow/providers/amazon/aws/triggers/s3.py

@@ -17,8 +17,9 @@
 from __future__ import annotations
 
 import asyncio
+from collections.abc import AsyncIterator
 from functools import cached_property
-from typing import TYPE_CHECKING, Any, AsyncIterator
+from typing import TYPE_CHECKING, Any
 
 from airflow.providers.amazon.aws.hooks.s3 import S3Hook
 from airflow.triggers.base import BaseTrigger, TriggerEvent

airflow/providers/amazon/aws/triggers/sagemaker.py

@@ -19,9 +19,10 @@ from __future__ import annotations
 
 import asyncio
 from collections import Counter
+from collections.abc import AsyncIterator
 from enum import IntEnum
 from functools import cached_property
-from typing import Any, AsyncIterator
+from typing import Any
 
 from botocore.exceptions import WaiterError
 

airflow/providers/amazon/aws/triggers/sqs.py

@@ -17,7 +17,8 @@
 from __future__ import annotations
 
 import asyncio
-from typing import TYPE_CHECKING, Any, AsyncIterator, Collection
+from collections.abc import AsyncIterator, Collection
+from typing import TYPE_CHECKING, Any
 
 from airflow.exceptions import AirflowException
 from airflow.providers.amazon.aws.hooks.sqs import SqsHook

airflow/providers/amazon/aws/utils/__init__.py

@@ -31,21 +31,7 @@ log = logging.getLogger(__name__)
 
 
 def trim_none_values(obj: dict):
-    from packaging.version import Version
-
-    from airflow.version import version
-
-    if Version(version) < Version("2.7"):
-        # before version 2.7, the behavior is not the same.
-        # Empty dict and lists are removed from the given dict.
-        return {key: val for key, val in obj.items() if val is not None}
-    else:
-        # once airflow 2.6 rolls out of compatibility support for provider packages,
-        # we can replace usages of this method with the core one in our code,
-        # and uncomment this warning for users who may use it.
-        # warnings.warn("use airflow.utils.helpers.prune_dict() instead",
-        #     AirflowProviderDeprecationWarning, stacklevel=2)
-        return prune_dict(obj)
+    return prune_dict(obj)
 
 
 def datetime_to_epoch(date_time: datetime) -> int:

airflow/providers/amazon/aws/utils/task_log_fetcher.py

@@ -18,9 +18,10 @@
 from __future__ import annotations
 
 import time
+from collections.abc import Generator
 from datetime import datetime, timedelta, timezone
 from threading import Event, Thread
-from typing import TYPE_CHECKING, Generator
+from typing import TYPE_CHECKING
 
 from botocore.exceptions import ClientError, ConnectionClosedError
 

airflow/providers/amazon/aws/utils/waiter.py

@@ -19,6 +19,7 @@ from __future__ import annotations
 
 import logging
 import time
+from enum import Enum
 from typing import Callable
 
 from airflow.exceptions import AirflowException
@@ -83,3 +84,22 @@ def get_state(response, keys) -> str:
         if value is not None:
             value = value.get(key, None)
     return value
+
+
+class WaitPolicy(str, Enum):
+    """
+    Used to control the waiting behaviour within EMRClusterJobFlowOperator.
+
+    Choices:
+    - WAIT_FOR_COMPLETION - Will wait for the cluster to report "Running" state
+    - WAIT_FOR_STEPS_COMPLETION - Will wait for the cluster to report "Terminated" state
+    """
+
+    WAIT_FOR_COMPLETION = "wait_for_completion"
+    WAIT_FOR_STEPS_COMPLETION = "wait_for_steps_completion"
+
+
+WAITER_POLICY_NAME_MAPPING: dict[WaitPolicy, str] = {
+    WaitPolicy.WAIT_FOR_COMPLETION: "job_flow_waiting",
+    WaitPolicy.WAIT_FOR_STEPS_COMPLETION: "job_flow_terminated",
+}

airflow/providers/amazon/aws/waiters/dms.json

@@ -0,0 +1,88 @@
+{
+    "version": 2,
+    "waiters": {
+        "replication_deprovisioned": {
+            "operation": "DescribeReplications",
+            "delay": 30,
+            "maxAttempts": 60,
+            "acceptors": [
+                {
+                    "matcher": "path",
+                    "argument": "Replications[0].ProvisionData.ProvisionState",
+                    "expected": "deprovisioned",
+                    "state": "success"
+                }
+            ]
+        },
+        "replication_terminal_status": {
+            "operation": "DescribeReplications",
+            "delay": 30,
+            "maxAttempts": 60,
+            "acceptors": [
+                {
+                    "matcher": "path",
+                    "argument": "Replications[0].ProvisionData.ProvisionState",
+                    "expected": "deprovisioning",
+                    "state": "retry"
+                },
+                {
+                    "matcher": "path",
+                    "argument": "Replications[0].Status",
+                    "expected": "failed",
+                    "state": "success"
+                },
+                {
+                    "matcher": "path",
+                    "argument": "Replications[0].Status",
+                    "expected": "stopped",
+                    "state": "success"
+                }
+            ]
+        },
+        "replication_complete": {
+            "operation": "DescribeReplications",
+            "delay": 30,
+            "maxAttempts": 60,
+            "acceptors": [
+                {
+                    "matcher": "path",
+                    "argument": "Replications[0].Status",
+                    "expected": "failed",
+                    "state": "failure"
+                },
+                {
+                    "matcher": "path",
+                    "argument": "Replications[0].Status",
+                    "expected": "stopped",
+                    "state": "success"
+                }
+            ]
+        },
+        "replication_config_deleted": {
+            "operation": "DescribeReplicationConfigs",
+            "delay": 5,
+            "maxAttempts": 60,
+            "acceptors": [
+                {
+                    "matcher": "error",
+                    "expected": "ResourceNotFoundFault",
+                    "state": "success",
+                    "argument": "Error.Code"
+                }
+            ]
+        },
+        "replication_stopped": {
+            "operation": "DescribeReplications",
+            "delay": 5,
+            "maxAttempts": 60,
+            "acceptors": [
+                {
+                    "matcher": "path",
+                    "argument": "Replications[0].Status",
+                    "expected": "stopped",
+                    "state": "success"
+                }
+            ]
+        }
+    }
+}

airflow/providers/amazon/get_provider_info.py

@@ -28,8 +28,9 @@ def get_provider_info():
         "name": "Amazon",
         "description": "Amazon integration (including `Amazon Web Services (AWS) <https://aws.amazon.com/>`__).\n",
         "state": "ready",
-        "source-date-epoch": 1730011042,
+        "source-date-epoch": 1734527035,
         "versions": [
+            "9.2.0",
             "9.1.0",
             "9.0.0",
             "8.29.0",
@@ -98,9 +99,9 @@ def get_provider_info():
             "1.0.0",
         ],
         "dependencies": [
-            "apache-airflow>=2.8.0",
-            "apache-airflow-providers-common-compat>=1.2.1",
-            "apache-airflow-providers-common-sql>=1.3.1",
+            "apache-airflow>=2.9.0",
+            "apache-airflow-providers-common-compat>=1.3.0",
+            "apache-airflow-providers-common-sql>=1.20.0",
             "apache-airflow-providers-http",
             "boto3>=1.34.90",
             "botocore>=1.34.90",
@@ -130,7 +131,7 @@ def get_provider_info():
             "aiobotocore>=2.13.0",
             "aws_xray_sdk>=2.12.0",
             "moto[cloudformation,glue]>=5.0.0",
-            "mypy-boto3-appflow>=1.34.0,<1.35.39",
+            "mypy-boto3-appflow>=1.35.39",
             "mypy-boto3-rds>=1.34.90",
             "mypy-boto3-redshift-data>=1.34.0",
             "mypy-boto3-s3>=1.34.90",
@@ -949,6 +950,10 @@ def get_provider_info():
                 "integration-name": "Amazon Neptune",
                 "python-modules": ["airflow.providers.amazon.aws.triggers.neptune"],
             },
+            {
+                "integration-name": "AWS Database Migration Service",
+                "python-modules": ["airflow.providers.amazon.aws.triggers.dms"],
+            },
         ],
         "transfers": [
             {

airflow/providers/amazon/version_compat.py

@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# NOTE! THIS FILE IS COPIED MANUALLY IN OTHER PROVIDERS DELIBERATELY TO AVOID ADDING UNNECESSARY
+# DEPENDENCIES BETWEEN PROVIDERS. IF YOU WANT TO ADD CONDITIONAL CODE IN YOUR PROVIDER THAT DEPENDS
+# ON AIRFLOW VERSION, PLEASE COPY THIS FILE TO THE ROOT PACKAGE OF YOUR PROVIDER AND IMPORT
+# THOSE CONSTANTS FROM IT RATHER THAN IMPORTING THEM FROM ANOTHER PROVIDER OR TEST CODE
+#
+from __future__ import annotations
+
+
+def get_base_airflow_version_tuple() -> tuple[int, int, int]:
+    from packaging.version import Version
+
+    from airflow import __version__
+
+    airflow_version = Version(__version__)
+    return airflow_version.major, airflow_version.minor, airflow_version.micro
+
+
+AIRFLOW_V_2_10_PLUS = get_base_airflow_version_tuple() >= (2, 10, 0)
+AIRFLOW_V_3_0_PLUS = get_base_airflow_version_tuple() >= (3, 0, 0)

{apache_airflow_providers_amazon-9.1.0rc3.dist-info → apache_airflow_providers_amazon-9.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: apache-airflow-providers-amazon
-Version: 9.1.0rc3
+Version: 9.2.0
 Summary: Provider package apache-airflow-providers-amazon for Apache Airflow
 Keywords: airflow-provider,amazon,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -21,10 +21,10 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: System :: Monitoring
 Requires-Dist: PyAthena>=3.0.10
-Requires-Dist: apache-airflow-providers-common-compat>=1.2.1rc0
-Requires-Dist: apache-airflow-providers-common-sql>=1.3.1rc0
+Requires-Dist: apache-airflow-providers-common-compat>=1.3.0
+Requires-Dist: apache-airflow-providers-common-sql>=1.20.0
 Requires-Dist: apache-airflow-providers-http
-Requires-Dist: apache-airflow>=2.8.0rc0
+Requires-Dist: apache-airflow>=2.9.0
 Requires-Dist: asgiref>=2.3.0
 Requires-Dist: boto3>=1.34.90
 Requires-Dist: botocore>=1.34.90
@@ -35,16 +35,13 @@ Requires-Dist: python3-saml>=1.16.0
 Requires-Dist: redshift_connector>=2.0.918
 Requires-Dist: watchtower>=3.0.0,!=3.3.0,<4
 Requires-Dist: aiobotocore[boto3]>=2.13.0 ; extra == "aiobotocore"
-Requires-Dist: apache-airflow-providers-apache-hive ; extra == "apache.hive"
-Requires-Dist: apache-airflow-providers-cncf-kubernetes>=7.2.0rc0 ; extra == "cncf.kubernetes"
-Requires-Dist: apache-airflow-providers-common-compat ; extra == "common.compat"
-Requires-Dist: apache-airflow-providers-common-sql ; extra == "common.sql"
+Requires-Dist: apache-airflow-providers-apache-hive ; extra == "apache-hive"
+Requires-Dist: apache-airflow-providers-cncf-kubernetes>=7.2.0 ; extra == "cncf-kubernetes"
 Requires-Dist: apache-airflow-providers-exasol ; extra == "exasol"
 Requires-Dist: apache-airflow-providers-ftp ; extra == "ftp"
 Requires-Dist: apache-airflow-providers-google ; extra == "google"
-Requires-Dist: apache-airflow-providers-http ; extra == "http"
 Requires-Dist: apache-airflow-providers-imap ; extra == "imap"
-Requires-Dist: apache-airflow-providers-microsoft-azure ; extra == "microsoft.azure"
+Requires-Dist: apache-airflow-providers-microsoft-azure ; extra == "microsoft-azure"
 Requires-Dist: apache-airflow-providers-mongo ; extra == "mongo"
 Requires-Dist: apache-airflow-providers-openlineage ; extra == "openlineage"
 Requires-Dist: pandas>=2.1.2,<2.2 ; extra == "pandas" and (python_version>="3.9")
@@ -54,23 +51,20 @@ Requires-Dist: s3fs>=2023.10.0 ; extra == "s3fs"
 Requires-Dist: apache-airflow-providers-salesforce ; extra == "salesforce"
 Requires-Dist: apache-airflow-providers-ssh ; extra == "ssh"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.1.0/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.1.0
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.2.0/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.2.0
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
-Project-URL: Twitter, https://twitter.com/ApacheAirflow
+Project-URL: Twitter, https://x.com/ApacheAirflow
 Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
 Provides-Extra: aiobotocore
-Provides-Extra: apache.hive
-Provides-Extra: cncf.kubernetes
-Provides-Extra: common.compat
-Provides-Extra: common.sql
+Provides-Extra: apache-hive
+Provides-Extra: cncf-kubernetes
 Provides-Extra: exasol
 Provides-Extra: ftp
 Provides-Extra: google
-Provides-Extra: http
 Provides-Extra: imap
-Provides-Extra: microsoft.azure
+Provides-Extra: microsoft-azure
 Provides-Extra: mongo
 Provides-Extra: openlineage
 Provides-Extra: pandas
@@ -123,7 +117,7 @@ Provides-Extra: ssh
 
 Package ``apache-airflow-providers-amazon``
 
-Release: ``9.1.0.rc3``
+Release: ``9.2.0``
 
 
 Amazon integration (including `Amazon Web Services (AWS) <https://aws.amazon.com/>`__).
@@ -136,7 +130,7 @@ This is a provider package for ``amazon`` provider. All classes for this provide
 are in ``airflow.providers.amazon`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.1.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.2.0/>`_.
 
 Installation
 ------------
@@ -153,9 +147,9 @@ Requirements
 ==========================================  ======================
 PIP package                                 Version required
 ==========================================  ======================
-``apache-airflow``                          ``>=2.8.0``
-``apache-airflow-providers-common-compat``  ``>=1.2.1``
-``apache-airflow-providers-common-sql``     ``>=1.3.1``
+``apache-airflow``                          ``>=2.9.0``
+``apache-airflow-providers-common-compat``  ``>=1.3.0``
+``apache-airflow-providers-common-sql``     ``>=1.20.0``
 ``apache-airflow-providers-http``
 ``boto3``                                   ``>=1.34.90``
 ``botocore``                                ``>=1.34.90``
@@ -202,4 +196,4 @@ Dependent package
 ======================================================================================================================  ===================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.1.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.2.0/changelog.html>`_.