apache-airflow-providers-amazon 9.14.0__py3-none-any.whl → 9.18.0rc2__py3-none-any.whl

airflow/providers/amazon/aws/triggers/s3.py

@@ -41,6 +41,11 @@ class S3KeyTrigger(BaseTrigger):
         Unix wildcard pattern
     :param aws_conn_id: reference to the s3 connection
     :param use_regex: whether to use regex to check bucket
+    :param metadata_keys: List of head_object attributes to gather and send to ``check_fn``.
+        Acceptable values: Any top level attribute returned by s3.head_object. Specify * to return
+        all available attributes.
+        Default value: "Size".
+        If the requested attribute is not found, the key is still included and the value is None.
     :param hook_params: params for hook its optional
     """
 
@@ -56,6 +61,7 @@ class S3KeyTrigger(BaseTrigger):
         region_name: str | None = None,
         verify: bool | str | None = None,
         botocore_config: dict | None = None,
+        metadata_keys: list[str] | None = None,
         **hook_params: Any,
     ):
         super().__init__()
@@ -70,6 +76,7 @@ class S3KeyTrigger(BaseTrigger):
         self.region_name = region_name
         self.verify = verify
         self.botocore_config = botocore_config
+        self.metadata_keys = metadata_keys if metadata_keys else ["Size", "Key"]
 
     def serialize(self) -> tuple[str, dict[str, Any]]:
         """Serialize S3KeyTrigger arguments and classpath."""
@@ -87,6 +94,7 @@ class S3KeyTrigger(BaseTrigger):
                 "region_name": self.region_name,
                 "verify": self.verify,
                 "botocore_config": self.botocore_config,
+                "metadata_keys": self.metadata_keys,
             },
         )
 
@@ -108,11 +116,30 @@ class S3KeyTrigger(BaseTrigger):
                         client, self.bucket_name, self.bucket_key, self.wildcard_match, self.use_regex
                     ):
                         if self.should_check_fn:
-                            s3_objects = await self.hook.get_files_async(
+                            raw_objects = await self.hook.get_files_async(
                                 client, self.bucket_name, self.bucket_key, self.wildcard_match
                             )
+                            files = []
+                            for f in raw_objects:
+                                metadata = {}
+                                obj = await self.hook.get_head_object_async(
+                                    client=client, key=f, bucket_name=self.bucket_name
+                                )
+                                if obj is None:
+                                    return
+
+                                if "*" in self.metadata_keys:
+                                    metadata = obj
+                                else:
+                                    for mk in self.metadata_keys:
+                                        if mk == "Size":
+                                            metadata[mk] = obj.get("ContentLength")
+                                        else:
+                                            metadata[mk] = obj.get(mk, None)
+                                metadata["Key"] = f
+                                files.append(metadata)
                             await asyncio.sleep(self.poke_interval)
-                            yield TriggerEvent({"status": "running", "files": s3_objects})
+                            yield TriggerEvent({"status": "running", "files": files})
                         else:
                             yield TriggerEvent({"status": "success"})
                         return

airflow/providers/amazon/aws/triggers/ssm.py

@@ -36,6 +36,11 @@ class SsmRunCommandTrigger(AwsBaseWaiterTrigger):
     :param waiter_delay: The amount of time in seconds to wait between attempts. (default: 120)
     :param waiter_max_attempts: The maximum number of attempts to be made. (default: 75)
     :param aws_conn_id: The Airflow connection used for AWS credentials.
+    :param region_name: AWS region_name. If not specified then the default boto3 behaviour is used.
+    :param verify: Whether or not to verify SSL certificates. See:
+        https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html
+    :param botocore_config: Configuration dictionary (key-values) for botocore client. See:
+        https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html
     """
 
     def __init__(
@@ -45,6 +50,9 @@ class SsmRunCommandTrigger(AwsBaseWaiterTrigger):
         waiter_delay: int = 120,
         waiter_max_attempts: int = 75,
         aws_conn_id: str | None = None,
+        region_name: str | None = None,
+        verify: bool | str | None = None,
+        botocore_config: dict | None = None,
     ) -> None:
         super().__init__(
             serialized_fields={"command_id": command_id},
@@ -58,11 +66,19 @@ class SsmRunCommandTrigger(AwsBaseWaiterTrigger):
             waiter_delay=waiter_delay,
             waiter_max_attempts=waiter_max_attempts,
             aws_conn_id=aws_conn_id,
+            region_name=region_name,
+            verify=verify,
+            botocore_config=botocore_config,
         )
         self.command_id = command_id
 
     def hook(self) -> AwsGenericHook:
-        return SsmHook(aws_conn_id=self.aws_conn_id)
+        return SsmHook(
+            aws_conn_id=self.aws_conn_id,
+            region_name=self.region_name,
+            verify=self.verify,
+            config=self.botocore_config,
+        )
 
     async def run(self) -> AsyncIterator[TriggerEvent]:
         hook = self.hook()

airflow/providers/amazon/aws/utils/connection_wrapper.py

@@ -28,14 +28,11 @@ from botocore.config import Config
 
 from airflow.exceptions import AirflowException
 from airflow.providers.amazon.aws.utils import trim_none_values
+from airflow.providers.amazon.version_compat import NOTSET, ArgNotSet
 from airflow.utils.log.logging_mixin import LoggingMixin
-from airflow.utils.types import NOTSET, ArgNotSet
 
 if TYPE_CHECKING:
-    try:
-        from airflow.sdk import Connection
-    except ImportError:
-        from airflow.models.connection import Connection  # type: ignore[assignment]
+    from airflow.providers.common.compat.sdk import Connection
 
 
 @dataclass

airflow/providers/amazon/aws/utils/mixins.py

@@ -158,4 +158,4 @@ def aws_template_fields(*template_fields: str) -> tuple[str, ...]:
             f"{', '.join(map(repr, template_fields))}."
         )
         raise TypeError(msg)
-    return tuple(sorted(list({"aws_conn_id", "region_name", "verify"} | set(template_fields))))
+    return tuple(sorted({"aws_conn_id", "region_name", "verify"} | set(template_fields)))

airflow/providers/amazon/aws/utils/waiter.py

@@ -51,8 +51,8 @@ def waiter(
     :param desired_state: Wait until the getter returns this value
     :param failure_states: A set of states which indicate failure and should throw an
         exception if any are reached before the desired_state
-    :param object_type: Used for the reporting string. What are you waiting for? (application, job, etc)
-    :param action: Used for the reporting string. What action are you waiting for? (created, deleted, etc)
+    :param object_type: Used for the reporting string. What are you waiting for? (application, job, etc.)
+    :param action: Used for the reporting string. What action are you waiting for? (created, deleted, etc.)
     :param countdown: Number of seconds the waiter should wait for the desired state before timing out.
         Defaults to 25 * 60 seconds. None = infinite.
     :param check_interval_seconds: Number of seconds waiter should wait before attempting

airflow/providers/amazon/aws/waiters/emr.json

@@ -8,19 +8,19 @@
             "acceptors": [
                 {
                     "matcher": "path",
-                    "argument": "notebookexecution.status",
+                    "argument": "NotebookExecution.Status",
                     "expected": "RUNNING",
                     "state": "success"
                 },
                 {
                     "matcher": "path",
-                    "argument": "notebookexecution.status",
+                    "argument": "NotebookExecution.Status",
                     "expected": "FINISHED",
                     "state": "success"
                 },
                 {
                     "matcher": "path",
-                    "argument": "notebookexecution.status",
+                    "argument": "NotebookExecution.Status",
                     "expected": "FAILED",
                     "state": "failure"
                 }
@@ -33,19 +33,19 @@
             "acceptors": [
                 {
                     "matcher": "path",
-                    "argument": "notebookexecution.status",
+                    "argument": "NotebookExecution.Status",
                     "expected": "STOPPED",
                     "state": "success"
                 },
                 {
                     "matcher": "path",
-                    "argument": "notebookexecution.status",
+                    "argument": "NotebookExecution.Status",
                     "expected": "FINISHED",
                     "state": "success"
                 },
                 {
                     "matcher": "path",
-                    "argument": "notebookexecution.status",
+                    "argument": "NotebookExecution.Status",
                     "expected": "FAILED",
                     "state": "failure"
                 }

airflow/providers/amazon/get_provider_info.py

@@ -340,6 +340,20 @@ def get_provider_info():
                 "how-to-guide": ["/docs/apache-airflow-providers-amazon/operators/neptune.rst"],
                 "tags": ["aws"],
             },
+            {
+                "integration-name": "Amazon Kinesis Data Stream",
+                "external-doc-url": "https://aws.amazon.com/kinesis/",
+                "logo": "/docs/integration-logos/Amazon-Kinesis-Data-Firehose_light-bg@4x.png",
+                "how-to-guide": ["/docs/apache-airflow-providers-amazon/operators/kinesis_analytics.rst"],
+                "tags": ["aws"],
+            },
+            {
+                "integration-name": "Amazon Managed Workflows for Apache Airflow (MWAA)",
+                "external-doc-url": "https://aws.amazon.com/managed-workflows-for-apache-airflow/",
+                "logo": "/docs/integration-logos/Amazon-MWAA.png",
+                "how-to-guide": ["/docs/apache-airflow-providers-amazon/operators/mwaa.rst"],
+                "tags": ["aws"],
+            },
         ],
         "operators": [
             {
@@ -695,9 +709,13 @@ def get_provider_info():
                 ],
             },
             {
-                "integration-name": "Amazon Kinesis Data Firehose",
+                "integration-name": "Amazon Kinesis Data Stream",
                 "python-modules": ["airflow.providers.amazon.aws.hooks.kinesis"],
             },
+            {
+                "integration-name": "Amazon Kinesis Data Firehose",
+                "python-modules": ["airflow.providers.amazon.aws.hooks.firehose"],
+            },
             {
                 "integration-name": "AWS Lambda",
                 "python-modules": ["airflow.providers.amazon.aws.hooks.lambda_function"],

airflow/providers/amazon/version_compat.py

@@ -20,9 +20,13 @@
 # ON AIRFLOW VERSION, PLEASE COPY THIS FILE TO THE ROOT PACKAGE OF YOUR PROVIDER AND IMPORT
 # THOSE CONSTANTS FROM IT RATHER THAN IMPORTING THEM FROM ANOTHER PROVIDER OR TEST CODE
 #
+
 from __future__ import annotations
 
+import functools
+
 
+@functools.cache
 def get_base_airflow_version_tuple() -> tuple[int, int, int]:
     from packaging.version import Version
 
@@ -34,26 +38,25 @@ def get_base_airflow_version_tuple() -> tuple[int, int, int]:
 
 AIRFLOW_V_3_0_PLUS = get_base_airflow_version_tuple() >= (3, 0, 0)
 AIRFLOW_V_3_1_PLUS: bool = get_base_airflow_version_tuple() >= (3, 1, 0)
+AIRFLOW_V_3_1_1_PLUS: bool = get_base_airflow_version_tuple() >= (3, 1, 1)
+
+try:
+    from airflow.sdk.definitions._internal.types import NOTSET, ArgNotSet
+except ImportError:
+    from airflow.utils.types import NOTSET, ArgNotSet  # type: ignore[attr-defined,no-redef]
+try:
+    from airflow.sdk.definitions._internal.types import is_arg_set
+except ImportError:
 
-if AIRFLOW_V_3_1_PLUS:
-    from airflow.sdk import BaseHook
-else:
-    from airflow.hooks.base import BaseHook  # type: ignore[attr-defined,no-redef]
+    def is_arg_set(value):  # type: ignore[misc,no-redef]
+        return value is not NOTSET
 
-if AIRFLOW_V_3_0_PLUS:
-    from airflow.sdk import BaseOperator, BaseOperatorLink, BaseSensorOperator
-    from airflow.sdk.execution_time.xcom import XCom
-else:
-    from airflow.models import BaseOperator, XCom
-    from airflow.models.baseoperatorlink import BaseOperatorLink  # type: ignore[no-redef]
-    from airflow.sensors.base import BaseSensorOperator  # type: ignore[no-redef]
 
 __all__ = [
     "AIRFLOW_V_3_0_PLUS",
     "AIRFLOW_V_3_1_PLUS",
-    "BaseHook",
-    "BaseOperator",
-    "BaseOperatorLink",
-    "BaseSensorOperator",
-    "XCom",
+    "AIRFLOW_V_3_1_1_PLUS",
+    "NOTSET",
+    "ArgNotSet",
+    "is_arg_set",
 ]

{apache_airflow_providers_amazon-9.14.0.dist-info → apache_airflow_providers_amazon-9.18.0rc2.dist-info}/METADATA

@@ -1,12 +1,13 @@
 Metadata-Version: 2.4
 Name: apache-airflow-providers-amazon
-Version: 9.14.0
+Version: 9.18.0rc2
 Summary: Provider package apache-airflow-providers-amazon for Apache Airflow
 Keywords: airflow-provider,amazon,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
 Maintainer-email: Apache Software Foundation <dev@airflow.apache.org>
 Requires-Python: >=3.10
 Description-Content-Type: text/x-rst
+License-Expression: Apache-2.0
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: Console
 Classifier: Environment :: Web Environment
@@ -14,15 +15,16 @@ Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: System Administrators
 Classifier: Framework :: Apache Airflow
 Classifier: Framework :: Apache Airflow :: Provider
-Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: System :: Monitoring
-Requires-Dist: apache-airflow>=2.10.0
-Requires-Dist: apache-airflow-providers-common-compat>=1.6.1
-Requires-Dist: apache-airflow-providers-common-sql>=1.27.0
+License-File: LICENSE
+License-File: NOTICE
+Requires-Dist: apache-airflow>=2.11.0rc1
+Requires-Dist: apache-airflow-providers-common-compat>=1.10.0rc1
+Requires-Dist: apache-airflow-providers-common-sql>=1.27.0rc1
 Requires-Dist: apache-airflow-providers-http
 Requires-Dist: boto3>=1.37.2
 Requires-Dist: botocore>=1.37.2
@@ -34,19 +36,21 @@ Requires-Dist: asgiref>=2.3.0
 Requires-Dist: PyAthena>=3.10.0
 Requires-Dist: jmespath>=0.7.0
 Requires-Dist: sagemaker-studio>=1.0.9
+Requires-Dist: pydynamodb>=0.7.5; python_version >= '3.13'
+Requires-Dist: sqlean.py>=3.47.0; python_version >= '3.13'
 Requires-Dist: marshmallow>=3
 Requires-Dist: aiobotocore[boto3]>=2.21.1 ; extra == "aiobotocore"
 Requires-Dist: apache-airflow-providers-apache-hive ; extra == "apache-hive"
-Requires-Dist: apache-airflow-providers-cncf-kubernetes>=7.2.0 ; extra == "cncf-kubernetes"
-Requires-Dist: apache-airflow-providers-common-messaging>=2.0.0 ; extra == "common-messaging"
+Requires-Dist: apache-airflow-providers-cncf-kubernetes>=7.2.0rc1 ; extra == "cncf-kubernetes"
+Requires-Dist: apache-airflow-providers-common-messaging>=2.0.0rc1 ; extra == "common-messaging"
 Requires-Dist: apache-airflow-providers-exasol ; extra == "exasol"
-Requires-Dist: apache-airflow-providers-fab>=2.2.0 ; extra == "fab" and ( python_version < '3.13')
+Requires-Dist: apache-airflow-providers-fab>=2.2.0rc1 ; extra == "fab" and ( python_version < '3.13')
 Requires-Dist: apache-airflow-providers-ftp ; extra == "ftp"
 Requires-Dist: apache-airflow-providers-google ; extra == "google"
 Requires-Dist: apache-airflow-providers-imap ; extra == "imap"
 Requires-Dist: apache-airflow-providers-microsoft-azure ; extra == "microsoft-azure"
 Requires-Dist: apache-airflow-providers-mongo ; extra == "mongo"
-Requires-Dist: apache-airflow-providers-openlineage>=2.3.0 ; extra == "openlineage"
+Requires-Dist: apache-airflow-providers-openlineage>=2.3.0rc1 ; extra == "openlineage"
 Requires-Dist: python3-saml>=1.16.0 ; extra == "python3-saml" and ( python_version < '3.13')
 Requires-Dist: xmlsec>=1.3.14 ; extra == "python3-saml" and ( python_version < '3.13')
 Requires-Dist: lxml>=6.0.0 ; extra == "python3-saml" and ( python_version < '3.13')
@@ -55,8 +59,8 @@ Requires-Dist: apache-airflow-providers-salesforce ; extra == "salesforce"
 Requires-Dist: apache-airflow-providers-ssh ; extra == "ssh"
 Requires-Dist: apache-airflow-providers-standard ; extra == "standard"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.14.0/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.14.0
+Project-URL: Changelog, https://airflow.staged.apache.org/docs/apache-airflow-providers-amazon/9.18.0/changelog.html
+Project-URL: Documentation, https://airflow.staged.apache.org/docs/apache-airflow-providers-amazon/9.18.0
 Project-URL: Mastodon, https://fosstodon.org/@airflow
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
@@ -104,7 +108,7 @@ Provides-Extra: standard
 
 Package ``apache-airflow-providers-amazon``
 
-Release: ``9.14.0``
+Release: ``9.18.0``
 
 
 Amazon integration (including `Amazon Web Services (AWS) <https://aws.amazon.com/>`__).
@@ -117,7 +121,7 @@ This is a provider package for ``amazon`` provider. All classes for this provide
 are in ``airflow.providers.amazon`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.14.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.18.0/>`_.
 
 Installation
 ------------
@@ -131,11 +135,11 @@ The package supports the following python versions: 3.10,3.11,3.12,3.13
 Requirements
 ------------
 
-==========================================  ==================
+==========================================  ======================================
 PIP package                                 Version required
-==========================================  ==================
-``apache-airflow``                          ``>=2.10.0``
-``apache-airflow-providers-common-compat``  ``>=1.6.1``
+==========================================  ======================================
+``apache-airflow``                          ``>=2.11.0``
+``apache-airflow-providers-common-compat``  ``>=1.10.0``
 ``apache-airflow-providers-common-sql``     ``>=1.27.0``
 ``apache-airflow-providers-http``
 ``boto3``                                   ``>=1.37.2``
@@ -148,8 +152,10 @@ PIP package                                 Version required
 ``PyAthena``                                ``>=3.10.0``
 ``jmespath``                                ``>=0.7.0``
 ``sagemaker-studio``                        ``>=1.0.9``
+``pydynamodb``                              ``>=0.7.5; python_version >= "3.13"``
+``sqlean.py``                               ``>=3.47.0; python_version >= "3.13"``
 ``marshmallow``                             ``>=3``
-==========================================  ==================
+==========================================  ======================================
 
 Cross provider package dependencies
 -----------------------------------
@@ -210,5 +216,5 @@ Extra                 Dependencies
 ====================  ========================================================================================================================================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.14.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.18.0/changelog.html>`_.