apache-airflow-providers-amazon 9.10.0__tar.gz → 9.11.0rc1__tar.gz

{apache_airflow_providers_amazon-9.10.0 → apache_airflow_providers_amazon-9.11.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apache-airflow-providers-amazon
-Version: 9.10.0
+Version: 9.11.0rc1
 Summary: Provider package apache-airflow-providers-amazon for Apache Airflow
 Keywords: airflow-provider,amazon,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -18,10 +18,11 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: System :: Monitoring
-Requires-Dist: apache-airflow>=2.10.0
-Requires-Dist: apache-airflow-providers-common-compat>=1.6.1
-Requires-Dist: apache-airflow-providers-common-sql>=1.27.0
+Requires-Dist: apache-airflow>=2.10.0rc1
+Requires-Dist: apache-airflow-providers-common-compat>=1.6.1rc1
+Requires-Dist: apache-airflow-providers-common-sql>=1.27.0rc1
 Requires-Dist: apache-airflow-providers-http
 Requires-Dist: boto3>=1.37.2
 Requires-Dist: botocore>=1.37.2
@@ -36,26 +37,26 @@ Requires-Dist: sagemaker-studio>=1.0.9
 Requires-Dist: marshmallow>=3
 Requires-Dist: aiobotocore[boto3]>=2.21.1 ; extra == "aiobotocore"
 Requires-Dist: apache-airflow-providers-apache-hive ; extra == "apache-hive"
-Requires-Dist: apache-airflow-providers-cncf-kubernetes>=7.2.0 ; extra == "cncf-kubernetes"
-Requires-Dist: apache-airflow-providers-common-messaging>=1.0.1 ; extra == "common-messaging"
+Requires-Dist: apache-airflow-providers-cncf-kubernetes>=7.2.0rc1 ; extra == "cncf-kubernetes"
+Requires-Dist: apache-airflow-providers-common-messaging>=1.0.1rc1 ; extra == "common-messaging"
 Requires-Dist: apache-airflow-providers-exasol ; extra == "exasol"
-Requires-Dist: apache-airflow-providers-fab ; extra == "fab"
+Requires-Dist: apache-airflow-providers-fab>=2.2.0rc1 ; extra == "fab" and ( python_version < '3.13')
 Requires-Dist: apache-airflow-providers-ftp ; extra == "ftp"
 Requires-Dist: apache-airflow-providers-google ; extra == "google"
 Requires-Dist: apache-airflow-providers-imap ; extra == "imap"
 Requires-Dist: apache-airflow-providers-microsoft-azure ; extra == "microsoft-azure"
 Requires-Dist: apache-airflow-providers-mongo ; extra == "mongo"
-Requires-Dist: apache-airflow-providers-openlineage>=2.3.0 ; extra == "openlineage"
-Requires-Dist: python3-saml>=1.16.0 ; extra == "python3-saml"
-Requires-Dist: xmlsec!=1.3.15,>=1.3.14 ; extra == "python3-saml"
-Requires-Dist: lxml<5.4.0,>=5.3.2 ; extra == "python3-saml"
+Requires-Dist: apache-airflow-providers-openlineage>=2.3.0rc1 ; extra == "openlineage"
+Requires-Dist: python3-saml>=1.16.0 ; extra == "python3-saml" and ( python_version < '3.13')
+Requires-Dist: xmlsec>=1.3.14 ; extra == "python3-saml" and ( python_version < '3.13')
+Requires-Dist: lxml>=6.0.0 ; extra == "python3-saml" and ( python_version < '3.13')
 Requires-Dist: s3fs>=2023.10.0 ; extra == "s3fs"
 Requires-Dist: apache-airflow-providers-salesforce ; extra == "salesforce"
 Requires-Dist: apache-airflow-providers-ssh ; extra == "ssh"
 Requires-Dist: apache-airflow-providers-standard ; extra == "standard"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.10.0/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.10.0
+Project-URL: Changelog, https://airflow.staged.apache.org/docs/apache-airflow-providers-amazon/9.11.0/changelog.html
+Project-URL: Documentation, https://airflow.staged.apache.org/docs/apache-airflow-providers-amazon/9.11.0
 Project-URL: Mastodon, https://fosstodon.org/@airflow
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
@@ -103,8 +104,9 @@ Provides-Extra: standard
 
 Package ``apache-airflow-providers-amazon``
 
-Release: ``9.10.0``
+Release: ``9.11.0``
 
+Release Date: ``|PypiReleaseDate|``
 
 Amazon integration (including `Amazon Web Services (AWS) <https://aws.amazon.com/>`__).
 
@@ -116,7 +118,7 @@ This is a provider package for ``amazon`` provider. All classes for this provide
 are in ``airflow.providers.amazon`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.10.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.11.0/>`_.
 
 Installation
 ------------
@@ -125,7 +127,7 @@ You can install this package on top of an existing Airflow 2 installation (see `
 for the minimum Airflow version supported) via
 ``pip install apache-airflow-providers-amazon``
 
-The package supports the following python versions: 3.10,3.11,3.12
+The package supports the following python versions: 3.10,3.11,3.12,3.13
 
 Requirements
 ------------
@@ -184,5 +186,5 @@ Dependent package
 ========================================================================================================================  ====================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.10.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.11.0/changelog.html>`_.
 

{apache_airflow_providers_amazon-9.10.0 → apache_airflow_providers_amazon-9.11.0rc1}/README.rst

@@ -23,8 +23,9 @@
 
 Package ``apache-airflow-providers-amazon``
 
-Release: ``9.10.0``
+Release: ``9.11.0``
 
+Release Date: ``|PypiReleaseDate|``
 
 Amazon integration (including `Amazon Web Services (AWS) <https://aws.amazon.com/>`__).
 
@@ -36,7 +37,7 @@ This is a provider package for ``amazon`` provider. All classes for this provide
 are in ``airflow.providers.amazon`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.10.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.11.0/>`_.
 
 Installation
 ------------
@@ -45,7 +46,7 @@ You can install this package on top of an existing Airflow 2 installation (see `
 for the minimum Airflow version supported) via
 ``pip install apache-airflow-providers-amazon``
 
-The package supports the following python versions: 3.10,3.11,3.12
+The package supports the following python versions: 3.10,3.11,3.12,3.13
 
 Requirements
 ------------
@@ -104,4 +105,4 @@ Dependent package
 ========================================================================================================================  ====================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.10.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.11.0/changelog.html>`_.

{apache_airflow_providers_amazon-9.10.0 → apache_airflow_providers_amazon-9.11.0rc1}/docs/changelog.rst

@@ -26,6 +26,50 @@
 Changelog
 ---------
 
+9.11.0
+......
+
+Features
+~~~~~~~~
+
+* ``Add full support for AWS SSM Run Command in Airflow (#52769)``
+* ``Enhancement: AWS Provider sql to s3 operator pd.read_sql kwargs (#53399)``
+* ``Support HA schedulers for the Lambda Executor (#53396)``
+
+Bug Fixes
+~~~~~~~~~
+
+* ``Fix variable name in EKS command for token expiration timestamp (#53720)``
+* ``Fix EMR operator parameter documentation and naming (#53446)``
+* ``remove ECS Operator retry mechanism on task failed to start (#53083)``
+* ``Resolve OOM When Reading Large Logs in Webserver (#49470)``
+
+Misc
+~~~~
+
+* ``Add Python 3.13 support for Airflow. (#46891)``
+* ``Fix unreachable code mypy warnings in amazon provider (#53414)``
+* ``Refactoring get con part dbapihook in providers (#53335)``
+* ``Remove type ignore across codebase after mypy upgrade (#53243)``
+* ``Cleanup type ignores in amazon provider where possible (#53239)``
+* ``Use standard library ''typing'' imports for Python 3.10+ (#53158)``
+* ``Always build xmlsec and lxml packages from sources in our images (#53137)``
+* ``Improve mypy typing for RedshiftHook (#53099)``
+* ``Make amazon provider compatible with mypy 1.16.1 (#53088)``
+
+.. Below changes are excluded from the changelog. Move them to
+   appropriate section above if needed. Do not delete the lines(!):
+   * ``Change import in provider example dag (#53772)``
+   * ``Deprecate decorators from Core (#53629)``
+   * ``Replace 'mock.patch("utcnow")' with time_machine. (#53642)``
+   * ``Fix typos 'aiflow' -> 'airflow' (#53603)``
+   * ``Add support to example_emr_eks for LambdaExecutor environment (#53394)``
+   * ``Cleanup mypy ignore in eks_test_utils (#53325)``
+   * ``Handle ruff PT028 changes (#53235)``
+   * ``Create connection with API instead of directly through Session (#53161)``
+   * ``Make dag_version_id in TI non-nullable (#50825)``
+   * ``Changing import path to 'airflow.sdk' in Amazon provider package``
+
 9.10.0
 ......
 

{apache_airflow_providers_amazon-9.10.0 → apache_airflow_providers_amazon-9.11.0rc1}/docs/index.rst

@@ -87,7 +87,9 @@ apache-airflow-providers-amazon package
 Amazon integration (including `Amazon Web Services (AWS) <https://aws.amazon.com/>`__).
 
 
-Release: 9.10.0
+Release: 9.11.0
+
+Release Date: ``|PypiReleaseDate|``
 
 Provider package
 ----------------
@@ -166,5 +168,5 @@ Downloading official packages
 You can download officially released packages and verify their checksums and signatures from the
 `Official Apache Download site <https://downloads.apache.org/airflow/providers/>`_
 
-* `The apache-airflow-providers-amazon 9.10.0 sdist package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_amazon-9.10.0.tar.gz>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_amazon-9.10.0.tar.gz.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_amazon-9.10.0.tar.gz.sha512>`__)
-* `The apache-airflow-providers-amazon 9.10.0 wheel package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_amazon-9.10.0-py3-none-any.whl>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_amazon-9.10.0-py3-none-any.whl.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_amazon-9.10.0-py3-none-any.whl.sha512>`__)
+* `The apache-airflow-providers-amazon 9.11.0 sdist package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_amazon-9.11.0.tar.gz>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_amazon-9.11.0.tar.gz.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_amazon-9.11.0.tar.gz.sha512>`__)
+* `The apache-airflow-providers-amazon 9.11.0 wheel package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_amazon-9.11.0-py3-none-any.whl>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_amazon-9.11.0-py3-none-any.whl.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_amazon-9.11.0-py3-none-any.whl.sha512>`__)

apache_airflow_providers_amazon-9.11.0rc1/docs/operators/ssm.rst

@@ -0,0 +1,103 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+ ..   http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+===================================
+Amazon Simple Systems Manager (SSM)
+===================================
+
+`Amazon Simple Systems Manager (Amazon SSM) <https://aws.amazon.com/systems-manager/>`__ is a service
+that helps centrally view, manage, and operate nodes at scale in AWS, on-premises, and multi-cloud
+environments. Systems Manager consolidates various tools to help complete common node tasks across AWS
+accounts and Regions.
+To use Systems Manager, nodes must be managed, which means SSM Agent is installed on the machine and
+the agent can communicate with the Systems Manager service.
+
+Prerequisite Tasks
+------------------
+
+.. include:: ../_partials/prerequisite_tasks.rst
+
+Generic Parameters
+------------------
+
+.. include:: ../_partials/generic_parameters.rst
+
+Operators
+---------
+
+.. _howto/operator:SsmRunCommandOperator:
+
+Runs commands on one or more managed nodes
+==========================================
+
+To run SSM run command, you can use
+:class:`~airflow.providers.amazon.aws.operators.ssm.SsmRunCommandOperator`.
+
+To monitor the state of the command for a specific instance, you can use the "command_executed"
+Waiter. Additionally, you can use the following components to track the status of the command execution:
+:class:`~airflow.providers.amazon.aws.sensors.ssm.SsmRunCommandCompletedSensor` Sensor,
+or the :class:`~airflow.providers.amazon.aws.triggers.ssm.SsmRunCommandTrigger` Trigger.
+
+
+.. exampleinclude:: /../../amazon/tests/system/amazon/aws/example_ssm.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_run_command]
+    :end-before: [END howto_operator_run_command]
+
+Sensors
+-------
+
+.. _howto/sensor:SsmRunCommandCompletedSensor:
+
+Wait for an Amazon SSM run command
+==================================
+
+To wait on the state of an Amazon SSM run command job until it reaches a terminal state you can use
+:class:`~airflow.providers.amazon.aws.sensors.SSM.SsmRunCommandCompletedSensor`
+
+.. exampleinclude:: /../../amazon/tests/system/amazon/aws/example_ssm.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_sensor_run_command]
+    :end-before: [END howto_sensor_run_command]
+
+IAM Permissions
+---------------
+
+You need to ensure the following IAM permissions are granted to allow Airflow to run and monitor SSM Run Command executions:
+
+.. code-block::
+
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ssm:SendCommand",
+        "ssm:ListCommandInvocations",
+        "ssm:GetCommandInvocation"
+      ],
+      "Resource": "*"
+    }
+
+This policy allows access to all SSM documents and managed instances. For production environments,
+it is recommended to restrict the ``Resource`` field to specific SSM document ARNs and, if applicable,
+to the ARNs of intended target resources (such as EC2 instances), in accordance with the principle of least privilege.
+
+Reference
+---------
+
+* `AWS boto3 library documentation for Amazon SSM <https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ssm.html>`__

{apache_airflow_providers_amazon-9.10.0 → apache_airflow_providers_amazon-9.11.0rc1}/provider.yaml

@@ -22,12 +22,13 @@ description: |
   Amazon integration (including `Amazon Web Services (AWS) <https://aws.amazon.com/>`__).
 
 state: ready
-source-date-epoch: 1751472351
+source-date-epoch: 1753688567
 # Note that those versions are maintained by release manager - do not update them manually
 # with the exception of case where other provider in sources has >= new provider version.
 # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have
 # to be done in the same PR
 versions:
+  - 9.11.0
   - 9.10.0
   - 9.9.0
   - 9.8.0
@@ -281,6 +282,8 @@ integrations:
   - integration-name: Amazon Systems Manager (SSM)
     external-doc-url: https://aws.amazon.com/systems-manager/
     logo: /docs/integration-logos/AWS-Systems-Manager_light-bg@4x.png
+    how-to-guide:
+      - /docs/apache-airflow-providers-amazon/operators/ssm.rst
     tags: [aws]
   - integration-name: Amazon Web Services
     external-doc-url: https://aws.amazon.com/
@@ -430,6 +433,9 @@ operators:
   - integration-name: AWS Step Functions
     python-modules:
       - airflow.providers.amazon.aws.operators.step_function
+  - integration-name: Amazon Systems Manager (SSM)
+    python-modules:
+      - airflow.providers.amazon.aws.operators.ssm
   - integration-name: Amazon RDS
     python-modules:
       - airflow.providers.amazon.aws.operators.rds
@@ -531,6 +537,9 @@ sensors:
   - integration-name: AWS Step Functions
     python-modules:
       - airflow.providers.amazon.aws.sensors.step_function
+  - integration-name: Amazon Systems Manager (SSM)
+    python-modules:
+      - airflow.providers.amazon.aws.sensors.ssm
   - integration-name: Amazon QuickSight
     python-modules:
       - airflow.providers.amazon.aws.sensors.quicksight
@@ -737,6 +746,9 @@ triggers:
   - integration-name: Amazon Simple Storage Service (S3)
     python-modules:
       - airflow.providers.amazon.aws.triggers.s3
+  - integration-name: Amazon Systems Manager (SSM)
+    python-modules:
+      - airflow.providers.amazon.aws.triggers.ssm
   - integration-name: Amazon EMR
     python-modules:
       - airflow.providers.amazon.aws.triggers.emr

{apache_airflow_providers_amazon-9.10.0 → apache_airflow_providers_amazon-9.11.0rc1}/pyproject.toml

@@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi"
 
 [project]
 name = "apache-airflow-providers-amazon"
-version = "9.10.0"
+version = "9.11.0rc1"
 description = "Provider package apache-airflow-providers-amazon for Apache Airflow"
 readme = "README.rst"
 authors = [
@@ -47,6 +47,7 @@ classifiers = [
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
     "Topic :: System :: Monitoring",
 ]
 requires-python = ">=3.10"
@@ -56,9 +57,9 @@ requires-python = ">=3.10"
 # Make sure to run ``breeze static-checks --type update-providers-dependencies --all-files``
 # After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build``
 dependencies = [
-    "apache-airflow>=2.10.0",
-    "apache-airflow-providers-common-compat>=1.6.1",
-    "apache-airflow-providers-common-sql>=1.27.0",
+    "apache-airflow>=2.10.0rc1",
+    "apache-airflow-providers-common-compat>=1.6.1rc1",
+    "apache-airflow-providers-common-sql>=1.27.0rc1",
     "apache-airflow-providers-http",
     # We should update minimum version of boto3 and here regularly to avoid `pip` backtracking with the number
     # of candidates to consider. Make sure to configure boto3 version here as well as in all the tools below
@@ -89,17 +90,18 @@ dependencies = [
     "aiobotocore[boto3]>=2.21.1",
 ]
 "cncf.kubernetes" = [
-    "apache-airflow-providers-cncf-kubernetes>=7.2.0",
+    "apache-airflow-providers-cncf-kubernetes>=7.2.0rc1",
 ]
 "s3fs" = [
     "s3fs>=2023.10.0",
 ]
 "python3-saml" = [
-    "python3-saml>=1.16.0",
+    # Python 3 saml is not compatible with Python 3.13 yet, so we pin it to < 3.13
+    "python3-saml>=1.16.0; python_version < '3.13'",
     # python3-saml is dependent on xmlsec and seems they do not pin it, pinning here would be needed
     # We can remove it after https://github.com/xmlsec/python-xmlsec/issues/344 is fixed
-    "xmlsec!=1.3.15,>=1.3.14",
-    "lxml<5.4.0,>=5.3.2",
+    "xmlsec>=1.3.14; python_version < '3.13'",
+    "lxml>=6.0.0; python_version < '3.13'",
 ]
 "apache.hive" = [
     "apache-airflow-providers-apache-hive"
@@ -108,7 +110,7 @@ dependencies = [
     "apache-airflow-providers-exasol"
 ]
 "fab" = [
-    "apache-airflow-providers-fab"
+    "apache-airflow-providers-fab>=2.2.0rc1; python_version < '3.13'"
 ]
 "ftp" = [
     "apache-airflow-providers-ftp"
@@ -126,7 +128,7 @@ dependencies = [
     "apache-airflow-providers-mongo"
 ]
 "openlineage" = [
-    "apache-airflow-providers-openlineage>=2.3.0"
+    "apache-airflow-providers-openlineage>=2.3.0rc1"
 ]
 "salesforce" = [
     "apache-airflow-providers-salesforce"
@@ -138,7 +140,7 @@ dependencies = [
     "apache-airflow-providers-standard"
 ]
 "common.messaging" = [
-    "apache-airflow-providers-common-messaging>=1.0.1"
+    "apache-airflow-providers-common-messaging>=1.0.1rc1"
 ]
 
 [dependency-groups]
@@ -204,8 +206,8 @@ apache-airflow-providers-common-sql = {workspace = true}
 apache-airflow-providers-standard = {workspace = true}
 
 [project.urls]
-"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.10.0"
-"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.10.0/changelog.html"
+"Documentation" = "https://airflow.staged.apache.org/docs/apache-airflow-providers-amazon/9.11.0"
+"Changelog" = "https://airflow.staged.apache.org/docs/apache-airflow-providers-amazon/9.11.0/changelog.html"
 "Bug Tracker" = "https://github.com/apache/airflow/issues"
 "Source Code" = "https://github.com/apache/airflow"
 "Slack Chat" = "https://s.apache.org/airflow-slack"

{apache_airflow_providers_amazon-9.10.0/src/airflow/providers → apache_airflow_providers_amazon-9.11.0rc1/src/airflow}/__init__.py

@@ -14,4 +14,4 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-__path__ = __import__("pkgutil").extend_path(__path__, __name__)  # type: ignore
+__path__ = __import__("pkgutil").extend_path(__path__, __name__)

{apache_airflow_providers_amazon-9.10.0/tests/system → apache_airflow_providers_amazon-9.11.0rc1/src/airflow/providers}/__init__.py

@@ -14,4 +14,4 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-__path__ = __import__("pkgutil").extend_path(__path__, __name__)  # type: ignore
+__path__ = __import__("pkgutil").extend_path(__path__, __name__)

{apache_airflow_providers_amazon-9.10.0 → apache_airflow_providers_amazon-9.11.0rc1}/src/airflow/providers/amazon/__init__.py

@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "9.10.0"
+__version__ = "9.11.0"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.10.0"

{apache_airflow_providers_amazon-9.10.0 → apache_airflow_providers_amazon-9.11.0rc1}/src/airflow/providers/amazon/aws/executors/aws_lambda/lambda_executor.py

@@ -362,26 +362,57 @@ class AwsLambdaExecutor(BaseExecutor):
             MaxNumberOfMessages=10,
         )
 
+        # Pagination? Maybe we don't need it. But we don't always delete messages after viewing them so we
+        # could possibly accumulate a lot of messages in the queue and get stuck if we don't read bigger
+        # chunks and paginate.
         messages = response.get("Messages", [])
-        # Pagination? Maybe we don't need it. Since we always delete messages after looking at them.
-        # But then that may delete messages that could have been adopted. Let's leave it for now and see how it goes.
+        # The keys that we validate in the messages below will be different depending on whether or not
+        # the message is from the dead letter queue or the main results queue.
+        message_keys = ("return_code", "task_key")
         if messages and queue_url == self.dlq_url:
             self.log.warning("%d messages received from the dead letter queue", len(messages))
+            message_keys = ("command", "task_key")
 
         for message in messages:
+            delete_message = False
             receipt_handle = message["ReceiptHandle"]
-            body = json.loads(message["Body"])
+            try:
+                body = json.loads(message["Body"])
+            except json.JSONDecodeError:
+                self.log.warning(
+                    "Received a message from the queue that could not be parsed as JSON: %s",
+                    message["Body"],
+                )
+                delete_message = True
+            # If the message is not already marked for deletion, check if it has the required keys.
+            if not delete_message and not all(key in body for key in message_keys):
+                self.log.warning(
+                    "Message is not formatted correctly, %s and/or %s are missing: %s", *message_keys, body
+                )
+                delete_message = True
+            if delete_message:
+                self.log.warning("Deleting the message to avoid processing it again.")
+                self.sqs_client.delete_message(QueueUrl=queue_url, ReceiptHandle=receipt_handle)
+                continue
             return_code = body.get("return_code")
             ser_task_key = body.get("task_key")
             # Fetch the real task key from the running_tasks dict, using the serialized task key.
             try:
                 task_key = self.running_tasks[ser_task_key]
             except KeyError:
-                self.log.warning(
-                    "Received task %s from the queue which is not found in running tasks. Removing message.",
+                self.log.debug(
+                    "Received task %s from the queue which is not found in running tasks, it is likely "
+                    "from another Lambda Executor sharing this queue or might be a stale message that needs "
+                    "deleting manually. Marking the message as visible again.",
                     ser_task_key,
                 )
-                task_key = None
+                # Mark task as visible again in SQS so that another executor can pick it up.
+                self.sqs_client.change_message_visibility(
+                    QueueUrl=queue_url,
+                    ReceiptHandle=receipt_handle,
+                    VisibilityTimeout=0,
+                )
+                continue
 
             if task_key:
                 if return_code == 0:

{apache_airflow_providers_amazon-9.10.0 → apache_airflow_providers_amazon-9.11.0rc1}/src/airflow/providers/amazon/aws/hooks/base_aws.py

@@ -74,7 +74,7 @@ from airflow.utils.log.logging_mixin import LoggingMixin
 # If we change to Union syntax then mypy is not happy with UP007 Use `X | Y` for type annotations
 # The only way to workaround it for now is to keep the union syntax with ignore for mypy
 # We should try to resolve this later.
-BaseAwsConnection = TypeVar("BaseAwsConnection", bound=Union[BaseClient, ServiceResource])  # type: ignore[operator] # noqa: UP007
+BaseAwsConnection = TypeVar("BaseAwsConnection", bound=Union[BaseClient, ServiceResource])  # noqa: UP007
 
 
 if AIRFLOW_V_3_0_PLUS:
@@ -636,7 +636,7 @@ class AwsGenericHook(BaseHook, Generic[BaseAwsConnection]):
                     raise
 
         return AwsConnectionWrapper(
-            conn=connection,  # type: ignore[arg-type]
+            conn=connection,
             region_name=self._region_name,
             botocore_config=self._config,
             verify=self._verify,
@@ -718,10 +718,10 @@ class AwsGenericHook(BaseHook, Generic[BaseAwsConnection]):
         # because the user_agent_extra field is generated at runtime.
         user_agent_config = Config(
             user_agent_extra=self._generate_user_agent_extra_field(
-                existing_user_agent_extra=config.user_agent_extra  # type: ignore[union-attr]
+                existing_user_agent_extra=config.user_agent_extra
             )
         )
-        return config.merge(user_agent_config)  # type: ignore[union-attr]
+        return config.merge(user_agent_config)
 
     def get_client_type(
         self,
@@ -1050,7 +1050,7 @@ class AwsGenericHook(BaseHook, Generic[BaseAwsConnection]):
             return WaiterModel(model_config).waiter_names
 
 
-class AwsBaseHook(AwsGenericHook[Union[boto3.client, boto3.resource]]):  # type: ignore[operator] # noqa: UP007
+class AwsBaseHook(AwsGenericHook[Union[boto3.client, boto3.resource]]):  # noqa: UP007
     """
     Base class for interact with AWS.
 

{apache_airflow_providers_amazon-9.10.0 → apache_airflow_providers_amazon-9.11.0rc1}/src/airflow/providers/amazon/aws/hooks/ec2.py

@@ -20,11 +20,10 @@ from __future__ import annotations
 import functools
 import time
 from collections.abc import Callable
-from typing import TypeVar
+from typing import ParamSpec, TypeVar
 
 from airflow.exceptions import AirflowException
 from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
-from airflow.typing_compat import ParamSpec
 
 PS = ParamSpec("PS")
 RT = TypeVar("RT")

{apache_airflow_providers_amazon-9.10.0 → apache_airflow_providers_amazon-9.11.0rc1}/src/airflow/providers/amazon/aws/hooks/ecs.py

@@ -19,7 +19,7 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING, Protocol, runtime_checkable
 
-from airflow.providers.amazon.aws.exceptions import EcsOperatorError, EcsTaskFailToStart
+from airflow.providers.amazon.aws.exceptions import EcsOperatorError
 from airflow.providers.amazon.aws.hooks.base_aws import AwsGenericHook
 from airflow.providers.amazon.aws.utils import _StringCompareEnum
 
@@ -38,16 +38,6 @@ def should_retry(exception: Exception):
     return False
 
 
-def should_retry_eni(exception: Exception):
-    """Check if exception is related to ENI (Elastic Network Interfaces)."""
-    if isinstance(exception, EcsTaskFailToStart):
-        return any(
-            eni_reason in exception.message
-            for eni_reason in ["network interface provisioning", "ResourceInitializationError"]
-        )
-    return False
-
-
 class EcsClusterStates(_StringCompareEnum):
     """Contains the possible State values of an ECS Cluster."""
 

{apache_airflow_providers_amazon-9.10.0 → apache_airflow_providers_amazon-9.11.0rc1}/src/airflow/providers/amazon/aws/hooks/eks.py

@@ -100,7 +100,7 @@ COMMAND = """
 
             json_string=$(printf '{{"kind": "ExecCredential","apiVersion": \
                 "client.authentication.k8s.io/v1alpha1","spec": {{}},"status": \
-                {{"expirationTimestamp": "%s","token": "%s"}}}}' "$expiration_timestamp" "$token")
+                {{"expirationTimestamp": "%s","token": "%s"}}}}' "$timestamp" "$token")
             echo $json_string
             """
 
@@ -523,7 +523,7 @@ class EksHook(AwsBaseHook):
         :return: A List of the combined results of the provided API call.
         """
         name_collection: list = []
-        token = DEFAULT_PAGINATION_TOKEN
+        token: str | None = DEFAULT_PAGINATION_TOKEN
 
         while token is not None:
             response = api_call(nextToken=token)