apache-airflow-providers-amazon 9.13.0__tar.gz → 9.14.0__tar.gz

{apache_airflow_providers_amazon-9.13.0 → apache_airflow_providers_amazon-9.14.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apache-airflow-providers-amazon
-Version: 9.13.0
+Version: 9.14.0
 Summary: Provider package apache-airflow-providers-amazon for Apache Airflow
 Keywords: airflow-provider,amazon,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -55,8 +55,8 @@ Requires-Dist: apache-airflow-providers-salesforce ; extra == "salesforce"
 Requires-Dist: apache-airflow-providers-ssh ; extra == "ssh"
 Requires-Dist: apache-airflow-providers-standard ; extra == "standard"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.13.0/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.13.0
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.14.0/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.14.0
 Project-URL: Mastodon, https://fosstodon.org/@airflow
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
@@ -104,7 +104,7 @@ Provides-Extra: standard
 
 Package ``apache-airflow-providers-amazon``
 
-Release: ``9.13.0``
+Release: ``9.14.0``
 
 
 Amazon integration (including `Amazon Web Services (AWS) <https://aws.amazon.com/>`__).
@@ -117,7 +117,7 @@ This is a provider package for ``amazon`` provider. All classes for this provide
 are in ``airflow.providers.amazon`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.13.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.14.0/>`_.
 
 Installation
 ------------
@@ -184,6 +184,31 @@ Dependent package
 `apache-airflow-providers-ssh <https://airflow.apache.org/docs/apache-airflow-providers-ssh>`_                            ``ssh``
 ========================================================================================================================  ====================
 
+Optional dependencies
+----------------------
+
+====================  ========================================================================================================================================
+Extra                 Dependencies
+====================  ========================================================================================================================================
+``aiobotocore``       ``aiobotocore[boto3]>=2.21.1``
+``cncf.kubernetes``   ``apache-airflow-providers-cncf-kubernetes>=7.2.0``
+``s3fs``              ``s3fs>=2023.10.0``
+``python3-saml``      ``python3-saml>=1.16.0; python_version < '3.13'``, ``xmlsec>=1.3.14; python_version < '3.13'``, ``lxml>=6.0.0; python_version < '3.13'``
+``apache.hive``       ``apache-airflow-providers-apache-hive``
+``exasol``            ``apache-airflow-providers-exasol``
+``fab``               ``apache-airflow-providers-fab>=2.2.0; python_version < '3.13'``
+``ftp``               ``apache-airflow-providers-ftp``
+``google``            ``apache-airflow-providers-google``
+``imap``              ``apache-airflow-providers-imap``
+``microsoft.azure``   ``apache-airflow-providers-microsoft-azure``
+``mongo``             ``apache-airflow-providers-mongo``
+``openlineage``       ``apache-airflow-providers-openlineage>=2.3.0``
+``salesforce``        ``apache-airflow-providers-salesforce``
+``ssh``               ``apache-airflow-providers-ssh``
+``standard``          ``apache-airflow-providers-standard``
+``common.messaging``  ``apache-airflow-providers-common-messaging>=2.0.0``
+====================  ========================================================================================================================================
+
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.13.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.14.0/changelog.html>`_.
 

{apache_airflow_providers_amazon-9.13.0 → apache_airflow_providers_amazon-9.14.0}/README.rst

@@ -23,7 +23,7 @@
 
 Package ``apache-airflow-providers-amazon``
 
-Release: ``9.13.0``
+Release: ``9.14.0``
 
 
 Amazon integration (including `Amazon Web Services (AWS) <https://aws.amazon.com/>`__).
@@ -36,7 +36,7 @@ This is a provider package for ``amazon`` provider. All classes for this provide
 are in ``airflow.providers.amazon`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.13.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.14.0/>`_.
 
 Installation
 ------------
@@ -103,5 +103,30 @@ Dependent package
 `apache-airflow-providers-ssh <https://airflow.apache.org/docs/apache-airflow-providers-ssh>`_                            ``ssh``
 ========================================================================================================================  ====================
 
+Optional dependencies
+----------------------
+
+====================  ========================================================================================================================================
+Extra                 Dependencies
+====================  ========================================================================================================================================
+``aiobotocore``       ``aiobotocore[boto3]>=2.21.1``
+``cncf.kubernetes``   ``apache-airflow-providers-cncf-kubernetes>=7.2.0``
+``s3fs``              ``s3fs>=2023.10.0``
+``python3-saml``      ``python3-saml>=1.16.0; python_version < '3.13'``, ``xmlsec>=1.3.14; python_version < '3.13'``, ``lxml>=6.0.0; python_version < '3.13'``
+``apache.hive``       ``apache-airflow-providers-apache-hive``
+``exasol``            ``apache-airflow-providers-exasol``
+``fab``               ``apache-airflow-providers-fab>=2.2.0; python_version < '3.13'``
+``ftp``               ``apache-airflow-providers-ftp``
+``google``            ``apache-airflow-providers-google``
+``imap``              ``apache-airflow-providers-imap``
+``microsoft.azure``   ``apache-airflow-providers-microsoft-azure``
+``mongo``             ``apache-airflow-providers-mongo``
+``openlineage``       ``apache-airflow-providers-openlineage>=2.3.0``
+``salesforce``        ``apache-airflow-providers-salesforce``
+``ssh``               ``apache-airflow-providers-ssh``
+``standard``          ``apache-airflow-providers-standard``
+``common.messaging``  ``apache-airflow-providers-common-messaging>=2.0.0``
+====================  ========================================================================================================================================
+
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.13.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.14.0/changelog.html>`_.

{apache_airflow_providers_amazon-9.13.0 → apache_airflow_providers_amazon-9.14.0}/docs/changelog.rst

@@ -26,12 +26,57 @@
 Changelog
 ---------
 
+9.14.0
+......
+
+
+Release Date: ``|PypiReleaseDate|``
+
+Features
+~~~~~~~~
+
+* ``Implement 'batch_is_authorized_' methods in AWS auth manager (#55307)``
+* ``Add configurable confirm parameter to 'S3ToSFTPOperator' (#55569)``
+
+Bug Fixes
+~~~~~~~~~
+
+* ``[OSSTaskHandler, CloudwatchTaskHandler, S3TaskHandler, HdfsTaskHandler, ElasticsearchTaskHandler, GCSTaskHandler, OpensearchTaskHandler, RedisTaskHandler, WasbTaskHandler] supports log file size handling (#55455)``
+* ``Catch 404/401 issues for Bedrock Operators (#55445)``
+* ``EcsRunTaskOperator fails when no containers are provided in the response (#51692)``
+* ``AWS BatchOperator does not fetch log entries for deferred jobs (#55703)``
+
+Misc
+~~~~
+
+* ``List only connections, pools and variables the user has access to (#55298)``
+* ``Switch all airflow logging to structlog (#52651)``
+* ``AIP-67 - Multi-team: Per team executor config (env var only) (#55003)``
+* ``Allow SSM operators and sensors to run in deferrable mode (#55649)``
+* ``Update EOL AWS Redshift cluster node types (#55741)``
+* ``improve logging in SqsSensorTrigger (#55705)``
+
+Doc-only
+~~~~~~~~
+
+* ``Add stable note to BatchExecutor (#55286)``
+* ``Add quotas section in lambda executor docs (#55740)``
+
+.. Below changes are excluded from the changelog. Move them to
+   appropriate section above if needed. Do not delete the lines(!):
+   * ``Migrate off Xen-based ECS instances (#55527)``
+   * ``Add missing test for aws batch utils (#55407)``
+   * ``README optional dependencies template (#55280)``
+
 9.13.0
 ......
 
 
 Release Date: ``|PypiReleaseDate|``
 
+.. note::
+  * ``The experimental BatchExecutor added in 8.20.0 is now stable``
+
 Features
 ~~~~~~~~
 

{apache_airflow_providers_amazon-9.13.0 → apache_airflow_providers_amazon-9.14.0}/docs/executors/lambda-executor.rst

@@ -56,7 +56,7 @@ When a task is queued to run with the Lambda executor a Lambda function is async
 invoked with a payload containing the task key being run and the Airflow CLI
 command/workload to run. The Lambda function executes the task and then must
 publish a message to the SQS results queue with the same task key it received as well as
-the return code of the airflow task execution process. The Lambda executor periodically
+the return code of the Airflow task execution process. The Lambda executor periodically
 polls this SQS results queue for messages and updates its internal state with the results
 it receives. The Lambda function must also be configured with a Dead Letter Queue (DLQ) to
 which Lambda will automatically send messages to for invocations that fail to be processed,
@@ -65,6 +65,13 @@ also poll this DLQ for messages and update the internal state with the results i
 
 .. image:: ../img/lambda_executor.png
 
+Quotas
+------
+
+When working with the Lambda Executor it is important to be familiar with the AWS Lambda quotas
+as they may affect performance and scalability. For more information,
+please refer to the `official AWS Lambda documentation on quotas <http://docs.aws.amazon.com/lambda/latest/dg/gettingstarted-limits.html>`__.
+
 .. _lambda_config-options:
 
 Config Options
@@ -358,4 +365,3 @@ To configure Airflow to utilize the Lambda Executor and leverage the resources w
 
 .. include:: general.rst
   :start-after: .. BEGIN INIT_DB
-  :end-before: .. END INIT_DB

{apache_airflow_providers_amazon-9.13.0 → apache_airflow_providers_amazon-9.14.0}/docs/index.rst

@@ -87,7 +87,7 @@ apache-airflow-providers-amazon package
 Amazon integration (including `Amazon Web Services (AWS) <https://aws.amazon.com/>`__).
 
 
-Release: 9.13.0
+Release: 9.14.0
 
 Provider package
 ----------------
@@ -166,5 +166,5 @@ Downloading official packages
 You can download officially released packages and verify their checksums and signatures from the
 `Official Apache Download site <https://downloads.apache.org/airflow/providers/>`_
 
-* `The apache-airflow-providers-amazon 9.13.0 sdist package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_amazon-9.13.0.tar.gz>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_amazon-9.13.0.tar.gz.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_amazon-9.13.0.tar.gz.sha512>`__)
-* `The apache-airflow-providers-amazon 9.13.0 wheel package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_amazon-9.13.0-py3-none-any.whl>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_amazon-9.13.0-py3-none-any.whl.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_amazon-9.13.0-py3-none-any.whl.sha512>`__)
+* `The apache-airflow-providers-amazon 9.14.0 sdist package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_amazon-9.14.0.tar.gz>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_amazon-9.14.0.tar.gz.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_amazon-9.14.0.tar.gz.sha512>`__)
+* `The apache-airflow-providers-amazon 9.14.0 wheel package <https://downloads.apache.org/airflow/providers/apache_airflow_providers_amazon-9.14.0-py3-none-any.whl>`_ (`asc <https://downloads.apache.org/airflow/providers/apache_airflow_providers_amazon-9.14.0-py3-none-any.whl.asc>`__, `sha512 <https://downloads.apache.org/airflow/providers/apache_airflow_providers_amazon-9.14.0-py3-none-any.whl.sha512>`__)

{apache_airflow_providers_amazon-9.13.0 → apache_airflow_providers_amazon-9.14.0}/provider.yaml

@@ -22,12 +22,13 @@ description: |
   Amazon integration (including `Amazon Web Services (AWS) <https://aws.amazon.com/>`__).
 
 state: ready
-source-date-epoch: 1756876382
+source-date-epoch: 1757949816
 # Note that those versions are maintained by release manager - do not update them manually
 # with the exception of case where other provider in sources has >= new provider version.
 # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have
 # to be done in the same PR
 versions:
+  - 9.14.0
   - 9.13.0
   - 9.12.0
   - 9.11.0

{apache_airflow_providers_amazon-9.13.0 → apache_airflow_providers_amazon-9.14.0}/pyproject.toml

@@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi"
 
 [project]
 name = "apache-airflow-providers-amazon"
-version = "9.13.0"
+version = "9.14.0"
 description = "Provider package apache-airflow-providers-amazon for Apache Airflow"
 readme = "README.rst"
 authors = [
@@ -206,8 +206,8 @@ apache-airflow-providers-common-sql = {workspace = true}
 apache-airflow-providers-standard = {workspace = true}
 
 [project.urls]
-"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.13.0"
-"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.13.0/changelog.html"
+"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.14.0"
+"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.14.0/changelog.html"
 "Bug Tracker" = "https://github.com/apache/airflow/issues"
 "Source Code" = "https://github.com/apache/airflow"
 "Slack Chat" = "https://s.apache.org/airflow-slack"

{apache_airflow_providers_amazon-9.13.0 → apache_airflow_providers_amazon-9.14.0}/src/airflow/providers/amazon/__init__.py

@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "9.13.0"
+__version__ = "9.14.0"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.10.0"

{apache_airflow_providers_amazon-9.13.0 → apache_airflow_providers_amazon-9.14.0}/src/airflow/providers/amazon/aws/auth_manager/aws_auth_manager.py

@@ -44,7 +44,10 @@ from airflow.providers.amazon.version_compat import AIRFLOW_V_3_0_PLUS
 if TYPE_CHECKING:
     from airflow.api_fastapi.auth.managers.base_auth_manager import ResourceMethod
     from airflow.api_fastapi.auth.managers.models.batch_apis import (
+        IsAuthorizedConnectionRequest,
         IsAuthorizedDagRequest,
+        IsAuthorizedPoolRequest,
+        IsAuthorizedVariableRequest,
     )
     from airflow.api_fastapi.auth.managers.models.resource_details import (
         AccessView,
@@ -244,6 +247,27 @@ class AwsAuthManager(BaseAuthManager[AwsAuthManagerUser]):
 
         return [menu_item for menu_item in menu_items if _has_access_to_menu_item(requests[menu_item.value])]
 
+    def batch_is_authorized_connection(
+        self,
+        requests: Sequence[IsAuthorizedConnectionRequest],
+        *,
+        user: AwsAuthManagerUser,
+    ) -> bool:
+        facade_requests: Sequence[IsAuthorizedRequest] = [
+            cast(
+                "IsAuthorizedRequest",
+                {
+                    "method": request["method"],
+                    "entity_type": AvpEntities.CONNECTION,
+                    "entity_id": cast("ConnectionDetails", request["details"]).conn_id
+                    if request.get("details")
+                    else None,
+                },
+            )
+            for request in requests
+        ]
+        return self.avp_facade.batch_is_authorized(requests=facade_requests, user=user)
+
     def batch_is_authorized_dag(
         self,
         requests: Sequence[IsAuthorizedDagRequest],
@@ -251,18 +275,65 @@ class AwsAuthManager(BaseAuthManager[AwsAuthManagerUser]):
         user: AwsAuthManagerUser,
     ) -> bool:
         facade_requests: Sequence[IsAuthorizedRequest] = [
-            {
-                "method": request["method"],
-                "entity_type": AvpEntities.DAG,
-                "entity_id": cast("DagDetails", request["details"]).id if request.get("details") else None,
-                "context": {
-                    "dag_entity": {
-                        "string": cast("DagAccessEntity", request["access_entity"]).value,
-                    },
-                }
-                if request.get("access_entity")
-                else None,
-            }
+            cast(
+                "IsAuthorizedRequest",
+                {
+                    "method": request["method"],
+                    "entity_type": AvpEntities.DAG,
+                    "entity_id": cast("DagDetails", request["details"]).id
+                    if request.get("details")
+                    else None,
+                    "context": {
+                        "dag_entity": {
+                            "string": cast("DagAccessEntity", request["access_entity"]).value,
+                        },
+                    }
+                    if request.get("access_entity")
+                    else None,
+                },
+            )
+            for request in requests
+        ]
+        return self.avp_facade.batch_is_authorized(requests=facade_requests, user=user)
+
+    def batch_is_authorized_pool(
+        self,
+        requests: Sequence[IsAuthorizedPoolRequest],
+        *,
+        user: AwsAuthManagerUser,
+    ) -> bool:
+        facade_requests: Sequence[IsAuthorizedRequest] = [
+            cast(
+                "IsAuthorizedRequest",
+                {
+                    "method": request["method"],
+                    "entity_type": AvpEntities.POOL,
+                    "entity_id": cast("PoolDetails", request["details"]).name
+                    if request.get("details")
+                    else None,
+                },
+            )
+            for request in requests
+        ]
+        return self.avp_facade.batch_is_authorized(requests=facade_requests, user=user)
+
+    def batch_is_authorized_variable(
+        self,
+        requests: Sequence[IsAuthorizedVariableRequest],
+        *,
+        user: AwsAuthManagerUser,
+    ) -> bool:
+        facade_requests: Sequence[IsAuthorizedRequest] = [
+            cast(
+                "IsAuthorizedRequest",
+                {
+                    "method": request["method"],
+                    "entity_type": AvpEntities.VARIABLE,
+                    "entity_id": cast("VariableDetails", request["details"]).key
+                    if request.get("details")
+                    else None,
+                },
+            )
             for request in requests
         ]
         return self.avp_facade.batch_is_authorized(requests=facade_requests, user=user)
@@ -273,6 +344,7 @@ class AwsAuthManager(BaseAuthManager[AwsAuthManagerUser]):
         dag_ids: set[str],
         user: AwsAuthManagerUser,
         method: ResourceMethod = "GET",
+        team_name: str | None = None,
     ):
         requests: dict[str, dict[ResourceMethod, IsAuthorizedRequest]] = defaultdict(dict)
         requests_list: list[IsAuthorizedRequest] = []

{apache_airflow_providers_amazon-9.13.0 → apache_airflow_providers_amazon-9.14.0}/src/airflow/providers/amazon/aws/executors/ecs/ecs_executor.py

@@ -32,7 +32,6 @@ from typing import TYPE_CHECKING
 
 from botocore.exceptions import ClientError, NoCredentialsError
 
-from airflow.configuration import conf
 from airflow.exceptions import AirflowException
 from airflow.executors.base_executor import BaseExecutor
 from airflow.providers.amazon.aws.executors.ecs.boto_schema import BotoDescribeTasksSchema, BotoRunTaskSchema
@@ -98,13 +97,6 @@ class AwsEcsExecutor(BaseExecutor):
      Airflow TaskInstance's executor_config.
     """
 
-    # Maximum number of retries to run an ECS task.
-    MAX_RUN_TASK_ATTEMPTS = conf.get(
-        CONFIG_GROUP_NAME,
-        AllEcsConfigKeys.MAX_RUN_TASK_ATTEMPTS,
-        fallback=CONFIG_DEFAULTS[AllEcsConfigKeys.MAX_RUN_TASK_ATTEMPTS],
-    )
-
     # AWS limits the maximum number of ARNs in the describe_tasks function.
     DESCRIBE_TASKS_BATCH_SIZE = 99
 
@@ -118,8 +110,18 @@ class AwsEcsExecutor(BaseExecutor):
         self.active_workers: EcsTaskCollection = EcsTaskCollection()
         self.pending_tasks: deque = deque()
 
-        self.cluster = conf.get(CONFIG_GROUP_NAME, AllEcsConfigKeys.CLUSTER)
-        self.container_name = conf.get(CONFIG_GROUP_NAME, AllEcsConfigKeys.CONTAINER_NAME)
+        # Check if self has the ExecutorConf set on the self.conf attribute, and if not, set it to the global
+        # configuration object. This allows the changes to be backwards compatible with older versions of
+        # Airflow.
+        # Can be removed when minimum supported provider version is equal to the version of core airflow
+        # which introduces multi-team configuration.
+        if not hasattr(self, "conf"):
+            from airflow.configuration import conf
+
+            self.conf = conf
+
+        self.cluster = self.conf.get(CONFIG_GROUP_NAME, AllEcsConfigKeys.CLUSTER)
+        self.container_name = self.conf.get(CONFIG_GROUP_NAME, AllEcsConfigKeys.CONTAINER_NAME)
         self.attempts_since_last_successful_connection = 0
 
         self.load_ecs_connection(check_connection=False)
@@ -127,6 +129,13 @@ class AwsEcsExecutor(BaseExecutor):
 
         self.run_task_kwargs = self._load_run_kwargs()
 
+        # Maximum number of retries to run an ECS task.
+        self.max_run_task_attempts = self.conf.get(
+            CONFIG_GROUP_NAME,
+            AllEcsConfigKeys.MAX_RUN_TASK_ATTEMPTS,
+            fallback=CONFIG_DEFAULTS[AllEcsConfigKeys.MAX_RUN_TASK_ATTEMPTS],
+        )
+
     def queue_workload(self, workload: workloads.All, session: Session | None) -> None:
         from airflow.executors import workloads
 
@@ -154,7 +163,7 @@ class AwsEcsExecutor(BaseExecutor):
 
     def start(self):
         """Call this when the Executor is run for the first time by the scheduler."""
-        check_health = conf.getboolean(
+        check_health = self.conf.getboolean(
             CONFIG_GROUP_NAME, AllEcsConfigKeys.CHECK_HEALTH_ON_STARTUP, fallback=False
         )
 
@@ -218,12 +227,12 @@ class AwsEcsExecutor(BaseExecutor):
 
     def load_ecs_connection(self, check_connection: bool = True):
         self.log.info("Loading Connection information")
-        aws_conn_id = conf.get(
+        aws_conn_id = self.conf.get(
             CONFIG_GROUP_NAME,
             AllEcsConfigKeys.AWS_CONN_ID,
             fallback=CONFIG_DEFAULTS[AllEcsConfigKeys.AWS_CONN_ID],
         )
-        region_name = conf.get(CONFIG_GROUP_NAME, AllEcsConfigKeys.REGION_NAME, fallback=None)
+        region_name = self.conf.get(CONFIG_GROUP_NAME, AllEcsConfigKeys.REGION_NAME, fallback=None)
         self.ecs = EcsHook(aws_conn_id=aws_conn_id, region_name=region_name).conn
         self.attempts_since_last_successful_connection += 1
         self.last_connection_reload = timezone.utcnow()
@@ -340,13 +349,13 @@ class AwsEcsExecutor(BaseExecutor):
         queue = task_info.queue
         exec_info = task_info.config
         failure_count = self.active_workers.failure_count_by_key(task_key)
-        if int(failure_count) < int(self.__class__.MAX_RUN_TASK_ATTEMPTS):
+        if int(failure_count) < int(self.max_run_task_attempts):
             self.log.warning(
                 "Airflow task %s failed due to %s. Failure %s out of %s occurred on %s. Rescheduling.",
                 task_key,
                 reason,
                 failure_count,
-                self.__class__.MAX_RUN_TASK_ATTEMPTS,
+                self.max_run_task_attempts,
                 task_arn,
             )
             self.pending_tasks.append(
@@ -416,8 +425,8 @@ class AwsEcsExecutor(BaseExecutor):
                     failure_reasons.extend([f["reason"] for f in run_task_response["failures"]])
 
             if failure_reasons:
-                # Make sure the number of attempts does not exceed MAX_RUN_TASK_ATTEMPTS
-                if int(attempt_number) < int(self.__class__.MAX_RUN_TASK_ATTEMPTS):
+                # Make sure the number of attempts does not exceed max_run_task_attempts
+                if int(attempt_number) < int(self.max_run_task_attempts):
                     ecs_task.attempt_number += 1
                     ecs_task.next_attempt_time = timezone.utcnow() + calculate_next_attempt_delay(
                         attempt_number
@@ -545,7 +554,7 @@ class AwsEcsExecutor(BaseExecutor):
     def _load_run_kwargs(self) -> dict:
         from airflow.providers.amazon.aws.executors.ecs.ecs_executor_config import build_task_kwargs
 
-        ecs_executor_run_task_kwargs = build_task_kwargs()
+        ecs_executor_run_task_kwargs = build_task_kwargs(self.conf)
 
         try:
             self.get_container(ecs_executor_run_task_kwargs["overrides"]["containerOverrides"])["command"]

{apache_airflow_providers_amazon-9.13.0 → apache_airflow_providers_amazon-9.14.0}/src/airflow/providers/amazon/aws/executors/ecs/ecs_executor_config.py

@@ -32,7 +32,6 @@ from __future__ import annotations
 import json
 from json import JSONDecodeError
 
-from airflow.configuration import conf
 from airflow.providers.amazon.aws.executors.ecs.utils import (
     CONFIG_GROUP_NAME,
     ECS_LAUNCH_TYPE_EC2,
@@ -46,23 +45,27 @@ from airflow.providers.amazon.aws.hooks.ecs import EcsHook
 from airflow.utils.helpers import prune_dict
 
 
-def _fetch_templated_kwargs() -> dict[str, str]:
-    run_task_kwargs_value = conf.get(CONFIG_GROUP_NAME, AllEcsConfigKeys.RUN_TASK_KWARGS, fallback=dict())
+def _fetch_templated_kwargs(conf) -> dict[str, str]:
+    run_task_kwargs_value = conf.get(
+        CONFIG_GROUP_NAME,
+        AllEcsConfigKeys.RUN_TASK_KWARGS,
+        fallback=dict(),
+    )
     return json.loads(str(run_task_kwargs_value))
 
 
-def _fetch_config_values() -> dict[str, str]:
+def _fetch_config_values(conf) -> dict[str, str]:
     return prune_dict(
         {key: conf.get(CONFIG_GROUP_NAME, key, fallback=None) for key in RunTaskKwargsConfigKeys()}
     )
 
 
-def build_task_kwargs() -> dict:
+def build_task_kwargs(conf) -> dict:
     all_config_keys = AllEcsConfigKeys()
     # This will put some kwargs at the root of the dictionary that do NOT belong there. However,
     # the code below expects them to be there and will rearrange them as necessary.
-    task_kwargs = _fetch_config_values()
-    task_kwargs.update(_fetch_templated_kwargs())
+    task_kwargs = _fetch_config_values(conf)
+    task_kwargs.update(_fetch_templated_kwargs(conf))
 
     has_launch_type: bool = all_config_keys.LAUNCH_TYPE in task_kwargs
     has_capacity_provider: bool = all_config_keys.CAPACITY_PROVIDER_STRATEGY in task_kwargs

{apache_airflow_providers_amazon-9.13.0 → apache_airflow_providers_amazon-9.14.0}/src/airflow/providers/amazon/aws/hooks/glue.py

@@ -109,7 +109,7 @@ class GlueJobHook(AwsBaseHook):
             "retry": retry_if_exception(self._should_retry_on_error),
             "wait": wait_exponential(multiplier=1, min=1, max=60),
             "stop": stop_after_attempt(5),
-            "before_sleep": before_sleep_log(self.log, log_level=20),
+            "before_sleep": before_sleep_log(self.log, log_level=20),  # type: ignore[arg-type]
             "reraise": True,
         }
 

{apache_airflow_providers_amazon-9.13.0 → apache_airflow_providers_amazon-9.14.0}/src/airflow/providers/amazon/aws/hooks/redshift_cluster.py

@@ -57,10 +57,9 @@ class RedshiftHook(AwsBaseHook):
             - :external+boto3:py:meth:`Redshift.Client.create_cluster`
 
         :param cluster_identifier: A unique identifier for the cluster.
-        :param node_type: The node type to be provisioned for the cluster.
-            Valid Values: ``ds2.xlarge``, ``ds2.8xlarge``, ``dc1.large``,
-            ``dc1.8xlarge``, ``dc2.large``, ``dc2.8xlarge``, ``ra3.xlplus``,
-            ``ra3.4xlarge``, and ``ra3.16xlarge``.
+        :param node_type: The node type to be provisioned for the cluster. Refer
+            https://docs.aws.amazon.com/redshift/latest/mgmt/working-with-clusters.html#rs-node-type-info
+            for the list of available node types.
         :param master_username: The username associated with the admin user account
             for the cluster that is being created.
         :param master_user_password: password associated with the admin user account

{apache_airflow_providers_amazon-9.13.0 → apache_airflow_providers_amazon-9.14.0}/src/airflow/providers/amazon/aws/log/cloudwatch_task_handler.py

@@ -230,8 +230,19 @@ class CloudwatchTaskHandler(FileTaskHandler, LoggingMixin):
 
     trigger_should_wrap = True
 
-    def __init__(self, base_log_folder: str, log_group_arn: str, **kwargs):
-        super().__init__(base_log_folder)
+    def __init__(
+        self,
+        base_log_folder: str,
+        log_group_arn: str,
+        max_bytes: int = 0,
+        backup_count: int = 0,
+        delay: bool = False,
+        **kwargs,
+    ) -> None:
+        # support log file size handling of FileTaskHandler
+        super().__init__(
+            base_log_folder=base_log_folder, max_bytes=max_bytes, backup_count=backup_count, delay=delay
+        )
         split_arn = log_group_arn.split(":")
 
         self.handler = None

{apache_airflow_providers_amazon-9.13.0 → apache_airflow_providers_amazon-9.14.0}/src/airflow/providers/amazon/aws/log/s3_task_handler.py

@@ -172,8 +172,19 @@ class S3TaskHandler(FileTaskHandler, LoggingMixin):
     It extends airflow FileTaskHandler and uploads to and reads from S3 remote storage.
     """
 
-    def __init__(self, base_log_folder: str, s3_log_folder: str, **kwargs):
-        super().__init__(base_log_folder)
+    def __init__(
+        self,
+        base_log_folder: str,
+        s3_log_folder: str,
+        max_bytes: int = 0,
+        backup_count: int = 0,
+        delay: bool = False,
+        **kwargs,
+    ) -> None:
+        # support log file size handling of FileTaskHandler
+        super().__init__(
+            base_log_folder=base_log_folder, max_bytes=max_bytes, backup_count=backup_count, delay=delay
+        )
         self.handler: logging.FileHandler | None = None
         self.remote_base = s3_log_folder
         self.log_relative_path = ""

{apache_airflow_providers_amazon-9.13.0 → apache_airflow_providers_amazon-9.14.0}/src/airflow/providers/amazon/aws/operators/batch.py

@@ -256,8 +256,14 @@ class BatchOperator(AwsBaseOperator[BatchClientHook]):
         if validated_event["status"] != "success":
             raise AirflowException(f"Error while running job: {validated_event}")
 
-        self.log.info("Job completed.")
-        return validated_event["job_id"]
+        self.job_id = validated_event["job_id"]
+
+        # Fetch logs if awslogs_enabled
+        if self.awslogs_enabled:
+            self.monitor_job(context)  # fetch logs, no need to return
+
+        self.log.info("Job completed successfully for job_id: %s", self.job_id)
+        return self.job_id
 
     def on_kill(self):
         response = self.hook.client.terminate_job(jobId=self.job_id, reason="Task killed by the user")