apache-airflow-providers-amazon 9.19.0__py3-none-any.whl → 9.21.0__py3-none-any.whl

airflow/providers/amazon/__init__.py

@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "9.19.0"
+__version__ = "9.21.0"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.11.0"

airflow/providers/amazon/aws/auth_manager/avp/facade.py

@@ -22,7 +22,6 @@ from functools import cached_property
 from pathlib import Path
 from typing import TYPE_CHECKING, TypedDict
 
-from airflow.configuration import conf
 from airflow.providers.amazon.aws.auth_manager.avp.entities import AvpEntities, get_action_id, get_entity_type
 from airflow.providers.amazon.aws.auth_manager.constants import (
     CONF_AVP_POLICY_STORE_ID_KEY,
@@ -31,7 +30,7 @@ from airflow.providers.amazon.aws.auth_manager.constants import (
     CONF_SECTION_NAME,
 )
 from airflow.providers.amazon.aws.hooks.verified_permissions import VerifiedPermissionsHook
-from airflow.providers.common.compat.sdk import AirflowException
+from airflow.providers.common.compat.sdk import AirflowException, conf
 from airflow.utils.helpers import prune_dict
 from airflow.utils.log.logging_mixin import LoggingMixin
 

airflow/providers/amazon/aws/auth_manager/aws_auth_manager.py

@@ -16,7 +16,6 @@
 # under the License.
 from __future__ import annotations
 
-import argparse
 from collections import defaultdict
 from collections.abc import Sequence
 from functools import cached_property
@@ -27,19 +26,15 @@ from fastapi import FastAPI
 
 from airflow.api_fastapi.app import AUTH_MANAGER_FASTAPI_APP_PREFIX
 from airflow.api_fastapi.auth.managers.base_auth_manager import BaseAuthManager
-from airflow.cli.cli_config import CLICommand, DefaultHelpParser, GroupCommand
-from airflow.configuration import conf
-from airflow.exceptions import AirflowOptionalProviderFeatureException
+from airflow.cli.cli_config import CLICommand
 from airflow.providers.amazon.aws.auth_manager.avp.entities import AvpEntities
 from airflow.providers.amazon.aws.auth_manager.avp.facade import (
     AwsAuthManagerAmazonVerifiedPermissionsFacade,
     IsAuthorizedRequest,
 )
-from airflow.providers.amazon.aws.auth_manager.cli.definition import (
-    AWS_AUTH_MANAGER_COMMANDS,
-)
 from airflow.providers.amazon.aws.auth_manager.user import AwsAuthManagerUser
 from airflow.providers.amazon.version_compat import AIRFLOW_V_3_0_PLUS
+from airflow.providers.common.compat.sdk import AirflowOptionalProviderFeatureException, conf
 
 if TYPE_CHECKING:
     from airflow.api_fastapi.auth.managers.base_auth_manager import ResourceMethod
@@ -62,6 +57,7 @@ if TYPE_CHECKING:
         VariableDetails,
     )
     from airflow.api_fastapi.common.types import MenuItem
+    from airflow.cli.cli_config import CLICommand
 
 
 class AwsAuthManager(BaseAuthManager[AwsAuthManagerUser]):
@@ -468,13 +464,9 @@ class AwsAuthManager(BaseAuthManager[AwsAuthManagerUser]):
     @staticmethod
     def get_cli_commands() -> list[CLICommand]:
         """Vends CLI commands to be included in Airflow CLI."""
-        return [
-            GroupCommand(
-                name="aws-auth-manager",
-                help="Manage resources used by AWS auth manager",
-                subcommands=AWS_AUTH_MANAGER_COMMANDS,
-            ),
-        ]
+        from airflow.providers.amazon.aws.cli.definition import get_aws_cli_commands
+
+        return get_aws_cli_commands()
 
     def get_fastapi_app(self) -> FastAPI | None:
         from airflow.providers.amazon.aws.auth_manager.routes.login import login_router
@@ -515,14 +507,3 @@ class AwsAuthManager(BaseAuthManager[AwsAuthManagerUser]):
                 "Please update it to its latest version. "
                 "See doc: https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/auth-manager/setup/amazon-verified-permissions.html#update-the-policy-store-schema."
             )
-
-
-def get_parser() -> argparse.ArgumentParser:
-    """Generate documentation; used by Sphinx argparse."""
-    from airflow.cli.cli_parser import AirflowHelpFormatter, _add_command
-
-    parser = DefaultHelpParser(prog="airflow", formatter_class=AirflowHelpFormatter)
-    subparsers = parser.add_subparsers(dest="subcommand", metavar="GROUP_OR_COMMAND")
-    for group_command in AwsAuthManager.get_cli_commands():
-        _add_command(subparsers, group_command)
-    return parser

airflow/providers/amazon/aws/auth_manager/cli/avp_commands.py

@@ -22,9 +22,8 @@ from typing import TYPE_CHECKING
 
 import boto3
 
-from airflow.configuration import conf
-from airflow.exceptions import AirflowOptionalProviderFeatureException
 from airflow.providers.amazon.aws.auth_manager.constants import CONF_REGION_NAME_KEY, CONF_SECTION_NAME
+from airflow.providers.common.compat.sdk import AirflowOptionalProviderFeatureException, conf
 from airflow.utils import cli as cli_utils
 
 try:

airflow/providers/amazon/aws/auth_manager/routes/login.py

@@ -21,9 +21,8 @@ import logging
 from typing import Any
 
 import anyio
-from fastapi import HTTPException, Request
-from starlette import status
-from starlette.responses import RedirectResponse
+from fastapi import HTTPException, Request, status
+from fastapi.responses import RedirectResponse
 
 from airflow.api_fastapi.app import (
     AUTH_MANAGER_FASTAPI_APP_PREFIX,
@@ -31,11 +30,11 @@ from airflow.api_fastapi.app import (
 )
 from airflow.api_fastapi.auth.managers.base_auth_manager import COOKIE_NAME_JWT_TOKEN
 from airflow.api_fastapi.common.router import AirflowRouter
-from airflow.configuration import conf
 from airflow.providers.amazon.aws.auth_manager.constants import CONF_SAML_METADATA_URL_KEY, CONF_SECTION_NAME
 from airflow.providers.amazon.aws.auth_manager.datamodels.login import LoginResponse
 from airflow.providers.amazon.aws.auth_manager.user import AwsAuthManagerUser
 from airflow.providers.amazon.version_compat import AIRFLOW_V_3_1_1_PLUS
+from airflow.providers.common.compat.sdk import conf
 
 try:
     from onelogin.saml2.auth import OneLogin_Saml2_Auth

airflow/providers/amazon/aws/auth_manager/user.py

@@ -16,7 +16,7 @@
 # under the License.
 from __future__ import annotations
 
-from airflow.exceptions import AirflowOptionalProviderFeatureException
+from airflow.providers.common.compat.sdk import AirflowOptionalProviderFeatureException
 
 try:
     from airflow.api_fastapi.auth.managers.models.base_user import BaseUser

airflow/providers/amazon/aws/cli/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

airflow/providers/amazon/aws/{auth_manager/cli → cli}/definition.py

@@ -17,12 +17,17 @@
 
 from __future__ import annotations
 
+from typing import TYPE_CHECKING
+
 from airflow.cli.cli_config import (
     ActionCommand,
     Arg,
     lazy_load_command,
 )
 
+if TYPE_CHECKING:
+    import argparse
+
 ############
 # # ARGS # #
 ############
@@ -58,3 +63,31 @@ AWS_AUTH_MANAGER_COMMANDS = (
         args=(ARG_POLICY_STORE_ID, ARG_DRY_RUN),
     ),
 )
+
+
+def get_aws_cli_commands():
+    """Return CLI commands for AWS auth manager."""
+    from airflow.cli.cli_config import GroupCommand
+
+    return [
+        GroupCommand(
+            name="aws-auth-manager",
+            help="Manage resources used by AWS auth manager",
+            subcommands=AWS_AUTH_MANAGER_COMMANDS,
+        ),
+    ]
+
+
+def get_parser() -> argparse.ArgumentParser:
+    """
+    Generate documentation; used by Sphinx argparse.
+
+    :meta private:
+    """
+    from airflow.cli.cli_parser import AirflowHelpFormatter, DefaultHelpParser, _add_command
+
+    parser = DefaultHelpParser(prog="airflow", formatter_class=AirflowHelpFormatter)
+    subparsers = parser.add_subparsers(dest="subcommand", metavar="GROUP_OR_COMMAND")
+    for group_command in get_aws_cli_commands():
+        _add_command(subparsers, group_command)
+    return parser

airflow/providers/amazon/aws/executors/aws_lambda/lambda_executor.py

@@ -25,7 +25,6 @@ from typing import TYPE_CHECKING
 from boto3.session import NoCredentialsError
 from botocore.utils import ClientError
 
-from airflow.configuration import conf
 from airflow.executors.base_executor import BaseExecutor
 from airflow.models.taskinstancekey import TaskInstanceKey
 from airflow.providers.amazon.aws.executors.aws_lambda.utils import (
@@ -41,14 +40,8 @@ from airflow.providers.amazon.aws.executors.utils.exponential_backoff_retry impo
 )
 from airflow.providers.amazon.aws.hooks.lambda_function import LambdaHook
 from airflow.providers.amazon.aws.hooks.sqs import SqsHook
-from airflow.providers.common.compat.sdk import AirflowException, Stats
-
-try:
-    from airflow.sdk import timezone
-except ImportError:
-    from airflow.utils import timezone  # type: ignore[attr-defined,no-redef]
-
 from airflow.providers.amazon.version_compat import AIRFLOW_V_3_0_PLUS
+from airflow.providers.common.compat.sdk import AirflowException, Stats, conf, timezone
 
 if TYPE_CHECKING:
     from sqlalchemy.orm import Session
@@ -499,7 +492,7 @@ class AwsLambdaExecutor(BaseExecutor):
         :param heartbeat_interval: The interval in seconds to wait between checks for task completion.
         """
         self.log.info("Received signal to end, waiting for outstanding tasks to finish.")
-        time_to_wait = int(conf.get(CONFIG_GROUP_NAME, AllLambdaConfigKeys.END_WAIT_TIMEOUT))
+        time_to_wait = int(conf.get(CONFIG_GROUP_NAME, AllLambdaConfigKeys.END_WAIT_TIMEOUT, fallback="0"))
         start_time = timezone.utcnow()
         while True:
             if time_to_wait:

airflow/providers/amazon/aws/executors/aws_lambda/utils.py

@@ -59,7 +59,7 @@ class AllLambdaConfigKeys(InvokeLambdaKwargsConfigKeys):
 
     AWS_CONN_ID = "conn_id"
     CHECK_HEALTH_ON_STARTUP = "check_health_on_startup"
-    MAX_INVOKE_ATTEMPTS = "max_run_task_attempts"
+    MAX_INVOKE_ATTEMPTS = "max_invoke_attempts"
     REGION_NAME = "region_name"
     QUEUE_URL = "queue_url"
     DLQ_URL = "dead_letter_queue_url"

airflow/providers/amazon/aws/executors/batch/batch_executor.py

@@ -22,13 +22,11 @@ from __future__ import annotations
 import time
 from collections import deque
 from collections.abc import Sequence
-from contextlib import suppress
 from copy import deepcopy
 from typing import TYPE_CHECKING, Any
 
 from botocore.exceptions import ClientError, NoCredentialsError
 
-from airflow.configuration import conf
 from airflow.executors.base_executor import BaseExecutor
 from airflow.providers.amazon.aws.executors.utils.exponential_backoff_retry import (
     calculate_next_attempt_delay,
@@ -36,12 +34,7 @@ from airflow.providers.amazon.aws.executors.utils.exponential_backoff_retry impo
 )
 from airflow.providers.amazon.aws.hooks.batch_client import BatchClientHook
 from airflow.providers.amazon.version_compat import AIRFLOW_V_3_0_PLUS
-from airflow.providers.common.compat.sdk import AirflowException, Stats
-
-try:
-    from airflow.sdk import timezone
-except ImportError:
-    from airflow.utils import timezone  # type: ignore[attr-defined,no-redef]
+from airflow.providers.common.compat.sdk import AirflowException, Stats, timezone
 from airflow.utils.helpers import merge_dicts
 
 if TYPE_CHECKING:
@@ -94,12 +87,7 @@ class AwsBatchExecutor(BaseExecutor):
     Airflow TaskInstance's executor_config.
     """
 
-    # Maximum number of retries to submit a Batch Job.
-    MAX_SUBMIT_JOB_ATTEMPTS = conf.get(
-        CONFIG_GROUP_NAME,
-        AllBatchConfigKeys.MAX_SUBMIT_JOB_ATTEMPTS,
-        fallback=CONFIG_DEFAULTS[AllBatchConfigKeys.MAX_SUBMIT_JOB_ATTEMPTS],
-    )
+    supports_multi_team: bool = True
 
     # AWS only allows a maximum number of JOBs in the describe_jobs function
     DESCRIBE_JOBS_BATCH_SIZE = 99
@@ -113,11 +101,29 @@ class AwsBatchExecutor(BaseExecutor):
         super().__init__(*args, **kwargs)
         self.active_workers = BatchJobCollection()
         self.pending_jobs: deque = deque()
+
+        # Check if self has the ExecutorConf set on the self.conf attribute, and if not, set it to the global
+        # configuration object. This allows the changes to be backwards compatible with older versions of
+        # Airflow.
+        # Can be removed when minimum supported provider version is equal to the version of core airflow
+        # which introduces multi-team configuration.
+        if not hasattr(self, "conf"):
+            from airflow.providers.common.compat.sdk import conf
+
+            self.conf = conf
+
         self.attempts_since_last_successful_connection = 0
         self.load_batch_connection(check_connection=False)
         self.IS_BOTO_CONNECTION_HEALTHY = False
         self.submit_job_kwargs = self._load_submit_kwargs()
 
+        # Maximum number of retries to submit a Batch job.
+        self.max_submit_job_attempts = self.conf.get(
+            CONFIG_GROUP_NAME,
+            AllBatchConfigKeys.MAX_SUBMIT_JOB_ATTEMPTS,
+            fallback=CONFIG_DEFAULTS[AllBatchConfigKeys.MAX_SUBMIT_JOB_ATTEMPTS],
+        )
+
     def queue_workload(self, workload: workloads.All, session: Session | None) -> None:
         from airflow.executors import workloads
 
@@ -171,7 +177,7 @@ class AwsBatchExecutor(BaseExecutor):
 
     def start(self):
         """Call this when the Executor is run for the first time by the scheduler."""
-        check_health = conf.getboolean(
+        check_health = self.conf.getboolean(
             CONFIG_GROUP_NAME, AllBatchConfigKeys.CHECK_HEALTH_ON_STARTUP, fallback=False
         )
 
@@ -187,12 +193,12 @@ class AwsBatchExecutor(BaseExecutor):
 
     def load_batch_connection(self, check_connection: bool = True):
         self.log.info("Loading Connection information")
-        aws_conn_id = conf.get(
+        aws_conn_id = self.conf.get(
             CONFIG_GROUP_NAME,
             AllBatchConfigKeys.AWS_CONN_ID,
             fallback=CONFIG_DEFAULTS[AllBatchConfigKeys.AWS_CONN_ID],
         )
-        region_name = conf.get(CONFIG_GROUP_NAME, AllBatchConfigKeys.REGION_NAME, fallback=None)
+        region_name = self.conf.get(CONFIG_GROUP_NAME, AllBatchConfigKeys.REGION_NAME, fallback=None)
         self.batch = BatchClientHook(aws_conn_id=aws_conn_id, region_name=region_name).conn
         self.attempts_since_last_successful_connection += 1
         self.last_connection_reload = timezone.utcnow()
@@ -262,13 +268,13 @@ class AwsBatchExecutor(BaseExecutor):
         queue = job_info.queue
         exec_info = job_info.config
         failure_count = self.active_workers.failure_count_by_id(job_id=job.job_id)
-        if int(failure_count) < int(self.__class__.MAX_SUBMIT_JOB_ATTEMPTS):
+        if int(failure_count) < int(self.max_submit_job_attempts):
             self.log.warning(
                 "Airflow task %s failed due to %s. Failure %s out of %s occurred on %s. Rescheduling.",
                 task_key,
                 job.status_reason,
                 failure_count,
-                self.__class__.MAX_SUBMIT_JOB_ATTEMPTS,
+                self.max_submit_job_attempts,
                 job.job_id,
             )
             self.active_workers.increment_failure_count(job_id=job.job_id)
@@ -327,7 +333,7 @@ class AwsBatchExecutor(BaseExecutor):
                 failure_reason = str(e)
 
             if failure_reason:
-                if attempt_number >= int(self.__class__.MAX_SUBMIT_JOB_ATTEMPTS):
+                if attempt_number >= int(self.max_submit_job_attempts):
                     self.log.error(
                         (
                             "This job has been unsuccessfully attempted too many times (%s). "
@@ -466,11 +472,10 @@ class AwsBatchExecutor(BaseExecutor):
             # up and kill the scheduler process.
             self.log.exception("Failed to terminate %s", self.__class__.__name__)
 
-    @staticmethod
-    def _load_submit_kwargs() -> dict:
+    def _load_submit_kwargs(self) -> dict:
         from airflow.providers.amazon.aws.executors.batch.batch_executor_config import build_submit_kwargs
 
-        submit_kwargs = build_submit_kwargs()
+        submit_kwargs = build_submit_kwargs(self.conf)
 
         if "containerOverrides" not in submit_kwargs or "command" not in submit_kwargs["containerOverrides"]:
             raise KeyError(
@@ -514,12 +519,3 @@ class AwsBatchExecutor(BaseExecutor):
 
             not_adopted_tis = [ti for ti in tis if ti not in adopted_tis]
             return not_adopted_tis
-
-    def log_task_event(self, *, event: str, extra: str, ti_key: TaskInstanceKey):
-        # TODO: remove this method when min_airflow_version is set to higher than 2.10.0
-        with suppress(AttributeError):
-            super().log_task_event(
-                event=event,
-                extra=extra,
-                ti_key=ti_key,
-            )

airflow/providers/amazon/aws/executors/batch/batch_executor_config.py

@@ -33,7 +33,6 @@ import json
 from json import JSONDecodeError
 from typing import TYPE_CHECKING
 
-from airflow.configuration import conf
 from airflow.providers.amazon.aws.executors.batch.utils import (
     CONFIG_GROUP_NAME,
     AllBatchConfigKeys,
@@ -43,22 +42,22 @@ from airflow.providers.amazon.aws.executors.ecs.utils import camelize_dict_keys
 from airflow.utils.helpers import prune_dict
 
 
-def _fetch_templated_kwargs() -> dict[str, str]:
+def _fetch_templated_kwargs(conf) -> dict[str, str]:
     submit_job_kwargs_value = conf.get(
         CONFIG_GROUP_NAME, AllBatchConfigKeys.SUBMIT_JOB_KWARGS, fallback=dict()
     )
     return json.loads(str(submit_job_kwargs_value))
 
 
-def _fetch_config_values() -> dict[str, str]:
+def _fetch_config_values(conf) -> dict[str, str]:
     return prune_dict(
         {key: conf.get(CONFIG_GROUP_NAME, key, fallback=None) for key in BatchSubmitJobKwargsConfigKeys()}
     )
 
 
-def build_submit_kwargs() -> dict:
-    job_kwargs = _fetch_config_values()
-    job_kwargs.update(_fetch_templated_kwargs())
+def build_submit_kwargs(conf) -> dict:
+    job_kwargs = _fetch_config_values(conf)
+    job_kwargs.update(_fetch_templated_kwargs(conf))
 
     if "containerOverrides" not in job_kwargs:
         job_kwargs["containerOverrides"] = {}  # type: ignore

airflow/providers/amazon/aws/executors/ecs/ecs_executor.py

@@ -26,7 +26,6 @@ from __future__ import annotations
 import time
 from collections import defaultdict, deque
 from collections.abc import Sequence
-from contextlib import suppress
 from copy import deepcopy
 from typing import TYPE_CHECKING
 
@@ -48,12 +47,7 @@ from airflow.providers.amazon.aws.executors.utils.exponential_backoff_retry impo
 )
 from airflow.providers.amazon.aws.hooks.ecs import EcsHook
 from airflow.providers.amazon.version_compat import AIRFLOW_V_3_0_PLUS
-from airflow.providers.common.compat.sdk import AirflowException, Stats
-
-try:
-    from airflow.sdk import timezone
-except ImportError:
-    from airflow.utils import timezone  # type: ignore[attr-defined,no-redef]
+from airflow.providers.common.compat.sdk import AirflowException, Stats, timezone
 from airflow.utils.helpers import merge_dicts
 from airflow.utils.state import State
 
@@ -96,6 +90,8 @@ class AwsEcsExecutor(BaseExecutor):
      Airflow TaskInstance's executor_config.
     """
 
+    supports_multi_team: bool = True
+
     # AWS limits the maximum number of ARNs in the describe_tasks function.
     DESCRIBE_TASKS_BATCH_SIZE = 99
 
@@ -115,7 +111,7 @@ class AwsEcsExecutor(BaseExecutor):
         # Can be removed when minimum supported provider version is equal to the version of core airflow
         # which introduces multi-team configuration.
         if not hasattr(self, "conf"):
-            from airflow.configuration import conf
+            from airflow.providers.common.compat.sdk import conf
 
             self.conf = conf
 
@@ -611,12 +607,3 @@ class AwsEcsExecutor(BaseExecutor):
 
             not_adopted_tis = [ti for ti in tis if ti not in adopted_tis]
             return not_adopted_tis
-
-    def log_task_event(self, *, event: str, extra: str, ti_key: TaskInstanceKey):
-        # TODO: remove this method when min_airflow_version is set to higher than 2.10.0
-        with suppress(AttributeError):
-            super().log_task_event(
-                event=event,
-                extra=extra,
-                ti_key=ti_key,
-            )

airflow/providers/amazon/aws/executors/ecs/ecs_executor_config.py

@@ -33,6 +33,7 @@ import json
 from json import JSONDecodeError
 
 from airflow.providers.amazon.aws.executors.ecs.utils import (
+    CONFIG_DEFAULTS,
     CONFIG_GROUP_NAME,
     ECS_LAUNCH_TYPE_EC2,
     ECS_LAUNCH_TYPE_FARGATE,
@@ -56,7 +57,10 @@ def _fetch_templated_kwargs(conf) -> dict[str, str]:
 
 def _fetch_config_values(conf) -> dict[str, str]:
     return prune_dict(
-        {key: conf.get(CONFIG_GROUP_NAME, key, fallback=None) for key in RunTaskKwargsConfigKeys()}
+        {
+            key: conf.get(CONFIG_GROUP_NAME, key, fallback=CONFIG_DEFAULTS.get(key, None))
+            for key in RunTaskKwargsConfigKeys()
+        }
     )
 
 

airflow/providers/amazon/aws/hooks/athena_sql.py

@@ -21,7 +21,11 @@ from functools import cached_property
 from typing import TYPE_CHECKING, Any
 
 import pyathena
-from sqlalchemy.engine.url import URL
+
+try:
+    from sqlalchemy.engine.url import URL
+except ImportError:
+    URL = None  # type: ignore[assignment,misc]
 
 from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
 from airflow.providers.amazon.aws.utils.connection_wrapper import AwsConnectionWrapper
@@ -152,9 +156,15 @@ class AthenaSQLHook(AwsBaseHook, DbApiHook):
 
     def get_uri(self) -> str:
         """Overridden to use the Athena dialect as driver name."""
+        from airflow.providers.common.compat.sdk import AirflowOptionalProviderFeatureException
+
+        if URL is None:
+            raise AirflowOptionalProviderFeatureException(
+                "sqlalchemy is required to generate the connection URI. "
+                "Install it with: pip install 'apache-airflow-providers-amazon[sqlalchemy]'"
+            )
         conn_params = self._get_conn_params()
         creds = self.get_credentials(region_name=conn_params["region_name"])
-
         return URL.create(
             f"awsathena+{conn_params['driver']}",
             username=creds.access_key,

airflow/providers/amazon/aws/hooks/base_aws.py

@@ -51,12 +51,11 @@ from botocore.waiter import Waiter, WaiterModel
 from dateutil.tz import tzlocal
 from slugify import slugify
 
-from airflow.configuration import conf
 from airflow.exceptions import AirflowProviderDeprecationWarning
 from airflow.providers.amazon.aws.utils.connection_wrapper import AwsConnectionWrapper
 from airflow.providers.amazon.aws.utils.identifiers import generate_uuid
 from airflow.providers.amazon.aws.utils.suppress import return_on_error
-from airflow.providers.common.compat.sdk import AirflowException, AirflowNotFoundException, BaseHook
+from airflow.providers.common.compat.sdk import AirflowException, AirflowNotFoundException, BaseHook, conf
 from airflow.providers_manager import ProvidersManager
 from airflow.utils.helpers import exactly_one
 from airflow.utils.log.logging_mixin import LoggingMixin

airflow/providers/amazon/aws/hooks/redshift_sql.py

@@ -22,11 +22,15 @@ from typing import TYPE_CHECKING
 import redshift_connector
 import tenacity
 from redshift_connector import Connection as RedshiftConnection, InterfaceError, OperationalError
-from sqlalchemy import create_engine
-from sqlalchemy.engine.url import URL
+
+try:
+    from sqlalchemy import create_engine
+    from sqlalchemy.engine.url import URL
+except ImportError:
+    URL = create_engine = None  # type: ignore[assignment,misc]
 
 from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
-from airflow.providers.common.compat.sdk import AirflowException
+from airflow.providers.common.compat.sdk import AirflowException, AirflowOptionalProviderFeatureException
 from airflow.providers.common.sql.hooks.sql import DbApiHook
 
 if TYPE_CHECKING:
@@ -151,6 +155,11 @@ class RedshiftSQLHook(DbApiHook):
 
     def get_uri(self) -> str:
         """Overridden to use the Redshift dialect as driver name."""
+        if URL is None:
+            raise AirflowOptionalProviderFeatureException(
+                "sqlalchemy is required to generate the connection URI. "
+                "Install it with: pip install 'apache-airflow-providers-amazon[sqlalchemy]'"
+            )
         conn_params = self._get_conn_params()
 
         if "user" in conn_params:
@@ -174,6 +183,11 @@ class RedshiftSQLHook(DbApiHook):
 
     def get_sqlalchemy_engine(self, engine_kwargs=None):
         """Overridden to pass Redshift-specific arguments."""
+        if create_engine is None:
+            raise AirflowOptionalProviderFeatureException(
+                "sqlalchemy is required for creating the engine. Install it with"
+                ": pip install 'apache-airflow-providers-amazon[sqlalchemy]'"
+            )
         conn_kwargs = self.conn.extra_dejson
         if engine_kwargs is None:
             engine_kwargs = {}

airflow/providers/amazon/aws/hooks/s3.py

@@ -42,6 +42,8 @@ from typing import TYPE_CHECKING, Any
 from urllib.parse import urlsplit
 from uuid import uuid4
 
+from airflow.providers.common.compat.connection import get_async_connection
+
 if TYPE_CHECKING:
     from aiobotocore.client import AioBaseClient
     from mypy_boto3_s3.service_resource import (
@@ -52,7 +54,6 @@ if TYPE_CHECKING:
     from airflow.providers.amazon.version_compat import ArgNotSet
 
 
-from asgiref.sync import sync_to_async
 from boto3.s3.transfer import S3Transfer, TransferConfig
 from botocore.exceptions import ClientError
 
@@ -90,7 +91,7 @@ def provide_bucket_name(func: Callable) -> Callable:
         if not bound_args.arguments.get("bucket_name"):
             self = args[0]
             if self.aws_conn_id:
-                connection = await sync_to_async(self.get_connection)(self.aws_conn_id)
+                connection = await get_async_connection(self.aws_conn_id)
                 if connection.schema:
                     bound_args.arguments["bucket_name"] = connection.schema
         return bound_args
@@ -1743,29 +1744,31 @@ class S3Hook(AwsBaseHook):
 
     def _sync_to_local_dir_if_changed(self, s3_bucket, s3_object, local_target_path: Path):
         should_download = False
-        download_msg = ""
+        download_logs: list[str] = []
+        download_log_params: list[Any] = []
+
         if not local_target_path.exists():
             should_download = True
-            download_msg = f"Local file {local_target_path} does not exist."
+            download_logs.append("Local file %s does not exist.")
+            download_log_params.append(local_target_path)
         else:
             local_stats = local_target_path.stat()
-
             if s3_object.size != local_stats.st_size:
                 should_download = True
-                download_msg = (
-                    f"S3 object size ({s3_object.size}) and local file size ({local_stats.st_size}) differ."
-                )
+                download_logs.append("S3 object size (%s) and local file size (%s) differ.")
+                download_log_params.extend([s3_object.size, local_stats.st_size])
 
             s3_last_modified = s3_object.last_modified
-            if local_stats.st_mtime < s3_last_modified.microsecond:
+            if local_stats.st_mtime < s3_last_modified.timestamp():
                 should_download = True
-                download_msg = f"S3 object last modified ({s3_last_modified.microsecond}) and local file last modified ({local_stats.st_mtime}) differ."
+                download_logs.append("S3 object last modified (%s) and local file last modified (%s) differ.")
+                download_log_params.extend([s3_last_modified.timestamp(), local_stats.st_mtime])
 
         if should_download:
             s3_bucket.download_file(s3_object.key, local_target_path)
-            self.log.debug(
-                "%s Downloaded %s to %s", download_msg, s3_object.key, local_target_path.as_posix()
-            )
+            download_logs.append("Downloaded %s to %s")
+            download_log_params.extend([s3_object.key, local_target_path.as_posix()])
+            self.log.debug(" ".join(download_logs), *download_log_params)
         else:
             self.log.debug(
                 "Local file %s is up-to-date with S3 object %s. Skipping download.",