apache-airflow-providers-amazon 8.29.0rc1__py3-none-any.whl → 9.0.0rc1__py3-none-any.whl

airflow/providers/amazon/aws/utils/connection_wrapper.py

@@ -25,12 +25,10 @@ from typing import TYPE_CHECKING, Any
 
 from botocore import UNSIGNED
 from botocore.config import Config
-from deprecated import deprecated
 
-from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
+from airflow.exceptions import AirflowException
 from airflow.providers.amazon.aws.utils import trim_none_values
 from airflow.utils.log.logging_mixin import LoggingMixin
-from airflow.utils.log.secrets_masker import mask_secret
 from airflow.utils.types import NOTSET, ArgNotSet
 
 if TYPE_CHECKING:
@@ -157,15 +155,6 @@ class AwsConnectionWrapper(LoggingMixin):
                     "Can't resolve STS endpoint when both "
                     "`sts_connection` and `sts_test_connection` set to True."
                 )
-            elif sts_test_connection:
-                if "test_endpoint_url" in self.extra_config:
-                    warnings.warn(
-                        "extra['test_endpoint_url'] is deprecated and will be removed in a future release."
-                        " Please set `endpoint_url` in `service_config.sts` within `extras`.",
-                        AirflowProviderDeprecationWarning,
-                        stacklevel=2,
-                    )
-                    global_endpoint_url = self.extra_config["test_endpoint_url"]
 
         return service_config.get("endpoint_url", global_endpoint_url)
 
@@ -208,15 +197,7 @@ class AwsConnectionWrapper(LoggingMixin):
         self.schema = conn.schema or None
         self.extra_config = deepcopy(conn.extra_dejson)
 
-        if self.conn_type.lower() == "s3":
-            warnings.warn(
-                f"{self.conn_repr} has connection type 's3', "
-                "which has been replaced by connection type 'aws'. "
-                "Please update your connection to have `conn_type='aws'`.",
-                AirflowProviderDeprecationWarning,
-                stacklevel=2,
-            )
-        elif self.conn_type != "aws":
+        if self.conn_type != "aws":
             warnings.warn(
                 f"{self.conn_repr} expected connection type 'aws', got {self.conn_type!r}. "
                 "This connection might not work correctly. "
@@ -228,15 +209,6 @@ class AwsConnectionWrapper(LoggingMixin):
         extra = deepcopy(conn.extra_dejson)
         self.service_config = extra.get("service_config", {})
 
-        session_kwargs = extra.get("session_kwargs", {})
-        if session_kwargs:
-            warnings.warn(
-                "'session_kwargs' in extra config is deprecated and will be removed in a future releases. "
-                f"Please specify arguments passed to boto3 Session directly in {self.conn_repr} extra.",
-                AirflowProviderDeprecationWarning,
-                stacklevel=2,
-            )
-
         # Retrieve initial connection credentials
         init_credentials = self._get_credentials(**extra)
         self.aws_access_key_id, self.aws_secret_access_key, self.aws_session_token = init_credentials
@@ -245,13 +217,6 @@ class AwsConnectionWrapper(LoggingMixin):
             if "region_name" in extra:
                 self.region_name = extra["region_name"]
                 self.log.debug("Retrieving region_name=%s from %s extra.", self.region_name, self.conn_repr)
-            elif "region_name" in session_kwargs:
-                self.region_name = session_kwargs["region_name"]
-                self.log.debug(
-                    "Retrieving region_name=%s from %s extra['session_kwargs'].",
-                    self.region_name,
-                    self.conn_repr,
-                )
 
         if self.verify is None and "verify" in extra:
             self.verify = extra["verify"]
@@ -260,13 +225,6 @@ class AwsConnectionWrapper(LoggingMixin):
         if "profile_name" in extra:
             self.profile_name = extra["profile_name"]
             self.log.debug("Retrieving profile_name=%s from %s extra.", self.profile_name, self.conn_repr)
-        elif "profile_name" in session_kwargs:
-            self.profile_name = session_kwargs["profile_name"]
-            self.log.debug(
-                "Retrieving profile_name=%s from %s extra['session_kwargs'].",
-                self.profile_name,
-                self.conn_repr,
-            )
 
         # Warn the user that an invalid parameter is being used which actually not related to 'profile_name'.
         # ToDo: Remove this check entirely as soon as drop support credentials from s3_config_file
@@ -287,24 +245,7 @@ class AwsConnectionWrapper(LoggingMixin):
                 config_kwargs["signature_version"] = UNSIGNED
             self.botocore_config = Config(**config_kwargs)
 
-        if conn.host:
-            warnings.warn(
-                f"Host {conn.host} specified in the connection is not used."
-                " Please, set it on extra['endpoint_url'] instead",
-                AirflowProviderDeprecationWarning,
-                stacklevel=2,
-            )
-
-        self.endpoint_url = extra.get("host")
-        if self.endpoint_url:
-            warnings.warn(
-                "extra['host'] is deprecated and will be removed in a future release."
-                " Please set extra['endpoint_url'] instead",
-                AirflowProviderDeprecationWarning,
-                stacklevel=2,
-            )
-        else:
-            self.endpoint_url = extra.get("endpoint_url")
+        self.endpoint_url = extra.get("endpoint_url")
 
         # Retrieve Assume Role Configuration
         assume_role_configs = self._get_assume_role_configs(**extra)
@@ -359,10 +300,6 @@ class AwsConnectionWrapper(LoggingMixin):
         aws_access_key_id: str | None = None,
         aws_secret_access_key: str | None = None,
         aws_session_token: str | None = None,
-        # Deprecated Values
-        s3_config_file: str | None = None,
-        s3_config_format: str | None = None,
-        profile: str | None = None,
         session_kwargs: dict[str, Any] | None = None,
         **kwargs,
     ) -> tuple[str | None, str | None, str | None]:
@@ -393,13 +330,6 @@ class AwsConnectionWrapper(LoggingMixin):
             aws_access_key_id = session_aws_access_key_id
             aws_secret_access_key = session_aws_secret_access_key
             self.log.info("%s credentials retrieved from extra['session_kwargs'].", self.conn_repr)
-        elif s3_config_file:
-            aws_access_key_id, aws_secret_access_key = _parse_s3_config(
-                s3_config_file,
-                s3_config_format,
-                profile,
-            )
-            self.log.info("%s credentials retrieved from extra['s3_config_file']", self.conn_repr)
 
         if aws_session_token:
             self.log.info(
@@ -422,31 +352,12 @@ class AwsConnectionWrapper(LoggingMixin):
         role_arn: str | None = None,
         assume_role_method: str = "assume_role",
         assume_role_kwargs: dict[str, Any] | None = None,
-        # Deprecated Values
-        aws_account_id: str | None = None,
-        aws_iam_role: str | None = None,
-        external_id: str | None = None,
         **kwargs,
     ) -> tuple[str | None, str | None, dict[Any, str]]:
         """Get assume role configs from Connection extra."""
         if role_arn:
             self.log.debug("Retrieving role_arn=%r from %s extra.", role_arn, self.conn_repr)
-        elif aws_account_id and aws_iam_role:
-            warnings.warn(
-                "Constructing 'role_arn' from extra['aws_account_id'] and extra['aws_iam_role'] is deprecated"
-                f" and will be removed in a future releases."
-                f" Please set 'role_arn' in {self.conn_repr} extra.",
-                AirflowProviderDeprecationWarning,
-                stacklevel=3,
-            )
-            role_arn = f"arn:aws:iam::{aws_account_id}:role/{aws_iam_role}"
-            self.log.debug(
-                "Constructions role_arn=%r from %s extra['aws_account_id'] and extra['aws_iam_role'].",
-                role_arn,
-                self.conn_repr,
-            )
-
-        if not role_arn:
+        else:
             # There is no reason obtain `assume_role_method` and `assume_role_kwargs` if `role_arn` not set.
             return None, None, {}
 
@@ -460,76 +371,5 @@ class AwsConnectionWrapper(LoggingMixin):
         self.log.debug("Retrieve assume_role_method=%r from %s.", assume_role_method, self.conn_repr)
 
         assume_role_kwargs = assume_role_kwargs or {}
-        if "ExternalId" not in assume_role_kwargs and external_id:
-            warnings.warn(
-                "'external_id' in extra config is deprecated and will be removed in a future releases. "
-                f"Please set 'ExternalId' in 'assume_role_kwargs' in {self.conn_repr} extra.",
-                AirflowProviderDeprecationWarning,
-                stacklevel=3,
-            )
-            assume_role_kwargs["ExternalId"] = external_id
 
         return role_arn, assume_role_method, assume_role_kwargs
-
-
-@deprecated(
-    reason=(
-        "Use local credentials file is never documented and well tested. "
-        "Obtain credentials by this way deprecated and will be removed in a future releases."
-    ),
-    category=AirflowProviderDeprecationWarning,
-)
-def _parse_s3_config(
-    config_file_name: str, config_format: str | None = "boto", profile: str | None = None
-) -> tuple[str | None, str | None]:
-    """
-    Parse a config file for S3 credentials.
-
-    Can currently parse boto, s3cmd.conf and AWS SDK config formats.
-
-    :param config_file_name: path to the config file
-    :param config_format: config type. One of "boto", "s3cmd" or "aws".
-        Defaults to "boto"
-    :param profile: profile name in AWS type config file
-    """
-    import configparser
-
-    config = configparser.ConfigParser()
-    try:
-        if config.read(config_file_name):  # pragma: no cover
-            sections = config.sections()
-        else:
-            raise AirflowException(f"Couldn't read {config_file_name}")
-    except Exception as e:
-        raise AirflowException("Exception when parsing %s: %s", config_file_name, e.__class__.__name__)
-    # Setting option names depending on file format
-    if config_format is None:
-        config_format = "boto"
-    conf_format = config_format.lower()
-    if conf_format == "boto":  # pragma: no cover
-        if profile is not None and "profile " + profile in sections:
-            cred_section = "profile " + profile
-        else:
-            cred_section = "Credentials"
-    elif conf_format == "aws" and profile is not None:
-        cred_section = profile
-    else:
-        cred_section = "default"
-    # Option names
-    if conf_format in ("boto", "aws"):  # pragma: no cover
-        key_id_option = "aws_access_key_id"
-        secret_key_option = "aws_secret_access_key"
-    else:
-        key_id_option = "access_key"
-        secret_key_option = "secret_key"
-    # Actual Parsing
-    if cred_section not in sections:
-        raise AirflowException("This config file format is not recognized")
-    else:
-        try:
-            access_key = config.get(cred_section, key_id_option)
-            secret_key = config.get(cred_section, secret_key_option)
-            mask_secret(secret_key)
-        except Exception:
-            raise AirflowException("Option Error in parsing s3 config file")
-        return access_key, secret_key

airflow/providers/amazon/aws/utils/mixins.py

@@ -27,19 +27,14 @@ This module contains different mixin classes for internal use within the Amazon
 
 from __future__ import annotations
 
-import warnings
-from functools import cached_property
+from functools import cache, cached_property
 from typing import Any, Generic, NamedTuple, TypeVar
 
-from deprecated import deprecated
 from typing_extensions import final
 
-from airflow.compat.functools import cache
-from airflow.exceptions import AirflowProviderDeprecationWarning
 from airflow.providers.amazon.aws.hooks.base_aws import AwsGenericHook
 
 AwsHookType = TypeVar("AwsHookType", bound=AwsGenericHook)
-REGION_MSG = "`region` is deprecated and will be removed in the future. Please use `region_name` instead."
 
 
 class AwsHookParams(NamedTuple):
@@ -90,13 +85,6 @@ class AwsHookParams(NamedTuple):
                     self.botocore_config = params.botocore_config
                     self.foo = foo
         """
-        if region := additional_params.pop("region", None):
-            warnings.warn(REGION_MSG, AirflowProviderDeprecationWarning, stacklevel=3)
-            if region_name and region_name != region:
-                raise ValueError(
-                    f"Conflicting `region_name` provided, region_name={region_name!r}, region={region!r}."
-                )
-            region_name = region
         return cls(aws_conn_id, region_name, verify, botocore_config)
 
 
@@ -160,16 +148,6 @@ class AwsBaseHookMixin(Generic[AwsHookType]):
         """
         return self.aws_hook_class(**self._hook_parameters)
 
-    @property
-    @final
-    @deprecated(
-        reason="`region` is deprecated and will be removed in the future. Please use `region_name` instead.",
-        category=AirflowProviderDeprecationWarning,
-    )
-    def region(self) -> str | None:
-        """Alias for ``region_name``, used for compatibility (deprecated)."""
-        return self.region_name
-
 
 @cache
 def aws_template_fields(*template_fields: str) -> tuple[str, ...]:

airflow/providers/amazon/aws/utils/openlineage.py

@@ -86,7 +86,9 @@ def get_facets_from_redshift_table(
             ]
         )
     else:
-        statement_id = redshift_hook.execute_query(sql=sql, poll_interval=1, **redshift_data_api_kwargs)
+        statement_id = redshift_hook.execute_query(
+            sql=sql, poll_interval=1, **redshift_data_api_kwargs
+        ).statement_id
         response = redshift_hook.conn.get_statement_result(Id=statement_id)
 
         table_schema = SchemaDatasetFacet(

airflow/providers/amazon/aws/utils/task_log_fetcher.py

@@ -70,7 +70,7 @@ class AwsTaskLogFetcher(Thread):
                     # timestamp)
                     # When a slight delay is added before logging the event, that solves the issue
                     # See https://github.com/apache/airflow/issues/40875
-                    time.sleep(0.1)
+                    time.sleep(0.001)
                 self.logger.info(self.event_to_str(log_event))
                 prev_timestamp_event = current_timestamp_event
 

airflow/providers/amazon/get_provider_info.py

@@ -28,8 +28,9 @@ def get_provider_info():
         "name": "Amazon",
         "description": "Amazon integration (including `Amazon Web Services (AWS) <https://aws.amazon.com/>`__).\n",
         "state": "ready",
-        "source-date-epoch": 1726859877,
+        "source-date-epoch": 1728484310,
         "versions": [
+            "9.0.0",
             "8.29.0",
             "8.28.0",
             "8.27.0",
@@ -672,12 +673,20 @@ def get_provider_info():
                 "python-modules": ["airflow.providers.amazon.aws.sensors.quicksight"],
             },
         ],
+        "asset-uris": [
+            {
+                "schemes": ["s3"],
+                "handler": "airflow.providers.amazon.aws.assets.s3.sanitize_uri",
+                "to_openlineage_converter": "airflow.providers.amazon.aws.assets.s3.convert_asset_to_openlineage",
+                "factory": "airflow.providers.amazon.aws.assets.s3.create_asset",
+            }
+        ],
         "dataset-uris": [
             {
                 "schemes": ["s3"],
-                "handler": "airflow.providers.amazon.aws.datasets.s3.sanitize_uri",
-                "to_openlineage_converter": "airflow.providers.amazon.aws.datasets.s3.convert_dataset_to_openlineage",
-                "factory": "airflow.providers.amazon.aws.datasets.s3.create_dataset",
+                "handler": "airflow.providers.amazon.aws.assets.s3.sanitize_uri",
+                "to_openlineage_converter": "airflow.providers.amazon.aws.assets.s3.convert_asset_to_openlineage",
+                "factory": "airflow.providers.amazon.aws.assets.s3.create_asset",
             }
         ],
         "filesystems": ["airflow.providers.amazon.aws.fs.s3"],

{apache_airflow_providers_amazon-8.29.0rc1.dist-info → apache_airflow_providers_amazon-9.0.0rc1.dist-info}/METADATA

@@ -1,11 +1,11 @@
 Metadata-Version: 2.1
 Name: apache-airflow-providers-amazon
-Version: 8.29.0rc1
+Version: 9.0.0rc1
 Summary: Provider package apache-airflow-providers-amazon for Apache Airflow
 Keywords: airflow-provider,amazon,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
 Maintainer-email: Apache Software Foundation <dev@airflow.apache.org>
-Requires-Python: ~=3.8
+Requires-Python: ~=3.9
 Description-Content-Type: text/x-rst
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: Console
@@ -15,7 +15,6 @@ Classifier: Intended Audience :: System Administrators
 Classifier: Framework :: Apache Airflow
 Classifier: Framework :: Apache Airflow :: Provider
 Classifier: License :: OSI Approved :: Apache Software License
-Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
@@ -56,8 +55,8 @@ Requires-Dist: s3fs>=2023.10.0 ; extra == "s3fs"
 Requires-Dist: apache-airflow-providers-salesforce ; extra == "salesforce"
 Requires-Dist: apache-airflow-providers-ssh ; extra == "ssh"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-amazon/8.29.0/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-amazon/8.29.0
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.0.0/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.0.0
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Twitter, https://twitter.com/ApacheAirflow
@@ -125,7 +124,7 @@ Provides-Extra: ssh
 
 Package ``apache-airflow-providers-amazon``
 
-Release: ``8.29.0.rc1``
+Release: ``9.0.0.rc1``
 
 
 Amazon integration (including `Amazon Web Services (AWS) <https://aws.amazon.com/>`__).
@@ -138,7 +137,7 @@ This is a provider package for ``amazon`` provider. All classes for this provide
 are in ``airflow.providers.amazon`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-amazon/8.29.0/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.0.0/>`_.
 
 Installation
 ------------
@@ -147,7 +146,7 @@ You can install this package on top of an existing Airflow 2 installation (see `
 for the minimum Airflow version supported) via
 ``pip install apache-airflow-providers-amazon``
 
-The package supports the following python versions: 3.8,3.9,3.10,3.11,3.12
+The package supports the following python versions: 3.9,3.10,3.11,3.12
 
 Requirements
 ------------
@@ -205,4 +204,4 @@ Dependent package
 ======================================================================================================================  ===================
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-amazon/8.29.0/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.0.0/changelog.html>`_.