apache-airflow-providers-amazon 9.1.0rc2__tar.gz → 9.1.0rc4__tar.gz

{apache_airflow_providers_amazon-9.1.0rc2 → apache_airflow_providers_amazon-9.1.0rc4}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: apache-airflow-providers-amazon
-Version: 9.1.0rc2
+Version: 9.1.0rc4
 Summary: Provider package apache-airflow-providers-amazon for Apache Airflow
 Keywords: airflow-provider,amazon,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -22,7 +22,7 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: System :: Monitoring
 Requires-Dist: PyAthena>=3.0.10
 Requires-Dist: apache-airflow-providers-common-compat>=1.2.1rc0
-Requires-Dist: apache-airflow-providers-common-sql>=1.3.1rc0
+Requires-Dist: apache-airflow-providers-common-sql>=1.20.0rc0
 Requires-Dist: apache-airflow-providers-http
 Requires-Dist: apache-airflow>=2.8.0rc0
 Requires-Dist: asgiref>=2.3.0
@@ -35,16 +35,16 @@ Requires-Dist: python3-saml>=1.16.0
 Requires-Dist: redshift_connector>=2.0.918
 Requires-Dist: watchtower>=3.0.0,!=3.3.0,<4
 Requires-Dist: aiobotocore[boto3]>=2.13.0 ; extra == "aiobotocore"
-Requires-Dist: apache-airflow-providers-apache-hive ; extra == "apache.hive"
-Requires-Dist: apache-airflow-providers-cncf-kubernetes>=7.2.0rc0 ; extra == "cncf.kubernetes"
-Requires-Dist: apache-airflow-providers-common-compat ; extra == "common.compat"
-Requires-Dist: apache-airflow-providers-common-sql ; extra == "common.sql"
+Requires-Dist: apache-airflow-providers-apache-hive ; extra == "apache-hive"
+Requires-Dist: apache-airflow-providers-cncf-kubernetes>=7.2.0rc0 ; extra == "cncf-kubernetes"
+Requires-Dist: apache-airflow-providers-common-compat ; extra == "common-compat"
+Requires-Dist: apache-airflow-providers-common-sql ; extra == "common-sql"
 Requires-Dist: apache-airflow-providers-exasol ; extra == "exasol"
 Requires-Dist: apache-airflow-providers-ftp ; extra == "ftp"
 Requires-Dist: apache-airflow-providers-google ; extra == "google"
 Requires-Dist: apache-airflow-providers-http ; extra == "http"
 Requires-Dist: apache-airflow-providers-imap ; extra == "imap"
-Requires-Dist: apache-airflow-providers-microsoft-azure ; extra == "microsoft.azure"
+Requires-Dist: apache-airflow-providers-microsoft-azure ; extra == "microsoft-azure"
 Requires-Dist: apache-airflow-providers-mongo ; extra == "mongo"
 Requires-Dist: apache-airflow-providers-openlineage ; extra == "openlineage"
 Requires-Dist: pandas>=2.1.2,<2.2 ; extra == "pandas" and (python_version>="3.9")
@@ -61,16 +61,16 @@ Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Twitter, https://twitter.com/ApacheAirflow
 Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
 Provides-Extra: aiobotocore
-Provides-Extra: apache.hive
-Provides-Extra: cncf.kubernetes
-Provides-Extra: common.compat
-Provides-Extra: common.sql
+Provides-Extra: apache-hive
+Provides-Extra: cncf-kubernetes
+Provides-Extra: common-compat
+Provides-Extra: common-sql
 Provides-Extra: exasol
 Provides-Extra: ftp
 Provides-Extra: google
 Provides-Extra: http
 Provides-Extra: imap
-Provides-Extra: microsoft.azure
+Provides-Extra: microsoft-azure
 Provides-Extra: mongo
 Provides-Extra: openlineage
 Provides-Extra: pandas
@@ -123,7 +123,7 @@ Provides-Extra: ssh
 
 Package ``apache-airflow-providers-amazon``
 
-Release: ``9.1.0.rc2``
+Release: ``9.1.0.rc4``
 
 
 Amazon integration (including `Amazon Web Services (AWS) <https://aws.amazon.com/>`__).
@@ -155,7 +155,7 @@ PIP package                                 Version required
 ==========================================  ======================
 ``apache-airflow``                          ``>=2.8.0``
 ``apache-airflow-providers-common-compat``  ``>=1.2.1``
-``apache-airflow-providers-common-sql``     ``>=1.3.1``
+``apache-airflow-providers-common-sql``     ``>=1.20.0``
 ``apache-airflow-providers-http``
 ``boto3``                                   ``>=1.34.90``
 ``botocore``                                ``>=1.34.90``

{apache_airflow_providers_amazon-9.1.0rc2 → apache_airflow_providers_amazon-9.1.0rc4}/README.rst

@@ -42,7 +42,7 @@
 
 Package ``apache-airflow-providers-amazon``
 
-Release: ``9.1.0.rc2``
+Release: ``9.1.0.rc4``
 
 
 Amazon integration (including `Amazon Web Services (AWS) <https://aws.amazon.com/>`__).
@@ -74,7 +74,7 @@ PIP package                                 Version required
 ==========================================  ======================
 ``apache-airflow``                          ``>=2.8.0``
 ``apache-airflow-providers-common-compat``  ``>=1.2.1``
-``apache-airflow-providers-common-sql``     ``>=1.3.1``
+``apache-airflow-providers-common-sql``     ``>=1.20.0``
 ``apache-airflow-providers-http``
 ``boto3``                                   ``>=1.34.90``
 ``botocore``                                ``>=1.34.90``

{apache_airflow_providers_amazon-9.1.0rc2 → apache_airflow_providers_amazon-9.1.0rc4}/airflow/providers/amazon/aws/hooks/eks.py

@@ -85,8 +85,8 @@ COMMAND = """
                 exit 1
             fi
 
-            expiration_timestamp=$(echo "$output" | grep -oP 'expirationTimestamp:\s*\K[^,]+')
-            token=$(echo "$output" | grep -oP 'token:\s*\K[^,]+')
+            expiration_timestamp=$(echo "$output" | grep -oP 'expirationTimestamp: \\K[^,]+')
+            token=$(echo "$output" | grep -oP 'token: \\K[^,]+')
 
             json_string=$(printf '{{"kind": "ExecCredential","apiVersion": \
                 "client.authentication.k8s.io/v1alpha1","spec": {{}},"status": \

{apache_airflow_providers_amazon-9.1.0rc2 → apache_airflow_providers_amazon-9.1.0rc4}/airflow/providers/amazon/aws/hooks/glue.py

@@ -282,13 +282,16 @@ class GlueJobHook(AwsBaseHook):
             log_group_error, continuation_tokens.error_stream_continuation
         )
 
-    def job_completion(self, job_name: str, run_id: str, verbose: bool = False) -> dict[str, str]:
+    def job_completion(
+        self, job_name: str, run_id: str, verbose: bool = False, sleep_before_return: int = 0
+    ) -> dict[str, str]:
         """
         Wait until Glue job with job_name finishes; return final state if finished or raises AirflowException.
 
         :param job_name: unique job name per AWS account
         :param run_id: The job-run ID of the predecessor job run
         :param verbose: If True, more Glue Job Run logs show in the Airflow Task Logs.  (default: False)
+        :param sleep_before_return: time in seconds to wait before returning final status.
         :return: Dict of JobRunState and JobRunId
         """
         next_log_tokens = self.LogContinuationTokens()
@@ -296,6 +299,7 @@ class GlueJobHook(AwsBaseHook):
             job_run_state = self.get_job_state(job_name, run_id)
             ret = self._handle_state(job_run_state, job_name, run_id, verbose, next_log_tokens)
             if ret:
+                time.sleep(sleep_before_return)
                 return ret
             else:
                 time.sleep(self.job_poll_interval)

{apache_airflow_providers_amazon-9.1.0rc2 → apache_airflow_providers_amazon-9.1.0rc4}/airflow/providers/amazon/aws/hooks/s3.py

@@ -86,7 +86,7 @@ def provide_bucket_name(func: Callable) -> Callable:
     async def maybe_add_bucket_name(*args, **kwargs):
         bound_args = function_signature.bind(*args, **kwargs)
 
-        if "bucket_name" not in bound_args.arguments:
+        if not bound_args.arguments.get("bucket_name"):
             self = args[0]
             if self.aws_conn_id:
                 connection = await sync_to_async(self.get_connection)(self.aws_conn_id)
@@ -116,7 +116,7 @@ def provide_bucket_name(func: Callable) -> Callable:
         def wrapper(*args, **kwargs) -> Callable:
             bound_args = function_signature.bind(*args, **kwargs)
 
-            if "bucket_name" not in bound_args.arguments:
+            if not bound_args.arguments.get("bucket_name"):
                 self = args[0]
 
                 if "bucket_name" in self.service_config:

{apache_airflow_providers_amazon-9.1.0rc2 → apache_airflow_providers_amazon-9.1.0rc4}/airflow/providers/amazon/aws/operators/appflow.py

@@ -21,11 +21,11 @@ from datetime import datetime, timedelta
 from typing import TYPE_CHECKING, cast
 
 from airflow.exceptions import AirflowException
-from airflow.operators.python import ShortCircuitOperator
 from airflow.providers.amazon.aws.hooks.appflow import AppflowHook
 from airflow.providers.amazon.aws.operators.base_aws import AwsBaseOperator
 from airflow.providers.amazon.aws.utils import datetime_to_epoch_ms
 from airflow.providers.amazon.aws.utils.mixins import AwsBaseHookMixin, AwsHookParams, aws_template_fields
+from airflow.providers.common.compat.standard.operators import ShortCircuitOperator
 
 if TYPE_CHECKING:
     from mypy_boto3_appflow.type_defs import (

{apache_airflow_providers_amazon-9.1.0rc2 → apache_airflow_providers_amazon-9.1.0rc4}/airflow/providers/amazon/aws/operators/athena.py

@@ -311,7 +311,9 @@ class AthenaOperator(AwsBaseOperator[AthenaHook]):
             }
             fields = [
                 SchemaDatasetFacetFields(
-                    name=column["Name"], type=column["Type"], description=column["Comment"]
+                    name=column["Name"],
+                    type=column["Type"],
+                    description=column.get("Comment"),
                 )
                 for column in table_metadata["TableMetadata"]["Columns"]
             ]

{apache_airflow_providers_amazon-9.1.0rc2 → apache_airflow_providers_amazon-9.1.0rc4}/airflow/providers/amazon/aws/operators/ecs.py

@@ -368,7 +368,7 @@ class EcsRunTaskOperator(EcsBaseOperator):
         If None, this is the same as the `region` parameter. If that is also None,
         this is the default AWS region based on your connection settings.
     :param awslogs_stream_prefix: the stream prefix that is used for the CloudWatch logs.
-        This is usually based on some custom name combined with the name of the container.
+        This should match the prefix specified in the log configuration of the task definition.
         Only required if you want logs to be shown in the Airflow UI after your job has
         finished.
     :param awslogs_fetch_interval: the interval that the ECS task log fetcher should wait
@@ -481,6 +481,7 @@ class EcsRunTaskOperator(EcsBaseOperator):
             self.awslogs_region = self.region_name
 
         self.arn: str | None = None
+        self.container_name: str | None = None
         self._started_by: str | None = None
 
         self.retry_args = quota_retry
@@ -597,10 +598,10 @@ class EcsRunTaskOperator(EcsBaseOperator):
 
         if self.capacity_provider_strategy:
             run_opts["capacityProviderStrategy"] = self.capacity_provider_strategy
-            if self.volume_configurations is not None:
-                run_opts["volumeConfigurations"] = self.volume_configurations
         elif self.launch_type:
             run_opts["launchType"] = self.launch_type
+        if self.volume_configurations is not None:
+            run_opts["volumeConfigurations"] = self.volume_configurations
         if self.platform_version is not None:
             run_opts["platformVersion"] = self.platform_version
         if self.group is not None:
@@ -624,6 +625,7 @@ class EcsRunTaskOperator(EcsBaseOperator):
         self.log.info("ECS Task started: %s", response)
 
         self.arn = response["tasks"][0]["taskArn"]
+        self.container_name = response["tasks"][0]["containers"][0]["name"]
         self.log.info("ECS task ID is: %s", self._get_ecs_task_id(self.arn))
 
     def _try_reattach_task(self, started_by: str):
@@ -659,6 +661,12 @@ class EcsRunTaskOperator(EcsBaseOperator):
         return self.awslogs_group and self.awslogs_stream_prefix
 
     def _get_logs_stream_name(self) -> str:
+        if (
+            self.awslogs_stream_prefix
+            and self.container_name
+            and not self.awslogs_stream_prefix.endswith(f"/{self.container_name}")
+        ):
+            return f"{self.awslogs_stream_prefix}/{self.container_name}/{self._get_ecs_task_id(self.arn)}"
         return f"{self.awslogs_stream_prefix}/{self._get_ecs_task_id(self.arn)}"
 
     def _get_task_log_fetcher(self) -> AwsTaskLogFetcher:

{apache_airflow_providers_amazon-9.1.0rc2 → apache_airflow_providers_amazon-9.1.0rc4}/airflow/providers/amazon/aws/operators/eks.py

@@ -45,8 +45,10 @@ from airflow.providers.cncf.kubernetes.utils.pod_manager import OnFinishAction
 try:
     from airflow.providers.cncf.kubernetes.operators.pod import KubernetesPodOperator
 except ImportError:
-    # preserve backward compatibility for older versions of cncf.kubernetes provider
-    from airflow.providers.cncf.kubernetes.operators.kubernetes_pod import KubernetesPodOperator
+    # preserve backward compatibility for older versions of cncf.kubernetes provider, remove this when minimum cncf.kubernetes provider is 10.0
+    from airflow.providers.cncf.kubernetes.operators.kubernetes_pod import (  # type: ignore[no-redef]
+        KubernetesPodOperator,
+    )
 
 if TYPE_CHECKING:
     from airflow.utils.context import Context

{apache_airflow_providers_amazon-9.1.0rc2 → apache_airflow_providers_amazon-9.1.0rc4}/airflow/providers/amazon/aws/operators/glue.py

@@ -74,6 +74,11 @@ class GlueJobOperator(BaseOperator):
     :param update_config: If True, Operator will update job configuration.  (default: False)
     :param replace_script_file: If True, the script file will be replaced in S3. (default: False)
     :param stop_job_run_on_kill: If True, Operator will stop the job run when task is killed.
+    :param sleep_before_return: time in seconds to wait before returning final status. This is meaningful in case
+        of limiting concurrency, Glue needs 5-10 seconds to clean up resources.
+        Thus if status is returned immediately it might end up in case of more than 1 concurrent run.
+        It is recommended to set this parameter to 10 when you are using concurrency=1.
+        For more information see: https://repost.aws/questions/QUaKgpLBMPSGWO0iq2Fob_bw/glue-run-concurrent-jobs#ANFpCL2fRnQRqgDFuIU_rpvA
     """
 
     template_fields: Sequence[str] = (
@@ -118,6 +123,7 @@ class GlueJobOperator(BaseOperator):
         update_config: bool = False,
         job_poll_interval: int | float = 6,
         stop_job_run_on_kill: bool = False,
+        sleep_before_return: int = 0,
         **kwargs,
     ):
         super().__init__(**kwargs)
@@ -145,6 +151,7 @@ class GlueJobOperator(BaseOperator):
         self.job_poll_interval = job_poll_interval
         self.stop_job_run_on_kill = stop_job_run_on_kill
         self._job_run_id: str | None = None
+        self.sleep_before_return: int = sleep_before_return
 
     @cached_property
     def glue_job_hook(self) -> GlueJobHook:
@@ -220,7 +227,9 @@ class GlueJobOperator(BaseOperator):
                 method_name="execute_complete",
             )
         elif self.wait_for_completion:
-            glue_job_run = self.glue_job_hook.job_completion(self.job_name, self._job_run_id, self.verbose)
+            glue_job_run = self.glue_job_hook.job_completion(
+                self.job_name, self._job_run_id, self.verbose, self.sleep_before_return
+            )
             self.log.info(
                 "AWS Glue Job: %s status: %s. Run Id: %s",
                 self.job_name,

{apache_airflow_providers_amazon-9.1.0rc2 → apache_airflow_providers_amazon-9.1.0rc4}/airflow/providers/amazon/aws/transfers/s3_to_dynamodb.py

@@ -240,7 +240,7 @@ class S3ToDynamoDBOperator(BaseOperator):
         finally:
             self.log.info("Delete tmp DynamoDB table %s", self.tmp_table_name)
             client.delete_table(TableName=self.tmp_table_name)
-            return dynamodb_hook.get_conn().Table(self.dynamodb_table_name).table_arn
+        return dynamodb_hook.get_conn().Table(self.dynamodb_table_name).table_arn
 
     def execute(self, context: Context) -> str:
         """

{apache_airflow_providers_amazon-9.1.0rc2 → apache_airflow_providers_amazon-9.1.0rc4}/airflow/providers/amazon/aws/triggers/athena.py

@@ -29,8 +29,7 @@ class AthenaTrigger(AwsBaseWaiterTrigger):
     """
     Trigger for AthenaOperator.
 
-    The trigger will asynchronously poll the boto3 API and wait for the
-    Redshift cluster to be in the `available` state.
+    The trigger will asynchronously poll the boto3 API
 
     :param query_execution_id:  ID of the Athena query execution to watch
     :param waiter_delay: The amount of time in seconds to wait between attempts.

{apache_airflow_providers_amazon-9.1.0rc2 → apache_airflow_providers_amazon-9.1.0rc4}/airflow/providers/amazon/aws/triggers/glue.py

@@ -62,7 +62,7 @@ class GlueJobCompleteTrigger(BaseTrigger):
             {
                 "job_name": self.job_name,
                 "run_id": self.run_id,
-                "verbose": str(self.verbose),
+                "verbose": self.verbose,
                 "aws_conn_id": self.aws_conn_id,
                 "job_poll_interval": self.job_poll_interval,
             },

{apache_airflow_providers_amazon-9.1.0rc2 → apache_airflow_providers_amazon-9.1.0rc4}/airflow/providers/amazon/get_provider_info.py

@@ -100,7 +100,7 @@ def get_provider_info():
         "dependencies": [
             "apache-airflow>=2.8.0",
             "apache-airflow-providers-common-compat>=1.2.1",
-            "apache-airflow-providers-common-sql>=1.3.1",
+            "apache-airflow-providers-common-sql>=1.20.0",
             "apache-airflow-providers-http",
             "boto3>=1.34.90",
             "botocore>=1.34.90",

{apache_airflow_providers_amazon-9.1.0rc2 → apache_airflow_providers_amazon-9.1.0rc4}/pyproject.toml

@@ -21,14 +21,13 @@
 
 # IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
 # `pyproject_TEMPLATE.toml.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
-#
 [build-system]
-requires = ["flit_core >=3.2,<4"]
+requires = ["flit_core==3.10.1"]
 build-backend = "flit_core.buildapi"
 
 [project]
 name = "apache-airflow-providers-amazon"
-version = "9.1.0.rc2"
+version = "9.1.0.rc4"
 description = "Provider package apache-airflow-providers-amazon for Apache Airflow"
 readme = "README.rst"
 authors = [
@@ -57,7 +56,7 @@ requires-python = "~=3.9"
 dependencies = [
     "PyAthena>=3.0.10",
     "apache-airflow-providers-common-compat>=1.2.1rc0",
-    "apache-airflow-providers-common-sql>=1.3.1rc0",
+    "apache-airflow-providers-common-sql>=1.20.0rc0",
     "apache-airflow-providers-http",
     "apache-airflow>=2.8.0rc0",
     "asgiref>=2.3.0",