apache-airflow-providers-microsoft-azure 10.1.2rc1__tar.gz → 10.2.0__tar.gz

{apache_airflow_providers_microsoft_azure-10.1.2rc1 → apache_airflow_providers_microsoft_azure-10.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apache-airflow-providers-microsoft-azure
-Version: 10.1.2rc1
+Version: 10.2.0
 Summary: Provider package apache-airflow-providers-microsoft-azure for Apache Airflow
 Keywords: airflow-provider,microsoft.azure,airflow,integration
 Author-email: Apache Software Foundation <dev@airflow.apache.org>
@@ -23,7 +23,7 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: System :: Monitoring
 Requires-Dist: adal>=1.2.7
 Requires-Dist: adlfs>=2023.10.0
-Requires-Dist: apache-airflow>=2.7.0rc0
+Requires-Dist: apache-airflow>=2.7.0
 Requires-Dist: azure-batch>=8.0.0
 Requires-Dist: azure-cosmos>=4.6.0
 Requires-Dist: azure-datalake-store>=0.0.45
@@ -44,16 +44,18 @@ Requires-Dist: azure-storage-file-share>=12.7.0
 Requires-Dist: azure-synapse-artifacts>=0.17.0
 Requires-Dist: azure-synapse-spark>=0.2.0
 Requires-Dist: msgraph-core>=1.0.0
+Requires-Dist: apache-airflow-providers-amazon ; extra == "amazon"
 Requires-Dist: apache-airflow-providers-google ; extra == "google"
 Requires-Dist: apache-airflow-providers-oracle ; extra == "oracle"
 Requires-Dist: apache-airflow-providers-sftp ; extra == "sftp"
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.1.2/changelog.html
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.1.2
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.2.0/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.2.0
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Twitter, https://twitter.com/ApacheAirflow
 Project-URL: YouTube, https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/
+Provides-Extra: amazon
 Provides-Extra: google
 Provides-Extra: oracle
 Provides-Extra: sftp
@@ -102,7 +104,7 @@ Provides-Extra: sftp
 
 Package ``apache-airflow-providers-microsoft-azure``
 
-Release: ``10.1.2.rc1``
+Release: ``10.2.0``
 
 
 `Microsoft Azure <https://azure.microsoft.com/>`__
@@ -115,7 +117,7 @@ This is a provider package for ``microsoft.azure`` provider. All classes for thi
 are in ``airflow.providers.microsoft.azure`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.1.2/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.2.0/>`_.
 
 Installation
 ------------
@@ -167,16 +169,17 @@ You can install such cross-provider dependencies when installing from PyPI. For
 
 .. code-block:: bash
 
-    pip install apache-airflow-providers-microsoft-azure[google]
+    pip install apache-airflow-providers-microsoft-azure[amazon]
 
 
 ====================================================================================================  ==========
 Dependent package                                                                                     Extra
 ====================================================================================================  ==========
+`apache-airflow-providers-amazon <https://airflow.apache.org/docs/apache-airflow-providers-amazon>`_  ``amazon``
 `apache-airflow-providers-google <https://airflow.apache.org/docs/apache-airflow-providers-google>`_  ``google``
 `apache-airflow-providers-oracle <https://airflow.apache.org/docs/apache-airflow-providers-oracle>`_  ``oracle``
 `apache-airflow-providers-sftp <https://airflow.apache.org/docs/apache-airflow-providers-sftp>`_      ``sftp``
 ====================================================================================================  ==========
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.1.2/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.2.0/changelog.html>`_.

{apache_airflow_providers_microsoft_azure-10.1.2rc1 → apache_airflow_providers_microsoft_azure-10.2.0}/README.rst

@@ -42,7 +42,7 @@
 
 Package ``apache-airflow-providers-microsoft-azure``
 
-Release: ``10.1.2.rc1``
+Release: ``10.2.0``
 
 
 `Microsoft Azure <https://azure.microsoft.com/>`__
@@ -55,7 +55,7 @@ This is a provider package for ``microsoft.azure`` provider. All classes for thi
 are in ``airflow.providers.microsoft.azure`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.1.2/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.2.0/>`_.
 
 Installation
 ------------
@@ -107,16 +107,17 @@ You can install such cross-provider dependencies when installing from PyPI. For
 
 .. code-block:: bash
 
-    pip install apache-airflow-providers-microsoft-azure[google]
+    pip install apache-airflow-providers-microsoft-azure[amazon]
 
 
 ====================================================================================================  ==========
 Dependent package                                                                                     Extra
 ====================================================================================================  ==========
+`apache-airflow-providers-amazon <https://airflow.apache.org/docs/apache-airflow-providers-amazon>`_  ``amazon``
 `apache-airflow-providers-google <https://airflow.apache.org/docs/apache-airflow-providers-google>`_  ``google``
 `apache-airflow-providers-oracle <https://airflow.apache.org/docs/apache-airflow-providers-oracle>`_  ``oracle``
 `apache-airflow-providers-sftp <https://airflow.apache.org/docs/apache-airflow-providers-sftp>`_      ``sftp``
 ====================================================================================================  ==========
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.1.2/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.2.0/changelog.html>`_.

{apache_airflow_providers_microsoft_azure-10.1.2rc1 → apache_airflow_providers_microsoft_azure-10.2.0}/airflow/providers/microsoft/azure/__init__.py

@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
 
 __all__ = ["__version__"]
 
-__version__ = "10.1.2"
+__version__ = "10.2.0"
 
 if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
     "2.7.0"

{apache_airflow_providers_microsoft_azure-10.1.2rc1 → apache_airflow_providers_microsoft_azure-10.2.0}/airflow/providers/microsoft/azure/get_provider_info.py

@@ -28,8 +28,9 @@ def get_provider_info():
         "name": "Microsoft Azure",
         "description": "`Microsoft Azure <https://azure.microsoft.com/>`__\n",
         "state": "ready",
-        "source-date-epoch": 1718604928,
+        "source-date-epoch": 1720503933,
         "versions": [
+            "10.2.0",
             "10.1.2",
             "10.1.1",
             "10.1.0",
@@ -363,6 +364,12 @@ def get_provider_info():
                 "how-to-guide": "/docs/apache-airflow-providers-microsoft-azure/transfer/sftp_to_wasb.rst",
                 "python-module": "airflow.providers.microsoft.azure.transfers.sftp_to_wasb",
             },
+            {
+                "source-integration-name": "Amazon Simple Storage Service (S3)",
+                "target-integration-name": "Microsoft Azure Blob Storage",
+                "how-to-guide": "/docs/apache-airflow-providers-microsoft-azure/transfer/s3_to_wasb.rst",
+                "python-module": "airflow.providers.microsoft.azure.transfers.s3_to_wasb",
+            },
             {
                 "source-integration-name": "Microsoft Azure Blob Storage",
                 "target-integration-name": "Google Cloud Storage (GCS)",

{apache_airflow_providers_microsoft_azure-10.1.2rc1 → apache_airflow_providers_microsoft_azure-10.2.0}/airflow/providers/microsoft/azure/hooks/asb.py

@@ -92,7 +92,8 @@ class BaseAzureServiceBusHook(BaseHook):
 
 
 class AdminClientHook(BaseAzureServiceBusHook):
-    """Interact with the ServiceBusAdministrationClient.
+    """
+    Interact with the ServiceBusAdministrationClient.
 
     This can create, update, list, and delete resources of a Service Bus
     namespace. This hook uses the same Azure Service Bus client connection
@@ -100,7 +101,8 @@ class AdminClientHook(BaseAzureServiceBusHook):
     """
 
     def get_conn(self) -> ServiceBusAdministrationClient:
-        """Create a ServiceBusAdministrationClient instance.
+        """
+        Create a ServiceBusAdministrationClient instance.
 
         This uses the connection string in connection details.
         """
@@ -190,7 +192,8 @@ class AdminClientHook(BaseAzureServiceBusHook):
 
 
 class MessageHook(BaseAzureServiceBusHook):
-    """Interact with ServiceBusClient.
+    """
+    Interact with ServiceBusClient.
 
     This acts as a high level interface for getting ServiceBusSender and ServiceBusReceiver.
     """
@@ -225,7 +228,8 @@ class MessageHook(BaseAzureServiceBusHook):
         return client
 
     def send_message(self, queue_name: str, messages: str | list[str], batch_message_flag: bool = False):
-        """Use ServiceBusClient Send to send message(s) to a Service Bus Queue.
+        """
+        Use ServiceBusClient Send to send message(s) to a Service Bus Queue.
 
         By using ``batch_message_flag``, it enables and send message as batch message.
 
@@ -295,7 +299,8 @@ class MessageHook(BaseAzureServiceBusHook):
         max_message_count: int | None,
         max_wait_time: float | None,
     ):
-        """Receive a batch of subscription message at once.
+        """
+        Receive a batch of subscription message at once.
 
         This approach is optimal if you wish to process multiple messages
         simultaneously, or perform an ad-hoc receive as a single call.

{apache_airflow_providers_microsoft_azure-10.1.2rc1 → apache_airflow_providers_microsoft_azure-10.2.0}/airflow/providers/microsoft/azure/hooks/data_factory.py

@@ -14,7 +14,8 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-"""Spelling exceptions.
+"""
+Spelling exceptions.
 
 .. spelling:word-list::
     CreateRunResponse

{apache_airflow_providers_microsoft_azure-10.1.2rc1 → apache_airflow_providers_microsoft_azure-10.2.0}/airflow/providers/microsoft/azure/hooks/data_lake.py

@@ -45,7 +45,8 @@ Credentials = Union[ClientSecretCredential, AzureIdentityCredentialAdapter, Defa
 
 
 class AzureDataLakeHook(BaseHook):
-    """Integration with Azure Data Lake.
+    """
+    Integration with Azure Data Lake.
 
     AzureDataLakeHook communicates via a REST API compatible with WebHDFS. Make
     sure that a Airflow connection of type ``azure_data_lake`` exists.
@@ -135,7 +136,8 @@ class AzureDataLakeHook(BaseHook):
         return self._conn
 
     def check_for_file(self, file_path: str) -> bool:
-        """Check if a file exists on Azure Data Lake.
+        """
+        Check if a file exists on Azure Data Lake.
 
         :param file_path: Path and name of the file.
         :return: True if the file exists, False otherwise.
@@ -156,7 +158,8 @@ class AzureDataLakeHook(BaseHook):
         blocksize: int = 4194304,
         **kwargs,
     ) -> None:
-        """Upload a file to Azure Data Lake.
+        """
+        Upload a file to Azure Data Lake.
 
         :param local_path: local path. Can be single file, directory (in which case,
             upload recursively) or glob pattern. Recursive glob patterns using `**`
@@ -196,7 +199,8 @@ class AzureDataLakeHook(BaseHook):
         blocksize: int = 4194304,
         **kwargs,
     ) -> None:
-        """Download a file from Azure Blob Storage.
+        """
+        Download a file from Azure Blob Storage.
 
         :param local_path: local path. If downloading a single file, will write to this
             specific file, unless it is an existing directory, in which case a file is
@@ -228,7 +232,8 @@ class AzureDataLakeHook(BaseHook):
         )
 
     def list(self, path: str) -> list:
-        """List files in Azure Data Lake Storage.
+        """
+        List files in Azure Data Lake Storage.
 
         :param path: full path/globstring to use to list files in ADLS
         """
@@ -238,7 +243,8 @@ class AzureDataLakeHook(BaseHook):
             return self.get_conn().walk(path)
 
     def remove(self, path: str, recursive: bool = False, ignore_not_found: bool = True) -> None:
-        """Remove files in Azure Data Lake Storage.
+        """
+        Remove files in Azure Data Lake Storage.
 
         :param path: A directory or file to remove in ADLS
         :param recursive: Whether to loop into directories in the location and remove the files
@@ -254,7 +260,8 @@ class AzureDataLakeHook(BaseHook):
 
 
 class AzureDataLakeStorageV2Hook(BaseHook):
-    """Interact with a ADLS gen2 storage account.
+    """
+    Interact with a ADLS gen2 storage account.
 
     It mainly helps to create and manage directories and files in storage
     accounts that have a hierarchical namespace. Using Adls_v2 connection
@@ -375,7 +382,8 @@ class AzureDataLakeStorageV2Hook(BaseHook):
         )
 
     def create_file_system(self, file_system_name: str) -> None:
-        """Create a new file system under the specified account.
+        """
+        Create a new file system under the specified account.
 
         A container acts as a file system for your files.
 
@@ -393,7 +401,8 @@ class AzureDataLakeStorageV2Hook(BaseHook):
             raise
 
     def get_file_system(self, file_system: FileSystemProperties | str) -> FileSystemClient:
-        """Get a client to interact with the specified file system.
+        """
+        Get a client to interact with the specified file system.
 
         :param file_system: This can either be the name of the file system
             or an instance of FileSystemProperties.
@@ -411,7 +420,8 @@ class AzureDataLakeStorageV2Hook(BaseHook):
     def create_directory(
         self, file_system_name: FileSystemProperties | str, directory_name: str, **kwargs
     ) -> DataLakeDirectoryClient:
-        """Create a directory under the specified file system.
+        """
+        Create a directory under the specified file system.
 
         :param file_system_name: Name of the file system or instance of FileSystemProperties.
         :param directory_name: Name of the directory which needs to be created in the file system.
@@ -424,7 +434,8 @@ class AzureDataLakeStorageV2Hook(BaseHook):
         file_system_name: FileSystemProperties | str,
         directory_name: DirectoryProperties | str,
     ) -> DataLakeDirectoryClient:
-        """Get the specific directory under the specified file system.
+        """
+        Get the specific directory under the specified file system.
 
         :param file_system_name: Name of the file system or instance of FileSystemProperties.
         :param directory_name: Name of the directory or instance of DirectoryProperties which needs to be
@@ -443,7 +454,8 @@ class AzureDataLakeStorageV2Hook(BaseHook):
             raise
 
     def create_file(self, file_system_name: FileSystemProperties | str, file_name: str) -> DataLakeFileClient:
-        """Create a file under the file system.
+        """
+        Create a file under the file system.
 
         :param file_system_name: Name of the file system or instance of FileSystemProperties.
         :param file_name: Name of the file which needs to be created in the file system.
@@ -459,7 +471,8 @@ class AzureDataLakeStorageV2Hook(BaseHook):
         overwrite: bool = False,
         **kwargs: Any,
     ) -> None:
-        """Create a file with data in the file system.
+        """
+        Create a file with data in the file system.
 
         :param file_system_name: Name of the file system or instance of FileSystemProperties.
         :param file_name: Name of the file to be created with name.
@@ -479,7 +492,8 @@ class AzureDataLakeStorageV2Hook(BaseHook):
         overwrite: bool = False,
         **kwargs: Any,
     ) -> None:
-        """Upload data to a file.
+        """
+        Upload data to a file.
 
         :param file_system_name: Name of the file system or instance of FileSystemProperties.
         :param directory_name: Name of the directory.
@@ -495,7 +509,8 @@ class AzureDataLakeStorageV2Hook(BaseHook):
     def list_files_directory(
         self, file_system_name: FileSystemProperties | str, directory_name: str
     ) -> list[str]:
-        """List files or directories under the specified file system.
+        """
+        List files or directories under the specified file system.
 
         :param file_system_name: Name of the file system or instance of FileSystemProperties.
         :param directory_name: Name of the directory.
@@ -509,7 +524,8 @@ class AzureDataLakeStorageV2Hook(BaseHook):
     def list_file_system(
         self, prefix: str | None = None, include_metadata: bool = False, **kwargs: Any
     ) -> list[str]:
-        """List file systems under the specified account.
+        """
+        List file systems under the specified account.
 
         :param prefix:
             Filters the results to return only file systems whose names
@@ -526,7 +542,8 @@ class AzureDataLakeStorageV2Hook(BaseHook):
         return file_system_list
 
     def delete_file_system(self, file_system_name: FileSystemProperties | str) -> None:
-        """Delete the file system.
+        """
+        Delete the file system.
 
         :param file_system_name: Name of the file system or instance of FileSystemProperties.
         """
@@ -540,7 +557,8 @@ class AzureDataLakeStorageV2Hook(BaseHook):
             raise
 
     def delete_directory(self, file_system_name: FileSystemProperties | str, directory_name: str) -> None:
-        """Delete the specified directory in a file system.
+        """
+        Delete the specified directory in a file system.
 
         :param file_system_name: Name of the file system or instance of FileSystemProperties.
         :param directory_name: Name of the directory.

{apache_airflow_providers_microsoft_azure-10.1.2rc1 → apache_airflow_providers_microsoft_azure-10.2.0}/airflow/providers/microsoft/azure/operators/asb.py

@@ -32,7 +32,8 @@ if TYPE_CHECKING:
 
 
 class AzureServiceBusCreateQueueOperator(BaseOperator):
-    """Create a Azure Service Bus queue under a Service Bus Namespace.
+    """
+    Create a Azure Service Bus queue under a Service Bus Namespace.
 
     .. seealso::
         For more information on how to use this operator, take a look at the guide:
@@ -84,7 +85,8 @@ class AzureServiceBusCreateQueueOperator(BaseOperator):
 
 
 class AzureServiceBusSendMessageOperator(BaseOperator):
-    """Send Message or batch message to the Service Bus queue.
+    """
+    Send Message or batch message to the Service Bus queue.
 
     .. seealso::
         For more information on how to use this operator, take a look at the guide:
@@ -126,7 +128,8 @@ class AzureServiceBusSendMessageOperator(BaseOperator):
 
 
 class AzureServiceBusReceiveMessageOperator(BaseOperator):
-    """Receive a batch of messages at once in a specified Queue name.
+    """
+    Receive a batch of messages at once in a specified Queue name.
 
     .. seealso::
         For more information on how to use this operator, take a look at the guide:
@@ -169,7 +172,8 @@ class AzureServiceBusReceiveMessageOperator(BaseOperator):
 
 
 class AzureServiceBusDeleteQueueOperator(BaseOperator):
-    """Delete the Queue in the Azure Service Bus namespace.
+    """
+    Delete the Queue in the Azure Service Bus namespace.
 
     .. seealso::
         For more information on how to use this operator, take a look at the guide:
@@ -204,7 +208,8 @@ class AzureServiceBusDeleteQueueOperator(BaseOperator):
 
 
 class AzureServiceBusTopicCreateOperator(BaseOperator):
-    """Create an Azure Service Bus Topic under a Service Bus Namespace.
+    """
+    Create an Azure Service Bus Topic under a Service Bus Namespace.
 
     .. seealso::
         For more information on how to use this operator, take a look at the guide:
@@ -325,7 +330,8 @@ class AzureServiceBusTopicCreateOperator(BaseOperator):
 
 
 class AzureServiceBusSubscriptionCreateOperator(BaseOperator):
-    """Create an Azure Service Bus Topic Subscription under a Service Bus Namespace.
+    """
+    Create an Azure Service Bus Topic Subscription under a Service Bus Namespace.
 
     .. seealso::
         For more information on how to use this operator, take a look at the guide:
@@ -431,7 +437,8 @@ class AzureServiceBusSubscriptionCreateOperator(BaseOperator):
 
 
 class AzureServiceBusUpdateSubscriptionOperator(BaseOperator):
-    """Update an Azure ServiceBus Topic Subscription under a ServiceBus Namespace.
+    """
+    Update an Azure ServiceBus Topic Subscription under a ServiceBus Namespace.
 
     .. seealso::
         For more information on how to use this operator, take a look at the guide:
@@ -490,7 +497,8 @@ class AzureServiceBusUpdateSubscriptionOperator(BaseOperator):
 
 
 class ASBReceiveSubscriptionMessageOperator(BaseOperator):
-    """Receive a Batch messages from a Service Bus Subscription under specific Topic.
+    """
+    Receive a Batch messages from a Service Bus Subscription under specific Topic.
 
     .. seealso::
         For more information on how to use this operator, take a look at the guide:
@@ -541,7 +549,8 @@ class ASBReceiveSubscriptionMessageOperator(BaseOperator):
 
 
 class AzureServiceBusSubscriptionDeleteOperator(BaseOperator):
-    """Delete the topic subscription in the Azure ServiceBus namespace.
+    """
+    Delete the topic subscription in the Azure ServiceBus namespace.
 
     .. seealso::
         For more information on how to use this operator, take a look at the guide:
@@ -579,7 +588,8 @@ class AzureServiceBusSubscriptionDeleteOperator(BaseOperator):
 
 
 class AzureServiceBusTopicDeleteOperator(BaseOperator):
-    """Delete the topic in the Azure Service Bus namespace.
+    """
+    Delete the topic in the Azure Service Bus namespace.
 
     .. seealso::
         For more information on how to use this operator, take a look at the guide:

{apache_airflow_providers_microsoft_azure-10.1.2rc1 → apache_airflow_providers_microsoft_azure-10.2.0}/airflow/providers/microsoft/azure/operators/cosmos.py

@@ -27,7 +27,8 @@ if TYPE_CHECKING:
 
 
 class AzureCosmosInsertDocumentOperator(BaseOperator):
-    """Insert a new document into the specified Cosmos database and collection.
+    """
+    Insert a new document into the specified Cosmos database and collection.
 
     Both the database and collection will be created automatically if they do
     not already exist.

{apache_airflow_providers_microsoft_azure-10.1.2rc1 → apache_airflow_providers_microsoft_azure-10.2.0}/airflow/providers/microsoft/azure/sensors/data_factory.py

@@ -101,7 +101,8 @@ class AzureDataFactoryPipelineRunStatusSensor(BaseSensorOperator):
         return pipeline_run_status == AzureDataFactoryPipelineRunStatus.SUCCEEDED
 
     def execute(self, context: Context) -> None:
-        """Poll for state of the job run.
+        """
+        Poll for state of the job run.
 
         In deferrable mode, the polling is deferred to the triggerer. Otherwise
         the sensor waits synchronously.

{apache_airflow_providers_microsoft_azure-10.1.2rc1 → apache_airflow_providers_microsoft_azure-10.2.0}/airflow/providers/microsoft/azure/sensors/wasb.py

@@ -74,7 +74,8 @@ class WasbBlobSensor(BaseSensorOperator):
         return hook.check_for_blob(self.container_name, self.blob_name, **self.check_options)
 
     def execute(self, context: Context) -> None:
-        """Poll for state of the job run.
+        """
+        Poll for state of the job run.
 
         In deferrable mode, the polling is deferred to the triggerer. Otherwise
         the sensor waits synchronously.
@@ -184,7 +185,8 @@ class WasbPrefixSensor(BaseSensorOperator):
         return hook.check_for_prefix(self.container_name, self.prefix, **self.check_options)
 
     def execute(self, context: Context) -> None:
-        """Poll for state of the job run.
+        """
+        Poll for state of the job run.
 
         In deferrable mode, the polling is deferred to the triggerer. Otherwise
         the sensor waits synchronously.

apache_airflow_providers_microsoft_azure-10.2.0/airflow/providers/microsoft/azure/transfers/s3_to_wasb.py

@@ -0,0 +1,266 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import tempfile
+from functools import cached_property
+from typing import TYPE_CHECKING, Sequence
+
+from airflow.models import BaseOperator
+from airflow.providers.amazon.aws.hooks.s3 import S3Hook
+from airflow.providers.microsoft.azure.hooks.wasb import WasbHook
+
+if TYPE_CHECKING:
+    from airflow.utils.context import Context
+
+
+# Create three custom exception that are
+class TooManyFilesToMoveException(Exception):
+    """Custom exception thrown when attempting to move multiple files from S3 to a single Azure Blob."""
+
+    def __init__(self, number_of_files: int):
+        # Call the parent constructor with a simple message
+        message: str = f"{number_of_files} cannot be moved to a single Azure Blob."
+        super().__init__(message)
+
+
+class InvalidAzureBlobParameters(Exception):
+    """Custom exception raised when neither a blob_prefix or blob_name are passed to the operator."""
+
+    def __init__(self):
+        message: str = "One of blob_name or blob_prefix must be provided."
+        super().__init__(message)
+
+
+class InvalidKeyComponents(Exception):
+    """Custom exception raised when neither a full_path or file_name + prefix are provided to _create_key."""
+
+    def __init__(self):
+        message = "Either full_path of prefix and file_name must not be None"
+        super().__init__(message)
+
+
+class S3ToAzureBlobStorageOperator(BaseOperator):
+    """
+    Operator to move data from and AWS S3 Bucket to Microsoft Azure Blob Storage.
+
+    A similar class exists to move data from Microsoft Azure Blob Storage to an AWS S3 Bucket, and lives in
+    the airflow/providers/amazon/aws/transfers/azure_blob_to_s3.py file
+
+    Either an explicit S3 key can be provided, or a prefix containing the files that are to be transferred to
+    Azure blob storage. The same holds for a Blob name; an explicit name can be passed, or a Blob prefix can
+    be provided for the file to be stored to
+
+    .. seealso:
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator::SFTPToWasbOperator`
+
+    :param aws_conn_id: ID for the AWS S3 connection to use.
+    :param wasb_conn_id: ID for the Azure Blob Storage connection to use.
+    :param s3_bucket: The name of the AWS S3 bucket that an object (or objects)  would be transferred from.
+        (templated)
+    :param container_name: The name of the Azure Storage Blob container an object (or objects) would be
+        transferred to. (templated)
+    :param s3_prefix: Prefix string that filters any S3 objects that begin with this prefix. (templated)
+    :param s3_key: An explicit S3 key (object) to be transferred. (templated)
+    :param blob_prefix: Prefix string that would provide a path in the Azure Storage Blob container for an
+        object (or objects) to be moved to. (templated)
+    :param blob_name: An explicit blob name that an object would be transferred to. This can only be used
+        if a single file is being moved. If there are multiple files in an S3 bucket that are to be moved
+        to a single Azure blob, an exception will be raised. (templated)
+    :param create_container: True if a container should be created if it did not already exist, False
+        otherwise.
+    :param replace: If a blob exists in the container and replace takes a value of true, it will be
+        overwritten. If replace is False and a blob exists in the container, the file will NOT be
+        overwritten.
+    :param s3_verify: Whether or not to verify SSL certificates for S3 connection.
+        By default, SSL certificates are verified.
+        You can provide the following values:
+
+        - ``False``: do not validate SSL certificates. SSL will still be used
+                 (unless use_ssl is False), but SSL certificates will not be
+                 verified.
+        - ``path/to/cert/bundle.pem``: A filename of the CA cert bundle to uses.
+                 You can specify this argument if you want to use a different
+                 CA cert bundle than the one used by botocore.
+    :param s3_extra_args: kwargs to pass to S3Hook.
+    :param wasb_extra_args: kwargs to pass to WasbHook.
+    """
+
+    template_fields: Sequence[str] = (
+        "s3_bucket",
+        "container_name",
+        "s3_prefix",
+        "s3_key",
+        "blob_prefix",
+        "blob_name",
+    )
+
+    def __init__(
+        self,
+        *,
+        aws_conn_id: str = "aws_default",
+        wasb_conn_id: str = "wasb_default",
+        s3_bucket: str,
+        container_name: str,
+        s3_prefix: str | None = None,  # Only use this to pull an entire directory of files
+        s3_key: str | None = None,  # Only use this to pull a single file
+        blob_prefix: str | None = None,
+        blob_name: str | None = None,
+        create_container: bool = False,
+        replace: bool = False,
+        s3_verify: bool = False,
+        s3_extra_args: dict | None = None,
+        wasb_extra_args: dict | None = None,
+        **kwargs,
+    ):
+        # Call to constructor of the inherited BaseOperator class
+        super().__init__(**kwargs)
+
+        self.aws_conn_id = aws_conn_id
+        self.wasb_conn_id = wasb_conn_id
+        self.s3_bucket = s3_bucket
+        self.container_name = container_name
+        self.s3_prefix = s3_prefix
+        self.s3_key = s3_key
+        self.blob_prefix = blob_prefix
+        self.blob_name = blob_name
+        self.create_container = create_container
+        self.replace = replace
+        self.s3_verify = s3_verify
+        self.s3_extra_args = s3_extra_args or {}
+        self.wasb_extra_args = wasb_extra_args or {}
+
+    # These cached properties come in handy when working with hooks. Rather than closing and opening new
+    # hooks, the same hook can be used across multiple methods (without having to use the constructor to
+    # create the hook)
+    @cached_property
+    def s3_hook(self) -> S3Hook:
+        """Create and return an S3Hook."""
+        return S3Hook(aws_conn_id=self.aws_conn_id, verify=self.s3_verify, **self.s3_extra_args)
+
+    @cached_property
+    def wasb_hook(self) -> WasbHook:
+        """Create and return a WasbHook."""
+        return WasbHook(wasb_conn_id=self.wasb_conn_id, **self.wasb_extra_args)
+
+    def execute(self, context: Context) -> list[str]:
+        """Execute logic below when operator is executed as a task."""
+        self.log.info(
+            "Getting %s from %s" if self.s3_key else "Getting all files start with %s from %s",
+            self.s3_key if self.s3_key else self.s3_prefix,
+            self.s3_bucket,
+        )
+
+        # Pull a list of files to move from S3 to Azure Blob storage
+        files_to_move: list[str] = self.get_files_to_move()
+
+        # Check to see if there are indeed files to move. If so, move each of these files. Otherwise, output
+        # a logging message that denotes there are no files to move
+        if files_to_move:
+            for file_name in files_to_move:
+                self.move_file(file_name)
+
+            # Assuming that files_to_move is a list (which it always should be), this will get "hit" after the
+            # last file is moved from S3 -> Azure Blob
+            self.log.info("All done, uploaded %s to Azure Blob.", len(files_to_move))
+
+        else:
+            # If there are no files to move, a message will be logged. May want to consider alternative
+            # functionality (should an exception instead be raised?)
+            self.log.info("There are no files to move!")
+
+        # Return a list of the files that were moved
+        return files_to_move
+
+    def get_files_to_move(self) -> list[str]:
+        """Determine the list of files that need to be moved, and return the name."""
+        if self.s3_key:
+            # Only pull the file name from the s3_key, drop the rest of the key
+            files_to_move: list[str] = [self.s3_key.split("/")[-1]]
+        else:
+            # Pull the keys from the s3_bucket using the provided prefix. Remove the prefix from the file
+            # name, and add to the list of files to move
+            s3_keys: list[str] = self.s3_hook.list_keys(bucket_name=self.s3_bucket, prefix=self.s3_prefix)
+            files_to_move = [s3_key.replace(f"{self.s3_prefix}/", "", 1) for s3_key in s3_keys]
+
+            # Now, make sure that there are not too many files to move to a single Azure blob
+            if self.blob_name and len(files_to_move) > 1:
+                raise TooManyFilesToMoveException(len(files_to_move))
+
+        if not self.replace:
+            # Only grab the files from S3 that are not in Azure Blob already. This will prevent any files that
+            # exist in both S3 and Azure Blob from being overwritten. If a blob_name is provided, check to
+            # see if that blob exists
+            azure_blob_files: list[str] = []
+
+            if self.blob_name:
+                # If the singular blob (stored at self.blob_name) exists, add it to azure_blob_files so it
+                # can be removed from the list of files to move
+                if self.wasb_hook.check_for_blob(self.container_name, self.blob_name):
+                    azure_blob_files.append(self.blob_name.split("/")[-1])
+
+            elif self.blob_prefix:
+                azure_blob_files += self.wasb_hook.get_blobs_list_recursive(
+                    container_name=self.container_name, prefix=self.blob_prefix
+                )
+            else:
+                raise InvalidAzureBlobParameters
+
+            # This conditional block only does one thing - it alters the elements in the files_to_move list.
+            # This list is being trimmed to remove the existing files in the Azure Blob (as mentioned above)
+            existing_files = azure_blob_files if azure_blob_files else []
+            files_to_move = list(set(files_to_move) - set(existing_files))
+
+        return files_to_move
+
+    def move_file(self, file_name: str) -> None:
+        """Move file from S3 to Azure Blob storage."""
+        with tempfile.NamedTemporaryFile("w") as temp_file:
+            # If using an s3_key, this creates a scenario where the only file in the files_to_move
+            # list is going to be the name pulled from the s3_key. It's not verbose, but provides
+            # standard implementation across the operator
+            source_s3_key: str = self._create_key(self.s3_key, self.s3_prefix, file_name)
+
+            # Create retrieve the S3 client itself, rather than directly using the hook. Download the file to
+            # the temp_file.name
+            s3_client = self.s3_hook.get_conn()
+            s3_client.download_file(self.s3_bucket, source_s3_key, temp_file.name)
+
+            # Load the file to Azure Blob using either the key that has been passed in, or the key
+            # from the list of files present in the s3_prefix, plus the blob_prefix. There may be
+            # desire to only pass in an S3 key, in which case, the blob_name should be derived from
+            # the S3 key
+            destination_azure_blob_name: str = self._create_key(self.blob_name, self.blob_prefix, file_name)
+            self.wasb_hook.load_file(
+                file_path=temp_file.name,
+                container_name=self.container_name,
+                blob_name=destination_azure_blob_name,
+                create_container=self.create_container,
+                **self.wasb_extra_args,
+            )
+
+    @staticmethod
+    def _create_key(full_path: str | None, prefix: str | None, file_name: str | None):
+        """Return a file key using its components."""
+        if full_path:
+            return full_path
+        elif prefix and file_name:
+            return f"{prefix}/{file_name}"
+        else:
+            raise InvalidKeyComponents

{apache_airflow_providers_microsoft_azure-10.1.2rc1 → apache_airflow_providers_microsoft_azure-10.2.0}/airflow/providers/microsoft/azure/triggers/data_factory.py

@@ -30,7 +30,8 @@ from airflow.triggers.base import BaseTrigger, TriggerEvent
 
 
 class ADFPipelineRunStatusSensorTrigger(BaseTrigger):
-    """Trigger with params to run the task when the ADF Pipeline is running.
+    """
+    Trigger with params to run the task when the ADF Pipeline is running.
 
     :param run_id: The pipeline run identifier.
     :param azure_data_factory_conn_id: The connection identifier for connecting to Azure Data Factory.
@@ -110,7 +111,8 @@ class ADFPipelineRunStatusSensorTrigger(BaseTrigger):
 
 
 class AzureDataFactoryTrigger(BaseTrigger):
-    """Trigger when the Azure data factory pipeline job finishes.
+    """
+    Trigger when the Azure data factory pipeline job finishes.
 
     When wait_for_termination is set to False, it triggers immediately with success status.
 

{apache_airflow_providers_microsoft_azure-10.1.2rc1 → apache_airflow_providers_microsoft_azure-10.2.0}/airflow/providers/microsoft/azure/utils.py

@@ -62,7 +62,8 @@ def _get_default_azure_credential(
     workload_identity_tenant_id: str | None = None,
     use_async: bool = False,
 ) -> DefaultAzureCredential | AsyncDefaultAzureCredential:
-    """Get DefaultAzureCredential based on provided arguments.
+    """
+    Get DefaultAzureCredential based on provided arguments.
 
     If managed_identity_client_id and workload_identity_tenant_id are provided, this function returns
     DefaultAzureCredential with managed identity.
@@ -115,7 +116,8 @@ def add_managed_identity_connection_widgets(func):
 
 
 class AzureIdentityCredentialAdapter(BasicTokenAuthentication):
-    """Adapt azure-identity credentials for backward compatibility.
+    """
+    Adapt azure-identity credentials for backward compatibility.
 
     Adapt credentials from azure-identity to be compatible with SD
     that needs msrestazure or azure.common.credentials
@@ -132,7 +134,8 @@ class AzureIdentityCredentialAdapter(BasicTokenAuthentication):
         workload_identity_tenant_id: str | None = None,
         **kwargs,
     ):
-        """Adapt azure-identity credentials for backward compatibility.
+        """
+        Adapt azure-identity credentials for backward compatibility.
 
         :param credential: Any azure-identity credential (DefaultAzureCredential by default)
         :param resource_id: The scope to use to get the token (default ARM)
@@ -156,7 +159,8 @@ class AzureIdentityCredentialAdapter(BasicTokenAuthentication):
         )
 
     def set_token(self):
-        """Ask the azure-core BearerTokenCredentialPolicy policy to get a token.
+        """
+        Ask the azure-core BearerTokenCredentialPolicy policy to get a token.
 
         Using the policy gives us for free the caching system of azure-core.
         We could make this code simpler by using private method, but by definition

{apache_airflow_providers_microsoft_azure-10.1.2rc1 → apache_airflow_providers_microsoft_azure-10.2.0}/pyproject.toml

@@ -28,7 +28,7 @@ build-backend = "flit_core.buildapi"
 
 [project]
 name = "apache-airflow-providers-microsoft-azure"
-version = "10.1.2.rc1"
+version = "10.2.0"
 description = "Provider package apache-airflow-providers-microsoft-azure for Apache Airflow"
 readme = "README.rst"
 authors = [
@@ -58,7 +58,7 @@ requires-python = "~=3.8"
 dependencies = [
     "adal>=1.2.7",
     "adlfs>=2023.10.0",
-    "apache-airflow>=2.7.0rc0",
+    "apache-airflow>=2.7.0",
     "azure-batch>=8.0.0",
     "azure-cosmos>=4.6.0",
     "azure-datalake-store>=0.0.45",
@@ -82,8 +82,8 @@ dependencies = [
 ]
 
 [project.urls]
-"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.1.2"
-"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.1.2/changelog.html"
+"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.2.0"
+"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.2.0/changelog.html"
 "Bug Tracker" = "https://github.com/apache/airflow/issues"
 "Source Code" = "https://github.com/apache/airflow"
 "Slack Chat" = "https://s.apache.org/airflow-slack"
@@ -93,6 +93,9 @@ dependencies = [
 [project.entry-points."apache_airflow_provider"]
 provider_info = "airflow.providers.microsoft.azure.get_provider_info:get_provider_info"
 [project.optional-dependencies]
+"amazon" = [
+    "apache-airflow-providers-amazon",
+]
 "google" = [
     "apache-airflow-providers-google",
 ]