apache-airflow-providers-microsoft-azure 6.2.4rc1__tar.gz → 6.3.0rc1__tar.gz

{apache-airflow-providers-microsoft-azure-6.2.4rc1/apache_airflow_providers_microsoft_azure.egg-info → apache-airflow-providers-microsoft-azure-6.3.0rc1}/PKG-INFO

@@ -1,14 +1,14 @@
 Metadata-Version: 2.1
 Name: apache-airflow-providers-microsoft-azure
-Version: 6.2.4rc1
+Version: 6.3.0rc1
 Summary: Provider for Apache Airflow. Implements apache-airflow-providers-microsoft-azure package
 Home-page: https://airflow.apache.org/
 Download-URL: https://archive.apache.org/dist/airflow/providers
 Author: Apache Software Foundation
 Author-email: dev@airflow.apache.org
 License: Apache License 2.0
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/6.2.4/
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/6.2.4/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/6.3.0/
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/6.3.0/changelog.html
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
@@ -73,7 +73,7 @@ License-File: NOTICE
 
 Package ``apache-airflow-providers-microsoft-azure``
 
-Release: ``6.2.4rc1``
+Release: ``6.3.0rc1``
 
 
 `Microsoft Azure <https://azure.microsoft.com/>`__
@@ -86,7 +86,7 @@ This is a provider package for ``microsoft.azure`` provider. All classes for thi
 are in ``airflow.providers.microsoft.azure`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/6.2.4/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/6.3.0/>`_.
 
 
 Installation
@@ -119,9 +119,9 @@ PIP package                       Version required
 ``azure-synapse-spark``
 ``adal``                          ``>=1.2.7``
 ``azure-storage-file-datalake``   ``>=12.9.1``
+``azure-kusto-data``              ``>=4.1.0``
 ``azure-mgmt-containerinstance``  ``>=1.5.0,<2.0``
 ``azure-mgmt-datafactory``        ``>=1.0.0,<2.0``
-``azure-kusto-data``              ``>=0.0.43,<0.1``
 ================================  ==================
 
 Cross provider package dependencies
@@ -146,4 +146,4 @@ Dependent package
 ====================================================================================================  ==========
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/6.2.4/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/6.3.0/changelog.html>`_.

{apache-airflow-providers-microsoft-azure-6.2.4rc1 → apache-airflow-providers-microsoft-azure-6.3.0rc1}/README.rst

@@ -36,7 +36,7 @@
 
 Package ``apache-airflow-providers-microsoft-azure``
 
-Release: ``6.2.4rc1``
+Release: ``6.3.0rc1``
 
 
 `Microsoft Azure <https://azure.microsoft.com/>`__
@@ -49,7 +49,7 @@ This is a provider package for ``microsoft.azure`` provider. All classes for thi
 are in ``airflow.providers.microsoft.azure`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/6.2.4/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/6.3.0/>`_.
 
 
 Installation
@@ -82,9 +82,9 @@ PIP package                       Version required
 ``azure-synapse-spark``
 ``adal``                          ``>=1.2.7``
 ``azure-storage-file-datalake``   ``>=12.9.1``
+``azure-kusto-data``              ``>=4.1.0``
 ``azure-mgmt-containerinstance``  ``>=1.5.0,<2.0``
 ``azure-mgmt-datafactory``        ``>=1.0.0,<2.0``
-``azure-kusto-data``              ``>=0.0.43,<0.1``
 ================================  ==================
 
 Cross provider package dependencies
@@ -109,4 +109,4 @@ Dependent package
 ====================================================================================================  ==========
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/6.2.4/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/6.3.0/changelog.html>`_.

{apache-airflow-providers-microsoft-azure-6.2.4rc1 → apache-airflow-providers-microsoft-azure-6.3.0rc1}/airflow/providers/microsoft/azure/__init__.py

@@ -28,7 +28,7 @@ import packaging.version
 
 __all__ = ["__version__"]
 
-__version__ = "6.2.4"
+__version__ = "6.3.0"
 
 try:
     from airflow import __version__ as airflow_version

{apache-airflow-providers-microsoft-azure-6.2.4rc1 → apache-airflow-providers-microsoft-azure-6.3.0rc1}/airflow/providers/microsoft/azure/get_provider_info.py

@@ -29,6 +29,7 @@ def get_provider_info():
         "description": "`Microsoft Azure <https://azure.microsoft.com/>`__\n",
         "suspended": False,
         "versions": [
+            "6.3.0",
             "6.2.4",
             "6.2.3",
             "6.2.2",
@@ -85,14 +86,15 @@ def get_provider_info():
             "azure-synapse-spark",
             "adal>=1.2.7",
             "azure-storage-file-datalake>=12.9.1",
+            "azure-kusto-data>=4.1.0",
             "azure-mgmt-containerinstance>=1.5.0,<2.0",
             "azure-mgmt-datafactory>=1.0.0,<2.0",
-            "azure-kusto-data>=0.0.43,<0.1",
         ],
         "integrations": [
             {
                 "integration-name": "Microsoft Azure Batch",
                 "external-doc-url": "https://azure.microsoft.com/en-us/services/batch/",
+                "how-to-guide": ["/docs/apache-airflow-providers-microsoft-azure/operators/batch.rst"],
                 "logo": "/integration-logos/azure/Microsoft-Azure-Batch.png",
                 "tags": ["azure"],
             },
@@ -310,11 +312,6 @@ def get_provider_info():
                 "how-to-guide": "/docs/apache-airflow-providers-microsoft-azure/transfer/local_to_wasb.rst",
                 "python-module": "airflow.providers.microsoft.azure.transfers.local_to_wasb",
             },
-            {
-                "source-integration-name": "Microsoft Azure Blob Storage",
-                "target-integration-name": "Google Cloud Storage (GCS)",
-                "python-module": "airflow.providers.microsoft.azure.transfers.azure_blob_to_gcs",
-            },
             {
                 "source-integration-name": "SSH File Transfer Protocol (SFTP)",
                 "target-integration-name": "Microsoft Azure Blob Storage",

{apache-airflow-providers-microsoft-azure-6.2.4rc1 → apache-airflow-providers-microsoft-azure-6.3.0rc1}/airflow/providers/microsoft/azure/hooks/adx.py

@@ -28,8 +28,9 @@ from __future__ import annotations
 import warnings
 from typing import Any
 
+from azure.identity import DefaultAzureCredential
+from azure.kusto.data import ClientRequestProperties, KustoClient, KustoConnectionStringBuilder
 from azure.kusto.data.exceptions import KustoServiceError
-from azure.kusto.data.request import ClientRequestProperties, KustoClient, KustoConnectionStringBuilder
 from azure.kusto.data.response import KustoResponseDataSetV2
 
 from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
@@ -105,7 +106,7 @@ class AzureDataExplorerHook(BaseHook):
             "placeholders": {
                 "login": "Varies with authentication method",
                 "password": "Varies with authentication method",
-                "auth_method": "AAD_APP/AAD_APP_CERT/AAD_CREDS/AAD_DEVICE",
+                "auth_method": "AAD_APP/AAD_APP_CERT/AAD_CREDS/AAD_DEVICE/AZURE_TOKEN_CRED",
                 "tenant": "Used with AAD_APP/AAD_APP_CERT/AAD_CREDS",
                 "certificate": "Used with AAD_APP_CERT",
                 "thumbprint": "Used with AAD_APP_CERT",
@@ -183,6 +184,11 @@ class AzureDataExplorerHook(BaseHook):
             )
         elif auth_method == "AAD_DEVICE":
             kcsb = KustoConnectionStringBuilder.with_aad_device_authentication(cluster)
+        elif auth_method == "AZURE_TOKEN_CRED":
+            kcsb = KustoConnectionStringBuilder.with_azure_token_credential(
+                connection_string=cluster,
+                credential=DefaultAzureCredential(),
+            )
         else:
             raise AirflowException(f"Unknown authentication method: {auth_method}")
 

{apache-airflow-providers-microsoft-azure-6.2.4rc1 → apache-airflow-providers-microsoft-azure-6.3.0rc1}/airflow/providers/microsoft/azure/hooks/base_azure.py

@@ -24,6 +24,7 @@ from azure.common.credentials import ServicePrincipalCredentials
 
 from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
 from airflow.hooks.base import BaseHook
+from airflow.providers.microsoft.azure.utils import AzureIdentityCredentialAdapter
 
 
 class AzureBaseHook(BaseHook):
@@ -124,10 +125,17 @@ class AzureBaseHook(BaseHook):
             self.log.info("Getting connection using a JSON config.")
             return get_client_from_json_dict(client_class=self.sdk_client, config_dict=key_json)
 
-        self.log.info("Getting connection using specific credentials and subscription_id.")
-        return self.sdk_client(
-            credentials=ServicePrincipalCredentials(
+        credentials: ServicePrincipalCredentials | AzureIdentityCredentialAdapter
+        if all([conn.login, conn.password, tenant]):
+            self.log.info("Getting connection using specific credentials and subscription_id.")
+            credentials = ServicePrincipalCredentials(
                 client_id=conn.login, secret=conn.password, tenant=tenant
-            ),
+            )
+        else:
+            self.log.info("Using DefaultAzureCredential as credential")
+            credentials = AzureIdentityCredentialAdapter()
+
+        return self.sdk_client(
+            credentials=credentials,
             subscription_id=subscription_id,
         )

{apache-airflow-providers-microsoft-azure-6.2.4rc1 → apache-airflow-providers-microsoft-azure-6.3.0rc1}/airflow/providers/microsoft/azure/hooks/batch.py

@@ -27,7 +27,7 @@ from azure.batch.models import JobAddParameter, PoolAddParameter, TaskAddParamet
 from airflow.exceptions import AirflowException
 from airflow.hooks.base import BaseHook
 from airflow.models import Connection
-from airflow.providers.microsoft.azure.utils import get_field
+from airflow.providers.microsoft.azure.utils import AzureIdentityCredentialAdapter, get_field
 from airflow.utils import timezone
 
 
@@ -96,7 +96,15 @@ class AzureBatchHook(BaseHook):
         if not batch_account_url:
             raise AirflowException("Batch Account URL parameter is missing.")
 
-        credentials = batch_auth.SharedKeyCredentials(conn.login, conn.password)
+        credentials: batch_auth.SharedKeyCredentials | AzureIdentityCredentialAdapter
+        if all([conn.login, conn.password]):
+            credentials = batch_auth.SharedKeyCredentials(conn.login, conn.password)
+        else:
+            credentials = AzureIdentityCredentialAdapter(
+                None, resource_id="https://batch.core.windows.net/.default"
+            )
+            # credentials = AzureIdentityCredentialAdapter()
+
         batch_client = BatchServiceClient(credentials, batch_url=batch_account_url)
         return batch_client
 
@@ -344,7 +352,6 @@ class AzureBatchHook(BaseHook):
         :param task: The task to add
         """
         try:
-
             self.connection.task.add(job_id=job_id, task=task)
         except batch_models.BatchErrorException as err:
             if not err.error or err.error.code != "TaskExists":

{apache-airflow-providers-microsoft-azure-6.2.4rc1 → apache-airflow-providers-microsoft-azure-6.3.0rc1}/airflow/providers/microsoft/azure/hooks/cosmos.py

@@ -25,7 +25,6 @@ the default database and collection to use (see connection `azure_cosmos_default
 """
 from __future__ import annotations
 
-import json
 import uuid
 from typing import Any
 
@@ -145,15 +144,17 @@ class AzureCosmosDBHook(BaseHook):
         if collection_name is None:
             raise AirflowBadRequest("Collection name cannot be None.")
 
+        # The ignores below is due to typing bug in azure-cosmos 9.2.0
+        # https://github.com/Azure/azure-sdk-for-python/issues/31811
         existing_container = list(
             self.get_conn()
             .get_database_client(self.__get_database_name(database_name))
             .query_containers(
                 "SELECT * FROM r WHERE r.id=@id",
-                parameters=[json.dumps({"name": "@id", "value": collection_name})],
+                parameters=[{"name": "@id", "value": collection_name}],  # type: ignore[list-item]
             )
         )
-        if len(existing_container) == 0:
+        if not existing_container:
             return False
 
         return True
@@ -170,17 +171,19 @@ class AzureCosmosDBHook(BaseHook):
 
         # We need to check to see if this container already exists so we don't try
         # to create it twice
+        # The ignores below is due to typing bug in azure-cosmos 9.2.0
+        # https://github.com/Azure/azure-sdk-for-python/issues/31811
         existing_container = list(
             self.get_conn()
             .get_database_client(self.__get_database_name(database_name))
             .query_containers(
                 "SELECT * FROM r WHERE r.id=@id",
-                parameters=[json.dumps({"name": "@id", "value": collection_name})],
+                parameters=[{"name": "@id", "value": collection_name}],  # type: ignore[list-item]
             )
         )
 
         # Only create if we did not find it already existing
-        if len(existing_container) == 0:
+        if not existing_container:
             self.get_conn().get_database_client(self.__get_database_name(database_name)).create_container(
                 collection_name, partition_key=partition_key
             )
@@ -190,13 +193,15 @@ class AzureCosmosDBHook(BaseHook):
         if database_name is None:
             raise AirflowBadRequest("Database name cannot be None.")
 
+        # The ignores below is due to typing bug in azure-cosmos 9.2.0
+        # https://github.com/Azure/azure-sdk-for-python/issues/31811
         existing_database = list(
             self.get_conn().query_databases(
                 "SELECT * FROM r WHERE r.id=@id",
-                parameters=[json.dumps({"name": "@id", "value": database_name})],
+                parameters=[{"name": "@id", "value": database_name}],  # type: ignore[list-item]
             )
         )
-        if len(existing_database) == 0:
+        if not existing_database:
             return False
 
         return True
@@ -208,15 +213,17 @@ class AzureCosmosDBHook(BaseHook):
 
         # We need to check to see if this database already exists so we don't try
         # to create it twice
+        # The ignores below is due to typing bug in azure-cosmos 9.2.0
+        # https://github.com/Azure/azure-sdk-for-python/issues/31811
         existing_database = list(
             self.get_conn().query_databases(
                 "SELECT * FROM r WHERE r.id=@id",
-                parameters=[json.dumps({"name": "@id", "value": database_name})],
+                parameters=[{"name": "@id", "value": database_name}],  # type: ignore[list-item]
             )
         )
 
         # Only create if we did not find it already existing
-        if len(existing_database) == 0:
+        if not existing_database:
             self.get_conn().create_database(database_name)
 
     def delete_database(self, database_name: str) -> None:

{apache-airflow-providers-microsoft-azure-6.2.4rc1 → apache-airflow-providers-microsoft-azure-6.3.0rc1}/airflow/providers/microsoft/azure/hooks/data_lake.py

@@ -241,7 +241,7 @@ class AzureDataLakeStorageV2Hook(BaseHook):
     accounts that have a hierarchical namespace. Using Adls_v2 connection
     details create DataLakeServiceClient object.
 
-    Due to Wasb is marked as legacy and and retirement of the (ADLS1), it would
+    Due to Wasb is marked as legacy and retirement of the (ADLS1), it would
     be nice to implement ADLS gen2 hook for interacting with the storage account.
 
     .. seealso::

{apache-airflow-providers-microsoft-azure-6.2.4rc1 → apache-airflow-providers-microsoft-azure-6.3.0rc1}/airflow/providers/microsoft/azure/hooks/synapse.py

@@ -183,7 +183,7 @@ class AzureSynapseHook(BaseHook):
                 )
 
             # Wait to check the status of the job run based on the ``check_interval`` configured.
-            self.log.info("Sleeping for %s seconds", str(check_interval))
+            self.log.info("Sleeping for %s seconds", check_interval)
             time.sleep(check_interval)
 
             job_run_status = self.get_job_run_status()

{apache-airflow-providers-microsoft-azure-6.2.4rc1 → apache-airflow-providers-microsoft-azure-6.3.0rc1}/airflow/providers/microsoft/azure/hooks/wasb.py

@@ -28,6 +28,7 @@ from __future__ import annotations
 import logging
 import os
 from typing import Any, Union
+from urllib.parse import urlparse
 
 from asgiref.sync import sync_to_async
 from azure.core.exceptions import HttpResponseError, ResourceExistsError, ResourceNotFoundError
@@ -152,11 +153,13 @@ class WasbHook(BaseHook):
             # connection_string auth takes priority
             return BlobServiceClient.from_connection_string(connection_string, **extra)
 
-        account_url = (
-            conn.host
-            if conn.host and conn.host.startswith("https://")
-            else f"https://{conn.login}.blob.core.windows.net/"
-        )
+        account_url = conn.host if conn.host else f"https://{conn.login}.blob.core.windows.net/"
+        parsed_url = urlparse(account_url)
+
+        if not parsed_url.netloc and "." not in parsed_url.path:
+            # if there's no netloc and no dots in the path, then user only
+            # provided the Active Directory ID, not the full URL or DNS name
+            account_url = f"https://{conn.login}.blob.core.windows.net/"
 
         tenant = self._get_field(extra, "tenant_id")
         if tenant:
@@ -238,7 +241,7 @@ class WasbHook(BaseHook):
         :return: True if blobs matching the prefix exist, False otherwise.
         """
         blobs = self.get_blobs_list(container_name=container_name, prefix=prefix, **kwargs)
-        return len(blobs) > 0
+        return bool(blobs)
 
     def get_blobs_list(
         self,
@@ -499,7 +502,7 @@ class WasbHook(BaseHook):
             blobs_to_delete = [blob_name]
         else:
             blobs_to_delete = []
-        if not ignore_if_missing and len(blobs_to_delete) == 0:
+        if not ignore_if_missing and not blobs_to_delete:
             raise AirflowException(f"Blob(s) not found: {blob_name}")
 
         # The maximum number of blobs that can be deleted in a single request is 256 using the underlying
@@ -555,11 +558,13 @@ class WasbAsyncHook(WasbHook):
             )
             return self.blob_service_client
 
-        account_url = (
-            conn.host
-            if conn.host and conn.host.startswith("https://")
-            else f"https://{conn.login}.blob.core.windows.net/"
-        )
+        account_url = conn.host if conn.host else f"https://{conn.login}.blob.core.windows.net/"
+        parsed_url = urlparse(account_url)
+
+        if not parsed_url.netloc and "." not in parsed_url.path:
+            # if there's no netloc and no dots in the path, then user only
+            # provided the Active Directory ID, not the full URL or DNS name
+            account_url = f"https://{conn.login}.blob.core.windows.net/"
 
         tenant = self._get_field(extra, "tenant_id")
         if tenant:
@@ -678,4 +683,4 @@ class WasbAsyncHook(WasbHook):
         :param kwargs: Optional keyword arguments for ``ContainerClient.walk_blobs``
         """
         blobs = await self.get_blobs_list_async(container_name=container_name, prefix=prefix, **kwargs)
-        return len(blobs) > 0
+        return bool(blobs)

{apache-airflow-providers-microsoft-azure-6.2.4rc1 → apache-airflow-providers-microsoft-azure-6.3.0rc1}/airflow/providers/microsoft/azure/operators/batch.py

@@ -17,6 +17,7 @@
 # under the License.
 from __future__ import annotations
 
+from functools import cached_property
 from typing import TYPE_CHECKING, Any, Sequence
 
 from azure.batch import models as batch_models
@@ -176,7 +177,10 @@ class AzureBatchOperator(BaseOperator):
         self.timeout = timeout
         self.should_delete_job = should_delete_job
         self.should_delete_pool = should_delete_pool
-        self.hook = self.get_hook()
+
+    @cached_property
+    def hook(self):
+        return self.get_hook()
 
     def _check_inputs(self) -> Any:
         if not self.os_family and not self.vm_publisher:
@@ -189,7 +193,7 @@ class AzureBatchOperator(BaseOperator):
             )
 
         if self.use_latest_image:
-            if not all(elem for elem in [self.vm_publisher, self.vm_offer]):
+            if not self.vm_publisher or not self.vm_offer:
                 raise AirflowException(
                     f"If use_latest_image_and_sku is set to True then the parameters vm_publisher, "
                     f"vm_offer, must all be set. "

{apache-airflow-providers-microsoft-azure-6.2.4rc1 → apache-airflow-providers-microsoft-azure-6.3.0rc1}/airflow/providers/microsoft/azure/operators/data_factory.py

@@ -92,7 +92,7 @@ class AzureDataFactoryRunPipelineOperator(BaseOperator):
         ``AzureDataFactoryHook`` will attempt to use the resource group name provided in the corresponding
         connection.
     :param factory_name: The data factory name. If a value is not passed in to the operator, the
-        ``AzureDataFactoryHook`` will attempt to use the factory name name provided in the corresponding
+        ``AzureDataFactoryHook`` will attempt to use the factory name provided in the corresponding
         connection.
     :param reference_pipeline_run_id: The pipeline run identifier. If this run ID is specified the parameters
         of the specified run will be used to create a new run.

{apache-airflow-providers-microsoft-azure-6.2.4rc1 → apache-airflow-providers-microsoft-azure-6.3.0rc1}/airflow/providers/microsoft/azure/secrets/key_vault.py

@@ -16,6 +16,8 @@
 # under the License.
 from __future__ import annotations
 
+import logging
+import os
 import re
 import warnings
 from functools import cached_property
@@ -95,6 +97,13 @@ class AzureKeyVaultBackend(BaseSecretsBackend, LoggingMixin):
             self.config_prefix = config_prefix.rstrip(sep)
         else:
             self.config_prefix = config_prefix
+
+        logger = logging.getLogger("azure.core.pipeline.policies.http_logging_policy")
+        try:
+            logger.setLevel(os.environ.get("AZURE_HTTP_LOGGING_LEVEL", logging.WARNING))
+        except ValueError:
+            logger.setLevel(logging.WARNING)
+
         self.sep = sep
         self.kwargs = kwargs
 

{apache-airflow-providers-microsoft-azure-6.2.4rc1 → apache-airflow-providers-microsoft-azure-6.3.0rc1}/airflow/providers/microsoft/azure/triggers/data_factory.py

@@ -233,7 +233,7 @@ class AzureDataFactoryTrigger(BaseTrigger):
                         resource_group_name=self.resource_group_name,
                         factory_name=self.factory_name,
                     )
-                    self.log.info("Unexpected error %s caught. Cancel pipeline run %s", str(e), self.run_id)
+                    self.log.info("Unexpected error %s caught. Cancel pipeline run %s", e, self.run_id)
                 except Exception as err:
                     yield TriggerEvent({"status": "error", "message": str(err), "run_id": self.run_id})
             yield TriggerEvent({"status": "error", "message": str(e), "run_id": self.run_id})

apache-airflow-providers-microsoft-azure-6.3.0rc1/airflow/providers/microsoft/azure/utils.py

@@ -0,0 +1,95 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import annotations
+
+import warnings
+
+from azure.core.pipeline import PipelineContext, PipelineRequest
+from azure.core.pipeline.policies import BearerTokenCredentialPolicy
+from azure.core.pipeline.transport import HttpRequest
+from azure.identity import DefaultAzureCredential
+from msrest.authentication import BasicTokenAuthentication
+
+
+def get_field(*, conn_id: str, conn_type: str, extras: dict, field_name: str):
+    """Get field from extra, first checking short name, then for backcompat we check for prefixed name."""
+    backcompat_prefix = f"extra__{conn_type}__"
+    backcompat_key = f"{backcompat_prefix}{field_name}"
+    ret = None
+    if field_name.startswith("extra__"):
+        raise ValueError(
+            f"Got prefixed name {field_name}; please remove the '{backcompat_prefix}' prefix "
+            "when using this method."
+        )
+    if field_name in extras:
+        if backcompat_key in extras:
+            warnings.warn(
+                f"Conflicting params `{field_name}` and `{backcompat_key}` found in extras for conn "
+                f"{conn_id}. Using value for `{field_name}`.  Please ensure this is the correct "
+                f"value and remove the backcompat key `{backcompat_key}`."
+            )
+        ret = extras[field_name]
+    elif backcompat_key in extras:
+        ret = extras.get(backcompat_key)
+    if ret == "":
+        return None
+    return ret
+
+
+class AzureIdentityCredentialAdapter(BasicTokenAuthentication):
+    """Adapt azure-identity credentials for backward compatibility.
+
+    Adapt credentials from azure-identity to be compatible with SD
+    that needs msrestazure or azure.common.credentials
+
+    Check https://stackoverflow.com/questions/63384092/exception-attributeerror-defaultazurecredential-object-has-no-attribute-sig
+    """
+
+    def __init__(self, credential=None, resource_id="https://management.azure.com/.default", **kwargs):
+        """Adapt azure-identity credentials for backward compatibility.
+
+        :param credential: Any azure-identity credential (DefaultAzureCredential by default)
+        :param str resource_id: The scope to use to get the token (default ARM)
+        """
+        super().__init__(None)
+        if credential is None:
+            credential = DefaultAzureCredential()
+        self._policy = BearerTokenCredentialPolicy(credential, resource_id, **kwargs)
+
+    def _make_request(self):
+        return PipelineRequest(
+            HttpRequest("AzureIdentityCredentialAdapter", "https://fakeurl"), PipelineContext(None)
+        )
+
+    def set_token(self):
+        """Ask the azure-core BearerTokenCredentialPolicy policy to get a token.
+
+        Using the policy gives us for free the caching system of azure-core.
+        We could make this code simpler by using private method, but by definition
+        I can't assure they will be there forever, so mocking a fake call to the policy
+        to extract the token, using 100% public API.
+        """
+        request = self._make_request()
+        self._policy.on_request(request)
+        # Read Authorization, and get the second part after Bearer
+        token = request.http_request.headers["Authorization"].split(" ", 1)[1]
+        self.token = {"access_token": token}
+
+    def signed_session(self, azure_session=None):
+        self.set_token()
+        return super().signed_session(azure_session)

{apache-airflow-providers-microsoft-azure-6.2.4rc1 → apache-airflow-providers-microsoft-azure-6.3.0rc1/apache_airflow_providers_microsoft_azure.egg-info}/PKG-INFO

@@ -1,14 +1,14 @@
 Metadata-Version: 2.1
 Name: apache-airflow-providers-microsoft-azure
-Version: 6.2.4rc1
+Version: 6.3.0rc1
 Summary: Provider for Apache Airflow. Implements apache-airflow-providers-microsoft-azure package
 Home-page: https://airflow.apache.org/
 Download-URL: https://archive.apache.org/dist/airflow/providers
 Author: Apache Software Foundation
 Author-email: dev@airflow.apache.org
 License: Apache License 2.0
-Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/6.2.4/
-Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/6.2.4/changelog.html
+Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/6.3.0/
+Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/6.3.0/changelog.html
 Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
 Project-URL: Source Code, https://github.com/apache/airflow
 Project-URL: Slack Chat, https://s.apache.org/airflow-slack
@@ -73,7 +73,7 @@ License-File: NOTICE
 
 Package ``apache-airflow-providers-microsoft-azure``
 
-Release: ``6.2.4rc1``
+Release: ``6.3.0rc1``
 
 
 `Microsoft Azure <https://azure.microsoft.com/>`__
@@ -86,7 +86,7 @@ This is a provider package for ``microsoft.azure`` provider. All classes for thi
 are in ``airflow.providers.microsoft.azure`` python package.
 
 You can find package information and changelog for the provider
-in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/6.2.4/>`_.
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/6.3.0/>`_.
 
 
 Installation
@@ -119,9 +119,9 @@ PIP package                       Version required
 ``azure-synapse-spark``
 ``adal``                          ``>=1.2.7``
 ``azure-storage-file-datalake``   ``>=12.9.1``
+``azure-kusto-data``              ``>=4.1.0``
 ``azure-mgmt-containerinstance``  ``>=1.5.0,<2.0``
 ``azure-mgmt-datafactory``        ``>=1.0.0,<2.0``
-``azure-kusto-data``              ``>=0.0.43,<0.1``
 ================================  ==================
 
 Cross provider package dependencies
@@ -146,4 +146,4 @@ Dependent package
 ====================================================================================================  ==========
 
 The changelog for the provider package can be found in the
-`changelog <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/6.2.4/changelog.html>`_.
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/6.3.0/changelog.html>`_.

{apache-airflow-providers-microsoft-azure-6.2.4rc1 → apache-airflow-providers-microsoft-azure-6.3.0rc1}/apache_airflow_providers_microsoft_azure.egg-info/SOURCES.txt

@@ -8,8 +8,6 @@ setup.py
 airflow/providers/microsoft/azure/__init__.py
 airflow/providers/microsoft/azure/get_provider_info.py
 airflow/providers/microsoft/azure/utils.py
-airflow/providers/microsoft/azure/example_dag/__init__.py
-airflow/providers/microsoft/azure/example_dag/example_wasb_sensors.py
 airflow/providers/microsoft/azure/hooks/__init__.py
 airflow/providers/microsoft/azure/hooks/adx.py
 airflow/providers/microsoft/azure/hooks/asb.py

{apache-airflow-providers-microsoft-azure-6.2.4rc1 → apache-airflow-providers-microsoft-azure-6.3.0rc1}/apache_airflow_providers_microsoft_azure.egg-info/requires.txt

@@ -5,7 +5,7 @@ azure-cosmos>=4.0.0
 azure-datalake-store>=0.0.45
 azure-identity>=1.3.1
 azure-keyvault-secrets>=4.1.0
-azure-kusto-data<0.1,>=0.0.43
+azure-kusto-data>=4.1.0
 azure-mgmt-containerinstance<2.0,>=1.5.0
 azure-mgmt-datafactory<2.0,>=1.0.0
 azure-mgmt-datalake-store>=0.5.0

{apache-airflow-providers-microsoft-azure-6.2.4rc1 → apache-airflow-providers-microsoft-azure-6.3.0rc1}/pyproject.toml

@@ -66,7 +66,6 @@ extend-select = [
 ]
 extend-ignore = [
     "D203",
-    "D205",
     "D212",
     "D213",
     "D214",
@@ -144,3 +143,29 @@ combine-as-imports = true
 "airflow/security/utils.py" = ["E402"]
 "tests/providers/elasticsearch/log/elasticmock/__init__.py" = ["E402"]
 "tests/providers/elasticsearch/log/elasticmock/utilities/__init__.py" = ["E402"]
+
+[tool.coverage.run]
+branch = true
+relative_files = true
+source = ["airflow"]
+omit = [
+    "airflow/_vendor/**",
+    "airflow/contrib/**",
+    "airflow/example_dags/**",
+    "airflow/migrations/**",
+    "airflow/providers/**/example_dags/**",
+    "airflow/www/node_modules/**",
+    "airflow/providers/google/ads/_vendor/**",
+]
+
+[tool.coverage.report]
+skip_empty = true
+exclude_also = [
+    "def __repr__",
+    "raise AssertionError",
+    "raise NotImplementedError",
+    "if __name__ == .__main__.:",
+    "@(abc\\.)?abstractmethod",
+    "@(typing(_extensions)?\\.)?overload",
+    "if TYPE_CHECKING:"
+]

{apache-airflow-providers-microsoft-azure-6.2.4rc1 → apache-airflow-providers-microsoft-azure-6.3.0rc1}/setup.cfg

@@ -27,8 +27,8 @@ classifiers =
 	Programming Language :: Python :: 3.11
 	Topic :: System :: Monitoring
 project_urls = 
-	Documentation=https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/6.2.4/
-	Changelog=https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/6.2.4/changelog.html
+	Documentation=https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/6.3.0/
+	Changelog=https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/6.3.0/changelog.html
 	Bug Tracker=https://github.com/apache/airflow/issues
 	Source Code=https://github.com/apache/airflow
 	Slack Chat=https://s.apache.org/airflow-slack
@@ -54,7 +54,7 @@ install_requires =
 	azure-datalake-store>=0.0.45
 	azure-identity>=1.3.1
 	azure-keyvault-secrets>=4.1.0
-	azure-kusto-data>=0.0.43,<0.1
+	azure-kusto-data>=4.1.0
 	azure-mgmt-containerinstance>=1.5.0,<2.0
 	azure-mgmt-datafactory>=1.0.0,<2.0
 	azure-mgmt-datalake-store>=0.5.0

{apache-airflow-providers-microsoft-azure-6.2.4rc1 → apache-airflow-providers-microsoft-azure-6.3.0rc1}/setup.py

@@ -26,7 +26,7 @@
 
 from setuptools import find_namespace_packages, setup
 
-version = "6.2.4"
+version = "6.3.0"
 
 
 def do_setup():

apache-airflow-providers-microsoft-azure-6.2.4rc1/airflow/providers/microsoft/azure/example_dag/example_wasb_sensors.py

@@ -1,60 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""
-Example Airflow DAG that senses blob(s) in Azure Blob Storage.
-
-This DAG relies on the following OS environment variables
-
-* CONTAINER_NAME - The container under which to look for the blob.
-* BLOB_NAME - The name of the blob to match.
-* PREFIX - The blob with the specified prefix to match.
-"""
-from __future__ import annotations
-
-import os
-from datetime import datetime
-
-from airflow.models import DAG
-from airflow.providers.microsoft.azure.sensors.wasb import WasbBlobSensor, WasbPrefixSensor
-
-CONTAINER_NAME = os.environ.get("CONTAINER_NAME", "example-container-name")
-BLOB_NAME = os.environ.get("BLOB_NAME", "example-blob-name")
-PREFIX = os.environ.get("PREFIX", "example-prefix")
-
-
-with DAG(
-    "example_wasb_sensors",
-    start_date=datetime(2022, 8, 8),
-    catchup=False,
-    tags=["example"],
-) as dag:
-    # [START wasb_blob_sensor]
-    azure_wasb_sensor = WasbBlobSensor(
-        container_name=CONTAINER_NAME,
-        blob_name=BLOB_NAME,
-        task_id="wasb_sense_blob",
-    )
-    # [END wasb_blob_sensor]
-
-    # [START wasb_prefix_sensor]
-    azure_wasb_prefix_sensor = WasbPrefixSensor(
-        container_name=CONTAINER_NAME,
-        prefix=PREFIX,
-        task_id="wasb_sense_prefix",
-    )
-    # [END wasb_prefix_sensor]

apache-airflow-providers-microsoft-azure-6.2.4rc1/airflow/providers/microsoft/azure/triggers/__init__.py

@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.

apache-airflow-providers-microsoft-azure-6.2.4rc1/airflow/providers/microsoft/azure/utils.py

@@ -1,45 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from __future__ import annotations
-
-import warnings
-
-
-def get_field(*, conn_id: str, conn_type: str, extras: dict, field_name: str):
-    """Get field from extra, first checking short name, then for backcompat we check for prefixed name."""
-    backcompat_prefix = f"extra__{conn_type}__"
-    backcompat_key = f"{backcompat_prefix}{field_name}"
-    ret = None
-    if field_name.startswith("extra__"):
-        raise ValueError(
-            f"Got prefixed name {field_name}; please remove the '{backcompat_prefix}' prefix "
-            "when using this method."
-        )
-    if field_name in extras:
-        if backcompat_key in extras:
-            warnings.warn(
-                f"Conflicting params `{field_name}` and `{backcompat_key}` found in extras for conn "
-                f"{conn_id}. Using value for `{field_name}`.  Please ensure this is the correct "
-                f"value and remove the backcompat key `{backcompat_key}`."
-            )
-        ret = extras[field_name]
-    elif backcompat_key in extras:
-        ret = extras.get(backcompat_key)
-    if ret == "":
-        return None
-    return ret