apache-airflow-providers-microsoft-azure 8.1.0rc1__py3-none-any.whl → 8.2.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/microsoft/azure/__init__.py +1 -1
- airflow/providers/microsoft/azure/get_provider_info.py +1 -0
- airflow/providers/microsoft/azure/hooks/adx.py +12 -2
- airflow/providers/microsoft/azure/hooks/asb.py +31 -6
- airflow/providers/microsoft/azure/hooks/base_azure.py +11 -2
- airflow/providers/microsoft/azure/hooks/batch.py +20 -10
- airflow/providers/microsoft/azure/hooks/container_instance.py +7 -1
- airflow/providers/microsoft/azure/hooks/container_registry.py +14 -3
- airflow/providers/microsoft/azure/hooks/container_volume.py +25 -17
- airflow/providers/microsoft/azure/hooks/cosmos.py +15 -4
- airflow/providers/microsoft/azure/hooks/data_factory.py +18 -2
- airflow/providers/microsoft/azure/hooks/data_lake.py +30 -14
- airflow/providers/microsoft/azure/hooks/fileshare.py +37 -21
- airflow/providers/microsoft/azure/hooks/synapse.py +17 -3
- airflow/providers/microsoft/azure/hooks/wasb.py +22 -4
- airflow/providers/microsoft/azure/operators/container_instances.py +5 -6
- airflow/providers/microsoft/azure/secrets/key_vault.py +22 -3
- airflow/providers/microsoft/azure/utils.py +80 -4
- {apache_airflow_providers_microsoft_azure-8.1.0rc1.dist-info → apache_airflow_providers_microsoft_azure-8.2.0rc1.dist-info}/METADATA +7 -9
- {apache_airflow_providers_microsoft_azure-8.1.0rc1.dist-info → apache_airflow_providers_microsoft_azure-8.2.0rc1.dist-info}/RECORD +25 -25
- {apache_airflow_providers_microsoft_azure-8.1.0rc1.dist-info → apache_airflow_providers_microsoft_azure-8.2.0rc1.dist-info}/WHEEL +1 -1
- {apache_airflow_providers_microsoft_azure-8.1.0rc1.dist-info → apache_airflow_providers_microsoft_azure-8.2.0rc1.dist-info}/LICENSE +0 -0
- {apache_airflow_providers_microsoft_azure-8.1.0rc1.dist-info → apache_airflow_providers_microsoft_azure-8.2.0rc1.dist-info}/NOTICE +0 -0
- {apache_airflow_providers_microsoft_azure-8.1.0rc1.dist-info → apache_airflow_providers_microsoft_azure-8.2.0rc1.dist-info}/entry_points.txt +0 -0
- {apache_airflow_providers_microsoft_azure-8.1.0rc1.dist-info → apache_airflow_providers_microsoft_azure-8.2.0rc1.dist-info}/top_level.txt +0 -0
@@ -29,12 +29,15 @@ import warnings
|
|
29
29
|
from functools import cached_property
|
30
30
|
from typing import TYPE_CHECKING, Any
|
31
31
|
|
32
|
-
from azure.identity import DefaultAzureCredential
|
33
32
|
from azure.kusto.data import ClientRequestProperties, KustoClient, KustoConnectionStringBuilder
|
34
33
|
from azure.kusto.data.exceptions import KustoServiceError
|
35
34
|
|
36
35
|
from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
|
37
36
|
from airflow.hooks.base import BaseHook
|
37
|
+
from airflow.providers.microsoft.azure.utils import (
|
38
|
+
add_managed_identity_connection_widgets,
|
39
|
+
get_sync_default_azure_credential,
|
40
|
+
)
|
38
41
|
|
39
42
|
if TYPE_CHECKING:
|
40
43
|
from azure.kusto.data.response import KustoResponseDataSetV2
|
@@ -80,6 +83,7 @@ class AzureDataExplorerHook(BaseHook):
|
|
80
83
|
hook_name = "Azure Data Explorer"
|
81
84
|
|
82
85
|
@classmethod
|
86
|
+
@add_managed_identity_connection_widgets
|
83
87
|
def get_connection_form_widgets(cls) -> dict[str, Any]:
|
84
88
|
"""Returns connection widgets to add to connection form."""
|
85
89
|
from flask_appbuilder.fieldwidgets import BS3PasswordFieldWidget, BS3TextFieldWidget
|
@@ -192,9 +196,15 @@ class AzureDataExplorerHook(BaseHook):
|
|
192
196
|
elif auth_method == "AAD_DEVICE":
|
193
197
|
kcsb = KustoConnectionStringBuilder.with_aad_device_authentication(cluster)
|
194
198
|
elif auth_method == "AZURE_TOKEN_CRED":
|
199
|
+
managed_identity_client_id = conn.extra_dejson.get("managed_identity_client_id")
|
200
|
+
workload_identity_tenant_id = conn.extra_dejson.get("workload_identity_tenant_id")
|
201
|
+
credential = get_sync_default_azure_credential(
|
202
|
+
managed_identity_client_id=managed_identity_client_id,
|
203
|
+
workload_identity_tenant_id=workload_identity_tenant_id,
|
204
|
+
)
|
195
205
|
kcsb = KustoConnectionStringBuilder.with_azure_token_credential(
|
196
206
|
connection_string=cluster,
|
197
|
-
credential=
|
207
|
+
credential=credential,
|
198
208
|
)
|
199
209
|
else:
|
200
210
|
raise AirflowException(f"Unknown authentication method: {auth_method}")
|
@@ -16,14 +16,20 @@
|
|
16
16
|
# under the License.
|
17
17
|
from __future__ import annotations
|
18
18
|
|
19
|
-
from typing import Any
|
19
|
+
from typing import TYPE_CHECKING, Any
|
20
20
|
|
21
|
-
from azure.identity import DefaultAzureCredential
|
22
21
|
from azure.servicebus import ServiceBusClient, ServiceBusMessage, ServiceBusSender
|
23
22
|
from azure.servicebus.management import QueueProperties, ServiceBusAdministrationClient
|
24
23
|
|
25
24
|
from airflow.hooks.base import BaseHook
|
26
|
-
from airflow.providers.microsoft.azure.utils import
|
25
|
+
from airflow.providers.microsoft.azure.utils import (
|
26
|
+
add_managed_identity_connection_widgets,
|
27
|
+
get_field,
|
28
|
+
get_sync_default_azure_credential,
|
29
|
+
)
|
30
|
+
|
31
|
+
if TYPE_CHECKING:
|
32
|
+
from azure.identity import DefaultAzureCredential
|
27
33
|
|
28
34
|
|
29
35
|
class BaseAzureServiceBusHook(BaseHook):
|
@@ -40,6 +46,7 @@ class BaseAzureServiceBusHook(BaseHook):
|
|
40
46
|
hook_name = "Azure Service Bus"
|
41
47
|
|
42
48
|
@staticmethod
|
49
|
+
@add_managed_identity_connection_widgets
|
43
50
|
def get_connection_form_widgets() -> dict[str, Any]:
|
44
51
|
"""Returns connection widgets to add to connection form."""
|
45
52
|
from flask_appbuilder.fieldwidgets import BS3TextFieldWidget
|
@@ -64,7 +71,7 @@ class BaseAzureServiceBusHook(BaseHook):
|
|
64
71
|
"<Resource group>.servicebus.windows.net (for Azure AD authenticaltion)"
|
65
72
|
),
|
66
73
|
"credential": "credential",
|
67
|
-
"schema": "Endpoint=sb://<Resource group>.servicebus.windows.net/;SharedAccessKeyName=<AccessKeyName>;SharedAccessKey=<SharedAccessKey>",
|
74
|
+
"schema": "Endpoint=sb://<Resource group>.servicebus.windows.net/;SharedAccessKeyName=<AccessKeyName>;SharedAccessKey=<SharedAccessKey>",
|
68
75
|
},
|
69
76
|
}
|
70
77
|
|
@@ -106,7 +113,16 @@ class AdminClientHook(BaseAzureServiceBusHook):
|
|
106
113
|
credential: str | DefaultAzureCredential = self._get_field(extras=extras, field_name="credential")
|
107
114
|
fully_qualified_namespace = self._get_field(extras=extras, field_name="fully_qualified_namespace")
|
108
115
|
if not credential:
|
109
|
-
|
116
|
+
managed_identity_client_id = self._get_field(
|
117
|
+
extras=extras, field_name="managed_identity_client_id"
|
118
|
+
)
|
119
|
+
workload_identity_tenant_id = self._get_field(
|
120
|
+
extras=extras, field_name="workload_identity_tenant_id"
|
121
|
+
)
|
122
|
+
credential = get_sync_default_azure_credential(
|
123
|
+
managed_identity_client_id=managed_identity_client_id,
|
124
|
+
workload_identity_tenant_id=workload_identity_tenant_id,
|
125
|
+
)
|
110
126
|
client = ServiceBusAdministrationClient(
|
111
127
|
fully_qualified_namespace=fully_qualified_namespace,
|
112
128
|
credential=credential, # type: ignore[arg-type]
|
@@ -190,7 +206,16 @@ class MessageHook(BaseAzureServiceBusHook):
|
|
190
206
|
credential: str | DefaultAzureCredential = self._get_field(extras=extras, field_name="credential")
|
191
207
|
fully_qualified_namespace = self._get_field(extras=extras, field_name="fully_qualified_namespace")
|
192
208
|
if not credential:
|
193
|
-
|
209
|
+
managed_identity_client_id = self._get_field(
|
210
|
+
extras=extras, field_name="managed_identity_client_id"
|
211
|
+
)
|
212
|
+
workload_identity_tenant_id = self._get_field(
|
213
|
+
extras=extras, field_name="workload_identity_tenant_id"
|
214
|
+
)
|
215
|
+
credential = get_sync_default_azure_credential(
|
216
|
+
managed_identity_client_id=managed_identity_client_id,
|
217
|
+
workload_identity_tenant_id=workload_identity_tenant_id,
|
218
|
+
)
|
194
219
|
client = ServiceBusClient(
|
195
220
|
fully_qualified_namespace=fully_qualified_namespace,
|
196
221
|
credential=credential, # type: ignore[arg-type]
|
@@ -24,7 +24,10 @@ from azure.common.credentials import ServicePrincipalCredentials
|
|
24
24
|
|
25
25
|
from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
|
26
26
|
from airflow.hooks.base import BaseHook
|
27
|
-
from airflow.providers.microsoft.azure.utils import
|
27
|
+
from airflow.providers.microsoft.azure.utils import (
|
28
|
+
AzureIdentityCredentialAdapter,
|
29
|
+
add_managed_identity_connection_widgets,
|
30
|
+
)
|
28
31
|
|
29
32
|
|
30
33
|
class AzureBaseHook(BaseHook):
|
@@ -45,6 +48,7 @@ class AzureBaseHook(BaseHook):
|
|
45
48
|
hook_name = "Azure"
|
46
49
|
|
47
50
|
@staticmethod
|
51
|
+
@add_managed_identity_connection_widgets
|
48
52
|
def get_connection_form_widgets() -> dict[str, Any]:
|
49
53
|
"""Returns connection widgets to add to connection form."""
|
50
54
|
from flask_appbuilder.fieldwidgets import BS3TextFieldWidget
|
@@ -133,7 +137,12 @@ class AzureBaseHook(BaseHook):
|
|
133
137
|
)
|
134
138
|
else:
|
135
139
|
self.log.info("Using DefaultAzureCredential as credential")
|
136
|
-
|
140
|
+
managed_identity_client_id = conn.extra_dejson.get("managed_identity_client_id")
|
141
|
+
workload_identity_tenant_id = conn.extra_dejson.get("workload_identity_tenant_id")
|
142
|
+
credentials = AzureIdentityCredentialAdapter(
|
143
|
+
managed_identity_client_id=managed_identity_client_id,
|
144
|
+
workload_identity_tenant_id=workload_identity_tenant_id,
|
145
|
+
)
|
137
146
|
|
138
147
|
return self.sdk_client(
|
139
148
|
credentials=credentials,
|
@@ -26,7 +26,11 @@ from azure.batch import BatchServiceClient, batch_auth, models as batch_models
|
|
26
26
|
|
27
27
|
from airflow.exceptions import AirflowException
|
28
28
|
from airflow.hooks.base import BaseHook
|
29
|
-
from airflow.providers.microsoft.azure.utils import
|
29
|
+
from airflow.providers.microsoft.azure.utils import (
|
30
|
+
AzureIdentityCredentialAdapter,
|
31
|
+
add_managed_identity_connection_widgets,
|
32
|
+
get_field,
|
33
|
+
)
|
30
34
|
from airflow.utils import timezone
|
31
35
|
|
32
36
|
if TYPE_CHECKING:
|
@@ -46,15 +50,8 @@ class AzureBatchHook(BaseHook):
|
|
46
50
|
conn_type = "azure_batch"
|
47
51
|
hook_name = "Azure Batch Service"
|
48
52
|
|
49
|
-
def _get_field(self, extras, name):
|
50
|
-
return get_field(
|
51
|
-
conn_id=self.conn_id,
|
52
|
-
conn_type=self.conn_type,
|
53
|
-
extras=extras,
|
54
|
-
field_name=name,
|
55
|
-
)
|
56
|
-
|
57
53
|
@classmethod
|
54
|
+
@add_managed_identity_connection_widgets
|
58
55
|
def get_connection_form_widgets(cls) -> dict[str, Any]:
|
59
56
|
"""Returns connection widgets to add to connection form."""
|
60
57
|
from flask_appbuilder.fieldwidgets import BS3TextFieldWidget
|
@@ -80,6 +77,14 @@ class AzureBatchHook(BaseHook):
|
|
80
77
|
super().__init__()
|
81
78
|
self.conn_id = azure_batch_conn_id
|
82
79
|
|
80
|
+
def _get_field(self, extras, name):
|
81
|
+
return get_field(
|
82
|
+
conn_id=self.conn_id,
|
83
|
+
conn_type=self.conn_type,
|
84
|
+
extras=extras,
|
85
|
+
field_name=name,
|
86
|
+
)
|
87
|
+
|
83
88
|
@cached_property
|
84
89
|
def connection(self) -> BatchServiceClient:
|
85
90
|
"""Get the Batch client connection (cached)."""
|
@@ -101,8 +106,13 @@ class AzureBatchHook(BaseHook):
|
|
101
106
|
if all([conn.login, conn.password]):
|
102
107
|
credentials = batch_auth.SharedKeyCredentials(conn.login, conn.password)
|
103
108
|
else:
|
109
|
+
managed_identity_client_id = conn.extra_dejson.get("managed_identity_client_id")
|
110
|
+
workload_identity_tenant_id = conn.extra_dejson.get("workload_identity_tenant_id")
|
104
111
|
credentials = AzureIdentityCredentialAdapter(
|
105
|
-
None,
|
112
|
+
None,
|
113
|
+
resource_id="https://batch.core.windows.net/.default",
|
114
|
+
managed_identity_client_id=managed_identity_client_id,
|
115
|
+
workload_identity_tenant_id=workload_identity_tenant_id,
|
106
116
|
)
|
107
117
|
|
108
118
|
batch_client = BatchServiceClient(credentials, batch_url=batch_account_url)
|
@@ -27,6 +27,7 @@ from azure.mgmt.containerinstance import ContainerInstanceManagementClient
|
|
27
27
|
|
28
28
|
from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
|
29
29
|
from airflow.providers.microsoft.azure.hooks.base_azure import AzureBaseHook
|
30
|
+
from airflow.providers.microsoft.azure.utils import get_sync_default_azure_credential
|
30
31
|
|
31
32
|
if TYPE_CHECKING:
|
32
33
|
from azure.mgmt.containerinstance.models import (
|
@@ -92,7 +93,12 @@ class AzureContainerInstanceHook(AzureBaseHook):
|
|
92
93
|
)
|
93
94
|
else:
|
94
95
|
self.log.info("Using DefaultAzureCredential as credential")
|
95
|
-
|
96
|
+
managed_identity_client_id = conn.extra_dejson.get("managed_identity_client_id")
|
97
|
+
workload_identity_tenant_id = conn.extra_dejson.get("workload_identity_tenant_id")
|
98
|
+
credential = get_sync_default_azure_credential(
|
99
|
+
managed_identity_client_id=managed_identity_client_id,
|
100
|
+
workload_identity_tenant_id=workload_identity_tenant_id,
|
101
|
+
)
|
96
102
|
|
97
103
|
subscription_id = cast(str, conn.extra_dejson.get("subscriptionId"))
|
98
104
|
return ContainerInstanceManagementClient(
|
@@ -21,12 +21,15 @@ from __future__ import annotations
|
|
21
21
|
from functools import cached_property
|
22
22
|
from typing import Any
|
23
23
|
|
24
|
-
from azure.identity import DefaultAzureCredential
|
25
24
|
from azure.mgmt.containerinstance.models import ImageRegistryCredential
|
26
25
|
from azure.mgmt.containerregistry import ContainerRegistryManagementClient
|
27
26
|
|
28
27
|
from airflow.hooks.base import BaseHook
|
29
|
-
from airflow.providers.microsoft.azure.utils import
|
28
|
+
from airflow.providers.microsoft.azure.utils import (
|
29
|
+
add_managed_identity_connection_widgets,
|
30
|
+
get_field,
|
31
|
+
get_sync_default_azure_credential,
|
32
|
+
)
|
30
33
|
|
31
34
|
|
32
35
|
class AzureContainerRegistryHook(BaseHook):
|
@@ -44,6 +47,7 @@ class AzureContainerRegistryHook(BaseHook):
|
|
44
47
|
hook_name = "Azure Container Registry"
|
45
48
|
|
46
49
|
@staticmethod
|
50
|
+
@add_managed_identity_connection_widgets
|
47
51
|
def get_connection_form_widgets() -> dict[str, Any]:
|
48
52
|
"""Returns connection widgets to add to connection form."""
|
49
53
|
from flask_appbuilder.fieldwidgets import BS3TextFieldWidget
|
@@ -103,8 +107,15 @@ class AzureContainerRegistryHook(BaseHook):
|
|
103
107
|
extras = conn.extra_dejson
|
104
108
|
subscription_id = self._get_field(extras, "subscription_id")
|
105
109
|
resource_group = self._get_field(extras, "resource_group")
|
110
|
+
managed_identity_client_id = self._get_field(extras, "managed_identity_client_id")
|
111
|
+
workload_identity_tenant_id = self._get_field(extras, "workload_identity_tenant_id")
|
112
|
+
credential = get_sync_default_azure_credential(
|
113
|
+
managed_identity_client_id=managed_identity_client_id,
|
114
|
+
workload_identity_tenant_id=workload_identity_tenant_id,
|
115
|
+
)
|
106
116
|
client = ContainerRegistryManagementClient(
|
107
|
-
credential=
|
117
|
+
credential=credential,
|
118
|
+
subscription_id=subscription_id,
|
108
119
|
)
|
109
120
|
credentials = client.registries.list_credentials(resource_group, conn.login).as_dict()
|
110
121
|
password = credentials["passwords"][0]["value"]
|
@@ -1,4 +1,3 @@
|
|
1
|
-
#
|
2
1
|
# Licensed to the Apache Software Foundation (ASF) under one
|
3
2
|
# or more contributor license agreements. See the NOTICE file
|
4
3
|
# distributed with this work for additional information
|
@@ -19,12 +18,15 @@ from __future__ import annotations
|
|
19
18
|
|
20
19
|
from typing import Any
|
21
20
|
|
22
|
-
from azure.identity import DefaultAzureCredential
|
23
21
|
from azure.mgmt.containerinstance.models import AzureFileVolume, Volume
|
24
22
|
from azure.mgmt.storage import StorageManagementClient
|
25
23
|
|
26
24
|
from airflow.hooks.base import BaseHook
|
27
|
-
from airflow.providers.microsoft.azure.utils import
|
25
|
+
from airflow.providers.microsoft.azure.utils import (
|
26
|
+
add_managed_identity_connection_widgets,
|
27
|
+
get_field,
|
28
|
+
get_sync_default_azure_credential,
|
29
|
+
)
|
28
30
|
|
29
31
|
|
30
32
|
class AzureContainerVolumeHook(BaseHook):
|
@@ -41,19 +43,8 @@ class AzureContainerVolumeHook(BaseHook):
|
|
41
43
|
conn_type = "azure_container_volume"
|
42
44
|
hook_name = "Azure Container Volume"
|
43
45
|
|
44
|
-
def __init__(self, azure_container_volume_conn_id: str = "azure_container_volume_default") -> None:
|
45
|
-
super().__init__()
|
46
|
-
self.conn_id = azure_container_volume_conn_id
|
47
|
-
|
48
|
-
def _get_field(self, extras, name):
|
49
|
-
return get_field(
|
50
|
-
conn_id=self.conn_id,
|
51
|
-
conn_type=self.conn_type,
|
52
|
-
extras=extras,
|
53
|
-
field_name=name,
|
54
|
-
)
|
55
|
-
|
56
46
|
@staticmethod
|
47
|
+
@add_managed_identity_connection_widgets
|
57
48
|
def get_connection_form_widgets() -> dict[str, Any]:
|
58
49
|
"""Returns connection widgets to add to connection form."""
|
59
50
|
from flask_appbuilder.fieldwidgets import BS3PasswordFieldWidget, BS3TextFieldWidget
|
@@ -92,6 +83,18 @@ class AzureContainerVolumeHook(BaseHook):
|
|
92
83
|
},
|
93
84
|
}
|
94
85
|
|
86
|
+
def __init__(self, azure_container_volume_conn_id: str = "azure_container_volume_default") -> None:
|
87
|
+
super().__init__()
|
88
|
+
self.conn_id = azure_container_volume_conn_id
|
89
|
+
|
90
|
+
def _get_field(self, extras, name):
|
91
|
+
return get_field(
|
92
|
+
conn_id=self.conn_id,
|
93
|
+
conn_type=self.conn_type,
|
94
|
+
extras=extras,
|
95
|
+
field_name=name,
|
96
|
+
)
|
97
|
+
|
95
98
|
def get_storagekey(self, *, storage_account_name: str | None = None) -> str:
|
96
99
|
"""Get Azure File Volume storage key."""
|
97
100
|
conn = self.get_connection(self.conn_id)
|
@@ -106,8 +109,13 @@ class AzureContainerVolumeHook(BaseHook):
|
|
106
109
|
subscription_id = self._get_field(extras, "subscription_id")
|
107
110
|
resource_group = self._get_field(extras, "resource_group")
|
108
111
|
if subscription_id and storage_account_name and resource_group:
|
109
|
-
|
110
|
-
|
112
|
+
managed_identity_client_id = self._get_field(extras, "managed_identity_client_id")
|
113
|
+
workload_identity_tenant_id = self._get_field(extras, "workload_identity_tenant_id")
|
114
|
+
credential = get_sync_default_azure_credential(
|
115
|
+
managed_identity_client_id=managed_identity_client_id,
|
116
|
+
workload_identity_tenant_id=workload_identity_tenant_id,
|
117
|
+
)
|
118
|
+
storage_client = StorageManagementClient(credential, subscription_id)
|
111
119
|
storage_account_list_keys_result = storage_client.storage_accounts.list_keys(
|
112
120
|
resource_group, storage_account_name
|
113
121
|
)
|
@@ -31,12 +31,15 @@ from urllib.parse import urlparse
|
|
31
31
|
|
32
32
|
from azure.cosmos.cosmos_client import CosmosClient
|
33
33
|
from azure.cosmos.exceptions import CosmosHttpResponseError
|
34
|
-
from azure.identity import DefaultAzureCredential
|
35
34
|
from azure.mgmt.cosmosdb import CosmosDBManagementClient
|
36
35
|
|
37
36
|
from airflow.exceptions import AirflowBadRequest, AirflowException
|
38
37
|
from airflow.hooks.base import BaseHook
|
39
|
-
from airflow.providers.microsoft.azure.utils import
|
38
|
+
from airflow.providers.microsoft.azure.utils import (
|
39
|
+
add_managed_identity_connection_widgets,
|
40
|
+
get_field,
|
41
|
+
get_sync_default_azure_credential,
|
42
|
+
)
|
40
43
|
|
41
44
|
|
42
45
|
class AzureCosmosDBHook(BaseHook):
|
@@ -57,6 +60,7 @@ class AzureCosmosDBHook(BaseHook):
|
|
57
60
|
hook_name = "Azure CosmosDB"
|
58
61
|
|
59
62
|
@staticmethod
|
63
|
+
@add_managed_identity_connection_widgets
|
60
64
|
def get_connection_form_widgets() -> dict[str, Any]:
|
61
65
|
"""Returns connection widgets to add to connection form."""
|
62
66
|
from flask_appbuilder.fieldwidgets import BS3TextFieldWidget
|
@@ -126,9 +130,16 @@ class AzureCosmosDBHook(BaseHook):
|
|
126
130
|
if conn.password:
|
127
131
|
master_key = conn.password
|
128
132
|
elif resource_group_name:
|
133
|
+
managed_identity_client_id = self._get_field(extras, "managed_identity_client_id")
|
134
|
+
workload_identity_tenant_id = self._get_field(extras, "workload_identity_tenant_id")
|
135
|
+
subscritption_id = self._get_field(extras, "subscription_id")
|
136
|
+
credential = get_sync_default_azure_credential(
|
137
|
+
managed_identity_client_id=managed_identity_client_id,
|
138
|
+
workload_identity_tenant_id=workload_identity_tenant_id,
|
139
|
+
)
|
129
140
|
management_client = CosmosDBManagementClient(
|
130
|
-
credential=
|
131
|
-
subscription_id=
|
141
|
+
credential=credential,
|
142
|
+
subscription_id=subscritption_id,
|
132
143
|
)
|
133
144
|
|
134
145
|
database_account = urlparse(conn.login).netloc.split(".")[0]
|
@@ -48,6 +48,11 @@ from azure.mgmt.datafactory.aio import DataFactoryManagementClient as AsyncDataF
|
|
48
48
|
|
49
49
|
from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
|
50
50
|
from airflow.hooks.base import BaseHook
|
51
|
+
from airflow.providers.microsoft.azure.utils import (
|
52
|
+
add_managed_identity_connection_widgets,
|
53
|
+
get_async_default_azure_credential,
|
54
|
+
get_sync_default_azure_credential,
|
55
|
+
)
|
51
56
|
|
52
57
|
if TYPE_CHECKING:
|
53
58
|
from azure.core.polling import LROPoller
|
@@ -152,6 +157,7 @@ class AzureDataFactoryHook(BaseHook):
|
|
152
157
|
hook_name: str = "Azure Data Factory"
|
153
158
|
|
154
159
|
@staticmethod
|
160
|
+
@add_managed_identity_connection_widgets
|
155
161
|
def get_connection_form_widgets() -> dict[str, Any]:
|
156
162
|
"""Returns connection widgets to add to connection form."""
|
157
163
|
from flask_appbuilder.fieldwidgets import BS3TextFieldWidget
|
@@ -205,7 +211,12 @@ class AzureDataFactoryHook(BaseHook):
|
|
205
211
|
client_id=conn.login, client_secret=conn.password, tenant_id=tenant
|
206
212
|
)
|
207
213
|
else:
|
208
|
-
|
214
|
+
managed_identity_client_id = get_field(extras, "managed_identity_client_id")
|
215
|
+
workload_identity_tenant_id = get_field(extras, "workload_identity_tenant_id")
|
216
|
+
credential = get_sync_default_azure_credential(
|
217
|
+
managed_identity_client_id=managed_identity_client_id,
|
218
|
+
workload_identity_tenant_id=workload_identity_tenant_id,
|
219
|
+
)
|
209
220
|
self._conn = self._create_client(credential, subscription_id)
|
210
221
|
|
211
222
|
return self._conn
|
@@ -1140,7 +1151,12 @@ class AzureDataFactoryAsyncHook(AzureDataFactoryHook):
|
|
1140
1151
|
client_id=conn.login, client_secret=conn.password, tenant_id=tenant
|
1141
1152
|
)
|
1142
1153
|
else:
|
1143
|
-
|
1154
|
+
managed_identity_client_id = get_field(extras, "managed_identity_client_id")
|
1155
|
+
workload_identity_tenant_id = get_field(extras, "workload_identity_tenant_id")
|
1156
|
+
credential = get_async_default_azure_credential(
|
1157
|
+
managed_identity_client_id=managed_identity_client_id,
|
1158
|
+
workload_identity_tenant_id=workload_identity_tenant_id,
|
1159
|
+
)
|
1144
1160
|
|
1145
1161
|
self._async_conn = AsyncDataFactoryManagementClient(
|
1146
1162
|
credential=credential,
|
@@ -34,7 +34,11 @@ from azure.storage.filedatalake import (
|
|
34
34
|
|
35
35
|
from airflow.exceptions import AirflowException
|
36
36
|
from airflow.hooks.base import BaseHook
|
37
|
-
from airflow.providers.microsoft.azure.utils import
|
37
|
+
from airflow.providers.microsoft.azure.utils import (
|
38
|
+
AzureIdentityCredentialAdapter,
|
39
|
+
add_managed_identity_connection_widgets,
|
40
|
+
get_field,
|
41
|
+
)
|
38
42
|
|
39
43
|
Credentials = Union[ClientSecretCredential, AzureIdentityCredentialAdapter]
|
40
44
|
|
@@ -62,6 +66,7 @@ class AzureDataLakeHook(BaseHook):
|
|
62
66
|
hook_name = "Azure Data Lake"
|
63
67
|
|
64
68
|
@staticmethod
|
69
|
+
@add_managed_identity_connection_widgets
|
65
70
|
def get_connection_form_widgets() -> dict[str, Any]:
|
66
71
|
"""Returns connection widgets to add to connection form."""
|
67
72
|
from flask_appbuilder.fieldwidgets import BS3TextFieldWidget
|
@@ -118,7 +123,12 @@ class AzureDataLakeHook(BaseHook):
|
|
118
123
|
if tenant:
|
119
124
|
credential = lib.auth(tenant_id=tenant, client_secret=conn.password, client_id=conn.login)
|
120
125
|
else:
|
121
|
-
|
126
|
+
managed_identity_client_id = self._get_field(extras, "managed_identity_client_id")
|
127
|
+
workload_identity_tenant_id = self._get_field(extras, "workload_identity_tenant_id")
|
128
|
+
credential = AzureIdentityCredentialAdapter(
|
129
|
+
managed_identity_client_id=managed_identity_client_id,
|
130
|
+
workload_identity_tenant_id=workload_identity_tenant_id,
|
131
|
+
)
|
122
132
|
self._conn = core.AzureDLFileSystem(credential, store_name=self.account_name)
|
123
133
|
self._conn.connect()
|
124
134
|
return self._conn
|
@@ -265,6 +275,7 @@ class AzureDataLakeStorageV2Hook(BaseHook):
|
|
265
275
|
hook_name = "Azure Date Lake Storage V2"
|
266
276
|
|
267
277
|
@classmethod
|
278
|
+
@add_managed_identity_connection_widgets
|
268
279
|
def get_connection_form_widgets(cls) -> dict[str, Any]:
|
269
280
|
"""Returns connection widgets to add to connection form."""
|
270
281
|
from flask_appbuilder.fieldwidgets import BS3PasswordFieldWidget, BS3TextFieldWidget
|
@@ -305,6 +316,17 @@ class AzureDataLakeStorageV2Hook(BaseHook):
|
|
305
316
|
self.conn_id = adls_conn_id
|
306
317
|
self.public_read = public_read
|
307
318
|
|
319
|
+
def _get_field(self, extra_dict, field_name):
|
320
|
+
prefix = "extra__adls__"
|
321
|
+
if field_name.startswith("extra__"):
|
322
|
+
raise ValueError(
|
323
|
+
f"Got prefixed name {field_name}; please remove the '{prefix}' prefix "
|
324
|
+
f"when using this method."
|
325
|
+
)
|
326
|
+
if field_name in extra_dict:
|
327
|
+
return extra_dict[field_name] or None
|
328
|
+
return extra_dict.get(f"{prefix}{field_name}") or None
|
329
|
+
|
308
330
|
@cached_property
|
309
331
|
def service_client(self) -> DataLakeServiceClient:
|
310
332
|
"""Return the DataLakeServiceClient object (cached)."""
|
@@ -330,7 +352,12 @@ class AzureDataLakeStorageV2Hook(BaseHook):
|
|
330
352
|
elif conn.password:
|
331
353
|
credential = conn.password
|
332
354
|
else:
|
333
|
-
|
355
|
+
managed_identity_client_id = self._get_field(extra, "managed_identity_client_id")
|
356
|
+
workload_identity_tenant_id = self._get_field(extra, "workload_identity_tenant_id")
|
357
|
+
credential = AzureIdentityCredentialAdapter(
|
358
|
+
managed_identity_client_id=managed_identity_client_id,
|
359
|
+
workload_identity_tenant_id=workload_identity_tenant_id,
|
360
|
+
)
|
334
361
|
|
335
362
|
return DataLakeServiceClient(
|
336
363
|
account_url=f"https://{conn.host}.dfs.core.windows.net",
|
@@ -338,17 +365,6 @@ class AzureDataLakeStorageV2Hook(BaseHook):
|
|
338
365
|
**extra,
|
339
366
|
)
|
340
367
|
|
341
|
-
def _get_field(self, extra_dict, field_name):
|
342
|
-
prefix = "extra__adls__"
|
343
|
-
if field_name.startswith("extra__"):
|
344
|
-
raise ValueError(
|
345
|
-
f"Got prefixed name {field_name}; please remove the '{prefix}' prefix "
|
346
|
-
f"when using this method."
|
347
|
-
)
|
348
|
-
if field_name in extra_dict:
|
349
|
-
return extra_dict[field_name] or None
|
350
|
-
return extra_dict.get(f"{prefix}{field_name}") or None
|
351
|
-
|
352
368
|
def create_file_system(self, file_system_name: str) -> None:
|
353
369
|
"""Create a new file system under the specified account.
|
354
370
|
|
@@ -19,10 +19,13 @@ from __future__ import annotations
|
|
19
19
|
|
20
20
|
from typing import IO, Any
|
21
21
|
|
22
|
-
from azure.identity import DefaultAzureCredential
|
23
22
|
from azure.storage.fileshare import FileProperties, ShareDirectoryClient, ShareFileClient, ShareServiceClient
|
24
23
|
|
25
24
|
from airflow.hooks.base import BaseHook
|
25
|
+
from airflow.providers.microsoft.azure.utils import (
|
26
|
+
add_managed_identity_connection_widgets,
|
27
|
+
get_sync_default_azure_credential,
|
28
|
+
)
|
26
29
|
|
27
30
|
|
28
31
|
class AzureFileShareHook(BaseHook):
|
@@ -39,24 +42,8 @@ class AzureFileShareHook(BaseHook):
|
|
39
42
|
conn_type = "azure_fileshare"
|
40
43
|
hook_name = "Azure FileShare"
|
41
44
|
|
42
|
-
def __init__(
|
43
|
-
self,
|
44
|
-
share_name: str | None = None,
|
45
|
-
file_path: str | None = None,
|
46
|
-
directory_path: str | None = None,
|
47
|
-
azure_fileshare_conn_id: str = "azure_fileshare_default",
|
48
|
-
) -> None:
|
49
|
-
super().__init__()
|
50
|
-
self._conn_id = azure_fileshare_conn_id
|
51
|
-
self.share_name = share_name
|
52
|
-
self.file_path = file_path
|
53
|
-
self.directory_path = directory_path
|
54
|
-
self._account_url: str | None = None
|
55
|
-
self._connection_string: str | None = None
|
56
|
-
self._account_access_key: str | None = None
|
57
|
-
self._sas_token: str | None = None
|
58
|
-
|
59
45
|
@staticmethod
|
46
|
+
@add_managed_identity_connection_widgets
|
60
47
|
def get_connection_form_widgets() -> dict[str, Any]:
|
61
48
|
"""Returns connection widgets to add to connection form."""
|
62
49
|
from flask_appbuilder.fieldwidgets import BS3PasswordFieldWidget, BS3TextFieldWidget
|
@@ -87,6 +74,23 @@ class AzureFileShareHook(BaseHook):
|
|
87
74
|
},
|
88
75
|
}
|
89
76
|
|
77
|
+
def __init__(
|
78
|
+
self,
|
79
|
+
share_name: str | None = None,
|
80
|
+
file_path: str | None = None,
|
81
|
+
directory_path: str | None = None,
|
82
|
+
azure_fileshare_conn_id: str = "azure_fileshare_default",
|
83
|
+
) -> None:
|
84
|
+
super().__init__()
|
85
|
+
self._conn_id = azure_fileshare_conn_id
|
86
|
+
self.share_name = share_name
|
87
|
+
self.file_path = file_path
|
88
|
+
self.directory_path = directory_path
|
89
|
+
self._account_url: str | None = None
|
90
|
+
self._connection_string: str | None = None
|
91
|
+
self._account_access_key: str | None = None
|
92
|
+
self._sas_token: str | None = None
|
93
|
+
|
90
94
|
def get_conn(self) -> None:
|
91
95
|
conn = self.get_connection(self._conn_id)
|
92
96
|
extras = conn.extra_dejson
|
@@ -102,6 +106,16 @@ class AzureFileShareHook(BaseHook):
|
|
102
106
|
return f"https://{account_url}.file.core.windows.net"
|
103
107
|
return account_url
|
104
108
|
|
109
|
+
def _get_sync_default_azure_credential(self):
|
110
|
+
conn = self.get_connection(self._conn_id)
|
111
|
+
extras = conn.extra_dejson
|
112
|
+
managed_identity_client_id = extras.get("managed_identity_client_id")
|
113
|
+
workload_identity_tenant_id = extras.get("workload_identity_tenant_id")
|
114
|
+
return get_sync_default_azure_credential(
|
115
|
+
managed_identity_client_id=managed_identity_client_id,
|
116
|
+
workload_identity_tenant_id=workload_identity_tenant_id,
|
117
|
+
)
|
118
|
+
|
105
119
|
@property
|
106
120
|
def share_service_client(self):
|
107
121
|
self.get_conn()
|
@@ -114,7 +128,9 @@ class AzureFileShareHook(BaseHook):
|
|
114
128
|
return ShareServiceClient(account_url=self._account_url, credential=credential)
|
115
129
|
else:
|
116
130
|
return ShareServiceClient(
|
117
|
-
account_url=self._account_url,
|
131
|
+
account_url=self._account_url,
|
132
|
+
credential=self._get_sync_default_azure_credential(),
|
133
|
+
token_intent="backup",
|
118
134
|
)
|
119
135
|
|
120
136
|
@property
|
@@ -138,7 +154,7 @@ class AzureFileShareHook(BaseHook):
|
|
138
154
|
account_url=self._account_url,
|
139
155
|
share_name=self.share_name,
|
140
156
|
directory_path=self.directory_path,
|
141
|
-
credential=
|
157
|
+
credential=self._get_sync_default_azure_credential(),
|
142
158
|
token_intent="backup",
|
143
159
|
)
|
144
160
|
|
@@ -163,7 +179,7 @@ class AzureFileShareHook(BaseHook):
|
|
163
179
|
account_url=self._account_url,
|
164
180
|
share_name=self.share_name,
|
165
181
|
file_path=self.file_path,
|
166
|
-
credential=
|
182
|
+
credential=self._get_sync_default_azure_credential(),
|
167
183
|
token_intent="backup",
|
168
184
|
)
|
169
185
|
|
@@ -24,7 +24,11 @@ from azure.synapse.spark import SparkClient
|
|
24
24
|
|
25
25
|
from airflow.exceptions import AirflowTaskTimeout
|
26
26
|
from airflow.hooks.base import BaseHook
|
27
|
-
from airflow.providers.microsoft.azure.utils import
|
27
|
+
from airflow.providers.microsoft.azure.utils import (
|
28
|
+
add_managed_identity_connection_widgets,
|
29
|
+
get_field,
|
30
|
+
get_sync_default_azure_credential,
|
31
|
+
)
|
28
32
|
|
29
33
|
if TYPE_CHECKING:
|
30
34
|
from azure.synapse.spark.models import SparkBatchJobOptions
|
@@ -63,6 +67,7 @@ class AzureSynapseHook(BaseHook):
|
|
63
67
|
hook_name: str = "Azure Synapse"
|
64
68
|
|
65
69
|
@staticmethod
|
70
|
+
@add_managed_identity_connection_widgets
|
66
71
|
def get_connection_form_widgets() -> dict[str, Any]:
|
67
72
|
"""Returns connection widgets to add to connection form."""
|
68
73
|
from flask_appbuilder.fieldwidgets import BS3TextFieldWidget
|
@@ -79,7 +84,11 @@ class AzureSynapseHook(BaseHook):
|
|
79
84
|
"""Returns custom field behaviour."""
|
80
85
|
return {
|
81
86
|
"hidden_fields": ["schema", "port", "extra"],
|
82
|
-
"relabeling": {
|
87
|
+
"relabeling": {
|
88
|
+
"login": "Client ID",
|
89
|
+
"password": "Secret",
|
90
|
+
"host": "Synapse Workspace URL",
|
91
|
+
},
|
83
92
|
}
|
84
93
|
|
85
94
|
def __init__(self, azure_synapse_conn_id: str = default_conn_name, spark_pool: str = ""):
|
@@ -120,7 +129,12 @@ class AzureSynapseHook(BaseHook):
|
|
120
129
|
client_id=conn.login, client_secret=conn.password, tenant_id=tenant
|
121
130
|
)
|
122
131
|
else:
|
123
|
-
|
132
|
+
managed_identity_client_id = self._get_field(extras, "managed_identity_client_id")
|
133
|
+
workload_identity_tenant_id = self._get_field(extras, "workload_identity_tenant_id")
|
134
|
+
credential = get_sync_default_azure_credential(
|
135
|
+
managed_identity_client_id=managed_identity_client_id,
|
136
|
+
workload_identity_tenant_id=workload_identity_tenant_id,
|
137
|
+
)
|
124
138
|
|
125
139
|
self._conn = self._create_client(credential, conn.host, spark_pool, livy_api_version, subscription_id)
|
126
140
|
|
@@ -33,7 +33,7 @@ from urllib.parse import urlparse
|
|
33
33
|
|
34
34
|
from asgiref.sync import sync_to_async
|
35
35
|
from azure.core.exceptions import HttpResponseError, ResourceExistsError, ResourceNotFoundError
|
36
|
-
from azure.identity import ClientSecretCredential
|
36
|
+
from azure.identity import ClientSecretCredential
|
37
37
|
from azure.identity.aio import (
|
38
38
|
ClientSecretCredential as AsyncClientSecretCredential,
|
39
39
|
DefaultAzureCredential as AsyncDefaultAzureCredential,
|
@@ -47,6 +47,11 @@ from azure.storage.blob.aio import (
|
|
47
47
|
|
48
48
|
from airflow.exceptions import AirflowException
|
49
49
|
from airflow.hooks.base import BaseHook
|
50
|
+
from airflow.providers.microsoft.azure.utils import (
|
51
|
+
add_managed_identity_connection_widgets,
|
52
|
+
get_async_default_azure_credential,
|
53
|
+
get_sync_default_azure_credential,
|
54
|
+
)
|
50
55
|
|
51
56
|
if TYPE_CHECKING:
|
52
57
|
from azure.storage.blob._models import BlobProperties
|
@@ -77,6 +82,7 @@ class WasbHook(BaseHook):
|
|
77
82
|
hook_name = "Azure Blob Storage"
|
78
83
|
|
79
84
|
@staticmethod
|
85
|
+
@add_managed_identity_connection_widgets
|
80
86
|
def get_connection_form_widgets() -> dict[str, Any]:
|
81
87
|
"""Returns connection widgets to add to connection form."""
|
82
88
|
from flask_appbuilder.fieldwidgets import BS3PasswordFieldWidget, BS3TextFieldWidget
|
@@ -207,7 +213,12 @@ class WasbHook(BaseHook):
|
|
207
213
|
# Fall back to old auth (password) or use managed identity if not provided.
|
208
214
|
credential = conn.password
|
209
215
|
if not credential:
|
210
|
-
|
216
|
+
managed_identity_client_id = self._get_field(extra, "managed_identity_client_id")
|
217
|
+
workload_identity_tenant_id = self._get_field(extra, "workload_identity_tenant_id")
|
218
|
+
credential = get_sync_default_azure_credential(
|
219
|
+
managed_identity_client_id=managed_identity_client_id,
|
220
|
+
workload_identity_tenant_id=workload_identity_tenant_id,
|
221
|
+
)
|
211
222
|
self.log.info("Using DefaultAzureCredential as credential")
|
212
223
|
return BlobServiceClient(
|
213
224
|
account_url=account_url,
|
@@ -600,7 +611,9 @@ class WasbAsyncHook(WasbHook):
|
|
600
611
|
tenant, app_id, app_secret, **client_secret_auth_config
|
601
612
|
)
|
602
613
|
self.blob_service_client = AsyncBlobServiceClient(
|
603
|
-
account_url=account_url,
|
614
|
+
account_url=account_url,
|
615
|
+
credential=token_credential,
|
616
|
+
**extra, # type:ignore[arg-type]
|
604
617
|
)
|
605
618
|
return self.blob_service_client
|
606
619
|
|
@@ -632,7 +645,12 @@ class WasbAsyncHook(WasbHook):
|
|
632
645
|
# Fall back to old auth (password) or use managed identity if not provided.
|
633
646
|
credential = conn.password
|
634
647
|
if not credential:
|
635
|
-
|
648
|
+
managed_identity_client_id = self._get_field(extra, "managed_identity_client_id")
|
649
|
+
workload_identity_tenant_id = self._get_field(extra, "workload_identity_tenant_id")
|
650
|
+
credential = get_async_default_azure_credential(
|
651
|
+
managed_identity_client_id=managed_identity_client_id,
|
652
|
+
workload_identity_tenant_id=workload_identity_tenant_id,
|
653
|
+
)
|
636
654
|
self.log.info("Using DefaultAzureCredential as credential")
|
637
655
|
self.blob_service_client = AsyncBlobServiceClient(
|
638
656
|
account_url=account_url,
|
@@ -278,12 +278,11 @@ class AzureContainerInstancesOperator(BaseOperator):
|
|
278
278
|
self.on_kill()
|
279
279
|
|
280
280
|
def on_kill(self) -> None:
|
281
|
-
|
282
|
-
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
self.log.exception("Could not delete container group")
|
281
|
+
self.log.info("Deleting container group")
|
282
|
+
try:
|
283
|
+
self._ci_hook.delete(self.resource_group, self.name)
|
284
|
+
except Exception:
|
285
|
+
self.log.exception("Could not delete container group")
|
287
286
|
|
288
287
|
def _monitor_logging(self, resource_group: str, name: str) -> int:
|
289
288
|
last_state = None
|
@@ -14,6 +14,14 @@
|
|
14
14
|
# KIND, either express or implied. See the License for the
|
15
15
|
# specific language governing permissions and limitations
|
16
16
|
# under the License.
|
17
|
+
"""
|
18
|
+
This module contains Azure Key Vault Backend.
|
19
|
+
|
20
|
+
.. spelling:word-list::
|
21
|
+
|
22
|
+
Entra
|
23
|
+
"""
|
24
|
+
|
17
25
|
from __future__ import annotations
|
18
26
|
|
19
27
|
import logging
|
@@ -27,6 +35,7 @@ from azure.identity import ClientSecretCredential, DefaultAzureCredential
|
|
27
35
|
from azure.keyvault.secrets import SecretClient
|
28
36
|
|
29
37
|
from airflow.exceptions import AirflowProviderDeprecationWarning
|
38
|
+
from airflow.providers.microsoft.azure.utils import get_sync_default_azure_credential
|
30
39
|
from airflow.secrets import BaseSecretsBackend
|
31
40
|
from airflow.utils.log.logging_mixin import LoggingMixin
|
32
41
|
from airflow.version import version as airflow_version
|
@@ -76,8 +85,11 @@ class AzureKeyVaultBackend(BaseSecretsBackend, LoggingMixin):
|
|
76
85
|
If not given, it falls back to ``DefaultAzureCredential``
|
77
86
|
:param client_id: The client id of an Azure Key Vault to use.
|
78
87
|
If not given, it falls back to ``DefaultAzureCredential``
|
79
|
-
:param
|
80
|
-
If
|
88
|
+
:param managed_identity_client_id: The client ID of a user-assigned managed identity.
|
89
|
+
If provided with `workload_identity_tenant_id`, they'll pass to ``DefaultAzureCredential``.
|
90
|
+
:param workload_identity_tenant_id: ID of the application's Microsoft Entra tenant.
|
91
|
+
Also called its "directory" ID.
|
92
|
+
If provided with `managed_identity_client_id`, they'll pass to ``DefaultAzureCredential``.
|
81
93
|
"""
|
82
94
|
|
83
95
|
def __init__(
|
@@ -91,6 +103,8 @@ class AzureKeyVaultBackend(BaseSecretsBackend, LoggingMixin):
|
|
91
103
|
tenant_id: str = "",
|
92
104
|
client_id: str = "",
|
93
105
|
client_secret: str = "",
|
106
|
+
managed_identity_client_id: str = "",
|
107
|
+
workload_identity_tenant_id: str = "",
|
94
108
|
**kwargs,
|
95
109
|
) -> None:
|
96
110
|
super().__init__()
|
@@ -118,6 +132,8 @@ class AzureKeyVaultBackend(BaseSecretsBackend, LoggingMixin):
|
|
118
132
|
self.tenant_id = tenant_id
|
119
133
|
self.client_id = client_id
|
120
134
|
self.client_secret = client_secret
|
135
|
+
self.managed_identity_client_id = managed_identity_client_id
|
136
|
+
self.workload_identity_tenant_id = workload_identity_tenant_id
|
121
137
|
self.kwargs = kwargs
|
122
138
|
|
123
139
|
@cached_property
|
@@ -127,7 +143,10 @@ class AzureKeyVaultBackend(BaseSecretsBackend, LoggingMixin):
|
|
127
143
|
if all([self.tenant_id, self.client_id, self.client_secret]):
|
128
144
|
credential = ClientSecretCredential(self.tenant_id, self.client_id, self.client_secret)
|
129
145
|
else:
|
130
|
-
credential =
|
146
|
+
credential = get_sync_default_azure_credential(
|
147
|
+
managed_identity_client_id=self.managed_identity_client_id,
|
148
|
+
workload_identity_tenant_id=self.workload_identity_tenant_id,
|
149
|
+
)
|
131
150
|
client = SecretClient(vault_url=self.vault_url, credential=credential, **self.kwargs)
|
132
151
|
return client
|
133
152
|
|
@@ -18,11 +18,13 @@
|
|
18
18
|
from __future__ import annotations
|
19
19
|
|
20
20
|
import warnings
|
21
|
+
from functools import partial, wraps
|
21
22
|
|
22
23
|
from azure.core.pipeline import PipelineContext, PipelineRequest
|
23
24
|
from azure.core.pipeline.policies import BearerTokenCredentialPolicy
|
24
25
|
from azure.core.pipeline.transport import HttpRequest
|
25
26
|
from azure.identity import DefaultAzureCredential
|
27
|
+
from azure.identity.aio import DefaultAzureCredential as AsyncDefaultAzureCredential
|
26
28
|
from msrest.authentication import BasicTokenAuthentication
|
27
29
|
|
28
30
|
|
@@ -51,6 +53,64 @@ def get_field(*, conn_id: str, conn_type: str, extras: dict, field_name: str):
|
|
51
53
|
return ret
|
52
54
|
|
53
55
|
|
56
|
+
def _get_default_azure_credential(
|
57
|
+
*,
|
58
|
+
managed_identity_client_id: str | None = None,
|
59
|
+
workload_identity_tenant_id: str | None = None,
|
60
|
+
use_async: bool = False,
|
61
|
+
) -> DefaultAzureCredential | AsyncDefaultAzureCredential:
|
62
|
+
"""Get DefaultAzureCredential based on provided arguments.
|
63
|
+
|
64
|
+
If managed_identity_client_id and workload_identity_tenant_id are provided, this function returns
|
65
|
+
DefaultAzureCredential with managed identity.
|
66
|
+
"""
|
67
|
+
credential_cls: type[AsyncDefaultAzureCredential] | type[DefaultAzureCredential] = (
|
68
|
+
AsyncDefaultAzureCredential if use_async else DefaultAzureCredential
|
69
|
+
)
|
70
|
+
if managed_identity_client_id and workload_identity_tenant_id:
|
71
|
+
return credential_cls(
|
72
|
+
managed_identity_client_id=managed_identity_client_id,
|
73
|
+
workload_identity_tenant_id=workload_identity_tenant_id,
|
74
|
+
additionally_allowed_tenants=[workload_identity_tenant_id],
|
75
|
+
)
|
76
|
+
else:
|
77
|
+
return credential_cls()
|
78
|
+
|
79
|
+
|
80
|
+
get_sync_default_azure_credential: partial[DefaultAzureCredential] = partial(
|
81
|
+
_get_default_azure_credential, # type: ignore[arg-type]
|
82
|
+
use_async=False,
|
83
|
+
)
|
84
|
+
|
85
|
+
get_async_default_azure_credential: partial[AsyncDefaultAzureCredential] = partial(
|
86
|
+
_get_default_azure_credential, # type: ignore[arg-type]
|
87
|
+
use_async=True,
|
88
|
+
)
|
89
|
+
|
90
|
+
|
91
|
+
def add_managed_identity_connection_widgets(func):
|
92
|
+
@wraps(func)
|
93
|
+
def wrapper(*args, **kwargs):
|
94
|
+
from flask_appbuilder.fieldwidgets import BS3TextFieldWidget
|
95
|
+
from flask_babel import lazy_gettext
|
96
|
+
from wtforms import StringField
|
97
|
+
|
98
|
+
widgets = func(*args, **kwargs)
|
99
|
+
widgets.update(
|
100
|
+
{
|
101
|
+
"managed_identity_client_id": StringField(
|
102
|
+
lazy_gettext("Managed Identity Client ID"), widget=BS3TextFieldWidget()
|
103
|
+
),
|
104
|
+
"workload_identity_tenant_id": StringField(
|
105
|
+
lazy_gettext("Workload Identity Tenant ID"), widget=BS3TextFieldWidget()
|
106
|
+
),
|
107
|
+
}
|
108
|
+
)
|
109
|
+
return widgets
|
110
|
+
|
111
|
+
return wrapper
|
112
|
+
|
113
|
+
|
54
114
|
class AzureIdentityCredentialAdapter(BasicTokenAuthentication):
|
55
115
|
"""Adapt azure-identity credentials for backward compatibility.
|
56
116
|
|
@@ -60,15 +120,31 @@ class AzureIdentityCredentialAdapter(BasicTokenAuthentication):
|
|
60
120
|
Check https://stackoverflow.com/questions/63384092/exception-attributeerror-defaultazurecredential-object-has-no-attribute-sig
|
61
121
|
"""
|
62
122
|
|
63
|
-
def __init__(
|
123
|
+
def __init__(
|
124
|
+
self,
|
125
|
+
credential=None,
|
126
|
+
resource_id="https://management.azure.com/.default",
|
127
|
+
*,
|
128
|
+
managed_identity_client_id: str | None = None,
|
129
|
+
workload_identity_tenant_id: str | None = None,
|
130
|
+
**kwargs,
|
131
|
+
):
|
64
132
|
"""Adapt azure-identity credentials for backward compatibility.
|
65
133
|
|
66
134
|
:param credential: Any azure-identity credential (DefaultAzureCredential by default)
|
67
|
-
:param
|
135
|
+
:param resource_id: The scope to use to get the token (default ARM)
|
136
|
+
:param managed_identity_client_id: The client ID of a user-assigned managed identity.
|
137
|
+
If provided with `workload_identity_tenant_id`, they'll pass to ``DefaultAzureCredential``.
|
138
|
+
:param workload_identity_tenant_id: ID of the application's Microsoft Entra tenant.
|
139
|
+
Also called its "directory" ID.
|
140
|
+
If provided with `managed_identity_client_id`, they'll pass to ``DefaultAzureCredential``.
|
68
141
|
"""
|
69
|
-
super().__init__(None)
|
142
|
+
super().__init__(None) # type: ignore[arg-type]
|
70
143
|
if credential is None:
|
71
|
-
credential =
|
144
|
+
credential = get_sync_default_azure_credential(
|
145
|
+
managed_identity_client_id=managed_identity_client_id,
|
146
|
+
workload_identity_tenant_id=workload_identity_tenant_id,
|
147
|
+
)
|
72
148
|
self._policy = BearerTokenCredentialPolicy(credential, resource_id, **kwargs)
|
73
149
|
|
74
150
|
def _make_request(self):
|
@@ -1,14 +1,14 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: apache-airflow-providers-microsoft-azure
|
3
|
-
Version: 8.
|
3
|
+
Version: 8.2.0rc1
|
4
4
|
Summary: Provider for Apache Airflow. Implements apache-airflow-providers-microsoft-azure package
|
5
5
|
Home-page: https://airflow.apache.org/
|
6
6
|
Download-URL: https://archive.apache.org/dist/airflow/providers
|
7
7
|
Author: Apache Software Foundation
|
8
8
|
Author-email: dev@airflow.apache.org
|
9
9
|
License: Apache License 2.0
|
10
|
-
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/8.
|
11
|
-
Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/8.
|
10
|
+
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/8.2.0/
|
11
|
+
Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/8.2.0/changelog.html
|
12
12
|
Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
|
13
13
|
Project-URL: Source Code, https://github.com/apache/airflow
|
14
14
|
Project-URL: Slack Chat, https://s.apache.org/airflow-slack
|
@@ -76,8 +76,7 @@ Requires-Dist: apache-airflow-providers-sftp ; extra == 'sftp'
|
|
76
76
|
KIND, either express or implied. See the License for the
|
77
77
|
specific language governing permissions and limitations
|
78
78
|
under the License.
|
79
|
-
|
80
|
-
.. Licensed to the Apache Software Foundation (ASF) under one
|
79
|
+
.. Licensed to the Apache Software Foundation (ASF) under one
|
81
80
|
or more contributor license agreements. See the NOTICE file
|
82
81
|
distributed with this work for additional information
|
83
82
|
regarding copyright ownership. The ASF licenses this file
|
@@ -97,7 +96,7 @@ Requires-Dist: apache-airflow-providers-sftp ; extra == 'sftp'
|
|
97
96
|
|
98
97
|
Package ``apache-airflow-providers-microsoft-azure``
|
99
98
|
|
100
|
-
Release: ``8.
|
99
|
+
Release: ``8.2.0rc1``
|
101
100
|
|
102
101
|
|
103
102
|
`Microsoft Azure <https://azure.microsoft.com/>`__
|
@@ -110,8 +109,7 @@ This is a provider package for ``microsoft.azure`` provider. All classes for thi
|
|
110
109
|
are in ``airflow.providers.microsoft.azure`` python package.
|
111
110
|
|
112
111
|
You can find package information and changelog for the provider
|
113
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/8.
|
114
|
-
|
112
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/8.2.0/>`_.
|
115
113
|
|
116
114
|
Installation
|
117
115
|
------------
|
@@ -173,4 +171,4 @@ Dependent package
|
|
173
171
|
==================================================================================================== ==========
|
174
172
|
|
175
173
|
The changelog for the provider package can be found in the
|
176
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/8.
|
174
|
+
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/8.2.0/changelog.html>`_.
|
@@ -1,22 +1,22 @@
|
|
1
|
-
airflow/providers/microsoft/azure/__init__.py,sha256=
|
2
|
-
airflow/providers/microsoft/azure/get_provider_info.py,sha256=
|
3
|
-
airflow/providers/microsoft/azure/utils.py,sha256=
|
1
|
+
airflow/providers/microsoft/azure/__init__.py,sha256=A4kvtHQo9XYh3bXkl1-QNHM5vSkAdX2833p-oSipxYA,1570
|
2
|
+
airflow/providers/microsoft/azure/get_provider_info.py,sha256=bzX27y8jPmWH9fU8t6vXTRX2lXprj2kzybcSrGHZb0M,18343
|
3
|
+
airflow/providers/microsoft/azure/utils.py,sha256=0JAjaDDZ0dIT_K9sMUsWaSZzBBQvGtkQUeG2SLrYdzg,7127
|
4
4
|
airflow/providers/microsoft/azure/fs/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
5
5
|
airflow/providers/microsoft/azure/fs/adls.py,sha256=XV4uWBXO8C6U2gy4KyZjf9ni9zJamoRpKaQIMT2ttdM,2183
|
6
6
|
airflow/providers/microsoft/azure/hooks/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
|
7
|
-
airflow/providers/microsoft/azure/hooks/adx.py,sha256=
|
8
|
-
airflow/providers/microsoft/azure/hooks/asb.py,sha256=
|
9
|
-
airflow/providers/microsoft/azure/hooks/base_azure.py,sha256=
|
10
|
-
airflow/providers/microsoft/azure/hooks/batch.py,sha256=
|
11
|
-
airflow/providers/microsoft/azure/hooks/container_instance.py,sha256=
|
12
|
-
airflow/providers/microsoft/azure/hooks/container_registry.py,sha256=
|
13
|
-
airflow/providers/microsoft/azure/hooks/container_volume.py,sha256=
|
14
|
-
airflow/providers/microsoft/azure/hooks/cosmos.py,sha256=
|
15
|
-
airflow/providers/microsoft/azure/hooks/data_factory.py,sha256=
|
16
|
-
airflow/providers/microsoft/azure/hooks/data_lake.py,sha256=
|
17
|
-
airflow/providers/microsoft/azure/hooks/fileshare.py,sha256=
|
18
|
-
airflow/providers/microsoft/azure/hooks/synapse.py,sha256=
|
19
|
-
airflow/providers/microsoft/azure/hooks/wasb.py,sha256=
|
7
|
+
airflow/providers/microsoft/azure/hooks/adx.py,sha256=4a2oBs-x3VNJrjxCqSA6l7hjSn8z8kbGAnmpC_hJCM0,9934
|
8
|
+
airflow/providers/microsoft/azure/hooks/asb.py,sha256=VKLLgGSj_SnBB-74vPxtc2N0mVduBJEtXqU_EAXvouE,14427
|
9
|
+
airflow/providers/microsoft/azure/hooks/base_azure.py,sha256=yVO6cXYcdaHwLLrXqtXvCN9HeQRGpnyk60lOmntOehA,6282
|
10
|
+
airflow/providers/microsoft/azure/hooks/batch.py,sha256=VdAix6NcsAH9_uDWPz7pUNeTh8sTmC8mUgo6zBI_f8c,16114
|
11
|
+
airflow/providers/microsoft/azure/hooks/container_instance.py,sha256=B9uCgpy-q7Pd_FQ0BdvcJ_opYQbRJfVvw0dOdTXfl3c,9054
|
12
|
+
airflow/providers/microsoft/azure/hooks/container_registry.py,sha256=QCoTcqVY5o1O5fR-5KxoHRcdgTtyM9Spc6ktOrz1Tho,4854
|
13
|
+
airflow/providers/microsoft/azure/hooks/container_volume.py,sha256=2b31RtjMQvLIyyRvKJ1yyykeMrPhWF-To5V8OkGefUM,5756
|
14
|
+
airflow/providers/microsoft/azure/hooks/cosmos.py,sha256=zgtGHqeAOiFpFhvIyumAB6c89vjeP28NkQZN9KV9H88,16642
|
15
|
+
airflow/providers/microsoft/azure/hooks/data_factory.py,sha256=Mpv6FsldMp_ToEJ1JkWZ-eJ4HxQhxzWrFK6U_DEnCqc,45553
|
16
|
+
airflow/providers/microsoft/azure/hooks/data_lake.py,sha256=_OpZUzng8SvV0FDOprBFFWRKuXubxbEouVIxhobpwYE,23335
|
17
|
+
airflow/providers/microsoft/azure/hooks/fileshare.py,sha256=8evFa4csjcRScW_XJBgBw-ZJWSH0SL9CGfJIcOkO9bU,10834
|
18
|
+
airflow/providers/microsoft/azure/hooks/synapse.py,sha256=MEzC4nsWjKbQHEydFljDVb6AZapKOtSXLj_wo1AmqwQ,7914
|
19
|
+
airflow/providers/microsoft/azure/hooks/wasb.py,sha256=ZgxbL0CXuzBWZG3hcwDSYoox60b6nzOZoxDlqM7gdMQ,31111
|
20
20
|
airflow/providers/microsoft/azure/log/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
21
21
|
airflow/providers/microsoft/azure/log/wasb_task_handler.py,sha256=uq7CPplHQCAJCMxxeTdTnmsKwq82olihBXYPh9LmbNI,9949
|
22
22
|
airflow/providers/microsoft/azure/operators/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
|
@@ -24,13 +24,13 @@ airflow/providers/microsoft/azure/operators/adls.py,sha256=msx7uZNlSshiZsNASESk6
|
|
24
24
|
airflow/providers/microsoft/azure/operators/adx.py,sha256=H7Q6BLdO-WZUV0UUxqtR8e-rRdvJ8qo2dk-6VpJj3K4,3408
|
25
25
|
airflow/providers/microsoft/azure/operators/asb.py,sha256=DDOH-EMSuQNn8PfSqdsFw5Ks81GAfavvxzKjchdZgTk,29420
|
26
26
|
airflow/providers/microsoft/azure/operators/batch.py,sha256=BxhI8H9vzCtvPZo7bjQtNirRAlRCOyo8UK2D7GZCJkk,16538
|
27
|
-
airflow/providers/microsoft/azure/operators/container_instances.py,sha256=
|
27
|
+
airflow/providers/microsoft/azure/operators/container_instances.py,sha256=uZahD2uIYO0sFeUWeM7Nk3J5QWzCcq2gcKv4SEgSfwY,16084
|
28
28
|
airflow/providers/microsoft/azure/operators/cosmos.py,sha256=Y1Hj4p6W8soVFaq1rx8LFgchNISjkq8vjdaQ0j8Tnqs,2782
|
29
29
|
airflow/providers/microsoft/azure/operators/data_factory.py,sha256=-AZNA51CTiFfyl0DUJL_0LlDHg6uGVVtt0YID7iX910,12377
|
30
30
|
airflow/providers/microsoft/azure/operators/synapse.py,sha256=j3HC4SbjJKXwso8gkw7gKwvnjPByWBKGxVecH5HaVvM,4541
|
31
31
|
airflow/providers/microsoft/azure/operators/wasb_delete_blob.py,sha256=GUfV9DLU1bGYvn2TE54iTYBTbxn3Jm_e985pXp_0IsE,2687
|
32
32
|
airflow/providers/microsoft/azure/secrets/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
33
|
-
airflow/providers/microsoft/azure/secrets/key_vault.py,sha256=
|
33
|
+
airflow/providers/microsoft/azure/secrets/key_vault.py,sha256=S9VpCe-qUqTyU-Ym8_kpAL3Q47uUL_w6yGaf480mAS8,9928
|
34
34
|
airflow/providers/microsoft/azure/sensors/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
|
35
35
|
airflow/providers/microsoft/azure/sensors/cosmos.py,sha256=vRrJ8zJnApvuKxHia53tNZUZ7wILWFT3_5cEyMA2M1I,2637
|
36
36
|
airflow/providers/microsoft/azure/sensors/data_factory.py,sha256=P2XkREfInc0kAvTB1kdKejs_7kvbBTcxmu_ShkW3cCo,5553
|
@@ -44,10 +44,10 @@ airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py,sha256=sx3fCZdjQ_AVx
|
|
44
44
|
airflow/providers/microsoft/azure/triggers/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
45
45
|
airflow/providers/microsoft/azure/triggers/data_factory.py,sha256=Td65obpw4fvXjjmF8kB-b9W994GueXTMm0mfR3Gy8FA,11130
|
46
46
|
airflow/providers/microsoft/azure/triggers/wasb.py,sha256=PizofSGAwdgCuswenlbzlH0Jr7GHEzT89BluucDt8dE,7382
|
47
|
-
apache_airflow_providers_microsoft_azure-8.
|
48
|
-
apache_airflow_providers_microsoft_azure-8.
|
49
|
-
apache_airflow_providers_microsoft_azure-8.
|
50
|
-
apache_airflow_providers_microsoft_azure-8.
|
51
|
-
apache_airflow_providers_microsoft_azure-8.
|
52
|
-
apache_airflow_providers_microsoft_azure-8.
|
53
|
-
apache_airflow_providers_microsoft_azure-8.
|
47
|
+
apache_airflow_providers_microsoft_azure-8.2.0rc1.dist-info/LICENSE,sha256=gXPVwptPlW1TJ4HSuG5OMPg-a3h43OGMkZRR1rpwfJA,10850
|
48
|
+
apache_airflow_providers_microsoft_azure-8.2.0rc1.dist-info/METADATA,sha256=7AUdA4dGMQb98FkFN1v-2lWMGvI8ojL5bPq09dwJ8Ro,7750
|
49
|
+
apache_airflow_providers_microsoft_azure-8.2.0rc1.dist-info/NOTICE,sha256=m-6s2XynUxVSUIxO4rVablAZCvFq-wmLrqV91DotRBw,240
|
50
|
+
apache_airflow_providers_microsoft_azure-8.2.0rc1.dist-info/WHEEL,sha256=Xo9-1PvkuimrydujYJAjF7pCkriuXBpUPEjma1nZyJ0,92
|
51
|
+
apache_airflow_providers_microsoft_azure-8.2.0rc1.dist-info/entry_points.txt,sha256=mAQpo-U_MYTG3pa7xmaY5vG849FnAziWsdGpfV-16NM,112
|
52
|
+
apache_airflow_providers_microsoft_azure-8.2.0rc1.dist-info/top_level.txt,sha256=OeMVH5md7fr2QQWpnZoOWWxWO-0WH1IP70lpTVwopPg,8
|
53
|
+
apache_airflow_providers_microsoft_azure-8.2.0rc1.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|