apache-airflow-providers-microsoft-azure 10.4.0rc1__tar.gz → 10.5.0rc1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/PKG-INFO +6 -6
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/README.rst +3 -3
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/__init__.py +1 -1
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/get_provider_info.py +2 -1
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/hooks/asb.py +51 -8
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/hooks/msgraph.py +22 -25
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/log/wasb_task_handler.py +1 -3
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/operators/asb.py +23 -3
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/operators/msgraph.py +1 -1
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/operators/powerbi.py +11 -1
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/sensors/data_factory.py +1 -10
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/sensors/msgraph.py +1 -1
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/sensors/wasb.py +1 -7
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/triggers/msgraph.py +3 -4
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/triggers/powerbi.py +3 -4
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/pyproject.toml +3 -3
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/LICENSE +0 -0
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/fs/__init__.py +0 -0
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/fs/adls.py +0 -0
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/hooks/__init__.py +0 -0
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/hooks/adx.py +0 -0
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/hooks/base_azure.py +0 -0
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/hooks/batch.py +0 -0
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/hooks/container_instance.py +0 -0
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/hooks/container_registry.py +0 -0
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/hooks/container_volume.py +0 -0
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/hooks/cosmos.py +0 -0
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/hooks/data_factory.py +0 -0
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/hooks/data_lake.py +0 -0
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/hooks/fileshare.py +0 -0
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/hooks/powerbi.py +0 -0
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/hooks/synapse.py +0 -0
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/hooks/wasb.py +0 -0
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/log/__init__.py +0 -0
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/operators/__init__.py +0 -0
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/operators/adls.py +0 -0
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/operators/adx.py +0 -0
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/operators/batch.py +0 -0
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/operators/container_instances.py +0 -0
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/operators/cosmos.py +0 -0
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/operators/data_factory.py +0 -0
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/operators/synapse.py +0 -0
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/operators/wasb_delete_blob.py +0 -0
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/secrets/__init__.py +0 -0
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/secrets/key_vault.py +0 -0
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/sensors/__init__.py +0 -0
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/sensors/cosmos.py +0 -0
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/transfers/__init__.py +0 -0
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/transfers/azure_blob_to_gcs.py +0 -0
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/transfers/local_to_adls.py +0 -0
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/transfers/local_to_wasb.py +0 -0
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/transfers/oracle_to_azure_data_lake.py +0 -0
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/transfers/s3_to_wasb.py +0 -0
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py +0 -0
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/triggers/__init__.py +0 -0
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/triggers/data_factory.py +0 -0
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/triggers/wasb.py +0 -0
- {apache_airflow_providers_microsoft_azure-10.4.0rc1 → apache_airflow_providers_microsoft_azure-10.5.0rc1}/airflow/providers/microsoft/azure/utils.py +0 -0
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: apache-airflow-providers-microsoft-azure
|
3
|
-
Version: 10.
|
3
|
+
Version: 10.5.0rc1
|
4
4
|
Summary: Provider package apache-airflow-providers-microsoft-azure for Apache Airflow
|
5
5
|
Keywords: airflow-provider,microsoft.azure,airflow,integration
|
6
6
|
Author-email: Apache Software Foundation <dev@airflow.apache.org>
|
@@ -49,8 +49,8 @@ Requires-Dist: apache-airflow-providers-google ; extra == "google"
|
|
49
49
|
Requires-Dist: apache-airflow-providers-oracle ; extra == "oracle"
|
50
50
|
Requires-Dist: apache-airflow-providers-sftp ; extra == "sftp"
|
51
51
|
Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
|
52
|
-
Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.
|
53
|
-
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.
|
52
|
+
Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.5.0/changelog.html
|
53
|
+
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.5.0
|
54
54
|
Project-URL: Slack Chat, https://s.apache.org/airflow-slack
|
55
55
|
Project-URL: Source Code, https://github.com/apache/airflow
|
56
56
|
Project-URL: Twitter, https://twitter.com/ApacheAirflow
|
@@ -104,7 +104,7 @@ Provides-Extra: sftp
|
|
104
104
|
|
105
105
|
Package ``apache-airflow-providers-microsoft-azure``
|
106
106
|
|
107
|
-
Release: ``10.
|
107
|
+
Release: ``10.5.0.rc1``
|
108
108
|
|
109
109
|
|
110
110
|
`Microsoft Azure <https://azure.microsoft.com/>`__
|
@@ -117,7 +117,7 @@ This is a provider package for ``microsoft.azure`` provider. All classes for thi
|
|
117
117
|
are in ``airflow.providers.microsoft.azure`` python package.
|
118
118
|
|
119
119
|
You can find package information and changelog for the provider
|
120
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.
|
120
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.5.0/>`_.
|
121
121
|
|
122
122
|
Installation
|
123
123
|
------------
|
@@ -182,4 +182,4 @@ Dependent package
|
|
182
182
|
==================================================================================================== ==========
|
183
183
|
|
184
184
|
The changelog for the provider package can be found in the
|
185
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.
|
185
|
+
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.5.0/changelog.html>`_.
|
@@ -42,7 +42,7 @@
|
|
42
42
|
|
43
43
|
Package ``apache-airflow-providers-microsoft-azure``
|
44
44
|
|
45
|
-
Release: ``10.
|
45
|
+
Release: ``10.5.0.rc1``
|
46
46
|
|
47
47
|
|
48
48
|
`Microsoft Azure <https://azure.microsoft.com/>`__
|
@@ -55,7 +55,7 @@ This is a provider package for ``microsoft.azure`` provider. All classes for thi
|
|
55
55
|
are in ``airflow.providers.microsoft.azure`` python package.
|
56
56
|
|
57
57
|
You can find package information and changelog for the provider
|
58
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.
|
58
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.5.0/>`_.
|
59
59
|
|
60
60
|
Installation
|
61
61
|
------------
|
@@ -120,4 +120,4 @@ Dependent package
|
|
120
120
|
==================================================================================================== ==========
|
121
121
|
|
122
122
|
The changelog for the provider package can be found in the
|
123
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.
|
123
|
+
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.5.0/changelog.html>`_.
|
@@ -29,7 +29,7 @@ from airflow import __version__ as airflow_version
|
|
29
29
|
|
30
30
|
__all__ = ["__version__"]
|
31
31
|
|
32
|
-
__version__ = "10.
|
32
|
+
__version__ = "10.5.0"
|
33
33
|
|
34
34
|
if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
|
35
35
|
"2.8.0"
|
@@ -28,8 +28,9 @@ def get_provider_info():
|
|
28
28
|
"name": "Microsoft Azure",
|
29
29
|
"description": "`Microsoft Azure <https://azure.microsoft.com/>`__\n",
|
30
30
|
"state": "ready",
|
31
|
-
"source-date-epoch":
|
31
|
+
"source-date-epoch": 1726861002,
|
32
32
|
"versions": [
|
33
|
+
"10.5.0",
|
33
34
|
"10.4.0",
|
34
35
|
"10.3.0",
|
35
36
|
"10.2.0",
|
@@ -16,9 +16,15 @@
|
|
16
16
|
# under the License.
|
17
17
|
from __future__ import annotations
|
18
18
|
|
19
|
-
from typing import TYPE_CHECKING, Any
|
20
|
-
|
21
|
-
from azure.servicebus import
|
19
|
+
from typing import TYPE_CHECKING, Any, Callable
|
20
|
+
|
21
|
+
from azure.servicebus import (
|
22
|
+
ServiceBusClient,
|
23
|
+
ServiceBusMessage,
|
24
|
+
ServiceBusReceivedMessage,
|
25
|
+
ServiceBusReceiver,
|
26
|
+
ServiceBusSender,
|
27
|
+
)
|
22
28
|
from azure.servicebus.management import QueueProperties, ServiceBusAdministrationClient
|
23
29
|
|
24
30
|
from airflow.hooks.base import BaseHook
|
@@ -28,6 +34,9 @@ from airflow.providers.microsoft.azure.utils import (
|
|
28
34
|
get_sync_default_azure_credential,
|
29
35
|
)
|
30
36
|
|
37
|
+
MessageCallback = Callable[[ServiceBusMessage], None]
|
38
|
+
|
39
|
+
|
31
40
|
if TYPE_CHECKING:
|
32
41
|
from azure.identity import DefaultAzureCredential
|
33
42
|
|
@@ -270,7 +279,11 @@ class MessageHook(BaseAzureServiceBusHook):
|
|
270
279
|
sender.send_messages(batch_message)
|
271
280
|
|
272
281
|
def receive_message(
|
273
|
-
self,
|
282
|
+
self,
|
283
|
+
queue_name: str,
|
284
|
+
max_message_count: int | None = 1,
|
285
|
+
max_wait_time: float | None = None,
|
286
|
+
message_callback: MessageCallback | None = None,
|
274
287
|
):
|
275
288
|
"""
|
276
289
|
Receive a batch of messages at once in a specified Queue name.
|
@@ -278,6 +291,9 @@ class MessageHook(BaseAzureServiceBusHook):
|
|
278
291
|
:param queue_name: The name of the queue name or a QueueProperties with name.
|
279
292
|
:param max_message_count: Maximum number of messages in the batch.
|
280
293
|
:param max_wait_time: Maximum time to wait in seconds for the first message to arrive.
|
294
|
+
:param message_callback: Optional callback to process each message. If not provided, then
|
295
|
+
the message will be logged and completed. If provided, and throws an exception, the
|
296
|
+
message will be abandoned for future redelivery.
|
281
297
|
"""
|
282
298
|
if queue_name is None:
|
283
299
|
raise TypeError("Queue name cannot be None.")
|
@@ -289,8 +305,7 @@ class MessageHook(BaseAzureServiceBusHook):
|
|
289
305
|
max_message_count=max_message_count, max_wait_time=max_wait_time
|
290
306
|
)
|
291
307
|
for msg in received_msgs:
|
292
|
-
self.
|
293
|
-
receiver.complete_message(msg)
|
308
|
+
self._process_message(msg, message_callback, receiver)
|
294
309
|
|
295
310
|
def receive_subscription_message(
|
296
311
|
self,
|
@@ -298,6 +313,7 @@ class MessageHook(BaseAzureServiceBusHook):
|
|
298
313
|
subscription_name: str,
|
299
314
|
max_message_count: int | None,
|
300
315
|
max_wait_time: float | None,
|
316
|
+
message_callback: MessageCallback | None = None,
|
301
317
|
):
|
302
318
|
"""
|
303
319
|
Receive a batch of subscription message at once.
|
@@ -326,5 +342,32 @@ class MessageHook(BaseAzureServiceBusHook):
|
|
326
342
|
max_message_count=max_message_count, max_wait_time=max_wait_time
|
327
343
|
)
|
328
344
|
for msg in received_msgs:
|
329
|
-
self.
|
330
|
-
|
345
|
+
self._process_message(msg, message_callback, subscription_receiver)
|
346
|
+
|
347
|
+
def _process_message(
|
348
|
+
self,
|
349
|
+
msg: ServiceBusReceivedMessage,
|
350
|
+
message_callback: MessageCallback | None,
|
351
|
+
receiver: ServiceBusReceiver,
|
352
|
+
):
|
353
|
+
"""
|
354
|
+
Process the message by calling the message_callback or logging the message.
|
355
|
+
|
356
|
+
:param msg: The message to process.
|
357
|
+
:param message_callback: Optional callback to process each message. If not provided, then
|
358
|
+
the message will be logged and completed. If provided, and throws an exception, the
|
359
|
+
message will be abandoned for future redelivery.
|
360
|
+
:param receiver: The receiver that received the message.
|
361
|
+
"""
|
362
|
+
if message_callback is None:
|
363
|
+
self.log.info(msg)
|
364
|
+
receiver.complete_message(msg)
|
365
|
+
else:
|
366
|
+
try:
|
367
|
+
message_callback(msg)
|
368
|
+
except Exception as e:
|
369
|
+
self.log.error("Error processing message: %s", e)
|
370
|
+
receiver.abandon_message(msg)
|
371
|
+
raise e
|
372
|
+
else:
|
373
|
+
receiver.complete_message(msg)
|
@@ -96,6 +96,8 @@ class KiotaRequestAdapterHook(BaseHook):
|
|
96
96
|
:param timeout: The HTTP timeout being used by the KiotaRequestAdapter (default is None).
|
97
97
|
When no timeout is specified or set to None then no HTTP timeout is applied on each request.
|
98
98
|
:param proxies: A Dict defining the HTTP proxies to be used (default is None).
|
99
|
+
:param host: The host to be used (default is "https://graph.microsoft.com").
|
100
|
+
:param scopes: The scopes to be used (default is ["https://graph.microsoft.com/.default"]).
|
99
101
|
:param api_version: The API version of the Microsoft Graph API to be used (default is v1).
|
100
102
|
You can pass an enum named APIVersion which has 2 possible members v1 and beta,
|
101
103
|
or you can pass a string as "v1.0" or "beta".
|
@@ -123,27 +125,22 @@ class KiotaRequestAdapterHook(BaseHook):
|
|
123
125
|
self._api_version = self.resolve_api_version_from_value(api_version)
|
124
126
|
|
125
127
|
@property
|
126
|
-
def api_version(self) ->
|
128
|
+
def api_version(self) -> str | None:
|
127
129
|
self.get_conn() # Make sure config has been loaded through get_conn to have correct api version!
|
128
130
|
return self._api_version
|
129
131
|
|
130
132
|
@staticmethod
|
131
133
|
def resolve_api_version_from_value(
|
132
|
-
api_version: APIVersion | str, default:
|
133
|
-
) ->
|
134
|
+
api_version: APIVersion | str, default: str | None = None
|
135
|
+
) -> str | None:
|
134
136
|
if isinstance(api_version, APIVersion):
|
135
|
-
return api_version
|
136
|
-
return
|
137
|
-
filter(lambda version: version.value == api_version, APIVersion),
|
138
|
-
default,
|
139
|
-
)
|
137
|
+
return api_version.value
|
138
|
+
return api_version or default
|
140
139
|
|
141
|
-
def get_api_version(self, config: dict) ->
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
)
|
146
|
-
return self._api_version
|
140
|
+
def get_api_version(self, config: dict) -> str:
|
141
|
+
return self._api_version or self.resolve_api_version_from_value(
|
142
|
+
config.get("api_version"), APIVersion.v1.value
|
143
|
+
) # type: ignore
|
147
144
|
|
148
145
|
def get_host(self, connection: Connection) -> str:
|
149
146
|
if connection.schema and connection.host:
|
@@ -169,15 +166,15 @@ class KiotaRequestAdapterHook(BaseHook):
|
|
169
166
|
return proxies
|
170
167
|
|
171
168
|
def to_msal_proxies(self, authority: str | None, proxies: dict):
|
172
|
-
self.log.
|
169
|
+
self.log.debug("authority: %s", authority)
|
173
170
|
if authority:
|
174
171
|
no_proxies = proxies.get("no")
|
175
|
-
self.log.
|
172
|
+
self.log.debug("no_proxies: %s", no_proxies)
|
176
173
|
if no_proxies:
|
177
174
|
for url in no_proxies.split(","):
|
178
175
|
self.log.info("url: %s", url)
|
179
176
|
domain_name = urlparse(url).path.replace("*", "")
|
180
|
-
self.log.
|
177
|
+
self.log.debug("domain_name: %s", domain_name)
|
181
178
|
if authority.endswith(domain_name):
|
182
179
|
return None
|
183
180
|
return proxies
|
@@ -193,10 +190,10 @@ class KiotaRequestAdapterHook(BaseHook):
|
|
193
190
|
client_id = connection.login
|
194
191
|
client_secret = connection.password
|
195
192
|
config = connection.extra_dejson if connection.extra else {}
|
196
|
-
tenant_id = config.get("tenant_id")
|
193
|
+
tenant_id = config.get("tenant_id") or config.get("tenantId")
|
197
194
|
api_version = self.get_api_version(config)
|
198
195
|
host = self.get_host(connection)
|
199
|
-
base_url = config.get("base_url", urljoin(host, api_version
|
196
|
+
base_url = config.get("base_url", urljoin(host, api_version))
|
200
197
|
authority = config.get("authority")
|
201
198
|
proxies = self.proxies or config.get("proxies", {})
|
202
199
|
msal_proxies = self.to_msal_proxies(authority=authority, proxies=proxies)
|
@@ -209,7 +206,7 @@ class KiotaRequestAdapterHook(BaseHook):
|
|
209
206
|
|
210
207
|
self.log.info(
|
211
208
|
"Creating Microsoft Graph SDK client %s for conn_id: %s",
|
212
|
-
api_version
|
209
|
+
api_version,
|
213
210
|
self.conn_id,
|
214
211
|
)
|
215
212
|
self.log.info("Host: %s", host)
|
@@ -217,7 +214,7 @@ class KiotaRequestAdapterHook(BaseHook):
|
|
217
214
|
self.log.info("Tenant id: %s", tenant_id)
|
218
215
|
self.log.info("Client id: %s", client_id)
|
219
216
|
self.log.info("Client secret: %s", client_secret)
|
220
|
-
self.log.info("API version: %s", api_version
|
217
|
+
self.log.info("API version: %s", api_version)
|
221
218
|
self.log.info("Scope: %s", scopes)
|
222
219
|
self.log.info("Verify: %s", verify)
|
223
220
|
self.log.info("Timeout: %s", self.timeout)
|
@@ -238,17 +235,17 @@ class KiotaRequestAdapterHook(BaseHook):
|
|
238
235
|
connection_verify=verify,
|
239
236
|
)
|
240
237
|
http_client = GraphClientFactory.create_with_default_middleware(
|
241
|
-
api_version=api_version,
|
238
|
+
api_version=api_version, # type: ignore
|
242
239
|
client=httpx.AsyncClient(
|
243
240
|
proxies=httpx_proxies,
|
244
241
|
timeout=Timeout(timeout=self.timeout),
|
245
242
|
verify=verify,
|
246
243
|
trust_env=trust_env,
|
247
244
|
),
|
248
|
-
host=host,
|
245
|
+
host=host, # type: ignore
|
249
246
|
)
|
250
247
|
auth_provider = AzureIdentityAuthenticationProvider(
|
251
|
-
credentials=credentials,
|
248
|
+
credentials=credentials, # type: ignore
|
252
249
|
scopes=scopes,
|
253
250
|
allowed_hosts=allowed_hosts,
|
254
251
|
)
|
@@ -295,7 +292,7 @@ class KiotaRequestAdapterHook(BaseHook):
|
|
295
292
|
error_map=self.error_mapping(),
|
296
293
|
)
|
297
294
|
|
298
|
-
self.log.
|
295
|
+
self.log.debug("response: %s", response)
|
299
296
|
|
300
297
|
return response
|
301
298
|
|
@@ -49,11 +49,9 @@ class WasbTaskHandler(FileTaskHandler, LoggingMixin):
|
|
49
49
|
base_log_folder: str,
|
50
50
|
wasb_log_folder: str,
|
51
51
|
wasb_container: str,
|
52
|
-
*,
|
53
|
-
filename_template: str | None = None,
|
54
52
|
**kwargs,
|
55
53
|
) -> None:
|
56
|
-
super().__init__(base_log_folder
|
54
|
+
super().__init__(base_log_folder)
|
57
55
|
self.handler: logging.FileHandler | None = None
|
58
56
|
self.wasb_container = wasb_container
|
59
57
|
self.remote_base = wasb_log_folder
|
@@ -16,7 +16,7 @@
|
|
16
16
|
# under the License.
|
17
17
|
from __future__ import annotations
|
18
18
|
|
19
|
-
from typing import TYPE_CHECKING, Any, Sequence
|
19
|
+
from typing import TYPE_CHECKING, Any, Callable, Sequence
|
20
20
|
|
21
21
|
from azure.core.exceptions import ResourceNotFoundError
|
22
22
|
|
@@ -26,10 +26,13 @@ from airflow.providers.microsoft.azure.hooks.asb import AdminClientHook, Message
|
|
26
26
|
if TYPE_CHECKING:
|
27
27
|
import datetime
|
28
28
|
|
29
|
+
from azure.servicebus import ServiceBusMessage
|
29
30
|
from azure.servicebus.management._models import AuthorizationRule
|
30
31
|
|
31
32
|
from airflow.utils.context import Context
|
32
33
|
|
34
|
+
MessageCallback = Callable[[ServiceBusMessage], None]
|
35
|
+
|
33
36
|
|
34
37
|
class AzureServiceBusCreateQueueOperator(BaseOperator):
|
35
38
|
"""
|
@@ -140,6 +143,9 @@ class AzureServiceBusReceiveMessageOperator(BaseOperator):
|
|
140
143
|
:param max_wait_time: Maximum time to wait in seconds for the first message to arrive.
|
141
144
|
:param azure_service_bus_conn_id: Reference to the
|
142
145
|
:ref: `Azure Service Bus connection <howto/connection:azure_service_bus>`.
|
146
|
+
:param message_callback: Optional callback to process each message. If not provided, then
|
147
|
+
the message will be logged and completed. If provided, and throws an exception, the
|
148
|
+
message will be abandoned for future redelivery.
|
143
149
|
"""
|
144
150
|
|
145
151
|
template_fields: Sequence[str] = ("queue_name",)
|
@@ -152,6 +158,7 @@ class AzureServiceBusReceiveMessageOperator(BaseOperator):
|
|
152
158
|
azure_service_bus_conn_id: str = "azure_service_bus_default",
|
153
159
|
max_message_count: int = 10,
|
154
160
|
max_wait_time: float = 5,
|
161
|
+
message_callback: MessageCallback | None = None,
|
155
162
|
**kwargs,
|
156
163
|
) -> None:
|
157
164
|
super().__init__(**kwargs)
|
@@ -159,6 +166,7 @@ class AzureServiceBusReceiveMessageOperator(BaseOperator):
|
|
159
166
|
self.azure_service_bus_conn_id = azure_service_bus_conn_id
|
160
167
|
self.max_message_count = max_message_count
|
161
168
|
self.max_wait_time = max_wait_time
|
169
|
+
self.message_callback = message_callback
|
162
170
|
|
163
171
|
def execute(self, context: Context) -> None:
|
164
172
|
"""Receive Message in specific queue in Service Bus namespace by connecting to Service Bus client."""
|
@@ -167,7 +175,10 @@ class AzureServiceBusReceiveMessageOperator(BaseOperator):
|
|
167
175
|
|
168
176
|
# Receive message
|
169
177
|
hook.receive_message(
|
170
|
-
self.queue_name,
|
178
|
+
self.queue_name,
|
179
|
+
max_message_count=self.max_message_count,
|
180
|
+
max_wait_time=self.max_wait_time,
|
181
|
+
message_callback=self.message_callback,
|
171
182
|
)
|
172
183
|
|
173
184
|
|
@@ -515,6 +526,9 @@ class ASBReceiveSubscriptionMessageOperator(BaseOperator):
|
|
515
526
|
an empty list will be returned.
|
516
527
|
:param azure_service_bus_conn_id: Reference to the
|
517
528
|
:ref:`Azure Service Bus connection <howto/connection:azure_service_bus>`.
|
529
|
+
:param message_callback: Optional callback to process each message. If not provided, then
|
530
|
+
the message will be logged and completed. If provided, and throws an exception, the
|
531
|
+
message will be abandoned for future redelivery.
|
518
532
|
"""
|
519
533
|
|
520
534
|
template_fields: Sequence[str] = ("topic_name", "subscription_name")
|
@@ -528,6 +542,7 @@ class ASBReceiveSubscriptionMessageOperator(BaseOperator):
|
|
528
542
|
max_message_count: int | None = 1,
|
529
543
|
max_wait_time: float | None = 5,
|
530
544
|
azure_service_bus_conn_id: str = "azure_service_bus_default",
|
545
|
+
message_callback: MessageCallback | None = None,
|
531
546
|
**kwargs,
|
532
547
|
) -> None:
|
533
548
|
super().__init__(**kwargs)
|
@@ -536,6 +551,7 @@ class ASBReceiveSubscriptionMessageOperator(BaseOperator):
|
|
536
551
|
self.max_message_count = max_message_count
|
537
552
|
self.max_wait_time = max_wait_time
|
538
553
|
self.azure_service_bus_conn_id = azure_service_bus_conn_id
|
554
|
+
self.message_callback = message_callback
|
539
555
|
|
540
556
|
def execute(self, context: Context) -> None:
|
541
557
|
"""Receive Message in specific queue in Service Bus namespace by connecting to Service Bus client."""
|
@@ -544,7 +560,11 @@ class ASBReceiveSubscriptionMessageOperator(BaseOperator):
|
|
544
560
|
|
545
561
|
# Receive message
|
546
562
|
hook.receive_subscription_message(
|
547
|
-
self.topic_name,
|
563
|
+
self.topic_name,
|
564
|
+
self.subscription_name,
|
565
|
+
self.max_message_count,
|
566
|
+
self.max_wait_time,
|
567
|
+
message_callback=self.message_callback,
|
548
568
|
)
|
549
569
|
|
550
570
|
|
@@ -99,7 +99,7 @@ class MSGraphAsyncOperator(BaseOperator):
|
|
99
99
|
key: str = XCOM_RETURN_KEY,
|
100
100
|
timeout: float | None = None,
|
101
101
|
proxies: dict | None = None,
|
102
|
-
api_version: APIVersion | None = None,
|
102
|
+
api_version: APIVersion | str | None = None,
|
103
103
|
pagination_function: Callable[[MSGraphAsyncOperator, dict], tuple[str, dict]] | None = None,
|
104
104
|
result_processor: Callable[[Context, Any], Any] = lambda context, result: result,
|
105
105
|
serializer: type[ResponseSerializer] = ResponseSerializer,
|
@@ -76,7 +76,7 @@ class PowerBIDatasetRefreshOperator(BaseOperator):
|
|
76
76
|
conn_id: str = PowerBIHook.default_conn_name,
|
77
77
|
timeout: float = 60 * 60 * 24 * 7,
|
78
78
|
proxies: dict | None = None,
|
79
|
-
api_version: APIVersion | None = None,
|
79
|
+
api_version: APIVersion | str | None = None,
|
80
80
|
check_interval: int = 60,
|
81
81
|
**kwargs,
|
82
82
|
) -> None:
|
@@ -89,6 +89,14 @@ class PowerBIDatasetRefreshOperator(BaseOperator):
|
|
89
89
|
self.timeout = timeout
|
90
90
|
self.check_interval = check_interval
|
91
91
|
|
92
|
+
@property
|
93
|
+
def proxies(self) -> dict | None:
|
94
|
+
return self.hook.proxies
|
95
|
+
|
96
|
+
@property
|
97
|
+
def api_version(self) -> str | None:
|
98
|
+
return self.hook.api_version
|
99
|
+
|
92
100
|
def execute(self, context: Context):
|
93
101
|
"""Refresh the Power BI Dataset."""
|
94
102
|
if self.wait_for_termination:
|
@@ -98,6 +106,8 @@ class PowerBIDatasetRefreshOperator(BaseOperator):
|
|
98
106
|
group_id=self.group_id,
|
99
107
|
dataset_id=self.dataset_id,
|
100
108
|
timeout=self.timeout,
|
109
|
+
proxies=self.proxies,
|
110
|
+
api_version=self.api_version,
|
101
111
|
check_interval=self.check_interval,
|
102
112
|
wait_for_termination=self.wait_for_termination,
|
103
113
|
),
|
@@ -21,7 +21,7 @@ from functools import cached_property
|
|
21
21
|
from typing import TYPE_CHECKING, Sequence
|
22
22
|
|
23
23
|
from airflow.configuration import conf
|
24
|
-
from airflow.exceptions import AirflowException
|
24
|
+
from airflow.exceptions import AirflowException
|
25
25
|
from airflow.providers.microsoft.azure.hooks.data_factory import (
|
26
26
|
AzureDataFactoryHook,
|
27
27
|
AzureDataFactoryPipelineRunException,
|
@@ -85,17 +85,11 @@ class AzureDataFactoryPipelineRunStatusSensor(BaseSensorOperator):
|
|
85
85
|
)
|
86
86
|
|
87
87
|
if pipeline_run_status == AzureDataFactoryPipelineRunStatus.FAILED:
|
88
|
-
# TODO: remove this if check when min_airflow_version is set to higher than 2.7.1
|
89
88
|
message = f"Pipeline run {self.run_id} has failed."
|
90
|
-
if self.soft_fail:
|
91
|
-
raise AirflowSkipException(message)
|
92
89
|
raise AzureDataFactoryPipelineRunException(message)
|
93
90
|
|
94
91
|
if pipeline_run_status == AzureDataFactoryPipelineRunStatus.CANCELLED:
|
95
|
-
# TODO: remove this if check when min_airflow_version is set to higher than 2.7.1
|
96
92
|
message = f"Pipeline run {self.run_id} has been cancelled."
|
97
|
-
if self.soft_fail:
|
98
|
-
raise AirflowSkipException(message)
|
99
93
|
raise AzureDataFactoryPipelineRunException(message)
|
100
94
|
|
101
95
|
return pipeline_run_status == AzureDataFactoryPipelineRunStatus.SUCCEEDED
|
@@ -131,9 +125,6 @@ class AzureDataFactoryPipelineRunStatusSensor(BaseSensorOperator):
|
|
131
125
|
"""
|
132
126
|
if event:
|
133
127
|
if event["status"] == "error":
|
134
|
-
# TODO: remove this if check when min_airflow_version is set to higher than 2.7.1
|
135
|
-
if self.soft_fail:
|
136
|
-
raise AirflowSkipException(event["message"])
|
137
128
|
raise AirflowException(event["message"])
|
138
129
|
self.log.info(event["message"])
|
139
130
|
return None
|
@@ -82,7 +82,7 @@ class MSGraphSensor(BaseSensorOperator):
|
|
82
82
|
data: dict[str, Any] | str | BytesIO | None = None,
|
83
83
|
conn_id: str = KiotaRequestAdapterHook.default_conn_name,
|
84
84
|
proxies: dict | None = None,
|
85
|
-
api_version: APIVersion | None = None,
|
85
|
+
api_version: APIVersion | str | None = None,
|
86
86
|
event_processor: Callable[[Context, Any], bool] = lambda context, e: e.get("status") == "Succeeded",
|
87
87
|
result_processor: Callable[[Context, Any], Any] = lambda context, result: result,
|
88
88
|
serializer: type[ResponseSerializer] = ResponseSerializer,
|
@@ -23,7 +23,7 @@ from typing import TYPE_CHECKING, Any, Sequence
|
|
23
23
|
from deprecated import deprecated
|
24
24
|
|
25
25
|
from airflow.configuration import conf
|
26
|
-
from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
|
26
|
+
from airflow.exceptions import AirflowException, AirflowProviderDeprecationWarning
|
27
27
|
from airflow.providers.microsoft.azure.hooks.wasb import WasbHook
|
28
28
|
from airflow.providers.microsoft.azure.triggers.wasb import WasbBlobSensorTrigger, WasbPrefixSensorTrigger
|
29
29
|
from airflow.sensors.base import BaseSensorOperator
|
@@ -104,9 +104,6 @@ class WasbBlobSensor(BaseSensorOperator):
|
|
104
104
|
"""
|
105
105
|
if event:
|
106
106
|
if event["status"] == "error":
|
107
|
-
# TODO: remove this if check when min_airflow_version is set to higher than 2.7.1
|
108
|
-
if self.soft_fail:
|
109
|
-
raise AirflowSkipException(event["message"])
|
110
107
|
raise AirflowException(event["message"])
|
111
108
|
self.log.info(event["message"])
|
112
109
|
else:
|
@@ -216,9 +213,6 @@ class WasbPrefixSensor(BaseSensorOperator):
|
|
216
213
|
"""
|
217
214
|
if event:
|
218
215
|
if event["status"] == "error":
|
219
|
-
# TODO: remove this if check when min_airflow_version is set to higher than 2.7.1
|
220
|
-
if self.soft_fail:
|
221
|
-
raise AirflowSkipException(event["message"])
|
222
216
|
raise AirflowException(event["message"])
|
223
217
|
self.log.info(event["message"])
|
224
218
|
else:
|
@@ -122,7 +122,7 @@ class MSGraphTrigger(BaseTrigger):
|
|
122
122
|
conn_id: str = KiotaRequestAdapterHook.default_conn_name,
|
123
123
|
timeout: float | None = None,
|
124
124
|
proxies: dict | None = None,
|
125
|
-
api_version: APIVersion | None = None,
|
125
|
+
api_version: APIVersion | str | None = None,
|
126
126
|
serializer: type[ResponseSerializer] = ResponseSerializer,
|
127
127
|
):
|
128
128
|
super().__init__()
|
@@ -152,14 +152,13 @@ class MSGraphTrigger(BaseTrigger):
|
|
152
152
|
|
153
153
|
def serialize(self) -> tuple[str, dict[str, Any]]:
|
154
154
|
"""Serialize the HttpTrigger arguments and classpath."""
|
155
|
-
api_version = self.api_version.value if self.api_version else None
|
156
155
|
return (
|
157
156
|
f"{self.__class__.__module__}.{self.__class__.__name__}",
|
158
157
|
{
|
159
158
|
"conn_id": self.conn_id,
|
160
159
|
"timeout": self.timeout,
|
161
160
|
"proxies": self.proxies,
|
162
|
-
"api_version": api_version,
|
161
|
+
"api_version": self.api_version,
|
163
162
|
"serializer": f"{self.serializer.__class__.__module__}.{self.serializer.__class__.__name__}",
|
164
163
|
"url": self.url,
|
165
164
|
"path_parameters": self.path_parameters,
|
@@ -188,7 +187,7 @@ class MSGraphTrigger(BaseTrigger):
|
|
188
187
|
return self.hook.proxies
|
189
188
|
|
190
189
|
@property
|
191
|
-
def api_version(self) -> APIVersion:
|
190
|
+
def api_version(self) -> APIVersion | str:
|
192
191
|
return self.hook.api_version
|
193
192
|
|
194
193
|
async def run(self) -> AsyncIterator[TriggerEvent]:
|
@@ -58,7 +58,7 @@ class PowerBITrigger(BaseTrigger):
|
|
58
58
|
group_id: str,
|
59
59
|
timeout: float = 60 * 60 * 24 * 7,
|
60
60
|
proxies: dict | None = None,
|
61
|
-
api_version: APIVersion | None = None,
|
61
|
+
api_version: APIVersion | str | None = None,
|
62
62
|
check_interval: int = 60,
|
63
63
|
wait_for_termination: bool = True,
|
64
64
|
):
|
@@ -72,13 +72,12 @@ class PowerBITrigger(BaseTrigger):
|
|
72
72
|
|
73
73
|
def serialize(self):
|
74
74
|
"""Serialize the trigger instance."""
|
75
|
-
api_version = self.api_version.value if self.api_version else None
|
76
75
|
return (
|
77
76
|
"airflow.providers.microsoft.azure.triggers.powerbi.PowerBITrigger",
|
78
77
|
{
|
79
78
|
"conn_id": self.conn_id,
|
80
79
|
"proxies": self.proxies,
|
81
|
-
"api_version": api_version,
|
80
|
+
"api_version": self.api_version,
|
82
81
|
"dataset_id": self.dataset_id,
|
83
82
|
"group_id": self.group_id,
|
84
83
|
"timeout": self.timeout,
|
@@ -96,7 +95,7 @@ class PowerBITrigger(BaseTrigger):
|
|
96
95
|
return self.hook.proxies
|
97
96
|
|
98
97
|
@property
|
99
|
-
def api_version(self) -> APIVersion:
|
98
|
+
def api_version(self) -> APIVersion | str:
|
100
99
|
return self.hook.api_version
|
101
100
|
|
102
101
|
async def run(self) -> AsyncIterator[TriggerEvent]:
|
@@ -28,7 +28,7 @@ build-backend = "flit_core.buildapi"
|
|
28
28
|
|
29
29
|
[project]
|
30
30
|
name = "apache-airflow-providers-microsoft-azure"
|
31
|
-
version = "10.
|
31
|
+
version = "10.5.0.rc1"
|
32
32
|
description = "Provider package apache-airflow-providers-microsoft-azure for Apache Airflow"
|
33
33
|
readme = "README.rst"
|
34
34
|
authors = [
|
@@ -82,8 +82,8 @@ dependencies = [
|
|
82
82
|
]
|
83
83
|
|
84
84
|
[project.urls]
|
85
|
-
"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.
|
86
|
-
"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.
|
85
|
+
"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.5.0"
|
86
|
+
"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/10.5.0/changelog.html"
|
87
87
|
"Bug Tracker" = "https://github.com/apache/airflow/issues"
|
88
88
|
"Source Code" = "https://github.com/apache/airflow"
|
89
89
|
"Slack Chat" = "https://s.apache.org/airflow-slack"
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|