apache-airflow-providers-microsoft-azure 12.3.1rc1__py3-none-any.whl → 12.4.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/microsoft/azure/__init__.py +3 -3
- airflow/providers/microsoft/azure/hooks/powerbi.py +49 -1
- airflow/providers/microsoft/azure/operators/container_instances.py +4 -0
- airflow/providers/microsoft/azure/operators/msgraph.py +22 -5
- airflow/providers/microsoft/azure/operators/powerbi.py +143 -1
- airflow/providers/microsoft/azure/triggers/powerbi.py +160 -0
- airflow/providers/microsoft/azure/version_compat.py +0 -1
- {apache_airflow_providers_microsoft_azure-12.3.1rc1.dist-info → apache_airflow_providers_microsoft_azure-12.4.0rc1.dist-info}/METADATA +48 -44
- {apache_airflow_providers_microsoft_azure-12.3.1rc1.dist-info → apache_airflow_providers_microsoft_azure-12.4.0rc1.dist-info}/RECORD +11 -11
- {apache_airflow_providers_microsoft_azure-12.3.1rc1.dist-info → apache_airflow_providers_microsoft_azure-12.4.0rc1.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_microsoft_azure-12.3.1rc1.dist-info → apache_airflow_providers_microsoft_azure-12.4.0rc1.dist-info}/entry_points.txt +0 -0
@@ -29,11 +29,11 @@ from airflow import __version__ as airflow_version
|
|
29
29
|
|
30
30
|
__all__ = ["__version__"]
|
31
31
|
|
32
|
-
__version__ = "12.
|
32
|
+
__version__ = "12.4.0"
|
33
33
|
|
34
34
|
if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse(
|
35
|
-
"2.
|
35
|
+
"2.10.0"
|
36
36
|
):
|
37
37
|
raise RuntimeError(
|
38
|
-
f"The package `apache-airflow-providers-microsoft-azure:{__version__}` needs Apache Airflow 2.
|
38
|
+
f"The package `apache-airflow-providers-microsoft-azure:{__version__}` needs Apache Airflow 2.10.0+"
|
39
39
|
)
|
@@ -51,11 +51,25 @@ class PowerBIDatasetRefreshException(AirflowException):
|
|
51
51
|
"""An exception that indicates a dataset refresh failed to complete."""
|
52
52
|
|
53
53
|
|
54
|
+
class PowerBIWorkspaceListException(AirflowException):
|
55
|
+
"""An exception that indicates a failure in getting the list of groups (workspaces)."""
|
56
|
+
|
57
|
+
|
58
|
+
class PowerBIDatasetListException(AirflowException):
|
59
|
+
"""An exception that indicates a failure in getting the list of datasets."""
|
60
|
+
|
61
|
+
|
54
62
|
class PowerBIHook(KiotaRequestAdapterHook):
|
55
63
|
"""
|
56
64
|
A async hook to interact with Power BI.
|
57
65
|
|
58
|
-
:param conn_id: The
|
66
|
+
:param conn_id: The connection Id to connect to PowerBI.
|
67
|
+
:param timeout: The HTTP timeout being used by the `KiotaRequestAdapter` (default is None).
|
68
|
+
When no timeout is specified or set to None then there is no HTTP timeout on each request.
|
69
|
+
:param proxies: A dict defining the HTTP proxies to be used (default is None).
|
70
|
+
:param api_version: The API version of the Microsoft Graph API to be used (default is v1).
|
71
|
+
You can pass an enum named APIVersion which has 2 possible members v1 and beta,
|
72
|
+
or you can pass a string as `v1.0` or `beta`.
|
59
73
|
"""
|
60
74
|
|
61
75
|
conn_type: str = "powerbi"
|
@@ -200,6 +214,40 @@ class PowerBIHook(KiotaRequestAdapterHook):
|
|
200
214
|
except AirflowException:
|
201
215
|
raise PowerBIDatasetRefreshException("Failed to trigger dataset refresh.")
|
202
216
|
|
217
|
+
async def get_workspace_list(self) -> list[str]:
|
218
|
+
"""
|
219
|
+
Triggers a request to get all available workspaces for the service principal.
|
220
|
+
|
221
|
+
:return: List of workspace IDs.
|
222
|
+
"""
|
223
|
+
try:
|
224
|
+
response = await self.run(url="myorg/groups", method="GET")
|
225
|
+
|
226
|
+
list_of_workspaces = response.get("value", [])
|
227
|
+
|
228
|
+
return [ws["id"] for ws in list_of_workspaces if "id" in ws]
|
229
|
+
|
230
|
+
except AirflowException:
|
231
|
+
raise PowerBIWorkspaceListException("Failed to get workspace ID list.")
|
232
|
+
|
233
|
+
async def get_dataset_list(self, *, group_id: str) -> list[str]:
|
234
|
+
"""
|
235
|
+
Triggers a request to get all datasets within a group (workspace).
|
236
|
+
|
237
|
+
:param group_id: Workspace ID.
|
238
|
+
|
239
|
+
:return: List of dataset IDs.
|
240
|
+
"""
|
241
|
+
try:
|
242
|
+
response = await self.run(url=f"myorg/groups/{group_id}/datasets", method="GET")
|
243
|
+
|
244
|
+
list_of_datasets = response.get("value", [])
|
245
|
+
|
246
|
+
return [ds["id"] for ds in list_of_datasets if "id" in ds]
|
247
|
+
|
248
|
+
except AirflowException:
|
249
|
+
raise PowerBIDatasetListException("Failed to get dataset ID list.")
|
250
|
+
|
203
251
|
async def cancel_dataset_refresh(self, dataset_id: str, group_id: str, dataset_refresh_id: str) -> None:
|
204
252
|
"""
|
205
253
|
Cancel the dataset refresh.
|
@@ -388,6 +388,10 @@ class AzureContainerInstancesOperator(BaseOperator):
|
|
388
388
|
self.log.info("Container exited with detail_status %s", detail_status)
|
389
389
|
return exit_code
|
390
390
|
|
391
|
+
if state == "Unhealthy":
|
392
|
+
self.log.error("Azure provision unhealthy")
|
393
|
+
return 1
|
394
|
+
|
391
395
|
if state == "Failed":
|
392
396
|
self.log.error("Azure provision failure")
|
393
397
|
return 1
|
@@ -19,6 +19,7 @@ from __future__ import annotations
|
|
19
19
|
|
20
20
|
import warnings
|
21
21
|
from collections.abc import Sequence
|
22
|
+
from contextlib import suppress
|
22
23
|
from copy import deepcopy
|
23
24
|
from typing import (
|
24
25
|
TYPE_CHECKING,
|
@@ -58,7 +59,16 @@ def execute_callable(
|
|
58
59
|
message: str,
|
59
60
|
) -> Any:
|
60
61
|
try:
|
61
|
-
|
62
|
+
with warnings.catch_warnings():
|
63
|
+
with suppress(AttributeError, ImportError):
|
64
|
+
from airflow.utils.context import ( # type: ignore[attr-defined]
|
65
|
+
AirflowContextDeprecationWarning,
|
66
|
+
)
|
67
|
+
|
68
|
+
warnings.filterwarnings("ignore", category=AirflowContextDeprecationWarning)
|
69
|
+
warnings.simplefilter("ignore", category=DeprecationWarning)
|
70
|
+
warnings.simplefilter("ignore", category=UserWarning)
|
71
|
+
return func(value, **context) # type: ignore
|
62
72
|
except TypeError:
|
63
73
|
warnings.warn(
|
64
74
|
message,
|
@@ -216,8 +226,6 @@ class MSGraphAsyncOperator(BaseOperator):
|
|
216
226
|
|
217
227
|
self.log.debug("processed response: %s", result)
|
218
228
|
|
219
|
-
event["response"] = result
|
220
|
-
|
221
229
|
try:
|
222
230
|
self.trigger_next_link(
|
223
231
|
response=response, method_name=self.execute_complete.__name__, context=context
|
@@ -293,7 +301,13 @@ class MSGraphAsyncOperator(BaseOperator):
|
|
293
301
|
def push_xcom(self, context: Any, value) -> None:
|
294
302
|
self.log.debug("do_xcom_push: %s", self.do_xcom_push)
|
295
303
|
if self.do_xcom_push:
|
296
|
-
self.log.info(
|
304
|
+
self.log.info(
|
305
|
+
"Pushing XCom with task_id '%s' and dag_id '%s' and key '%s': %s",
|
306
|
+
self.task_id,
|
307
|
+
self.dag_id,
|
308
|
+
self.key,
|
309
|
+
value,
|
310
|
+
)
|
297
311
|
self.xcom_push(context=context, key=self.key, value=value)
|
298
312
|
|
299
313
|
@staticmethod
|
@@ -316,7 +330,10 @@ class MSGraphAsyncOperator(BaseOperator):
|
|
316
330
|
def trigger_next_link(self, response, method_name: str, context: Context) -> None:
|
317
331
|
if isinstance(response, dict):
|
318
332
|
try:
|
319
|
-
|
333
|
+
with warnings.catch_warnings():
|
334
|
+
warnings.simplefilter("ignore", category=DeprecationWarning)
|
335
|
+
warnings.filterwarnings("ignore", category=UserWarning)
|
336
|
+
url, query_parameters = self.pagination_function(self, response, **context) # type: ignore
|
320
337
|
except TypeError:
|
321
338
|
warnings.warn(
|
322
339
|
"pagination_function signature has changed, context parameter should be a kwargs argument!",
|
@@ -23,7 +23,11 @@ from typing import TYPE_CHECKING, Any
|
|
23
23
|
from airflow.exceptions import AirflowException
|
24
24
|
from airflow.models import BaseOperator
|
25
25
|
from airflow.providers.microsoft.azure.hooks.powerbi import PowerBIHook
|
26
|
-
from airflow.providers.microsoft.azure.triggers.powerbi import
|
26
|
+
from airflow.providers.microsoft.azure.triggers.powerbi import (
|
27
|
+
PowerBIDatasetListTrigger,
|
28
|
+
PowerBITrigger,
|
29
|
+
PowerBIWorkspaceListTrigger,
|
30
|
+
)
|
27
31
|
|
28
32
|
if TYPE_CHECKING:
|
29
33
|
from msgraph_core import APIVersion
|
@@ -167,3 +171,141 @@ class PowerBIDatasetRefreshOperator(BaseOperator):
|
|
167
171
|
)
|
168
172
|
if event["status"] == "error":
|
169
173
|
raise AirflowException(event["message"])
|
174
|
+
|
175
|
+
|
176
|
+
class PowerBIWorkspaceListOperator(BaseOperator):
|
177
|
+
"""
|
178
|
+
Gets a list of workspaces where the service principal from the connection is assigned as admin.
|
179
|
+
|
180
|
+
.. seealso::
|
181
|
+
For more information on how to use this operator, take a look at the guide:
|
182
|
+
:ref:`howto/operator:PowerBIWorkspaceListOperator`
|
183
|
+
|
184
|
+
:param conn_id: The connection Id to connect to PowerBI.
|
185
|
+
:param timeout: The HTTP timeout being used by the `KiotaRequestAdapter`. Default is 1 week (60s * 60m * 24h * 7d).
|
186
|
+
When no timeout is specified or set to None then there is no HTTP timeout on each request.
|
187
|
+
:param proxies: A dict defining the HTTP proxies to be used (default is None).
|
188
|
+
:param api_version: The API version of the Microsoft Graph API to be used (default is v1).
|
189
|
+
You can pass an enum named APIVersion which has 2 possible members v1 and beta,
|
190
|
+
or you can pass a string as `v1.0` or `beta`.
|
191
|
+
"""
|
192
|
+
|
193
|
+
def __init__(
|
194
|
+
self,
|
195
|
+
*,
|
196
|
+
conn_id: str = PowerBIHook.default_conn_name,
|
197
|
+
timeout: float = 60 * 60 * 24 * 7,
|
198
|
+
proxies: dict | None = None,
|
199
|
+
api_version: APIVersion | str | None = None,
|
200
|
+
**kwargs,
|
201
|
+
) -> None:
|
202
|
+
super().__init__(**kwargs)
|
203
|
+
self.hook = PowerBIHook(conn_id=conn_id, proxies=proxies, api_version=api_version, timeout=timeout)
|
204
|
+
self.conn_id = conn_id
|
205
|
+
self.timeout = timeout
|
206
|
+
|
207
|
+
@property
|
208
|
+
def proxies(self) -> dict | None:
|
209
|
+
return self.hook.proxies
|
210
|
+
|
211
|
+
@property
|
212
|
+
def api_version(self) -> str | None:
|
213
|
+
return self.hook.api_version
|
214
|
+
|
215
|
+
def execute(self, context: Context):
|
216
|
+
"""List visible PowerBI Workspaces."""
|
217
|
+
self.defer(
|
218
|
+
trigger=PowerBIWorkspaceListTrigger(
|
219
|
+
conn_id=self.conn_id,
|
220
|
+
timeout=self.timeout,
|
221
|
+
proxies=self.proxies,
|
222
|
+
api_version=self.api_version,
|
223
|
+
),
|
224
|
+
method_name=self.execute_complete.__name__,
|
225
|
+
)
|
226
|
+
|
227
|
+
def execute_complete(self, context: Context, event: dict[str, str]) -> Any:
|
228
|
+
"""
|
229
|
+
Return immediately - callback for when the trigger fires.
|
230
|
+
|
231
|
+
Relies on trigger to throw an exception, otherwise it assumes execution was successful.
|
232
|
+
"""
|
233
|
+
if event:
|
234
|
+
self.xcom_push(
|
235
|
+
context=context,
|
236
|
+
key=f"{self.task_id}.powerbi_workspace_ids",
|
237
|
+
value=event["workspace_ids"],
|
238
|
+
)
|
239
|
+
if event["status"] == "error":
|
240
|
+
raise AirflowException(event["message"])
|
241
|
+
|
242
|
+
|
243
|
+
class PowerBIDatasetListOperator(BaseOperator):
|
244
|
+
"""
|
245
|
+
Gets a list of datasets where the service principal from the connection is assigned as admin.
|
246
|
+
|
247
|
+
.. seealso::
|
248
|
+
For more information on how to use this operator, take a look at the guide:
|
249
|
+
:ref:`howto/operator:PowerBIDatasetListOperator`
|
250
|
+
|
251
|
+
:param conn_id: The connection Id to connect to PowerBI.
|
252
|
+
:param group_id: The group Id to list discoverable datasets.
|
253
|
+
:param timeout: The HTTP timeout being used by the `KiotaRequestAdapter`. Default is 1 week (60s * 60m * 24h * 7d).
|
254
|
+
When no timeout is specified or set to None then there is no HTTP timeout on each request.
|
255
|
+
:param proxies: A dict defining the HTTP proxies to be used (default is None).
|
256
|
+
:param api_version: The API version of the Microsoft Graph API to be used (default is v1).
|
257
|
+
You can pass an enum named APIVersion which has 2 possible members v1 and beta,
|
258
|
+
or you can pass a string as `v1.0` or `beta`.
|
259
|
+
"""
|
260
|
+
|
261
|
+
def __init__(
|
262
|
+
self,
|
263
|
+
*,
|
264
|
+
group_id: str,
|
265
|
+
conn_id: str = PowerBIHook.default_conn_name,
|
266
|
+
timeout: float = 60 * 60 * 24 * 7,
|
267
|
+
proxies: dict | None = None,
|
268
|
+
api_version: APIVersion | str | None = None,
|
269
|
+
**kwargs,
|
270
|
+
) -> None:
|
271
|
+
super().__init__(**kwargs)
|
272
|
+
self.hook = PowerBIHook(conn_id=conn_id, proxies=proxies, api_version=api_version, timeout=timeout)
|
273
|
+
self.conn_id = conn_id
|
274
|
+
self.group_id = group_id
|
275
|
+
self.timeout = timeout
|
276
|
+
|
277
|
+
@property
|
278
|
+
def proxies(self) -> dict | None:
|
279
|
+
return self.hook.proxies
|
280
|
+
|
281
|
+
@property
|
282
|
+
def api_version(self) -> str | None:
|
283
|
+
return self.hook.api_version
|
284
|
+
|
285
|
+
def execute(self, context: Context):
|
286
|
+
"""List visible PowerBI datasets within group (Workspace)."""
|
287
|
+
self.defer(
|
288
|
+
trigger=PowerBIDatasetListTrigger(
|
289
|
+
conn_id=self.conn_id,
|
290
|
+
timeout=self.timeout,
|
291
|
+
proxies=self.proxies,
|
292
|
+
api_version=self.api_version,
|
293
|
+
group_id=self.group_id,
|
294
|
+
),
|
295
|
+
method_name=self.execute_complete.__name__,
|
296
|
+
)
|
297
|
+
|
298
|
+
def execute_complete(self, context: Context, event: dict[str, str]) -> Any:
|
299
|
+
"""
|
300
|
+
Return immediately - callback for when the trigger fires.
|
301
|
+
|
302
|
+
Relies on trigger to throw an exception, otherwise it assumes execution was successful.
|
303
|
+
"""
|
304
|
+
if event:
|
305
|
+
self.xcom_push(
|
306
|
+
context=context,
|
307
|
+
key=f"{self.task_id}.powerbi_dataset_ids",
|
308
|
+
value=event["dataset_ids"],
|
309
|
+
)
|
310
|
+
if event["status"] == "error":
|
311
|
+
raise AirflowException(event["message"])
|
@@ -230,3 +230,163 @@ class PowerBITrigger(BaseTrigger):
|
|
230
230
|
"dataset_refresh_id": self.dataset_refresh_id,
|
231
231
|
}
|
232
232
|
)
|
233
|
+
|
234
|
+
|
235
|
+
class PowerBIWorkspaceListTrigger(BaseTrigger):
|
236
|
+
"""
|
237
|
+
Triggers a call to the API to request the available workspace IDs.
|
238
|
+
|
239
|
+
:param conn_id: The connection Id to connect to PowerBI.
|
240
|
+
:param timeout: The HTTP timeout being used by the `KiotaRequestAdapter`. Default is 1 week (60s * 60m * 24h * 7d).
|
241
|
+
When no timeout is specified or set to None then there is no HTTP timeout on each request.
|
242
|
+
:param proxies: A dict defining the HTTP proxies to be used (default is None).
|
243
|
+
:param api_version: The API version of the Microsoft Graph API to be used (default is v1).
|
244
|
+
You can pass an enum named APIVersion which has 2 possible members v1 and beta,
|
245
|
+
or you can pass a string as `v1.0` or `beta`.
|
246
|
+
"""
|
247
|
+
|
248
|
+
def __init__(
|
249
|
+
self,
|
250
|
+
conn_id: str,
|
251
|
+
workspace_ids: list[str] | None = None,
|
252
|
+
timeout: float = 60 * 60 * 24 * 7,
|
253
|
+
proxies: dict | None = None,
|
254
|
+
api_version: APIVersion | str | None = None,
|
255
|
+
):
|
256
|
+
super().__init__()
|
257
|
+
self.hook = PowerBIHook(conn_id=conn_id, proxies=proxies, api_version=api_version, timeout=timeout)
|
258
|
+
self.timeout = timeout
|
259
|
+
self.workspace_ids = workspace_ids
|
260
|
+
|
261
|
+
def serialize(self):
|
262
|
+
"""Serialize the trigger instance."""
|
263
|
+
return (
|
264
|
+
"airflow.providers.microsoft.azure.triggers.powerbi.PowerBIWorkspaceListTrigger",
|
265
|
+
{
|
266
|
+
"conn_id": self.conn_id,
|
267
|
+
"proxies": self.proxies,
|
268
|
+
"api_version": self.api_version,
|
269
|
+
"timeout": self.timeout,
|
270
|
+
"workspace_ids": self.workspace_ids,
|
271
|
+
},
|
272
|
+
)
|
273
|
+
|
274
|
+
@property
|
275
|
+
def conn_id(self) -> str:
|
276
|
+
return self.hook.conn_id
|
277
|
+
|
278
|
+
@property
|
279
|
+
def proxies(self) -> dict | None:
|
280
|
+
return self.hook.proxies
|
281
|
+
|
282
|
+
@property
|
283
|
+
def api_version(self) -> APIVersion | str:
|
284
|
+
return self.hook.api_version
|
285
|
+
|
286
|
+
async def run(self) -> AsyncIterator[TriggerEvent]:
|
287
|
+
"""Make async connection to the PowerBI and polls for the list of workspace IDs."""
|
288
|
+
# Trigger the API to get the workspace list
|
289
|
+
workspace_ids = await self.hook.get_workspace_list()
|
290
|
+
|
291
|
+
if workspace_ids:
|
292
|
+
self.log.info("Triggered request to get workspace list.")
|
293
|
+
yield TriggerEvent(
|
294
|
+
{
|
295
|
+
"status": "success",
|
296
|
+
"message": "The workspace list get request has been successful.",
|
297
|
+
"workspace_ids": workspace_ids,
|
298
|
+
}
|
299
|
+
)
|
300
|
+
return
|
301
|
+
|
302
|
+
yield TriggerEvent(
|
303
|
+
{
|
304
|
+
"status": "error",
|
305
|
+
"message": "Error grabbing the workspace list.",
|
306
|
+
"workspace_ids": None,
|
307
|
+
}
|
308
|
+
)
|
309
|
+
return
|
310
|
+
|
311
|
+
|
312
|
+
class PowerBIDatasetListTrigger(BaseTrigger):
|
313
|
+
"""
|
314
|
+
Triggers a call to the API to request the available dataset IDs.
|
315
|
+
|
316
|
+
:param conn_id: The connection Id to connect to PowerBI.
|
317
|
+
:param group_id: The group Id to list discoverable datasets.
|
318
|
+
:param timeout: The HTTP timeout being used by the `KiotaRequestAdapter`. Default is 1 week (60s * 60m * 24h * 7d).
|
319
|
+
When no timeout is specified or set to None then there is no HTTP timeout on each request.
|
320
|
+
:param proxies: A dict defining the HTTP proxies to be used (default is None).
|
321
|
+
:param api_version: The API version of the Microsoft Graph API to be used (default is v1).
|
322
|
+
You can pass an enum named APIVersion which has 2 possible members v1 and beta,
|
323
|
+
or you can pass a string as `v1.0` or `beta`.
|
324
|
+
"""
|
325
|
+
|
326
|
+
def __init__(
|
327
|
+
self,
|
328
|
+
conn_id: str,
|
329
|
+
group_id: str,
|
330
|
+
dataset_ids: list[str] | None = None,
|
331
|
+
timeout: float = 60 * 60 * 24 * 7,
|
332
|
+
proxies: dict | None = None,
|
333
|
+
api_version: APIVersion | str | None = None,
|
334
|
+
):
|
335
|
+
super().__init__()
|
336
|
+
self.hook = PowerBIHook(conn_id=conn_id, proxies=proxies, api_version=api_version, timeout=timeout)
|
337
|
+
self.timeout = timeout
|
338
|
+
self.group_id = group_id
|
339
|
+
self.dataset_ids = dataset_ids
|
340
|
+
|
341
|
+
def serialize(self):
|
342
|
+
"""Serialize the trigger instance."""
|
343
|
+
return (
|
344
|
+
"airflow.providers.microsoft.azure.triggers.powerbi.PowerBIDatasetListTrigger",
|
345
|
+
{
|
346
|
+
"conn_id": self.conn_id,
|
347
|
+
"proxies": self.proxies,
|
348
|
+
"api_version": self.api_version,
|
349
|
+
"timeout": self.timeout,
|
350
|
+
"group_id": self.group_id,
|
351
|
+
"dataset_ids": self.dataset_ids,
|
352
|
+
},
|
353
|
+
)
|
354
|
+
|
355
|
+
@property
|
356
|
+
def conn_id(self) -> str:
|
357
|
+
return self.hook.conn_id
|
358
|
+
|
359
|
+
@property
|
360
|
+
def proxies(self) -> dict | None:
|
361
|
+
return self.hook.proxies
|
362
|
+
|
363
|
+
@property
|
364
|
+
def api_version(self) -> APIVersion | str:
|
365
|
+
return self.hook.api_version
|
366
|
+
|
367
|
+
async def run(self) -> AsyncIterator[TriggerEvent]:
|
368
|
+
"""Make async connection to the PowerBI and polls for the list of dataset IDs."""
|
369
|
+
# Trigger the API to get the dataset list
|
370
|
+
dataset_ids = await self.hook.get_dataset_list(
|
371
|
+
group_id=self.group_id,
|
372
|
+
)
|
373
|
+
|
374
|
+
if dataset_ids:
|
375
|
+
self.log.info("Triggered request to get dataset list.")
|
376
|
+
yield TriggerEvent(
|
377
|
+
{
|
378
|
+
"status": "success",
|
379
|
+
"message": f"The dataset list get request from workspace {self.group_id} has been successful.",
|
380
|
+
"dataset_ids": dataset_ids,
|
381
|
+
}
|
382
|
+
)
|
383
|
+
return
|
384
|
+
|
385
|
+
yield TriggerEvent(
|
386
|
+
{
|
387
|
+
"status": "error",
|
388
|
+
"message": "Error grabbing the dataset list.",
|
389
|
+
"dataset_ids": None,
|
390
|
+
}
|
391
|
+
)
|
392
|
+
return
|
@@ -32,5 +32,4 @@ def get_base_airflow_version_tuple() -> tuple[int, int, int]:
|
|
32
32
|
return airflow_version.major, airflow_version.minor, airflow_version.micro
|
33
33
|
|
34
34
|
|
35
|
-
AIRFLOW_V_2_10_PLUS = get_base_airflow_version_tuple() >= (2, 10, 0)
|
36
35
|
AIRFLOW_V_3_0_PLUS = get_base_airflow_version_tuple() >= (3, 0, 0)
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: apache-airflow-providers-microsoft-azure
|
3
|
-
Version: 12.
|
3
|
+
Version: 12.4.0rc1
|
4
4
|
Summary: Provider package apache-airflow-providers-microsoft-azure for Apache Airflow
|
5
5
|
Keywords: airflow-provider,microsoft.azure,airflow,integration
|
6
6
|
Author-email: Apache Software Foundation <dev@airflow.apache.org>
|
@@ -20,7 +20,7 @@ Classifier: Programming Language :: Python :: 3.10
|
|
20
20
|
Classifier: Programming Language :: Python :: 3.11
|
21
21
|
Classifier: Programming Language :: Python :: 3.12
|
22
22
|
Classifier: Topic :: System :: Monitoring
|
23
|
-
Requires-Dist: apache-airflow>=2.
|
23
|
+
Requires-Dist: apache-airflow>=2.10.0rc1
|
24
24
|
Requires-Dist: adlfs>=2023.10.0
|
25
25
|
Requires-Dist: azure-batch>=8.0.0
|
26
26
|
Requires-Dist: azure-cosmos>=4.6.0
|
@@ -44,18 +44,20 @@ Requires-Dist: azure-mgmt-containerregistry>=8.0.0
|
|
44
44
|
Requires-Dist: azure-mgmt-containerinstance>=10.1.0
|
45
45
|
Requires-Dist: flask-appbuilder>=4.0.0
|
46
46
|
Requires-Dist: msgraph-core>=1.3.3
|
47
|
-
Requires-Dist: microsoft-kiota-http>=1.
|
48
|
-
Requires-Dist: microsoft-kiota-serialization-json>=1.
|
49
|
-
Requires-Dist: microsoft-kiota-serialization-text>=1.
|
50
|
-
Requires-Dist: microsoft-kiota-abstractions>=1.
|
47
|
+
Requires-Dist: microsoft-kiota-http>=1.9.0,<2.0.0
|
48
|
+
Requires-Dist: microsoft-kiota-serialization-json>=1.9.0
|
49
|
+
Requires-Dist: microsoft-kiota-serialization-text>=1.9.0
|
50
|
+
Requires-Dist: microsoft-kiota-abstractions>=1.9.0,<2.0.0
|
51
|
+
Requires-Dist: microsoft-kiota-authentication-azure>=1.9.0,<2.0.0
|
51
52
|
Requires-Dist: msal-extensions>=1.1.0
|
53
|
+
Requires-Dist: portalocker>=2.8.1
|
52
54
|
Requires-Dist: apache-airflow-providers-amazon ; extra == "amazon"
|
53
55
|
Requires-Dist: apache-airflow-providers-common-compat ; extra == "common-compat"
|
54
56
|
Requires-Dist: apache-airflow-providers-oracle ; extra == "oracle"
|
55
57
|
Requires-Dist: apache-airflow-providers-sftp ; extra == "sftp"
|
56
58
|
Project-URL: Bug Tracker, https://github.com/apache/airflow/issues
|
57
|
-
Project-URL: Changelog, https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/12.
|
58
|
-
Project-URL: Documentation, https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/12.
|
59
|
+
Project-URL: Changelog, https://airflow.staged.apache.org/docs/apache-airflow-providers-microsoft-azure/12.4.0/changelog.html
|
60
|
+
Project-URL: Documentation, https://airflow.staged.apache.org/docs/apache-airflow-providers-microsoft-azure/12.4.0
|
59
61
|
Project-URL: Mastodon, https://fosstodon.org/@airflow
|
60
62
|
Project-URL: Slack Chat, https://s.apache.org/airflow-slack
|
61
63
|
Project-URL: Source Code, https://github.com/apache/airflow
|
@@ -90,7 +92,7 @@ Provides-Extra: sftp
|
|
90
92
|
|
91
93
|
Package ``apache-airflow-providers-microsoft-azure``
|
92
94
|
|
93
|
-
Release: ``12.
|
95
|
+
Release: ``12.4.0``
|
94
96
|
|
95
97
|
|
96
98
|
`Microsoft Azure <https://azure.microsoft.com/>`__
|
@@ -103,7 +105,7 @@ This is a provider package for ``microsoft.azure`` provider. All classes for thi
|
|
103
105
|
are in ``airflow.providers.microsoft.azure`` python package.
|
104
106
|
|
105
107
|
You can find package information and changelog for the provider
|
106
|
-
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/12.
|
108
|
+
in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/12.4.0/>`_.
|
107
109
|
|
108
110
|
Installation
|
109
111
|
------------
|
@@ -117,39 +119,41 @@ The package supports the following python versions: 3.9,3.10,3.11,3.12
|
|
117
119
|
Requirements
|
118
120
|
------------
|
119
121
|
|
120
|
-
|
121
|
-
PIP package
|
122
|
-
|
123
|
-
``apache-airflow``
|
124
|
-
``adlfs``
|
125
|
-
``azure-batch``
|
126
|
-
``azure-cosmos``
|
127
|
-
``azure-mgmt-cosmosdb``
|
128
|
-
``azure-datalake-store``
|
129
|
-
``azure-identity``
|
130
|
-
``azure-keyvault-secrets``
|
131
|
-
``azure-mgmt-datalake-store``
|
132
|
-
``azure-mgmt-resource``
|
133
|
-
``azure-storage-blob``
|
134
|
-
``azure-mgmt-storage``
|
135
|
-
``azure-storage-file-share``
|
136
|
-
``azure-servicebus``
|
137
|
-
``azure-synapse-spark``
|
138
|
-
``azure-synapse-artifacts``
|
139
|
-
``adal``
|
140
|
-
``azure-storage-file-datalake``
|
141
|
-
``azure-kusto-data``
|
142
|
-
``azure-mgmt-datafactory``
|
143
|
-
``azure-mgmt-containerregistry``
|
144
|
-
``azure-mgmt-containerinstance``
|
145
|
-
``flask-appbuilder``
|
146
|
-
``msgraph-core``
|
147
|
-
``microsoft-kiota-http``
|
148
|
-
``microsoft-kiota-serialization-json``
|
149
|
-
``microsoft-kiota-serialization-text``
|
150
|
-
``microsoft-kiota-abstractions``
|
151
|
-
``
|
152
|
-
|
122
|
+
======================================== ===================
|
123
|
+
PIP package Version required
|
124
|
+
======================================== ===================
|
125
|
+
``apache-airflow`` ``>=2.10.0``
|
126
|
+
``adlfs`` ``>=2023.10.0``
|
127
|
+
``azure-batch`` ``>=8.0.0``
|
128
|
+
``azure-cosmos`` ``>=4.6.0``
|
129
|
+
``azure-mgmt-cosmosdb`` ``>=3.0.0``
|
130
|
+
``azure-datalake-store`` ``>=0.0.45``
|
131
|
+
``azure-identity`` ``>=1.3.1``
|
132
|
+
``azure-keyvault-secrets`` ``>=4.1.0``
|
133
|
+
``azure-mgmt-datalake-store`` ``>=0.5.0``
|
134
|
+
``azure-mgmt-resource`` ``>=2.2.0``
|
135
|
+
``azure-storage-blob`` ``>=12.14.0``
|
136
|
+
``azure-mgmt-storage`` ``>=16.0.0``
|
137
|
+
``azure-storage-file-share`` ``>=12.7.0``
|
138
|
+
``azure-servicebus`` ``>=7.12.1``
|
139
|
+
``azure-synapse-spark`` ``>=0.2.0``
|
140
|
+
``azure-synapse-artifacts`` ``>=0.17.0``
|
141
|
+
``adal`` ``>=1.2.7``
|
142
|
+
``azure-storage-file-datalake`` ``>=12.9.1``
|
143
|
+
``azure-kusto-data`` ``>=4.1.0,!=4.6.0``
|
144
|
+
``azure-mgmt-datafactory`` ``>=2.0.0``
|
145
|
+
``azure-mgmt-containerregistry`` ``>=8.0.0``
|
146
|
+
``azure-mgmt-containerinstance`` ``>=10.1.0``
|
147
|
+
``flask-appbuilder`` ``>=4.0.0``
|
148
|
+
``msgraph-core`` ``>=1.3.3``
|
149
|
+
``microsoft-kiota-http`` ``>=1.9.0,<2.0.0``
|
150
|
+
``microsoft-kiota-serialization-json`` ``>=1.9.0``
|
151
|
+
``microsoft-kiota-serialization-text`` ``>=1.9.0``
|
152
|
+
``microsoft-kiota-abstractions`` ``>=1.9.0,<2.0.0``
|
153
|
+
``microsoft-kiota-authentication-azure`` ``>=1.9.0,<2.0.0``
|
154
|
+
``msal-extensions`` ``>=1.1.0``
|
155
|
+
``portalocker`` ``>=2.8.1``
|
156
|
+
======================================== ===================
|
153
157
|
|
154
158
|
Cross provider package dependencies
|
155
159
|
-----------------------------------
|
@@ -174,5 +178,5 @@ Dependent package
|
|
174
178
|
================================================================================================================== =================
|
175
179
|
|
176
180
|
The changelog for the provider package can be found in the
|
177
|
-
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/12.
|
181
|
+
`changelog <https://airflow.apache.org/docs/apache-airflow-providers-microsoft-azure/12.4.0/changelog.html>`_.
|
178
182
|
|
@@ -1,8 +1,8 @@
|
|
1
1
|
airflow/providers/microsoft/azure/LICENSE,sha256=gXPVwptPlW1TJ4HSuG5OMPg-a3h43OGMkZRR1rpwfJA,10850
|
2
|
-
airflow/providers/microsoft/azure/__init__.py,sha256=
|
2
|
+
airflow/providers/microsoft/azure/__init__.py,sha256=AjZAtXVVp4wrefKhFnz2VuVtjmSALdqD8lJ4GbHVw6Y,1505
|
3
3
|
airflow/providers/microsoft/azure/get_provider_info.py,sha256=AeIwe8L-5p1z7IVF9Yc_mmQyAe149r6YVCyRpjfc3u0,18946
|
4
4
|
airflow/providers/microsoft/azure/utils.py,sha256=0iDdxz-TujH181MoI2gRf6ppOVQfj_fKGzrHAcnx7uU,8415
|
5
|
-
airflow/providers/microsoft/azure/version_compat.py,sha256=
|
5
|
+
airflow/providers/microsoft/azure/version_compat.py,sha256=j5PCtXvZ71aBjixu-EFTNtVDPsngzzs7os0ZQDgFVDk,1536
|
6
6
|
airflow/providers/microsoft/azure/fs/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
7
7
|
airflow/providers/microsoft/azure/fs/adls.py,sha256=kXZOVulLNfqwJNR0X9MBN23kcYr3dyv8K_0KqAcWvnk,3679
|
8
8
|
airflow/providers/microsoft/azure/hooks/__init__.py,sha256=mlJxuZLkd5x-iq2SBwD3mvRQpt3YR7wjz_nceyF1IaI,787
|
@@ -18,7 +18,7 @@ airflow/providers/microsoft/azure/hooks/data_factory.py,sha256=zlkvYVVakmQGBpk5H
|
|
18
18
|
airflow/providers/microsoft/azure/hooks/data_lake.py,sha256=Te71mF7xsWv-s6tMOncGvZmrMvuu8hwEoFbRnTO8BEk,23786
|
19
19
|
airflow/providers/microsoft/azure/hooks/fileshare.py,sha256=UIz7Jn2fWt_yXi4KVuwqe8sGS4DxKaZrMcCVJbeWf_w,10760
|
20
20
|
airflow/providers/microsoft/azure/hooks/msgraph.py,sha256=b_O0C5A0oDGmrotcITv8_nK-xX8YYA6MruyECIhn_jo,21083
|
21
|
-
airflow/providers/microsoft/azure/hooks/powerbi.py,sha256=
|
21
|
+
airflow/providers/microsoft/azure/hooks/powerbi.py,sha256=vCdKq_4aBJXVNxYp83avgqy_Gsfh25LVQ9JXAP8loGc,9490
|
22
22
|
airflow/providers/microsoft/azure/hooks/synapse.py,sha256=XHSTy_oPo2E7WkfKtOBsYqRosD71IJL1mtsr6wbykiY,16060
|
23
23
|
airflow/providers/microsoft/azure/hooks/wasb.py,sha256=eb6FGL08xcJzNQHCanVbI12_sauGaEmXFz84Vh75hdU,30785
|
24
24
|
airflow/providers/microsoft/azure/log/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
@@ -28,11 +28,11 @@ airflow/providers/microsoft/azure/operators/adls.py,sha256=xLcaYKMFkygveTSGzFY1Y
|
|
28
28
|
airflow/providers/microsoft/azure/operators/adx.py,sha256=wVhg36vgSmT5pKDCva5BXYbz9AAePJofNSwlQ1q7Ba8,3155
|
29
29
|
airflow/providers/microsoft/azure/operators/asb.py,sha256=7yj35zwQRr2EE0zIaKTbEjikhJ0v7KrQUyNCnJosqac,30477
|
30
30
|
airflow/providers/microsoft/azure/operators/batch.py,sha256=QgtmaUYgejLtER68o0dDhLRJSGM4WqmstW5Guy7I30U,16270
|
31
|
-
airflow/providers/microsoft/azure/operators/container_instances.py,sha256=
|
31
|
+
airflow/providers/microsoft/azure/operators/container_instances.py,sha256=1wh_hV3fl7m9y1sADghQ4LbiBgEVM9m-c2S7ejPg-UY,18638
|
32
32
|
airflow/providers/microsoft/azure/operators/cosmos.py,sha256=apq0wVYslRaO-ThXGnVyf7v_DovK951vaE7LxcVYU4E,2814
|
33
33
|
airflow/providers/microsoft/azure/operators/data_factory.py,sha256=kiVoQ1hYPG725rVhCfpE0MuxiufIQYwXNurCkwtKfY8,12806
|
34
|
-
airflow/providers/microsoft/azure/operators/msgraph.py,sha256=
|
35
|
-
airflow/providers/microsoft/azure/operators/powerbi.py,sha256=
|
34
|
+
airflow/providers/microsoft/azure/operators/msgraph.py,sha256=Y3C4jcAkYrtfbSIVFhWNVlfHdvxH1WBeIuEUUAbUu8s,14187
|
35
|
+
airflow/providers/microsoft/azure/operators/powerbi.py,sha256=ESDtzMTlUf2WhavsnO_BflFq8lIr245MzNsjYMLTFkc,11619
|
36
36
|
airflow/providers/microsoft/azure/operators/synapse.py,sha256=fqd3W9uHQYidJY-LAJZaQv1okr-pImXokyMd7iLeY-Q,12751
|
37
37
|
airflow/providers/microsoft/azure/operators/wasb_delete_blob.py,sha256=RQQz_xvcjyfY3AuIVkK-J-rkPF3rT-z4SJNPY-fpyZQ,2714
|
38
38
|
airflow/providers/microsoft/azure/secrets/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
@@ -51,9 +51,9 @@ airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py,sha256=UU_PFJlmsd8xz
|
|
51
51
|
airflow/providers/microsoft/azure/triggers/__init__.py,sha256=9hdXHABrVpkbpjZgUft39kOFL2xSGeG4GEua0Hmelus,785
|
52
52
|
airflow/providers/microsoft/azure/triggers/data_factory.py,sha256=U3vY_pj4yORxE7X6YR7CP3Jl73K9euCdUczy3dLmikU,11165
|
53
53
|
airflow/providers/microsoft/azure/triggers/msgraph.py,sha256=l7A50JoBebiKhhsxILtLvuoulyIn59BVdjTvdAezdpk,8704
|
54
|
-
airflow/providers/microsoft/azure/triggers/powerbi.py,sha256=
|
54
|
+
airflow/providers/microsoft/azure/triggers/powerbi.py,sha256=X-AZRyXomY1CCdXgw7w_Li86RKiqvtNaSUrt6_6-Mjw,15481
|
55
55
|
airflow/providers/microsoft/azure/triggers/wasb.py,sha256=RF-C6iqDEs6_pWireCWZXqxcqWK-sFJ695Okdd_EJOA,7456
|
56
|
-
apache_airflow_providers_microsoft_azure-12.
|
57
|
-
apache_airflow_providers_microsoft_azure-12.
|
58
|
-
apache_airflow_providers_microsoft_azure-12.
|
59
|
-
apache_airflow_providers_microsoft_azure-12.
|
56
|
+
apache_airflow_providers_microsoft_azure-12.4.0rc1.dist-info/entry_points.txt,sha256=6iWHenOoUC3YZBb3OKn6g0HlJsV58Ba56i8USmQrcJI,111
|
57
|
+
apache_airflow_providers_microsoft_azure-12.4.0rc1.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
|
58
|
+
apache_airflow_providers_microsoft_azure-12.4.0rc1.dist-info/METADATA,sha256=d7bN5Cq1XcmAKCB9pVyou738kiP2GqyavhkkeC87ksQ,8844
|
59
|
+
apache_airflow_providers_microsoft_azure-12.4.0rc1.dist-info/RECORD,,
|
File without changes
|