apache-airflow-providers-microsoft-azure 6.1.1__py3-none-any.whl → 6.1.2rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airflow/providers/microsoft/azure/__init__.py +1 -1
- airflow/providers/microsoft/azure/get_provider_info.py +11 -0
- airflow/providers/microsoft/azure/hooks/adx.py +3 -3
- airflow/providers/microsoft/azure/hooks/asb.py +25 -24
- airflow/providers/microsoft/azure/hooks/base_azure.py +2 -2
- airflow/providers/microsoft/azure/hooks/batch.py +14 -14
- airflow/providers/microsoft/azure/hooks/container_instance.py +7 -7
- airflow/providers/microsoft/azure/hooks/container_registry.py +2 -2
- airflow/providers/microsoft/azure/hooks/container_volume.py +4 -4
- airflow/providers/microsoft/azure/hooks/cosmos.py +5 -5
- airflow/providers/microsoft/azure/hooks/data_factory.py +8 -8
- airflow/providers/microsoft/azure/hooks/data_lake.py +39 -55
- airflow/providers/microsoft/azure/hooks/fileshare.py +2 -2
- airflow/providers/microsoft/azure/hooks/synapse.py +2 -2
- airflow/providers/microsoft/azure/hooks/wasb.py +8 -8
- airflow/providers/microsoft/azure/log/wasb_task_handler.py +5 -4
- airflow/providers/microsoft/azure/operators/adls.py +1 -1
- airflow/providers/microsoft/azure/operators/adx.py +6 -4
- airflow/providers/microsoft/azure/operators/asb.py +20 -35
- airflow/providers/microsoft/azure/operators/batch.py +2 -2
- airflow/providers/microsoft/azure/operators/container_instances.py +1 -1
- airflow/providers/microsoft/azure/operators/cosmos.py +4 -3
- airflow/providers/microsoft/azure/secrets/key_vault.py +4 -4
- airflow/providers/microsoft/azure/sensors/cosmos.py +2 -3
- airflow/providers/microsoft/azure/sensors/data_factory.py +4 -2
- airflow/providers/microsoft/azure/sensors/wasb.py +8 -4
- airflow/providers/microsoft/azure/transfers/azure_blob_to_gcs.py +1 -5
- airflow/providers/microsoft/azure/transfers/local_to_adls.py +1 -1
- airflow/providers/microsoft/azure/transfers/oracle_to_azure_data_lake.py +4 -6
- airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py +3 -3
- airflow/providers/microsoft/azure/triggers/data_factory.py +6 -8
- {apache_airflow_providers_microsoft_azure-6.1.1.dist-info → apache_airflow_providers_microsoft_azure-6.1.2rc1.dist-info}/METADATA +29 -8
- apache_airflow_providers_microsoft_azure-6.1.2rc1.dist-info/RECORD +51 -0
- apache_airflow_providers_microsoft_azure-6.1.1.dist-info/RECORD +0 -51
- {apache_airflow_providers_microsoft_azure-6.1.1.dist-info → apache_airflow_providers_microsoft_azure-6.1.2rc1.dist-info}/LICENSE +0 -0
- {apache_airflow_providers_microsoft_azure-6.1.1.dist-info → apache_airflow_providers_microsoft_azure-6.1.2rc1.dist-info}/NOTICE +0 -0
- {apache_airflow_providers_microsoft_azure-6.1.1.dist-info → apache_airflow_providers_microsoft_azure-6.1.2rc1.dist-info}/WHEEL +0 -0
- {apache_airflow_providers_microsoft_azure-6.1.1.dist-info → apache_airflow_providers_microsoft_azure-6.1.2rc1.dist-info}/entry_points.txt +0 -0
- {apache_airflow_providers_microsoft_azure-6.1.1.dist-info → apache_airflow_providers_microsoft_azure-6.1.2rc1.dist-info}/top_level.txt +0 -0
@@ -37,21 +37,20 @@ from airflow.providers.microsoft.azure.utils import _ensure_prefixes, get_field
|
|
37
37
|
|
38
38
|
|
39
39
|
class AzureDataLakeHook(BaseHook):
|
40
|
-
"""
|
41
|
-
This module contains integration with Azure Data Lake.
|
42
|
-
|
43
|
-
AzureDataLakeHook communicates via a REST API compatible with WebHDFS. Make sure that a
|
44
|
-
Airflow connection of type `azure_data_lake` exists. Authorization can be done by supplying a
|
45
|
-
login (=Client ID), password (=Client Secret) and extra fields tenant (Tenant) and account_name
|
46
|
-
(Account Name)(see connection `azure_data_lake_default` for an example).
|
40
|
+
"""Integration with Azure Data Lake.
|
47
41
|
|
48
|
-
|
42
|
+
AzureDataLakeHook communicates via a REST API compatible with WebHDFS. Make
|
43
|
+
sure that a Airflow connection of type ``azure_data_lake`` exists.
|
44
|
+
Authorization can be done by supplying a *login* (=Client ID), *password*
|
45
|
+
(=Client Secret), and extra fields *tenant* (Tenant) and *account_name*
|
46
|
+
(Account Name). See connection ``azure_data_lake_default`` for an example.
|
49
47
|
|
50
|
-
Client ID and
|
48
|
+
Client ID and secret should be in user and password parameters.
|
51
49
|
Tenant and account name should be extra field as
|
52
|
-
{"tenant": "<TENANT>", "account_name": "ACCOUNT_NAME"}
|
50
|
+
``{"tenant": "<TENANT>", "account_name": "ACCOUNT_NAME"}``.
|
53
51
|
|
54
|
-
:param azure_data_lake_conn_id: Reference to
|
52
|
+
:param azure_data_lake_conn_id: Reference to
|
53
|
+
:ref:`Azure Data Lake connection<howto/connection:adl>`.
|
55
54
|
"""
|
56
55
|
|
57
56
|
conn_name_attr = "azure_data_lake_conn_id"
|
@@ -61,7 +60,7 @@ class AzureDataLakeHook(BaseHook):
|
|
61
60
|
|
62
61
|
@staticmethod
|
63
62
|
def get_connection_form_widgets() -> dict[str, Any]:
|
64
|
-
"""Returns connection widgets to add to connection form"""
|
63
|
+
"""Returns connection widgets to add to connection form."""
|
65
64
|
from flask_appbuilder.fieldwidgets import BS3TextFieldWidget
|
66
65
|
from flask_babel import lazy_gettext
|
67
66
|
from wtforms import StringField
|
@@ -76,7 +75,7 @@ class AzureDataLakeHook(BaseHook):
|
|
76
75
|
@staticmethod
|
77
76
|
@_ensure_prefixes(conn_type="azure_data_lake")
|
78
77
|
def get_ui_field_behaviour() -> dict[str, Any]:
|
79
|
-
"""Returns custom field behaviour"""
|
78
|
+
"""Returns custom field behaviour."""
|
80
79
|
return {
|
81
80
|
"hidden_fields": ["schema", "port", "host", "extra"],
|
82
81
|
"relabeling": {
|
@@ -118,8 +117,7 @@ class AzureDataLakeHook(BaseHook):
|
|
118
117
|
return self._conn
|
119
118
|
|
120
119
|
def check_for_file(self, file_path: str) -> bool:
|
121
|
-
"""
|
122
|
-
Check if a file exists on Azure Data Lake.
|
120
|
+
"""Check if a file exists on Azure Data Lake.
|
123
121
|
|
124
122
|
:param file_path: Path and name of the file.
|
125
123
|
:return: True if the file exists, False otherwise.
|
@@ -140,8 +138,7 @@ class AzureDataLakeHook(BaseHook):
|
|
140
138
|
blocksize: int = 4194304,
|
141
139
|
**kwargs,
|
142
140
|
) -> None:
|
143
|
-
"""
|
144
|
-
Upload a file to Azure Data Lake.
|
141
|
+
"""Upload a file to Azure Data Lake.
|
145
142
|
|
146
143
|
:param local_path: local path. Can be single file, directory (in which case,
|
147
144
|
upload recursively) or glob pattern. Recursive glob patterns using `**`
|
@@ -181,8 +178,7 @@ class AzureDataLakeHook(BaseHook):
|
|
181
178
|
blocksize: int = 4194304,
|
182
179
|
**kwargs,
|
183
180
|
) -> None:
|
184
|
-
"""
|
185
|
-
Download a file from Azure Blob Storage.
|
181
|
+
"""Download a file from Azure Blob Storage.
|
186
182
|
|
187
183
|
:param local_path: local path. If downloading a single file, will write to this
|
188
184
|
specific file, unless it is an existing directory, in which case a file is
|
@@ -214,8 +210,7 @@ class AzureDataLakeHook(BaseHook):
|
|
214
210
|
)
|
215
211
|
|
216
212
|
def list(self, path: str) -> list:
|
217
|
-
"""
|
218
|
-
List files in Azure Data Lake Storage
|
213
|
+
"""List files in Azure Data Lake Storage.
|
219
214
|
|
220
215
|
:param path: full path/globstring to use to list files in ADLS
|
221
216
|
"""
|
@@ -225,8 +220,7 @@ class AzureDataLakeHook(BaseHook):
|
|
225
220
|
return self.get_conn().walk(path)
|
226
221
|
|
227
222
|
def remove(self, path: str, recursive: bool = False, ignore_not_found: bool = True) -> None:
|
228
|
-
"""
|
229
|
-
Remove files in Azure Data Lake Storage
|
223
|
+
"""Remove files in Azure Data Lake Storage.
|
230
224
|
|
231
225
|
:param path: A directory or file to remove in ADLS
|
232
226
|
:param recursive: Whether to loop into directories in the location and remove the files
|
@@ -242,13 +236,14 @@ class AzureDataLakeHook(BaseHook):
|
|
242
236
|
|
243
237
|
|
244
238
|
class AzureDataLakeStorageV2Hook(BaseHook):
|
245
|
-
"""
|
246
|
-
|
247
|
-
|
248
|
-
|
239
|
+
"""Interact with a ADLS gen2 storage account.
|
240
|
+
|
241
|
+
It mainly helps to create and manage directories and files in storage
|
242
|
+
accounts that have a hierarchical namespace. Using Adls_v2 connection
|
243
|
+
details create DataLakeServiceClient object.
|
249
244
|
|
250
|
-
Due to Wasb is marked as legacy and and retirement of the (ADLS1) it would
|
251
|
-
implement ADLS gen2 hook for interacting with the storage account.
|
245
|
+
Due to Wasb is marked as legacy and and retirement of the (ADLS1), it would
|
246
|
+
be nice to implement ADLS gen2 hook for interacting with the storage account.
|
252
247
|
|
253
248
|
.. seealso::
|
254
249
|
https://learn.microsoft.com/en-us/azure/storage/blobs/data-lake-storage-directory-file-acl-python
|
@@ -264,7 +259,7 @@ class AzureDataLakeStorageV2Hook(BaseHook):
|
|
264
259
|
|
265
260
|
@staticmethod
|
266
261
|
def get_connection_form_widgets() -> dict[str, Any]:
|
267
|
-
"""Returns connection widgets to add to connection form"""
|
262
|
+
"""Returns connection widgets to add to connection form."""
|
268
263
|
from flask_appbuilder.fieldwidgets import BS3PasswordFieldWidget, BS3TextFieldWidget
|
269
264
|
from flask_babel import lazy_gettext
|
270
265
|
from wtforms import PasswordField, StringField
|
@@ -280,7 +275,7 @@ class AzureDataLakeStorageV2Hook(BaseHook):
|
|
280
275
|
|
281
276
|
@staticmethod
|
282
277
|
def get_ui_field_behaviour() -> dict[str, Any]:
|
283
|
-
"""Returns custom field behaviour"""
|
278
|
+
"""Returns custom field behaviour."""
|
284
279
|
return {
|
285
280
|
"hidden_fields": ["schema", "port"],
|
286
281
|
"relabeling": {
|
@@ -342,9 +337,9 @@ class AzureDataLakeStorageV2Hook(BaseHook):
|
|
342
337
|
return extra_dict.get(f"{prefix}{field_name}") or None
|
343
338
|
|
344
339
|
def create_file_system(self, file_system_name: str) -> None:
|
345
|
-
"""
|
346
|
-
|
347
|
-
|
340
|
+
"""Create a new file system under the specified account.
|
341
|
+
|
342
|
+
A container acts as a file system for your files.
|
348
343
|
|
349
344
|
If the file system with the same name already exists, a ResourceExistsError will
|
350
345
|
be raised. This method returns a client with which to interact with the newly
|
@@ -360,8 +355,7 @@ class AzureDataLakeStorageV2Hook(BaseHook):
|
|
360
355
|
raise
|
361
356
|
|
362
357
|
def get_file_system(self, file_system: FileSystemProperties | str) -> FileSystemClient:
|
363
|
-
"""
|
364
|
-
Get a client to interact with the specified file system
|
358
|
+
"""Get a client to interact with the specified file system.
|
365
359
|
|
366
360
|
:param file_system: This can either be the name of the file system
|
367
361
|
or an instance of FileSystemProperties.
|
@@ -379,8 +373,7 @@ class AzureDataLakeStorageV2Hook(BaseHook):
|
|
379
373
|
def create_directory(
|
380
374
|
self, file_system_name: FileSystemProperties | str, directory_name: str, **kwargs
|
381
375
|
) -> DataLakeDirectoryClient:
|
382
|
-
"""
|
383
|
-
Create a directory under the specified file system.
|
376
|
+
"""Create a directory under the specified file system.
|
384
377
|
|
385
378
|
:param file_system_name: Name of the file system or instance of FileSystemProperties.
|
386
379
|
:param directory_name: Name of the directory which needs to be created in the file system.
|
@@ -393,8 +386,7 @@ class AzureDataLakeStorageV2Hook(BaseHook):
|
|
393
386
|
file_system_name: FileSystemProperties | str,
|
394
387
|
directory_name: DirectoryProperties | str,
|
395
388
|
) -> DataLakeDirectoryClient:
|
396
|
-
"""
|
397
|
-
Get the specific directory under the specified file system.
|
389
|
+
"""Get the specific directory under the specified file system.
|
398
390
|
|
399
391
|
:param file_system_name: Name of the file system or instance of FileSystemProperties.
|
400
392
|
:param directory_name: Name of the directory or instance of DirectoryProperties which needs to be
|
@@ -413,8 +405,7 @@ class AzureDataLakeStorageV2Hook(BaseHook):
|
|
413
405
|
raise
|
414
406
|
|
415
407
|
def create_file(self, file_system_name: FileSystemProperties | str, file_name: str) -> DataLakeFileClient:
|
416
|
-
"""
|
417
|
-
Creates a file under the file system
|
408
|
+
"""Create a file under the file system.
|
418
409
|
|
419
410
|
:param file_system_name: Name of the file system or instance of FileSystemProperties.
|
420
411
|
:param file_name: Name of the file which needs to be created in the file system.
|
@@ -430,8 +421,7 @@ class AzureDataLakeStorageV2Hook(BaseHook):
|
|
430
421
|
overwrite: bool = False,
|
431
422
|
**kwargs: Any,
|
432
423
|
) -> None:
|
433
|
-
"""
|
434
|
-
Create a file with data in the file system
|
424
|
+
"""Create a file with data in the file system.
|
435
425
|
|
436
426
|
:param file_system_name: Name of the file system or instance of FileSystemProperties.
|
437
427
|
:param file_name: Name of the file to be created with name.
|
@@ -451,9 +441,7 @@ class AzureDataLakeStorageV2Hook(BaseHook):
|
|
451
441
|
overwrite: bool = False,
|
452
442
|
**kwargs: Any,
|
453
443
|
) -> None:
|
454
|
-
"""
|
455
|
-
Create a new file and return the file client to be interacted with and then
|
456
|
-
upload data to a file
|
444
|
+
"""Upload data to a file.
|
457
445
|
|
458
446
|
:param file_system_name: Name of the file system or instance of FileSystemProperties.
|
459
447
|
:param directory_name: Name of the directory.
|
@@ -469,8 +457,7 @@ class AzureDataLakeStorageV2Hook(BaseHook):
|
|
469
457
|
def list_files_directory(
|
470
458
|
self, file_system_name: FileSystemProperties | str, directory_name: str
|
471
459
|
) -> list[str]:
|
472
|
-
"""
|
473
|
-
Get the list of files or directories under the specified file system
|
460
|
+
"""List files or directories under the specified file system.
|
474
461
|
|
475
462
|
:param file_system_name: Name of the file system or instance of FileSystemProperties.
|
476
463
|
:param directory_name: Name of the directory.
|
@@ -484,8 +471,7 @@ class AzureDataLakeStorageV2Hook(BaseHook):
|
|
484
471
|
def list_file_system(
|
485
472
|
self, prefix: str | None = None, include_metadata: bool = False, **kwargs: Any
|
486
473
|
) -> list[str]:
|
487
|
-
"""
|
488
|
-
Get the list the file systems under the specified account.
|
474
|
+
"""List file systems under the specified account.
|
489
475
|
|
490
476
|
:param prefix:
|
491
477
|
Filters the results to return only file systems whose names
|
@@ -502,8 +488,7 @@ class AzureDataLakeStorageV2Hook(BaseHook):
|
|
502
488
|
return file_system_list
|
503
489
|
|
504
490
|
def delete_file_system(self, file_system_name: FileSystemProperties | str) -> None:
|
505
|
-
"""
|
506
|
-
Deletes the file system
|
491
|
+
"""Delete the file system.
|
507
492
|
|
508
493
|
:param file_system_name: Name of the file system or instance of FileSystemProperties.
|
509
494
|
"""
|
@@ -517,8 +502,7 @@ class AzureDataLakeStorageV2Hook(BaseHook):
|
|
517
502
|
raise
|
518
503
|
|
519
504
|
def delete_directory(self, file_system_name: FileSystemProperties | str, directory_name: str) -> None:
|
520
|
-
"""
|
521
|
-
Deletes specified directory in file system
|
505
|
+
"""Delete the specified directory in a file system.
|
522
506
|
|
523
507
|
:param file_system_name: Name of the file system or instance of FileSystemProperties.
|
524
508
|
:param directory_name: Name of the directory.
|
@@ -76,7 +76,7 @@ class AzureFileShareHook(BaseHook):
|
|
76
76
|
|
77
77
|
@staticmethod
|
78
78
|
def get_connection_form_widgets() -> dict[str, Any]:
|
79
|
-
"""Returns connection widgets to add to connection form"""
|
79
|
+
"""Returns connection widgets to add to connection form."""
|
80
80
|
from flask_appbuilder.fieldwidgets import BS3PasswordFieldWidget, BS3TextFieldWidget
|
81
81
|
from flask_babel import lazy_gettext
|
82
82
|
from wtforms import PasswordField, StringField
|
@@ -94,7 +94,7 @@ class AzureFileShareHook(BaseHook):
|
|
94
94
|
@staticmethod
|
95
95
|
@_ensure_prefixes(conn_type="azure_fileshare")
|
96
96
|
def get_ui_field_behaviour() -> dict[str, Any]:
|
97
|
-
"""Returns custom field behaviour"""
|
97
|
+
"""Returns custom field behaviour."""
|
98
98
|
return {
|
99
99
|
"hidden_fields": ["schema", "port", "host", "extra"],
|
100
100
|
"relabeling": {
|
@@ -62,7 +62,7 @@ class AzureSynapseHook(BaseHook):
|
|
62
62
|
|
63
63
|
@staticmethod
|
64
64
|
def get_connection_form_widgets() -> dict[str, Any]:
|
65
|
-
"""Returns connection widgets to add to connection form"""
|
65
|
+
"""Returns connection widgets to add to connection form."""
|
66
66
|
from flask_appbuilder.fieldwidgets import BS3TextFieldWidget
|
67
67
|
from flask_babel import lazy_gettext
|
68
68
|
from wtforms import StringField
|
@@ -74,7 +74,7 @@ class AzureSynapseHook(BaseHook):
|
|
74
74
|
|
75
75
|
@staticmethod
|
76
76
|
def get_ui_field_behaviour() -> dict[str, Any]:
|
77
|
-
"""Returns custom field behaviour"""
|
77
|
+
"""Returns custom field behaviour."""
|
78
78
|
return {
|
79
79
|
"hidden_fields": ["schema", "port", "extra"],
|
80
80
|
"relabeling": {"login": "Client ID", "password": "Secret", "host": "Synapse Workspace URL"},
|
@@ -103,7 +103,7 @@ class WasbHook(BaseHook):
|
|
103
103
|
|
104
104
|
@staticmethod
|
105
105
|
def get_connection_form_widgets() -> dict[str, Any]:
|
106
|
-
"""Returns connection widgets to add to connection form"""
|
106
|
+
"""Returns connection widgets to add to connection form."""
|
107
107
|
from flask_appbuilder.fieldwidgets import BS3PasswordFieldWidget, BS3TextFieldWidget
|
108
108
|
from flask_babel import lazy_gettext
|
109
109
|
from wtforms import PasswordField, StringField
|
@@ -124,7 +124,7 @@ class WasbHook(BaseHook):
|
|
124
124
|
@staticmethod
|
125
125
|
@_ensure_prefixes(conn_type="wasb")
|
126
126
|
def get_ui_field_behaviour() -> dict[str, Any]:
|
127
|
-
"""Returns custom field behaviour"""
|
127
|
+
"""Returns custom field behaviour."""
|
128
128
|
return {
|
129
129
|
"hidden_fields": ["schema", "port", "extra"],
|
130
130
|
"relabeling": {
|
@@ -221,7 +221,7 @@ class WasbHook(BaseHook):
|
|
221
221
|
|
222
222
|
def _get_container_client(self, container_name: str) -> ContainerClient:
|
223
223
|
"""
|
224
|
-
Instantiates a container client
|
224
|
+
Instantiates a container client.
|
225
225
|
|
226
226
|
:param container_name: The name of the container
|
227
227
|
:return: ContainerClient
|
@@ -230,7 +230,7 @@ class WasbHook(BaseHook):
|
|
230
230
|
|
231
231
|
def _get_blob_client(self, container_name: str, blob_name: str) -> BlobClient:
|
232
232
|
"""
|
233
|
-
Instantiates a blob client
|
233
|
+
Instantiates a blob client.
|
234
234
|
|
235
235
|
:param container_name: The name of the blob container
|
236
236
|
:param blob_name: The name of the blob. This needs not be existing
|
@@ -273,7 +273,7 @@ class WasbHook(BaseHook):
|
|
273
273
|
**kwargs,
|
274
274
|
) -> list:
|
275
275
|
"""
|
276
|
-
List blobs in a given container
|
276
|
+
List blobs in a given container.
|
277
277
|
|
278
278
|
:param container_name: The name of the container
|
279
279
|
:param prefix: Filters the results to return only blobs whose names
|
@@ -400,7 +400,7 @@ class WasbHook(BaseHook):
|
|
400
400
|
self, container_name, blob_name, offset: int | None = None, length: int | None = None, **kwargs
|
401
401
|
) -> StorageStreamDownloader:
|
402
402
|
"""
|
403
|
-
Downloads a blob to the StorageStreamDownloader
|
403
|
+
Downloads a blob to the StorageStreamDownloader.
|
404
404
|
|
405
405
|
:param container_name: The name of the container containing the blob
|
406
406
|
:param blob_name: The name of the blob to download
|
@@ -413,7 +413,7 @@ class WasbHook(BaseHook):
|
|
413
413
|
|
414
414
|
def create_container(self, container_name: str) -> None:
|
415
415
|
"""
|
416
|
-
Create container object if not already existing
|
416
|
+
Create container object if not already existing.
|
417
417
|
|
418
418
|
:param container_name: The name of the container to create
|
419
419
|
"""
|
@@ -444,7 +444,7 @@ class WasbHook(BaseHook):
|
|
444
444
|
|
445
445
|
def delete_container(self, container_name: str) -> None:
|
446
446
|
"""
|
447
|
-
Delete a container object
|
447
|
+
Delete a container object.
|
448
448
|
|
449
449
|
:param container_name: The name of the container
|
450
450
|
"""
|
@@ -19,20 +19,21 @@ from __future__ import annotations
|
|
19
19
|
|
20
20
|
import os
|
21
21
|
import shutil
|
22
|
+
from functools import cached_property
|
22
23
|
from pathlib import Path
|
23
24
|
from typing import TYPE_CHECKING, Any
|
24
25
|
|
25
26
|
from azure.core.exceptions import HttpResponseError
|
26
27
|
from packaging.version import Version
|
27
28
|
|
28
|
-
from airflow.compat.functools import cached_property
|
29
29
|
from airflow.configuration import conf
|
30
30
|
from airflow.utils.log.file_task_handler import FileTaskHandler
|
31
31
|
from airflow.utils.log.logging_mixin import LoggingMixin
|
32
32
|
|
33
33
|
|
34
34
|
def get_default_delete_local_copy():
|
35
|
-
"""Load delete_local_logs conf if Airflow version > 2.6 and return False if not
|
35
|
+
"""Load delete_local_logs conf if Airflow version > 2.6 and return False if not.
|
36
|
+
|
36
37
|
TODO: delete this function when min airflow version >= 2.6
|
37
38
|
"""
|
38
39
|
from airflow.version import version
|
@@ -128,7 +129,7 @@ class WasbTaskHandler(FileTaskHandler, LoggingMixin):
|
|
128
129
|
# Mark closed so we don't double write if close is called twice
|
129
130
|
self.closed = True
|
130
131
|
|
131
|
-
def _read_remote_logs(self, ti, try_number, metadata=None):
|
132
|
+
def _read_remote_logs(self, ti, try_number, metadata=None) -> tuple[list[str], list[str]]:
|
132
133
|
messages = []
|
133
134
|
logs = []
|
134
135
|
worker_log_relative_path = self._render_filename(ti, try_number)
|
@@ -192,7 +193,7 @@ class WasbTaskHandler(FileTaskHandler, LoggingMixin):
|
|
192
193
|
|
193
194
|
def wasb_log_exists(self, remote_log_location: str) -> bool:
|
194
195
|
"""
|
195
|
-
Check if remote_log_location exists in remote storage
|
196
|
+
Check if remote_log_location exists in remote storage.
|
196
197
|
|
197
198
|
:param remote_log_location: log's location in remote storage
|
198
199
|
:return: True if location exists else False
|
@@ -64,7 +64,7 @@ class ADLSDeleteOperator(BaseOperator):
|
|
64
64
|
|
65
65
|
class ADLSListOperator(BaseOperator):
|
66
66
|
"""
|
67
|
-
List all files from the specified path
|
67
|
+
List all files from the specified path.
|
68
68
|
|
69
69
|
This operator returns a python list with the names of files which can be used by
|
70
70
|
`xcom` in the downstream tasks.
|
@@ -15,7 +15,7 @@
|
|
15
15
|
# KIND, either express or implied. See the License for the
|
16
16
|
# specific language governing permissions and limitations
|
17
17
|
# under the License.
|
18
|
-
"""This module contains Azure Data Explorer operators"""
|
18
|
+
"""This module contains Azure Data Explorer operators."""
|
19
19
|
from __future__ import annotations
|
20
20
|
|
21
21
|
from typing import TYPE_CHECKING, Sequence
|
@@ -62,14 +62,16 @@ class AzureDataExplorerQueryOperator(BaseOperator):
|
|
62
62
|
self.azure_data_explorer_conn_id = azure_data_explorer_conn_id
|
63
63
|
|
64
64
|
def get_hook(self) -> AzureDataExplorerHook:
|
65
|
-
"""Returns new instance of AzureDataExplorerHook"""
|
65
|
+
"""Returns new instance of AzureDataExplorerHook."""
|
66
66
|
return AzureDataExplorerHook(self.azure_data_explorer_conn_id)
|
67
67
|
|
68
68
|
def execute(self, context: Context) -> KustoResultTable | str:
|
69
69
|
"""
|
70
70
|
Run KQL Query on Azure Data Explorer (Kusto).
|
71
|
-
|
72
|
-
|
71
|
+
|
72
|
+
Returns `PrimaryResult` of Query v2 HTTP response contents.
|
73
|
+
|
74
|
+
https://docs.microsoft.com/en-us/azure/kusto/api/rest/response2
|
73
75
|
"""
|
74
76
|
hook = self.get_hook()
|
75
77
|
response = hook.run_query(self.query, self.database, self.options)
|
@@ -29,8 +29,7 @@ if TYPE_CHECKING:
|
|
29
29
|
|
30
30
|
|
31
31
|
class AzureServiceBusCreateQueueOperator(BaseOperator):
|
32
|
-
"""
|
33
|
-
Creates a Azure Service Bus queue under a Service Bus Namespace by using ServiceBusAdministrationClient
|
32
|
+
"""Create a Azure Service Bus queue under a Service Bus Namespace.
|
34
33
|
|
35
34
|
.. seealso::
|
36
35
|
For more information on how to use this operator, take a look at the guide:
|
@@ -68,7 +67,7 @@ class AzureServiceBusCreateQueueOperator(BaseOperator):
|
|
68
67
|
self.azure_service_bus_conn_id = azure_service_bus_conn_id
|
69
68
|
|
70
69
|
def execute(self, context: Context) -> None:
|
71
|
-
"""Creates Queue in Azure Service Bus namespace, by connecting to Service Bus Admin client in hook"""
|
70
|
+
"""Creates Queue in Azure Service Bus namespace, by connecting to Service Bus Admin client in hook."""
|
72
71
|
hook = AdminClientHook(azure_service_bus_conn_id=self.azure_service_bus_conn_id)
|
73
72
|
|
74
73
|
# create queue with name
|
@@ -82,8 +81,7 @@ class AzureServiceBusCreateQueueOperator(BaseOperator):
|
|
82
81
|
|
83
82
|
|
84
83
|
class AzureServiceBusSendMessageOperator(BaseOperator):
|
85
|
-
"""
|
86
|
-
Send Message or batch message to the Service Bus queue
|
84
|
+
"""Send Message or batch message to the Service Bus queue.
|
87
85
|
|
88
86
|
.. seealso::
|
89
87
|
For more information on how to use this operator, take a look at the guide:
|
@@ -116,10 +114,7 @@ class AzureServiceBusSendMessageOperator(BaseOperator):
|
|
116
114
|
self.azure_service_bus_conn_id = azure_service_bus_conn_id
|
117
115
|
|
118
116
|
def execute(self, context: Context) -> None:
|
119
|
-
"""
|
120
|
-
Sends Message to the specific queue in Service Bus namespace, by
|
121
|
-
connecting to Service Bus client
|
122
|
-
"""
|
117
|
+
"""Sends Message to the specific queue in Service Bus namespace."""
|
123
118
|
# Create the hook
|
124
119
|
hook = MessageHook(azure_service_bus_conn_id=self.azure_service_bus_conn_id)
|
125
120
|
|
@@ -128,8 +123,7 @@ class AzureServiceBusSendMessageOperator(BaseOperator):
|
|
128
123
|
|
129
124
|
|
130
125
|
class AzureServiceBusReceiveMessageOperator(BaseOperator):
|
131
|
-
"""
|
132
|
-
Receive a batch of messages at once in a specified Queue name
|
126
|
+
"""Receive a batch of messages at once in a specified Queue name.
|
133
127
|
|
134
128
|
.. seealso::
|
135
129
|
For more information on how to use this operator, take a look at the guide:
|
@@ -163,7 +157,7 @@ class AzureServiceBusReceiveMessageOperator(BaseOperator):
|
|
163
157
|
def execute(self, context: Context) -> None:
|
164
158
|
"""
|
165
159
|
Receive Message in specific queue in Service Bus namespace,
|
166
|
-
by connecting to Service Bus client
|
160
|
+
by connecting to Service Bus client.
|
167
161
|
"""
|
168
162
|
# Create the hook
|
169
163
|
hook = MessageHook(azure_service_bus_conn_id=self.azure_service_bus_conn_id)
|
@@ -175,8 +169,7 @@ class AzureServiceBusReceiveMessageOperator(BaseOperator):
|
|
175
169
|
|
176
170
|
|
177
171
|
class AzureServiceBusDeleteQueueOperator(BaseOperator):
|
178
|
-
"""
|
179
|
-
Deletes the Queue in the Azure Service Bus namespace
|
172
|
+
"""Delete the Queue in the Azure Service Bus namespace.
|
180
173
|
|
181
174
|
.. seealso::
|
182
175
|
For more information on how to use this operator, take a look at the guide:
|
@@ -202,7 +195,7 @@ class AzureServiceBusDeleteQueueOperator(BaseOperator):
|
|
202
195
|
self.azure_service_bus_conn_id = azure_service_bus_conn_id
|
203
196
|
|
204
197
|
def execute(self, context: Context) -> None:
|
205
|
-
"""Delete Queue in Service Bus namespace, by connecting to Service Bus Admin client"""
|
198
|
+
"""Delete Queue in Service Bus namespace, by connecting to Service Bus Admin client."""
|
206
199
|
# Create the hook
|
207
200
|
hook = AdminClientHook(azure_service_bus_conn_id=self.azure_service_bus_conn_id)
|
208
201
|
|
@@ -211,8 +204,7 @@ class AzureServiceBusDeleteQueueOperator(BaseOperator):
|
|
211
204
|
|
212
205
|
|
213
206
|
class AzureServiceBusTopicCreateOperator(BaseOperator):
|
214
|
-
"""
|
215
|
-
Create an Azure Service Bus Topic under a Service Bus Namespace by using ServiceBusAdministrationClient
|
207
|
+
"""Create an Azure Service Bus Topic under a Service Bus Namespace.
|
216
208
|
|
217
209
|
.. seealso::
|
218
210
|
For more information on how to use this operator, take a look at the guide:
|
@@ -296,7 +288,7 @@ class AzureServiceBusTopicCreateOperator(BaseOperator):
|
|
296
288
|
self.max_message_size_in_kilobytes = max_message_size_in_kilobytes
|
297
289
|
|
298
290
|
def execute(self, context: Context) -> str:
|
299
|
-
"""Creates Topic in Service Bus namespace, by connecting to Service Bus Admin client"""
|
291
|
+
"""Creates Topic in Service Bus namespace, by connecting to Service Bus Admin client."""
|
300
292
|
if self.topic_name is None:
|
301
293
|
raise TypeError("Topic name cannot be None.")
|
302
294
|
|
@@ -330,9 +322,7 @@ class AzureServiceBusTopicCreateOperator(BaseOperator):
|
|
330
322
|
|
331
323
|
|
332
324
|
class AzureServiceBusSubscriptionCreateOperator(BaseOperator):
|
333
|
-
"""
|
334
|
-
Create an Azure Service Bus Topic Subscription under a Service Bus Namespace
|
335
|
-
by using ServiceBusAdministrationClient
|
325
|
+
"""Create an Azure Service Bus Topic Subscription under a Service Bus Namespace.
|
336
326
|
|
337
327
|
.. seealso::
|
338
328
|
For more information on how to use this operator, take a look at the guide:
|
@@ -409,7 +399,7 @@ class AzureServiceBusSubscriptionCreateOperator(BaseOperator):
|
|
409
399
|
self.azure_service_bus_conn_id = azure_service_bus_conn_id
|
410
400
|
|
411
401
|
def execute(self, context: Context) -> None:
|
412
|
-
"""Creates Subscription in Service Bus namespace, by connecting to Service Bus Admin client"""
|
402
|
+
"""Creates Subscription in Service Bus namespace, by connecting to Service Bus Admin client."""
|
413
403
|
if self.subscription_name is None:
|
414
404
|
raise TypeError("Subscription name cannot be None.")
|
415
405
|
if self.topic_name is None:
|
@@ -438,9 +428,7 @@ class AzureServiceBusSubscriptionCreateOperator(BaseOperator):
|
|
438
428
|
|
439
429
|
|
440
430
|
class AzureServiceBusUpdateSubscriptionOperator(BaseOperator):
|
441
|
-
"""
|
442
|
-
Update an Azure ServiceBus Topic Subscription under a ServiceBus Namespace
|
443
|
-
by using ServiceBusAdministrationClient
|
431
|
+
"""Update an Azure ServiceBus Topic Subscription under a ServiceBus Namespace.
|
444
432
|
|
445
433
|
.. seealso::
|
446
434
|
For more information on how to use this operator, take a look at the guide:
|
@@ -481,7 +469,7 @@ class AzureServiceBusUpdateSubscriptionOperator(BaseOperator):
|
|
481
469
|
self.azure_service_bus_conn_id = azure_service_bus_conn_id
|
482
470
|
|
483
471
|
def execute(self, context: Context) -> None:
|
484
|
-
"""Updates Subscription properties, by connecting to Service Bus Admin client"""
|
472
|
+
"""Updates Subscription properties, by connecting to Service Bus Admin client."""
|
485
473
|
hook = AdminClientHook(azure_service_bus_conn_id=self.azure_service_bus_conn_id)
|
486
474
|
|
487
475
|
with hook.get_conn() as service_mgmt_conn:
|
@@ -499,8 +487,7 @@ class AzureServiceBusUpdateSubscriptionOperator(BaseOperator):
|
|
499
487
|
|
500
488
|
|
501
489
|
class ASBReceiveSubscriptionMessageOperator(BaseOperator):
|
502
|
-
"""
|
503
|
-
Receive a Batch messages from a Service Bus Subscription under specific Topic.
|
490
|
+
"""Receive a Batch messages from a Service Bus Subscription under specific Topic.
|
504
491
|
|
505
492
|
.. seealso::
|
506
493
|
For more information on how to use this operator, take a look at the guide:
|
@@ -542,7 +529,7 @@ class ASBReceiveSubscriptionMessageOperator(BaseOperator):
|
|
542
529
|
def execute(self, context: Context) -> None:
|
543
530
|
"""
|
544
531
|
Receive Message in specific queue in Service Bus namespace,
|
545
|
-
by connecting to Service Bus client
|
532
|
+
by connecting to Service Bus client.
|
546
533
|
"""
|
547
534
|
# Create the hook
|
548
535
|
hook = MessageHook(azure_service_bus_conn_id=self.azure_service_bus_conn_id)
|
@@ -554,8 +541,7 @@ class ASBReceiveSubscriptionMessageOperator(BaseOperator):
|
|
554
541
|
|
555
542
|
|
556
543
|
class AzureServiceBusSubscriptionDeleteOperator(BaseOperator):
|
557
|
-
"""
|
558
|
-
Deletes the topic subscription in the Azure ServiceBus namespace
|
544
|
+
"""Delete the topic subscription in the Azure ServiceBus namespace.
|
559
545
|
|
560
546
|
.. seealso::
|
561
547
|
For more information on how to use this operator, take a look at the guide:
|
@@ -584,7 +570,7 @@ class AzureServiceBusSubscriptionDeleteOperator(BaseOperator):
|
|
584
570
|
self.azure_service_bus_conn_id = azure_service_bus_conn_id
|
585
571
|
|
586
572
|
def execute(self, context: Context) -> None:
|
587
|
-
"""Delete topic subscription in Service Bus namespace, by connecting to Service Bus Admin client"""
|
573
|
+
"""Delete topic subscription in Service Bus namespace, by connecting to Service Bus Admin client."""
|
588
574
|
# Create the hook
|
589
575
|
hook = AdminClientHook(azure_service_bus_conn_id=self.azure_service_bus_conn_id)
|
590
576
|
|
@@ -593,8 +579,7 @@ class AzureServiceBusSubscriptionDeleteOperator(BaseOperator):
|
|
593
579
|
|
594
580
|
|
595
581
|
class AzureServiceBusTopicDeleteOperator(BaseOperator):
|
596
|
-
"""
|
597
|
-
Deletes the topic in the Azure Service Bus namespace
|
582
|
+
"""Delete the topic in the Azure Service Bus namespace.
|
598
583
|
|
599
584
|
.. seealso::
|
600
585
|
For more information on how to use this operator, take a look at the guide:
|
@@ -620,7 +605,7 @@ class AzureServiceBusTopicDeleteOperator(BaseOperator):
|
|
620
605
|
self.azure_service_bus_conn_id = azure_service_bus_conn_id
|
621
606
|
|
622
607
|
def execute(self, context: Context) -> None:
|
623
|
-
"""Delete topic in Service Bus namespace, by connecting to Service Bus Admin client"""
|
608
|
+
"""Delete topic in Service Bus namespace, by connecting to Service Bus Admin client."""
|
624
609
|
if self.topic_name is None:
|
625
610
|
raise TypeError("Topic name cannot be None.")
|
626
611
|
hook = AdminClientHook(azure_service_bus_conn_id=self.azure_service_bus_conn_id)
|
@@ -31,7 +31,7 @@ if TYPE_CHECKING:
|
|
31
31
|
|
32
32
|
class AzureBatchOperator(BaseOperator):
|
33
33
|
"""
|
34
|
-
Executes a job on Azure Batch Service
|
34
|
+
Executes a job on Azure Batch Service.
|
35
35
|
|
36
36
|
:param batch_pool_id: A string that uniquely identifies the Pool within the Account.
|
37
37
|
:param batch_pool_vm_size: The size of virtual machines in the Pool
|
@@ -316,7 +316,7 @@ class AzureBatchOperator(BaseOperator):
|
|
316
316
|
|
317
317
|
def clean_up(self, pool_id: str | None = None, job_id: str | None = None) -> None:
|
318
318
|
"""
|
319
|
-
Delete the given pool and job in the batch account
|
319
|
+
Delete the given pool and job in the batch account.
|
320
320
|
|
321
321
|
:param pool_id: The id of the pool to delete
|
322
322
|
:param job_id: The id of the job to delete
|
@@ -60,7 +60,7 @@ DEFAULT_CPU = 1.0
|
|
60
60
|
|
61
61
|
class AzureContainerInstancesOperator(BaseOperator):
|
62
62
|
"""
|
63
|
-
Start a container on Azure Container Instances
|
63
|
+
Start a container on Azure Container Instances.
|
64
64
|
|
65
65
|
:param ci_conn_id: connection id of a service principal which will be used
|
66
66
|
to start the container instance
|