azure-storage-blob 12.25.1__py3-none-any.whl → 12.26.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- azure/storage/blob/_blob_client.py +51 -3
- azure/storage/blob/_blob_client_helpers.py +15 -1
- azure/storage/blob/_generated/_azure_blob_storage.py +1 -1
- azure/storage/blob/_generated/_configuration.py +2 -2
- azure/storage/blob/_generated/_serialization.py +3 -3
- azure/storage/blob/_generated/aio/_azure_blob_storage.py +1 -1
- azure/storage/blob/_generated/aio/_configuration.py +2 -2
- azure/storage/blob/_generated/aio/operations/_append_blob_operations.py +5 -4
- azure/storage/blob/_generated/aio/operations/_blob_operations.py +5 -25
- azure/storage/blob/_generated/aio/operations/_block_blob_operations.py +9 -7
- azure/storage/blob/_generated/aio/operations/_container_operations.py +1 -19
- azure/storage/blob/_generated/aio/operations/_page_blob_operations.py +5 -10
- azure/storage/blob/_generated/aio/operations/_service_operations.py +1 -8
- azure/storage/blob/_generated/models/__init__.py +2 -0
- azure/storage/blob/_generated/models/_azure_blob_storage_enums.py +6 -0
- azure/storage/blob/_generated/operations/_append_blob_operations.py +12 -9
- azure/storage/blob/_generated/operations/_blob_operations.py +32 -49
- azure/storage/blob/_generated/operations/_block_blob_operations.py +21 -13
- azure/storage/blob/_generated/operations/_container_operations.py +19 -37
- azure/storage/blob/_generated/operations/_page_blob_operations.py +17 -19
- azure/storage/blob/_generated/operations/_service_operations.py +9 -17
- azure/storage/blob/_quick_query_helper.py +17 -21
- azure/storage/blob/_serialize.py +1 -0
- azure/storage/blob/_shared/base_client.py +1 -0
- azure/storage/blob/_shared/shared_access_signature.py +4 -0
- azure/storage/blob/_shared_access_signature.py +3 -1
- azure/storage/blob/_version.py +1 -1
- azure/storage/blob/aio/_blob_client_async.py +189 -5
- azure/storage/blob/aio/_quick_query_helper_async.py +194 -0
- {azure_storage_blob-12.25.1.dist-info → azure_storage_blob-12.26.0b1.dist-info}/METADATA +2 -2
- {azure_storage_blob-12.25.1.dist-info → azure_storage_blob-12.26.0b1.dist-info}/RECORD +34 -33
- {azure_storage_blob-12.25.1.dist-info → azure_storage_blob-12.26.0b1.dist-info}/LICENSE +0 -0
- {azure_storage_blob-12.25.1.dist-info → azure_storage_blob-12.26.0b1.dist-info}/WHEEL +0 -0
- {azure_storage_blob-12.25.1.dist-info → azure_storage_blob-12.26.0b1.dist-info}/top_level.txt +0 -0
@@ -5,7 +5,10 @@
|
|
5
5
|
# --------------------------------------------------------------------------
|
6
6
|
|
7
7
|
from io import BytesIO
|
8
|
-
from typing import
|
8
|
+
from typing import (
|
9
|
+
Any, Dict, Generator, IO, Iterable, Optional, Type,
|
10
|
+
TYPE_CHECKING
|
11
|
+
)
|
9
12
|
|
10
13
|
from ._shared.avro.avro_io import DatumReader
|
11
14
|
from ._shared.avro.datafile import DataFileReader
|
@@ -14,11 +17,11 @@ if TYPE_CHECKING:
|
|
14
17
|
from ._models import BlobQueryError
|
15
18
|
|
16
19
|
|
17
|
-
class BlobQueryReader
|
20
|
+
class BlobQueryReader: # pylint: disable=too-many-instance-attributes
|
18
21
|
"""A streaming object to read query results."""
|
19
22
|
|
20
23
|
name: str
|
21
|
-
"""The name of the blob being
|
24
|
+
"""The name of the blob being queried."""
|
22
25
|
container: str
|
23
26
|
"""The name of the container where the blob is."""
|
24
27
|
response_headers: Dict[str, Any]
|
@@ -28,8 +31,7 @@ class BlobQueryReader(object): # pylint: disable=too-many-instance-attributes
|
|
28
31
|
method will return these lines via a generator."""
|
29
32
|
|
30
33
|
def __init__(
|
31
|
-
self,
|
32
|
-
name: str = None, # type: ignore [assignment]
|
34
|
+
self, name: str = None, # type: ignore [assignment]
|
33
35
|
container: str = None, # type: ignore [assignment]
|
34
36
|
errors: Any = None,
|
35
37
|
record_delimiter: str = '\n',
|
@@ -50,7 +52,7 @@ class BlobQueryReader(object): # pylint: disable=too-many-instance-attributes
|
|
50
52
|
self._first_result = self._process_record(next(self._parsed_results))
|
51
53
|
self._error_cls = error_cls
|
52
54
|
|
53
|
-
def __len__(self):
|
55
|
+
def __len__(self) -> int:
|
54
56
|
return self._size
|
55
57
|
|
56
58
|
def _process_record(self, result: Dict[str, Any]) -> Optional[bytes]:
|
@@ -77,21 +79,19 @@ class BlobQueryReader(object): # pylint: disable=too-many-instance-attributes
|
|
77
79
|
if processed_result is not None:
|
78
80
|
yield processed_result
|
79
81
|
|
80
|
-
def readall(self) ->
|
82
|
+
def readall(self) -> bytes:
|
81
83
|
"""Return all query results.
|
82
84
|
|
83
85
|
This operation is blocking until all data is downloaded.
|
84
|
-
If encoding has been configured - this will be used to decode individual
|
85
|
-
records are they are received.
|
86
86
|
|
87
87
|
:returns: The query results.
|
88
|
-
:rtype:
|
88
|
+
:rtype: bytes
|
89
89
|
"""
|
90
90
|
stream = BytesIO()
|
91
91
|
self.readinto(stream)
|
92
92
|
data = stream.getvalue()
|
93
93
|
if self._encoding:
|
94
|
-
return data.decode(self._encoding)
|
94
|
+
return data.decode(self._encoding) # type: ignore [return-value]
|
95
95
|
return data
|
96
96
|
|
97
97
|
def readinto(self, stream: IO) -> None:
|
@@ -105,29 +105,25 @@ class BlobQueryReader(object): # pylint: disable=too-many-instance-attributes
|
|
105
105
|
for record in self._iter_stream():
|
106
106
|
stream.write(record)
|
107
107
|
|
108
|
-
def records(self) -> Iterable[
|
108
|
+
def records(self) -> Iterable[bytes]:
|
109
109
|
"""Returns a record generator for the query result.
|
110
110
|
|
111
111
|
Records will be returned line by line.
|
112
|
-
If encoding has been configured - this will be used to decode individual
|
113
|
-
records are they are received.
|
114
112
|
|
115
113
|
:returns: A record generator for the query result.
|
116
|
-
:rtype: Iterable[
|
114
|
+
:rtype: Iterable[bytes]
|
117
115
|
"""
|
118
116
|
delimiter = self.record_delimiter.encode('utf-8')
|
119
117
|
for record_chunk in self._iter_stream():
|
120
118
|
for record in record_chunk.split(delimiter):
|
121
119
|
if self._encoding:
|
122
|
-
yield record.decode(self._encoding)
|
120
|
+
yield record.decode(self._encoding) # type: ignore [misc]
|
123
121
|
else:
|
124
122
|
yield record
|
125
123
|
|
126
124
|
|
127
|
-
class QuickQueryStreamer
|
128
|
-
"""
|
129
|
-
File-like streaming iterator.
|
130
|
-
"""
|
125
|
+
class QuickQueryStreamer:
|
126
|
+
"""File-like streaming iterator."""
|
131
127
|
|
132
128
|
def __init__(self, generator):
|
133
129
|
self.generator = generator
|
@@ -183,7 +179,7 @@ class QuickQueryStreamer(object):
|
|
183
179
|
if relative_start < 0:
|
184
180
|
raise ValueError("Buffer has dumped too much data")
|
185
181
|
relative_end = relative_start + size
|
186
|
-
data = self._buf[relative_start:
|
182
|
+
data = self._buf[relative_start:relative_end]
|
187
183
|
|
188
184
|
# dump the extra data in buffer
|
189
185
|
# buffer start--------------------16bytes----current read position
|
azure/storage/blob/_serialize.py
CHANGED
@@ -131,6 +131,7 @@ class StorageAccountHostsMixin(object):
|
|
131
131
|
|
132
132
|
This could be either the primary endpoint,
|
133
133
|
or the secondary endpoint depending on the current :func:`location_mode`.
|
134
|
+
|
134
135
|
:returns: The full endpoint URL to this entity, including SAS token if used.
|
135
136
|
:rtype: str
|
136
137
|
"""
|
@@ -41,6 +41,8 @@ class QueryStringConstants(object):
|
|
41
41
|
SIGNED_KEY_SERVICE = 'sks'
|
42
42
|
SIGNED_KEY_VERSION = 'skv'
|
43
43
|
SIGNED_ENCRYPTION_SCOPE = 'ses'
|
44
|
+
SIGNED_KEY_DELEGATED_USER_TID = 'skdutid'
|
45
|
+
SIGNED_DELEGATED_USER_OID = 'sduoid'
|
44
46
|
|
45
47
|
# for ADLS
|
46
48
|
SIGNED_AUTHORIZED_OID = 'saoid'
|
@@ -78,6 +80,8 @@ class QueryStringConstants(object):
|
|
78
80
|
QueryStringConstants.SIGNED_KEY_SERVICE,
|
79
81
|
QueryStringConstants.SIGNED_KEY_VERSION,
|
80
82
|
QueryStringConstants.SIGNED_ENCRYPTION_SCOPE,
|
83
|
+
QueryStringConstants.SIGNED_KEY_DELEGATED_USER_TID,
|
84
|
+
QueryStringConstants.SIGNED_DELEGATED_USER_OID,
|
81
85
|
# for ADLS
|
82
86
|
QueryStringConstants.SIGNED_AUTHORIZED_OID,
|
83
87
|
QueryStringConstants.SIGNED_UNAUTHORIZED_OID,
|
@@ -311,7 +311,9 @@ class _BlobSharedAccessHelper(_SharedAccessHelper):
|
|
311
311
|
self.get_value_to_append(QueryStringConstants.SIGNED_KEY_VERSION) +
|
312
312
|
self.get_value_to_append(QueryStringConstants.SIGNED_AUTHORIZED_OID) +
|
313
313
|
self.get_value_to_append(QueryStringConstants.SIGNED_UNAUTHORIZED_OID) +
|
314
|
-
self.get_value_to_append(QueryStringConstants.SIGNED_CORRELATION_ID)
|
314
|
+
self.get_value_to_append(QueryStringConstants.SIGNED_CORRELATION_ID) +
|
315
|
+
self.get_value_to_append(QueryStringConstants.SIGNED_KEY_DELEGATED_USER_TID) +
|
316
|
+
self.get_value_to_append(QueryStringConstants.SIGNED_DELEGATED_USER_OID))
|
315
317
|
else:
|
316
318
|
string_to_sign += self.get_value_to_append(QueryStringConstants.SIGNED_IDENTIFIER)
|
317
319
|
|
azure/storage/blob/_version.py
CHANGED
@@ -9,7 +9,8 @@ import warnings
|
|
9
9
|
from datetime import datetime
|
10
10
|
from functools import partial
|
11
11
|
from typing import (
|
12
|
-
Any, AnyStr, AsyncIterable, cast, Dict, IO,
|
12
|
+
Any, AnyStr, AsyncIterable, Callable, cast, Dict, IO,
|
13
|
+
Iterable, List, Optional, overload, Tuple, Union,
|
13
14
|
TYPE_CHECKING
|
14
15
|
)
|
15
16
|
from typing_extensions import Self
|
@@ -23,6 +24,7 @@ from azure.core.tracing.decorator_async import distributed_trace_async
|
|
23
24
|
from ._download_async import StorageStreamDownloader
|
24
25
|
from ._lease_async import BlobLeaseClient
|
25
26
|
from ._models import PageRangePaged
|
27
|
+
from ._quick_query_helper_async import BlobQueryReader
|
26
28
|
from ._upload_helpers import (
|
27
29
|
upload_append_blob,
|
28
30
|
upload_block_blob,
|
@@ -46,6 +48,7 @@ from .._blob_client_helpers import (
|
|
46
48
|
_get_block_list_result,
|
47
49
|
_get_page_ranges_options,
|
48
50
|
_parse_url,
|
51
|
+
_quick_query_options,
|
49
52
|
_resize_blob_options,
|
50
53
|
_seal_append_blob_options,
|
51
54
|
_set_blob_metadata_options,
|
@@ -69,21 +72,27 @@ from .._deserialize import (
|
|
69
72
|
from .._encryption import StorageEncryptionMixin, _ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION
|
70
73
|
from .._generated.aio import AzureBlobStorage
|
71
74
|
from .._generated.models import CpkInfo
|
72
|
-
from .._models import BlobType, BlobBlock, BlobProperties, PageRange
|
75
|
+
from .._models import BlobType, BlobBlock, BlobProperties, BlobQueryError, PageRange
|
73
76
|
from .._serialize import get_access_conditions, get_api_version, get_modify_conditions, get_version_id
|
74
77
|
from .._shared.base_client_async import AsyncStorageAccountHostsMixin, AsyncTransportWrapper, parse_connection_str
|
75
78
|
from .._shared.policies_async import ExponentialRetry
|
76
79
|
from .._shared.response_handlers import process_storage_error, return_response_headers
|
77
80
|
|
78
81
|
if TYPE_CHECKING:
|
82
|
+
from azure.core import MatchConditions
|
79
83
|
from azure.core.credentials import AzureNamedKeyCredential, AzureSasCredential
|
80
84
|
from azure.core.credentials_async import AsyncTokenCredential
|
81
85
|
from azure.core.pipeline.policies import AsyncHTTPPolicy
|
86
|
+
from azure.storage.blob import CustomerProvidedEncryptionKey
|
82
87
|
from azure.storage.blob.aio import ContainerClient
|
83
88
|
from .._models import (
|
89
|
+
ArrowDialect,
|
84
90
|
ContentSettings,
|
91
|
+
DelimitedJsonDialect,
|
92
|
+
DelimitedTextDialect,
|
85
93
|
ImmutabilityPolicy,
|
86
94
|
PremiumPageBlobTier,
|
95
|
+
QuickQueryDialect,
|
87
96
|
SequenceNumberAction,
|
88
97
|
StandardBlobTier
|
89
98
|
)
|
@@ -412,6 +421,15 @@ class BlobClient(AsyncStorageAccountHostsMixin, StorageAccountHostsMixin, Storag
|
|
412
421
|
:keyword str source_authorization:
|
413
422
|
Authenticate as a service principal using a client secret to access a source blob. Ensure "bearer " is
|
414
423
|
the prefix of the source_authorization string.
|
424
|
+
:keyword source_token_intent:
|
425
|
+
Required when source is Azure Storage Files and using `TokenCredential` for authentication.
|
426
|
+
This is ignored for other forms of authentication.
|
427
|
+
Specifies the intent for all requests when using `TokenCredential` authentication. Possible values are:
|
428
|
+
|
429
|
+
backup - Specifies requests are intended for backup/admin type operations, meaning that all file/directory
|
430
|
+
ACLs are bypassed and full permissions are granted. User must also have required RBAC permission.
|
431
|
+
|
432
|
+
:paramtype source_token_intent: Literal['backup']
|
415
433
|
:returns: Response from creating a new block blob for a given URL.
|
416
434
|
:rtype: Dict[str, Any]
|
417
435
|
"""
|
@@ -420,7 +438,8 @@ class BlobClient(AsyncStorageAccountHostsMixin, StorageAccountHostsMixin, Storag
|
|
420
438
|
options = _upload_blob_from_url_options(
|
421
439
|
source_url=source_url,
|
422
440
|
metadata=metadata,
|
423
|
-
**kwargs
|
441
|
+
**kwargs
|
442
|
+
)
|
424
443
|
try:
|
425
444
|
return cast(Dict[str, Any], await self._client.block_blob.put_blob_from_url(**options))
|
426
445
|
except HttpResponseError as error:
|
@@ -746,6 +765,133 @@ class BlobClient(AsyncStorageAccountHostsMixin, StorageAccountHostsMixin, Storag
|
|
746
765
|
await downloader._setup() # pylint: disable=protected-access
|
747
766
|
return downloader
|
748
767
|
|
768
|
+
@distributed_trace_async
|
769
|
+
async def query_blob(
|
770
|
+
self, query_expression: str,
|
771
|
+
*,
|
772
|
+
on_error: Optional[Callable[[BlobQueryError], None]] = None,
|
773
|
+
blob_format: Optional[Union["DelimitedTextDialect", "DelimitedJsonDialect", "QuickQueryDialect", str]] = None,
|
774
|
+
output_format: Optional[Union["DelimitedTextDialect", "DelimitedJsonDialect", "QuickQueryDialect", List["ArrowDialect"], str]] = None, # pylint: disable=line-too-long
|
775
|
+
lease: Optional[Union[BlobLeaseClient, str]] = None,
|
776
|
+
if_modified_since: Optional[datetime] = None,
|
777
|
+
if_unmodified_since: Optional[datetime] = None,
|
778
|
+
etag: Optional[str] = None,
|
779
|
+
match_condition: Optional["MatchConditions"] = None,
|
780
|
+
if_tags_match_condition: Optional[str] = None,
|
781
|
+
cpk: Optional["CustomerProvidedEncryptionKey"] = None,
|
782
|
+
timeout: Optional[int] = None,
|
783
|
+
**kwargs: Any
|
784
|
+
) -> BlobQueryReader:
|
785
|
+
"""Enables users to select/project on blob/or blob snapshot data by providing simple query expressions.
|
786
|
+
This operation returns a BlobQueryReader, users need to use readall() or readinto() to get query data.
|
787
|
+
|
788
|
+
:param str query_expression:
|
789
|
+
Required. a query statement. For more details see
|
790
|
+
https://learn.microsoft.com/azure/storage/blobs/query-acceleration-sql-reference.
|
791
|
+
:keyword Callable[~azure.storage.blob.BlobQueryError] on_error:
|
792
|
+
A function to be called on any processing errors returned by the service.
|
793
|
+
:keyword blob_format:
|
794
|
+
Optional. Defines the serialization of the data currently stored in the blob. The default is to
|
795
|
+
treat the blob data as CSV data formatted in the default dialect. This can be overridden with
|
796
|
+
a custom DelimitedTextDialect, or DelimitedJsonDialect or "ParquetDialect" (passed as a string or enum).
|
797
|
+
These dialects can be passed through their respective classes, the QuickQueryDialect enum or as a string.
|
798
|
+
|
799
|
+
.. note::
|
800
|
+
"ParquetDialect" is in preview, so some features may not work as intended.
|
801
|
+
|
802
|
+
:paramtype blob_format:
|
803
|
+
~azure.storage.blob.DelimitedTextDialect or
|
804
|
+
~azure.storage.blob.DelimitedJsonDialect or
|
805
|
+
~azure.storage.blob.QuickQueryDialect or
|
806
|
+
str
|
807
|
+
:keyword output_format:
|
808
|
+
Optional. Defines the output serialization for the data stream. By default the data will be returned
|
809
|
+
as it is represented in the blob (Parquet formats default to DelimitedTextDialect).
|
810
|
+
By providing an output format, the blob data will be reformatted according to that profile.
|
811
|
+
This value can be a DelimitedTextDialect or a DelimitedJsonDialect or ArrowDialect.
|
812
|
+
These dialects can be passed through their respective classes, the QuickQueryDialect enum or as a string.
|
813
|
+
:paramtype output_format:
|
814
|
+
~azure.storage.blob.DelimitedTextDialect or
|
815
|
+
~azure.storage.blob.DelimitedJsonDialect or
|
816
|
+
~azure.storage.blob.QuickQueryDialect or
|
817
|
+
List[~azure.storage.blob.ArrowDialect] or
|
818
|
+
str
|
819
|
+
:keyword lease:
|
820
|
+
Required if the blob has an active lease. Value can be a BlobLeaseClient object
|
821
|
+
or the lease ID as a string.
|
822
|
+
:keyword ~datetime.datetime if_modified_since:
|
823
|
+
A DateTime value. Azure expects the date value passed in to be UTC.
|
824
|
+
If timezone is included, any non-UTC datetime will be converted to UTC.
|
825
|
+
If a date is passed in without timezone info, it is assumed to be UTC.
|
826
|
+
Specify this header to perform the operation only if
|
827
|
+
the resource has been modified since the specified date/time.
|
828
|
+
:keyword ~datetime.datetime if_unmodified_since:
|
829
|
+
A DateTime value. Azure expects the date value passed in to be UTC.
|
830
|
+
If timezone is included, any non-UTC datetime will be converted to UTC.
|
831
|
+
If a date is passed in without timezone info, it is assumed to be UTC.
|
832
|
+
Specify this header to perform the operation only if
|
833
|
+
the resource has not been modified since the specified date/time.
|
834
|
+
:keyword str etag:
|
835
|
+
An ETag value, or the wildcard character (*). Used to check if the resource has changed,
|
836
|
+
and act according to the condition specified by the `match_condition` parameter.
|
837
|
+
:keyword ~azure.core.MatchConditions match_condition:
|
838
|
+
The match condition to use upon the etag.
|
839
|
+
:keyword str if_tags_match_condition:
|
840
|
+
Specify a SQL where clause on blob tags to operate only on blob with a matching value.
|
841
|
+
eg. ``\"\\\"tagname\\\"='my tag'\"``
|
842
|
+
|
843
|
+
.. versionadded:: 12.4.0
|
844
|
+
|
845
|
+
:keyword ~azure.storage.blob.CustomerProvidedEncryptionKey cpk:
|
846
|
+
Encrypts the data on the service-side with the given key.
|
847
|
+
Use of customer-provided keys must be done over HTTPS.
|
848
|
+
As the encryption key itself is provided in the request,
|
849
|
+
a secure connection must be established to transfer the key.
|
850
|
+
:keyword int timeout:
|
851
|
+
Sets the server-side timeout for the operation in seconds. For more details see
|
852
|
+
https://learn.microsoft.com/rest/api/storageservices/setting-timeouts-for-blob-service-operations.
|
853
|
+
This value is not tracked or validated on the client. To configure client-side network timeouts
|
854
|
+
see `here <https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/storage/azure-storage-blob
|
855
|
+
#other-client--per-operation-configuration>`__.
|
856
|
+
:returns: A streaming object (BlobQueryReader)
|
857
|
+
:rtype: ~azure.storage.blob.aio.BlobQueryReader
|
858
|
+
"""
|
859
|
+
error_cls = kwargs.pop("error_cls", BlobQueryError)
|
860
|
+
encoding = kwargs.pop("encoding", None)
|
861
|
+
if cpk and self.scheme.lower() != 'https':
|
862
|
+
raise ValueError("Customer provided encryption key must be used over HTTPS.")
|
863
|
+
options, delimiter = _quick_query_options(
|
864
|
+
self.snapshot,
|
865
|
+
query_expression,
|
866
|
+
blob_format=blob_format,
|
867
|
+
output_format=output_format,
|
868
|
+
lease=lease,
|
869
|
+
if_modified_since=if_modified_since,
|
870
|
+
if_unmodified_since=if_unmodified_since,
|
871
|
+
etag=etag,
|
872
|
+
match_condition=match_condition,
|
873
|
+
if_tags_match_condition=if_tags_match_condition,
|
874
|
+
cpk=cpk,
|
875
|
+
timeout=timeout,
|
876
|
+
**kwargs
|
877
|
+
)
|
878
|
+
try:
|
879
|
+
headers, raw_response_body = await self._client.blob.query(**options)
|
880
|
+
except HttpResponseError as error:
|
881
|
+
process_storage_error(error)
|
882
|
+
blob_query_reader = BlobQueryReader(
|
883
|
+
name=self.blob_name,
|
884
|
+
container=self.container_name,
|
885
|
+
errors=on_error,
|
886
|
+
record_delimiter=delimiter,
|
887
|
+
encoding=encoding,
|
888
|
+
headers=headers,
|
889
|
+
response=raw_response_body,
|
890
|
+
error_cls=error_cls
|
891
|
+
)
|
892
|
+
await blob_query_reader._setup() # pylint: disable=protected-access
|
893
|
+
return blob_query_reader
|
894
|
+
|
749
895
|
@distributed_trace_async
|
750
896
|
async def delete_blob(self, delete_snapshots: Optional[str] = None, **kwargs: Any) -> None:
|
751
897
|
"""Marks the specified blob for deletion.
|
@@ -1650,6 +1796,15 @@ class BlobClient(AsyncStorageAccountHostsMixin, StorageAccountHostsMixin, Storag
|
|
1650
1796
|
|
1651
1797
|
.. versionadded:: 12.9.0
|
1652
1798
|
|
1799
|
+
:keyword source_token_intent:
|
1800
|
+
Required when source is Azure Storage Files and using `TokenCredential` for authentication.
|
1801
|
+
This is ignored for other forms of authentication.
|
1802
|
+
Specifies the intent for all requests when using `TokenCredential` authentication. Possible values are:
|
1803
|
+
|
1804
|
+
backup - Specifies requests are intended for backup/admin type operations, meaning that all file/directory
|
1805
|
+
ACLs are bypassed and full permissions are granted. User must also have required RBAC permission.
|
1806
|
+
|
1807
|
+
:paramtype source_token_intent: Literal['backup']
|
1653
1808
|
:keyword str encryption_scope:
|
1654
1809
|
A predefined encryption scope used to encrypt the data on the sync copied blob. An encryption
|
1655
1810
|
scope can be created using the Management API and referenced here by name. If a default
|
@@ -1674,7 +1829,8 @@ class BlobClient(AsyncStorageAccountHostsMixin, StorageAccountHostsMixin, Storag
|
|
1674
1829
|
source_url=source_url,
|
1675
1830
|
metadata=metadata,
|
1676
1831
|
incremental_copy=incremental_copy,
|
1677
|
-
**kwargs
|
1832
|
+
**kwargs
|
1833
|
+
)
|
1678
1834
|
try:
|
1679
1835
|
if incremental_copy:
|
1680
1836
|
return cast(Dict[str, Union[str, datetime]], await self._client.page_blob.copy_incremental(**options))
|
@@ -1944,6 +2100,15 @@ class BlobClient(AsyncStorageAccountHostsMixin, StorageAccountHostsMixin, Storag
|
|
1944
2100
|
:keyword str source_authorization:
|
1945
2101
|
Authenticate as a service principal using a client secret to access a source blob. Ensure "bearer " is
|
1946
2102
|
the prefix of the source_authorization string.
|
2103
|
+
:keyword source_token_intent:
|
2104
|
+
Required when source is Azure Storage Files and using `TokenCredential` for authentication.
|
2105
|
+
This is ignored for other forms of authentication.
|
2106
|
+
Specifies the intent for all requests when using `TokenCredential` authentication. Possible values are:
|
2107
|
+
|
2108
|
+
backup - Specifies requests are intended for backup/admin type operations, meaning that all file/directory
|
2109
|
+
ACLs are bypassed and full permissions are granted. User must also have required RBAC permission.
|
2110
|
+
|
2111
|
+
:paramtype source_token_intent: Literal['backup']
|
1947
2112
|
:returns: Blob property dict.
|
1948
2113
|
:rtype: Dict[str, Any]
|
1949
2114
|
"""
|
@@ -1955,7 +2120,8 @@ class BlobClient(AsyncStorageAccountHostsMixin, StorageAccountHostsMixin, Storag
|
|
1955
2120
|
source_offset=source_offset,
|
1956
2121
|
source_length=source_length,
|
1957
2122
|
source_content_md5=source_content_md5,
|
1958
|
-
**kwargs
|
2123
|
+
**kwargs
|
2124
|
+
)
|
1959
2125
|
try:
|
1960
2126
|
return cast(Dict[str, Any], await self._client.block_blob.stage_block_from_url(**options))
|
1961
2127
|
except HttpResponseError as error:
|
@@ -2819,6 +2985,15 @@ class BlobClient(AsyncStorageAccountHostsMixin, StorageAccountHostsMixin, Storag
|
|
2819
2985
|
:keyword str source_authorization:
|
2820
2986
|
Authenticate as a service principal using a client secret to access a source blob. Ensure "bearer " is
|
2821
2987
|
the prefix of the source_authorization string.
|
2988
|
+
:keyword source_token_intent:
|
2989
|
+
Required when source is Azure Storage Files and using `TokenCredential` for authentication.
|
2990
|
+
This is ignored for other forms of authentication.
|
2991
|
+
Specifies the intent for all requests when using `TokenCredential` authentication. Possible values are:
|
2992
|
+
|
2993
|
+
backup - Specifies requests are intended for backup/admin type operations, meaning that all file/directory
|
2994
|
+
ACLs are bypassed and full permissions are granted. User must also have required RBAC permission.
|
2995
|
+
|
2996
|
+
:paramtype source_token_intent: Literal['backup']
|
2822
2997
|
:returns: Response after uploading pages from specified URL.
|
2823
2998
|
:rtype: Dict[str, Any]
|
2824
2999
|
"""
|
@@ -3112,6 +3287,15 @@ class BlobClient(AsyncStorageAccountHostsMixin, StorageAccountHostsMixin, Storag
|
|
3112
3287
|
:keyword str source_authorization:
|
3113
3288
|
Authenticate as a service principal using a client secret to access a source blob. Ensure "bearer " is
|
3114
3289
|
the prefix of the source_authorization string.
|
3290
|
+
:keyword source_token_intent:
|
3291
|
+
Required when source is Azure Storage Files and using `TokenCredential` for authentication.
|
3292
|
+
This is ignored for other forms of authentication.
|
3293
|
+
Specifies the intent for all requests when using `TokenCredential` authentication. Possible values are:
|
3294
|
+
|
3295
|
+
backup - Specifies requests are intended for backup/admin type operations, meaning that all file/directory
|
3296
|
+
ACLs are bypassed and full permissions are granted. User must also have required RBAC permission.
|
3297
|
+
|
3298
|
+
:paramtype source_token_intent: Literal['backup']
|
3115
3299
|
:returns: Result after appending a new block.
|
3116
3300
|
:rtype: Dict[str, Union[str, datetime, int]]
|
3117
3301
|
"""
|
@@ -0,0 +1,194 @@
|
|
1
|
+
# -------------------------------------------------------------------------
|
2
|
+
# Copyright (c) Microsoft Corporation. All rights reserved.
|
3
|
+
# Licensed under the MIT License. See License.txt in the project root for
|
4
|
+
# license information.
|
5
|
+
# --------------------------------------------------------------------------
|
6
|
+
|
7
|
+
from io import BytesIO
|
8
|
+
from typing import (
|
9
|
+
Any, AsyncGenerator, AsyncIterable, Dict, IO, Optional, Type,
|
10
|
+
TYPE_CHECKING
|
11
|
+
)
|
12
|
+
|
13
|
+
from .._shared.avro.avro_io_async import AsyncDatumReader
|
14
|
+
from .._shared.avro.datafile_async import AsyncDataFileReader
|
15
|
+
|
16
|
+
if TYPE_CHECKING:
|
17
|
+
from .._models import BlobQueryError
|
18
|
+
|
19
|
+
|
20
|
+
class BlobQueryReader: # pylint: disable=too-many-instance-attributes
|
21
|
+
"""A streaming object to read query results."""
|
22
|
+
|
23
|
+
name: str
|
24
|
+
"""The name of the blob being queried."""
|
25
|
+
container: str
|
26
|
+
"""The name of the container where the blob is."""
|
27
|
+
response_headers: Dict[str, Any]
|
28
|
+
"""The response headers of the quick query request."""
|
29
|
+
record_delimiter: str
|
30
|
+
"""The delimiter used to separate lines, or records with the data. The `records`
|
31
|
+
method will return these lines via a generator."""
|
32
|
+
|
33
|
+
def __init__(
|
34
|
+
self, name: str = None, # type: ignore [assignment]
|
35
|
+
container: str = None, # type: ignore [assignment]
|
36
|
+
errors: Any = None,
|
37
|
+
record_delimiter: str = '\n',
|
38
|
+
encoding: Optional[str] = None,
|
39
|
+
headers: Dict[str, Any] = None, # type: ignore [assignment]
|
40
|
+
response: Any = None,
|
41
|
+
error_cls: Type["BlobQueryError"] = None, # type: ignore [assignment]
|
42
|
+
) -> None:
|
43
|
+
self.name = name
|
44
|
+
self.container = container
|
45
|
+
self.response_headers = headers
|
46
|
+
self.record_delimiter = record_delimiter
|
47
|
+
self._size = 0
|
48
|
+
self._bytes_processed = 0
|
49
|
+
self._errors = errors
|
50
|
+
self._encoding = encoding
|
51
|
+
self._parsed_results = AsyncDataFileReader(QuickQueryStreamer(response), AsyncDatumReader())
|
52
|
+
self._error_cls = error_cls
|
53
|
+
|
54
|
+
async def _setup(self):
|
55
|
+
self._parsed_results = await self._parsed_results.init()
|
56
|
+
first_result = await self._parsed_results.__anext__()
|
57
|
+
self._first_result = self._process_record(first_result) # pylint: disable=attribute-defined-outside-init
|
58
|
+
|
59
|
+
def __len__(self) -> int:
|
60
|
+
return self._size
|
61
|
+
|
62
|
+
def _process_record(self, result: Dict[str, Any]) -> Optional[bytes]:
|
63
|
+
self._size = result.get('totalBytes', self._size)
|
64
|
+
self._bytes_processed = result.get('bytesScanned', self._bytes_processed)
|
65
|
+
if 'data' in result:
|
66
|
+
return result.get('data')
|
67
|
+
if 'fatal' in result:
|
68
|
+
error = self._error_cls(
|
69
|
+
error=result['name'],
|
70
|
+
is_fatal=result['fatal'],
|
71
|
+
description=result['description'],
|
72
|
+
position=result['position']
|
73
|
+
)
|
74
|
+
if self._errors:
|
75
|
+
self._errors(error)
|
76
|
+
return None
|
77
|
+
|
78
|
+
async def _aiter_stream(self) -> AsyncGenerator[bytes, None]:
|
79
|
+
if self._first_result is not None:
|
80
|
+
yield self._first_result
|
81
|
+
async for next_result in self._parsed_results:
|
82
|
+
processed_result = self._process_record(next_result)
|
83
|
+
if processed_result is not None:
|
84
|
+
yield processed_result
|
85
|
+
|
86
|
+
async def readall(self) -> bytes:
|
87
|
+
"""Return all query results.
|
88
|
+
|
89
|
+
This operation is blocking until all data is downloaded.
|
90
|
+
|
91
|
+
:returns: The query results.
|
92
|
+
:rtype: bytes
|
93
|
+
"""
|
94
|
+
stream = BytesIO()
|
95
|
+
await self.readinto(stream)
|
96
|
+
data = stream.getvalue()
|
97
|
+
if self._encoding:
|
98
|
+
return data.decode(self._encoding) # type: ignore [return-value]
|
99
|
+
return data
|
100
|
+
|
101
|
+
async def readinto(self, stream: IO) -> None:
|
102
|
+
"""Download the query result to a stream.
|
103
|
+
|
104
|
+
:param IO stream:
|
105
|
+
The stream to download to. This can be an open file-handle,
|
106
|
+
or any writable stream.
|
107
|
+
:returns: None
|
108
|
+
"""
|
109
|
+
async for record in self._aiter_stream():
|
110
|
+
stream.write(record)
|
111
|
+
|
112
|
+
async def records(self) -> AsyncIterable[bytes]:
|
113
|
+
"""Returns a record generator for the query result.
|
114
|
+
|
115
|
+
Records will be returned line by line.
|
116
|
+
|
117
|
+
:returns: A record generator for the query result.
|
118
|
+
:rtype: AsyncIterable[bytes]
|
119
|
+
"""
|
120
|
+
delimiter = self.record_delimiter.encode('utf-8')
|
121
|
+
async for record_chunk in self._aiter_stream():
|
122
|
+
for record in record_chunk.split(delimiter):
|
123
|
+
if self._encoding:
|
124
|
+
yield record.decode(self._encoding) # type: ignore [misc]
|
125
|
+
else:
|
126
|
+
yield record
|
127
|
+
|
128
|
+
|
129
|
+
class QuickQueryStreamer:
|
130
|
+
"""File-like streaming iterator."""
|
131
|
+
|
132
|
+
def __init__(self, generator):
|
133
|
+
self.generator = generator
|
134
|
+
self.iterator = generator.__aiter__()
|
135
|
+
self._buf = b""
|
136
|
+
self._point = 0
|
137
|
+
self._download_offset = 0
|
138
|
+
self._buf_start = 0
|
139
|
+
self.file_length = None
|
140
|
+
|
141
|
+
def __len__(self):
|
142
|
+
return self.file_length
|
143
|
+
|
144
|
+
def __aiter__(self):
|
145
|
+
return self.iterator
|
146
|
+
|
147
|
+
@staticmethod
|
148
|
+
def seekable():
|
149
|
+
return True
|
150
|
+
|
151
|
+
async def __anext__(self):
|
152
|
+
next_part = await self.iterator.__anext__()
|
153
|
+
self._download_offset += len(next_part)
|
154
|
+
return next_part
|
155
|
+
|
156
|
+
def tell(self):
|
157
|
+
return self._point
|
158
|
+
|
159
|
+
async def seek(self, offset, whence=0):
|
160
|
+
if whence == 0:
|
161
|
+
self._point = offset
|
162
|
+
elif whence == 1:
|
163
|
+
self._point += offset
|
164
|
+
else:
|
165
|
+
raise ValueError("whence must be 0 or 1")
|
166
|
+
if self._point < 0: # pylint: disable=consider-using-max-builtin
|
167
|
+
self._point = 0
|
168
|
+
|
169
|
+
async def read(self, size):
|
170
|
+
try:
|
171
|
+
# keep reading from the generator until the buffer of this stream has enough data to read
|
172
|
+
while self._point + size > self._download_offset:
|
173
|
+
self._buf += await self.__anext__()
|
174
|
+
except StopAsyncIteration:
|
175
|
+
self.file_length = self._download_offset
|
176
|
+
|
177
|
+
start_point = self._point
|
178
|
+
|
179
|
+
# EOF
|
180
|
+
self._point = min(self._point + size, self._download_offset)
|
181
|
+
|
182
|
+
relative_start = start_point - self._buf_start
|
183
|
+
if relative_start < 0:
|
184
|
+
raise ValueError("Buffer has dumped too much data")
|
185
|
+
relative_end = relative_start + size
|
186
|
+
data = self._buf[relative_start:relative_end]
|
187
|
+
|
188
|
+
# dump the extra data in buffer
|
189
|
+
# buffer start--------------------16bytes----current read position
|
190
|
+
dumped_size = max(relative_end - 16 - relative_start, 0)
|
191
|
+
self._buf_start += dumped_size
|
192
|
+
self._buf = self._buf[dumped_size:]
|
193
|
+
|
194
|
+
return data
|
@@ -1,13 +1,13 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: azure-storage-blob
|
3
|
-
Version: 12.
|
3
|
+
Version: 12.26.0b1
|
4
4
|
Summary: Microsoft Azure Blob Storage Client Library for Python
|
5
5
|
Home-page: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/storage/azure-storage-blob
|
6
6
|
Author: Microsoft Corporation
|
7
7
|
Author-email: ascl@microsoft.com
|
8
8
|
License: MIT License
|
9
9
|
Keywords: azure,azure sdk
|
10
|
-
Classifier: Development Status ::
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
11
11
|
Classifier: Programming Language :: Python
|
12
12
|
Classifier: Programming Language :: Python :: 3 :: Only
|
13
13
|
Classifier: Programming Language :: Python :: 3
|