azure-storage-blob 12.26.0b1__py3-none-any.whl → 12.27.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- azure/storage/blob/__init__.py +3 -2
- azure/storage/blob/_blob_client.py +43 -38
- azure/storage/blob/_blob_client_helpers.py +4 -3
- azure/storage/blob/_blob_service_client.py +16 -13
- azure/storage/blob/_container_client.py +25 -22
- azure/storage/blob/_deserialize.py +1 -1
- azure/storage/blob/_download.py +7 -7
- azure/storage/blob/_encryption.py +177 -184
- azure/storage/blob/_generated/_azure_blob_storage.py +3 -2
- azure/storage/blob/_generated/_configuration.py +2 -2
- azure/storage/blob/_generated/_utils/__init__.py +6 -0
- azure/storage/blob/_generated/{_serialization.py → _utils/serialization.py} +4 -22
- azure/storage/blob/_generated/aio/_azure_blob_storage.py +3 -2
- azure/storage/blob/_generated/aio/_configuration.py +2 -2
- azure/storage/blob/_generated/aio/operations/_append_blob_operations.py +6 -10
- azure/storage/blob/_generated/aio/operations/_blob_operations.py +35 -39
- azure/storage/blob/_generated/aio/operations/_block_blob_operations.py +9 -13
- azure/storage/blob/_generated/aio/operations/_container_operations.py +20 -24
- azure/storage/blob/_generated/aio/operations/_page_blob_operations.py +13 -17
- azure/storage/blob/_generated/aio/operations/_service_operations.py +10 -14
- azure/storage/blob/_generated/models/_models_py3.py +30 -9
- azure/storage/blob/_generated/operations/_append_blob_operations.py +11 -15
- azure/storage/blob/_generated/operations/_blob_operations.py +60 -64
- azure/storage/blob/_generated/operations/_block_blob_operations.py +16 -20
- azure/storage/blob/_generated/operations/_container_operations.py +39 -43
- azure/storage/blob/_generated/operations/_page_blob_operations.py +23 -27
- azure/storage/blob/_generated/operations/_service_operations.py +19 -23
- azure/storage/blob/_lease.py +1 -0
- azure/storage/blob/_list_blobs_helper.py +1 -1
- azure/storage/blob/_quick_query_helper.py +3 -3
- azure/storage/blob/_serialize.py +1 -0
- azure/storage/blob/_shared/__init__.py +7 -7
- azure/storage/blob/_shared/authentication.py +49 -32
- azure/storage/blob/_shared/avro/avro_io.py +44 -42
- azure/storage/blob/_shared/avro/avro_io_async.py +42 -41
- azure/storage/blob/_shared/avro/datafile.py +24 -21
- azure/storage/blob/_shared/avro/datafile_async.py +15 -15
- azure/storage/blob/_shared/avro/schema.py +196 -217
- azure/storage/blob/_shared/base_client.py +81 -59
- azure/storage/blob/_shared/base_client_async.py +58 -51
- azure/storage/blob/_shared/constants.py +1 -1
- azure/storage/blob/_shared/models.py +94 -92
- azure/storage/blob/_shared/parser.py +3 -3
- azure/storage/blob/_shared/policies.py +186 -147
- azure/storage/blob/_shared/policies_async.py +53 -65
- azure/storage/blob/_shared/request_handlers.py +50 -45
- azure/storage/blob/_shared/response_handlers.py +54 -45
- azure/storage/blob/_shared/shared_access_signature.py +65 -73
- azure/storage/blob/_shared/uploads.py +56 -49
- azure/storage/blob/_shared/uploads_async.py +70 -58
- azure/storage/blob/_version.py +1 -1
- azure/storage/blob/aio/__init__.py +3 -2
- azure/storage/blob/aio/_blob_client_async.py +53 -40
- azure/storage/blob/aio/_blob_service_client_async.py +13 -11
- azure/storage/blob/aio/_container_client_async.py +28 -25
- azure/storage/blob/aio/_download_async.py +7 -7
- azure/storage/blob/aio/_lease_async.py +1 -0
- azure/storage/blob/aio/_quick_query_helper_async.py +3 -3
- {azure_storage_blob-12.26.0b1.dist-info → azure_storage_blob-12.27.0b1.dist-info}/METADATA +3 -4
- azure_storage_blob-12.27.0b1.dist-info/RECORD +86 -0
- azure_storage_blob-12.26.0b1.dist-info/RECORD +0 -85
- {azure_storage_blob-12.26.0b1.dist-info → azure_storage_blob-12.27.0b1.dist-info}/LICENSE +0 -0
- {azure_storage_blob-12.26.0b1.dist-info → azure_storage_blob-12.27.0b1.dist-info}/WHEEL +0 -0
- {azure_storage_blob-12.26.0b1.dist-info → azure_storage_blob-12.27.0b1.dist-info}/top_level.txt +0 -0
@@ -29,6 +29,7 @@ import abc
|
|
29
29
|
import json
|
30
30
|
import logging
|
31
31
|
import re
|
32
|
+
|
32
33
|
logger = logging.getLogger(__name__)
|
33
34
|
|
34
35
|
# ------------------------------------------------------------------------------
|
@@ -37,43 +38,47 @@ logger = logging.getLogger(__name__)
|
|
37
38
|
# Log level more verbose than DEBUG=10, INFO=20, etc.
|
38
39
|
DEBUG_VERBOSE = 5
|
39
40
|
|
40
|
-
NULL =
|
41
|
-
BOOLEAN =
|
42
|
-
STRING =
|
43
|
-
BYTES =
|
44
|
-
INT =
|
45
|
-
LONG =
|
46
|
-
FLOAT =
|
47
|
-
DOUBLE =
|
48
|
-
FIXED =
|
49
|
-
ENUM =
|
50
|
-
RECORD =
|
51
|
-
ERROR =
|
52
|
-
ARRAY =
|
53
|
-
MAP =
|
54
|
-
UNION =
|
41
|
+
NULL = "null"
|
42
|
+
BOOLEAN = "boolean"
|
43
|
+
STRING = "string"
|
44
|
+
BYTES = "bytes"
|
45
|
+
INT = "int"
|
46
|
+
LONG = "long"
|
47
|
+
FLOAT = "float"
|
48
|
+
DOUBLE = "double"
|
49
|
+
FIXED = "fixed"
|
50
|
+
ENUM = "enum"
|
51
|
+
RECORD = "record"
|
52
|
+
ERROR = "error"
|
53
|
+
ARRAY = "array"
|
54
|
+
MAP = "map"
|
55
|
+
UNION = "union"
|
55
56
|
|
56
57
|
# Request and error unions are part of Avro protocols:
|
57
|
-
REQUEST =
|
58
|
-
ERROR_UNION =
|
59
|
-
|
60
|
-
PRIMITIVE_TYPES = frozenset(
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
58
|
+
REQUEST = "request"
|
59
|
+
ERROR_UNION = "error_union"
|
60
|
+
|
61
|
+
PRIMITIVE_TYPES = frozenset(
|
62
|
+
[
|
63
|
+
NULL,
|
64
|
+
BOOLEAN,
|
65
|
+
STRING,
|
66
|
+
BYTES,
|
67
|
+
INT,
|
68
|
+
LONG,
|
69
|
+
FLOAT,
|
70
|
+
DOUBLE,
|
71
|
+
]
|
72
|
+
)
|
73
|
+
|
74
|
+
NAMED_TYPES = frozenset(
|
75
|
+
[
|
76
|
+
FIXED,
|
77
|
+
ENUM,
|
78
|
+
RECORD,
|
79
|
+
ERROR,
|
80
|
+
]
|
81
|
+
)
|
77
82
|
|
78
83
|
VALID_TYPES = frozenset.union(
|
79
84
|
PRIMITIVE_TYPES,
|
@@ -87,31 +92,37 @@ VALID_TYPES = frozenset.union(
|
|
87
92
|
],
|
88
93
|
)
|
89
94
|
|
90
|
-
SCHEMA_RESERVED_PROPS = frozenset(
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
95
|
+
SCHEMA_RESERVED_PROPS = frozenset(
|
96
|
+
[
|
97
|
+
"type",
|
98
|
+
"name",
|
99
|
+
"namespace",
|
100
|
+
"fields", # Record
|
101
|
+
"items", # Array
|
102
|
+
"size", # Fixed
|
103
|
+
"symbols", # Enum
|
104
|
+
"values", # Map
|
105
|
+
"doc",
|
106
|
+
]
|
107
|
+
)
|
108
|
+
|
109
|
+
FIELD_RESERVED_PROPS = frozenset(
|
110
|
+
[
|
111
|
+
"default",
|
112
|
+
"name",
|
113
|
+
"doc",
|
114
|
+
"order",
|
115
|
+
"type",
|
116
|
+
]
|
117
|
+
)
|
118
|
+
|
119
|
+
VALID_FIELD_SORT_ORDERS = frozenset(
|
120
|
+
[
|
121
|
+
"ascending",
|
122
|
+
"descending",
|
123
|
+
"ignore",
|
124
|
+
]
|
125
|
+
)
|
115
126
|
|
116
127
|
|
117
128
|
# ------------------------------------------------------------------------------
|
@@ -141,12 +152,12 @@ class Schema(metaclass=abc.ABCMeta):
|
|
141
152
|
other_props: Optional dictionary of additional properties.
|
142
153
|
"""
|
143
154
|
if data_type not in VALID_TYPES:
|
144
|
-
raise SchemaParseException(f
|
155
|
+
raise SchemaParseException(f"{data_type!r} is not a valid Avro type.")
|
145
156
|
|
146
157
|
# All properties of this schema, as a map: property name -> property value
|
147
158
|
self._props = {}
|
148
159
|
|
149
|
-
self._props[
|
160
|
+
self._props["type"] = data_type
|
150
161
|
self._type = data_type
|
151
162
|
|
152
163
|
if other_props:
|
@@ -155,7 +166,7 @@ class Schema(metaclass=abc.ABCMeta):
|
|
155
166
|
@property
|
156
167
|
def namespace(self):
|
157
168
|
"""Returns: the namespace this schema belongs to, if any, or None."""
|
158
|
-
return self._props.get(
|
169
|
+
return self._props.get("namespace", None)
|
159
170
|
|
160
171
|
@property
|
161
172
|
def type(self):
|
@@ -165,7 +176,7 @@ class Schema(metaclass=abc.ABCMeta):
|
|
165
176
|
@property
|
166
177
|
def doc(self):
|
167
178
|
"""Returns: the documentation associated to this schema, if any, or None."""
|
168
|
-
return self._props.get(
|
179
|
+
return self._props.get("doc", None)
|
169
180
|
|
170
181
|
@property
|
171
182
|
def props(self):
|
@@ -193,20 +204,19 @@ class Schema(metaclass=abc.ABCMeta):
|
|
193
204
|
# Schema types that have names (records, enums, and fixed) must be aware of not
|
194
205
|
# re-defining schemas that are already listed in the parameter names.
|
195
206
|
@abc.abstractmethod
|
196
|
-
def to_json(self, names):
|
197
|
-
...
|
207
|
+
def to_json(self, names): ...
|
198
208
|
|
199
209
|
|
200
210
|
# ------------------------------------------------------------------------------
|
201
211
|
|
202
212
|
|
203
|
-
_RE_NAME = re.compile(r
|
213
|
+
_RE_NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
|
204
214
|
|
205
215
|
_RE_FULL_NAME = re.compile(
|
206
|
-
r
|
207
|
-
r
|
208
|
-
r
|
209
|
-
r
|
216
|
+
r"^"
|
217
|
+
r"[.]?(?:[A-Za-z_][A-Za-z0-9_]*[.])*" # optional namespace
|
218
|
+
r"([A-Za-z_][A-Za-z0-9_]*)" # name
|
219
|
+
r"$"
|
210
220
|
)
|
211
221
|
|
212
222
|
|
@@ -222,32 +232,31 @@ class Name(object):
|
|
222
232
|
"""
|
223
233
|
# Normalize: namespace is always defined as a string, possibly empty.
|
224
234
|
if namespace is None:
|
225
|
-
namespace =
|
235
|
+
namespace = ""
|
226
236
|
|
227
|
-
if
|
237
|
+
if "." in name:
|
228
238
|
# name is absolute, namespace is ignored:
|
229
239
|
self._fullname = name
|
230
240
|
|
231
241
|
match = _RE_FULL_NAME.match(self._fullname)
|
232
242
|
if match is None:
|
233
|
-
raise SchemaParseException(
|
234
|
-
f'Invalid absolute schema name: {self._fullname!r}.')
|
243
|
+
raise SchemaParseException(f"Invalid absolute schema name: {self._fullname!r}.")
|
235
244
|
|
236
245
|
self._name = match.group(1)
|
237
|
-
self._namespace = self._fullname[
|
246
|
+
self._namespace = self._fullname[: -(len(self._name) + 1)]
|
238
247
|
|
239
248
|
else:
|
240
249
|
# name is relative, combine with explicit namespace:
|
241
250
|
self._name = name
|
242
251
|
self._namespace = namespace
|
243
|
-
self._fullname = (self._name
|
244
|
-
if (not self._namespace) else
|
245
|
-
f'{self._namespace}.{self._name}')
|
252
|
+
self._fullname = self._name if (not self._namespace) else f"{self._namespace}.{self._name}"
|
246
253
|
|
247
254
|
# Validate the fullname:
|
248
255
|
if _RE_FULL_NAME.match(self._fullname) is None:
|
249
|
-
raise SchemaParseException(
|
250
|
-
|
256
|
+
raise SchemaParseException(
|
257
|
+
f"Invalid schema name {self._fullname!r} inferred from "
|
258
|
+
f"name {self._name!r} and namespace {self._namespace!r}."
|
259
|
+
)
|
251
260
|
|
252
261
|
def __eq__(self, other):
|
253
262
|
if not isinstance(other, Name):
|
@@ -302,7 +311,7 @@ class Names(object):
|
|
302
311
|
"""Creates a new name tracker from this tracker, but with a new default ns.
|
303
312
|
|
304
313
|
:param Any namespace: New default namespace to use.
|
305
|
-
:
|
314
|
+
:return: New name tracker with the specified default namespace.
|
306
315
|
:rtype: Names
|
307
316
|
"""
|
308
317
|
return Names(names=self._names, default_namespace=namespace)
|
@@ -312,7 +321,7 @@ class Names(object):
|
|
312
321
|
|
313
322
|
:param Any name: Name to resolve (absolute or relative).
|
314
323
|
:param Optional[Any] namespace: Optional explicit namespace.
|
315
|
-
:
|
324
|
+
:return: The specified name, resolved according to this tracker.
|
316
325
|
:rtype: Name
|
317
326
|
"""
|
318
327
|
if namespace is None:
|
@@ -324,7 +333,7 @@ class Names(object):
|
|
324
333
|
|
325
334
|
:param Any name: Name (absolute or relative) of the Avro schema to look up.
|
326
335
|
:param Optional[Any] namespace: Optional explicit namespace.
|
327
|
-
:
|
336
|
+
:return: The schema with the specified name, if any, or None
|
328
337
|
:rtype: Union[Any, None]
|
329
338
|
"""
|
330
339
|
avro_name = self.get_name(name=name, namespace=namespace)
|
@@ -335,15 +344,15 @@ class Names(object):
|
|
335
344
|
if self.default_namespace is None:
|
336
345
|
# I have no default -- no change
|
337
346
|
return properties
|
338
|
-
if
|
347
|
+
if "namespace" not in properties:
|
339
348
|
# he has no namespace - no change
|
340
349
|
return properties
|
341
|
-
if properties[
|
350
|
+
if properties["namespace"] != self.default_namespace:
|
342
351
|
# we're different - leave his stuff alone
|
343
352
|
return properties
|
344
353
|
# we each have a namespace and it's redundant. delete his.
|
345
354
|
prunable = properties.copy()
|
346
|
-
del prunable[
|
355
|
+
del prunable["namespace"]
|
347
356
|
return prunable
|
348
357
|
|
349
358
|
def register(self, schema):
|
@@ -352,13 +361,11 @@ class Names(object):
|
|
352
361
|
:param Any schema: Named Avro schema to register in this tracker.
|
353
362
|
"""
|
354
363
|
if schema.fullname in VALID_TYPES:
|
355
|
-
raise SchemaParseException(
|
356
|
-
f'{schema.fullname} is a reserved type name.')
|
364
|
+
raise SchemaParseException(f"{schema.fullname} is a reserved type name.")
|
357
365
|
if schema.fullname in self.names:
|
358
|
-
raise SchemaParseException(
|
359
|
-
f'Avro name {schema.fullname!r} already exists.')
|
366
|
+
raise SchemaParseException(f"Avro name {schema.fullname!r} already exists.")
|
360
367
|
|
361
|
-
logger.log(DEBUG_VERBOSE,
|
368
|
+
logger.log(DEBUG_VERBOSE, "Register new name for %r", schema.fullname)
|
362
369
|
self._names[schema.fullname] = schema
|
363
370
|
|
364
371
|
|
@@ -372,12 +379,12 @@ class NamedSchema(Schema):
|
|
372
379
|
"""
|
373
380
|
|
374
381
|
def __init__(
|
375
|
-
|
376
|
-
|
377
|
-
|
378
|
-
|
379
|
-
|
380
|
-
|
382
|
+
self,
|
383
|
+
data_type,
|
384
|
+
name=None,
|
385
|
+
namespace=None,
|
386
|
+
names=None,
|
387
|
+
other_props=None,
|
381
388
|
):
|
382
389
|
"""Initializes a new named schema object.
|
383
390
|
|
@@ -388,16 +395,16 @@ class NamedSchema(Schema):
|
|
388
395
|
names: Tracker to resolve and register Avro names.
|
389
396
|
other_props: Optional map of additional properties of the schema.
|
390
397
|
"""
|
391
|
-
assert
|
398
|
+
assert data_type in NAMED_TYPES, f"Invalid named type: {data_type!r}"
|
392
399
|
self._avro_name = names.get_name(name=name, namespace=namespace)
|
393
400
|
|
394
401
|
super(NamedSchema, self).__init__(data_type, other_props)
|
395
402
|
|
396
403
|
names.register(self)
|
397
404
|
|
398
|
-
self._props[
|
405
|
+
self._props["name"] = self.name
|
399
406
|
if self.namespace:
|
400
|
-
self._props[
|
407
|
+
self._props["namespace"] = self.namespace
|
401
408
|
|
402
409
|
@property
|
403
410
|
def avro_name(self):
|
@@ -420,7 +427,7 @@ class NamedSchema(Schema):
|
|
420
427
|
"""Reports this schema name relative to the specified name tracker.
|
421
428
|
|
422
429
|
:param Any names: Avro name tracker to relativize this schema name against.
|
423
|
-
:
|
430
|
+
:return: This schema name, relativized against the specified name tracker.
|
424
431
|
:rtype: Any
|
425
432
|
"""
|
426
433
|
if self.namespace == names.default_namespace:
|
@@ -432,8 +439,8 @@ class NamedSchema(Schema):
|
|
432
439
|
# Schema types that have names (records, enums, and fixed) must be aware
|
433
440
|
# of not re-defining schemas that are already listed in the parameter names.
|
434
441
|
@abc.abstractmethod
|
435
|
-
def to_json(self, names):
|
436
|
-
|
442
|
+
def to_json(self, names): ...
|
443
|
+
|
437
444
|
|
438
445
|
# ------------------------------------------------------------------------------
|
439
446
|
|
@@ -445,15 +452,7 @@ class Field(object):
|
|
445
452
|
"""Representation of the schema of a field in a record."""
|
446
453
|
|
447
454
|
def __init__(
|
448
|
-
|
449
|
-
data_type,
|
450
|
-
name,
|
451
|
-
index,
|
452
|
-
has_default,
|
453
|
-
default=_NO_DEFAULT,
|
454
|
-
order=None,
|
455
|
-
doc=None,
|
456
|
-
other_props=None
|
455
|
+
self, data_type, name, index, has_default, default=_NO_DEFAULT, order=None, doc=None, other_props=None
|
457
456
|
):
|
458
457
|
"""Initializes a new Field object.
|
459
458
|
|
@@ -468,9 +467,9 @@ class Field(object):
|
|
468
467
|
other_props:
|
469
468
|
"""
|
470
469
|
if (not isinstance(name, str)) or (not name):
|
471
|
-
raise SchemaParseException(f
|
470
|
+
raise SchemaParseException(f"Invalid record field name: {name!r}.")
|
472
471
|
if (order is not None) and (order not in VALID_FIELD_SORT_ORDERS):
|
473
|
-
raise SchemaParseException(f
|
472
|
+
raise SchemaParseException(f"Invalid record field order: {order!r}.")
|
474
473
|
|
475
474
|
# All properties of this record field:
|
476
475
|
self._props = {}
|
@@ -480,17 +479,17 @@ class Field(object):
|
|
480
479
|
self._props.update(other_props)
|
481
480
|
|
482
481
|
self._index = index
|
483
|
-
self._type = self._props[
|
484
|
-
self._name = self._props[
|
482
|
+
self._type = self._props["type"] = data_type
|
483
|
+
self._name = self._props["name"] = name
|
485
484
|
|
486
485
|
if has_default:
|
487
|
-
self._props[
|
486
|
+
self._props["default"] = default
|
488
487
|
|
489
488
|
if order is not None:
|
490
|
-
self._props[
|
489
|
+
self._props["order"] = order
|
491
490
|
|
492
491
|
if doc is not None:
|
493
|
-
self._props[
|
492
|
+
self._props["doc"] = doc
|
494
493
|
|
495
494
|
@property
|
496
495
|
def type(self):
|
@@ -509,7 +508,7 @@ class Field(object):
|
|
509
508
|
|
510
509
|
@property
|
511
510
|
def default(self):
|
512
|
-
return self._props[
|
511
|
+
return self._props["default"]
|
513
512
|
|
514
513
|
@property
|
515
514
|
def has_default(self):
|
@@ -517,11 +516,11 @@ class Field(object):
|
|
517
516
|
|
518
517
|
@property
|
519
518
|
def order(self):
|
520
|
-
return self._props.get(
|
519
|
+
return self._props.get("order", None)
|
521
520
|
|
522
521
|
@property
|
523
522
|
def doc(self):
|
524
|
-
return self._props.get(
|
523
|
+
return self._props.get("doc", None)
|
525
524
|
|
526
525
|
@property
|
527
526
|
def props(self):
|
@@ -538,7 +537,7 @@ class Field(object):
|
|
538
537
|
if names is None:
|
539
538
|
names = Names()
|
540
539
|
to_dump = self.props.copy()
|
541
|
-
to_dump[
|
540
|
+
to_dump["type"] = self.type.to_json(names)
|
542
541
|
return to_dump
|
543
542
|
|
544
543
|
def __eq__(self, that):
|
@@ -563,7 +562,7 @@ class PrimitiveSchema(Schema):
|
|
563
562
|
data_type: Type of the schema to construct. Must be primitive.
|
564
563
|
"""
|
565
564
|
if data_type not in PRIMITIVE_TYPES:
|
566
|
-
raise AvroException(f
|
565
|
+
raise AvroException(f"{data_type!r} is not a valid primitive type.")
|
567
566
|
super(PrimitiveSchema, self).__init__(data_type, other_props=other_props)
|
568
567
|
|
569
568
|
@property
|
@@ -593,16 +592,16 @@ class PrimitiveSchema(Schema):
|
|
593
592
|
|
594
593
|
class FixedSchema(NamedSchema):
|
595
594
|
def __init__(
|
596
|
-
|
597
|
-
|
598
|
-
|
599
|
-
|
600
|
-
|
601
|
-
|
595
|
+
self,
|
596
|
+
name,
|
597
|
+
namespace,
|
598
|
+
size,
|
599
|
+
names=None,
|
600
|
+
other_props=None,
|
602
601
|
):
|
603
602
|
# Ensure valid ctor args
|
604
603
|
if not isinstance(size, int):
|
605
|
-
fail_msg =
|
604
|
+
fail_msg = "Fixed Schema requires a valid integer for size property."
|
606
605
|
raise AvroException(fail_msg)
|
607
606
|
|
608
607
|
super(FixedSchema, self).__init__(
|
@@ -612,12 +611,12 @@ class FixedSchema(NamedSchema):
|
|
612
611
|
names=names,
|
613
612
|
other_props=other_props,
|
614
613
|
)
|
615
|
-
self._props[
|
614
|
+
self._props["size"] = size
|
616
615
|
|
617
616
|
@property
|
618
617
|
def size(self):
|
619
618
|
"""Returns: the size of this fixed schema, in bytes."""
|
620
|
-
return self._props[
|
619
|
+
return self._props["size"]
|
621
620
|
|
622
621
|
def to_json(self, names=None):
|
623
622
|
if names is None:
|
@@ -636,13 +635,13 @@ class FixedSchema(NamedSchema):
|
|
636
635
|
|
637
636
|
class EnumSchema(NamedSchema):
|
638
637
|
def __init__(
|
639
|
-
|
640
|
-
|
641
|
-
|
642
|
-
|
643
|
-
|
644
|
-
|
645
|
-
|
638
|
+
self,
|
639
|
+
name,
|
640
|
+
namespace,
|
641
|
+
symbols,
|
642
|
+
names=None,
|
643
|
+
doc=None,
|
644
|
+
other_props=None,
|
646
645
|
):
|
647
646
|
"""Initializes a new enumeration schema object.
|
648
647
|
|
@@ -656,10 +655,8 @@ class EnumSchema(NamedSchema):
|
|
656
655
|
"""
|
657
656
|
symbols = tuple(symbols)
|
658
657
|
symbol_set = frozenset(symbols)
|
659
|
-
if
|
660
|
-
|
661
|
-
raise AvroException(
|
662
|
-
f'Invalid symbols for enum schema: {symbols!r}.')
|
658
|
+
if len(symbol_set) != len(symbols) or not all(map(lambda symbol: isinstance(symbol, str), symbols)):
|
659
|
+
raise AvroException(f"Invalid symbols for enum schema: {symbols!r}.")
|
663
660
|
|
664
661
|
super(EnumSchema, self).__init__(
|
665
662
|
data_type=ENUM,
|
@@ -669,14 +666,14 @@ class EnumSchema(NamedSchema):
|
|
669
666
|
other_props=other_props,
|
670
667
|
)
|
671
668
|
|
672
|
-
self._props[
|
669
|
+
self._props["symbols"] = symbols
|
673
670
|
if doc is not None:
|
674
|
-
self._props[
|
671
|
+
self._props["doc"] = doc
|
675
672
|
|
676
673
|
@property
|
677
674
|
def symbols(self):
|
678
675
|
"""Returns: the symbols defined in this enum."""
|
679
|
-
return self._props[
|
676
|
+
return self._props["symbols"]
|
680
677
|
|
681
678
|
def to_json(self, names=None):
|
682
679
|
if names is None:
|
@@ -709,7 +706,7 @@ class ArraySchema(Schema):
|
|
709
706
|
other_props=other_props,
|
710
707
|
)
|
711
708
|
self._items_schema = items
|
712
|
-
self._props[
|
709
|
+
self._props["items"] = items
|
713
710
|
|
714
711
|
@property
|
715
712
|
def items(self):
|
@@ -721,7 +718,7 @@ class ArraySchema(Schema):
|
|
721
718
|
names = Names()
|
722
719
|
to_dump = self.props.copy()
|
723
720
|
item_schema = self.items
|
724
|
-
to_dump[
|
721
|
+
to_dump["items"] = item_schema.to_json(names)
|
725
722
|
return to_dump
|
726
723
|
|
727
724
|
def __eq__(self, that):
|
@@ -747,7 +744,7 @@ class MapSchema(Schema):
|
|
747
744
|
other_props=other_props,
|
748
745
|
)
|
749
746
|
self._values_schema = values
|
750
|
-
self._props[
|
747
|
+
self._props["values"] = values
|
751
748
|
|
752
749
|
@property
|
753
750
|
def values(self):
|
@@ -758,7 +755,7 @@ class MapSchema(Schema):
|
|
758
755
|
if names is None:
|
759
756
|
names = Names()
|
760
757
|
to_dump = self.props.copy()
|
761
|
-
to_dump[
|
758
|
+
to_dump["values"] = self.values.to_json(names)
|
762
759
|
return to_dump
|
763
760
|
|
764
761
|
def __eq__(self, that):
|
@@ -784,23 +781,21 @@ class UnionSchema(Schema):
|
|
784
781
|
# Validate the schema branches:
|
785
782
|
|
786
783
|
# All named schema names are unique:
|
787
|
-
named_branches = tuple(
|
788
|
-
filter(lambda schema: schema.type in NAMED_TYPES, self._schemas))
|
784
|
+
named_branches = tuple(filter(lambda schema: schema.type in NAMED_TYPES, self._schemas))
|
789
785
|
unique_names = frozenset(map(lambda schema: schema.fullname, named_branches))
|
790
786
|
if len(unique_names) != len(named_branches):
|
791
|
-
schemas =
|
792
|
-
raise AvroException(f
|
787
|
+
schemas = "".join(map(lambda schema: (f"\n\t - {schema}"), self._schemas))
|
788
|
+
raise AvroException(f"Invalid union branches with duplicate schema name:{schemas}")
|
793
789
|
|
794
790
|
# Types are unique within unnamed schemas, and union is not allowed:
|
795
|
-
unnamed_branches = tuple(
|
796
|
-
filter(lambda schema: schema.type not in NAMED_TYPES, self._schemas))
|
791
|
+
unnamed_branches = tuple(filter(lambda schema: schema.type not in NAMED_TYPES, self._schemas))
|
797
792
|
unique_types = frozenset(map(lambda schema: schema.type, unnamed_branches))
|
798
793
|
if UNION in unique_types:
|
799
|
-
schemas =
|
800
|
-
raise AvroException(f
|
794
|
+
schemas = "".join(map(lambda schema: (f"\n\t - {schema}"), self._schemas))
|
795
|
+
raise AvroException(f"Invalid union branches contain other unions:{schemas}")
|
801
796
|
if len(unique_types) != len(unnamed_branches):
|
802
|
-
schemas =
|
803
|
-
raise AvroException(f
|
797
|
+
schemas = "".join(map(lambda schema: (f"\n\t - {schema}"), self._schemas))
|
798
|
+
raise AvroException(f"Invalid union branches with duplicate type:{schemas}")
|
804
799
|
|
805
800
|
@property
|
806
801
|
def schemas(self):
|
@@ -861,23 +856,22 @@ class RecordSchema(NamedSchema):
|
|
861
856
|
:param int index: 0-based index of the field in the record.
|
862
857
|
:param Any field_desc: JSON descriptors of a record field.
|
863
858
|
:param Any names: The names for this schema.
|
864
|
-
:
|
859
|
+
:return: The field schema.
|
865
860
|
:rtype: Field
|
866
861
|
"""
|
867
862
|
field_schema = schema_from_json_data(
|
868
|
-
json_data=field_desc[
|
863
|
+
json_data=field_desc["type"],
|
869
864
|
names=names,
|
870
865
|
)
|
871
|
-
other_props = (
|
872
|
-
dict(filter_keys_out(items=field_desc, keys=FIELD_RESERVED_PROPS)))
|
866
|
+
other_props = dict(filter_keys_out(items=field_desc, keys=FIELD_RESERVED_PROPS))
|
873
867
|
return Field(
|
874
868
|
data_type=field_schema,
|
875
|
-
name=field_desc[
|
869
|
+
name=field_desc["name"],
|
876
870
|
index=index,
|
877
|
-
has_default=(
|
878
|
-
default=field_desc.get(
|
879
|
-
order=field_desc.get(
|
880
|
-
doc=field_desc.get(
|
871
|
+
has_default=("default" in field_desc),
|
872
|
+
default=field_desc.get("default", _NO_DEFAULT),
|
873
|
+
order=field_desc.get("order", None),
|
874
|
+
doc=field_desc.get("doc", None),
|
881
875
|
other_props=other_props,
|
882
876
|
)
|
883
877
|
|
@@ -888,7 +882,7 @@ class RecordSchema(NamedSchema):
|
|
888
882
|
|
889
883
|
:param Any field_desc_list: Collection of field JSON descriptors.
|
890
884
|
:param Any names: The names for this schema.
|
891
|
-
:
|
885
|
+
:return: Field schemas.
|
892
886
|
:rtype: Field
|
893
887
|
"""
|
894
888
|
for index, field_desc in enumerate(field_desc_list):
|
@@ -900,27 +894,18 @@ class RecordSchema(NamedSchema):
|
|
900
894
|
Guarantees field name unicity.
|
901
895
|
|
902
896
|
:param Any fields: Iterable of field schema.
|
903
|
-
:
|
897
|
+
:return: A map of field schemas, indexed by name.
|
904
898
|
:rtype: Dict[Any, Any]
|
905
899
|
"""
|
906
900
|
field_map = {}
|
907
901
|
for field in fields:
|
908
902
|
if field.name in field_map:
|
909
|
-
raise SchemaParseException(
|
910
|
-
f'Duplicate record field name {field.name!r}.')
|
903
|
+
raise SchemaParseException(f"Duplicate record field name {field.name!r}.")
|
911
904
|
field_map[field.name] = field
|
912
905
|
return field_map
|
913
906
|
|
914
907
|
def __init__(
|
915
|
-
|
916
|
-
name,
|
917
|
-
namespace,
|
918
|
-
fields=None,
|
919
|
-
make_fields=None,
|
920
|
-
names=None,
|
921
|
-
record_type=RECORD,
|
922
|
-
doc=None,
|
923
|
-
other_props=None
|
908
|
+
self, name, namespace, fields=None, make_fields=None, names=None, record_type=RECORD, doc=None, other_props=None
|
924
909
|
):
|
925
910
|
"""Initializes a new record schema object.
|
926
911
|
|
@@ -954,8 +939,7 @@ class RecordSchema(NamedSchema):
|
|
954
939
|
other_props=other_props,
|
955
940
|
)
|
956
941
|
else:
|
957
|
-
raise SchemaParseException(
|
958
|
-
f'Invalid record type: {record_type!r}.')
|
942
|
+
raise SchemaParseException(f"Invalid record type: {record_type!r}.")
|
959
943
|
|
960
944
|
nested_names = []
|
961
945
|
if record_type in [RECORD, ERROR]:
|
@@ -973,9 +957,9 @@ class RecordSchema(NamedSchema):
|
|
973
957
|
|
974
958
|
self._field_map = RecordSchema._make_field_map(self._fields)
|
975
959
|
|
976
|
-
self._props[
|
960
|
+
self._props["fields"] = fields
|
977
961
|
if doc is not None:
|
978
|
-
self._props[
|
962
|
+
self._props["doc"] = doc
|
979
963
|
|
980
964
|
@property
|
981
965
|
def fields(self):
|
@@ -999,7 +983,7 @@ class RecordSchema(NamedSchema):
|
|
999
983
|
names.names[self.fullname] = self
|
1000
984
|
|
1001
985
|
to_dump = names.prune_namespace(self.props.copy())
|
1002
|
-
to_dump[
|
986
|
+
to_dump["fields"] = [f.to_json(names) for f in self.fields]
|
1003
987
|
return to_dump
|
1004
988
|
|
1005
989
|
def __eq__(self, that):
|
@@ -1017,7 +1001,7 @@ def filter_keys_out(items, keys):
|
|
1017
1001
|
|
1018
1002
|
:param Dict[Any, Any] items: Dictionary of items to filter the keys out of.
|
1019
1003
|
:param Dict[Any, Any] keys: Dictionary of keys to filter the extracted keys against.
|
1020
|
-
:
|
1004
|
+
:return: Filtered items.
|
1021
1005
|
:rtype: Tuple(Any, Any)
|
1022
1006
|
"""
|
1023
1007
|
for key, value in items.items():
|
@@ -1048,31 +1032,29 @@ def _schema_from_json_array(json_array, names):
|
|
1048
1032
|
|
1049
1033
|
|
1050
1034
|
def _schema_from_json_object(json_object, names):
|
1051
|
-
data_type = json_object.get(
|
1035
|
+
data_type = json_object.get("type")
|
1052
1036
|
if data_type is None:
|
1053
|
-
raise SchemaParseException(
|
1054
|
-
f'Avro schema JSON descriptor has no "type" property: {json_object!r}')
|
1037
|
+
raise SchemaParseException(f'Avro schema JSON descriptor has no "type" property: {json_object!r}')
|
1055
1038
|
|
1056
|
-
other_props = dict(
|
1057
|
-
filter_keys_out(items=json_object, keys=SCHEMA_RESERVED_PROPS))
|
1039
|
+
other_props = dict(filter_keys_out(items=json_object, keys=SCHEMA_RESERVED_PROPS))
|
1058
1040
|
|
1059
1041
|
if data_type in PRIMITIVE_TYPES:
|
1060
1042
|
# FIXME should not ignore other properties
|
1061
1043
|
result = PrimitiveSchema(data_type, other_props=other_props)
|
1062
1044
|
|
1063
1045
|
elif data_type in NAMED_TYPES:
|
1064
|
-
name = json_object.get(
|
1065
|
-
namespace = json_object.get(
|
1046
|
+
name = json_object.get("name")
|
1047
|
+
namespace = json_object.get("namespace", names.default_namespace)
|
1066
1048
|
if data_type == FIXED:
|
1067
|
-
size = json_object.get(
|
1049
|
+
size = json_object.get("size")
|
1068
1050
|
result = FixedSchema(name, namespace, size, names, other_props)
|
1069
1051
|
elif data_type == ENUM:
|
1070
|
-
symbols = json_object.get(
|
1071
|
-
doc = json_object.get(
|
1052
|
+
symbols = json_object.get("symbols")
|
1053
|
+
doc = json_object.get("doc")
|
1072
1054
|
result = EnumSchema(name, namespace, symbols, names, doc, other_props)
|
1073
1055
|
|
1074
1056
|
elif data_type in [RECORD, ERROR]:
|
1075
|
-
field_desc_list = json_object.get(
|
1057
|
+
field_desc_list = json_object.get("fields", ())
|
1076
1058
|
|
1077
1059
|
def MakeFields(names):
|
1078
1060
|
return tuple(RecordSchema.make_field_list(field_desc_list, names))
|
@@ -1083,17 +1065,17 @@ def _schema_from_json_object(json_object, names):
|
|
1083
1065
|
make_fields=MakeFields,
|
1084
1066
|
names=names,
|
1085
1067
|
record_type=data_type,
|
1086
|
-
doc=json_object.get(
|
1068
|
+
doc=json_object.get("doc"),
|
1087
1069
|
other_props=other_props,
|
1088
1070
|
)
|
1089
1071
|
else:
|
1090
|
-
raise ValueError(f
|
1072
|
+
raise ValueError(f"Internal error: unknown type {data_type!r}.")
|
1091
1073
|
|
1092
1074
|
elif data_type in VALID_TYPES:
|
1093
1075
|
# Unnamed, non-primitive Avro type:
|
1094
1076
|
|
1095
1077
|
if data_type == ARRAY:
|
1096
|
-
items_desc = json_object.get(
|
1078
|
+
items_desc = json_object.get("items")
|
1097
1079
|
if items_desc is None:
|
1098
1080
|
raise SchemaParseException(f'Invalid array schema descriptor with no "items" : {json_object!r}.')
|
1099
1081
|
result = ArraySchema(
|
@@ -1102,7 +1084,7 @@ def _schema_from_json_object(json_object, names):
|
|
1102
1084
|
)
|
1103
1085
|
|
1104
1086
|
elif data_type == MAP:
|
1105
|
-
values_desc = json_object.get(
|
1087
|
+
values_desc = json_object.get("values")
|
1106
1088
|
if values_desc is None:
|
1107
1089
|
raise SchemaParseException(f'Invalid map schema descriptor with no "values" : {json_object!r}.')
|
1108
1090
|
result = MapSchema(
|
@@ -1111,17 +1093,15 @@ def _schema_from_json_object(json_object, names):
|
|
1111
1093
|
)
|
1112
1094
|
|
1113
1095
|
elif data_type == ERROR_UNION:
|
1114
|
-
error_desc_list = json_object.get(
|
1096
|
+
error_desc_list = json_object.get("declared_errors")
|
1115
1097
|
assert error_desc_list is not None
|
1116
|
-
error_schemas = map(
|
1117
|
-
lambda desc: schema_from_json_data(desc, names=names),
|
1118
|
-
error_desc_list)
|
1098
|
+
error_schemas = map(lambda desc: schema_from_json_data(desc, names=names), error_desc_list)
|
1119
1099
|
result = ErrorUnionSchema(schemas=error_schemas)
|
1120
1100
|
|
1121
1101
|
else:
|
1122
|
-
raise ValueError(f
|
1102
|
+
raise ValueError(f"Internal error: unknown type {data_type!r}.")
|
1123
1103
|
else:
|
1124
|
-
raise SchemaParseException(f
|
1104
|
+
raise SchemaParseException(f"Invalid JSON descriptor for an Avro schema: {json_object!r}")
|
1125
1105
|
return result
|
1126
1106
|
|
1127
1107
|
|
@@ -1139,7 +1119,7 @@ def schema_from_json_data(json_data, names=None):
|
|
1139
1119
|
|
1140
1120
|
:param Any json_data: JSON data representing the descriptor of the Avro schema.
|
1141
1121
|
:param Any names: Optional tracker for Avro named schemas.
|
1142
|
-
:
|
1122
|
+
:return: The Avro schema parsed from the JSON descriptor.
|
1143
1123
|
:rtype: Any
|
1144
1124
|
"""
|
1145
1125
|
if names is None:
|
@@ -1148,8 +1128,7 @@ def schema_from_json_data(json_data, names=None):
|
|
1148
1128
|
# Select the appropriate parser based on the JSON data type:
|
1149
1129
|
parser = _JSONDataParserTypeMap.get(type(json_data))
|
1150
1130
|
if parser is None:
|
1151
|
-
raise SchemaParseException(
|
1152
|
-
f'Invalid JSON descriptor for an Avro schema: {json_data!r}.')
|
1131
|
+
raise SchemaParseException(f"Invalid JSON descriptor for an Avro schema: {json_data!r}.")
|
1153
1132
|
return parser(json_data, names=names)
|
1154
1133
|
|
1155
1134
|
|
@@ -1161,15 +1140,15 @@ def parse(json_string):
|
|
1161
1140
|
Raises SchemaParseException if a JSON parsing error is met, or if the JSON descriptor is invalid.
|
1162
1141
|
|
1163
1142
|
:param str json_string: String representation of the JSON descriptor of the schema.
|
1164
|
-
:
|
1143
|
+
:return: The parsed schema.
|
1165
1144
|
:rtype: Any
|
1166
1145
|
"""
|
1167
1146
|
try:
|
1168
1147
|
json_data = json.loads(json_string)
|
1169
1148
|
except Exception as exn:
|
1170
1149
|
raise SchemaParseException(
|
1171
|
-
f
|
1172
|
-
|
1150
|
+
f"Error parsing schema from JSON: {json_string!r}. " f"Error message: {exn!r}."
|
1151
|
+
) from exn
|
1173
1152
|
|
1174
1153
|
# Initialize the names object
|
1175
1154
|
names = Names()
|