dcicutils 8.8.4.1b34__py3-none-any.whl → 8.8.4.1b36__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dcicutils/misc_utils.py +15 -6
- dcicutils/structured_data.py +23 -4
- {dcicutils-8.8.4.1b34.dist-info → dcicutils-8.8.4.1b36.dist-info}/METADATA +1 -1
- {dcicutils-8.8.4.1b34.dist-info → dcicutils-8.8.4.1b36.dist-info}/RECORD +7 -7
- {dcicutils-8.8.4.1b34.dist-info → dcicutils-8.8.4.1b36.dist-info}/LICENSE.txt +0 -0
- {dcicutils-8.8.4.1b34.dist-info → dcicutils-8.8.4.1b36.dist-info}/WHEEL +0 -0
- {dcicutils-8.8.4.1b34.dist-info → dcicutils-8.8.4.1b36.dist-info}/entry_points.txt +0 -0
dcicutils/misc_utils.py
CHANGED
@@ -1155,7 +1155,8 @@ def remove_suffix(suffix: str, text: str, required: bool = False):
|
|
1155
1155
|
|
1156
1156
|
def remove_empty_properties(data: Optional[Union[list, dict]],
|
1157
1157
|
isempty: Optional[Callable] = None,
|
1158
|
-
isempty_array_element: Optional[Callable] = None
|
1158
|
+
isempty_array_element: Optional[Callable] = None,
|
1159
|
+
raise_exception_on_nonempty_array_element_after_empty: bool = False) -> None:
|
1159
1160
|
def _isempty(value: Any) -> bool: # noqa
|
1160
1161
|
return isempty(value) if callable(isempty) else value in [None, "", {}, []]
|
1161
1162
|
if isinstance(data, dict):
|
@@ -1163,11 +1164,22 @@ def remove_empty_properties(data: Optional[Union[list, dict]],
|
|
1163
1164
|
if _isempty(value := data[key]):
|
1164
1165
|
del data[key]
|
1165
1166
|
else:
|
1166
|
-
remove_empty_properties(value, isempty=isempty, isempty_array_element=isempty_array_element
|
1167
|
+
remove_empty_properties(value, isempty=isempty, isempty_array_element=isempty_array_element,
|
1168
|
+
raise_exception_on_nonempty_array_element_after_empty= # noqa
|
1169
|
+
raise_exception_on_nonempty_array_element_after_empty)
|
1167
1170
|
elif isinstance(data, list):
|
1168
1171
|
for item in data:
|
1169
|
-
remove_empty_properties(item, isempty=isempty, isempty_array_element=isempty_array_element
|
1172
|
+
remove_empty_properties(item, isempty=isempty, isempty_array_element=isempty_array_element,
|
1173
|
+
raise_exception_on_nonempty_array_element_after_empty= # noqa
|
1174
|
+
raise_exception_on_nonempty_array_element_after_empty)
|
1170
1175
|
if callable(isempty_array_element):
|
1176
|
+
if raise_exception_on_nonempty_array_element_after_empty is True:
|
1177
|
+
empty_element_seen = False
|
1178
|
+
for item in data:
|
1179
|
+
if not empty_element_seen and isempty_array_element(item):
|
1180
|
+
empty_element_seen = True
|
1181
|
+
elif empty_element_seen and not isempty_array_element(item):
|
1182
|
+
raise Exception("Non-empty element found after empty element.")
|
1171
1183
|
data[:] = [item for item in data if not isempty_array_element(item)]
|
1172
1184
|
|
1173
1185
|
|
@@ -2735,6 +2747,3 @@ def create_short_uuid(length: Optional[int] = None, upper: bool = False):
|
|
2735
2747
|
if upper is True:
|
2736
2748
|
value = value.upper()
|
2737
2749
|
return value
|
2738
|
-
|
2739
|
-
|
2740
|
-
print(format_size(1024 * 1024))
|
dcicutils/structured_data.py
CHANGED
@@ -53,6 +53,7 @@ class StructuredDataSet:
|
|
53
53
|
def __init__(self, file: Optional[str] = None, portal: Optional[Union[VirtualApp, TestApp, Portal]] = None,
|
54
54
|
schemas: Optional[List[dict]] = None, autoadd: Optional[dict] = None,
|
55
55
|
order: Optional[List[str]] = None, prune: bool = True,
|
56
|
+
remove_empty_objects_from_lists: bool = True,
|
56
57
|
ref_lookup_strategy: Optional[Callable] = None,
|
57
58
|
ref_lookup_nocache: bool = False,
|
58
59
|
norefs: bool = False,
|
@@ -65,7 +66,8 @@ class StructuredDataSet:
|
|
65
66
|
ref_lookup_nocache=ref_lookup_nocache) if portal else None
|
66
67
|
self._ref_lookup_strategy = ref_lookup_strategy
|
67
68
|
self._order = order
|
68
|
-
self._prune = prune
|
69
|
+
self._prune = prune is True
|
70
|
+
self._remove_empty_objects_from_lists = remove_empty_objects_from_lists is True
|
69
71
|
self._warnings = {}
|
70
72
|
self._errors = {}
|
71
73
|
self._resolved_refs = set()
|
@@ -93,12 +95,14 @@ class StructuredDataSet:
|
|
93
95
|
def load(file: str, portal: Optional[Union[VirtualApp, TestApp, Portal]] = None,
|
94
96
|
schemas: Optional[List[dict]] = None, autoadd: Optional[dict] = None,
|
95
97
|
order: Optional[List[str]] = None, prune: bool = True,
|
98
|
+
remove_empty_objects_from_lists: bool = True,
|
96
99
|
ref_lookup_strategy: Optional[Callable] = None,
|
97
100
|
ref_lookup_nocache: bool = False,
|
98
101
|
norefs: bool = False,
|
99
102
|
progress: Optional[Callable] = None,
|
100
103
|
debug_sleep: Optional[str] = None) -> StructuredDataSet:
|
101
104
|
return StructuredDataSet(file=file, portal=portal, schemas=schemas, autoadd=autoadd, order=order, prune=prune,
|
105
|
+
remove_empty_objects_from_lists=remove_empty_objects_from_lists,
|
102
106
|
ref_lookup_strategy=ref_lookup_strategy, ref_lookup_nocache=ref_lookup_nocache,
|
103
107
|
norefs=norefs, progress=progress, debug_sleep=debug_sleep)
|
104
108
|
|
@@ -368,7 +372,11 @@ class StructuredDataSet:
|
|
368
372
|
structured_row_template.set_value(structured_row, column_name, value, reader.file, reader.row_number)
|
369
373
|
if self._autoadd_properties:
|
370
374
|
self._add_properties(structured_row, self._autoadd_properties, schema)
|
371
|
-
self.
|
375
|
+
if (prune_error := self._prune_structured_row(structured_row)) is not None:
|
376
|
+
self._note_error({"src": create_dict(type=schema_name, row=reader.row_number),
|
377
|
+
"error": prune_error}, "validation")
|
378
|
+
else:
|
379
|
+
self._add(type_name, structured_row)
|
372
380
|
if self._progress:
|
373
381
|
self._progress({
|
374
382
|
PROGRESS.LOAD_ITEM: self._nrows,
|
@@ -385,9 +393,20 @@ class StructuredDataSet:
|
|
385
393
|
self._note_error(schema._unresolved_refs, "ref")
|
386
394
|
self._resolved_refs.update(schema._resolved_refs)
|
387
395
|
|
388
|
-
def
|
389
|
-
if self._prune:
|
396
|
+
def _prune_structured_row(self, data: dict) -> Optional[str]:
|
397
|
+
if not self._prune:
|
398
|
+
return None
|
399
|
+
if not self._remove_empty_objects_from_lists:
|
390
400
|
remove_empty_properties(data)
|
401
|
+
return None
|
402
|
+
try:
|
403
|
+
remove_empty_properties(data, isempty_array_element=lambda element: element == {},
|
404
|
+
raise_exception_on_nonempty_array_element_after_empty=True)
|
405
|
+
except Exception as e:
|
406
|
+
return str(e)
|
407
|
+
return None
|
408
|
+
|
409
|
+
def _add(self, type_name: str, data: Union[dict, List[dict]]) -> None:
|
391
410
|
if type_name in self._data:
|
392
411
|
self._data[type_name].extend([data] if isinstance(data, dict) else data)
|
393
412
|
else:
|
@@ -44,7 +44,7 @@ dcicutils/license_policies/park-lab-gpl-pipeline.jsonc,sha256=vLZkwm3Js-kjV44nug
|
|
44
44
|
dcicutils/license_policies/park-lab-pipeline.jsonc,sha256=9qlY0ASy3iUMQlr3gorVcXrSfRHnVGbLhkS427UaRy4,283
|
45
45
|
dcicutils/license_utils.py,sha256=d1cq6iwv5Ju-VjdoINi6q7CPNNL7Oz6rcJdLMY38RX0,46978
|
46
46
|
dcicutils/log_utils.py,sha256=7pWMc6vyrorUZQf-V-M3YC6zrPgNhuV_fzm9xqTPph0,10883
|
47
|
-
dcicutils/misc_utils.py,sha256=
|
47
|
+
dcicutils/misc_utils.py,sha256=zHwsxxEn24muLBP7mDvMa8I9VdMejwW8HMuCL5xbhhw,107690
|
48
48
|
dcicutils/obfuscation_utils.py,sha256=fo2jOmDRC6xWpYX49u80bVNisqRRoPskFNX3ymFAmjw,5963
|
49
49
|
dcicutils/opensearch_utils.py,sha256=V2exmFYW8Xl2_pGFixF4I2Cc549Opwe4PhFi5twC0M8,1017
|
50
50
|
dcicutils/portal_object_utils.py,sha256=gDXRgPsRvqCFwbC8WatsuflAxNiigOnqr0Hi93k3AgE,15422
|
@@ -64,7 +64,7 @@ dcicutils/secrets_utils.py,sha256=8dppXAsiHhJzI6NmOcvJV5ldvKkQZzh3Fl-cb8Wm7MI,19
|
|
64
64
|
dcicutils/sheet_utils.py,sha256=VlmzteONW5VF_Q4vo0yA5vesz1ViUah1MZ_yA1rwZ0M,33629
|
65
65
|
dcicutils/snapshot_utils.py,sha256=ymP7PXH6-yEiXAt75w0ldQFciGNqWBClNxC5gfX2FnY,22961
|
66
66
|
dcicutils/ssl_certificate_utils.py,sha256=F0ifz_wnRRN9dfrfsz7aCp4UDLgHEY8LaK7PjnNvrAQ,9707
|
67
|
-
dcicutils/structured_data.py,sha256=
|
67
|
+
dcicutils/structured_data.py,sha256=klpphnw4-mIZk8rGd5YBsaYkwS-YJ-6c1OasvWnqYhE,62284
|
68
68
|
dcicutils/submitr/progress_constants.py,sha256=5bxyX77ql8qEJearfHEvsvXl7D0GuUODW0T65mbRmnE,2895
|
69
69
|
dcicutils/submitr/ref_lookup_strategy.py,sha256=Js2cVznTmgjciLWBPLCvMiwLIHXjDn3jww-gJPjYuFw,3467
|
70
70
|
dcicutils/task_utils.py,sha256=MF8ujmTD6-O2AC2gRGPHyGdUrVKgtr8epT5XU8WtNjk,8082
|
@@ -73,8 +73,8 @@ dcicutils/trace_utils.py,sha256=g8kwV4ebEy5kXW6oOrEAUsurBcCROvwtZqz9fczsGRE,1769
|
|
73
73
|
dcicutils/validation_utils.py,sha256=cMZIU2cY98FYtzK52z5WUYck7urH6JcqOuz9jkXpqzg,14797
|
74
74
|
dcicutils/variant_utils.py,sha256=2H9azNx3xAj-MySg-uZ2SFqbWs4kZvf61JnK6b-h4Qw,4343
|
75
75
|
dcicutils/zip_utils.py,sha256=_Y9EmL3D2dUZhxucxHvrtmmlbZmK4FpSsHEb7rGSJLU,3265
|
76
|
-
dcicutils-8.8.4.
|
77
|
-
dcicutils-8.8.4.
|
78
|
-
dcicutils-8.8.4.
|
79
|
-
dcicutils-8.8.4.
|
80
|
-
dcicutils-8.8.4.
|
76
|
+
dcicutils-8.8.4.1b36.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
|
77
|
+
dcicutils-8.8.4.1b36.dist-info/METADATA,sha256=LrZoMbnXJRhcUEnTg2Z8KIBARN_dA_8e_cI4CH8CWKE,3440
|
78
|
+
dcicutils-8.8.4.1b36.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
|
79
|
+
dcicutils-8.8.4.1b36.dist-info/entry_points.txt,sha256=51Q4F_2V10L0282W7HFjP4jdzW4K8lnWDARJQVFy_hw,270
|
80
|
+
dcicutils-8.8.4.1b36.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|