dcicutils 8.8.4.1b34__py3-none-any.whl → 8.8.4.1b36__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
dcicutils/misc_utils.py CHANGED
@@ -1155,7 +1155,8 @@ def remove_suffix(suffix: str, text: str, required: bool = False):
1155
1155
 
1156
1156
  def remove_empty_properties(data: Optional[Union[list, dict]],
1157
1157
  isempty: Optional[Callable] = None,
1158
- isempty_array_element: Optional[Callable] = None) -> None:
1158
+ isempty_array_element: Optional[Callable] = None,
1159
+ raise_exception_on_nonempty_array_element_after_empty: bool = False) -> None:
1159
1160
  def _isempty(value: Any) -> bool: # noqa
1160
1161
  return isempty(value) if callable(isempty) else value in [None, "", {}, []]
1161
1162
  if isinstance(data, dict):
@@ -1163,11 +1164,22 @@ def remove_empty_properties(data: Optional[Union[list, dict]],
1163
1164
  if _isempty(value := data[key]):
1164
1165
  del data[key]
1165
1166
  else:
1166
- remove_empty_properties(value, isempty=isempty, isempty_array_element=isempty_array_element)
1167
+ remove_empty_properties(value, isempty=isempty, isempty_array_element=isempty_array_element,
1168
+ raise_exception_on_nonempty_array_element_after_empty= # noqa
1169
+ raise_exception_on_nonempty_array_element_after_empty)
1167
1170
  elif isinstance(data, list):
1168
1171
  for item in data:
1169
- remove_empty_properties(item, isempty=isempty, isempty_array_element=isempty_array_element)
1172
+ remove_empty_properties(item, isempty=isempty, isempty_array_element=isempty_array_element,
1173
+ raise_exception_on_nonempty_array_element_after_empty= # noqa
1174
+ raise_exception_on_nonempty_array_element_after_empty)
1170
1175
  if callable(isempty_array_element):
1176
+ if raise_exception_on_nonempty_array_element_after_empty is True:
1177
+ empty_element_seen = False
1178
+ for item in data:
1179
+ if not empty_element_seen and isempty_array_element(item):
1180
+ empty_element_seen = True
1181
+ elif empty_element_seen and not isempty_array_element(item):
1182
+ raise Exception("Non-empty element found after empty element.")
1171
1183
  data[:] = [item for item in data if not isempty_array_element(item)]
1172
1184
 
1173
1185
 
@@ -2735,6 +2747,3 @@ def create_short_uuid(length: Optional[int] = None, upper: bool = False):
2735
2747
  if upper is True:
2736
2748
  value = value.upper()
2737
2749
  return value
2738
-
2739
-
2740
- print(format_size(1024 * 1024))
@@ -53,6 +53,7 @@ class StructuredDataSet:
53
53
  def __init__(self, file: Optional[str] = None, portal: Optional[Union[VirtualApp, TestApp, Portal]] = None,
54
54
  schemas: Optional[List[dict]] = None, autoadd: Optional[dict] = None,
55
55
  order: Optional[List[str]] = None, prune: bool = True,
56
+ remove_empty_objects_from_lists: bool = True,
56
57
  ref_lookup_strategy: Optional[Callable] = None,
57
58
  ref_lookup_nocache: bool = False,
58
59
  norefs: bool = False,
@@ -65,7 +66,8 @@ class StructuredDataSet:
65
66
  ref_lookup_nocache=ref_lookup_nocache) if portal else None
66
67
  self._ref_lookup_strategy = ref_lookup_strategy
67
68
  self._order = order
68
- self._prune = prune
69
+ self._prune = prune is True
70
+ self._remove_empty_objects_from_lists = remove_empty_objects_from_lists is True
69
71
  self._warnings = {}
70
72
  self._errors = {}
71
73
  self._resolved_refs = set()
@@ -93,12 +95,14 @@ class StructuredDataSet:
93
95
  def load(file: str, portal: Optional[Union[VirtualApp, TestApp, Portal]] = None,
94
96
  schemas: Optional[List[dict]] = None, autoadd: Optional[dict] = None,
95
97
  order: Optional[List[str]] = None, prune: bool = True,
98
+ remove_empty_objects_from_lists: bool = True,
96
99
  ref_lookup_strategy: Optional[Callable] = None,
97
100
  ref_lookup_nocache: bool = False,
98
101
  norefs: bool = False,
99
102
  progress: Optional[Callable] = None,
100
103
  debug_sleep: Optional[str] = None) -> StructuredDataSet:
101
104
  return StructuredDataSet(file=file, portal=portal, schemas=schemas, autoadd=autoadd, order=order, prune=prune,
105
+ remove_empty_objects_from_lists=remove_empty_objects_from_lists,
102
106
  ref_lookup_strategy=ref_lookup_strategy, ref_lookup_nocache=ref_lookup_nocache,
103
107
  norefs=norefs, progress=progress, debug_sleep=debug_sleep)
104
108
 
@@ -368,7 +372,11 @@ class StructuredDataSet:
368
372
  structured_row_template.set_value(structured_row, column_name, value, reader.file, reader.row_number)
369
373
  if self._autoadd_properties:
370
374
  self._add_properties(structured_row, self._autoadd_properties, schema)
371
- self._add(type_name, structured_row)
375
+ if (prune_error := self._prune_structured_row(structured_row)) is not None:
376
+ self._note_error({"src": create_dict(type=schema_name, row=reader.row_number),
377
+ "error": prune_error}, "validation")
378
+ else:
379
+ self._add(type_name, structured_row)
372
380
  if self._progress:
373
381
  self._progress({
374
382
  PROGRESS.LOAD_ITEM: self._nrows,
@@ -385,9 +393,20 @@ class StructuredDataSet:
385
393
  self._note_error(schema._unresolved_refs, "ref")
386
394
  self._resolved_refs.update(schema._resolved_refs)
387
395
 
388
- def _add(self, type_name: str, data: Union[dict, List[dict]]) -> None:
389
- if self._prune:
396
+ def _prune_structured_row(self, data: dict) -> Optional[str]:
397
+ if not self._prune:
398
+ return None
399
+ if not self._remove_empty_objects_from_lists:
390
400
  remove_empty_properties(data)
401
+ return None
402
+ try:
403
+ remove_empty_properties(data, isempty_array_element=lambda element: element == {},
404
+ raise_exception_on_nonempty_array_element_after_empty=True)
405
+ except Exception as e:
406
+ return str(e)
407
+ return None
408
+
409
+ def _add(self, type_name: str, data: Union[dict, List[dict]]) -> None:
391
410
  if type_name in self._data:
392
411
  self._data[type_name].extend([data] if isinstance(data, dict) else data)
393
412
  else:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: dcicutils
3
- Version: 8.8.4.1b34
3
+ Version: 8.8.4.1b36
4
4
  Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
5
5
  Home-page: https://github.com/4dn-dcic/utils
6
6
  License: MIT
@@ -44,7 +44,7 @@ dcicutils/license_policies/park-lab-gpl-pipeline.jsonc,sha256=vLZkwm3Js-kjV44nug
44
44
  dcicutils/license_policies/park-lab-pipeline.jsonc,sha256=9qlY0ASy3iUMQlr3gorVcXrSfRHnVGbLhkS427UaRy4,283
45
45
  dcicutils/license_utils.py,sha256=d1cq6iwv5Ju-VjdoINi6q7CPNNL7Oz6rcJdLMY38RX0,46978
46
46
  dcicutils/log_utils.py,sha256=7pWMc6vyrorUZQf-V-M3YC6zrPgNhuV_fzm9xqTPph0,10883
47
- dcicutils/misc_utils.py,sha256=6TWxOOWav47An4X2XCgohYtGEncjU7XdYqnoYSRfyG4,106786
47
+ dcicutils/misc_utils.py,sha256=zHwsxxEn24muLBP7mDvMa8I9VdMejwW8HMuCL5xbhhw,107690
48
48
  dcicutils/obfuscation_utils.py,sha256=fo2jOmDRC6xWpYX49u80bVNisqRRoPskFNX3ymFAmjw,5963
49
49
  dcicutils/opensearch_utils.py,sha256=V2exmFYW8Xl2_pGFixF4I2Cc549Opwe4PhFi5twC0M8,1017
50
50
  dcicutils/portal_object_utils.py,sha256=gDXRgPsRvqCFwbC8WatsuflAxNiigOnqr0Hi93k3AgE,15422
@@ -64,7 +64,7 @@ dcicutils/secrets_utils.py,sha256=8dppXAsiHhJzI6NmOcvJV5ldvKkQZzh3Fl-cb8Wm7MI,19
64
64
  dcicutils/sheet_utils.py,sha256=VlmzteONW5VF_Q4vo0yA5vesz1ViUah1MZ_yA1rwZ0M,33629
65
65
  dcicutils/snapshot_utils.py,sha256=ymP7PXH6-yEiXAt75w0ldQFciGNqWBClNxC5gfX2FnY,22961
66
66
  dcicutils/ssl_certificate_utils.py,sha256=F0ifz_wnRRN9dfrfsz7aCp4UDLgHEY8LaK7PjnNvrAQ,9707
67
- dcicutils/structured_data.py,sha256=BQuIMv6OPySsn6YxtXE2Er-zLE2QJuCYhEQ3V0u_UXY,61238
67
+ dcicutils/structured_data.py,sha256=klpphnw4-mIZk8rGd5YBsaYkwS-YJ-6c1OasvWnqYhE,62284
68
68
  dcicutils/submitr/progress_constants.py,sha256=5bxyX77ql8qEJearfHEvsvXl7D0GuUODW0T65mbRmnE,2895
69
69
  dcicutils/submitr/ref_lookup_strategy.py,sha256=Js2cVznTmgjciLWBPLCvMiwLIHXjDn3jww-gJPjYuFw,3467
70
70
  dcicutils/task_utils.py,sha256=MF8ujmTD6-O2AC2gRGPHyGdUrVKgtr8epT5XU8WtNjk,8082
@@ -73,8 +73,8 @@ dcicutils/trace_utils.py,sha256=g8kwV4ebEy5kXW6oOrEAUsurBcCROvwtZqz9fczsGRE,1769
73
73
  dcicutils/validation_utils.py,sha256=cMZIU2cY98FYtzK52z5WUYck7urH6JcqOuz9jkXpqzg,14797
74
74
  dcicutils/variant_utils.py,sha256=2H9azNx3xAj-MySg-uZ2SFqbWs4kZvf61JnK6b-h4Qw,4343
75
75
  dcicutils/zip_utils.py,sha256=_Y9EmL3D2dUZhxucxHvrtmmlbZmK4FpSsHEb7rGSJLU,3265
76
- dcicutils-8.8.4.1b34.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
77
- dcicutils-8.8.4.1b34.dist-info/METADATA,sha256=hkWY8lqw4aHKXxtqn81dHbQpU2PeJ_3a-ZurdInqtB4,3440
78
- dcicutils-8.8.4.1b34.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
79
- dcicutils-8.8.4.1b34.dist-info/entry_points.txt,sha256=51Q4F_2V10L0282W7HFjP4jdzW4K8lnWDARJQVFy_hw,270
80
- dcicutils-8.8.4.1b34.dist-info/RECORD,,
76
+ dcicutils-8.8.4.1b36.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
77
+ dcicutils-8.8.4.1b36.dist-info/METADATA,sha256=LrZoMbnXJRhcUEnTg2Z8KIBARN_dA_8e_cI4CH8CWKE,3440
78
+ dcicutils-8.8.4.1b36.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
79
+ dcicutils-8.8.4.1b36.dist-info/entry_points.txt,sha256=51Q4F_2V10L0282W7HFjP4jdzW4K8lnWDARJQVFy_hw,270
80
+ dcicutils-8.8.4.1b36.dist-info/RECORD,,