dcicutils 8.7.1.1b2__py3-none-any.whl → 8.7.1.1b4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
dcicutils/data_readers.py CHANGED
@@ -10,6 +10,9 @@ Excel = Type["Excel"]
10
10
 
11
11
  class RowReader(abc.ABC):
12
12
 
13
+ CELL_DELETION_VALUES = ["*delete*"] # cell values(s) indicating property deletion
14
+ CELL_DELETION_SENTINEL = object() # special cell deletion sentinel value
15
+
13
16
  def __init__(self):
14
17
  self.header = None
15
18
  self.row_number = 0
@@ -45,8 +48,13 @@ class RowReader(abc.ABC):
45
48
  def is_terminating_row(self, row: Union[List[Optional[Any]], Tuple[Optional[Any]]]) -> bool:
46
49
  return False
47
50
 
48
- def cell_value(self, value: Optional[Any]) -> Optional[Any]:
49
- return str(value).strip() if value is not None else ""
51
+ def cell_value(self, value: Optional[Any]) -> str:
52
+ if value is None:
53
+ return ""
54
+ elif (value := str(value).strip()) in RowReader.CELL_DELETION_VALUES:
55
+ return RowReader.CELL_DELETION_SENTINEL
56
+ else:
57
+ return value
50
58
 
51
59
  def open(self) -> None:
52
60
  pass
@@ -1,6 +1,7 @@
1
1
  from functools import lru_cache
2
2
  import re
3
3
  from typing import Any, Callable, List, Optional, Tuple, Type, Union
4
+ from dcicutils.data_readers import RowReader
4
5
  from dcicutils.portal_utils import Portal
5
6
  from dcicutils.schema_utils import Schema
6
7
 
@@ -157,21 +158,33 @@ class PortalObject:
157
158
  return f"{_path}.{key}" if _path else key
158
159
  def list_to_dictionary(value: list) -> dict: # noqa
159
160
  result = {}
160
- for index, item in enumerate(sorted(value)): # ignore array order
161
+ for index, item in enumerate(value):
161
162
  result[f"#{index}"] = item
162
163
  return result
163
164
  diffs = {}
164
165
  for key in a:
165
166
  path = key_to_path(key)
166
167
  if key not in b:
167
- diffs[path] = {"value": a[key], "missing_value": True}
168
+ if a[key] != RowReader.CELL_DELETION_SENTINEL:
169
+ diffs[path] = {"value": a[key], "creating_value": True}
168
170
  else:
169
171
  if isinstance(a[key], dict) and isinstance(b[key], dict):
170
172
  diffs.update(PortalObject._compare(a[key], b[key], compare=compare, _path=path))
171
173
  elif isinstance(a[key], list) and isinstance(b[key], list):
174
+ # Note that lists will be compared in order, which means the when dealing with
175
+ # insertions/deletions to/from the list, we my easily mistakenly regard elements
176
+ # of the list to be different when they are really the same, since they occupy
177
+ # different indices within the array. This is just a known restriction of this
178
+ # comparison functionality; and perhaps actually technically correct, but probably
179
+ # in practice, at the application/semantic level, we likely regard the order of
180
+ # lists as unimportant, and with a little more work here we could try to detect
181
+ # and exclude from the diffs for a list, those elements in the list which are
182
+ # equal to each other but which reside at different indices with then two lists.
172
183
  diffs.update(PortalObject._compare(list_to_dictionary(a[key]),
173
184
  list_to_dictionary(b[key]), compare=compare, _path=path))
174
185
  elif a[key] != b[key]:
175
- if not callable(compare) or not compare(path, a[key], b[key]):
176
- diffs[path] = {"value": a[key], "differing_value": b[key]}
186
+ if a[key] == RowReader.CELL_DELETION_SENTINEL:
187
+ diffs[path] = {"value": b[key], "deleting_value": True}
188
+ elif not callable(compare) or not compare(path, a[key], b[key]):
189
+ diffs[path] = {"value": a[key], "updating_value": b[key]}
177
190
  return diffs
dcicutils/portal_utils.py CHANGED
@@ -264,12 +264,12 @@ class Portal:
264
264
  add_on = ""
265
265
  return get_metadata(obj_id=object_id, vapp=self.vapp, key=self.key, add_on=add_on)
266
266
 
267
- def patch_metadata(self, object_id: str, data: str) -> Optional[dict]:
267
+ def patch_metadata(self, object_id: str, data: dict) -> Optional[dict]:
268
268
  if self.key:
269
269
  return patch_metadata(obj_id=object_id, patch_item=data, key=self.key)
270
270
  return self.patch(f"/{object_id}", data).json()
271
271
 
272
- def post_metadata(self, object_type: str, data: str) -> Optional[dict]:
272
+ def post_metadata(self, object_type: str, data: dict) -> Optional[dict]:
273
273
  if self.key:
274
274
  return post_metadata(schema_name=object_type, post_item=data, key=self.key)
275
275
  return self.post(f"/{object_type}", data).json()
@@ -64,6 +64,12 @@ class StructuredDataSet:
64
64
  return StructuredDataSet(file=file, portal=portal, schemas=schemas, autoadd=autoadd, order=order, prune=prune)
65
65
 
66
66
  def validate(self, force: bool = False) -> None:
67
+ def data_without_deleted_properties(data: dict) -> dict:
68
+ nonlocal self
69
+ if self._prune:
70
+ return {key: value for key, value in data.items() if value != RowReader.CELL_DELETION_SENTINEL}
71
+ else:
72
+ return {key: "" if value == RowReader.CELL_DELETION_SENTINEL else value for key, value in data.items()}
67
73
  if self._validated and not force:
68
74
  return
69
75
  self._validated = True
@@ -71,6 +77,7 @@ class StructuredDataSet:
71
77
  if (schema := Schema.load_by_name(type_name, portal=self._portal)):
72
78
  row_number = 0
73
79
  for data in self.data[type_name]:
80
+ data = data_without_deleted_properties(data)
74
81
  row_number += 1
75
82
  if (validation_errors := schema.validate(data)) is not None:
76
83
  for validation_error in validation_errors:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: dcicutils
3
- Version: 8.7.1.1b2
3
+ Version: 8.7.1.1b4
4
4
  Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
5
5
  Home-page: https://github.com/4dn-dcic/utils
6
6
  License: MIT
@@ -9,7 +9,7 @@ dcicutils/common.py,sha256=YE8Mt5-vaZWWz4uaChSVhqGFbFtW5QKtnIyOr4zG4vM,3955
9
9
  dcicutils/contribution_scripts.py,sha256=0k5Gw1TumcD5SAcXVkDd6-yvuMEw-jUp5Kfb7FJH6XQ,2015
10
10
  dcicutils/contribution_utils.py,sha256=vYLS1JUB3sKd24BUxZ29qUBqYeQBLK9cwo8x3k64uPg,25653
11
11
  dcicutils/creds_utils.py,sha256=xrLekD49Ex0GOpL9n7LlJA4gvNcY7txTVFOSYD7LvEU,11113
12
- dcicutils/data_readers.py,sha256=-XVkptkHbiyKg0uQTokXa0gl0PbPMwKe-geiK92yfcw,5668
12
+ dcicutils/data_readers.py,sha256=02b_EnTHnRFpzqXzd847ArUGieL5PgXc6269L9xxF9w,5978
13
13
  dcicutils/data_utils.py,sha256=k2OxOlsx7AJ6jF-YNlMyGus_JqSUBe4_n1s65Mv1gQQ,3098
14
14
  dcicutils/deployment_utils.py,sha256=rcNUFMe_tsrG4CHEtgBe41cZx4Pk4JqISPsjrJRMoEs,68891
15
15
  dcicutils/diff_utils.py,sha256=sQx-yz56DHAcQWOChYbAG3clXu7TbiZKlw-GggeveO0,8118
@@ -44,8 +44,8 @@ dcicutils/log_utils.py,sha256=7pWMc6vyrorUZQf-V-M3YC6zrPgNhuV_fzm9xqTPph0,10883
44
44
  dcicutils/misc_utils.py,sha256=bMRWWxdbhuF3PKdCZEH-H4U1ecgT3Nag3EL92D9XGoY,100973
45
45
  dcicutils/obfuscation_utils.py,sha256=fo2jOmDRC6xWpYX49u80bVNisqRRoPskFNX3ymFAmjw,5963
46
46
  dcicutils/opensearch_utils.py,sha256=V2exmFYW8Xl2_pGFixF4I2Cc549Opwe4PhFi5twC0M8,1017
47
- dcicutils/portal_object_utils.py,sha256=H5YAxGuYTJxnY3Pji58edIuA4QFPvJ5EZqDGMLnxXTE,9420
48
- dcicutils/portal_utils.py,sha256=jKYgZUYVdkg6VOs1hsiX4bSULLguOIBJFFRpvvZEklU,26704
47
+ dcicutils/portal_object_utils.py,sha256=zc0pEacq-J8D1QE7pH1xnHC50fF3_VQ7VSKySOvRr0c,10551
48
+ dcicutils/portal_utils.py,sha256=8XVRCy882RrB8QT2gGvW36c4nT1RJCgUApt_wLP-EG8,26706
49
49
  dcicutils/project_utils.py,sha256=qPdCaFmWUVBJw4rw342iUytwdQC0P-XKpK4mhyIulMM,31250
50
50
  dcicutils/qa_checkers.py,sha256=cdXjeL0jCDFDLT8VR8Px78aS10hwNISOO5G_Zv2TZ6M,20534
51
51
  dcicutils/qa_utils.py,sha256=TT0SiJWiuxYvbsIyhK9VO4uV_suxhB6CpuC4qPacCzQ,160208
@@ -59,15 +59,15 @@ dcicutils/secrets_utils.py,sha256=8dppXAsiHhJzI6NmOcvJV5ldvKkQZzh3Fl-cb8Wm7MI,19
59
59
  dcicutils/sheet_utils.py,sha256=VlmzteONW5VF_Q4vo0yA5vesz1ViUah1MZ_yA1rwZ0M,33629
60
60
  dcicutils/snapshot_utils.py,sha256=ymP7PXH6-yEiXAt75w0ldQFciGNqWBClNxC5gfX2FnY,22961
61
61
  dcicutils/ssl_certificate_utils.py,sha256=F0ifz_wnRRN9dfrfsz7aCp4UDLgHEY8LaK7PjnNvrAQ,9707
62
- dcicutils/structured_data.py,sha256=VxkyBBMFoRIEoPZxQuYU1iVO3piQwSsG-W6_pKy3qXE,34225
62
+ dcicutils/structured_data.py,sha256=ljXvLy1yUcrAvguAlbAfQ_mqHILw84HEQ1_NcMnXBRk,34659
63
63
  dcicutils/task_utils.py,sha256=MF8ujmTD6-O2AC2gRGPHyGdUrVKgtr8epT5XU8WtNjk,8082
64
64
  dcicutils/tmpfile_utils.py,sha256=n95XF8dZVbQRSXBZTGToXXfSs3JUVRyN6c3ZZ0nhAWI,1403
65
65
  dcicutils/trace_utils.py,sha256=g8kwV4ebEy5kXW6oOrEAUsurBcCROvwtZqz9fczsGRE,1769
66
66
  dcicutils/validation_utils.py,sha256=cMZIU2cY98FYtzK52z5WUYck7urH6JcqOuz9jkXpqzg,14797
67
67
  dcicutils/variant_utils.py,sha256=2H9azNx3xAj-MySg-uZ2SFqbWs4kZvf61JnK6b-h4Qw,4343
68
68
  dcicutils/zip_utils.py,sha256=rnjNv_k6L9jT2SjDSgVXp4BEJYLtz9XN6Cl2Fy-tqnM,2027
69
- dcicutils-8.7.1.1b2.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
70
- dcicutils-8.7.1.1b2.dist-info/METADATA,sha256=p9ZOsaryGCxzrudhx2lRFFlWY0mW1TiGu2k76qUufyM,3314
71
- dcicutils-8.7.1.1b2.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
72
- dcicutils-8.7.1.1b2.dist-info/entry_points.txt,sha256=8wbw5csMIgBXhkwfgsgJeuFcoUc0WsucUxmOyml2aoA,209
73
- dcicutils-8.7.1.1b2.dist-info/RECORD,,
69
+ dcicutils-8.7.1.1b4.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
70
+ dcicutils-8.7.1.1b4.dist-info/METADATA,sha256=tdGvBos2PdrsL-KAOgu_qcVDh_4j8AcUdXpsxrEEWDU,3314
71
+ dcicutils-8.7.1.1b4.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
72
+ dcicutils-8.7.1.1b4.dist-info/entry_points.txt,sha256=8wbw5csMIgBXhkwfgsgJeuFcoUc0WsucUxmOyml2aoA,209
73
+ dcicutils-8.7.1.1b4.dist-info/RECORD,,