dcicutils 8.7.1.1b2__py3-none-any.whl → 8.7.1.1b4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dcicutils/data_readers.py +10 -2
- dcicutils/portal_object_utils.py +17 -4
- dcicutils/portal_utils.py +2 -2
- dcicutils/structured_data.py +7 -0
- {dcicutils-8.7.1.1b2.dist-info → dcicutils-8.7.1.1b4.dist-info}/METADATA +1 -1
- {dcicutils-8.7.1.1b2.dist-info → dcicutils-8.7.1.1b4.dist-info}/RECORD +9 -9
- {dcicutils-8.7.1.1b2.dist-info → dcicutils-8.7.1.1b4.dist-info}/LICENSE.txt +0 -0
- {dcicutils-8.7.1.1b2.dist-info → dcicutils-8.7.1.1b4.dist-info}/WHEEL +0 -0
- {dcicutils-8.7.1.1b2.dist-info → dcicutils-8.7.1.1b4.dist-info}/entry_points.txt +0 -0
dcicutils/data_readers.py
CHANGED
@@ -10,6 +10,9 @@ Excel = Type["Excel"]
|
|
10
10
|
|
11
11
|
class RowReader(abc.ABC):
|
12
12
|
|
13
|
+
CELL_DELETION_VALUES = ["*delete*"] # cell values(s) indicating property deletion
|
14
|
+
CELL_DELETION_SENTINEL = object() # special cell deletion sentinel value
|
15
|
+
|
13
16
|
def __init__(self):
|
14
17
|
self.header = None
|
15
18
|
self.row_number = 0
|
@@ -45,8 +48,13 @@ class RowReader(abc.ABC):
|
|
45
48
|
def is_terminating_row(self, row: Union[List[Optional[Any]], Tuple[Optional[Any]]]) -> bool:
|
46
49
|
return False
|
47
50
|
|
48
|
-
def cell_value(self, value: Optional[Any]) ->
|
49
|
-
|
51
|
+
def cell_value(self, value: Optional[Any]) -> str:
|
52
|
+
if value is None:
|
53
|
+
return ""
|
54
|
+
elif (value := str(value).strip()) in RowReader.CELL_DELETION_VALUES:
|
55
|
+
return RowReader.CELL_DELETION_SENTINEL
|
56
|
+
else:
|
57
|
+
return value
|
50
58
|
|
51
59
|
def open(self) -> None:
|
52
60
|
pass
|
dcicutils/portal_object_utils.py
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
from functools import lru_cache
|
2
2
|
import re
|
3
3
|
from typing import Any, Callable, List, Optional, Tuple, Type, Union
|
4
|
+
from dcicutils.data_readers import RowReader
|
4
5
|
from dcicutils.portal_utils import Portal
|
5
6
|
from dcicutils.schema_utils import Schema
|
6
7
|
|
@@ -157,21 +158,33 @@ class PortalObject:
|
|
157
158
|
return f"{_path}.{key}" if _path else key
|
158
159
|
def list_to_dictionary(value: list) -> dict: # noqa
|
159
160
|
result = {}
|
160
|
-
for index, item in enumerate(
|
161
|
+
for index, item in enumerate(value):
|
161
162
|
result[f"#{index}"] = item
|
162
163
|
return result
|
163
164
|
diffs = {}
|
164
165
|
for key in a:
|
165
166
|
path = key_to_path(key)
|
166
167
|
if key not in b:
|
167
|
-
|
168
|
+
if a[key] != RowReader.CELL_DELETION_SENTINEL:
|
169
|
+
diffs[path] = {"value": a[key], "creating_value": True}
|
168
170
|
else:
|
169
171
|
if isinstance(a[key], dict) and isinstance(b[key], dict):
|
170
172
|
diffs.update(PortalObject._compare(a[key], b[key], compare=compare, _path=path))
|
171
173
|
elif isinstance(a[key], list) and isinstance(b[key], list):
|
174
|
+
# Note that lists will be compared in order, which means the when dealing with
|
175
|
+
# insertions/deletions to/from the list, we my easily mistakenly regard elements
|
176
|
+
# of the list to be different when they are really the same, since they occupy
|
177
|
+
# different indices within the array. This is just a known restriction of this
|
178
|
+
# comparison functionality; and perhaps actually technically correct, but probably
|
179
|
+
# in practice, at the application/semantic level, we likely regard the order of
|
180
|
+
# lists as unimportant, and with a little more work here we could try to detect
|
181
|
+
# and exclude from the diffs for a list, those elements in the list which are
|
182
|
+
# equal to each other but which reside at different indices with then two lists.
|
172
183
|
diffs.update(PortalObject._compare(list_to_dictionary(a[key]),
|
173
184
|
list_to_dictionary(b[key]), compare=compare, _path=path))
|
174
185
|
elif a[key] != b[key]:
|
175
|
-
if
|
176
|
-
diffs[path] = {"value":
|
186
|
+
if a[key] == RowReader.CELL_DELETION_SENTINEL:
|
187
|
+
diffs[path] = {"value": b[key], "deleting_value": True}
|
188
|
+
elif not callable(compare) or not compare(path, a[key], b[key]):
|
189
|
+
diffs[path] = {"value": a[key], "updating_value": b[key]}
|
177
190
|
return diffs
|
dcicutils/portal_utils.py
CHANGED
@@ -264,12 +264,12 @@ class Portal:
|
|
264
264
|
add_on = ""
|
265
265
|
return get_metadata(obj_id=object_id, vapp=self.vapp, key=self.key, add_on=add_on)
|
266
266
|
|
267
|
-
def patch_metadata(self, object_id: str, data:
|
267
|
+
def patch_metadata(self, object_id: str, data: dict) -> Optional[dict]:
|
268
268
|
if self.key:
|
269
269
|
return patch_metadata(obj_id=object_id, patch_item=data, key=self.key)
|
270
270
|
return self.patch(f"/{object_id}", data).json()
|
271
271
|
|
272
|
-
def post_metadata(self, object_type: str, data:
|
272
|
+
def post_metadata(self, object_type: str, data: dict) -> Optional[dict]:
|
273
273
|
if self.key:
|
274
274
|
return post_metadata(schema_name=object_type, post_item=data, key=self.key)
|
275
275
|
return self.post(f"/{object_type}", data).json()
|
dcicutils/structured_data.py
CHANGED
@@ -64,6 +64,12 @@ class StructuredDataSet:
|
|
64
64
|
return StructuredDataSet(file=file, portal=portal, schemas=schemas, autoadd=autoadd, order=order, prune=prune)
|
65
65
|
|
66
66
|
def validate(self, force: bool = False) -> None:
|
67
|
+
def data_without_deleted_properties(data: dict) -> dict:
|
68
|
+
nonlocal self
|
69
|
+
if self._prune:
|
70
|
+
return {key: value for key, value in data.items() if value != RowReader.CELL_DELETION_SENTINEL}
|
71
|
+
else:
|
72
|
+
return {key: "" if value == RowReader.CELL_DELETION_SENTINEL else value for key, value in data.items()}
|
67
73
|
if self._validated and not force:
|
68
74
|
return
|
69
75
|
self._validated = True
|
@@ -71,6 +77,7 @@ class StructuredDataSet:
|
|
71
77
|
if (schema := Schema.load_by_name(type_name, portal=self._portal)):
|
72
78
|
row_number = 0
|
73
79
|
for data in self.data[type_name]:
|
80
|
+
data = data_without_deleted_properties(data)
|
74
81
|
row_number += 1
|
75
82
|
if (validation_errors := schema.validate(data)) is not None:
|
76
83
|
for validation_error in validation_errors:
|
@@ -9,7 +9,7 @@ dcicutils/common.py,sha256=YE8Mt5-vaZWWz4uaChSVhqGFbFtW5QKtnIyOr4zG4vM,3955
|
|
9
9
|
dcicutils/contribution_scripts.py,sha256=0k5Gw1TumcD5SAcXVkDd6-yvuMEw-jUp5Kfb7FJH6XQ,2015
|
10
10
|
dcicutils/contribution_utils.py,sha256=vYLS1JUB3sKd24BUxZ29qUBqYeQBLK9cwo8x3k64uPg,25653
|
11
11
|
dcicutils/creds_utils.py,sha256=xrLekD49Ex0GOpL9n7LlJA4gvNcY7txTVFOSYD7LvEU,11113
|
12
|
-
dcicutils/data_readers.py,sha256
|
12
|
+
dcicutils/data_readers.py,sha256=02b_EnTHnRFpzqXzd847ArUGieL5PgXc6269L9xxF9w,5978
|
13
13
|
dcicutils/data_utils.py,sha256=k2OxOlsx7AJ6jF-YNlMyGus_JqSUBe4_n1s65Mv1gQQ,3098
|
14
14
|
dcicutils/deployment_utils.py,sha256=rcNUFMe_tsrG4CHEtgBe41cZx4Pk4JqISPsjrJRMoEs,68891
|
15
15
|
dcicutils/diff_utils.py,sha256=sQx-yz56DHAcQWOChYbAG3clXu7TbiZKlw-GggeveO0,8118
|
@@ -44,8 +44,8 @@ dcicutils/log_utils.py,sha256=7pWMc6vyrorUZQf-V-M3YC6zrPgNhuV_fzm9xqTPph0,10883
|
|
44
44
|
dcicutils/misc_utils.py,sha256=bMRWWxdbhuF3PKdCZEH-H4U1ecgT3Nag3EL92D9XGoY,100973
|
45
45
|
dcicutils/obfuscation_utils.py,sha256=fo2jOmDRC6xWpYX49u80bVNisqRRoPskFNX3ymFAmjw,5963
|
46
46
|
dcicutils/opensearch_utils.py,sha256=V2exmFYW8Xl2_pGFixF4I2Cc549Opwe4PhFi5twC0M8,1017
|
47
|
-
dcicutils/portal_object_utils.py,sha256=
|
48
|
-
dcicutils/portal_utils.py,sha256=
|
47
|
+
dcicutils/portal_object_utils.py,sha256=zc0pEacq-J8D1QE7pH1xnHC50fF3_VQ7VSKySOvRr0c,10551
|
48
|
+
dcicutils/portal_utils.py,sha256=8XVRCy882RrB8QT2gGvW36c4nT1RJCgUApt_wLP-EG8,26706
|
49
49
|
dcicutils/project_utils.py,sha256=qPdCaFmWUVBJw4rw342iUytwdQC0P-XKpK4mhyIulMM,31250
|
50
50
|
dcicutils/qa_checkers.py,sha256=cdXjeL0jCDFDLT8VR8Px78aS10hwNISOO5G_Zv2TZ6M,20534
|
51
51
|
dcicutils/qa_utils.py,sha256=TT0SiJWiuxYvbsIyhK9VO4uV_suxhB6CpuC4qPacCzQ,160208
|
@@ -59,15 +59,15 @@ dcicutils/secrets_utils.py,sha256=8dppXAsiHhJzI6NmOcvJV5ldvKkQZzh3Fl-cb8Wm7MI,19
|
|
59
59
|
dcicutils/sheet_utils.py,sha256=VlmzteONW5VF_Q4vo0yA5vesz1ViUah1MZ_yA1rwZ0M,33629
|
60
60
|
dcicutils/snapshot_utils.py,sha256=ymP7PXH6-yEiXAt75w0ldQFciGNqWBClNxC5gfX2FnY,22961
|
61
61
|
dcicutils/ssl_certificate_utils.py,sha256=F0ifz_wnRRN9dfrfsz7aCp4UDLgHEY8LaK7PjnNvrAQ,9707
|
62
|
-
dcicutils/structured_data.py,sha256=
|
62
|
+
dcicutils/structured_data.py,sha256=ljXvLy1yUcrAvguAlbAfQ_mqHILw84HEQ1_NcMnXBRk,34659
|
63
63
|
dcicutils/task_utils.py,sha256=MF8ujmTD6-O2AC2gRGPHyGdUrVKgtr8epT5XU8WtNjk,8082
|
64
64
|
dcicutils/tmpfile_utils.py,sha256=n95XF8dZVbQRSXBZTGToXXfSs3JUVRyN6c3ZZ0nhAWI,1403
|
65
65
|
dcicutils/trace_utils.py,sha256=g8kwV4ebEy5kXW6oOrEAUsurBcCROvwtZqz9fczsGRE,1769
|
66
66
|
dcicutils/validation_utils.py,sha256=cMZIU2cY98FYtzK52z5WUYck7urH6JcqOuz9jkXpqzg,14797
|
67
67
|
dcicutils/variant_utils.py,sha256=2H9azNx3xAj-MySg-uZ2SFqbWs4kZvf61JnK6b-h4Qw,4343
|
68
68
|
dcicutils/zip_utils.py,sha256=rnjNv_k6L9jT2SjDSgVXp4BEJYLtz9XN6Cl2Fy-tqnM,2027
|
69
|
-
dcicutils-8.7.1.
|
70
|
-
dcicutils-8.7.1.
|
71
|
-
dcicutils-8.7.1.
|
72
|
-
dcicutils-8.7.1.
|
73
|
-
dcicutils-8.7.1.
|
69
|
+
dcicutils-8.7.1.1b4.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
|
70
|
+
dcicutils-8.7.1.1b4.dist-info/METADATA,sha256=tdGvBos2PdrsL-KAOgu_qcVDh_4j8AcUdXpsxrEEWDU,3314
|
71
|
+
dcicutils-8.7.1.1b4.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
|
72
|
+
dcicutils-8.7.1.1b4.dist-info/entry_points.txt,sha256=8wbw5csMIgBXhkwfgsgJeuFcoUc0WsucUxmOyml2aoA,209
|
73
|
+
dcicutils-8.7.1.1b4.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|