dcicutils 8.7.1.1b2__py3-none-any.whl → 8.7.1.1b4__py3-none-any.whl

dcicutils/data_readers.py

@@ -10,6 +10,9 @@ Excel = Type["Excel"]
 
 class RowReader(abc.ABC):
 
+    CELL_DELETION_VALUES = ["*delete*"]  # cell values(s) indicating property deletion
+    CELL_DELETION_SENTINEL = object()  # special cell deletion sentinel value
+
     def __init__(self):
         self.header = None
         self.row_number = 0
@@ -45,8 +48,13 @@ class RowReader(abc.ABC):
     def is_terminating_row(self, row: Union[List[Optional[Any]], Tuple[Optional[Any]]]) -> bool:
         return False
 
-    def cell_value(self, value: Optional[Any]) -> Optional[Any]:
-        return str(value).strip() if value is not None else ""
+    def cell_value(self, value: Optional[Any]) -> str:
+        if value is None:
+            return ""
+        elif (value := str(value).strip()) in RowReader.CELL_DELETION_VALUES:
+            return RowReader.CELL_DELETION_SENTINEL
+        else:
+            return value
 
     def open(self) -> None:
         pass

dcicutils/portal_object_utils.py

@@ -1,6 +1,7 @@
 from functools import lru_cache
 import re
 from typing import Any, Callable, List, Optional, Tuple, Type, Union
+from dcicutils.data_readers import RowReader
 from dcicutils.portal_utils import Portal
 from dcicutils.schema_utils import Schema
 
@@ -157,21 +158,33 @@ class PortalObject:
             return f"{_path}.{key}" if _path else key
         def list_to_dictionary(value: list) -> dict:  # noqa
             result = {}
-            for index, item in enumerate(sorted(value)):  # ignore array order
+            for index, item in enumerate(value):
                 result[f"#{index}"] = item
             return result
         diffs = {}
         for key in a:
             path = key_to_path(key)
             if key not in b:
-                diffs[path] = {"value": a[key], "missing_value": True}
+                if a[key] != RowReader.CELL_DELETION_SENTINEL:
+                    diffs[path] = {"value": a[key], "creating_value": True}
             else:
                 if isinstance(a[key], dict) and isinstance(b[key], dict):
                     diffs.update(PortalObject._compare(a[key], b[key], compare=compare, _path=path))
                 elif isinstance(a[key], list) and isinstance(b[key], list):
+                    # Note that lists will be compared in order, which means the when dealing with
+                    # insertions/deletions to/from the list, we my easily mistakenly regard elements
+                    # of the list to be different when they are really the same, since they occupy
+                    # different indices within the array. This is just a known restriction of this
+                    # comparison functionality; and perhaps actually technically correct, but probably
+                    # in practice, at the application/semantic level, we likely regard the order of
+                    # lists as unimportant, and with a little more work here we could try to detect
+                    # and exclude from the diffs for a list, those elements in the list which are
+                    # equal to each other but which reside at different indices with then two lists.
                     diffs.update(PortalObject._compare(list_to_dictionary(a[key]),
                                                        list_to_dictionary(b[key]), compare=compare, _path=path))
                 elif a[key] != b[key]:
-                    if not callable(compare) or not compare(path, a[key], b[key]):
-                        diffs[path] = {"value": a[key], "differing_value": b[key]}
+                    if a[key] == RowReader.CELL_DELETION_SENTINEL:
+                        diffs[path] = {"value": b[key], "deleting_value": True}
+                    elif not callable(compare) or not compare(path, a[key], b[key]):
+                        diffs[path] = {"value": a[key], "updating_value": b[key]}
         return diffs

dcicutils/portal_utils.py

@@ -264,12 +264,12 @@ class Portal:
             add_on = ""
         return get_metadata(obj_id=object_id, vapp=self.vapp, key=self.key, add_on=add_on)
 
-    def patch_metadata(self, object_id: str, data: str) -> Optional[dict]:
+    def patch_metadata(self, object_id: str, data: dict) -> Optional[dict]:
         if self.key:
             return patch_metadata(obj_id=object_id, patch_item=data, key=self.key)
         return self.patch(f"/{object_id}", data).json()
 
-    def post_metadata(self, object_type: str, data: str) -> Optional[dict]:
+    def post_metadata(self, object_type: str, data: dict) -> Optional[dict]:
         if self.key:
             return post_metadata(schema_name=object_type, post_item=data, key=self.key)
         return self.post(f"/{object_type}", data).json()

dcicutils/structured_data.py

@@ -64,6 +64,12 @@ class StructuredDataSet:
         return StructuredDataSet(file=file, portal=portal, schemas=schemas, autoadd=autoadd, order=order, prune=prune)
 
     def validate(self, force: bool = False) -> None:
+        def data_without_deleted_properties(data: dict) -> dict:
+            nonlocal self
+            if self._prune:
+                return {key: value for key, value in data.items() if value != RowReader.CELL_DELETION_SENTINEL}
+            else:
+                return {key: "" if value == RowReader.CELL_DELETION_SENTINEL else value for key, value in data.items()}
         if self._validated and not force:
             return
         self._validated = True
@@ -71,6 +77,7 @@ class StructuredDataSet:
             if (schema := Schema.load_by_name(type_name, portal=self._portal)):
                 row_number = 0
                 for data in self.data[type_name]:
+                    data = data_without_deleted_properties(data)
                     row_number += 1
                     if (validation_errors := schema.validate(data)) is not None:
                         for validation_error in validation_errors:

{dcicutils-8.7.1.1b2.dist-info → dcicutils-8.7.1.1b4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcicutils
-Version: 8.7.1.1b2
+Version: 8.7.1.1b4
 Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
 Home-page: https://github.com/4dn-dcic/utils
 License: MIT

{dcicutils-8.7.1.1b2.dist-info → dcicutils-8.7.1.1b4.dist-info}/RECORD

@@ -9,7 +9,7 @@ dcicutils/common.py,sha256=YE8Mt5-vaZWWz4uaChSVhqGFbFtW5QKtnIyOr4zG4vM,3955
 dcicutils/contribution_scripts.py,sha256=0k5Gw1TumcD5SAcXVkDd6-yvuMEw-jUp5Kfb7FJH6XQ,2015
 dcicutils/contribution_utils.py,sha256=vYLS1JUB3sKd24BUxZ29qUBqYeQBLK9cwo8x3k64uPg,25653
 dcicutils/creds_utils.py,sha256=xrLekD49Ex0GOpL9n7LlJA4gvNcY7txTVFOSYD7LvEU,11113
-dcicutils/data_readers.py,sha256=-XVkptkHbiyKg0uQTokXa0gl0PbPMwKe-geiK92yfcw,5668
+dcicutils/data_readers.py,sha256=02b_EnTHnRFpzqXzd847ArUGieL5PgXc6269L9xxF9w,5978
 dcicutils/data_utils.py,sha256=k2OxOlsx7AJ6jF-YNlMyGus_JqSUBe4_n1s65Mv1gQQ,3098
 dcicutils/deployment_utils.py,sha256=rcNUFMe_tsrG4CHEtgBe41cZx4Pk4JqISPsjrJRMoEs,68891
 dcicutils/diff_utils.py,sha256=sQx-yz56DHAcQWOChYbAG3clXu7TbiZKlw-GggeveO0,8118
@@ -44,8 +44,8 @@ dcicutils/log_utils.py,sha256=7pWMc6vyrorUZQf-V-M3YC6zrPgNhuV_fzm9xqTPph0,10883
 dcicutils/misc_utils.py,sha256=bMRWWxdbhuF3PKdCZEH-H4U1ecgT3Nag3EL92D9XGoY,100973
 dcicutils/obfuscation_utils.py,sha256=fo2jOmDRC6xWpYX49u80bVNisqRRoPskFNX3ymFAmjw,5963
 dcicutils/opensearch_utils.py,sha256=V2exmFYW8Xl2_pGFixF4I2Cc549Opwe4PhFi5twC0M8,1017
-dcicutils/portal_object_utils.py,sha256=H5YAxGuYTJxnY3Pji58edIuA4QFPvJ5EZqDGMLnxXTE,9420
-dcicutils/portal_utils.py,sha256=jKYgZUYVdkg6VOs1hsiX4bSULLguOIBJFFRpvvZEklU,26704
+dcicutils/portal_object_utils.py,sha256=zc0pEacq-J8D1QE7pH1xnHC50fF3_VQ7VSKySOvRr0c,10551
+dcicutils/portal_utils.py,sha256=8XVRCy882RrB8QT2gGvW36c4nT1RJCgUApt_wLP-EG8,26706
 dcicutils/project_utils.py,sha256=qPdCaFmWUVBJw4rw342iUytwdQC0P-XKpK4mhyIulMM,31250
 dcicutils/qa_checkers.py,sha256=cdXjeL0jCDFDLT8VR8Px78aS10hwNISOO5G_Zv2TZ6M,20534
 dcicutils/qa_utils.py,sha256=TT0SiJWiuxYvbsIyhK9VO4uV_suxhB6CpuC4qPacCzQ,160208
@@ -59,15 +59,15 @@ dcicutils/secrets_utils.py,sha256=8dppXAsiHhJzI6NmOcvJV5ldvKkQZzh3Fl-cb8Wm7MI,19
 dcicutils/sheet_utils.py,sha256=VlmzteONW5VF_Q4vo0yA5vesz1ViUah1MZ_yA1rwZ0M,33629
 dcicutils/snapshot_utils.py,sha256=ymP7PXH6-yEiXAt75w0ldQFciGNqWBClNxC5gfX2FnY,22961
 dcicutils/ssl_certificate_utils.py,sha256=F0ifz_wnRRN9dfrfsz7aCp4UDLgHEY8LaK7PjnNvrAQ,9707
-dcicutils/structured_data.py,sha256=VxkyBBMFoRIEoPZxQuYU1iVO3piQwSsG-W6_pKy3qXE,34225
+dcicutils/structured_data.py,sha256=ljXvLy1yUcrAvguAlbAfQ_mqHILw84HEQ1_NcMnXBRk,34659
 dcicutils/task_utils.py,sha256=MF8ujmTD6-O2AC2gRGPHyGdUrVKgtr8epT5XU8WtNjk,8082
 dcicutils/tmpfile_utils.py,sha256=n95XF8dZVbQRSXBZTGToXXfSs3JUVRyN6c3ZZ0nhAWI,1403
 dcicutils/trace_utils.py,sha256=g8kwV4ebEy5kXW6oOrEAUsurBcCROvwtZqz9fczsGRE,1769
 dcicutils/validation_utils.py,sha256=cMZIU2cY98FYtzK52z5WUYck7urH6JcqOuz9jkXpqzg,14797
 dcicutils/variant_utils.py,sha256=2H9azNx3xAj-MySg-uZ2SFqbWs4kZvf61JnK6b-h4Qw,4343
 dcicutils/zip_utils.py,sha256=rnjNv_k6L9jT2SjDSgVXp4BEJYLtz9XN6Cl2Fy-tqnM,2027
-dcicutils-8.7.1.1b2.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
-dcicutils-8.7.1.1b2.dist-info/METADATA,sha256=p9ZOsaryGCxzrudhx2lRFFlWY0mW1TiGu2k76qUufyM,3314
-dcicutils-8.7.1.1b2.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
-dcicutils-8.7.1.1b2.dist-info/entry_points.txt,sha256=8wbw5csMIgBXhkwfgsgJeuFcoUc0WsucUxmOyml2aoA,209
-dcicutils-8.7.1.1b2.dist-info/RECORD,,
+dcicutils-8.7.1.1b4.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
+dcicutils-8.7.1.1b4.dist-info/METADATA,sha256=tdGvBos2PdrsL-KAOgu_qcVDh_4j8AcUdXpsxrEEWDU,3314
+dcicutils-8.7.1.1b4.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+dcicutils-8.7.1.1b4.dist-info/entry_points.txt,sha256=8wbw5csMIgBXhkwfgsgJeuFcoUc0WsucUxmOyml2aoA,209
+dcicutils-8.7.1.1b4.dist-info/RECORD,,