dcicutils 8.7.1.1b2__tar.gz → 8.7.1.1b4__tar.gz

{dcicutils-8.7.1.1b2 → dcicutils-8.7.1.1b4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcicutils
-Version: 8.7.1.1b2
+Version: 8.7.1.1b4
 Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
 Home-page: https://github.com/4dn-dcic/utils
 License: MIT

{dcicutils-8.7.1.1b2 → dcicutils-8.7.1.1b4}/dcicutils/data_readers.py

@@ -10,6 +10,9 @@ Excel = Type["Excel"]
 
 class RowReader(abc.ABC):
 
+    CELL_DELETION_VALUES = ["*delete*"]  # cell values(s) indicating property deletion
+    CELL_DELETION_SENTINEL = object()  # special cell deletion sentinel value
+
     def __init__(self):
         self.header = None
         self.row_number = 0
@@ -45,8 +48,13 @@ class RowReader(abc.ABC):
     def is_terminating_row(self, row: Union[List[Optional[Any]], Tuple[Optional[Any]]]) -> bool:
         return False
 
-    def cell_value(self, value: Optional[Any]) -> Optional[Any]:
-        return str(value).strip() if value is not None else ""
+    def cell_value(self, value: Optional[Any]) -> str:
+        if value is None:
+            return ""
+        elif (value := str(value).strip()) in RowReader.CELL_DELETION_VALUES:
+            return RowReader.CELL_DELETION_SENTINEL
+        else:
+            return value
 
     def open(self) -> None:
         pass

{dcicutils-8.7.1.1b2 → dcicutils-8.7.1.1b4}/dcicutils/portal_object_utils.py

@@ -1,6 +1,7 @@
 from functools import lru_cache
 import re
 from typing import Any, Callable, List, Optional, Tuple, Type, Union
+from dcicutils.data_readers import RowReader
 from dcicutils.portal_utils import Portal
 from dcicutils.schema_utils import Schema
 
@@ -157,21 +158,33 @@ class PortalObject:
             return f"{_path}.{key}" if _path else key
         def list_to_dictionary(value: list) -> dict:  # noqa
             result = {}
-            for index, item in enumerate(sorted(value)):  # ignore array order
+            for index, item in enumerate(value):
                 result[f"#{index}"] = item
             return result
         diffs = {}
         for key in a:
             path = key_to_path(key)
             if key not in b:
-                diffs[path] = {"value": a[key], "missing_value": True}
+                if a[key] != RowReader.CELL_DELETION_SENTINEL:
+                    diffs[path] = {"value": a[key], "creating_value": True}
             else:
                 if isinstance(a[key], dict) and isinstance(b[key], dict):
                     diffs.update(PortalObject._compare(a[key], b[key], compare=compare, _path=path))
                 elif isinstance(a[key], list) and isinstance(b[key], list):
+                    # Note that lists will be compared in order, which means the when dealing with
+                    # insertions/deletions to/from the list, we my easily mistakenly regard elements
+                    # of the list to be different when they are really the same, since they occupy
+                    # different indices within the array. This is just a known restriction of this
+                    # comparison functionality; and perhaps actually technically correct, but probably
+                    # in practice, at the application/semantic level, we likely regard the order of
+                    # lists as unimportant, and with a little more work here we could try to detect
+                    # and exclude from the diffs for a list, those elements in the list which are
+                    # equal to each other but which reside at different indices with then two lists.
                     diffs.update(PortalObject._compare(list_to_dictionary(a[key]),
                                                        list_to_dictionary(b[key]), compare=compare, _path=path))
                 elif a[key] != b[key]:
-                    if not callable(compare) or not compare(path, a[key], b[key]):
-                        diffs[path] = {"value": a[key], "differing_value": b[key]}
+                    if a[key] == RowReader.CELL_DELETION_SENTINEL:
+                        diffs[path] = {"value": b[key], "deleting_value": True}
+                    elif not callable(compare) or not compare(path, a[key], b[key]):
+                        diffs[path] = {"value": a[key], "updating_value": b[key]}
         return diffs

{dcicutils-8.7.1.1b2 → dcicutils-8.7.1.1b4}/dcicutils/portal_utils.py

@@ -264,12 +264,12 @@ class Portal:
             add_on = ""
         return get_metadata(obj_id=object_id, vapp=self.vapp, key=self.key, add_on=add_on)
 
-    def patch_metadata(self, object_id: str, data: str) -> Optional[dict]:
+    def patch_metadata(self, object_id: str, data: dict) -> Optional[dict]:
         if self.key:
             return patch_metadata(obj_id=object_id, patch_item=data, key=self.key)
         return self.patch(f"/{object_id}", data).json()
 
-    def post_metadata(self, object_type: str, data: str) -> Optional[dict]:
+    def post_metadata(self, object_type: str, data: dict) -> Optional[dict]:
         if self.key:
             return post_metadata(schema_name=object_type, post_item=data, key=self.key)
         return self.post(f"/{object_type}", data).json()

{dcicutils-8.7.1.1b2 → dcicutils-8.7.1.1b4}/dcicutils/structured_data.py

@@ -64,6 +64,12 @@ class StructuredDataSet:
         return StructuredDataSet(file=file, portal=portal, schemas=schemas, autoadd=autoadd, order=order, prune=prune)
 
     def validate(self, force: bool = False) -> None:
+        def data_without_deleted_properties(data: dict) -> dict:
+            nonlocal self
+            if self._prune:
+                return {key: value for key, value in data.items() if value != RowReader.CELL_DELETION_SENTINEL}
+            else:
+                return {key: "" if value == RowReader.CELL_DELETION_SENTINEL else value for key, value in data.items()}
         if self._validated and not force:
             return
         self._validated = True
@@ -71,6 +77,7 @@ class StructuredDataSet:
             if (schema := Schema.load_by_name(type_name, portal=self._portal)):
                 row_number = 0
                 for data in self.data[type_name]:
+                    data = data_without_deleted_properties(data)
                     row_number += 1
                     if (validation_errors := schema.validate(data)) is not None:
                         for validation_error in validation_errors:

{dcicutils-8.7.1.1b2 → dcicutils-8.7.1.1b4}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dcicutils"
-version = "8.7.1.1b2"  # TODO: To become 8.7.2
+version = "8.7.1.1b4"  # TODO: To become 8.7.2
 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources"
 authors = ["4DN-DCIC Team <support@4dnucleome.org>"]
 license = "MIT"