dcicutils 8.7.0.1b33__tar.gz → 8.7.0.1b35__tar.gz

{dcicutils-8.7.0.1b33 → dcicutils-8.7.0.1b35}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcicutils
-Version: 8.7.0.1b33
+Version: 8.7.0.1b35
 Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
 Home-page: https://github.com/4dn-dcic/utils
 License: MIT

dcicutils-8.7.0.1b35/dcicutils/portal_object_utils.py

@@ -0,0 +1,165 @@
+from functools import lru_cache
+import re
+from typing import Any, Callable, List, Optional, Tuple, Type, Union
+from dcicutils.portal_utils import Portal
+from dcicutils.schema_utils import Schema
+
+PortalObject = Type["PortalObject"]  # Forward type reference for type hints.
+
+
+class PortalObject:
+
+    def __init__(self, portal: Portal, portal_object: dict, portal_object_type: Optional[str] = None) -> None:
+        self._portal = portal
+        self._data = portal_object
+        self._type = portal_object_type if isinstance(portal_object_type, str) and portal_object_type else None
+
+    @property
+    def data(self):
+        return self._data
+
+    @property
+    @lru_cache(maxsize=1)
+    def type(self):
+        return self._type or Portal.get_schema_type(self._data)
+
+    @property
+    @lru_cache(maxsize=1)
+    def types(self):
+        return self._type or Portal.get_schema_types(self._data)
+
+    @property
+    @lru_cache(maxsize=1)
+    def uuid(self) -> Optional[str]:
+        return self._data.get("uuid") if isinstance(self._data, dict) else None
+
+    @property
+    @lru_cache(maxsize=1)
+    def schema(self):
+        return self._portal.get_schema(self.type)
+
+    @property
+    @lru_cache(maxsize=1)
+    def identifying_properties(self) -> List[str]:
+        """
+        Returns the list of all identifying property names of this Portal object which actually have values.
+        Implicitly include "uuid" and "identifier" properties as identifying properties if they are actually
+        properties in the object schema, and favor these (first); defavor "aliases"; no other ordering defined.
+        """
+        if not (schema := self.schema) or not (schema_identifying_properties := schema.get("identifyingProperties")):
+            return []
+        identifying_properties = []
+        for identifying_property in schema_identifying_properties:
+            if identifying_property not in ["uuid", "identifier", "aliases"]:
+                if self._data.get(identifying_property):
+                    identifying_properties.append(identifying_property)
+        if self._data.get("identifier"):
+            identifying_properties.insert(0, "identifier")
+        if self._data.get("uuid"):
+            identifying_properties.insert(0, "uuid")
+        if "aliases" in schema_identifying_properties and self._data.get("aliases"):
+            identifying_properties.append("aliases")
+        return identifying_properties
+
+    @property
+    @lru_cache(maxsize=1)
+    def identifying_paths(self) -> List[str]:
+        """
+        Returns a list of the possible Portal URL paths identifying this Portal object.
+        """
+        if not (identifying_properties := self.identifying_properties):
+            return []
+        identifying_paths = []
+        for identifying_property in identifying_properties:
+            if (identifying_value := self._data.get(identifying_property)):
+                if identifying_property == "uuid":
+                    identifying_paths.append(f"/{identifying_value}")
+                # For now at least we include the path both with and without the schema type component,
+                # as for some identifying values, it works (only) with, and some, it works (only) without.
+                # For example: If we have FileSet with "accession", an identifying property, with value
+                # SMAFSFXF1RO4 then /SMAFSFXF1RO4 works but /FileSet/SMAFSFXF1RO4 does not; and
+                # conversely using "submitted_id", also an identifying property, with value
+                # UW_FILE-SET_COLO-829BL_HI-C_1 then /UW_FILE-SET_COLO-829BL_HI-C_1 does
+                # not work but /FileSet/UW_FILE-SET_COLO-829BL_HI-C_1 does work.
+                elif isinstance(identifying_value, list):
+                    for identifying_value_item in identifying_value:
+                        identifying_paths.append(f"/{self.type}/{identifying_value_item}")
+                        identifying_paths.append(f"/{identifying_value_item}")
+                else:
+                    identifying_paths.append(f"/{self.type}/{identifying_value}")
+                    identifying_paths.append(f"/{identifying_value}")
+        return identifying_paths
+
+    @property
+    @lru_cache(maxsize=1)
+    def identifying_path(self) -> Optional[str]:
+        if identifying_paths := self.identifying_paths:
+            return identifying_paths[0]
+
+    def lookup(self, include_identifying_path: bool = False,
+               raw: bool = False) -> Optional[Union[Tuple[PortalObject, str], PortalObject]]:
+        return self._lookup(raw=raw) if include_identifying_path else self._lookup(raw=raw)[0]
+
+    def lookup_identifying_path(self) -> Optional[str]:
+        return self._lookup()[1]
+
+    def _lookup(self, raw: bool = False) -> Tuple[Optional[PortalObject], Optional[str]]:
+        try:
+            for identifying_path in self.identifying_paths:
+                if (value := self._portal.get(identifying_path, raw=raw)) and (value.status_code == 200):
+                    return PortalObject(self._portal, value.json(), self.type if raw else None), identifying_path
+        except Exception:
+            pass
+        return None, self.identifying_path
+
+    def compare(self, value: Union[dict, PortalObject], consider_link_to: bool = False) -> dict:
+        """
+        Compares this Portal object against the given Portal object value; noting differences values of properites
+        which they have in common; and properties which are in this Portal object and not in the given Portal object;
+        we do NOT check the converse, i.e. properties in the given Portal object which are not in this Portal object.
+        Returns a dictionary with a description of the differences.
+        """
+        def are_properties_equal(property_path: str, property_value_a: Any, property_value_b: Any) -> bool:
+            if property_value_a == property_value_b:
+                return True
+            nonlocal self
+            if (schema := self.schema) and (property_type := Schema.get_property_by_path(schema, property_path)):
+                if link_to := property_type.get("linkTo"):
+                    if a := self._portal.get(f"/{link_to}/{property_value_a}", raw=True):
+                        if (a.status_code == 200) and (a := a.json()):
+                            if b := self._portal.get(f"/{link_to}/{property_value_b}", raw=True):
+                                if (b.status_code == 200) and (b := b.json()):
+                                    return a == b
+            return False
+        return PortalObject._compare(self._data, value.data if isinstance(value, PortalObject) else value,
+                                     compare=are_properties_equal if consider_link_to else None)
+
+    _ARRAY_KEY_REGULAR_EXPRESSION = re.compile(rf"^({Schema._ARRAY_NAME_SUFFIX_CHAR}\d+)$")
+
+    @staticmethod
+    def _compare(a: dict, b: dict, compare: Optional[Callable] = None, _path: Optional[str] = None) -> dict:
+        def key_to_path(key: str) -> Optional[str]:  # noqa
+            nonlocal _path
+            if match := PortalObject._ARRAY_KEY_REGULAR_EXPRESSION.search(key):
+                return f"{_path}{match.group(1)}" if _path else match.group(1)
+            return f"{_path}.{key}" if _path else key
+        def list_to_dictionary(value: list) -> dict:  # noqa
+            result = {}
+            for index, item in enumerate(sorted(value)):  # ignore array order
+                result[f"#{index}"] = item
+            return result
+        diffs = {}
+        for key in a:
+            path = key_to_path(key)
+            if key not in b:
+                diffs[path] = {"value": a[key], "missing_value": True}
+            else:
+                if isinstance(a[key], dict) and isinstance(b[key], dict):
+                    diffs.update(PortalObject._compare(a[key], b[key], compare=compare, _path=path))
+                elif isinstance(a[key], list) and isinstance(b[key], list):
+                    diffs.update(PortalObject._compare(list_to_dictionary(a[key]),
+                                                       list_to_dictionary(b[key]), compare=compare, _path=path))
+                elif a[key] != b[key]:
+                    if not callable(compare) or not compare(path, a[key], b[key]):
+                        diffs[path] = {"value": a[key], "differing_value": b[key]}
+        return diffs

{dcicutils-8.7.0.1b33 → dcicutils-8.7.0.1b35}/dcicutils/schema_utils.py

@@ -1,4 +1,6 @@
-from typing import Any, Dict, List
+import os
+from typing import Any, Dict, List, Optional, Tuple
+from dcicutils.misc_utils import to_camel_case
 
 
 class JsonSchemaConstants:
@@ -183,3 +185,79 @@ def get_one_of_formats(schema: Dict[str, Any]) -> List[str]:
         for one_of_schema in get_one_of(schema)
         if get_format(one_of_schema)
     ]
+
+
+class Schema:
+
+    def __init__(self, schema: dict, schema_type: Optional[str] = None) -> None:
+        self._data = schema if isinstance(schema, dict) else (schema.data if isinstance(schema, Schema) else {})
+        self._type = (isinstance(schema_type, str) and schema_type) or Schema.type_name(self._data.get("title", ""))
+
+    @property
+    def data(self) -> dict:
+        return self._data
+
+    @property
+    def type(self) -> str:
+        return self._type
+
+    @staticmethod
+    def type_name(value: str) -> Optional[str]:  # File or other name.
+        if isinstance(value, str) and (value := os.path.basename(value.replace(" ", ""))):
+            return to_camel_case(value[0:dot] if (dot := value.rfind(".")) >= 0 else value)
+
+    def property_by_path(self, property_path: str) -> Optional[dict]:
+        """
+        TODO
+        """
+        return Schema.get_property_by_path(self._data, property_path)
+
+    _ARRAY_NAME_SUFFIX_CHAR = "#"
+    _DOTTED_NAME_DELIMITER_CHAR = "."
+
+    @staticmethod
+    def get_property_by_path(schema: dict, property_path: str) -> Optional[dict]:
+        if not isinstance(schema, dict) or not isinstance(property_path, str):
+            return None
+        elif not (schema_properties := schema.get("properties")):
+            return None
+        property_paths = property_path.split(Schema._DOTTED_NAME_DELIMITER_CHAR)
+        for property_index, property_name in enumerate(property_paths):
+            property_name, array_specifiers = Schema._unarrayize_property_name(property_name)
+            if not (property_value := schema_properties.get(property_name)):
+                return None
+            elif (property_type := property_value.get("type")) == "object":
+                property_paths_tail = Schema._DOTTED_NAME_DELIMITER_CHAR.join(property_paths[property_index + 1:])
+                return Schema.get_property_by_path(property_value, property_paths_tail)
+            elif (property_type := property_value.get("type")) == "array":
+                if not array_specifiers:
+                    if property_index == len(property_paths) - 1:
+                        return property_value
+                    return None
+                for array_index in range(len(array_specifiers)):
+                    if property_type != "array":
+                        return None
+                    elif not (array_items := property_value.get("items")):
+                        return None
+                    property_type = (property_value := array_items).get("type")
+                if property_type == "object":
+                    if property_index == len(property_paths) - 1:
+                        return property_value
+                    property_paths_tail = Schema._DOTTED_NAME_DELIMITER_CHAR.join(property_paths[property_index + 1:])
+                    return Schema.get_property_by_path(property_value, property_paths_tail)
+        return property_value
+
+    @staticmethod
+    def _unarrayize_property_name(property_name: str) -> Tuple[str, Optional[List[int]]]:
+        if len(components := (property_name := property_name.strip()).split(Schema._ARRAY_NAME_SUFFIX_CHAR)) < 2:
+            return property_name, None
+        unarrayized_property_name = components[0].strip()
+        array_specifiers = []
+        for component in components[1:]:
+            if component.isdigit():
+                array_specifiers.append(int(component))
+            elif component == "":
+                array_specifiers.append(0)
+            else:
+                return property_name, None
+        return unarrayized_property_name, array_specifiers

{dcicutils-8.7.0.1b33 → dcicutils-8.7.0.1b35}/dcicutils/structured_data.py

@@ -74,7 +74,7 @@ class StructuredDataSet:
                     row_number += 1
                     if (validation_errors := schema.validate(data)) is not None:
                         for validation_error in validation_errors:
-                            self._note_error({"src": create_dict(type=schema.name, row=row_number),
+                            self._note_error({"src": create_dict(type=schema.type, row=row_number),
                                               "error": validation_error}, "validation")
 
     @property
@@ -168,7 +168,7 @@ class StructuredDataSet:
             if not structured_row_template:  # Delay creation just so we don't reference schema if there are no rows.
                 if not schema and not noschema and not (schema := Schema.load_by_name(type_name, portal=self._portal)):
                     noschema = True
-                elif schema and (schema_name := schema.name):
+                elif schema and (schema_name := schema.type):
                     type_name = schema_name
                 structured_row_template = _StructuredRowTemplate(reader.header, schema)
             structured_row = structured_row_template.create_row()
@@ -222,7 +222,7 @@ class _StructuredRowTemplate:
 
     def set_value(self, data: dict, column_name: str, value: str, file: Optional[str], row_number: int = -1) -> None:
         if (set_value_function := self._set_value_functions.get(column_name)):
-            src = create_dict(type=self._schema.name if self._schema else None,
+            src = create_dict(type=self._schema.type if self._schema else None,
                               column=column_name, file=file, row=row_number)
             set_value_function(data, value, src)
 
@@ -319,8 +319,8 @@ class _StructuredRowTemplate:
 class Schema:
 
     def __init__(self, schema_json: dict, portal: Optional[Portal] = None) -> None:
-        self.data = schema_json
-        self.name = Schema.type_name(schema_json.get("title", "")) if schema_json else ""
+        self._data = schema_json if isinstance(schema_json, dict) else {}
+        self._type = Schema.type_name(schema_json.get("title", ""))
         self._portal = portal  # Needed only to resolve linkTo references.
         self._map_value_functions = {
             "boolean": self._map_function_boolean,
@@ -333,6 +333,14 @@ class Schema:
         self._unresolved_refs = []
         self._typeinfo = self._create_typeinfo(schema_json)
 
+    @property
+    def data(self) -> dict:
+        return self._data
+
+    @property
+    def type(self) -> str:
+        return self._type
+
     @staticmethod
     def load_by_name(name: str, portal: Portal) -> Optional[dict]:
         schema_json = portal.get_schema(Schema.type_name(name)) if portal else None

{dcicutils-8.7.0.1b33 → dcicutils-8.7.0.1b35}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dcicutils"
-version = "8.7.0.1b33"  # TODO: To become 8.7.1
+version = "8.7.0.1b35"  # TODO: To become 8.7.1
 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources"
 authors = ["4DN-DCIC Team <support@4dnucleome.org>"]
 license = "MIT"

dcicutils-8.7.0.1b33/dcicutils/portal_object_utils.py

@@ -1,119 +0,0 @@
-from dcicutils.schema_utils import get_identifying_properties
-from dcicutils.portal_utils import Portal
-from functools import lru_cache
-from typing import List, Optional, Tuple, Union
-
-
-class PortalObject:
-
-    def __init__(self, portal: Portal, portal_object: dict, portal_object_type: Optional[str] = None) -> None:
-        self._portal = portal
-        self._data = portal_object
-        self._type = portal_object_type if isinstance(portal_object_type, str) and portal_object_type else None
-
-    @property
-    def data(self):
-        return self._data
-
-    @property
-    @lru_cache(maxsize=1)
-    def schema(self):
-        return self._portal.get_schema(self.schema_type)
-
-    @property
-    @lru_cache(maxsize=1)
-    def schema_type(self):
-        return self._type or Portal.get_schema_type(self._data)
-
-    @property
-    @lru_cache(maxsize=1)
-    def schema_types(self):
-        return self._type or Portal.get_schema_types(self._data)
-
-    @property
-    @lru_cache(maxsize=1)
-    def schema_identifying_properties(self) -> list:
-        if not (schema := self.schema):
-            return []
-        return get_identifying_properties(schema)
-
-    @property
-    @lru_cache(maxsize=1)
-    def uuid(self) -> Optional[str]:
-        return PortalObject.get_uuid(self._data)
-
-    @staticmethod
-    def get_uuid(portal_object: dict) -> Optional[str]:
-        return portal_object.get("uuid") if isinstance(portal_object, dict) else None
-
-    @property
-    @lru_cache(maxsize=1)
-    def identifying_properties(self) -> List[str]:
-        """
-        Returns the list of all identifying property names of this Portal object which actually have values.
-        Implicitly include "uuid" and "identifier" properties as identifying properties if they are actually
-        properties in the object schema, and favor these (first); defavor "aliases"; no other ordering defined.
-        """
-        identifying_properties = []
-        for identifying_property in self.schema_identifying_properties:
-            if identifying_property not in ["uuid", "identifier", "aliases"]:
-                if self._data.get(identifying_property):
-                    identifying_properties.append(identifying_property)
-        if self._data.get("identifier"):
-            identifying_properties.insert(0, "identifier")
-        if self._data.get("uuid"):
-            identifying_properties.insert(0, "uuid")
-        if "aliases" in self.schema_identifying_properties and self._data.get("aliases"):
-            identifying_properties.append("aliases")
-        return identifying_properties
-
-    @property
-    @lru_cache(maxsize=1)
-    def identifying_paths(self) -> List[str]:
-        """
-        Returns a list of the possible Portal URL paths identifying this Portal object.
-        """
-        if not (identifying_properties := self.identifying_properties):
-            return []
-        identifying_paths = []
-        for identifying_property in identifying_properties:
-            if (identifying_value := self._data.get(identifying_property)):
-                if identifying_property == "uuid":
-                    identifying_paths.append(f"/{identifying_value}")
-                # For now at least we include the path both with and without the schema type component
-                # as for some identifying values it works (only) with and some it works (only) without.
-                # For example: If we have FileSet with "accession", an identifying property, with value
-                # SMAFSFXF1RO4 then /SMAFSFXF1RO4 works but /FileSet/SMAFSFXF1RO4 does not; and
-                # conversely using "submitted_id", also an identifying property, with value
-                # UW_FILE-SET_COLO-829BL_HI-C_1 then /UW_FILE-SET_COLO-829BL_HI-C_1 does
-                # not work but /FileSet/UW_FILE-SET_COLO-829BL_HI-C_1 does work.
-                elif isinstance(identifying_value, list):
-                    for identifying_value_item in identifying_value:
-                        identifying_paths.append(f"/{self.schema_type}/{identifying_value_item}")
-                        identifying_paths.append(f"/{identifying_value_item}")
-                else:
-                    identifying_paths.append(f"/{self.schema_type}/{identifying_value}")
-                    identifying_paths.append(f"/{identifying_value}")
-        return identifying_paths
-
-    @property
-    @lru_cache(maxsize=1)
-    def identifying_path(self) -> Optional[str]:
-        if identifying_paths := self.identifying_paths:
-            return identifying_paths[0]
-
-    def lookup(self, include_identifying_path: bool = False,
-               raw: bool = False) -> Optional[Union[Tuple[dict, str], dict]]:
-        return self._lookup(raw=raw) if include_identifying_path else self._lookup(raw=raw)[0]
-
-    def lookup_identifying_path(self) -> Optional[str]:
-        return self._lookup()[1]
-
-    def _lookup(self, raw: bool = False) -> Tuple[Optional[dict], Optional[str]]:
-        try:
-            for identifying_path in self.identifying_paths:
-                if (value := self._portal.get(identifying_path, raw=raw)) and (value.status_code == 200):
-                    return value.json(), identifying_path
-        except Exception:
-            pass
-        return None, self.identifying_path