dcicutils 8.7.0.1b33__py3-none-any.whl → 8.7.0.1b35__py3-none-any.whl

dcicutils/portal_object_utils.py

@@ -1,7 +1,10 @@
-from dcicutils.schema_utils import get_identifying_properties
-from dcicutils.portal_utils import Portal
 from functools import lru_cache
-from typing import List, Optional, Tuple, Union
+import re
+from typing import Any, Callable, List, Optional, Tuple, Type, Union
+from dcicutils.portal_utils import Portal
+from dcicutils.schema_utils import Schema
+
+PortalObject = Type["PortalObject"]  # Forward type reference for type hints.
 
 
 class PortalObject:
@@ -17,34 +20,23 @@ class PortalObject:
 
     @property
     @lru_cache(maxsize=1)
-    def schema(self):
-        return self._portal.get_schema(self.schema_type)
-
-    @property
-    @lru_cache(maxsize=1)
-    def schema_type(self):
+    def type(self):
         return self._type or Portal.get_schema_type(self._data)
 
     @property
     @lru_cache(maxsize=1)
-    def schema_types(self):
+    def types(self):
         return self._type or Portal.get_schema_types(self._data)
 
     @property
     @lru_cache(maxsize=1)
-    def schema_identifying_properties(self) -> list:
-        if not (schema := self.schema):
-            return []
-        return get_identifying_properties(schema)
+    def uuid(self) -> Optional[str]:
+        return self._data.get("uuid") if isinstance(self._data, dict) else None
 
     @property
     @lru_cache(maxsize=1)
-    def uuid(self) -> Optional[str]:
-        return PortalObject.get_uuid(self._data)
-
-    @staticmethod
-    def get_uuid(portal_object: dict) -> Optional[str]:
-        return portal_object.get("uuid") if isinstance(portal_object, dict) else None
+    def schema(self):
+        return self._portal.get_schema(self.type)
 
     @property
     @lru_cache(maxsize=1)
@@ -54,8 +46,10 @@ class PortalObject:
         Implicitly include "uuid" and "identifier" properties as identifying properties if they are actually
         properties in the object schema, and favor these (first); defavor "aliases"; no other ordering defined.
         """
+        if not (schema := self.schema) or not (schema_identifying_properties := schema.get("identifyingProperties")):
+            return []
         identifying_properties = []
-        for identifying_property in self.schema_identifying_properties:
+        for identifying_property in schema_identifying_properties:
             if identifying_property not in ["uuid", "identifier", "aliases"]:
                 if self._data.get(identifying_property):
                     identifying_properties.append(identifying_property)
@@ -63,7 +57,7 @@ class PortalObject:
             identifying_properties.insert(0, "identifier")
         if self._data.get("uuid"):
             identifying_properties.insert(0, "uuid")
-        if "aliases" in self.schema_identifying_properties and self._data.get("aliases"):
+        if "aliases" in schema_identifying_properties and self._data.get("aliases"):
             identifying_properties.append("aliases")
         return identifying_properties
 
@@ -80,8 +74,8 @@ class PortalObject:
             if (identifying_value := self._data.get(identifying_property)):
                 if identifying_property == "uuid":
                     identifying_paths.append(f"/{identifying_value}")
-                # For now at least we include the path both with and without the schema type component
-                # as for some identifying values it works (only) with and some it works (only) without.
+                # For now at least we include the path both with and without the schema type component,
+                # as for some identifying values, it works (only) with, and some, it works (only) without.
                 # For example: If we have FileSet with "accession", an identifying property, with value
                 # SMAFSFXF1RO4 then /SMAFSFXF1RO4 works but /FileSet/SMAFSFXF1RO4 does not; and
                 # conversely using "submitted_id", also an identifying property, with value
@@ -89,10 +83,10 @@ class PortalObject:
                 # not work but /FileSet/UW_FILE-SET_COLO-829BL_HI-C_1 does work.
                 elif isinstance(identifying_value, list):
                     for identifying_value_item in identifying_value:
-                        identifying_paths.append(f"/{self.schema_type}/{identifying_value_item}")
+                        identifying_paths.append(f"/{self.type}/{identifying_value_item}")
                         identifying_paths.append(f"/{identifying_value_item}")
                 else:
-                    identifying_paths.append(f"/{self.schema_type}/{identifying_value}")
+                    identifying_paths.append(f"/{self.type}/{identifying_value}")
                     identifying_paths.append(f"/{identifying_value}")
         return identifying_paths
 
@@ -103,17 +97,69 @@ class PortalObject:
             return identifying_paths[0]
 
     def lookup(self, include_identifying_path: bool = False,
-               raw: bool = False) -> Optional[Union[Tuple[dict, str], dict]]:
+               raw: bool = False) -> Optional[Union[Tuple[PortalObject, str], PortalObject]]:
         return self._lookup(raw=raw) if include_identifying_path else self._lookup(raw=raw)[0]
 
     def lookup_identifying_path(self) -> Optional[str]:
         return self._lookup()[1]
 
-    def _lookup(self, raw: bool = False) -> Tuple[Optional[dict], Optional[str]]:
+    def _lookup(self, raw: bool = False) -> Tuple[Optional[PortalObject], Optional[str]]:
         try:
             for identifying_path in self.identifying_paths:
                 if (value := self._portal.get(identifying_path, raw=raw)) and (value.status_code == 200):
-                    return value.json(), identifying_path
+                    return PortalObject(self._portal, value.json(), self.type if raw else None), identifying_path
         except Exception:
             pass
         return None, self.identifying_path
+
+    def compare(self, value: Union[dict, PortalObject], consider_link_to: bool = False) -> dict:
+        """
+        Compares this Portal object against the given Portal object value; noting differences values of properites
+        which they have in common; and properties which are in this Portal object and not in the given Portal object;
+        we do NOT check the converse, i.e. properties in the given Portal object which are not in this Portal object.
+        Returns a dictionary with a description of the differences.
+        """
+        def are_properties_equal(property_path: str, property_value_a: Any, property_value_b: Any) -> bool:
+            if property_value_a == property_value_b:
+                return True
+            nonlocal self
+            if (schema := self.schema) and (property_type := Schema.get_property_by_path(schema, property_path)):
+                if link_to := property_type.get("linkTo"):
+                    if a := self._portal.get(f"/{link_to}/{property_value_a}", raw=True):
+                        if (a.status_code == 200) and (a := a.json()):
+                            if b := self._portal.get(f"/{link_to}/{property_value_b}", raw=True):
+                                if (b.status_code == 200) and (b := b.json()):
+                                    return a == b
+            return False
+        return PortalObject._compare(self._data, value.data if isinstance(value, PortalObject) else value,
+                                     compare=are_properties_equal if consider_link_to else None)
+
+    _ARRAY_KEY_REGULAR_EXPRESSION = re.compile(rf"^({Schema._ARRAY_NAME_SUFFIX_CHAR}\d+)$")
+
+    @staticmethod
+    def _compare(a: dict, b: dict, compare: Optional[Callable] = None, _path: Optional[str] = None) -> dict:
+        def key_to_path(key: str) -> Optional[str]:  # noqa
+            nonlocal _path
+            if match := PortalObject._ARRAY_KEY_REGULAR_EXPRESSION.search(key):
+                return f"{_path}{match.group(1)}" if _path else match.group(1)
+            return f"{_path}.{key}" if _path else key
+        def list_to_dictionary(value: list) -> dict:  # noqa
+            result = {}
+            for index, item in enumerate(sorted(value)):  # ignore array order
+                result[f"#{index}"] = item
+            return result
+        diffs = {}
+        for key in a:
+            path = key_to_path(key)
+            if key not in b:
+                diffs[path] = {"value": a[key], "missing_value": True}
+            else:
+                if isinstance(a[key], dict) and isinstance(b[key], dict):
+                    diffs.update(PortalObject._compare(a[key], b[key], compare=compare, _path=path))
+                elif isinstance(a[key], list) and isinstance(b[key], list):
+                    diffs.update(PortalObject._compare(list_to_dictionary(a[key]),
+                                                       list_to_dictionary(b[key]), compare=compare, _path=path))
+                elif a[key] != b[key]:
+                    if not callable(compare) or not compare(path, a[key], b[key]):
+                        diffs[path] = {"value": a[key], "differing_value": b[key]}
+        return diffs

dcicutils/schema_utils.py

@@ -1,4 +1,6 @@
-from typing import Any, Dict, List
+import os
+from typing import Any, Dict, List, Optional, Tuple
+from dcicutils.misc_utils import to_camel_case
 
 
 class JsonSchemaConstants:
@@ -183,3 +185,79 @@ def get_one_of_formats(schema: Dict[str, Any]) -> List[str]:
         for one_of_schema in get_one_of(schema)
         if get_format(one_of_schema)
     ]
+
+
+class Schema:
+
+    def __init__(self, schema: dict, schema_type: Optional[str] = None) -> None:
+        self._data = schema if isinstance(schema, dict) else (schema.data if isinstance(schema, Schema) else {})
+        self._type = (isinstance(schema_type, str) and schema_type) or Schema.type_name(self._data.get("title", ""))
+
+    @property
+    def data(self) -> dict:
+        return self._data
+
+    @property
+    def type(self) -> str:
+        return self._type
+
+    @staticmethod
+    def type_name(value: str) -> Optional[str]:  # File or other name.
+        if isinstance(value, str) and (value := os.path.basename(value.replace(" ", ""))):
+            return to_camel_case(value[0:dot] if (dot := value.rfind(".")) >= 0 else value)
+
+    def property_by_path(self, property_path: str) -> Optional[dict]:
+        """
+        TODO
+        """
+        return Schema.get_property_by_path(self._data, property_path)
+
+    _ARRAY_NAME_SUFFIX_CHAR = "#"
+    _DOTTED_NAME_DELIMITER_CHAR = "."
+
+    @staticmethod
+    def get_property_by_path(schema: dict, property_path: str) -> Optional[dict]:
+        if not isinstance(schema, dict) or not isinstance(property_path, str):
+            return None
+        elif not (schema_properties := schema.get("properties")):
+            return None
+        property_paths = property_path.split(Schema._DOTTED_NAME_DELIMITER_CHAR)
+        for property_index, property_name in enumerate(property_paths):
+            property_name, array_specifiers = Schema._unarrayize_property_name(property_name)
+            if not (property_value := schema_properties.get(property_name)):
+                return None
+            elif (property_type := property_value.get("type")) == "object":
+                property_paths_tail = Schema._DOTTED_NAME_DELIMITER_CHAR.join(property_paths[property_index + 1:])
+                return Schema.get_property_by_path(property_value, property_paths_tail)
+            elif (property_type := property_value.get("type")) == "array":
+                if not array_specifiers:
+                    if property_index == len(property_paths) - 1:
+                        return property_value
+                    return None
+                for array_index in range(len(array_specifiers)):
+                    if property_type != "array":
+                        return None
+                    elif not (array_items := property_value.get("items")):
+                        return None
+                    property_type = (property_value := array_items).get("type")
+                if property_type == "object":
+                    if property_index == len(property_paths) - 1:
+                        return property_value
+                    property_paths_tail = Schema._DOTTED_NAME_DELIMITER_CHAR.join(property_paths[property_index + 1:])
+                    return Schema.get_property_by_path(property_value, property_paths_tail)
+        return property_value
+
+    @staticmethod
+    def _unarrayize_property_name(property_name: str) -> Tuple[str, Optional[List[int]]]:
+        if len(components := (property_name := property_name.strip()).split(Schema._ARRAY_NAME_SUFFIX_CHAR)) < 2:
+            return property_name, None
+        unarrayized_property_name = components[0].strip()
+        array_specifiers = []
+        for component in components[1:]:
+            if component.isdigit():
+                array_specifiers.append(int(component))
+            elif component == "":
+                array_specifiers.append(0)
+            else:
+                return property_name, None
+        return unarrayized_property_name, array_specifiers

dcicutils/structured_data.py

@@ -74,7 +74,7 @@ class StructuredDataSet:
                     row_number += 1
                     if (validation_errors := schema.validate(data)) is not None:
                         for validation_error in validation_errors:
-                            self._note_error({"src": create_dict(type=schema.name, row=row_number),
+                            self._note_error({"src": create_dict(type=schema.type, row=row_number),
                                               "error": validation_error}, "validation")
 
     @property
@@ -168,7 +168,7 @@ class StructuredDataSet:
             if not structured_row_template:  # Delay creation just so we don't reference schema if there are no rows.
                 if not schema and not noschema and not (schema := Schema.load_by_name(type_name, portal=self._portal)):
                     noschema = True
-                elif schema and (schema_name := schema.name):
+                elif schema and (schema_name := schema.type):
                     type_name = schema_name
                 structured_row_template = _StructuredRowTemplate(reader.header, schema)
             structured_row = structured_row_template.create_row()
@@ -222,7 +222,7 @@ class _StructuredRowTemplate:
 
     def set_value(self, data: dict, column_name: str, value: str, file: Optional[str], row_number: int = -1) -> None:
         if (set_value_function := self._set_value_functions.get(column_name)):
-            src = create_dict(type=self._schema.name if self._schema else None,
+            src = create_dict(type=self._schema.type if self._schema else None,
                               column=column_name, file=file, row=row_number)
             set_value_function(data, value, src)
 
@@ -319,8 +319,8 @@ class _StructuredRowTemplate:
 class Schema:
 
     def __init__(self, schema_json: dict, portal: Optional[Portal] = None) -> None:
-        self.data = schema_json
-        self.name = Schema.type_name(schema_json.get("title", "")) if schema_json else ""
+        self._data = schema_json if isinstance(schema_json, dict) else {}
+        self._type = Schema.type_name(schema_json.get("title", ""))
         self._portal = portal  # Needed only to resolve linkTo references.
         self._map_value_functions = {
             "boolean": self._map_function_boolean,
@@ -333,6 +333,14 @@ class Schema:
         self._unresolved_refs = []
         self._typeinfo = self._create_typeinfo(schema_json)
 
+    @property
+    def data(self) -> dict:
+        return self._data
+
+    @property
+    def type(self) -> str:
+        return self._type
+
     @staticmethod
     def load_by_name(name: str, portal: Portal) -> Optional[dict]:
         schema_json = portal.get_schema(Schema.type_name(name)) if portal else None

{dcicutils-8.7.0.1b33.dist-info → dcicutils-8.7.0.1b35.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcicutils
-Version: 8.7.0.1b33
+Version: 8.7.0.1b35
 Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
 Home-page: https://github.com/4dn-dcic/utils
 License: MIT

{dcicutils-8.7.0.1b33.dist-info → dcicutils-8.7.0.1b35.dist-info}/RECORD

@@ -44,7 +44,7 @@ dcicutils/log_utils.py,sha256=7pWMc6vyrorUZQf-V-M3YC6zrPgNhuV_fzm9xqTPph0,10883
 dcicutils/misc_utils.py,sha256=bMRWWxdbhuF3PKdCZEH-H4U1ecgT3Nag3EL92D9XGoY,100973
 dcicutils/obfuscation_utils.py,sha256=fo2jOmDRC6xWpYX49u80bVNisqRRoPskFNX3ymFAmjw,5963
 dcicutils/opensearch_utils.py,sha256=V2exmFYW8Xl2_pGFixF4I2Cc549Opwe4PhFi5twC0M8,1017
-dcicutils/portal_object_utils.py,sha256=io5XCdfHarFb1H2S3IyDNb25n9LPw2IZjcsiBowmeN4,5251
+dcicutils/portal_object_utils.py,sha256=udjT_-IYWInTc-oXoh0Ie_xVYp28YEg3BRIlS88eUwQ,8352
 dcicutils/portal_utils.py,sha256=jKYgZUYVdkg6VOs1hsiX4bSULLguOIBJFFRpvvZEklU,26704
 dcicutils/project_utils.py,sha256=qPdCaFmWUVBJw4rw342iUytwdQC0P-XKpK4mhyIulMM,31250
 dcicutils/qa_checkers.py,sha256=cdXjeL0jCDFDLT8VR8Px78aS10hwNISOO5G_Zv2TZ6M,20534
@@ -52,22 +52,22 @@ dcicutils/qa_utils.py,sha256=TT0SiJWiuxYvbsIyhK9VO4uV_suxhB6CpuC4qPacCzQ,160208
 dcicutils/redis_tools.py,sha256=qkcSNMtvqkpvts-Cm9gWhneK523Q_oHwhNUud1be1qk,7055
 dcicutils/redis_utils.py,sha256=VJ-7g8pOZqR1ZCtdcjKz3-6as2DMUcs1b1zG6wSprH4,6462
 dcicutils/s3_utils.py,sha256=LauLFQGvZLfpBJ81tYMikjLd3SJRz2R_FrL1n4xSlyI,28868
-dcicutils/schema_utils.py,sha256=3Gd9QboOjQ3FHFawerylvYYU8Lor1Ma2pFv4JmezCdg,5501
+dcicutils/schema_utils.py,sha256=h3VlIiBxE8EmxnfcXHF5KZhNeIZIA71LvqeIM-04gKY,9169
 dcicutils/scripts/publish_to_pypi.py,sha256=LFzNHIQK2EXFr88YcfctyA_WKEBFc1ElnSjWrCXedPM,13889
 dcicutils/scripts/run_license_checker.py,sha256=z2keYnRDZsHQbTeo1XORAXSXNJK5axVzL5LjiNqZ7jE,4184
 dcicutils/secrets_utils.py,sha256=8dppXAsiHhJzI6NmOcvJV5ldvKkQZzh3Fl-cb8Wm7MI,19745
 dcicutils/sheet_utils.py,sha256=VlmzteONW5VF_Q4vo0yA5vesz1ViUah1MZ_yA1rwZ0M,33629
 dcicutils/snapshot_utils.py,sha256=ymP7PXH6-yEiXAt75w0ldQFciGNqWBClNxC5gfX2FnY,22961
 dcicutils/ssl_certificate_utils.py,sha256=F0ifz_wnRRN9dfrfsz7aCp4UDLgHEY8LaK7PjnNvrAQ,9707
-dcicutils/structured_data.py,sha256=0YYNwGJeev7xAV9sQj0ioP4m_h1gcx--Sj_dzLpO9us,34068
+dcicutils/structured_data.py,sha256=VxkyBBMFoRIEoPZxQuYU1iVO3piQwSsG-W6_pKy3qXE,34225
 dcicutils/task_utils.py,sha256=MF8ujmTD6-O2AC2gRGPHyGdUrVKgtr8epT5XU8WtNjk,8082
 dcicutils/tmpfile_utils.py,sha256=n95XF8dZVbQRSXBZTGToXXfSs3JUVRyN6c3ZZ0nhAWI,1403
 dcicutils/trace_utils.py,sha256=g8kwV4ebEy5kXW6oOrEAUsurBcCROvwtZqz9fczsGRE,1769
 dcicutils/validation_utils.py,sha256=cMZIU2cY98FYtzK52z5WUYck7urH6JcqOuz9jkXpqzg,14797
 dcicutils/variant_utils.py,sha256=2H9azNx3xAj-MySg-uZ2SFqbWs4kZvf61JnK6b-h4Qw,4343
 dcicutils/zip_utils.py,sha256=rnjNv_k6L9jT2SjDSgVXp4BEJYLtz9XN6Cl2Fy-tqnM,2027
-dcicutils-8.7.0.1b33.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
-dcicutils-8.7.0.1b33.dist-info/METADATA,sha256=oBnl8kr-p0xukoz_JLaF6utwsnX0CK79u9ykMwe8rCU,3315
-dcicutils-8.7.0.1b33.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
-dcicutils-8.7.0.1b33.dist-info/entry_points.txt,sha256=8wbw5csMIgBXhkwfgsgJeuFcoUc0WsucUxmOyml2aoA,209
-dcicutils-8.7.0.1b33.dist-info/RECORD,,
+dcicutils-8.7.0.1b35.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
+dcicutils-8.7.0.1b35.dist-info/METADATA,sha256=v9eG3K-Qg11ZBSXujKZI_CGs6AYp8YCAp1OAe6Eywgo,3315
+dcicutils-8.7.0.1b35.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+dcicutils-8.7.0.1b35.dist-info/entry_points.txt,sha256=8wbw5csMIgBXhkwfgsgJeuFcoUc0WsucUxmOyml2aoA,209
+dcicutils-8.7.0.1b35.dist-info/RECORD,,