dcicutils 8.8.0.1b22__py3-none-any.whl → 8.8.0.1b24__py3-none-any.whl

dcicutils/portal_object_utils.py

@@ -1,7 +1,7 @@
 from copy import deepcopy
 from functools import lru_cache
 import re
-from typing import Any, List, Optional, Tuple, Type, Union
+from typing import Any, Callable, List, Optional, Tuple, Type, Union
 from dcicutils.data_readers import RowReader
 from dcicutils.misc_utils import create_readonly_object
 from dcicutils.portal_utils import Portal
@@ -19,7 +19,7 @@ class PortalObject:
         self._data = data if isinstance(data, dict) else {}
         self._portal = portal if isinstance(portal, Portal) else None
         self._schema = schema if isinstance(schema, dict) else (schema.data if isinstance(schema, Schema) else None)
-        self._type = type if isinstance(type, str) and type else None
+        self._type = type if isinstance(type, str) else ""
 
     @property
     def data(self) -> dict:
@@ -31,8 +31,8 @@ class PortalObject:
 
     @property
     @lru_cache(maxsize=1)
-    def type(self) -> Optional[str]:
-        return self._type or Portal.get_schema_type(self._data) or (Schema(self._schema).type if self._schema else None)
+    def type(self) -> str:
+        return self._type or Portal.get_schema_type(self._data) or (Schema(self._schema).type if self._schema else "")
 
     @property
     @lru_cache(maxsize=1)
@@ -75,86 +75,42 @@ class PortalObject:
             identifying_properties.append("aliases")
         return identifying_properties or None
 
-    @property
-    @lru_cache(maxsize=1)
-    def identifying_paths(self) -> Optional[List[str]]:
-        """
-        Returns a list of the possible Portal URL paths identifying this Portal object.
-        """
-        identifying_paths = []
-        if not (identifying_properties := self.identifying_properties):
-            if self.uuid:
-                if self.type:
-                    identifying_paths.append(f"/{self.type}/{self.uuid}")
-                identifying_paths.append(f"/{self.uuid}")
-            return identifying_paths
-        for identifying_property in identifying_properties:
-            if (identifying_value := self._data.get(identifying_property)):
-                if identifying_property == "uuid":
-                    identifying_paths.append(f"/{self.type}/{identifying_value}")
-                    identifying_paths.append(f"/{identifying_value}")
-                # For now at least we include the path both with and without the schema type component,
-                # as for some identifying values, it works (only) with, and some, it works (only) without.
-                # For example: If we have FileSet with "accession", an identifying property, with value
-                # SMAFSFXF1RO4 then /SMAFSFXF1RO4 works but /FileSet/SMAFSFXF1RO4 does not; and
-                # conversely using "submitted_id", also an identifying property, with value
-                # UW_FILE-SET_COLO-829BL_HI-C_1 then /UW_FILE-SET_COLO-829BL_HI-C_1 does
-                # not work but /FileSet/UW_FILE-SET_COLO-829BL_HI-C_1 does work.
-                elif isinstance(identifying_value, list):
-                    for identifying_value_item in identifying_value:
-                        if self.type:
-                            identifying_paths.append(f"/{self.type}/{identifying_value_item}")
-                        identifying_paths.append(f"/{identifying_value_item}")
-                else:
-                    if (schema := self.schema):
-                        if pattern := schema.get("properties", {}).get(identifying_property, {}).get("pattern"):
-                            if not re.match(pattern, identifying_value):
-                                # If this identifying value is for a (identifying) property which has a
-                                # pattern, and the value does NOT match the pattern, then do NOT include
-                                # this value as an identifying path, since it cannot possibly be found.
-                                continue
-                    if self.type:
-                        identifying_paths.append(f"/{self.type}/{identifying_value}")
-                    identifying_paths.append(f"/{identifying_value}")
-        return identifying_paths or None
-
-    @property
-    @lru_cache(maxsize=1)
-    def identifying_path(self) -> Optional[str]:
-        if identifying_paths := self.identifying_paths:
-            return identifying_paths[0]
-
-    def lookup(self, include_identifying_path: bool = False,
-               raw: bool = False) -> Optional[Union[Tuple[PortalObject, str], PortalObject]]:
-        return self._lookup(raw=raw) if include_identifying_path else self._lookup(raw=raw)[0]
-
-    def lookup_identifying_path(self) -> Optional[str]:
-        return self._lookup()[1]
-
-    def _lookup(self, raw: bool = False) -> Tuple[Optional[PortalObject], Optional[str]]:
+    @lru_cache(maxsize=8192)
+    def lookup(self, raw: bool = False,
+               ref_lookup_strategy: Optional[Callable] = None) -> Tuple[Optional[PortalObject], Optional[str], int]:
+        nlookups = 0
+        first_identifying_path = None
         try:
-            if identifying_paths := self.identifying_paths:
+            if identifying_paths := self._get_identifying_paths(ref_lookup_strategy=ref_lookup_strategy):
                 for identifying_path in identifying_paths:
+                    if not first_identifying_path:
+                        first_identifying_path = identifying_path
+                    nlookups += 1
                     if (value := self._portal.get(identifying_path, raw=raw)) and (value.status_code == 200):
-                        return PortalObject(value.json(),
-                                            portal=self._portal, type=self.type if raw else None), identifying_path
+                        return (
+                            PortalObject(value.json(), portal=self._portal, type=self.type if raw else None),
+                            identifying_path,
+                            nlookups
+                        )
         except Exception:
             pass
-        return None, self.identifying_path
+        return None, first_identifying_path, nlookups
 
     def compare(self, value: Union[dict, PortalObject],
-                consider_refs: bool = False, resolved_refs: List[dict] = None) -> dict:
+                consider_refs: bool = False, resolved_refs: List[dict] = None) -> Tuple[dict, int]:
         if consider_refs and isinstance(resolved_refs, list):
-            this_data = self.normalized_refs(refs=resolved_refs).data
+            normlized_portal_object, nlookups = self._normalized_refs(refs=resolved_refs)
+            this_data = normlized_portal_object.data
         else:
             this_data = self.data
+            nlookups = 0
         if isinstance(value, PortalObject):
             comparing_data = value.data
         elif isinstance(value, dict):
             comparing_data = value
         else:
-            return {}
-        return PortalObject._compare(this_data, comparing_data)
+            return {}, nlookups
+        return PortalObject._compare(this_data, comparing_data), nlookups
 
     @staticmethod
     def _compare(a: Any, b: Any, _path: Optional[str] = None) -> dict:
@@ -201,42 +157,106 @@ class PortalObject:
                 diffs[_path] = diff_updating(a, b)
         return diffs
 
-    def normalize_refs(self, refs: List[dict]) -> None:
+    @lru_cache(maxsize=1)
+    def _get_identifying_paths(self, ref_lookup_strategy: Optional[Callable] = None) -> Optional[List[str]]:
         """
-        Turns any (linkTo) references which are paths (e.g. /SubmissionCenter/uwsc_gcc) within
-        this Portal object into the uuid style reference (e.g. d1b67068-300f-483f-bfe8-63d23c93801f),
-        based on the given "refs" list which is assumed to be a list of dictionaries, where each
-        contains a "path" and a "uuid" property; this list is typically (for our first usage of
-        this function) the value of structured_data.StructuredDataSet.resolved_refs_with_uuid.
-        Changes are made to this Portal object in place; use normalized_refs function to make a copy.
-        If there are no "refs" (None or empty) or if the speicified reference is not found in this
-        list then the references will be looked up via Portal calls (via Portal.get_metadata).
+        Returns a list of the possible Portal URL paths identifying this Portal object.
         """
-        PortalObject._normalize_refs(self.data, refs=refs, schema=self.schema, portal=self.portal)
+        identifying_paths = []
+        if not (identifying_properties := self.identifying_properties):
+            if self.uuid:
+                if self.type:
+                    identifying_paths.append(f"/{self.type}/{self.uuid}")
+                identifying_paths.append(f"/{self.uuid}")
+            return identifying_paths
+        for identifying_property in identifying_properties:
+            if identifying_value := self._data.get(identifying_property):
+                if identifying_property == "uuid":
+                    if self.type:
+                        identifying_paths.append(f"/{self.type}/{identifying_value}")
+                    identifying_paths.append(f"/{identifying_value}")
+                # For now at least we include the path both with and without the schema type component,
+                # as for some identifying values, it works (only) with, and some, it works (only) without.
+                # For example: If we have FileSet with "accession", an identifying property, with value
+                # SMAFSFXF1RO4 then /SMAFSFXF1RO4 works but /FileSet/SMAFSFXF1RO4 does not; and
+                # conversely using "submitted_id", also an identifying property, with value
+                # UW_FILE-SET_COLO-829BL_HI-C_1 then /UW_FILE-SET_COLO-829BL_HI-C_1 does
+                # not work but /FileSet/UW_FILE-SET_COLO-829BL_HI-C_1 does work.
+                elif isinstance(identifying_value, list):
+                    for identifying_value_item in identifying_value:
+                        if self.type:
+                            identifying_paths.append(f"/{self.type}/{identifying_value_item}")
+                        identifying_paths.append(f"/{identifying_value_item}")
+                else:
+                    # TODO: Import from somewhere ...
+                    lookup_options = 0
+                    if schema := self.schema:
+                        # TODO: Hook into the ref_lookup_strategy thing in structured_data to make
+                        # sure we check accession format (since it does not have a pattern).
+                        if callable(ref_lookup_strategy):
+                            lookup_options, ref_validator = ref_lookup_strategy(
+                                self.type, schema, identifying_value)
+                            if callable(ref_validator):
+                                if ref_validator(schema, identifying_property, identifying_value) is False:
+                                    continue
+                        if pattern := schema.get("properties", {}).get(identifying_property, {}).get("pattern"):
+                            if not re.match(pattern, identifying_value):
+                                # If this identifying value is for a (identifying) property which has a
+                                # pattern, and the value does NOT match the pattern, then do NOT include
+                                # this value as an identifying path, since it cannot possibly be found.
+                                continue
+                    if not lookup_options:
+                        lookup_options = Portal.LOOKUP_DEFAULT
+                    if Portal.is_lookup_root_first(lookup_options):
+                        identifying_paths.append(f"/{identifying_value}")
+                    if Portal.is_lookup_specified_type(lookup_options) and self.type:
+                        identifying_paths.append(f"/{self.type}/{identifying_value}")
+                    if Portal.is_lookup_root(lookup_options) and not Portal.is_lookup_root_first(lookup_options):
+                        identifying_paths.append(f"/{identifying_value}")
+                    if Portal.is_lookup_subtypes(lookup_options):
+                        for subtype_name in self._portal.get_schema_subtype_names(self.type):
+                            identifying_paths.append(f"/{subtype_name}/{identifying_value}")
+        return identifying_paths or None
 
-    def normalized_refs(self, refs: List[dict]) -> PortalObject:
+    def _normalized_refs(self, refs: List[dict]) -> Tuple[PortalObject, int]:
         """
-        Same as normalize_ref but does not make this change to this Portal object in place,
+        Same as _normalize_ref but does NOT make this change to this Portal object IN PLACE,
         rather it returns a new instance of this Portal object wrapped in a new PortalObject.
         """
         portal_object = self.copy()
-        portal_object.normalize_refs(refs)
-        return portal_object
+        nlookups = portal_object._normalize_refs(refs)
+        return portal_object, nlookups
+
+    def _normalize_refs(self, refs: List[dict]) -> int:
+        """
+        Turns any (linkTo) references which are paths (e.g. /SubmissionCenter/uwsc_gcc) within this
+        object IN PLACE into the uuid style reference (e.g. d1b67068-300f-483f-bfe8-63d23c93801f),
+        based on the given "refs" list which is assumed to be a list of dictionaries, where each
+        contains a "path" and a "uuid" property; this list is typically (for our first usage of
+        this function) the value of structured_data.StructuredDataSet.resolved_refs_with_uuid.
+        Changes are made to this Portal object IN PLACE; use _normalized_refs function to make a copy.
+        If there are no "refs" (None or empty) or if the speicified reference is not found in this
+        list then the references will be looked up via Portal calls (via Portal.get_metadata).
+        """
+        _, nlookups = PortalObject._normalize_data_refs(self.data, refs=refs, schema=self.schema, portal=self.portal)
+        return nlookups
 
     @staticmethod
-    def _normalize_refs(value: Any, refs: List[dict], schema: dict, portal: Portal, _path: Optional[str] = None) -> Any:
+    def _normalize_data_refs(value: Any, refs: List[dict], schema: dict,
+                             portal: Portal, _path: Optional[str] = None) -> Tuple[Any, int]:
+        nlookups = 0
         if not value or not isinstance(schema, dict):
-            return value
+            return value, nlookups
         if isinstance(value, dict):
             for key in value:
                 path = f"{_path}.{key}" if _path else key
-                value[key] = PortalObject._normalize_refs(value[key], refs=refs,
-                                                          schema=schema, portal=portal, _path=path)
+                value[key], nlookups = PortalObject._normalize_data_refs(value[key], refs=refs,
+                                                                         schema=schema, portal=portal, _path=path)
         elif isinstance(value, list):
             for index in range(len(value)):
                 path = f"{_path or ''}#{index}"
-                value[index] = PortalObject._normalize_refs(value[index], refs=refs,
-                                                            schema=schema, portal=portal, _path=path)
+                value[index], nlookups = PortalObject._normalize_data_refs(value[index], refs=refs,
+                                                                           schema=schema, portal=portal, _path=path)
         elif value_type := Schema.get_property_by_path(schema, _path):
             if link_to := value_type.get("linkTo"):
                 ref_path = f"/{link_to}/{value}"
@@ -247,7 +267,7 @@ class PortalObject:
                 else:
                     ref_uuid = None
                 if ref_uuid:
-                    return ref_uuid
+                    return ref_uuid, nlookups
                 # Here our (linkTo) reference appears not to be in the given refs; if these refs came
                 # from structured_data.StructuredDataSet.resolved_refs_with_uuid (in the context of
                 # smaht-submitr, which is the typical/first use case for this function) then this could
@@ -255,6 +275,7 @@ class PortalObject:
                 # the data/spreadsheet being submitted. In any case, we don't have the associated uuid
                 # so let us look it up here.
                 if isinstance(portal, Portal):
+                    nlookups += 1
                     if (ref_object := portal.get_metadata(ref_path)) and (ref_uuid := ref_object.get("uuid")):
-                        return ref_uuid
-        return value
+                        return ref_uuid, nlookups
+        return value, nlookups

dcicutils/portal_utils.py

@@ -46,6 +46,22 @@ class Portal:
     KEYS_FILE_DIRECTORY = "~"
     MIME_TYPE_JSON = "application/json"
 
+    # Object lookup strategies; on a per-reference (type/value) basis, used currently ONLY by
+    # structured_data.py; controlled by an optional ref_lookup_strategy callable; default is
+    # lookup at root path but after the specified type path lookup, and then lookup all subtypes;
+    # can choose to lookup root path first, or not lookup root path at all, or not lookup
+    # subtypes at all; the ref_lookup_strategy callable if specified should take a type_name
+    # and value (string) arguements and return an integer of any of the below ORed together.
+    # The main purpose of this is optimization; to minimize portal lookups; since for example,
+    # currently at least, /{type}/{accession} does not work but /{accession} does; so we
+    # currently (smaht-portal/.../ingestion_processors) use LOOKUP_ROOT_FIRST for this.
+    # And current usage NEVER has LOOKUP_SUBTYPES turned OFF; but support just in case.
+    LOOKUP_SPECIFIED_TYPE = 0x0001
+    LOOKUP_ROOT = 0x0002
+    LOOKUP_ROOT_FIRST = 0x0004 | LOOKUP_ROOT
+    LOOKUP_SUBTYPES = 0x0008
+    LOOKUP_DEFAULT = LOOKUP_SPECIFIED_TYPE | LOOKUP_ROOT | LOOKUP_SUBTYPES
+
     def __init__(self,
                  arg: Optional[Union[Portal, TestApp, VirtualApp, PyramidRouter, dict, tuple, str]] = None,
                  env: Optional[str] = None, server: Optional[str] = None,
@@ -188,9 +204,27 @@ class Portal:
     def vapp(self) -> Optional[TestApp]:
         return self._vapp
 
+    @staticmethod
+    def is_lookup_specified_type(lookup_options: int) -> bool:
+        return (lookup_options &
+                Portal.LOOKUP_SPECIFIED_TYPE) == Portal.LOOKUP_SPECIFIED_TYPE
+
+    @staticmethod
+    def is_lookup_root(lookup_options: int) -> bool:
+        return (lookup_options & Portal.LOOKUP_ROOT) == Portal.LOOKUP_ROOT
+
+    @staticmethod
+    def is_lookup_root_first(lookup_options: int) -> bool:
+        return (lookup_options & Portal.LOOKUP_ROOT_FIRST) == Portal.LOOKUP_ROOT_FIRST
+
+    @staticmethod
+    def is_lookup_subtypes(lookup_options: int) -> bool:
+        return (lookup_options & Portal.LOOKUP_SUBTYPES) == Portal.LOOKUP_SUBTYPES
+
     def get(self, url: str, follow: bool = True,
             raw: bool = False, database: bool = False, raise_for_status: bool = False, **kwargs) -> OptionalResponse:
         url = self.url(url, raw, database)
+        # print(f'xyzzy.portal.get({url})')
         if not self.vapp:
             response = requests.get(url, allow_redirects=follow, **self._kwargs(**kwargs))
         else:
@@ -205,6 +239,7 @@ class Portal:
     def patch(self, url: str, data: Optional[dict] = None, json: Optional[dict] = None,
               raise_for_status: bool = False, **kwargs) -> OptionalResponse:
         url = self.url(url)
+        # print(f'xyzzy.portal.patch({url})')
         if not self.vapp:
             response = requests.patch(url, data=data, json=json, **self._kwargs(**kwargs))
         else:
@@ -217,6 +252,7 @@ class Portal:
     def post(self, url: str, data: Optional[dict] = None, json: Optional[dict] = None, files: Optional[dict] = None,
              raise_for_status: bool = False, **kwargs) -> OptionalResponse:
         url = self.url(url)
+        # print(f'xyzzy.portal.post({url})')
         if files and not ("headers" in kwargs):
             # Setting headers to None when using files implies content-type multipart/form-data.
             kwargs["headers"] = None
@@ -233,6 +269,7 @@ class Portal:
         return response
 
     def get_metadata(self, object_id: str, raw: bool = False, database: bool = False) -> Optional[dict]:
+        # print(f'xyzzy.portal.get_metadata({object_id})')
         if isinstance(raw, bool) and raw:
             add_on = "frame=raw" + ("&datastore=database" if isinstance(database, bool) and database else "")
         elif database:
@@ -242,11 +279,13 @@ class Portal:
         return get_metadata(obj_id=object_id, vapp=self.vapp, key=self.key, add_on=add_on)
 
     def patch_metadata(self, object_id: str, data: dict) -> Optional[dict]:
+        # print(f'xyzzy.portal.patch_metadata({object_id})')
         if self.key:
             return patch_metadata(obj_id=object_id, patch_item=data, key=self.key)
         return self.patch(f"/{object_id}", data).json()
 
     def post_metadata(self, object_type: str, data: dict) -> Optional[dict]:
+        # print(f'xyzzy.portal.post_metadata({object_id})')
         if self.key:
             return post_metadata(schema_name=object_type, post_item=data, key=self.key)
         return self.post(f"/{object_type}", data).json()
@@ -358,6 +397,12 @@ class Portal:
             super_type_map_flattened[super_type_name] = list_breadth_first(super_type_map, super_type_name)
         return super_type_map_flattened
 
+    @lru_cache(maxsize=64)
+    def get_schema_subtype_names(self, type_name: str) -> List[str]:
+        if not (schemas_super_type_map := self.get_schemas_super_type_map()):
+            return []
+        return schemas_super_type_map.get(type_name, [])
+
     def url(self, url: str, raw: bool = False, database: bool = False) -> str:
         if not isinstance(url, str) or not url:
             return "/"

dcicutils/structured_data.py

@@ -48,22 +48,6 @@ StructuredDataSet = Type["StructuredDataSet"]
 
 class StructuredDataSet:
 
-    # Reference (linkTo) lookup strategies; on a per-reference (type/value) basis;
-    # controlled by optional ref_lookup_strategy callable; default is lookup at root path
-    # but after the named reference (linkTo) type path lookup, and then lookup all subtypes;
-    # can choose to lookup root path first, or not lookup root path at all, or not lookup
-    # subtypes at all; the ref_lookup_strategy callable if specified should take a type_name
-    # and value (string) arguements and return an integer of any of the below ORed together.
-    # The main purpose of this is optimization; to minimize portal lookups; since for example,
-    # currently at least, /{type}/{accession} does not work but /{accession} does; so we
-    # currently (smaht-portal/.../ingestion_processors) use REF_LOOKUP_ROOT_FIRST for this.
-    # And current usage NEVER has REF_LOOKUP_SUBTYPES turned OFF; but support just in case.
-    REF_LOOKUP_SPECIFIED_TYPE = 0x0001
-    REF_LOOKUP_ROOT = 0x0002
-    REF_LOOKUP_ROOT_FIRST = 0x0004 | REF_LOOKUP_ROOT
-    REF_LOOKUP_SUBTYPES = 0x0008
-    REF_LOOKUP_DEFAULT = REF_LOOKUP_SPECIFIED_TYPE | REF_LOOKUP_ROOT | REF_LOOKUP_SUBTYPES
-
     def __init__(self, file: Optional[str] = None, portal: Optional[Union[VirtualApp, TestApp, Portal]] = None,
                  schemas: Optional[List[dict]] = None, autoadd: Optional[dict] = None,
                  order: Optional[List[str]] = None, prune: bool = True,
@@ -76,6 +60,7 @@ class StructuredDataSet:
         self._portal = Portal(portal, data=self._data, schemas=schemas,
                               ref_lookup_strategy=ref_lookup_strategy,
                               ref_lookup_nocache=ref_lookup_nocache) if portal else None
+        self._ref_lookup_strategy = ref_lookup_strategy
         self._order = order
         self._prune = prune
         self._warnings = {}
@@ -199,25 +184,49 @@ class StructuredDataSet:
                 upload_file["path"] = file_path
         return upload_files
 
-    def compare(self) -> dict:
+    def compare(self, progress: Optional[Callable] = None) -> dict:
+        def get_counts() -> int:
+            ntypes = 0
+            nobjects = 0
+            if self.data:
+                ntypes = len(self.data)
+                for type_name in self.data:
+                    nobjects += len(self.data[type_name])
+            return ntypes, nobjects
         diffs = {}
-        if self.data or self.portal:
+        if callable(progress):
+            ntypes, nobjects = get_counts()
+            progress({"start": True, "types": ntypes, "objects": nobjects})
+        if self.data or self.portal:  # TODO: what is this OR biz?
             refs = self.resolved_refs_with_uuids
-            for object_type in self.data:
-                if not diffs.get(object_type):
-                    diffs[object_type] = []
-                for portal_object in self.data[object_type]:
-                    portal_object = PortalObject(portal_object, portal=self.portal, type=object_type)
-                    existing_object, identifying_path = portal_object.lookup(include_identifying_path=True, raw=True)
+            # TODO: Need feedback/progress tracking mechanism here.
+            # TODO: Check validity of reference; actually check that earlier on even maybe.
+            for type_name in self.data:
+                if not diffs.get(type_name):
+                    diffs[type_name] = []
+                for portal_object in self.data[type_name]:
+                    portal_object = PortalObject(portal_object, portal=self.portal, type=type_name)
+                    existing_object, identifying_path, nlookups = (
+                        portal_object.lookup(raw=True, ref_lookup_strategy=self._ref_lookup_strategy))
                     if existing_object:
-                        object_diffs = portal_object.compare(existing_object, consider_refs=True, resolved_refs=refs)
-                        diffs[object_type].append(create_readonly_object(path=identifying_path,
-                                                                         uuid=existing_object.uuid,
-                                                                         diffs=object_diffs or None))
+                        object_diffs, nlookups_compare = portal_object.compare(
+                            existing_object, consider_refs=True, resolved_refs=refs)
+                        diffs[type_name].append(create_readonly_object(path=identifying_path,
+                                                                       uuid=existing_object.uuid,
+                                                                       diffs=object_diffs or None))
+                        if callable(progress):
+                            progress({"update": True, "lookups": nlookups + nlookups_compare})
                     elif identifying_path:
                         # If there is no existing object we still create a record for this object
                         # but with no uuid which will be the indication that it does not exist.
-                        diffs[object_type].append(create_readonly_object(path=identifying_path, uuid=None, diffs=None))
+                        diffs[type_name].append(create_readonly_object(path=identifying_path, uuid=None, diffs=None))
+                        if callable(progress):
+                            progress({"create": True, "lookups": nlookups})
+                    else:
+                        if callable(progress):
+                            progress({"lookups": nlookups})
+        if callable(progress):
+            progress({"finish": True})
         return diffs
 
     def _load_file(self, file: str) -> None:
@@ -251,7 +260,7 @@ class StructuredDataSet:
         self._load_reader(CsvReader(file), type_name=Schema.type_name(file))
 
     def _load_excel_file(self, file: str) -> None:
-        def calculate_total_rows_to_process() -> Tuple[int, int]:
+        def get_counts() -> Tuple[int, int]:
             nonlocal file
             excel = Excel(file)
             nrows = 0
@@ -260,18 +269,27 @@ class StructuredDataSet:
                     nrows += 1
             return nrows, len(excel.sheet_names)
         if self._progress:
-            self._progress_update(calculate_total_rows_to_process)
+            nrows, nsheets = get_counts()
+            self._progress({"start": True, "sheets": nsheets, "rows": nrows})
+        """
+        if self._progress:
+            self._progress_update(get_counts)
+        """
         excel = Excel(file)  # Order the sheet names by any specified ordering (e.g. ala snovault.loadxl).
         order = {Schema.type_name(key): index for index, key in enumerate(self._order)} if self._order else {}
         for sheet_name in sorted(excel.sheet_names, key=lambda key: order.get(Schema.type_name(key), sys.maxsize)):
             self._load_reader(excel.sheet_reader(sheet_name), type_name=Schema.type_name(sheet_name))
+        if self._progress:
+            self._progress({"finish": True})
+        # TODO: Do we really need progress reporting for the below?
         # Check for unresolved reference errors which really are not because of ordering.
         # Yes such internal references will be handled correctly on actual database update via snovault.loadxl.
         if ref_errors := self.ref_errors:
             ref_errors_actual = []
             for ref_error in ref_errors:
                 if not (resolved := self.portal.ref_exists(ref := ref_error["error"])):
-                    # if not (resolved := self.portal.ref_exists_internally(ref := ref_error["error"])):  # TODO
+                    # TODO: Probably do this instead; and if so then no progress needed (per question above).
+                    # if not (resolved := self.portal.ref_exists_internally(ref := ref_error["error"])):
                     ref_errors_actual.append(ref_error)
                 else:
                     self._resolved_refs.add((ref, resolved.get("uuid")))
@@ -304,13 +322,15 @@ class StructuredDataSet:
                     self._add_properties(structured_row, self._autoadd_properties, schema)
             self._add(type_name, structured_row)
             if self._progress:
-                self._progress_update(-1,
-                                      self.ref_total_count,
-                                      self.ref_total_found_count,
-                                      self.ref_total_notfound_count,
-                                      self.ref_lookup_count,
-                                      self.ref_lookup_cache_hit_count,
-                                      self.ref_invalid_identifying_property_count)
+                self._progress({
+                    "parse": True,
+                    "refs": self.ref_total_count,
+                    "refs_found": self.ref_total_found_count,
+                    "refs_not_found": self.ref_total_notfound_count,
+                    "refs_lookup": self.ref_lookup_count,
+                    "refs_cache_hit": self.ref_lookup_cache_hit_count,
+                    "refs_invalid": self.ref_invalid_identifying_property_count
+                })
         self._note_warning(reader.warnings, "reader")
         if schema:
             self._note_error(schema._unresolved_refs, "ref")
@@ -331,16 +351,16 @@ class StructuredDataSet:
 
     def _is_ref_lookup_specified_type(ref_lookup_flags: int) -> bool:
         return (ref_lookup_flags &
-                StructuredDataSet.REF_LOOKUP_SPECIFIED_TYPE) == StructuredDataSet.REF_LOOKUP_SPECIFIED_TYPE
+                Portal.LOOKUP_SPECIFIED_TYPE) == Portal.LOOKUP_SPECIFIED_TYPE
 
     def _is_ref_lookup_root(ref_lookup_flags: int) -> bool:
-        return (ref_lookup_flags & StructuredDataSet.REF_LOOKUP_ROOT) == StructuredDataSet.REF_LOOKUP_ROOT
+        return (ref_lookup_flags & Portal.LOOKUP_ROOT) == Portal.LOOKUP_ROOT
 
     def _is_ref_lookup_root_first(ref_lookup_flags: int) -> bool:
-        return (ref_lookup_flags & StructuredDataSet.REF_LOOKUP_ROOT_FIRST) == StructuredDataSet.REF_LOOKUP_ROOT_FIRST
+        return (ref_lookup_flags & Portal.LOOKUP_ROOT_FIRST) == Portal.LOOKUP_ROOT_FIRST
 
     def _is_ref_lookup_subtypes(ref_lookup_flags: int) -> bool:
-        return (ref_lookup_flags & StructuredDataSet.REF_LOOKUP_SUBTYPES) == StructuredDataSet.REF_LOOKUP_SUBTYPES
+        return (ref_lookup_flags & Portal.LOOKUP_SUBTYPES) == Portal.LOOKUP_SUBTYPES
 
     @property
     def ref_total_count(self) -> int:
@@ -786,7 +806,7 @@ class Portal(PortalBase):
         if callable(ref_lookup_strategy):
             self._ref_lookup_strategy = ref_lookup_strategy
         else:
-            self._ref_lookup_strategy = lambda type_name, schema, value: (StructuredDataSet.REF_LOOKUP_DEFAULT, None)
+            self._ref_lookup_strategy = lambda type_name, schema, value: (Portal.LOOKUP_DEFAULT, None)
         if ref_lookup_nocache is True:
             self.ref_lookup = self.ref_lookup_uncached
             self._ref_cache = None
@@ -844,7 +864,7 @@ class Portal(PortalBase):
         return schemas
 
     @lru_cache(maxsize=64)
-    def _get_schema_subtypes_names(self, type_name: str) -> List[str]:
+    def _get_schema_subtype_names(self, type_name: str) -> List[str]:
         if not (schemas_super_type_map := self.get_schemas_super_type_map()):
             return []
         return schemas_super_type_map.get(type_name, [])
@@ -907,7 +927,7 @@ class Portal(PortalBase):
             lookup_paths.append(f"/{type_name}/{value}")
         if is_ref_lookup_root and not is_ref_lookup_root_first:
             lookup_paths.append(f"/{value}")
-        subtype_names = self._get_schema_subtypes_names(type_name) if is_ref_lookup_subtypes else []
+        subtype_names = self._get_schema_subtype_names(type_name) if is_ref_lookup_subtypes else []
         for subtype_name in subtype_names:
             lookup_paths.append(f"/{subtype_name}/{value}")
         if not lookup_paths:
@@ -946,7 +966,7 @@ class Portal(PortalBase):
         ref_lookup_strategy, ref_validator = (
             self._ref_lookup_strategy(type_name, self.get_schema(type_name), value))
         is_ref_lookup_subtypes = StructuredDataSet._is_ref_lookup_subtypes(ref_lookup_strategy)
-        subtype_names = self._get_schema_subtypes_names(type_name) if is_ref_lookup_subtypes else []
+        subtype_names = self._get_schema_subtype_names(type_name) if is_ref_lookup_subtypes else []
         for type_name in [type_name] + subtype_names:
             is_resolved, resolved_item = self._ref_exists_single_internally(type_name, value)
             if is_resolved:
@@ -1008,7 +1028,7 @@ class Portal(PortalBase):
                         if not is_uuid(property_value):
                             return False
             return True
-        for schema_name in [type_name] + self._get_schema_subtypes_names(type_name):
+        for schema_name in [type_name] + self._get_schema_subtype_names(type_name):
             if schema := self.get_schema(schema_name):
                 if identifying_properties := schema.get("identifyingProperties"):
                     for identifying_property in identifying_properties:
@@ -1033,7 +1053,7 @@ class Portal(PortalBase):
 
     def _cache_ref(self, type_name: str, value: str, resolved: List[str]) -> None:
         if self._ref_cache is not None:
-            subtype_names = self._get_schema_subtypes_names(type_name)
+            subtype_names = self._get_schema_subtype_names(type_name)
             for type_name in [type_name] + subtype_names:
                 self._ref_cache[f"/{type_name}/{value}"] = resolved
 

{dcicutils-8.8.0.1b22.dist-info → dcicutils-8.8.0.1b24.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcicutils
-Version: 8.8.0.1b22
+Version: 8.8.0.1b24
 Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
 Home-page: https://github.com/4dn-dcic/utils
 License: MIT

{dcicutils-8.8.0.1b22.dist-info → dcicutils-8.8.0.1b24.dist-info}/RECORD

@@ -46,8 +46,8 @@ dcicutils/log_utils.py,sha256=7pWMc6vyrorUZQf-V-M3YC6zrPgNhuV_fzm9xqTPph0,10883
 dcicutils/misc_utils.py,sha256=zVc4urdVGgnWjQ4UQlrGH-URAzr2l_PwZWI3u_GJdFE,102210
 dcicutils/obfuscation_utils.py,sha256=fo2jOmDRC6xWpYX49u80bVNisqRRoPskFNX3ymFAmjw,5963
 dcicutils/opensearch_utils.py,sha256=V2exmFYW8Xl2_pGFixF4I2Cc549Opwe4PhFi5twC0M8,1017
-dcicutils/portal_object_utils.py,sha256=khp7E0nb-aXuZWkL8_siftSlYCTOMHAd_2xHStM-maY,13613
-dcicutils/portal_utils.py,sha256=OR1wJNiZVEsxcweArA4-K6yiuy7_bCqawxhZnuFsUtM,27686
+dcicutils/portal_object_utils.py,sha256=7gteQ5CM6IVDfHx-UPFiOfeE1fJYOir_uwWdRTykExQ,15374
+dcicutils/portal_utils.py,sha256=trM8L9J1CPXntdsKSL56hy7SMpftNNpIReAn5iihGes,30050
 dcicutils/project_utils.py,sha256=qPdCaFmWUVBJw4rw342iUytwdQC0P-XKpK4mhyIulMM,31250
 dcicutils/qa_checkers.py,sha256=cdXjeL0jCDFDLT8VR8Px78aS10hwNISOO5G_Zv2TZ6M,20534
 dcicutils/qa_utils.py,sha256=TT0SiJWiuxYvbsIyhK9VO4uV_suxhB6CpuC4qPacCzQ,160208
@@ -62,15 +62,15 @@ dcicutils/secrets_utils.py,sha256=8dppXAsiHhJzI6NmOcvJV5ldvKkQZzh3Fl-cb8Wm7MI,19
 dcicutils/sheet_utils.py,sha256=VlmzteONW5VF_Q4vo0yA5vesz1ViUah1MZ_yA1rwZ0M,33629
 dcicutils/snapshot_utils.py,sha256=ymP7PXH6-yEiXAt75w0ldQFciGNqWBClNxC5gfX2FnY,22961
 dcicutils/ssl_certificate_utils.py,sha256=F0ifz_wnRRN9dfrfsz7aCp4UDLgHEY8LaK7PjnNvrAQ,9707
-dcicutils/structured_data.py,sha256=Yrpv1EgCOhwSaNk2ZMuzbgyxpmAVXpuJ6_4g4LMC2Q8,57180
+dcicutils/structured_data.py,sha256=aXyLqYlgp5DInGiWw6WZk08vfvKI0IW2eWW57n0zhz0,57549
 dcicutils/task_utils.py,sha256=MF8ujmTD6-O2AC2gRGPHyGdUrVKgtr8epT5XU8WtNjk,8082
 dcicutils/tmpfile_utils.py,sha256=n95XF8dZVbQRSXBZTGToXXfSs3JUVRyN6c3ZZ0nhAWI,1403
 dcicutils/trace_utils.py,sha256=g8kwV4ebEy5kXW6oOrEAUsurBcCROvwtZqz9fczsGRE,1769
 dcicutils/validation_utils.py,sha256=cMZIU2cY98FYtzK52z5WUYck7urH6JcqOuz9jkXpqzg,14797
 dcicutils/variant_utils.py,sha256=2H9azNx3xAj-MySg-uZ2SFqbWs4kZvf61JnK6b-h4Qw,4343
 dcicutils/zip_utils.py,sha256=rnjNv_k6L9jT2SjDSgVXp4BEJYLtz9XN6Cl2Fy-tqnM,2027
-dcicutils-8.8.0.1b22.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
-dcicutils-8.8.0.1b22.dist-info/METADATA,sha256=LKPWDLxkixPVLiNnypRH46KEEcwkr_beWBf6hEwz22E,3357
-dcicutils-8.8.0.1b22.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
-dcicutils-8.8.0.1b22.dist-info/entry_points.txt,sha256=51Q4F_2V10L0282W7HFjP4jdzW4K8lnWDARJQVFy_hw,270
-dcicutils-8.8.0.1b22.dist-info/RECORD,,
+dcicutils-8.8.0.1b24.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
+dcicutils-8.8.0.1b24.dist-info/METADATA,sha256=z5fS6VG0wLUCd3NTEvsMF5rdWwLg7TmWlTs1Syg-7Sk,3357
+dcicutils-8.8.0.1b24.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+dcicutils-8.8.0.1b24.dist-info/entry_points.txt,sha256=51Q4F_2V10L0282W7HFjP4jdzW4K8lnWDARJQVFy_hw,270
+dcicutils-8.8.0.1b24.dist-info/RECORD,,