dcicutils 8.8.0.1b23__py3-none-any.whl → 8.8.0.1b24__py3-none-any.whl

dcicutils/portal_object_utils.py

@@ -99,7 +99,8 @@ class PortalObject:
     def compare(self, value: Union[dict, PortalObject],
                 consider_refs: bool = False, resolved_refs: List[dict] = None) -> Tuple[dict, int]:
         if consider_refs and isinstance(resolved_refs, list):
-            this_data, nlookups = self._normalized_refs(refs=resolved_refs).data
+            normlized_portal_object, nlookups = self._normalized_refs(refs=resolved_refs)
+            this_data = normlized_portal_object.data
         else:
             this_data = self.data
             nlookups = 0
@@ -213,7 +214,7 @@ class PortalObject:
                     if Portal.is_lookup_root(lookup_options) and not Portal.is_lookup_root_first(lookup_options):
                         identifying_paths.append(f"/{identifying_value}")
                     if Portal.is_lookup_subtypes(lookup_options):
-                        for subtype_name in self._portal.get_schemas_subtype_names():
+                        for subtype_name in self._portal.get_schema_subtype_names(self.type):
                             identifying_paths.append(f"/{subtype_name}/{identifying_value}")
         return identifying_paths or None
 
@@ -249,13 +250,13 @@ class PortalObject:
         if isinstance(value, dict):
             for key in value:
                 path = f"{_path}.{key}" if _path else key
-                value[key] = PortalObject._normalize_data_refs(value[key], refs=refs,
-                                                               schema=schema, portal=portal, _path=path)
+                value[key], nlookups = PortalObject._normalize_data_refs(value[key], refs=refs,
+                                                                         schema=schema, portal=portal, _path=path)
         elif isinstance(value, list):
             for index in range(len(value)):
                 path = f"{_path or ''}#{index}"
-                value[index] = PortalObject._normalize_data_refs(value[index], refs=refs,
-                                                                 schema=schema, portal=portal, _path=path)
+                value[index], nlookups = PortalObject._normalize_data_refs(value[index], refs=refs,
+                                                                           schema=schema, portal=portal, _path=path)
         elif value_type := Schema.get_property_by_path(schema, _path):
             if link_to := value_type.get("linkTo"):
                 ref_path = f"/{link_to}/{value}"

dcicutils/portal_utils.py

@@ -398,7 +398,7 @@ class Portal:
         return super_type_map_flattened
 
     @lru_cache(maxsize=64)
-    def get_schemas_subtype_names(self, type_name: str) -> List[str]:
+    def get_schema_subtype_names(self, type_name: str) -> List[str]:
         if not (schemas_super_type_map := self.get_schemas_super_type_map()):
             return []
         return schemas_super_type_map.get(type_name, [])

dcicutils/structured_data.py

@@ -222,6 +222,9 @@ class StructuredDataSet:
                         diffs[type_name].append(create_readonly_object(path=identifying_path, uuid=None, diffs=None))
                         if callable(progress):
                             progress({"create": True, "lookups": nlookups})
+                    else:
+                        if callable(progress):
+                            progress({"lookups": nlookups})
         if callable(progress):
             progress({"finish": True})
         return diffs
@@ -257,7 +260,7 @@ class StructuredDataSet:
         self._load_reader(CsvReader(file), type_name=Schema.type_name(file))
 
     def _load_excel_file(self, file: str) -> None:
-        def calculate_total_rows_to_process() -> Tuple[int, int]:
+        def get_counts() -> Tuple[int, int]:
             nonlocal file
             excel = Excel(file)
             nrows = 0
@@ -266,18 +269,27 @@ class StructuredDataSet:
                     nrows += 1
             return nrows, len(excel.sheet_names)
         if self._progress:
-            self._progress_update(calculate_total_rows_to_process)
+            nrows, nsheets = get_counts()
+            self._progress({"start": True, "sheets": nsheets, "rows": nrows})
+        """
+        if self._progress:
+            self._progress_update(get_counts)
+        """
         excel = Excel(file)  # Order the sheet names by any specified ordering (e.g. ala snovault.loadxl).
         order = {Schema.type_name(key): index for index, key in enumerate(self._order)} if self._order else {}
         for sheet_name in sorted(excel.sheet_names, key=lambda key: order.get(Schema.type_name(key), sys.maxsize)):
             self._load_reader(excel.sheet_reader(sheet_name), type_name=Schema.type_name(sheet_name))
+        if self._progress:
+            self._progress({"finish": True})
+        # TODO: Do we really need progress reporting for the below?
         # Check for unresolved reference errors which really are not because of ordering.
         # Yes such internal references will be handled correctly on actual database update via snovault.loadxl.
         if ref_errors := self.ref_errors:
             ref_errors_actual = []
             for ref_error in ref_errors:
                 if not (resolved := self.portal.ref_exists(ref := ref_error["error"])):
-                    # if not (resolved := self.portal.ref_exists_internally(ref := ref_error["error"])):  # TODO
+                    # TODO: Probably do this instead; and if so then no progress needed (per question above).
+                    # if not (resolved := self.portal.ref_exists_internally(ref := ref_error["error"])):
                     ref_errors_actual.append(ref_error)
                 else:
                     self._resolved_refs.add((ref, resolved.get("uuid")))
@@ -310,13 +322,15 @@ class StructuredDataSet:
                     self._add_properties(structured_row, self._autoadd_properties, schema)
             self._add(type_name, structured_row)
             if self._progress:
-                self._progress_update(-1,
-                                      self.ref_total_count,
-                                      self.ref_total_found_count,
-                                      self.ref_total_notfound_count,
-                                      self.ref_lookup_count,
-                                      self.ref_lookup_cache_hit_count,
-                                      self.ref_invalid_identifying_property_count)
+                self._progress({
+                    "parse": True,
+                    "refs": self.ref_total_count,
+                    "refs_found": self.ref_total_found_count,
+                    "refs_not_found": self.ref_total_notfound_count,
+                    "refs_lookup": self.ref_lookup_count,
+                    "refs_cache_hit": self.ref_lookup_cache_hit_count,
+                    "refs_invalid": self.ref_invalid_identifying_property_count
+                })
         self._note_warning(reader.warnings, "reader")
         if schema:
             self._note_error(schema._unresolved_refs, "ref")
@@ -850,7 +864,7 @@ class Portal(PortalBase):
         return schemas
 
     @lru_cache(maxsize=64)
-    def _get_schema_subtypes_names(self, type_name: str) -> List[str]:
+    def _get_schema_subtype_names(self, type_name: str) -> List[str]:
         if not (schemas_super_type_map := self.get_schemas_super_type_map()):
             return []
         return schemas_super_type_map.get(type_name, [])
@@ -913,7 +927,7 @@ class Portal(PortalBase):
             lookup_paths.append(f"/{type_name}/{value}")
         if is_ref_lookup_root and not is_ref_lookup_root_first:
             lookup_paths.append(f"/{value}")
-        subtype_names = self._get_schema_subtypes_names(type_name) if is_ref_lookup_subtypes else []
+        subtype_names = self._get_schema_subtype_names(type_name) if is_ref_lookup_subtypes else []
         for subtype_name in subtype_names:
             lookup_paths.append(f"/{subtype_name}/{value}")
         if not lookup_paths:
@@ -952,7 +966,7 @@ class Portal(PortalBase):
         ref_lookup_strategy, ref_validator = (
             self._ref_lookup_strategy(type_name, self.get_schema(type_name), value))
         is_ref_lookup_subtypes = StructuredDataSet._is_ref_lookup_subtypes(ref_lookup_strategy)
-        subtype_names = self._get_schema_subtypes_names(type_name) if is_ref_lookup_subtypes else []
+        subtype_names = self._get_schema_subtype_names(type_name) if is_ref_lookup_subtypes else []
         for type_name in [type_name] + subtype_names:
             is_resolved, resolved_item = self._ref_exists_single_internally(type_name, value)
             if is_resolved:
@@ -1014,7 +1028,7 @@ class Portal(PortalBase):
                         if not is_uuid(property_value):
                             return False
             return True
-        for schema_name in [type_name] + self._get_schema_subtypes_names(type_name):
+        for schema_name in [type_name] + self._get_schema_subtype_names(type_name):
             if schema := self.get_schema(schema_name):
                 if identifying_properties := schema.get("identifyingProperties"):
                     for identifying_property in identifying_properties:
@@ -1039,7 +1053,7 @@ class Portal(PortalBase):
 
     def _cache_ref(self, type_name: str, value: str, resolved: List[str]) -> None:
         if self._ref_cache is not None:
-            subtype_names = self._get_schema_subtypes_names(type_name)
+            subtype_names = self._get_schema_subtype_names(type_name)
             for type_name in [type_name] + subtype_names:
                 self._ref_cache[f"/{type_name}/{value}"] = resolved
 

{dcicutils-8.8.0.1b23.dist-info → dcicutils-8.8.0.1b24.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcicutils
-Version: 8.8.0.1b23
+Version: 8.8.0.1b24
 Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
 Home-page: https://github.com/4dn-dcic/utils
 License: MIT

{dcicutils-8.8.0.1b23.dist-info → dcicutils-8.8.0.1b24.dist-info}/RECORD

@@ -46,8 +46,8 @@ dcicutils/log_utils.py,sha256=7pWMc6vyrorUZQf-V-M3YC6zrPgNhuV_fzm9xqTPph0,10883
 dcicutils/misc_utils.py,sha256=zVc4urdVGgnWjQ4UQlrGH-URAzr2l_PwZWI3u_GJdFE,102210
 dcicutils/obfuscation_utils.py,sha256=fo2jOmDRC6xWpYX49u80bVNisqRRoPskFNX3ymFAmjw,5963
 dcicutils/opensearch_utils.py,sha256=V2exmFYW8Xl2_pGFixF4I2Cc549Opwe4PhFi5twC0M8,1017
-dcicutils/portal_object_utils.py,sha256=5FEE334DXWDUsl-RAQbV9-QZr-uwvKGqzXFZLZUJ87I,15264
-dcicutils/portal_utils.py,sha256=tG-rq-0oZws14XbUcuTnSjvdoH07KfJb0TWsaUug3g4,30051
+dcicutils/portal_object_utils.py,sha256=7gteQ5CM6IVDfHx-UPFiOfeE1fJYOir_uwWdRTykExQ,15374
+dcicutils/portal_utils.py,sha256=trM8L9J1CPXntdsKSL56hy7SMpftNNpIReAn5iihGes,30050
 dcicutils/project_utils.py,sha256=qPdCaFmWUVBJw4rw342iUytwdQC0P-XKpK4mhyIulMM,31250
 dcicutils/qa_checkers.py,sha256=cdXjeL0jCDFDLT8VR8Px78aS10hwNISOO5G_Zv2TZ6M,20534
 dcicutils/qa_utils.py,sha256=TT0SiJWiuxYvbsIyhK9VO4uV_suxhB6CpuC4qPacCzQ,160208
@@ -62,15 +62,15 @@ dcicutils/secrets_utils.py,sha256=8dppXAsiHhJzI6NmOcvJV5ldvKkQZzh3Fl-cb8Wm7MI,19
 dcicutils/sheet_utils.py,sha256=VlmzteONW5VF_Q4vo0yA5vesz1ViUah1MZ_yA1rwZ0M,33629
 dcicutils/snapshot_utils.py,sha256=ymP7PXH6-yEiXAt75w0ldQFciGNqWBClNxC5gfX2FnY,22961
 dcicutils/ssl_certificate_utils.py,sha256=F0ifz_wnRRN9dfrfsz7aCp4UDLgHEY8LaK7PjnNvrAQ,9707
-dcicutils/structured_data.py,sha256=B8TdvOMA4i2Spq05JAz5xr6jrFKMswyBxTOAJPdILDI,57025
+dcicutils/structured_data.py,sha256=aXyLqYlgp5DInGiWw6WZk08vfvKI0IW2eWW57n0zhz0,57549
 dcicutils/task_utils.py,sha256=MF8ujmTD6-O2AC2gRGPHyGdUrVKgtr8epT5XU8WtNjk,8082
 dcicutils/tmpfile_utils.py,sha256=n95XF8dZVbQRSXBZTGToXXfSs3JUVRyN6c3ZZ0nhAWI,1403
 dcicutils/trace_utils.py,sha256=g8kwV4ebEy5kXW6oOrEAUsurBcCROvwtZqz9fczsGRE,1769
 dcicutils/validation_utils.py,sha256=cMZIU2cY98FYtzK52z5WUYck7urH6JcqOuz9jkXpqzg,14797
 dcicutils/variant_utils.py,sha256=2H9azNx3xAj-MySg-uZ2SFqbWs4kZvf61JnK6b-h4Qw,4343
 dcicutils/zip_utils.py,sha256=rnjNv_k6L9jT2SjDSgVXp4BEJYLtz9XN6Cl2Fy-tqnM,2027
-dcicutils-8.8.0.1b23.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
-dcicutils-8.8.0.1b23.dist-info/METADATA,sha256=zF1tJk8HgMrWpLI8ah39bG8vJi2egOZ8OqRJ33H9e1o,3357
-dcicutils-8.8.0.1b23.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
-dcicutils-8.8.0.1b23.dist-info/entry_points.txt,sha256=51Q4F_2V10L0282W7HFjP4jdzW4K8lnWDARJQVFy_hw,270
-dcicutils-8.8.0.1b23.dist-info/RECORD,,
+dcicutils-8.8.0.1b24.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
+dcicutils-8.8.0.1b24.dist-info/METADATA,sha256=z5fS6VG0wLUCd3NTEvsMF5rdWwLg7TmWlTs1Syg-7Sk,3357
+dcicutils-8.8.0.1b24.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+dcicutils-8.8.0.1b24.dist-info/entry_points.txt,sha256=51Q4F_2V10L0282W7HFjP4jdzW4K8lnWDARJQVFy_hw,270
+dcicutils-8.8.0.1b24.dist-info/RECORD,,