dcicutils 8.8.1.1b4__tar.gz → 8.8.1.1b6__tar.gz

{dcicutils-8.8.1.1b4 → dcicutils-8.8.1.1b6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcicutils
-Version: 8.8.1.1b4
+Version: 8.8.1.1b6
 Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
 Home-page: https://github.com/4dn-dcic/utils
 License: MIT

{dcicutils-8.8.1.1b4 → dcicutils-8.8.1.1b6}/dcicutils/data_readers.py

@@ -77,7 +77,11 @@ class RowReader(abc.ABC):
     def warnings(self) -> List[str]:
         warnings = []
         if self._warning_empty_headers:
-            warnings.append({"src": create_dict(file=self.file),
+            if hasattr(self, "sheet_name") and self.sheet_name:
+                src = {"sheet": self.sheet_name}
+            else:
+                src = {"file": self.file}
+            warnings.append({"src": src,
                              "warning": "Empty header column encountered; ignoring it and all subsequent columns."})
         if self._warning_extra_values:
             for row_number in self._warning_extra_values:

{dcicutils-8.8.1.1b4 → dcicutils-8.8.1.1b6}/dcicutils/structured_data.py

@@ -44,7 +44,7 @@ FILE_TYPE_PROPERTY_NAME = "filename"
 EXTRA_FILE_TYPE_NAME = "ExtraFile"
 EXTRA_FILE_TYPE_PROPERTY_NAME = "extra_files"
 
-ENABLE_ARRAY_SHEET_REFS = True
+ENABLE_ARRAY_SHEET_REFS = False
 
 # The ExtraFile pseudo-type schema.
 EXTRA_FILE_SCHEMA = {
@@ -91,6 +91,7 @@ class StructuredDataSet:
         self._errors = {}
         self._resolved_refs = set()
         self._validated = False
+        self._nrows = 0
         self._autoadd_properties = autoadd if isinstance(autoadd, dict) and autoadd else None
         self._norefs = True if norefs is True else False
         self._debug_sleep = None
@@ -194,6 +195,10 @@ class StructuredDataSet:
                 upload_file["path"] = file_path
         return upload_files
 
+    @property
+    def nrows(self) -> int:
+        return self._nrows
+
     def compare(self, progress: Optional[Callable] = None) -> dict:
         def get_counts() -> int:
             ntypes = 0
@@ -288,8 +293,6 @@ class StructuredDataSet:
         order = {Schema.type_name(key): index for index, key in enumerate(self._order)} if self._order else {}
         for sheet_name in sorted(excel.sheet_names, key=lambda key: order.get(Schema.type_name(key), sys.maxsize)):
             self._load_reader(excel.sheet_reader(sheet_name), type_name=Schema.type_name(sheet_name))
-        if self._progress:
-            self._progress({"finish": True})
         # TODO: Do we really need progress reporting for the below?
         # Check for unresolved reference errors which really are not because of ordering.
         # Yes such internal references will be handled correctly on actual database update via snovault.loadxl.
@@ -301,11 +304,25 @@ class StructuredDataSet:
                     # if not (resolved := self.portal.ref_exists_internally(ref := ref_error["error"])):
                     ref_errors_actual.append(ref_error)
                 else:
+                    # Now found so subtract off from ref_total_notfound_count.
+                    self.portal._ref_total_notfound_count -= 1
                     self._resolved_refs.add((ref, resolved.get("uuid")))
             if ref_errors_actual:
                 self._errors["ref"] = ref_errors_actual
             else:
                 del self._errors["ref"]
+        if self._progress:
+            # TODO: Refactor with same thing below in _load_reader.
+            self._progress({
+                "finish": True,
+                "refs": self.ref_total_count,
+                "refs_found": self.ref_total_found_count,
+                "refs_not_found": self.ref_total_notfound_count,
+                "refs_lookup": self.ref_lookup_count,
+                "refs_lookup_cache_hit": self.ref_lookup_cache_hit_count,
+                "refs_exists_cache_hit": self.ref_exists_cache_hit_count,
+                "refs_invalid": self.ref_invalid_identifying_property_count
+            })
 
     def _load_json_file(self, file: str) -> None:
         with open(file) as f:
@@ -316,6 +333,7 @@ class StructuredDataSet:
         noschema = False
         structured_row_template = None
         for row in reader:
+            self._nrows += 1
             if self._debug_sleep:
                 time.sleep(float(self._debug_sleep))
             if not structured_row_template:  # Delay creation just so we don't reference schema if there are no rows.
@@ -338,7 +356,8 @@ class StructuredDataSet:
                     "refs_found": self.ref_total_found_count,
                     "refs_not_found": self.ref_total_notfound_count,
                     "refs_lookup": self.ref_lookup_count,
-                    "refs_cache_hit": self.ref_exists_cache_hit_count,
+                    "refs_lookup_cache_hit": self.ref_lookup_cache_hit_count,
+                    "refs_exists_cache_hit": self.ref_exists_cache_hit_count,
                     "refs_invalid": self.ref_invalid_identifying_property_count
                 })
         self._note_warning(reader.warnings, "reader")
@@ -510,15 +529,16 @@ class _StructuredRowTemplate:
                     set_value_backtrack_object(i, p)
                 data = data[p]
             if (p := path[-1]) == -1 and isinstance(value, str):
-                if ENABLE_ARRAY_SHEET_REFS and value.lower().startswith("[ref:") and value.endswith("]"):
-                    if self._obtain_array_values:
-                        values = self._obtain_array_values(value)
-                    if sheet_name_containing_array := value[5:].strip():
-                        if dot := sheet_name_containing_array.find(".") > 0:
-                            if sheet_name_containing_array := sheet_name_containing_array[0:dot].strip():
-                                pass
-                                # sheet_column_containing_array = sheet_name_containing_array[dot + 1:].strip()
-                    pass
+                if ENABLE_ARRAY_SHEET_REFS and False:
+                    # TODO: IN PROGRESS. DISABLED FOR NOW.
+                    if isinstance(value, str) and value.lower().startswith("[ref:") and value.endswith("]"):
+                        if self._obtain_array_values:
+                            values = self._obtain_array_values(value)
+                        if sheet_name_containing_array := value[5:].strip():
+                            if dot := sheet_name_containing_array.find(".") > 0:
+                                if sheet_name_containing_array := sheet_name_containing_array[0:dot].strip():
+                                    pass
+                                    # sheet_column_containing_array = sheet_name_containing_array[dot + 1:].strip()
                 values = _split_array_string(value, unique=typeinfo.get("unique") if typeinfo else False)
                 if mapv:
                     values = [mapv(value, src) for value in values]
@@ -891,7 +911,6 @@ class Portal(PortalBase):
 
     def ref_exists(self, type_name: str, value: Optional[str] = None,
                    called_from_map_ref: bool = False) -> Optional[dict]:
-        # print(f"\033[Kxyzzy:ref_exists({type_name}/{value})")
         if not value:
             type_name, value = Portal._get_type_name_and_value_from_path(type_name)
             if not type_name or not value:
@@ -919,7 +938,9 @@ class Portal(PortalBase):
             # self._data can change, i.e. as data (e.g. spreadsheet sheets) are parsed.
             return self.ref_exists_internally(type_name, value, update_counts=called_from_map_ref) or {}
         # Reference is NOT cached here; lookup INTERNALLY first.
-        if resolved := self.ref_exists_internally(type_name, value, update_counts=called_from_map_ref):
+        # Skip updating _ref_total_notfound_count here as if not found we look in portal below.
+        if resolved := self.ref_exists_internally(type_name, value, update_counts=called_from_map_ref,
+                                                  skip_total_notfound_count=True):
             # Reference was resolved internally (note: here only if resolved is not an empty dictionary).
             if called_from_map_ref:
                 self._ref_total_found_count += 1
@@ -965,13 +986,13 @@ class Portal(PortalBase):
         return None
 
     def ref_exists_internally(self, type_name: str, value: Optional[str] = None,
-                              update_counts: bool = False) -> Optional[dict]:
+                              update_counts: bool = False,
+                              skip_total_notfound_count: bool = False) -> Optional[dict]:
         """
         Looks up the given reference (type/value) internally (i.e. with this data parsed thus far).
         If found then returns a dictionary containing the (given) type name and the uuid (if any)
         of the resolved item.
         """
-        # print(f"\033[Kxyzzy:ref_exists_internally({type_name}/{value})")
         if not value:
             type_name, value = Portal._get_type_name_and_value_from_path(type_name)
             if not type_name or not value:
@@ -990,6 +1011,9 @@ class Portal(PortalBase):
                 resolved = {"type": type_name, "uuid": resolved_item.get("uuid")}
                 self._cache_ref(type_name, value, resolved)
                 return resolved
+        if update_counts:
+            if not skip_total_notfound_count:
+                self._ref_total_notfound_count += 1
         return {}  # Empty return means not resolved internally.
 
     def _ref_exists_single_internally(self, type_name: str, value: str) -> Tuple[bool, Optional[dict]]:

{dcicutils-8.8.1.1b4 → dcicutils-8.8.1.1b6}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dcicutils"
-version = "8.8.1.1b4"  # TODO: To become 8.8.2
+version = "8.8.1.1b6"  # TODO: To become 8.8.2
 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources"
 authors = ["4DN-DCIC Team <support@4dnucleome.org>"]
 license = "MIT"