dcicutils 8.8.0.1b14__py3-none-any.whl → 8.8.0.1b16__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -7,7 +7,6 @@ from pyramid.router import Router
7
7
  import re
8
8
  import sys
9
9
  import time
10
- from tqdm import tqdm
11
10
  from typing import Any, Callable, List, Optional, Tuple, Type, Union
12
11
  from webtest.app import TestApp
13
12
  from dcicutils.common import OrchestratedApp
@@ -70,10 +69,9 @@ class StructuredDataSet:
70
69
  order: Optional[List[str]] = None, prune: bool = True,
71
70
  ref_lookup_strategy: Optional[Callable] = None,
72
71
  ref_lookup_nocache: bool = False,
73
- progress: bool = False,
72
+ progress: Optional[Callable] = None,
74
73
  debug_sleep: Optional[str] = None) -> None:
75
- progress = False
76
- self._progress = progress
74
+ self._progress = progress if callable(progress) else None
77
75
  self._data = {}
78
76
  self._portal = Portal(portal, data=self._data, schemas=schemas,
79
77
  ref_lookup_strategy=ref_lookup_strategy,
@@ -93,20 +91,15 @@ class StructuredDataSet:
93
91
  self._debug_sleep = None
94
92
  self._load_file(file) if file else None
95
93
 
96
- def _progress_add(self, amount: Union[int, Callable]) -> None:
97
- if self._progress is not False and self._progress is not None:
98
- if callable(amount):
99
- amount = amount()
100
- if not isinstance(amount, int):
101
- return
102
- if self._progress is True:
103
- if amount > 0:
104
- self._progress = tqdm(total=amount)
105
- elif isinstance(self._progress, tqdm):
106
- if amount > 0:
107
- self._progress.total += amount
108
- elif amount < 0:
109
- self._progress.update(-amount)
94
+ def _progress_update(self, nrows: Union[int, Callable],
95
+ nrefs_resolved: Optional[int] = None,
96
+ nrefs_unresolved: Optional[int] = None,
97
+ nlookups: Optional[int] = None) -> None:
98
+ if self._progress:
99
+ if callable(nrows):
100
+ nrows = nrows()
101
+ if isinstance(nrows, int) and nrows != 0:
102
+ self._progress(nrows, nrefs_resolved, nrefs_unresolved, nlookups)
110
103
 
111
104
  @property
112
105
  def data(self) -> dict:
@@ -259,7 +252,8 @@ class StructuredDataSet:
259
252
  for row in excel.sheet_reader(sheet_name):
260
253
  nrows += 1
261
254
  return nrows
262
- self._progress_add(calculate_total_rows_to_process)
255
+ if self._progress:
256
+ self._progress_update(calculate_total_rows_to_process)
263
257
  excel = Excel(file) # Order the sheet names by any specified ordering (e.g. ala snovault.loadxl).
264
258
  order = {Schema.type_name(key): index for index, key in enumerate(self._order)} if self._order else {}
265
259
  for sheet_name in sorted(excel.sheet_names, key=lambda key: order.get(Schema.type_name(key), sys.maxsize)):
@@ -270,9 +264,10 @@ class StructuredDataSet:
270
264
  ref_errors_actual = []
271
265
  for ref_error in ref_errors:
272
266
  if not (resolved := self.portal.ref_exists(ref := ref_error["error"])):
267
+ # if not (resolved := self.portal.ref_exists_internally(ref := ref_error["error"])):
273
268
  ref_errors_actual.append(ref_error)
274
269
  else:
275
- self._resolved_refs.add((ref, resolved[0].get("uuid")))
270
+ self._resolved_refs.add((ref, resolved.get("uuid")))
276
271
  if ref_errors_actual:
277
272
  self._errors["ref"] = ref_errors_actual
278
273
  else:
@@ -301,7 +296,8 @@ class StructuredDataSet:
301
296
  if self._autoadd_properties:
302
297
  self._add_properties(structured_row, self._autoadd_properties, schema)
303
298
  self._add(type_name, structured_row)
304
- self._progress_add(-1)
299
+ if self._progress:
300
+ self._progress_update(-1, self.ref_total_count, self.ref_total_notfound_count, self.ref_lookup_count)
305
301
  self._note_warning(reader.warnings, "reader")
306
302
  if schema:
307
303
  self._note_error(schema._unresolved_refs, "ref")
@@ -333,6 +329,18 @@ class StructuredDataSet:
333
329
  def _is_ref_lookup_subtypes(ref_lookup_flags: int) -> bool:
334
330
  return (ref_lookup_flags & StructuredDataSet.REF_LOOKUP_SUBTYPES) == StructuredDataSet.REF_LOOKUP_SUBTYPES
335
331
 
332
+ @property
333
+ def ref_total_count(self) -> int:
334
+ return self.portal.ref_total_count if self.portal else -1
335
+
336
+ @property
337
+ def ref_total_found_count(self) -> int:
338
+ return self.portal.ref_total_found_count if self.portal else -1
339
+
340
+ @property
341
+ def ref_total_notfound_count(self) -> int:
342
+ return self.portal.ref_total_notfound_count if self.portal else -1
343
+
336
344
  @property
337
345
  def ref_lookup_cache_hit_count(self) -> int:
338
346
  return self.portal.ref_lookup_cache_hit_count if self.portal else -1
@@ -361,6 +369,10 @@ class StructuredDataSet:
361
369
  def ref_exists_internal_count(self) -> int:
362
370
  return self.portal.ref_exists_internal_count if self.portal else -1
363
371
 
372
+ @property
373
+ def ref_exists_external_count(self) -> int:
374
+ return self.portal.ref_exists_external_count if self.portal else -1
375
+
364
376
  @property
365
377
  def ref_exists_cache_hit_count(self) -> int:
366
378
  return self.portal.ref_exists_cache_hit_count if self.portal else -1
@@ -600,18 +612,11 @@ class Schema(SchemaBase):
600
612
  # TODO: If think we do have the column (and type?) name(s) originating the ref yes?
601
613
  self._unresolved_refs.append({"src": src, "error": f"/{link_to}/<null>"})
602
614
  elif portal:
603
- if not (resolved := portal.ref_exists(link_to, value)):
615
+ if not (resolved := portal.ref_exists(link_to, value, True)):
604
616
  self._unresolved_refs.append({"src": src, "error": f"/{link_to}/{value}"})
605
- elif len(resolved) > 1:
606
- # TODO: Don't think we need this anymore; see TODO on Portal.ref_exists.
607
- self._unresolved_refs.append({
608
- "src": src,
609
- "error": f"/{link_to}/{value}",
610
- "types": [resolved_ref["type"] for resolved_ref in resolved]})
611
617
  else:
612
618
  # A resolved-ref set value is a tuple of the reference path and its uuid.
613
- self._resolved_refs.add((f"/{link_to}/{value}", resolved[0].get("uuid")))
614
- # self._resolved_refs.add((f"/{link_to}/{value}", resolved[0].get("uuid"), resolved[0].get("data")))
619
+ self._resolved_refs.add((f"/{link_to}/{value}", resolved.get("uuid")))
615
620
  return value
616
621
  return lambda value, src: map_ref(value, typeinfo.get("linkTo"), self._portal, src)
617
622
 
@@ -772,24 +777,28 @@ class Portal(PortalBase):
772
777
  else:
773
778
  self._ref_lookup_strategy = lambda type_name, schema, value: (StructuredDataSet.REF_LOOKUP_DEFAULT, None)
774
779
  if ref_lookup_nocache is True:
775
- self.ref_lookup = self.ref_lookup_nocache
780
+ self.ref_lookup = self.ref_lookup_uncached
776
781
  self._ref_cache = None
777
782
  else:
778
- self.ref_lookup = self.ref_lookup_cache
783
+ self.ref_lookup = self.ref_lookup_cached
779
784
  self._ref_cache = {}
780
785
  self._ref_lookup_found_count = 0
781
786
  self._ref_lookup_notfound_count = 0
782
787
  self._ref_lookup_error_count = 0
783
788
  self._ref_exists_internal_count = 0
789
+ self._ref_exists_external_count = 0
784
790
  self._ref_exists_cache_hit_count = 0
785
791
  self._ref_exists_cache_miss_count = 0
786
792
  self._ref_incorrect_identifying_property_count = 0
793
+ self._ref_total_count = 0
794
+ self._ref_total_found_count = 0
795
+ self._ref_total_notfound_count = 0
787
796
 
788
797
  @lru_cache(maxsize=8092)
789
- def ref_lookup_cache(self, object_name: str) -> Optional[dict]:
790
- return self.ref_lookup_nocache(object_name)
798
+ def ref_lookup_cached(self, object_name: str) -> Optional[dict]:
799
+ return self.ref_lookup_uncached(object_name)
791
800
 
792
- def ref_lookup_nocache(self, object_name: str) -> Optional[dict]:
801
+ def ref_lookup_uncached(self, object_name: str) -> Optional[dict]:
793
802
  try:
794
803
  result = super().get_metadata(object_name, raw=True)
795
804
  self._ref_lookup_found_count += 1
@@ -824,10 +833,10 @@ class Portal(PortalBase):
824
833
  return schemas
825
834
 
826
835
  @lru_cache(maxsize=64)
827
- def _get_schema_subtypes(self, type_name: str) -> Optional[List[str]]:
836
+ def _get_schema_subtypes_names(self, type_name: str) -> List[str]:
828
837
  if not (schemas_super_type_map := self.get_schemas_super_type_map()):
829
838
  return []
830
- return schemas_super_type_map.get(type_name)
839
+ return schemas_super_type_map.get(type_name, [])
831
840
 
832
841
  def is_file_schema(self, schema_name: str) -> bool:
833
842
  """
@@ -835,74 +844,47 @@ class Portal(PortalBase):
835
844
  """
836
845
  return self.is_schema_type(schema_name, FILE_SCHEMA_NAME)
837
846
 
838
- def _ref_exists_from_cache(self, type_name: str, value: str) -> Optional[List[dict]]:
839
- if self._ref_cache is not None:
840
- return self._ref_cache.get(f"/{type_name}/{value}", None)
841
- return None
842
-
843
- def _cache_ref(self, type_name: str, value: str, resolved: List[str], subtype_names: Optional[List[str]]) -> None:
844
- if self._ref_cache is not None:
845
- for type_name in [type_name] + (subtype_names if subtype_names else []):
846
- self._ref_cache[f"/{type_name}/{value}"] = resolved
847
-
848
- def ref_exists(self, type_name: str, value: Optional[str] = None) -> List[dict]:
847
+ def ref_exists(self, type_name: str, value: Optional[str] = None,
848
+ called_from_map_ref: bool = False) -> Optional[dict]:
849
+ # print(f"\033[Kxyzzy:ref_exists({type_name}/{value})")
849
850
  if not value:
850
- if type_name.startswith("/") and len(parts := type_name[1:].split("/")) == 2:
851
- if not (type_name := parts[0]) or not (value := parts[1]):
852
- return []
853
- else:
854
- return []
851
+ type_name, value = Portal._get_type_name_and_value_from_path(type_name)
852
+ if not type_name or not value:
853
+ return None
854
+ if called_from_map_ref:
855
+ self._ref_total_count += 1
856
+ # First check our reference cache.
855
857
  if (resolved := self._ref_exists_from_cache(type_name, value)) is not None:
856
- # Found cached resolved reference.
857
- if not resolved:
858
- # Cached resolved reference is empty ([]).
859
- # It might NOW be found internally, since the portal self._data
860
- # can change, as the data (e.g. spreadsheet sheets) are parsed.
861
- # TODO: Consolidate this with the below similar usage.
862
- ref_lookup_strategy, incorrect_identifying_property = (
863
- self._ref_lookup_strategy(type_name, self.get_schema(type_name), value))
864
- is_ref_lookup_subtypes = StructuredDataSet._is_ref_lookup_subtypes(ref_lookup_strategy)
865
- subtype_names = self._get_schema_subtypes(type_name) if is_ref_lookup_subtypes else None
866
- is_resolved, identifying_property, resolved_uuid = (
867
- self._ref_exists_internally(type_name, value, subtype_names,
868
- incorrect_identifying_property=incorrect_identifying_property))
869
- if is_resolved:
870
- if identifying_property == incorrect_identifying_property:
871
- # Not REALLY resolved as it resolved to a property which is NOT an identifying
872
- # property, but may be commonly mistaken for one (e.g. UnalignedReads.filename).
873
- self._ref_incorrect_identifying_property_count += 1
874
- return []
875
- resolved = [{"type": type_name, "uuid": resolved_uuid}]
876
- self._cache_ref(type_name, value, resolved, subtype_names)
877
- return resolved
878
- self._ref_exists_cache_hit_count += 1
858
+ # Found CACHED reference.
859
+ if resolved:
860
+ # Found cached RESOLVED reference (non-empty object).
861
+ if called_from_map_ref:
862
+ self._ref_total_found_count += 1
863
+ return resolved
864
+ # Found cached UNRESOLVED reference (empty object); meaning it was looked
865
+ # up but not found. It might NOW be found INTERNALLY, since the portal
866
+ # self._data can change, i.e. as data (e.g. spreadsheet sheets) are parsed.
867
+ return self.ref_exists_internally(type_name, value, update_counts=called_from_map_ref) or {}
868
+ # Reference is NOT cached here; lookup INTERNALLY first.
869
+ if (resolved := self.ref_exists_internally(type_name, value, update_counts=called_from_map_ref)) is None:
870
+ # Reference was resolved (internally) INCORRECTLY.
871
+ if called_from_map_ref:
872
+ self._ref_total_notfound_count += 1
873
+ return None
874
+ if resolved:
875
+ # Reference was resolved internally.
876
+ if called_from_map_ref:
877
+ self._ref_total_found_count += 1
879
878
  return resolved
880
- # Not cached here.
881
- self._ref_exists_cache_miss_count += 1
879
+ # Reference is NOT cached and was NOT resolved internally; lookup in PORTAL.
882
880
  # Get the lookup strategy; i.e. should do we lookup by root path, and if so, should
883
881
  # we do this first, and do we lookup by subtypes; by default we lookup by root path
884
- # but not first, and we do lookup by subtypes.
885
- ref_lookup_strategy, incorrect_identifying_property = (
886
- self._ref_lookup_strategy(type_name, self.get_schema(type_name), value))
882
+ # but not first, and we also lookup by subtypes by default.
883
+ ref_lookup_strategy, _ = self._ref_lookup_strategy(type_name, self.get_schema(type_name), value)
887
884
  is_ref_lookup_specified_type = StructuredDataSet._is_ref_lookup_specified_type(ref_lookup_strategy)
888
885
  is_ref_lookup_root = StructuredDataSet._is_ref_lookup_root(ref_lookup_strategy)
889
886
  is_ref_lookup_root_first = StructuredDataSet._is_ref_lookup_root_first(ref_lookup_strategy)
890
887
  is_ref_lookup_subtypes = StructuredDataSet._is_ref_lookup_subtypes(ref_lookup_strategy)
891
- subtype_names = self._get_schema_subtypes(type_name) if is_ref_lookup_subtypes else None
892
- # Lookup internally first (including at subtypes if desired; root lookup not applicable here).
893
- is_resolved, identifying_property, resolved_uuid = (
894
- self._ref_exists_internally(type_name, value, subtype_names,
895
- incorrect_identifying_property=incorrect_identifying_property))
896
- if is_resolved:
897
- if identifying_property == incorrect_identifying_property:
898
- # Not REALLY resolved as it resolved to a property which is NOT an identifying
899
- # property, but may be commonly mistaken for one (e.g. UnalignedReads.filename).
900
- self._ref_incorrect_identifying_property_count += 1
901
- return []
902
- resolved = [{"type": type_name, "uuid": resolved_uuid}]
903
- self._cache_ref(type_name, value, resolved, subtype_names)
904
- return resolved
905
- # Not found internally; perform actual portal lookup (including at root and subtypes if desired).
906
888
  # First construct the list of lookup paths at which to look for the referenced item.
907
889
  lookup_paths = []
908
890
  if is_ref_lookup_root_first:
@@ -911,37 +893,81 @@ class Portal(PortalBase):
911
893
  lookup_paths.append(f"/{type_name}/{value}")
912
894
  if is_ref_lookup_root and not is_ref_lookup_root_first:
913
895
  lookup_paths.append(f"/{value}")
914
- if subtype_names:
915
- for subtype_name in subtype_names:
916
- lookup_paths.append(f"/{subtype_name}/{value}")
896
+ subtype_names = self._get_schema_subtypes_names(type_name) if is_ref_lookup_subtypes else []
897
+ for subtype_name in subtype_names:
898
+ lookup_paths.append(f"/{subtype_name}/{value}")
917
899
  if not lookup_paths:
918
900
  # No (i.e. zero) lookup strategy means no ref lookup at all.
919
- return []
920
- # Do the actual lookup in the portal for each of the desired lookup paths.
901
+ if called_from_map_ref:
902
+ self._ref_total_notfound_count += 1
903
+ return None
904
+ # Do the actual lookup in portal for each of the desired lookup paths.
921
905
  for lookup_path in lookup_paths:
922
- if isinstance(item := self.ref_lookup(lookup_path), dict):
923
- resolved = [{"type": type_name, "uuid": item.get("uuid", None)}]
924
- self._cache_ref(type_name, value, resolved, subtype_names)
906
+ if isinstance(resolved_item := self.ref_lookup(lookup_path), dict):
907
+ resolved = {"type": type_name, "uuid": resolved_item.get("uuid", None)}
908
+ self._cache_ref(type_name, value, resolved)
909
+ self._ref_exists_external_count += 1
910
+ if called_from_map_ref:
911
+ self._ref_total_found_count += 1
925
912
  return resolved
926
- # Not found at all; note that we cache this ([]) too; indicates lookup has been done.
927
- self._cache_ref(type_name, value, [], subtype_names)
928
- return []
929
-
930
- def _ref_exists_internally(
931
- self, type_name: str, value: str,
932
- subtype_names: Optional[List[str]] = None,
933
- incorrect_identifying_property: Optional[str] = None) -> Tuple[bool, Optional[str], Optional[str]]:
934
- for type_name in [type_name] + (subtype_names or []):
935
- is_resolved, identifying_property, resolved_uuid = self._ref_exists_single_internally(
936
- type_name, value, incorrect_identifying_property=incorrect_identifying_property)
937
- if is_resolved:
938
- return True, identifying_property, resolved_uuid
939
- return False, None, None
913
+ # Not found at all; note that we cache this ({}) too; indicates lookup has been done.
914
+ self._cache_ref(type_name, value, {})
915
+ if called_from_map_ref:
916
+ self._ref_total_notfound_count += 1
917
+ return None
940
918
 
941
- def _ref_exists_single_internally(
942
- self, type_name: str, value: str,
943
- incorrect_identifying_property:
944
- Optional[str] = None) -> Tuple[bool, Optional[str], Optional[str]]:
919
+ def ref_exists_internally(self, type_name: str, value: Optional[str] = None,
920
+ update_counts: bool = False) -> Optional[dict]:
921
+ """
922
+ Looks up the given reference (type/value) internally (i.e. with this data parsed thus far).
923
+ If found then returns a list of a single dictionary containing the (given) type name and
924
+ the uuid (if any) of the resolved item. If not found then returns an empty list; however,
925
+ if not found, but found using an "incorrect" identifying property, then returns None.
926
+ """
927
+ # print(f"\033[Kxyzzy:ref_exists_internally({type_name}/{value})")
928
+ if not value:
929
+ type_name, value = Portal._get_type_name_and_value_from_path(type_name)
930
+ if not type_name or not value:
931
+ return None
932
+ # Note that root lookup not applicable here.
933
+ ref_lookup_strategy, incorrect_identifying_property = (
934
+ self._ref_lookup_strategy(type_name, self.get_schema(type_name), value))
935
+ is_ref_lookup_subtypes = StructuredDataSet._is_ref_lookup_subtypes(ref_lookup_strategy)
936
+ subtype_names = self._get_schema_subtypes_names(type_name) if is_ref_lookup_subtypes else []
937
+ for type_name in [type_name] + subtype_names:
938
+ is_resolved, resolved_item = self._ref_exists_single_internally(type_name, value)
939
+ if is_resolved:
940
+ if update_counts:
941
+ self._ref_exists_internal_count += 1
942
+ self._ref_total_found_count += 1
943
+ resolved = {"type": type_name, "uuid": resolved_item.get("uuid")}
944
+ self._cache_ref(type_name, value, resolved)
945
+ return resolved
946
+ # Here this reference is not resolved internally; but let us check any specified incorrect
947
+ # property to see if it would have been resolved using that; for example, if we pretend that
948
+ # UnalignedReads.filename were an identifying property (which it is not), then we see if this
949
+ # reference, which would otherwise be unresolved, would be resolved; in which case we have an
950
+ # incorrect reference; doing this can cut down considerably on useless lookups (at least for
951
+ # a case from He Li, early March 2024).
952
+ for type_name in [type_name] + subtype_names:
953
+ if incorrect_identifying_property:
954
+ if self._data and (items := self._data.get(type_name)):
955
+ for item in items:
956
+ if (identifying_value := item.get(incorrect_identifying_property, None)) is not None:
957
+ if ((identifying_value == value) or
958
+ (isinstance(identifying_value, list) and (value in identifying_value))): # noqa
959
+ # Not REALLY resolved as it resolved to a property which is NOT an identifying
960
+ # property, but may be commonly mistaken for one (e.g. UnalignedReads.filename).
961
+ # Return value to prevent actual portal lookup from happening.
962
+ if update_counts:
963
+ self._ref_incorrect_identifying_property_count += 1
964
+ self._ref_total_notfound_count += 1
965
+ return None # None return means resolved internally incorrectly.
966
+ if update_counts:
967
+ self._ref_total_notfound_count += 1
968
+ return {} # Empty return means not resolved internally.
969
+
970
+ def _ref_exists_single_internally(self, type_name: str, value: str) -> Tuple[bool, Optional[dict]]:
945
971
  if self._data and (items := self._data.get(type_name)) and (schema := self.get_schema(type_name)):
946
972
  identifying_properties = set(schema.get("identifyingProperties", [])) | {"identifier", "uuid"}
947
973
  for item in items:
@@ -949,32 +975,41 @@ class Portal(PortalBase):
949
975
  if (identifying_value := item.get(identifying_property, None)) is not None:
950
976
  if ((identifying_value == value) or
951
977
  (isinstance(identifying_value, list) and (value in identifying_value))): # noqa
952
- self._ref_exists_internal_count += 1
953
- return True, identifying_property, item.get("uuid", None)
954
- if incorrect_identifying_property:
955
- if (identifying_value := item.get(incorrect_identifying_property, None)) is not None:
956
- if ((identifying_value == value) or
957
- (isinstance(identifying_value, list) and (value in identifying_value))): # noqa
958
- return True, incorrect_identifying_property, item.get("uuid", None)
959
- return False, None, None
978
+ return True, item
979
+ return False, None
980
+
981
+ @staticmethod
982
+ def _get_type_name_and_value_from_path(path: str) -> Tuple[Optional[str], Optional[str]]:
983
+ if path.startswith("/") and len(parts := path[1:].split("/")) == 2:
984
+ if not (type_name := parts[0]) or not (value := parts[1]):
985
+ return None
986
+ return type_name, value
987
+ return None, None
988
+
989
+ def _ref_exists_from_cache(self, type_name: str, value: str) -> Optional[List[dict]]:
990
+ if self._ref_cache is not None:
991
+ self._ref_exists_cache_hit_count += 1
992
+ return self._ref_cache.get(f"/{type_name}/{value}", None)
993
+ self._ref_exists_cache_miss_count += 1
994
+ return None
995
+
996
+ def _cache_ref(self, type_name: str, value: str, resolved: List[str]) -> None:
997
+ subtype_names = self._get_schema_subtypes_names(type_name)
998
+ if self._ref_cache is not None:
999
+ for type_name in [type_name] + subtype_names:
1000
+ self._ref_cache[f"/{type_name}/{value}"] = resolved
960
1001
 
961
1002
  @property
962
- def ref_lookup_cache_hit_count(self) -> int:
963
- if self._ref_cache is None:
964
- return 0
965
- try:
966
- return self.ref_lookup_cache.cache_info().hits
967
- except Exception:
968
- return -1
1003
+ def ref_total_count(self) -> int:
1004
+ return self._ref_total_count
969
1005
 
970
1006
  @property
971
- def ref_lookup_cache_miss_count(self) -> int:
972
- if self._ref_cache is None:
973
- return self.ref_lookup_count
974
- try:
975
- return self.ref_lookup_cache.cache_info().misses
976
- except Exception:
977
- return -1
1007
+ def ref_total_found_count(self) -> int:
1008
+ return self._ref_total_found_count
1009
+
1010
+ @property
1011
+ def ref_total_notfound_count(self) -> int:
1012
+ return self._ref_total_notfound_count
978
1013
 
979
1014
  @property
980
1015
  def ref_lookup_count(self) -> int:
@@ -992,10 +1027,32 @@ class Portal(PortalBase):
992
1027
  def ref_lookup_error_count(self) -> int:
993
1028
  return self._ref_lookup_error_count
994
1029
 
1030
+ @property
1031
+ def ref_lookup_cache_hit_count(self) -> int:
1032
+ if self._ref_cache is None:
1033
+ return -1
1034
+ try:
1035
+ return self.ref_lookup_cached.cache_info().hits
1036
+ except Exception:
1037
+ return -1
1038
+
1039
+ @property
1040
+ def ref_lookup_cache_miss_count(self) -> int:
1041
+ if self._ref_cache is None:
1042
+ return -1
1043
+ try:
1044
+ return self.ref_lookup_cached.cache_info().misses
1045
+ except Exception:
1046
+ return -1
1047
+
995
1048
  @property
996
1049
  def ref_exists_internal_count(self) -> int:
997
1050
  return self._ref_exists_internal_count
998
1051
 
1052
+ @property
1053
+ def ref_exists_external_count(self) -> int:
1054
+ return self._ref_exists_external_count
1055
+
999
1056
  @property
1000
1057
  def ref_exists_cache_hit_count(self) -> int:
1001
1058
  return self._ref_exists_cache_hit_count
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: dcicutils
3
- Version: 8.8.0.1b14
3
+ Version: 8.8.0.1b16
4
4
  Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
5
5
  Home-page: https://github.com/4dn-dcic/utils
6
6
  License: MIT
@@ -62,15 +62,15 @@ dcicutils/secrets_utils.py,sha256=8dppXAsiHhJzI6NmOcvJV5ldvKkQZzh3Fl-cb8Wm7MI,19
62
62
  dcicutils/sheet_utils.py,sha256=VlmzteONW5VF_Q4vo0yA5vesz1ViUah1MZ_yA1rwZ0M,33629
63
63
  dcicutils/snapshot_utils.py,sha256=ymP7PXH6-yEiXAt75w0ldQFciGNqWBClNxC5gfX2FnY,22961
64
64
  dcicutils/ssl_certificate_utils.py,sha256=F0ifz_wnRRN9dfrfsz7aCp4UDLgHEY8LaK7PjnNvrAQ,9707
65
- dcicutils/structured_data.py,sha256=72nX3ijfc2QACqfyjWFlQIX-VX1B8RPRc4GA-Ly2G5E,52998
65
+ dcicutils/structured_data.py,sha256=QK04rpuoqyMv5NO-x4yFgCzHdZIGp3g1eR8XjNmJNvM,55302
66
66
  dcicutils/task_utils.py,sha256=MF8ujmTD6-O2AC2gRGPHyGdUrVKgtr8epT5XU8WtNjk,8082
67
67
  dcicutils/tmpfile_utils.py,sha256=n95XF8dZVbQRSXBZTGToXXfSs3JUVRyN6c3ZZ0nhAWI,1403
68
68
  dcicutils/trace_utils.py,sha256=g8kwV4ebEy5kXW6oOrEAUsurBcCROvwtZqz9fczsGRE,1769
69
69
  dcicutils/validation_utils.py,sha256=cMZIU2cY98FYtzK52z5WUYck7urH6JcqOuz9jkXpqzg,14797
70
70
  dcicutils/variant_utils.py,sha256=2H9azNx3xAj-MySg-uZ2SFqbWs4kZvf61JnK6b-h4Qw,4343
71
71
  dcicutils/zip_utils.py,sha256=rnjNv_k6L9jT2SjDSgVXp4BEJYLtz9XN6Cl2Fy-tqnM,2027
72
- dcicutils-8.8.0.1b14.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
73
- dcicutils-8.8.0.1b14.dist-info/METADATA,sha256=wESy8_hN46GF7JnZwBkIFBvZc3W8GDACj99M-X09ULA,3357
74
- dcicutils-8.8.0.1b14.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
75
- dcicutils-8.8.0.1b14.dist-info/entry_points.txt,sha256=51Q4F_2V10L0282W7HFjP4jdzW4K8lnWDARJQVFy_hw,270
76
- dcicutils-8.8.0.1b14.dist-info/RECORD,,
72
+ dcicutils-8.8.0.1b16.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
73
+ dcicutils-8.8.0.1b16.dist-info/METADATA,sha256=eNewQj8kA_d5SZ89aKo4bOA1rdT64sUMNQKWRy-dHj8,3357
74
+ dcicutils-8.8.0.1b16.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
75
+ dcicutils-8.8.0.1b16.dist-info/entry_points.txt,sha256=51Q4F_2V10L0282W7HFjP4jdzW4K8lnWDARJQVFy_hw,270
76
+ dcicutils-8.8.0.1b16.dist-info/RECORD,,