dcicutils 8.8.0.1b14__py3-none-any.whl → 8.8.0.1b16__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dcicutils/structured_data.py +201 -144
- {dcicutils-8.8.0.1b14.dist-info → dcicutils-8.8.0.1b16.dist-info}/METADATA +1 -1
- {dcicutils-8.8.0.1b14.dist-info → dcicutils-8.8.0.1b16.dist-info}/RECORD +6 -6
- {dcicutils-8.8.0.1b14.dist-info → dcicutils-8.8.0.1b16.dist-info}/LICENSE.txt +0 -0
- {dcicutils-8.8.0.1b14.dist-info → dcicutils-8.8.0.1b16.dist-info}/WHEEL +0 -0
- {dcicutils-8.8.0.1b14.dist-info → dcicutils-8.8.0.1b16.dist-info}/entry_points.txt +0 -0
dcicutils/structured_data.py
CHANGED
@@ -7,7 +7,6 @@ from pyramid.router import Router
|
|
7
7
|
import re
|
8
8
|
import sys
|
9
9
|
import time
|
10
|
-
from tqdm import tqdm
|
11
10
|
from typing import Any, Callable, List, Optional, Tuple, Type, Union
|
12
11
|
from webtest.app import TestApp
|
13
12
|
from dcicutils.common import OrchestratedApp
|
@@ -70,10 +69,9 @@ class StructuredDataSet:
|
|
70
69
|
order: Optional[List[str]] = None, prune: bool = True,
|
71
70
|
ref_lookup_strategy: Optional[Callable] = None,
|
72
71
|
ref_lookup_nocache: bool = False,
|
73
|
-
progress:
|
72
|
+
progress: Optional[Callable] = None,
|
74
73
|
debug_sleep: Optional[str] = None) -> None:
|
75
|
-
|
76
|
-
self._progress = progress
|
74
|
+
self._progress = progress if callable(progress) else None
|
77
75
|
self._data = {}
|
78
76
|
self._portal = Portal(portal, data=self._data, schemas=schemas,
|
79
77
|
ref_lookup_strategy=ref_lookup_strategy,
|
@@ -93,20 +91,15 @@ class StructuredDataSet:
|
|
93
91
|
self._debug_sleep = None
|
94
92
|
self._load_file(file) if file else None
|
95
93
|
|
96
|
-
def
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
elif isinstance(self._progress, tqdm):
|
106
|
-
if amount > 0:
|
107
|
-
self._progress.total += amount
|
108
|
-
elif amount < 0:
|
109
|
-
self._progress.update(-amount)
|
94
|
+
def _progress_update(self, nrows: Union[int, Callable],
|
95
|
+
nrefs_resolved: Optional[int] = None,
|
96
|
+
nrefs_unresolved: Optional[int] = None,
|
97
|
+
nlookups: Optional[int] = None) -> None:
|
98
|
+
if self._progress:
|
99
|
+
if callable(nrows):
|
100
|
+
nrows = nrows()
|
101
|
+
if isinstance(nrows, int) and nrows != 0:
|
102
|
+
self._progress(nrows, nrefs_resolved, nrefs_unresolved, nlookups)
|
110
103
|
|
111
104
|
@property
|
112
105
|
def data(self) -> dict:
|
@@ -259,7 +252,8 @@ class StructuredDataSet:
|
|
259
252
|
for row in excel.sheet_reader(sheet_name):
|
260
253
|
nrows += 1
|
261
254
|
return nrows
|
262
|
-
self.
|
255
|
+
if self._progress:
|
256
|
+
self._progress_update(calculate_total_rows_to_process)
|
263
257
|
excel = Excel(file) # Order the sheet names by any specified ordering (e.g. ala snovault.loadxl).
|
264
258
|
order = {Schema.type_name(key): index for index, key in enumerate(self._order)} if self._order else {}
|
265
259
|
for sheet_name in sorted(excel.sheet_names, key=lambda key: order.get(Schema.type_name(key), sys.maxsize)):
|
@@ -270,9 +264,10 @@ class StructuredDataSet:
|
|
270
264
|
ref_errors_actual = []
|
271
265
|
for ref_error in ref_errors:
|
272
266
|
if not (resolved := self.portal.ref_exists(ref := ref_error["error"])):
|
267
|
+
# if not (resolved := self.portal.ref_exists_internally(ref := ref_error["error"])):
|
273
268
|
ref_errors_actual.append(ref_error)
|
274
269
|
else:
|
275
|
-
self._resolved_refs.add((ref, resolved
|
270
|
+
self._resolved_refs.add((ref, resolved.get("uuid")))
|
276
271
|
if ref_errors_actual:
|
277
272
|
self._errors["ref"] = ref_errors_actual
|
278
273
|
else:
|
@@ -301,7 +296,8 @@ class StructuredDataSet:
|
|
301
296
|
if self._autoadd_properties:
|
302
297
|
self._add_properties(structured_row, self._autoadd_properties, schema)
|
303
298
|
self._add(type_name, structured_row)
|
304
|
-
self.
|
299
|
+
if self._progress:
|
300
|
+
self._progress_update(-1, self.ref_total_count, self.ref_total_notfound_count, self.ref_lookup_count)
|
305
301
|
self._note_warning(reader.warnings, "reader")
|
306
302
|
if schema:
|
307
303
|
self._note_error(schema._unresolved_refs, "ref")
|
@@ -333,6 +329,18 @@ class StructuredDataSet:
|
|
333
329
|
def _is_ref_lookup_subtypes(ref_lookup_flags: int) -> bool:
|
334
330
|
return (ref_lookup_flags & StructuredDataSet.REF_LOOKUP_SUBTYPES) == StructuredDataSet.REF_LOOKUP_SUBTYPES
|
335
331
|
|
332
|
+
@property
|
333
|
+
def ref_total_count(self) -> int:
|
334
|
+
return self.portal.ref_total_count if self.portal else -1
|
335
|
+
|
336
|
+
@property
|
337
|
+
def ref_total_found_count(self) -> int:
|
338
|
+
return self.portal.ref_total_found_count if self.portal else -1
|
339
|
+
|
340
|
+
@property
|
341
|
+
def ref_total_notfound_count(self) -> int:
|
342
|
+
return self.portal.ref_total_notfound_count if self.portal else -1
|
343
|
+
|
336
344
|
@property
|
337
345
|
def ref_lookup_cache_hit_count(self) -> int:
|
338
346
|
return self.portal.ref_lookup_cache_hit_count if self.portal else -1
|
@@ -361,6 +369,10 @@ class StructuredDataSet:
|
|
361
369
|
def ref_exists_internal_count(self) -> int:
|
362
370
|
return self.portal.ref_exists_internal_count if self.portal else -1
|
363
371
|
|
372
|
+
@property
|
373
|
+
def ref_exists_external_count(self) -> int:
|
374
|
+
return self.portal.ref_exists_external_count if self.portal else -1
|
375
|
+
|
364
376
|
@property
|
365
377
|
def ref_exists_cache_hit_count(self) -> int:
|
366
378
|
return self.portal.ref_exists_cache_hit_count if self.portal else -1
|
@@ -600,18 +612,11 @@ class Schema(SchemaBase):
|
|
600
612
|
# TODO: If think we do have the column (and type?) name(s) originating the ref yes?
|
601
613
|
self._unresolved_refs.append({"src": src, "error": f"/{link_to}/<null>"})
|
602
614
|
elif portal:
|
603
|
-
if not (resolved := portal.ref_exists(link_to, value)):
|
615
|
+
if not (resolved := portal.ref_exists(link_to, value, True)):
|
604
616
|
self._unresolved_refs.append({"src": src, "error": f"/{link_to}/{value}"})
|
605
|
-
elif len(resolved) > 1:
|
606
|
-
# TODO: Don't think we need this anymore; see TODO on Portal.ref_exists.
|
607
|
-
self._unresolved_refs.append({
|
608
|
-
"src": src,
|
609
|
-
"error": f"/{link_to}/{value}",
|
610
|
-
"types": [resolved_ref["type"] for resolved_ref in resolved]})
|
611
617
|
else:
|
612
618
|
# A resolved-ref set value is a tuple of the reference path and its uuid.
|
613
|
-
self._resolved_refs.add((f"/{link_to}/{value}", resolved
|
614
|
-
# self._resolved_refs.add((f"/{link_to}/{value}", resolved[0].get("uuid"), resolved[0].get("data")))
|
619
|
+
self._resolved_refs.add((f"/{link_to}/{value}", resolved.get("uuid")))
|
615
620
|
return value
|
616
621
|
return lambda value, src: map_ref(value, typeinfo.get("linkTo"), self._portal, src)
|
617
622
|
|
@@ -772,24 +777,28 @@ class Portal(PortalBase):
|
|
772
777
|
else:
|
773
778
|
self._ref_lookup_strategy = lambda type_name, schema, value: (StructuredDataSet.REF_LOOKUP_DEFAULT, None)
|
774
779
|
if ref_lookup_nocache is True:
|
775
|
-
self.ref_lookup = self.
|
780
|
+
self.ref_lookup = self.ref_lookup_uncached
|
776
781
|
self._ref_cache = None
|
777
782
|
else:
|
778
|
-
self.ref_lookup = self.
|
783
|
+
self.ref_lookup = self.ref_lookup_cached
|
779
784
|
self._ref_cache = {}
|
780
785
|
self._ref_lookup_found_count = 0
|
781
786
|
self._ref_lookup_notfound_count = 0
|
782
787
|
self._ref_lookup_error_count = 0
|
783
788
|
self._ref_exists_internal_count = 0
|
789
|
+
self._ref_exists_external_count = 0
|
784
790
|
self._ref_exists_cache_hit_count = 0
|
785
791
|
self._ref_exists_cache_miss_count = 0
|
786
792
|
self._ref_incorrect_identifying_property_count = 0
|
793
|
+
self._ref_total_count = 0
|
794
|
+
self._ref_total_found_count = 0
|
795
|
+
self._ref_total_notfound_count = 0
|
787
796
|
|
788
797
|
@lru_cache(maxsize=8092)
|
789
|
-
def
|
790
|
-
return self.
|
798
|
+
def ref_lookup_cached(self, object_name: str) -> Optional[dict]:
|
799
|
+
return self.ref_lookup_uncached(object_name)
|
791
800
|
|
792
|
-
def
|
801
|
+
def ref_lookup_uncached(self, object_name: str) -> Optional[dict]:
|
793
802
|
try:
|
794
803
|
result = super().get_metadata(object_name, raw=True)
|
795
804
|
self._ref_lookup_found_count += 1
|
@@ -824,10 +833,10 @@ class Portal(PortalBase):
|
|
824
833
|
return schemas
|
825
834
|
|
826
835
|
@lru_cache(maxsize=64)
|
827
|
-
def
|
836
|
+
def _get_schema_subtypes_names(self, type_name: str) -> List[str]:
|
828
837
|
if not (schemas_super_type_map := self.get_schemas_super_type_map()):
|
829
838
|
return []
|
830
|
-
return schemas_super_type_map.get(type_name)
|
839
|
+
return schemas_super_type_map.get(type_name, [])
|
831
840
|
|
832
841
|
def is_file_schema(self, schema_name: str) -> bool:
|
833
842
|
"""
|
@@ -835,74 +844,47 @@ class Portal(PortalBase):
|
|
835
844
|
"""
|
836
845
|
return self.is_schema_type(schema_name, FILE_SCHEMA_NAME)
|
837
846
|
|
838
|
-
def
|
839
|
-
|
840
|
-
|
841
|
-
return None
|
842
|
-
|
843
|
-
def _cache_ref(self, type_name: str, value: str, resolved: List[str], subtype_names: Optional[List[str]]) -> None:
|
844
|
-
if self._ref_cache is not None:
|
845
|
-
for type_name in [type_name] + (subtype_names if subtype_names else []):
|
846
|
-
self._ref_cache[f"/{type_name}/{value}"] = resolved
|
847
|
-
|
848
|
-
def ref_exists(self, type_name: str, value: Optional[str] = None) -> List[dict]:
|
847
|
+
def ref_exists(self, type_name: str, value: Optional[str] = None,
|
848
|
+
called_from_map_ref: bool = False) -> Optional[dict]:
|
849
|
+
# print(f"\033[Kxyzzy:ref_exists({type_name}/{value})")
|
849
850
|
if not value:
|
850
|
-
|
851
|
-
|
852
|
-
|
853
|
-
|
854
|
-
|
851
|
+
type_name, value = Portal._get_type_name_and_value_from_path(type_name)
|
852
|
+
if not type_name or not value:
|
853
|
+
return None
|
854
|
+
if called_from_map_ref:
|
855
|
+
self._ref_total_count += 1
|
856
|
+
# First check our reference cache.
|
855
857
|
if (resolved := self._ref_exists_from_cache(type_name, value)) is not None:
|
856
|
-
# Found
|
857
|
-
if
|
858
|
-
#
|
859
|
-
|
860
|
-
|
861
|
-
|
862
|
-
|
863
|
-
|
864
|
-
|
865
|
-
|
866
|
-
|
867
|
-
|
868
|
-
|
869
|
-
|
870
|
-
|
871
|
-
|
872
|
-
|
873
|
-
|
874
|
-
|
875
|
-
|
876
|
-
self._cache_ref(type_name, value, resolved, subtype_names)
|
877
|
-
return resolved
|
878
|
-
self._ref_exists_cache_hit_count += 1
|
858
|
+
# Found CACHED reference.
|
859
|
+
if resolved:
|
860
|
+
# Found cached RESOLVED reference (non-empty object).
|
861
|
+
if called_from_map_ref:
|
862
|
+
self._ref_total_found_count += 1
|
863
|
+
return resolved
|
864
|
+
# Found cached UNRESOLVED reference (empty object); meaning it was looked
|
865
|
+
# up but not found. It might NOW be found INTERNALLY, since the portal
|
866
|
+
# self._data can change, i.e. as data (e.g. spreadsheet sheets) are parsed.
|
867
|
+
return self.ref_exists_internally(type_name, value, update_counts=called_from_map_ref) or {}
|
868
|
+
# Reference is NOT cached here; lookup INTERNALLY first.
|
869
|
+
if (resolved := self.ref_exists_internally(type_name, value, update_counts=called_from_map_ref)) is None:
|
870
|
+
# Reference was resolved (internally) INCORRECTLY.
|
871
|
+
if called_from_map_ref:
|
872
|
+
self._ref_total_notfound_count += 1
|
873
|
+
return None
|
874
|
+
if resolved:
|
875
|
+
# Reference was resolved internally.
|
876
|
+
if called_from_map_ref:
|
877
|
+
self._ref_total_found_count += 1
|
879
878
|
return resolved
|
880
|
-
#
|
881
|
-
self._ref_exists_cache_miss_count += 1
|
879
|
+
# Reference is NOT cached and was NOT resolved internally; lookup in PORTAL.
|
882
880
|
# Get the lookup strategy; i.e. should do we lookup by root path, and if so, should
|
883
881
|
# we do this first, and do we lookup by subtypes; by default we lookup by root path
|
884
|
-
# but not first, and we
|
885
|
-
ref_lookup_strategy,
|
886
|
-
self._ref_lookup_strategy(type_name, self.get_schema(type_name), value))
|
882
|
+
# but not first, and we also lookup by subtypes by default.
|
883
|
+
ref_lookup_strategy, _ = self._ref_lookup_strategy(type_name, self.get_schema(type_name), value)
|
887
884
|
is_ref_lookup_specified_type = StructuredDataSet._is_ref_lookup_specified_type(ref_lookup_strategy)
|
888
885
|
is_ref_lookup_root = StructuredDataSet._is_ref_lookup_root(ref_lookup_strategy)
|
889
886
|
is_ref_lookup_root_first = StructuredDataSet._is_ref_lookup_root_first(ref_lookup_strategy)
|
890
887
|
is_ref_lookup_subtypes = StructuredDataSet._is_ref_lookup_subtypes(ref_lookup_strategy)
|
891
|
-
subtype_names = self._get_schema_subtypes(type_name) if is_ref_lookup_subtypes else None
|
892
|
-
# Lookup internally first (including at subtypes if desired; root lookup not applicable here).
|
893
|
-
is_resolved, identifying_property, resolved_uuid = (
|
894
|
-
self._ref_exists_internally(type_name, value, subtype_names,
|
895
|
-
incorrect_identifying_property=incorrect_identifying_property))
|
896
|
-
if is_resolved:
|
897
|
-
if identifying_property == incorrect_identifying_property:
|
898
|
-
# Not REALLY resolved as it resolved to a property which is NOT an identifying
|
899
|
-
# property, but may be commonly mistaken for one (e.g. UnalignedReads.filename).
|
900
|
-
self._ref_incorrect_identifying_property_count += 1
|
901
|
-
return []
|
902
|
-
resolved = [{"type": type_name, "uuid": resolved_uuid}]
|
903
|
-
self._cache_ref(type_name, value, resolved, subtype_names)
|
904
|
-
return resolved
|
905
|
-
# Not found internally; perform actual portal lookup (including at root and subtypes if desired).
|
906
888
|
# First construct the list of lookup paths at which to look for the referenced item.
|
907
889
|
lookup_paths = []
|
908
890
|
if is_ref_lookup_root_first:
|
@@ -911,37 +893,81 @@ class Portal(PortalBase):
|
|
911
893
|
lookup_paths.append(f"/{type_name}/{value}")
|
912
894
|
if is_ref_lookup_root and not is_ref_lookup_root_first:
|
913
895
|
lookup_paths.append(f"/{value}")
|
914
|
-
if
|
915
|
-
|
916
|
-
|
896
|
+
subtype_names = self._get_schema_subtypes_names(type_name) if is_ref_lookup_subtypes else []
|
897
|
+
for subtype_name in subtype_names:
|
898
|
+
lookup_paths.append(f"/{subtype_name}/{value}")
|
917
899
|
if not lookup_paths:
|
918
900
|
# No (i.e. zero) lookup strategy means no ref lookup at all.
|
919
|
-
|
920
|
-
|
901
|
+
if called_from_map_ref:
|
902
|
+
self._ref_total_notfound_count += 1
|
903
|
+
return None
|
904
|
+
# Do the actual lookup in portal for each of the desired lookup paths.
|
921
905
|
for lookup_path in lookup_paths:
|
922
|
-
if isinstance(
|
923
|
-
resolved =
|
924
|
-
self._cache_ref(type_name, value, resolved
|
906
|
+
if isinstance(resolved_item := self.ref_lookup(lookup_path), dict):
|
907
|
+
resolved = {"type": type_name, "uuid": resolved_item.get("uuid", None)}
|
908
|
+
self._cache_ref(type_name, value, resolved)
|
909
|
+
self._ref_exists_external_count += 1
|
910
|
+
if called_from_map_ref:
|
911
|
+
self._ref_total_found_count += 1
|
925
912
|
return resolved
|
926
|
-
# Not found at all; note that we cache this (
|
927
|
-
self._cache_ref(type_name, value,
|
928
|
-
|
929
|
-
|
930
|
-
|
931
|
-
self, type_name: str, value: str,
|
932
|
-
subtype_names: Optional[List[str]] = None,
|
933
|
-
incorrect_identifying_property: Optional[str] = None) -> Tuple[bool, Optional[str], Optional[str]]:
|
934
|
-
for type_name in [type_name] + (subtype_names or []):
|
935
|
-
is_resolved, identifying_property, resolved_uuid = self._ref_exists_single_internally(
|
936
|
-
type_name, value, incorrect_identifying_property=incorrect_identifying_property)
|
937
|
-
if is_resolved:
|
938
|
-
return True, identifying_property, resolved_uuid
|
939
|
-
return False, None, None
|
913
|
+
# Not found at all; note that we cache this ({}) too; indicates lookup has been done.
|
914
|
+
self._cache_ref(type_name, value, {})
|
915
|
+
if called_from_map_ref:
|
916
|
+
self._ref_total_notfound_count += 1
|
917
|
+
return None
|
940
918
|
|
941
|
-
def
|
942
|
-
|
943
|
-
|
944
|
-
|
919
|
+
def ref_exists_internally(self, type_name: str, value: Optional[str] = None,
|
920
|
+
update_counts: bool = False) -> Optional[dict]:
|
921
|
+
"""
|
922
|
+
Looks up the given reference (type/value) internally (i.e. with this data parsed thus far).
|
923
|
+
If found then returns a list of a single dictionary containing the (given) type name and
|
924
|
+
the uuid (if any) of the resolved item. If not found then returns an empty list; however,
|
925
|
+
if not found, but found using an "incorrect" identifying property, then returns None.
|
926
|
+
"""
|
927
|
+
# print(f"\033[Kxyzzy:ref_exists_internally({type_name}/{value})")
|
928
|
+
if not value:
|
929
|
+
type_name, value = Portal._get_type_name_and_value_from_path(type_name)
|
930
|
+
if not type_name or not value:
|
931
|
+
return None
|
932
|
+
# Note that root lookup not applicable here.
|
933
|
+
ref_lookup_strategy, incorrect_identifying_property = (
|
934
|
+
self._ref_lookup_strategy(type_name, self.get_schema(type_name), value))
|
935
|
+
is_ref_lookup_subtypes = StructuredDataSet._is_ref_lookup_subtypes(ref_lookup_strategy)
|
936
|
+
subtype_names = self._get_schema_subtypes_names(type_name) if is_ref_lookup_subtypes else []
|
937
|
+
for type_name in [type_name] + subtype_names:
|
938
|
+
is_resolved, resolved_item = self._ref_exists_single_internally(type_name, value)
|
939
|
+
if is_resolved:
|
940
|
+
if update_counts:
|
941
|
+
self._ref_exists_internal_count += 1
|
942
|
+
self._ref_total_found_count += 1
|
943
|
+
resolved = {"type": type_name, "uuid": resolved_item.get("uuid")}
|
944
|
+
self._cache_ref(type_name, value, resolved)
|
945
|
+
return resolved
|
946
|
+
# Here this reference is not resolved internally; but let us check any specified incorrect
|
947
|
+
# property to see if it would have been resolved using that; for example, if we pretend that
|
948
|
+
# UnalignedReads.filename were an identifying property (which it is not), then we see if this
|
949
|
+
# reference, which would otherwise be unresolved, would be resolved; in which case we have an
|
950
|
+
# incorrect reference; doing this can cut down considerably on useless lookups (at least for
|
951
|
+
# a case from He Li, early March 2024).
|
952
|
+
for type_name in [type_name] + subtype_names:
|
953
|
+
if incorrect_identifying_property:
|
954
|
+
if self._data and (items := self._data.get(type_name)):
|
955
|
+
for item in items:
|
956
|
+
if (identifying_value := item.get(incorrect_identifying_property, None)) is not None:
|
957
|
+
if ((identifying_value == value) or
|
958
|
+
(isinstance(identifying_value, list) and (value in identifying_value))): # noqa
|
959
|
+
# Not REALLY resolved as it resolved to a property which is NOT an identifying
|
960
|
+
# property, but may be commonly mistaken for one (e.g. UnalignedReads.filename).
|
961
|
+
# Return value to prevent actual portal lookup from happening.
|
962
|
+
if update_counts:
|
963
|
+
self._ref_incorrect_identifying_property_count += 1
|
964
|
+
self._ref_total_notfound_count += 1
|
965
|
+
return None # None return means resolved internally incorrectly.
|
966
|
+
if update_counts:
|
967
|
+
self._ref_total_notfound_count += 1
|
968
|
+
return {} # Empty return means not resolved internally.
|
969
|
+
|
970
|
+
def _ref_exists_single_internally(self, type_name: str, value: str) -> Tuple[bool, Optional[dict]]:
|
945
971
|
if self._data and (items := self._data.get(type_name)) and (schema := self.get_schema(type_name)):
|
946
972
|
identifying_properties = set(schema.get("identifyingProperties", [])) | {"identifier", "uuid"}
|
947
973
|
for item in items:
|
@@ -949,32 +975,41 @@ class Portal(PortalBase):
|
|
949
975
|
if (identifying_value := item.get(identifying_property, None)) is not None:
|
950
976
|
if ((identifying_value == value) or
|
951
977
|
(isinstance(identifying_value, list) and (value in identifying_value))): # noqa
|
952
|
-
|
953
|
-
|
954
|
-
|
955
|
-
|
956
|
-
|
957
|
-
|
958
|
-
|
959
|
-
|
978
|
+
return True, item
|
979
|
+
return False, None
|
980
|
+
|
981
|
+
@staticmethod
|
982
|
+
def _get_type_name_and_value_from_path(path: str) -> Tuple[Optional[str], Optional[str]]:
|
983
|
+
if path.startswith("/") and len(parts := path[1:].split("/")) == 2:
|
984
|
+
if not (type_name := parts[0]) or not (value := parts[1]):
|
985
|
+
return None
|
986
|
+
return type_name, value
|
987
|
+
return None, None
|
988
|
+
|
989
|
+
def _ref_exists_from_cache(self, type_name: str, value: str) -> Optional[List[dict]]:
|
990
|
+
if self._ref_cache is not None:
|
991
|
+
self._ref_exists_cache_hit_count += 1
|
992
|
+
return self._ref_cache.get(f"/{type_name}/{value}", None)
|
993
|
+
self._ref_exists_cache_miss_count += 1
|
994
|
+
return None
|
995
|
+
|
996
|
+
def _cache_ref(self, type_name: str, value: str, resolved: List[str]) -> None:
|
997
|
+
subtype_names = self._get_schema_subtypes_names(type_name)
|
998
|
+
if self._ref_cache is not None:
|
999
|
+
for type_name in [type_name] + subtype_names:
|
1000
|
+
self._ref_cache[f"/{type_name}/{value}"] = resolved
|
960
1001
|
|
961
1002
|
@property
|
962
|
-
def
|
963
|
-
|
964
|
-
return 0
|
965
|
-
try:
|
966
|
-
return self.ref_lookup_cache.cache_info().hits
|
967
|
-
except Exception:
|
968
|
-
return -1
|
1003
|
+
def ref_total_count(self) -> int:
|
1004
|
+
return self._ref_total_count
|
969
1005
|
|
970
1006
|
@property
|
971
|
-
def
|
972
|
-
|
973
|
-
|
974
|
-
|
975
|
-
|
976
|
-
|
977
|
-
return -1
|
1007
|
+
def ref_total_found_count(self) -> int:
|
1008
|
+
return self._ref_total_found_count
|
1009
|
+
|
1010
|
+
@property
|
1011
|
+
def ref_total_notfound_count(self) -> int:
|
1012
|
+
return self._ref_total_notfound_count
|
978
1013
|
|
979
1014
|
@property
|
980
1015
|
def ref_lookup_count(self) -> int:
|
@@ -992,10 +1027,32 @@ class Portal(PortalBase):
|
|
992
1027
|
def ref_lookup_error_count(self) -> int:
|
993
1028
|
return self._ref_lookup_error_count
|
994
1029
|
|
1030
|
+
@property
|
1031
|
+
def ref_lookup_cache_hit_count(self) -> int:
|
1032
|
+
if self._ref_cache is None:
|
1033
|
+
return -1
|
1034
|
+
try:
|
1035
|
+
return self.ref_lookup_cached.cache_info().hits
|
1036
|
+
except Exception:
|
1037
|
+
return -1
|
1038
|
+
|
1039
|
+
@property
|
1040
|
+
def ref_lookup_cache_miss_count(self) -> int:
|
1041
|
+
if self._ref_cache is None:
|
1042
|
+
return -1
|
1043
|
+
try:
|
1044
|
+
return self.ref_lookup_cached.cache_info().misses
|
1045
|
+
except Exception:
|
1046
|
+
return -1
|
1047
|
+
|
995
1048
|
@property
|
996
1049
|
def ref_exists_internal_count(self) -> int:
|
997
1050
|
return self._ref_exists_internal_count
|
998
1051
|
|
1052
|
+
@property
|
1053
|
+
def ref_exists_external_count(self) -> int:
|
1054
|
+
return self._ref_exists_external_count
|
1055
|
+
|
999
1056
|
@property
|
1000
1057
|
def ref_exists_cache_hit_count(self) -> int:
|
1001
1058
|
return self._ref_exists_cache_hit_count
|
@@ -62,15 +62,15 @@ dcicutils/secrets_utils.py,sha256=8dppXAsiHhJzI6NmOcvJV5ldvKkQZzh3Fl-cb8Wm7MI,19
|
|
62
62
|
dcicutils/sheet_utils.py,sha256=VlmzteONW5VF_Q4vo0yA5vesz1ViUah1MZ_yA1rwZ0M,33629
|
63
63
|
dcicutils/snapshot_utils.py,sha256=ymP7PXH6-yEiXAt75w0ldQFciGNqWBClNxC5gfX2FnY,22961
|
64
64
|
dcicutils/ssl_certificate_utils.py,sha256=F0ifz_wnRRN9dfrfsz7aCp4UDLgHEY8LaK7PjnNvrAQ,9707
|
65
|
-
dcicutils/structured_data.py,sha256=
|
65
|
+
dcicutils/structured_data.py,sha256=QK04rpuoqyMv5NO-x4yFgCzHdZIGp3g1eR8XjNmJNvM,55302
|
66
66
|
dcicutils/task_utils.py,sha256=MF8ujmTD6-O2AC2gRGPHyGdUrVKgtr8epT5XU8WtNjk,8082
|
67
67
|
dcicutils/tmpfile_utils.py,sha256=n95XF8dZVbQRSXBZTGToXXfSs3JUVRyN6c3ZZ0nhAWI,1403
|
68
68
|
dcicutils/trace_utils.py,sha256=g8kwV4ebEy5kXW6oOrEAUsurBcCROvwtZqz9fczsGRE,1769
|
69
69
|
dcicutils/validation_utils.py,sha256=cMZIU2cY98FYtzK52z5WUYck7urH6JcqOuz9jkXpqzg,14797
|
70
70
|
dcicutils/variant_utils.py,sha256=2H9azNx3xAj-MySg-uZ2SFqbWs4kZvf61JnK6b-h4Qw,4343
|
71
71
|
dcicutils/zip_utils.py,sha256=rnjNv_k6L9jT2SjDSgVXp4BEJYLtz9XN6Cl2Fy-tqnM,2027
|
72
|
-
dcicutils-8.8.0.
|
73
|
-
dcicutils-8.8.0.
|
74
|
-
dcicutils-8.8.0.
|
75
|
-
dcicutils-8.8.0.
|
76
|
-
dcicutils-8.8.0.
|
72
|
+
dcicutils-8.8.0.1b16.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
|
73
|
+
dcicutils-8.8.0.1b16.dist-info/METADATA,sha256=eNewQj8kA_d5SZ89aKo4bOA1rdT64sUMNQKWRy-dHj8,3357
|
74
|
+
dcicutils-8.8.0.1b16.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
|
75
|
+
dcicutils-8.8.0.1b16.dist-info/entry_points.txt,sha256=51Q4F_2V10L0282W7HFjP4jdzW4K8lnWDARJQVFy_hw,270
|
76
|
+
dcicutils-8.8.0.1b16.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|