dcicutils 8.8.1.1b15__py3-none-any.whl → 8.8.1.1b17__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dcicutils/progress_constants.py +10 -4
- dcicutils/structured_data.py +26 -26
- {dcicutils-8.8.1.1b15.dist-info → dcicutils-8.8.1.1b17.dist-info}/METADATA +1 -1
- {dcicutils-8.8.1.1b15.dist-info → dcicutils-8.8.1.1b17.dist-info}/RECORD +7 -7
- {dcicutils-8.8.1.1b15.dist-info → dcicutils-8.8.1.1b17.dist-info}/LICENSE.txt +0 -0
- {dcicutils-8.8.1.1b15.dist-info → dcicutils-8.8.1.1b17.dist-info}/WHEEL +0 -0
- {dcicutils-8.8.1.1b15.dist-info → dcicutils-8.8.1.1b17.dist-info}/entry_points.txt +0 -0
dcicutils/progress_constants.py
CHANGED
|
@@ -1,11 +1,17 @@
|
|
|
1
1
|
from datetime import datetime
|
|
2
2
|
from enum import Enum
|
|
3
3
|
|
|
4
|
-
|
|
5
4
|
# Constants for progress tracking for smaht-submitr.
|
|
6
5
|
# Here only to share between smaht-portal, snovault, and smaht-submitr.
|
|
7
6
|
|
|
8
|
-
|
|
7
|
+
|
|
8
|
+
class _Enum(Enum):
|
|
9
|
+
# Automatically make enumerators within the enumeration resolve to its value property.
|
|
10
|
+
def __get__(self, instance, owner):
|
|
11
|
+
return self.value
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class PROGRESS_INGESTER(_Enum):
|
|
9
15
|
VALIDATION = "ingester_validation"
|
|
10
16
|
INITIATE = "ingester_initiate"
|
|
11
17
|
PARSE_LOAD_INITIATE = "ingester_parse_initiate"
|
|
@@ -20,7 +26,7 @@ class PROGRESS_INGESTER(Enum):
|
|
|
20
26
|
NOW = lambda: str(datetime.utcnow()) # noqa
|
|
21
27
|
|
|
22
28
|
|
|
23
|
-
class PROGRESS_PARSE(
|
|
29
|
+
class PROGRESS_PARSE(_Enum):
|
|
24
30
|
LOAD_START = "parse_start"
|
|
25
31
|
LOAD_ITEM = "parse_item"
|
|
26
32
|
LOAD_DONE = "parse_done"
|
|
@@ -46,7 +52,7 @@ class PROGRESS_PARSE(Enum):
|
|
|
46
52
|
NOW = lambda: str(datetime.utcnow()) # noqa
|
|
47
53
|
|
|
48
54
|
|
|
49
|
-
class PROGRESS_LOADXL(
|
|
55
|
+
class PROGRESS_LOADXL(_Enum):
|
|
50
56
|
START = "loadxl_start"
|
|
51
57
|
START_SECOND_ROUND = "loadxl_start_second_round"
|
|
52
58
|
ITEM = "loadxl_item"
|
dcicutils/structured_data.py
CHANGED
|
@@ -192,8 +192,8 @@ class StructuredDataSet:
|
|
|
192
192
|
diffs = {}
|
|
193
193
|
if callable(progress):
|
|
194
194
|
ntypes, nobjects = get_counts()
|
|
195
|
-
progress({PROGRESS.ANALYZE_START
|
|
196
|
-
PROGRESS.ANALYZE_COUNT_TYPES
|
|
195
|
+
progress({PROGRESS.ANALYZE_START: PROGRESS.NOW(),
|
|
196
|
+
PROGRESS.ANALYZE_COUNT_TYPES: ntypes, PROGRESS.ANALYZE_COUNT_ITEMS: nobjects})
|
|
197
197
|
if self.data or self.portal: # TODO: what is this OR biz?
|
|
198
198
|
refs = self.resolved_refs_with_uuids
|
|
199
199
|
# TODO: Need feedback/progress tracking mechanism here.
|
|
@@ -212,20 +212,20 @@ class StructuredDataSet:
|
|
|
212
212
|
uuid=existing_object.uuid,
|
|
213
213
|
diffs=object_diffs or None))
|
|
214
214
|
if callable(progress):
|
|
215
|
-
progress({PROGRESS.ANALYZE_UPDATE
|
|
216
|
-
PROGRESS.ANALYZE_COUNT_LOOKUP
|
|
215
|
+
progress({PROGRESS.ANALYZE_UPDATE: True,
|
|
216
|
+
PROGRESS.ANALYZE_COUNT_LOOKUP: nlookups + nlookups_compare})
|
|
217
217
|
elif identifying_path:
|
|
218
218
|
# If there is no existing object we still create a record for this object
|
|
219
219
|
# but with no uuid which will be the indication that it does not exist.
|
|
220
220
|
diffs[type_name].append(create_readonly_object(path=identifying_path, uuid=None, diffs=None))
|
|
221
221
|
if callable(progress):
|
|
222
|
-
progress({PROGRESS.ANALYZE_CREATE
|
|
223
|
-
PROGRESS.ANALYZE_COUNT_LOOKUP
|
|
222
|
+
progress({PROGRESS.ANALYZE_CREATE: True,
|
|
223
|
+
PROGRESS.ANALYZE_COUNT_LOOKUP: nlookups})
|
|
224
224
|
else:
|
|
225
225
|
if callable(progress):
|
|
226
|
-
progress({PROGRESS.ANALYZE_COUNT_LOOKUP
|
|
226
|
+
progress({PROGRESS.ANALYZE_COUNT_LOOKUP: nlookups})
|
|
227
227
|
if callable(progress):
|
|
228
|
-
progress({PROGRESS.ANALYZE_DONE
|
|
228
|
+
progress({PROGRESS.ANALYZE_DONE: PROGRESS.NOW()})
|
|
229
229
|
return diffs
|
|
230
230
|
|
|
231
231
|
def load_file(self, file: str) -> None:
|
|
@@ -272,8 +272,8 @@ class StructuredDataSet:
|
|
|
272
272
|
return nrows, len(excel.sheet_names)
|
|
273
273
|
if self._progress: # TODO: Move to _load_reader
|
|
274
274
|
nrows, nsheets = get_counts()
|
|
275
|
-
self._progress({PROGRESS.LOAD_START
|
|
276
|
-
PROGRESS.LOAD_COUNT_SHEETS
|
|
275
|
+
self._progress({PROGRESS.LOAD_START: PROGRESS.NOW(),
|
|
276
|
+
PROGRESS.LOAD_COUNT_SHEETS: nsheets, PROGRESS.LOAD_COUNT_ROWS: nrows})
|
|
277
277
|
excel = Excel(file) # Order the sheet names by any specified ordering (e.g. ala snovault.loadxl).
|
|
278
278
|
order = {Schema.type_name(key): index for index, key in enumerate(self._order)} if self._order else {}
|
|
279
279
|
for sheet_name in sorted(excel.sheet_names, key=lambda key: order.get(Schema.type_name(key), sys.maxsize)):
|
|
@@ -298,14 +298,14 @@ class StructuredDataSet:
|
|
|
298
298
|
del self._errors["ref"]
|
|
299
299
|
if self._progress:
|
|
300
300
|
self._progress({ # TODO: Refactor with same thing below in _load_reader.
|
|
301
|
-
PROGRESS.LOAD_DONE
|
|
302
|
-
PROGRESS.LOAD_COUNT_REFS
|
|
303
|
-
PROGRESS.LOAD_COUNT_REFS_FOUND
|
|
304
|
-
PROGRESS.LOAD_COUNT_REFS_NOT_FOUND
|
|
305
|
-
PROGRESS.LOAD_COUNT_REFS_LOOKUP
|
|
306
|
-
PROGRESS.LOAD_COUNT_REFS_LOOKUP_CACHE_HIT
|
|
307
|
-
PROGRESS.LOAD_COUNT_REFS_EXISTS_CACHE_HIT
|
|
308
|
-
PROGRESS.LOAD_COUNT_REFS_INVALID
|
|
301
|
+
PROGRESS.LOAD_DONE: PROGRESS.NOW(),
|
|
302
|
+
PROGRESS.LOAD_COUNT_REFS: self.ref_total_count,
|
|
303
|
+
PROGRESS.LOAD_COUNT_REFS_FOUND: self.ref_total_found_count,
|
|
304
|
+
PROGRESS.LOAD_COUNT_REFS_NOT_FOUND: self.ref_total_notfound_count,
|
|
305
|
+
PROGRESS.LOAD_COUNT_REFS_LOOKUP: self.ref_lookup_count,
|
|
306
|
+
PROGRESS.LOAD_COUNT_REFS_LOOKUP_CACHE_HIT: self.ref_lookup_cache_hit_count,
|
|
307
|
+
PROGRESS.LOAD_COUNT_REFS_EXISTS_CACHE_HIT: self.ref_exists_cache_hit_count,
|
|
308
|
+
PROGRESS.LOAD_COUNT_REFS_INVALID: self.ref_invalid_identifying_property_count
|
|
309
309
|
})
|
|
310
310
|
|
|
311
311
|
def _load_json_file(self, file: str) -> None:
|
|
@@ -335,14 +335,14 @@ class StructuredDataSet:
|
|
|
335
335
|
self._add(type_name, structured_row)
|
|
336
336
|
if self._progress:
|
|
337
337
|
self._progress({
|
|
338
|
-
PROGRESS.LOAD_ITEM
|
|
339
|
-
PROGRESS.LOAD_COUNT_REFS
|
|
340
|
-
PROGRESS.LOAD_COUNT_REFS_FOUND
|
|
341
|
-
PROGRESS.LOAD_COUNT_REFS_NOT_FOUND
|
|
342
|
-
PROGRESS.LOAD_COUNT_REFS_LOOKUP
|
|
343
|
-
PROGRESS.LOAD_COUNT_REFS_LOOKUP_CACHE_HIT
|
|
344
|
-
PROGRESS.LOAD_COUNT_REFS_EXISTS_CACHE_HIT
|
|
345
|
-
PROGRESS.LOAD_COUNT_REFS_INVALID
|
|
338
|
+
PROGRESS.LOAD_ITEM: True,
|
|
339
|
+
PROGRESS.LOAD_COUNT_REFS: self.ref_total_count,
|
|
340
|
+
PROGRESS.LOAD_COUNT_REFS_FOUND: self.ref_total_found_count,
|
|
341
|
+
PROGRESS.LOAD_COUNT_REFS_NOT_FOUND: self.ref_total_notfound_count,
|
|
342
|
+
PROGRESS.LOAD_COUNT_REFS_LOOKUP: self.ref_lookup_count,
|
|
343
|
+
PROGRESS.LOAD_COUNT_REFS_LOOKUP_CACHE_HIT: self.ref_lookup_cache_hit_count,
|
|
344
|
+
PROGRESS.LOAD_COUNT_REFS_EXISTS_CACHE_HIT: self.ref_exists_cache_hit_count,
|
|
345
|
+
PROGRESS.LOAD_COUNT_REFS_INVALID: self.ref_invalid_identifying_property_count
|
|
346
346
|
})
|
|
347
347
|
self._note_warning(reader.warnings, "reader")
|
|
348
348
|
if schema:
|
|
@@ -48,7 +48,7 @@ dcicutils/obfuscation_utils.py,sha256=fo2jOmDRC6xWpYX49u80bVNisqRRoPskFNX3ymFAmj
|
|
|
48
48
|
dcicutils/opensearch_utils.py,sha256=V2exmFYW8Xl2_pGFixF4I2Cc549Opwe4PhFi5twC0M8,1017
|
|
49
49
|
dcicutils/portal_object_utils.py,sha256=MF6MTZ6yxakZFDjbkTKCsF4q4p11dLDVvT5JBV9m6RQ,15408
|
|
50
50
|
dcicutils/portal_utils.py,sha256=oBoI3KWRp6YrbsuVGbmPQ3kATB5cVVsQo7-qmnYXWqg,30260
|
|
51
|
-
dcicutils/progress_constants.py,sha256=
|
|
51
|
+
dcicutils/progress_constants.py,sha256=yHstS8vdYgZPxGJ_dk36Tu2Yzvl9G78Q4lNdz3_RAAw,2504
|
|
52
52
|
dcicutils/project_utils.py,sha256=qPdCaFmWUVBJw4rw342iUytwdQC0P-XKpK4mhyIulMM,31250
|
|
53
53
|
dcicutils/qa_checkers.py,sha256=cdXjeL0jCDFDLT8VR8Px78aS10hwNISOO5G_Zv2TZ6M,20534
|
|
54
54
|
dcicutils/qa_utils.py,sha256=TT0SiJWiuxYvbsIyhK9VO4uV_suxhB6CpuC4qPacCzQ,160208
|
|
@@ -63,15 +63,15 @@ dcicutils/secrets_utils.py,sha256=8dppXAsiHhJzI6NmOcvJV5ldvKkQZzh3Fl-cb8Wm7MI,19
|
|
|
63
63
|
dcicutils/sheet_utils.py,sha256=VlmzteONW5VF_Q4vo0yA5vesz1ViUah1MZ_yA1rwZ0M,33629
|
|
64
64
|
dcicutils/snapshot_utils.py,sha256=ymP7PXH6-yEiXAt75w0ldQFciGNqWBClNxC5gfX2FnY,22961
|
|
65
65
|
dcicutils/ssl_certificate_utils.py,sha256=F0ifz_wnRRN9dfrfsz7aCp4UDLgHEY8LaK7PjnNvrAQ,9707
|
|
66
|
-
dcicutils/structured_data.py,sha256=
|
|
66
|
+
dcicutils/structured_data.py,sha256=_AFFDColRikKygOTD5uk1SBj8tvv5dOr5UqicKxFZJo,58637
|
|
67
67
|
dcicutils/task_utils.py,sha256=MF8ujmTD6-O2AC2gRGPHyGdUrVKgtr8epT5XU8WtNjk,8082
|
|
68
68
|
dcicutils/tmpfile_utils.py,sha256=n95XF8dZVbQRSXBZTGToXXfSs3JUVRyN6c3ZZ0nhAWI,1403
|
|
69
69
|
dcicutils/trace_utils.py,sha256=g8kwV4ebEy5kXW6oOrEAUsurBcCROvwtZqz9fczsGRE,1769
|
|
70
70
|
dcicutils/validation_utils.py,sha256=cMZIU2cY98FYtzK52z5WUYck7urH6JcqOuz9jkXpqzg,14797
|
|
71
71
|
dcicutils/variant_utils.py,sha256=2H9azNx3xAj-MySg-uZ2SFqbWs4kZvf61JnK6b-h4Qw,4343
|
|
72
72
|
dcicutils/zip_utils.py,sha256=rnjNv_k6L9jT2SjDSgVXp4BEJYLtz9XN6Cl2Fy-tqnM,2027
|
|
73
|
-
dcicutils-8.8.1.
|
|
74
|
-
dcicutils-8.8.1.
|
|
75
|
-
dcicutils-8.8.1.
|
|
76
|
-
dcicutils-8.8.1.
|
|
77
|
-
dcicutils-8.8.1.
|
|
73
|
+
dcicutils-8.8.1.1b17.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
|
|
74
|
+
dcicutils-8.8.1.1b17.dist-info/METADATA,sha256=wtzaNYisXWNdxGFwEfC9fEOfG4WrZzG4R9EAKv6j90g,3357
|
|
75
|
+
dcicutils-8.8.1.1b17.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
|
|
76
|
+
dcicutils-8.8.1.1b17.dist-info/entry_points.txt,sha256=51Q4F_2V10L0282W7HFjP4jdzW4K8lnWDARJQVFy_hw,270
|
|
77
|
+
dcicutils-8.8.1.1b17.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|