dcicutils 8.8.1.1b10__py3-none-any.whl → 8.8.1.1b12__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dcicutils/progress_constants.py +25 -19
- dcicutils/structured_data.py +26 -25
- {dcicutils-8.8.1.1b10.dist-info → dcicutils-8.8.1.1b12.dist-info}/METADATA +1 -1
- {dcicutils-8.8.1.1b10.dist-info → dcicutils-8.8.1.1b12.dist-info}/RECORD +7 -7
- {dcicutils-8.8.1.1b10.dist-info → dcicutils-8.8.1.1b12.dist-info}/LICENSE.txt +0 -0
- {dcicutils-8.8.1.1b10.dist-info → dcicutils-8.8.1.1b12.dist-info}/WHEEL +0 -0
- {dcicutils-8.8.1.1b10.dist-info → dcicutils-8.8.1.1b12.dist-info}/entry_points.txt +0 -0
dcicutils/progress_constants.py
CHANGED
@@ -13,28 +13,34 @@ class PROGRESS_INGESTER(Enum):
|
|
13
13
|
VALIDATE_LOAD_DONE = "ingester_validate_done"
|
14
14
|
LOADXL_INITIATE = "ingester_loadxl_initiate"
|
15
15
|
LOADXL_DONE = "ingester_loadxl_done"
|
16
|
+
MESSAGE = "ingester_message"
|
17
|
+
MESSAGE_VERBOSE = "ingester_message_verbose"
|
18
|
+
MESSAGE_DEBUG = "ingester_message_debug"
|
16
19
|
|
17
20
|
|
18
21
|
class PROGRESS_PARSE(Enum):
|
19
|
-
LOAD_START = "
|
20
|
-
LOAD_ITEM = "
|
21
|
-
LOAD_DONE = "
|
22
|
-
LOAD_COUNT_SHEETS = "
|
23
|
-
LOAD_COUNT_ROWS = "
|
24
|
-
LOAD_COUNT_REFS = "
|
25
|
-
LOAD_COUNT_REFS_FOUND = "
|
26
|
-
LOAD_COUNT_REFS_NOT_FOUND = "
|
27
|
-
LOAD_COUNT_REFS_LOOKUP = "
|
28
|
-
LOAD_COUNT_REFS_LOOKUP_CACHE_HIT = "
|
29
|
-
LOAD_COUNT_REFS_EXISTS_CACHE_HIT = "
|
30
|
-
LOAD_COUNT_REFS_INVALID = "
|
31
|
-
ANALYZE_START = "
|
32
|
-
ANALYZE_COUNT_TYPES = "
|
33
|
-
ANALYZE_COUNT_ITEMS = "
|
34
|
-
|
35
|
-
|
36
|
-
ANALYZE_UPDATE = "
|
37
|
-
ANALYZE_DONE = "
|
22
|
+
LOAD_START = "parse_start"
|
23
|
+
LOAD_ITEM = "parse_item"
|
24
|
+
LOAD_DONE = "parse_done"
|
25
|
+
LOAD_COUNT_SHEETS = "parse_sheets"
|
26
|
+
LOAD_COUNT_ROWS = "parse_rows"
|
27
|
+
LOAD_COUNT_REFS = "parse_refs"
|
28
|
+
LOAD_COUNT_REFS_FOUND = "parse_refs_found"
|
29
|
+
LOAD_COUNT_REFS_NOT_FOUND = "parse_refs_not_found"
|
30
|
+
LOAD_COUNT_REFS_LOOKUP = "parse_refs_lookup"
|
31
|
+
LOAD_COUNT_REFS_LOOKUP_CACHE_HIT = "parse_refs_lookup_cache_hit"
|
32
|
+
LOAD_COUNT_REFS_EXISTS_CACHE_HIT = "parse_refs_exists_cache_hit"
|
33
|
+
LOAD_COUNT_REFS_INVALID = "parse_refs_invalid"
|
34
|
+
ANALYZE_START = "parse_analyze_start"
|
35
|
+
ANALYZE_COUNT_TYPES = "parse_analyze_types"
|
36
|
+
ANALYZE_COUNT_ITEMS = "parse_analyze_objects"
|
37
|
+
ANALYZE_COUNT_LOOKUP = "parse_analyze_lookups"
|
38
|
+
ANALYZE_CREATE = "parse_analyze_create"
|
39
|
+
ANALYZE_UPDATE = "parse_analyze_update"
|
40
|
+
ANALYZE_DONE = "parse_analyze_done"
|
41
|
+
MESSAGE = "parse_message"
|
42
|
+
MESSAGE_VERBOSE = "parse_message_verbose"
|
43
|
+
MESSAGE_DEBUG = "parse_message_debug"
|
38
44
|
|
39
45
|
|
40
46
|
class PROGRESS_LOADXL(Enum):
|
dcicutils/structured_data.py
CHANGED
@@ -192,8 +192,8 @@ class StructuredDataSet:
|
|
192
192
|
diffs = {}
|
193
193
|
if callable(progress):
|
194
194
|
ntypes, nobjects = get_counts()
|
195
|
-
progress({PROGRESS.ANALYZE_START: True,
|
196
|
-
PROGRESS.ANALYZE_COUNT_TYPES: ntypes, PROGRESS.ANALYZE_COUNT_ITEMS: nobjects})
|
195
|
+
progress({PROGRESS.ANALYZE_START.value: True,
|
196
|
+
PROGRESS.ANALYZE_COUNT_TYPES.value: ntypes, PROGRESS.ANALYZE_COUNT_ITEMS.value: nobjects})
|
197
197
|
if self.data or self.portal: # TODO: what is this OR biz?
|
198
198
|
refs = self.resolved_refs_with_uuids
|
199
199
|
# TODO: Need feedback/progress tracking mechanism here.
|
@@ -212,19 +212,20 @@ class StructuredDataSet:
|
|
212
212
|
uuid=existing_object.uuid,
|
213
213
|
diffs=object_diffs or None))
|
214
214
|
if callable(progress):
|
215
|
-
progress({PROGRESS.ANALYZE_UPDATE: True,
|
216
|
-
PROGRESS.ANALYZE_COUNT_LOOKUP: nlookups + nlookups_compare})
|
215
|
+
progress({PROGRESS.ANALYZE_UPDATE.value: True,
|
216
|
+
PROGRESS.ANALYZE_COUNT_LOOKUP.value: nlookups + nlookups_compare})
|
217
217
|
elif identifying_path:
|
218
218
|
# If there is no existing object we still create a record for this object
|
219
219
|
# but with no uuid which will be the indication that it does not exist.
|
220
220
|
diffs[type_name].append(create_readonly_object(path=identifying_path, uuid=None, diffs=None))
|
221
221
|
if callable(progress):
|
222
|
-
progress({PROGRESS.ANALYZE_CREATE: True,
|
222
|
+
progress({PROGRESS.ANALYZE_CREATE.value: True,
|
223
|
+
PROGRESS.ANALYZE_COUNT_LOOKUP.value: nlookups})
|
223
224
|
else:
|
224
225
|
if callable(progress):
|
225
|
-
progress({PROGRESS.ANALYZE_COUNT_LOOKUP: nlookups})
|
226
|
+
progress({PROGRESS.ANALYZE_COUNT_LOOKUP.value: nlookups})
|
226
227
|
if callable(progress):
|
227
|
-
progress({PROGRESS.ANALYZE_DONE: True})
|
228
|
+
progress({PROGRESS.ANALYZE_DONE.value: True})
|
228
229
|
return diffs
|
229
230
|
|
230
231
|
def load_file(self, file: str) -> None:
|
@@ -271,8 +272,8 @@ class StructuredDataSet:
|
|
271
272
|
return nrows, len(excel.sheet_names)
|
272
273
|
if self._progress: # TODO: Move to _load_reader
|
273
274
|
nrows, nsheets = get_counts()
|
274
|
-
self._progress({PROGRESS.LOAD_START: True,
|
275
|
-
PROGRESS.LOAD_COUNT_SHEETS: nsheets, PROGRESS.LOAD_COUNT_ROWS: nrows})
|
275
|
+
self._progress({PROGRESS.LOAD_START.value: True,
|
276
|
+
PROGRESS.LOAD_COUNT_SHEETS.value: nsheets, PROGRESS.LOAD_COUNT_ROWS.value: nrows})
|
276
277
|
excel = Excel(file) # Order the sheet names by any specified ordering (e.g. ala snovault.loadxl).
|
277
278
|
order = {Schema.type_name(key): index for index, key in enumerate(self._order)} if self._order else {}
|
278
279
|
for sheet_name in sorted(excel.sheet_names, key=lambda key: order.get(Schema.type_name(key), sys.maxsize)):
|
@@ -297,14 +298,14 @@ class StructuredDataSet:
|
|
297
298
|
del self._errors["ref"]
|
298
299
|
if self._progress:
|
299
300
|
self._progress({ # TODO: Refactor with same thing below in _load_reader.
|
300
|
-
PROGRESS.LOAD_DONE: True,
|
301
|
-
PROGRESS.LOAD_COUNT_REFS: self.ref_total_count,
|
302
|
-
PROGRESS.LOAD_COUNT_REFS_FOUND: self.ref_total_found_count,
|
303
|
-
PROGRESS.LOAD_COUNT_REFS_NOT_FOUND: self.ref_total_notfound_count,
|
304
|
-
PROGRESS.LOAD_COUNT_REFS_LOOKUP: self.ref_lookup_count,
|
305
|
-
PROGRESS.LOAD_COUNT_REFS_LOOKUP_CACHE_HIT: self.ref_lookup_cache_hit_count,
|
306
|
-
PROGRESS.LOAD_COUNT_REFS_EXISTS_CACHE_HIT: self.ref_exists_cache_hit_count,
|
307
|
-
PROGRESS.LOAD_COUNT_REFS_INVALID: self.ref_invalid_identifying_property_count
|
301
|
+
PROGRESS.LOAD_DONE.value: True,
|
302
|
+
PROGRESS.LOAD_COUNT_REFS.value: self.ref_total_count,
|
303
|
+
PROGRESS.LOAD_COUNT_REFS_FOUND.value: self.ref_total_found_count,
|
304
|
+
PROGRESS.LOAD_COUNT_REFS_NOT_FOUND.value: self.ref_total_notfound_count,
|
305
|
+
PROGRESS.LOAD_COUNT_REFS_LOOKUP.value: self.ref_lookup_count,
|
306
|
+
PROGRESS.LOAD_COUNT_REFS_LOOKUP_CACHE_HIT.value: self.ref_lookup_cache_hit_count,
|
307
|
+
PROGRESS.LOAD_COUNT_REFS_EXISTS_CACHE_HIT.value: self.ref_exists_cache_hit_count,
|
308
|
+
PROGRESS.LOAD_COUNT_REFS_INVALID.value: self.ref_invalid_identifying_property_count
|
308
309
|
})
|
309
310
|
|
310
311
|
def _load_json_file(self, file: str) -> None:
|
@@ -334,14 +335,14 @@ class StructuredDataSet:
|
|
334
335
|
self._add(type_name, structured_row)
|
335
336
|
if self._progress:
|
336
337
|
self._progress({
|
337
|
-
PROGRESS.LOAD_ITEM: True,
|
338
|
-
PROGRESS.LOAD_COUNT_REFS: self.ref_total_count,
|
339
|
-
PROGRESS.LOAD_COUNT_REFS_FOUND: self.ref_total_found_count,
|
340
|
-
PROGRESS.LOAD_COUNT_REFS_NOT_FOUND: self.ref_total_notfound_count,
|
341
|
-
PROGRESS.LOAD_COUNT_REFS_LOOKUP: self.ref_lookup_count,
|
342
|
-
PROGRESS.LOAD_COUNT_REFS_LOOKUP_CACHE_HIT: self.ref_lookup_cache_hit_count,
|
343
|
-
PROGRESS.LOAD_COUNT_REFS_EXISTS_CACHE_HIT: self.ref_exists_cache_hit_count,
|
344
|
-
PROGRESS.LOAD_COUNT_REFS_INVALID: self.ref_invalid_identifying_property_count
|
338
|
+
PROGRESS.LOAD_ITEM.value: True,
|
339
|
+
PROGRESS.LOAD_COUNT_REFS.value: self.ref_total_count,
|
340
|
+
PROGRESS.LOAD_COUNT_REFS_FOUND.value: self.ref_total_found_count,
|
341
|
+
PROGRESS.LOAD_COUNT_REFS_NOT_FOUND.value: self.ref_total_notfound_count,
|
342
|
+
PROGRESS.LOAD_COUNT_REFS_LOOKUP.value: self.ref_lookup_count,
|
343
|
+
PROGRESS.LOAD_COUNT_REFS_LOOKUP_CACHE_HIT.value: self.ref_lookup_cache_hit_count,
|
344
|
+
PROGRESS.LOAD_COUNT_REFS_EXISTS_CACHE_HIT.value: self.ref_exists_cache_hit_count,
|
345
|
+
PROGRESS.LOAD_COUNT_REFS_INVALID.value: self.ref_invalid_identifying_property_count
|
345
346
|
})
|
346
347
|
self._note_warning(reader.warnings, "reader")
|
347
348
|
if schema:
|
@@ -48,7 +48,7 @@ dcicutils/obfuscation_utils.py,sha256=fo2jOmDRC6xWpYX49u80bVNisqRRoPskFNX3ymFAmj
|
|
48
48
|
dcicutils/opensearch_utils.py,sha256=V2exmFYW8Xl2_pGFixF4I2Cc549Opwe4PhFi5twC0M8,1017
|
49
49
|
dcicutils/portal_object_utils.py,sha256=MF6MTZ6yxakZFDjbkTKCsF4q4p11dLDVvT5JBV9m6RQ,15408
|
50
50
|
dcicutils/portal_utils.py,sha256=oBoI3KWRp6YrbsuVGbmPQ3kATB5cVVsQo7-qmnYXWqg,30260
|
51
|
-
dcicutils/progress_constants.py,sha256=
|
51
|
+
dcicutils/progress_constants.py,sha256=oD6JKkPWDM22sFDlEus-JlDl9-jUC82MezVV8nJq870,2146
|
52
52
|
dcicutils/project_utils.py,sha256=qPdCaFmWUVBJw4rw342iUytwdQC0P-XKpK4mhyIulMM,31250
|
53
53
|
dcicutils/qa_checkers.py,sha256=cdXjeL0jCDFDLT8VR8Px78aS10hwNISOO5G_Zv2TZ6M,20534
|
54
54
|
dcicutils/qa_utils.py,sha256=TT0SiJWiuxYvbsIyhK9VO4uV_suxhB6CpuC4qPacCzQ,160208
|
@@ -63,15 +63,15 @@ dcicutils/secrets_utils.py,sha256=8dppXAsiHhJzI6NmOcvJV5ldvKkQZzh3Fl-cb8Wm7MI,19
|
|
63
63
|
dcicutils/sheet_utils.py,sha256=VlmzteONW5VF_Q4vo0yA5vesz1ViUah1MZ_yA1rwZ0M,33629
|
64
64
|
dcicutils/snapshot_utils.py,sha256=ymP7PXH6-yEiXAt75w0ldQFciGNqWBClNxC5gfX2FnY,22961
|
65
65
|
dcicutils/ssl_certificate_utils.py,sha256=F0ifz_wnRRN9dfrfsz7aCp4UDLgHEY8LaK7PjnNvrAQ,9707
|
66
|
-
dcicutils/structured_data.py,sha256=
|
66
|
+
dcicutils/structured_data.py,sha256=uJA-uU1SsPHQTDYm4hwV2d49HwjZvppYKsHcTyKxGPE,58765
|
67
67
|
dcicutils/task_utils.py,sha256=MF8ujmTD6-O2AC2gRGPHyGdUrVKgtr8epT5XU8WtNjk,8082
|
68
68
|
dcicutils/tmpfile_utils.py,sha256=n95XF8dZVbQRSXBZTGToXXfSs3JUVRyN6c3ZZ0nhAWI,1403
|
69
69
|
dcicutils/trace_utils.py,sha256=g8kwV4ebEy5kXW6oOrEAUsurBcCROvwtZqz9fczsGRE,1769
|
70
70
|
dcicutils/validation_utils.py,sha256=cMZIU2cY98FYtzK52z5WUYck7urH6JcqOuz9jkXpqzg,14797
|
71
71
|
dcicutils/variant_utils.py,sha256=2H9azNx3xAj-MySg-uZ2SFqbWs4kZvf61JnK6b-h4Qw,4343
|
72
72
|
dcicutils/zip_utils.py,sha256=rnjNv_k6L9jT2SjDSgVXp4BEJYLtz9XN6Cl2Fy-tqnM,2027
|
73
|
-
dcicutils-8.8.1.
|
74
|
-
dcicutils-8.8.1.
|
75
|
-
dcicutils-8.8.1.
|
76
|
-
dcicutils-8.8.1.
|
77
|
-
dcicutils-8.8.1.
|
73
|
+
dcicutils-8.8.1.1b12.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
|
74
|
+
dcicutils-8.8.1.1b12.dist-info/METADATA,sha256=8CSebRTKE4rRjLBB7zCcZ-ISjDa7OElpjrC61nr2Dhg,3357
|
75
|
+
dcicutils-8.8.1.1b12.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
|
76
|
+
dcicutils-8.8.1.1b12.dist-info/entry_points.txt,sha256=51Q4F_2V10L0282W7HFjP4jdzW4K8lnWDARJQVFy_hw,270
|
77
|
+
dcicutils-8.8.1.1b12.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|