dcicutils 8.8.1.1b10__py3-none-any.whl → 8.8.1.1b12__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. dcicutils/progress_constants.py +25 -19
  2. dcicutils/structured_data.py +26 -25
  3. {dcicutils-8.8.1.1b10.dist-info → dcicutils-8.8.1.1b12.dist-info}/METADATA +1 -1
  4. {dcicutils-8.8.1.1b10.dist-info → dcicutils-8.8.1.1b12.dist-info}/RECORD +7 -7
  5. {dcicutils-8.8.1.1b10.dist-info → dcicutils-8.8.1.1b12.dist-info}/LICENSE.txt +0 -0
  6. {dcicutils-8.8.1.1b10.dist-info → dcicutils-8.8.1.1b12.dist-info}/WHEEL +0 -0
  7. {dcicutils-8.8.1.1b10.dist-info → dcicutils-8.8.1.1b12.dist-info}/entry_points.txt +0 -0
dcicutils/progress_constants.py CHANGED
@@ -13,28 +13,34 @@ class PROGRESS_INGESTER(Enum):
13
13
  VALIDATE_LOAD_DONE = "ingester_validate_done"
14
14
  LOADXL_INITIATE = "ingester_loadxl_initiate"
15
15
  LOADXL_DONE = "ingester_loadxl_done"
16
+ MESSAGE = "ingester_message"
17
+ MESSAGE_VERBOSE = "ingester_message_verbose"
18
+ MESSAGE_DEBUG = "ingester_message_debug"
16
19
 
17
20
 
18
21
  class PROGRESS_PARSE(Enum):
19
- LOAD_START = "start"
20
- LOAD_ITEM = "parse"
21
- LOAD_DONE = "finish"
22
- LOAD_COUNT_SHEETS = "sheets"
23
- LOAD_COUNT_ROWS = "rows"
24
- LOAD_COUNT_REFS = "refs"
25
- LOAD_COUNT_REFS_FOUND = "refs_found"
26
- LOAD_COUNT_REFS_NOT_FOUND = "refs_not_found"
27
- LOAD_COUNT_REFS_LOOKUP = "refs_lookup"
28
- LOAD_COUNT_REFS_LOOKUP_CACHE_HIT = "refs_lookup_cache_hit"
29
- LOAD_COUNT_REFS_EXISTS_CACHE_HIT = "refs_exists_cache_hit"
30
- LOAD_COUNT_REFS_INVALID = "refs_invalid"
31
- ANALYZE_START = "start"
32
- ANALYZE_COUNT_TYPES = "types"
33
- ANALYZE_COUNT_ITEMS = "objects"
34
- ANALYZE_CREATE = "create"
35
- ANALYZE_COUNT_LOOKUP = "lookups"
36
- ANALYZE_UPDATE = "update"
37
- ANALYZE_DONE = "finish"
22
+ LOAD_START = "parse_start"
23
+ LOAD_ITEM = "parse_item"
24
+ LOAD_DONE = "parse_done"
25
+ LOAD_COUNT_SHEETS = "parse_sheets"
26
+ LOAD_COUNT_ROWS = "parse_rows"
27
+ LOAD_COUNT_REFS = "parse_refs"
28
+ LOAD_COUNT_REFS_FOUND = "parse_refs_found"
29
+ LOAD_COUNT_REFS_NOT_FOUND = "parse_refs_not_found"
30
+ LOAD_COUNT_REFS_LOOKUP = "parse_refs_lookup"
31
+ LOAD_COUNT_REFS_LOOKUP_CACHE_HIT = "parse_refs_lookup_cache_hit"
32
+ LOAD_COUNT_REFS_EXISTS_CACHE_HIT = "parse_refs_exists_cache_hit"
33
+ LOAD_COUNT_REFS_INVALID = "parse_refs_invalid"
34
+ ANALYZE_START = "parse_analyze_start"
35
+ ANALYZE_COUNT_TYPES = "parse_analyze_types"
36
+ ANALYZE_COUNT_ITEMS = "parse_analyze_objects"
37
+ ANALYZE_COUNT_LOOKUP = "parse_analyze_lookups"
38
+ ANALYZE_CREATE = "parse_analyze_create"
39
+ ANALYZE_UPDATE = "parse_analyze_update"
40
+ ANALYZE_DONE = "parse_analyze_done"
41
+ MESSAGE = "parse_message"
42
+ MESSAGE_VERBOSE = "parse_message_verbose"
43
+ MESSAGE_DEBUG = "parse_message_debug"
38
44
 
39
45
 
40
46
  class PROGRESS_LOADXL(Enum):
dcicutils/structured_data.py CHANGED
@@ -192,8 +192,8 @@ class StructuredDataSet:
192
192
  diffs = {}
193
193
  if callable(progress):
194
194
  ntypes, nobjects = get_counts()
195
- progress({PROGRESS.ANALYZE_START: True,
196
- PROGRESS.ANALYZE_COUNT_TYPES: ntypes, PROGRESS.ANALYZE_COUNT_ITEMS: nobjects})
195
+ progress({PROGRESS.ANALYZE_START.value: True,
196
+ PROGRESS.ANALYZE_COUNT_TYPES.value: ntypes, PROGRESS.ANALYZE_COUNT_ITEMS.value: nobjects})
197
197
  if self.data or self.portal: # TODO: what is this OR biz?
198
198
  refs = self.resolved_refs_with_uuids
199
199
  # TODO: Need feedback/progress tracking mechanism here.
@@ -212,19 +212,20 @@ class StructuredDataSet:
212
212
  uuid=existing_object.uuid,
213
213
  diffs=object_diffs or None))
214
214
  if callable(progress):
215
- progress({PROGRESS.ANALYZE_UPDATE: True,
216
- PROGRESS.ANALYZE_COUNT_LOOKUP: nlookups + nlookups_compare})
215
+ progress({PROGRESS.ANALYZE_UPDATE.value: True,
216
+ PROGRESS.ANALYZE_COUNT_LOOKUP.value: nlookups + nlookups_compare})
217
217
  elif identifying_path:
218
218
  # If there is no existing object we still create a record for this object
219
219
  # but with no uuid which will be the indication that it does not exist.
220
220
  diffs[type_name].append(create_readonly_object(path=identifying_path, uuid=None, diffs=None))
221
221
  if callable(progress):
222
- progress({PROGRESS.ANALYZE_CREATE: True, PROGRESS.ANALYZE_COUNT_LOOKUP: nlookups})
222
+ progress({PROGRESS.ANALYZE_CREATE.value: True,
223
+ PROGRESS.ANALYZE_COUNT_LOOKUP.value: nlookups})
223
224
  else:
224
225
  if callable(progress):
225
- progress({PROGRESS.ANALYZE_COUNT_LOOKUP: nlookups})
226
+ progress({PROGRESS.ANALYZE_COUNT_LOOKUP.value: nlookups})
226
227
  if callable(progress):
227
- progress({PROGRESS.ANALYZE_DONE: True})
228
+ progress({PROGRESS.ANALYZE_DONE.value: True})
228
229
  return diffs
229
230
 
230
231
  def load_file(self, file: str) -> None:
@@ -271,8 +272,8 @@ class StructuredDataSet:
271
272
  return nrows, len(excel.sheet_names)
272
273
  if self._progress: # TODO: Move to _load_reader
273
274
  nrows, nsheets = get_counts()
274
- self._progress({PROGRESS.LOAD_START: True,
275
- PROGRESS.LOAD_COUNT_SHEETS: nsheets, PROGRESS.LOAD_COUNT_ROWS: nrows})
275
+ self._progress({PROGRESS.LOAD_START.value: True,
276
+ PROGRESS.LOAD_COUNT_SHEETS.value: nsheets, PROGRESS.LOAD_COUNT_ROWS.value: nrows})
276
277
  excel = Excel(file) # Order the sheet names by any specified ordering (e.g. ala snovault.loadxl).
277
278
  order = {Schema.type_name(key): index for index, key in enumerate(self._order)} if self._order else {}
278
279
  for sheet_name in sorted(excel.sheet_names, key=lambda key: order.get(Schema.type_name(key), sys.maxsize)):
@@ -297,14 +298,14 @@ class StructuredDataSet:
297
298
  del self._errors["ref"]
298
299
  if self._progress:
299
300
  self._progress({ # TODO: Refactor with same thing below in _load_reader.
300
- PROGRESS.LOAD_DONE: True,
301
- PROGRESS.LOAD_COUNT_REFS: self.ref_total_count,
302
- PROGRESS.LOAD_COUNT_REFS_FOUND: self.ref_total_found_count,
303
- PROGRESS.LOAD_COUNT_REFS_NOT_FOUND: self.ref_total_notfound_count,
304
- PROGRESS.LOAD_COUNT_REFS_LOOKUP: self.ref_lookup_count,
305
- PROGRESS.LOAD_COUNT_REFS_LOOKUP_CACHE_HIT: self.ref_lookup_cache_hit_count,
306
- PROGRESS.LOAD_COUNT_REFS_EXISTS_CACHE_HIT: self.ref_exists_cache_hit_count,
307
- PROGRESS.LOAD_COUNT_REFS_INVALID: self.ref_invalid_identifying_property_count
301
+ PROGRESS.LOAD_DONE.value: True,
302
+ PROGRESS.LOAD_COUNT_REFS.value: self.ref_total_count,
303
+ PROGRESS.LOAD_COUNT_REFS_FOUND.value: self.ref_total_found_count,
304
+ PROGRESS.LOAD_COUNT_REFS_NOT_FOUND.value: self.ref_total_notfound_count,
305
+ PROGRESS.LOAD_COUNT_REFS_LOOKUP.value: self.ref_lookup_count,
306
+ PROGRESS.LOAD_COUNT_REFS_LOOKUP_CACHE_HIT.value: self.ref_lookup_cache_hit_count,
307
+ PROGRESS.LOAD_COUNT_REFS_EXISTS_CACHE_HIT.value: self.ref_exists_cache_hit_count,
308
+ PROGRESS.LOAD_COUNT_REFS_INVALID.value: self.ref_invalid_identifying_property_count
308
309
  })
309
310
 
310
311
  def _load_json_file(self, file: str) -> None:
@@ -334,14 +335,14 @@ class StructuredDataSet:
334
335
  self._add(type_name, structured_row)
335
336
  if self._progress:
336
337
  self._progress({
337
- PROGRESS.LOAD_ITEM: True,
338
- PROGRESS.LOAD_COUNT_REFS: self.ref_total_count,
339
- PROGRESS.LOAD_COUNT_REFS_FOUND: self.ref_total_found_count,
340
- PROGRESS.LOAD_COUNT_REFS_NOT_FOUND: self.ref_total_notfound_count,
341
- PROGRESS.LOAD_COUNT_REFS_LOOKUP: self.ref_lookup_count,
342
- PROGRESS.LOAD_COUNT_REFS_LOOKUP_CACHE_HIT: self.ref_lookup_cache_hit_count,
343
- PROGRESS.LOAD_COUNT_REFS_EXISTS_CACHE_HIT: self.ref_exists_cache_hit_count,
344
- PROGRESS.LOAD_COUNT_REFS_INVALID: self.ref_invalid_identifying_property_count
338
+ PROGRESS.LOAD_ITEM.value: True,
339
+ PROGRESS.LOAD_COUNT_REFS.value: self.ref_total_count,
340
+ PROGRESS.LOAD_COUNT_REFS_FOUND.value: self.ref_total_found_count,
341
+ PROGRESS.LOAD_COUNT_REFS_NOT_FOUND.value: self.ref_total_notfound_count,
342
+ PROGRESS.LOAD_COUNT_REFS_LOOKUP.value: self.ref_lookup_count,
343
+ PROGRESS.LOAD_COUNT_REFS_LOOKUP_CACHE_HIT.value: self.ref_lookup_cache_hit_count,
344
+ PROGRESS.LOAD_COUNT_REFS_EXISTS_CACHE_HIT.value: self.ref_exists_cache_hit_count,
345
+ PROGRESS.LOAD_COUNT_REFS_INVALID.value: self.ref_invalid_identifying_property_count
345
346
  })
346
347
  self._note_warning(reader.warnings, "reader")
347
348
  if schema:
{dcicutils-8.8.1.1b10.dist-info → dcicutils-8.8.1.1b12.dist-info}/METADATA RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: dcicutils
3
- Version: 8.8.1.1b10
3
+ Version: 8.8.1.1b12
4
4
  Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
5
5
  Home-page: https://github.com/4dn-dcic/utils
6
6
  License: MIT
{dcicutils-8.8.1.1b10.dist-info → dcicutils-8.8.1.1b12.dist-info}/RECORD RENAMED
@@ -48,7 +48,7 @@ dcicutils/obfuscation_utils.py,sha256=fo2jOmDRC6xWpYX49u80bVNisqRRoPskFNX3ymFAmj
48
48
  dcicutils/opensearch_utils.py,sha256=V2exmFYW8Xl2_pGFixF4I2Cc549Opwe4PhFi5twC0M8,1017
49
49
  dcicutils/portal_object_utils.py,sha256=MF6MTZ6yxakZFDjbkTKCsF4q4p11dLDVvT5JBV9m6RQ,15408
50
50
  dcicutils/portal_utils.py,sha256=oBoI3KWRp6YrbsuVGbmPQ3kATB5cVVsQo7-qmnYXWqg,30260
51
- dcicutils/progress_constants.py,sha256=1N3BOnViX4AlrNK4lq7O-zRBqoZ0BTvC_vpFk2rOxD4,1736
51
+ dcicutils/progress_constants.py,sha256=oD6JKkPWDM22sFDlEus-JlDl9-jUC82MezVV8nJq870,2146
52
52
  dcicutils/project_utils.py,sha256=qPdCaFmWUVBJw4rw342iUytwdQC0P-XKpK4mhyIulMM,31250
53
53
  dcicutils/qa_checkers.py,sha256=cdXjeL0jCDFDLT8VR8Px78aS10hwNISOO5G_Zv2TZ6M,20534
54
54
  dcicutils/qa_utils.py,sha256=TT0SiJWiuxYvbsIyhK9VO4uV_suxhB6CpuC4qPacCzQ,160208
@@ -63,15 +63,15 @@ dcicutils/secrets_utils.py,sha256=8dppXAsiHhJzI6NmOcvJV5ldvKkQZzh3Fl-cb8Wm7MI,19
63
63
  dcicutils/sheet_utils.py,sha256=VlmzteONW5VF_Q4vo0yA5vesz1ViUah1MZ_yA1rwZ0M,33629
64
64
  dcicutils/snapshot_utils.py,sha256=ymP7PXH6-yEiXAt75w0ldQFciGNqWBClNxC5gfX2FnY,22961
65
65
  dcicutils/ssl_certificate_utils.py,sha256=F0ifz_wnRRN9dfrfsz7aCp4UDLgHEY8LaK7PjnNvrAQ,9707
66
- dcicutils/structured_data.py,sha256=1guVNDzIVxJkQA_m0jSh9xI2FB5oVXR4m7sqrqF8A5w,58559
66
+ dcicutils/structured_data.py,sha256=uJA-uU1SsPHQTDYm4hwV2d49HwjZvppYKsHcTyKxGPE,58765
67
67
  dcicutils/task_utils.py,sha256=MF8ujmTD6-O2AC2gRGPHyGdUrVKgtr8epT5XU8WtNjk,8082
68
68
  dcicutils/tmpfile_utils.py,sha256=n95XF8dZVbQRSXBZTGToXXfSs3JUVRyN6c3ZZ0nhAWI,1403
69
69
  dcicutils/trace_utils.py,sha256=g8kwV4ebEy5kXW6oOrEAUsurBcCROvwtZqz9fczsGRE,1769
70
70
  dcicutils/validation_utils.py,sha256=cMZIU2cY98FYtzK52z5WUYck7urH6JcqOuz9jkXpqzg,14797
71
71
  dcicutils/variant_utils.py,sha256=2H9azNx3xAj-MySg-uZ2SFqbWs4kZvf61JnK6b-h4Qw,4343
72
72
  dcicutils/zip_utils.py,sha256=rnjNv_k6L9jT2SjDSgVXp4BEJYLtz9XN6Cl2Fy-tqnM,2027
73
- dcicutils-8.8.1.1b10.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
74
- dcicutils-8.8.1.1b10.dist-info/METADATA,sha256=ZpO-aF0tzLRyns4vaIGBO5VsQq1hX8bw4rWDv8_v5Hc,3357
75
- dcicutils-8.8.1.1b10.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
76
- dcicutils-8.8.1.1b10.dist-info/entry_points.txt,sha256=51Q4F_2V10L0282W7HFjP4jdzW4K8lnWDARJQVFy_hw,270
77
- dcicutils-8.8.1.1b10.dist-info/RECORD,,
73
+ dcicutils-8.8.1.1b12.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
74
+ dcicutils-8.8.1.1b12.dist-info/METADATA,sha256=8CSebRTKE4rRjLBB7zCcZ-ISjDa7OElpjrC61nr2Dhg,3357
75
+ dcicutils-8.8.1.1b12.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
76
+ dcicutils-8.8.1.1b12.dist-info/entry_points.txt,sha256=51Q4F_2V10L0282W7HFjP4jdzW4K8lnWDARJQVFy_hw,270
77
+ dcicutils-8.8.1.1b12.dist-info/RECORD,,