dcicutils 8.8.1.1b16__tar.gz → 8.8.1.1b18__tar.gz
Sign up to get free protection for your applications and to get access to all the features.
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/PKG-INFO +1 -1
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/progress_constants.py +4 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/structured_data.py +26 -26
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/pyproject.toml +1 -1
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/LICENSE.txt +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/README.rst +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/__init__.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/base.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/beanstalk_utils.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/bundle_utils.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/captured_output.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/cloudformation_utils.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/codebuild_utils.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/command_utils.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/common.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/contribution_scripts.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/contribution_utils.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/creds_utils.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/data_readers.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/data_utils.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/datetime_utils.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/deployment_utils.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/diff_utils.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/docker_utils.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/ecr_scripts.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/ecr_utils.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/ecs_utils.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/env_base.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/env_manager.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/env_scripts.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/env_utils.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/env_utils_legacy.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/es_utils.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/exceptions.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/ff_mocks.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/ff_utils.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/file_utils.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/function_cache_decorator.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/glacier_utils.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/jh_utils.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/kibana/dashboards.json +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/kibana/readme.md +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/lang_utils.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/license_policies/c4-infrastructure.jsonc +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/license_policies/c4-python-infrastructure.jsonc +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/license_policies/park-lab-common-server.jsonc +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/license_policies/park-lab-common.jsonc +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/license_policies/park-lab-gpl-pipeline.jsonc +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/license_policies/park-lab-pipeline.jsonc +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/license_utils.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/log_utils.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/misc_utils.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/obfuscation_utils.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/opensearch_utils.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/portal_object_utils.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/portal_utils.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/project_utils.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/qa_checkers.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/qa_utils.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/redis_tools.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/redis_utils.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/s3_utils.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/schema_utils.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/scripts/publish_to_pypi.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/scripts/run_license_checker.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/scripts/view_portal_object.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/secrets_utils.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/sheet_utils.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/snapshot_utils.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/ssl_certificate_utils.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/task_utils.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/tmpfile_utils.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/trace_utils.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/validation_utils.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/variant_utils.py +0 -0
- {dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/zip_utils.py +0 -0
@@ -9,6 +9,10 @@ class _Enum(Enum):
|
|
9
9
|
# Automatically make enumerators within the enumeration resolve to its value property.
|
10
10
|
def __get__(self, instance, owner):
|
11
11
|
return self.value
|
12
|
+
# But doing the above does not take when iterating; so make provide a values method.
|
13
|
+
@classmethod # noqa
|
14
|
+
def values(cls):
|
15
|
+
return [enumerator.value for enumerator in cls]
|
12
16
|
|
13
17
|
|
14
18
|
class PROGRESS_INGESTER(_Enum):
|
@@ -192,8 +192,8 @@ class StructuredDataSet:
|
|
192
192
|
diffs = {}
|
193
193
|
if callable(progress):
|
194
194
|
ntypes, nobjects = get_counts()
|
195
|
-
progress({PROGRESS.ANALYZE_START
|
196
|
-
PROGRESS.ANALYZE_COUNT_TYPES
|
195
|
+
progress({PROGRESS.ANALYZE_START: PROGRESS.NOW(),
|
196
|
+
PROGRESS.ANALYZE_COUNT_TYPES: ntypes, PROGRESS.ANALYZE_COUNT_ITEMS: nobjects})
|
197
197
|
if self.data or self.portal: # TODO: what is this OR biz?
|
198
198
|
refs = self.resolved_refs_with_uuids
|
199
199
|
# TODO: Need feedback/progress tracking mechanism here.
|
@@ -212,20 +212,20 @@ class StructuredDataSet:
|
|
212
212
|
uuid=existing_object.uuid,
|
213
213
|
diffs=object_diffs or None))
|
214
214
|
if callable(progress):
|
215
|
-
progress({PROGRESS.ANALYZE_UPDATE
|
216
|
-
PROGRESS.ANALYZE_COUNT_LOOKUP
|
215
|
+
progress({PROGRESS.ANALYZE_UPDATE: True,
|
216
|
+
PROGRESS.ANALYZE_COUNT_LOOKUP: nlookups + nlookups_compare})
|
217
217
|
elif identifying_path:
|
218
218
|
# If there is no existing object we still create a record for this object
|
219
219
|
# but with no uuid which will be the indication that it does not exist.
|
220
220
|
diffs[type_name].append(create_readonly_object(path=identifying_path, uuid=None, diffs=None))
|
221
221
|
if callable(progress):
|
222
|
-
progress({PROGRESS.ANALYZE_CREATE
|
223
|
-
PROGRESS.ANALYZE_COUNT_LOOKUP
|
222
|
+
progress({PROGRESS.ANALYZE_CREATE: True,
|
223
|
+
PROGRESS.ANALYZE_COUNT_LOOKUP: nlookups})
|
224
224
|
else:
|
225
225
|
if callable(progress):
|
226
|
-
progress({PROGRESS.ANALYZE_COUNT_LOOKUP
|
226
|
+
progress({PROGRESS.ANALYZE_COUNT_LOOKUP: nlookups})
|
227
227
|
if callable(progress):
|
228
|
-
progress({PROGRESS.ANALYZE_DONE
|
228
|
+
progress({PROGRESS.ANALYZE_DONE: PROGRESS.NOW()})
|
229
229
|
return diffs
|
230
230
|
|
231
231
|
def load_file(self, file: str) -> None:
|
@@ -272,8 +272,8 @@ class StructuredDataSet:
|
|
272
272
|
return nrows, len(excel.sheet_names)
|
273
273
|
if self._progress: # TODO: Move to _load_reader
|
274
274
|
nrows, nsheets = get_counts()
|
275
|
-
self._progress({PROGRESS.LOAD_START
|
276
|
-
PROGRESS.LOAD_COUNT_SHEETS
|
275
|
+
self._progress({PROGRESS.LOAD_START: PROGRESS.NOW(),
|
276
|
+
PROGRESS.LOAD_COUNT_SHEETS: nsheets, PROGRESS.LOAD_COUNT_ROWS: nrows})
|
277
277
|
excel = Excel(file) # Order the sheet names by any specified ordering (e.g. ala snovault.loadxl).
|
278
278
|
order = {Schema.type_name(key): index for index, key in enumerate(self._order)} if self._order else {}
|
279
279
|
for sheet_name in sorted(excel.sheet_names, key=lambda key: order.get(Schema.type_name(key), sys.maxsize)):
|
@@ -298,14 +298,14 @@ class StructuredDataSet:
|
|
298
298
|
del self._errors["ref"]
|
299
299
|
if self._progress:
|
300
300
|
self._progress({ # TODO: Refactor with same thing below in _load_reader.
|
301
|
-
PROGRESS.LOAD_DONE
|
302
|
-
PROGRESS.LOAD_COUNT_REFS
|
303
|
-
PROGRESS.LOAD_COUNT_REFS_FOUND
|
304
|
-
PROGRESS.LOAD_COUNT_REFS_NOT_FOUND
|
305
|
-
PROGRESS.LOAD_COUNT_REFS_LOOKUP
|
306
|
-
PROGRESS.LOAD_COUNT_REFS_LOOKUP_CACHE_HIT
|
307
|
-
PROGRESS.LOAD_COUNT_REFS_EXISTS_CACHE_HIT
|
308
|
-
PROGRESS.LOAD_COUNT_REFS_INVALID
|
301
|
+
PROGRESS.LOAD_DONE: PROGRESS.NOW(),
|
302
|
+
PROGRESS.LOAD_COUNT_REFS: self.ref_total_count,
|
303
|
+
PROGRESS.LOAD_COUNT_REFS_FOUND: self.ref_total_found_count,
|
304
|
+
PROGRESS.LOAD_COUNT_REFS_NOT_FOUND: self.ref_total_notfound_count,
|
305
|
+
PROGRESS.LOAD_COUNT_REFS_LOOKUP: self.ref_lookup_count,
|
306
|
+
PROGRESS.LOAD_COUNT_REFS_LOOKUP_CACHE_HIT: self.ref_lookup_cache_hit_count,
|
307
|
+
PROGRESS.LOAD_COUNT_REFS_EXISTS_CACHE_HIT: self.ref_exists_cache_hit_count,
|
308
|
+
PROGRESS.LOAD_COUNT_REFS_INVALID: self.ref_invalid_identifying_property_count
|
309
309
|
})
|
310
310
|
|
311
311
|
def _load_json_file(self, file: str) -> None:
|
@@ -335,14 +335,14 @@ class StructuredDataSet:
|
|
335
335
|
self._add(type_name, structured_row)
|
336
336
|
if self._progress:
|
337
337
|
self._progress({
|
338
|
-
PROGRESS.LOAD_ITEM
|
339
|
-
PROGRESS.LOAD_COUNT_REFS
|
340
|
-
PROGRESS.LOAD_COUNT_REFS_FOUND
|
341
|
-
PROGRESS.LOAD_COUNT_REFS_NOT_FOUND
|
342
|
-
PROGRESS.LOAD_COUNT_REFS_LOOKUP
|
343
|
-
PROGRESS.LOAD_COUNT_REFS_LOOKUP_CACHE_HIT
|
344
|
-
PROGRESS.LOAD_COUNT_REFS_EXISTS_CACHE_HIT
|
345
|
-
PROGRESS.LOAD_COUNT_REFS_INVALID
|
338
|
+
PROGRESS.LOAD_ITEM: True,
|
339
|
+
PROGRESS.LOAD_COUNT_REFS: self.ref_total_count,
|
340
|
+
PROGRESS.LOAD_COUNT_REFS_FOUND: self.ref_total_found_count,
|
341
|
+
PROGRESS.LOAD_COUNT_REFS_NOT_FOUND: self.ref_total_notfound_count,
|
342
|
+
PROGRESS.LOAD_COUNT_REFS_LOOKUP: self.ref_lookup_count,
|
343
|
+
PROGRESS.LOAD_COUNT_REFS_LOOKUP_CACHE_HIT: self.ref_lookup_cache_hit_count,
|
344
|
+
PROGRESS.LOAD_COUNT_REFS_EXISTS_CACHE_HIT: self.ref_exists_cache_hit_count,
|
345
|
+
PROGRESS.LOAD_COUNT_REFS_INVALID: self.ref_invalid_identifying_property_count
|
346
346
|
})
|
347
347
|
self._note_warning(reader.warnings, "reader")
|
348
348
|
if schema:
|
@@ -1,6 +1,6 @@
|
|
1
1
|
[tool.poetry]
|
2
2
|
name = "dcicutils"
|
3
|
-
version = "8.8.1.
|
3
|
+
version = "8.8.1.1b18" # TODO: To become 8.8.2
|
4
4
|
description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources"
|
5
5
|
authors = ["4DN-DCIC Team <support@4dnucleome.org>"]
|
6
6
|
license = "MIT"
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
{dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/license_policies/c4-infrastructure.jsonc
RENAMED
File without changes
|
File without changes
|
File without changes
|
{dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/license_policies/park-lab-common.jsonc
RENAMED
File without changes
|
{dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/license_policies/park-lab-gpl-pipeline.jsonc
RENAMED
File without changes
|
{dcicutils-8.8.1.1b16 → dcicutils-8.8.1.1b18}/dcicutils/license_policies/park-lab-pipeline.jsonc
RENAMED
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|