dcicutils 8.14.0.1b12__tar.gz → 8.14.0.1b14__tar.gz
Sign up to get free protection for your applications and to get access to all the features.
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/PKG-INFO +1 -1
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/scripts/update_portal_object.py +27 -13
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/pyproject.toml +1 -1
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/LICENSE.txt +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/README.rst +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/__init__.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/base.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/beanstalk_utils.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/bundle_utils.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/captured_output.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/cloudformation_utils.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/codebuild_utils.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/command_utils.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/common.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/contribution_scripts.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/contribution_utils.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/creds_utils.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/data_readers.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/data_utils.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/datetime_utils.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/deployment_utils.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/diff_utils.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/docker_utils.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/ecr_scripts.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/ecr_utils.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/ecs_utils.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/env_base.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/env_manager.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/env_scripts.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/env_utils.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/env_utils_legacy.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/es_utils.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/exceptions.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/ff_mocks.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/ff_utils.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/file_utils.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/function_cache_decorator.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/glacier_utils.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/hack_for_elasticsearch_numpy_usage.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/http_utils.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/jh_utils.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/kibana/dashboards.json +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/kibana/readme.md +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/lang_utils.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/license_policies/c4-infrastructure.jsonc +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/license_policies/c4-python-infrastructure.jsonc +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/license_policies/park-lab-common-server.jsonc +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/license_policies/park-lab-common.jsonc +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/license_policies/park-lab-gpl-pipeline.jsonc +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/license_policies/park-lab-pipeline.jsonc +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/license_utils.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/log_utils.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/misc_utils.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/obfuscation_utils.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/opensearch_utils.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/portal_object_utils.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/portal_utils.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/progress_bar.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/project_utils.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/qa_checkers.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/qa_utils.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/redis_tools.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/redis_utils.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/s3_utils.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/schema_utils.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/scripts/publish_to_pypi.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/scripts/run_license_checker.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/scripts/view_portal_object.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/secrets_utils.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/sheet_utils.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/snapshot_utils.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/ssl_certificate_utils.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/structured_data.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/submitr/progress_constants.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/submitr/ref_lookup_strategy.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/task_utils.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/tmpfile_utils.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/trace_utils.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/validation_utils.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/variant_utils.py +0 -0
- {dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/zip_utils.py +0 -0
@@ -123,7 +123,8 @@ def main():
|
|
123
123
|
parser.add_argument("--post", type=str, required=False, default=None, help="POST data.")
|
124
124
|
parser.add_argument("--patch", type=str, required=False, default=None, help="PATCH data.")
|
125
125
|
parser.add_argument("--upsert", type=str, required=False, default=None, help="Upsert data.")
|
126
|
-
parser.add_argument("--load", type=str, required=False, default=None,
|
126
|
+
parser.add_argument("--load", "--loadxl", type=str, required=False, default=None,
|
127
|
+
help="Load data via snovault.loadxl.")
|
127
128
|
parser.add_argument("--ini", type=str, required=False, default=None, help="INI file for data via snovault.loadxl.")
|
128
129
|
parser.add_argument("--delete", type=str, required=False, default=None, help="Delete data.")
|
129
130
|
parser.add_argument("--purge", type=str, required=False, default=None, help="Purge data.")
|
@@ -159,11 +160,12 @@ def main():
|
|
159
160
|
"when using the --load option (to load data via snovault.loadxl).")
|
160
161
|
exit(1)
|
161
162
|
if args.env:
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
_load_data(load=args.load, ini_file=args.ini,
|
166
|
-
|
163
|
+
if args.ini:
|
164
|
+
_print("The --env is not used for the --load option (to load data via snovault.loadxl).")
|
165
|
+
args.ini = args.env
|
166
|
+
if not _load_data(load=args.load, ini_file=args.ini, explicit_schema_name=args.schema,
|
167
|
+
verbose=args.verbose, debug=args.debug, noprogress=args.noprogress):
|
168
|
+
exit(1)
|
167
169
|
exit(0)
|
168
170
|
|
169
171
|
portal = _create_portal(env=args.env, app=app, verbose=args.verbose, debug=args.debug)
|
@@ -171,7 +173,7 @@ def main():
|
|
171
173
|
if explicit_schema_name := args.schema:
|
172
174
|
schema, explicit_schema_name = _get_schema(portal, explicit_schema_name)
|
173
175
|
if not schema:
|
174
|
-
usage(f"
|
176
|
+
usage(f"Unknown specified schema name: {args.schema}")
|
175
177
|
|
176
178
|
if args.post:
|
177
179
|
_post_or_patch_or_upsert(portal=portal,
|
@@ -494,15 +496,25 @@ def _load_data(load: str, ini_file: str, explicit_schema_name: Optional[str] = N
|
|
494
496
|
if not (schema_name := _get_schema_name_from_schema_named_json_file_name(portal, inserts_file)):
|
495
497
|
_print("Unable to determine schema name for JSON data file: {inserts_file}")
|
496
498
|
return False
|
499
|
+
elif not (schema_name := _get_schema(portal, explicit_schema_name)[1]):
|
500
|
+
_print(f"Unknown specified schema name: {explicit_schema_name}")
|
501
|
+
return False
|
497
502
|
with temporary_directory() as tmpdir:
|
498
503
|
file_name = os.path.join(tmpdir, f"{to_snake_case(schema_name)}.json")
|
499
504
|
with io.open(file_name, "w") as f:
|
500
505
|
json.dump(data, f)
|
501
|
-
return _load_data(load=tmpdir, ini_file=ini_file, explicit_schema_name=
|
506
|
+
return _load_data(load=tmpdir, ini_file=ini_file, explicit_schema_name=schema_name,
|
502
507
|
verbose=verbose, debug=debug, noprogress=noprogress,
|
503
508
|
_portal=portal, _single_insert_file=inserts_file)
|
504
509
|
elif isinstance(data, dict):
|
505
|
-
|
510
|
+
if schema_name := explicit_schema_name:
|
511
|
+
if _is_schema_name_list(portal, schema_names := list(data.keys())):
|
512
|
+
_print(f"Ignoring specify --schema: {schema_name}")
|
513
|
+
elif not (schema_name := _get_schema(portal, schema_name)[1]):
|
514
|
+
_print(f"Unknown specified schema name: {explicit_schema_name}")
|
515
|
+
return False
|
516
|
+
else:
|
517
|
+
data = {schema_name: [data]}
|
506
518
|
if not _is_schema_name_list(portal, schema_names := list(data.keys())):
|
507
519
|
_print(f"Unrecognized types in JSON data file: {inserts_file}")
|
508
520
|
return False
|
@@ -521,7 +533,6 @@ def _load_data(load: str, ini_file: str, explicit_schema_name: Optional[str] = N
|
|
521
533
|
return _load_data(load=tmpdir, ini_file=ini_file,
|
522
534
|
verbose=verbose, debug=debug, noprogress=noprogress,
|
523
535
|
_portal=portal, _single_insert_file=inserts_file)
|
524
|
-
# TODO
|
525
536
|
return True
|
526
537
|
else:
|
527
538
|
_print(f"Unrecognized JSON data in file: {inserts_file}")
|
@@ -529,9 +540,9 @@ def _load_data(load: str, ini_file: str, explicit_schema_name: Optional[str] = N
|
|
529
540
|
return True
|
530
541
|
if verbose:
|
531
542
|
if _single_insert_file:
|
532
|
-
_print(f"Loading data
|
543
|
+
_print(f"Loading data into Portal (via snovault.loadxl) from file: {_single_insert_file}")
|
533
544
|
else:
|
534
|
-
_print(f"Loading data
|
545
|
+
_print(f"Loading data into Portal (via snovault.loadxl) from directory: {inserts_directory}")
|
535
546
|
_print(f"Portal INI file for load is: {ini_file}")
|
536
547
|
|
537
548
|
schema_names = list(_get_schemas(portal).keys())
|
@@ -575,7 +586,10 @@ def _load_data(load: str, ini_file: str, explicit_schema_name: Optional[str] = N
|
|
575
586
|
else:
|
576
587
|
loadxl(portal=portal, inserts_directory=inserts_directory, schema_names_to_load=schema_names_to_load)
|
577
588
|
if verbose:
|
578
|
-
|
589
|
+
if _single_insert_file:
|
590
|
+
_print(f"Done loading data into Portal (via snovault.loadxl) from file: {_single_insert_file}")
|
591
|
+
else:
|
592
|
+
_print(f"Done loading data into Portal (via snovault.loadxl) from directory: {inserts_directory}")
|
579
593
|
return True
|
580
594
|
|
581
595
|
|
@@ -1,6 +1,6 @@
|
|
1
1
|
[tool.poetry]
|
2
2
|
name = "dcicutils"
|
3
|
-
version = "8.14.0.
|
3
|
+
version = "8.14.0.1b14" # TODO: To become 8.14.1
|
4
4
|
description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources"
|
5
5
|
authors = ["4DN-DCIC Team <support@4dnucleome.org>"]
|
6
6
|
license = "MIT"
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
{dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/hack_for_elasticsearch_numpy_usage.py
RENAMED
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
{dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/license_policies/c4-infrastructure.jsonc
RENAMED
File without changes
|
File without changes
|
File without changes
|
{dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/license_policies/park-lab-common.jsonc
RENAMED
File without changes
|
File without changes
|
{dcicutils-8.14.0.1b12 → dcicutils-8.14.0.1b14}/dcicutils/license_policies/park-lab-pipeline.jsonc
RENAMED
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|