dcicutils 8.14.0.1b10__py3-none-any.whl → 8.14.0.1b11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dcicutils/scripts/update_portal_object.py +82 -26
- {dcicutils-8.14.0.1b10.dist-info → dcicutils-8.14.0.1b11.dist-info}/METADATA +1 -1
- {dcicutils-8.14.0.1b10.dist-info → dcicutils-8.14.0.1b11.dist-info}/RECORD +6 -6
- {dcicutils-8.14.0.1b10.dist-info → dcicutils-8.14.0.1b11.dist-info}/LICENSE.txt +0 -0
- {dcicutils-8.14.0.1b10.dist-info → dcicutils-8.14.0.1b11.dist-info}/WHEEL +0 -0
- {dcicutils-8.14.0.1b10.dist-info → dcicutils-8.14.0.1b11.dist-info}/entry_points.txt +0 -0
@@ -162,7 +162,7 @@ def main():
|
|
162
162
|
_print("The --env is not used for the --load option (to load data via snovault.loadxl).")
|
163
163
|
if args.schema:
|
164
164
|
_print("The --schema is not used for the --load option (to load data via snovault.loadxl).")
|
165
|
-
_load_data(
|
165
|
+
_load_data(load=args.load, ini_file=args.ini,
|
166
166
|
verbose=args.verbose, debug=args.debug, noprogress=args.noprogress)
|
167
167
|
exit(0)
|
168
168
|
|
@@ -226,14 +226,6 @@ def _post_or_patch_or_upsert(portal: Portal, file_or_directory: str,
|
|
226
226
|
confirm: bool = False, verbose: bool = False,
|
227
227
|
quiet: bool = False, debug: bool = False) -> None:
|
228
228
|
|
229
|
-
def is_schema_name_list(portal: Portal, keys: list) -> bool:
|
230
|
-
if isinstance(keys, list):
|
231
|
-
for key in keys:
|
232
|
-
if portal.get_schema(key) is None:
|
233
|
-
return False
|
234
|
-
return True
|
235
|
-
return False
|
236
|
-
|
237
229
|
def post_or_patch_or_upsert(portal: Portal, file: str, schema_name: Optional[str],
|
238
230
|
patch_delete_fields: Optional[str] = None,
|
239
231
|
confirm: bool = False, verbose: bool = False,
|
@@ -251,7 +243,7 @@ def _post_or_patch_or_upsert(portal: Portal, file_or_directory: str,
|
|
251
243
|
patch_delete_fields=patch_delete_fields,
|
252
244
|
noignore=noignore, ignore=ignore,
|
253
245
|
confirm=confirm, verbose=verbose, debug=debug)
|
254
|
-
elif
|
246
|
+
elif _is_schema_name_list(portal, list(data.keys())):
|
255
247
|
if debug:
|
256
248
|
_print(f"DEBUG: File ({file}) contains a dictionary of schema names.")
|
257
249
|
for schema_name in data:
|
@@ -408,8 +400,8 @@ def _upsert_data(portal: Portal, data: dict, schema_name: str,
|
|
408
400
|
return
|
409
401
|
|
410
402
|
|
411
|
-
def _load_data(
|
412
|
-
verbose: bool = False, debug: bool = False, noprogress: bool = False) ->
|
403
|
+
def _load_data(load: str, ini_file: str, explicit_schema_name: Optional[str] = None,
|
404
|
+
verbose: bool = False, debug: bool = False, noprogress: bool = False) -> bool:
|
413
405
|
|
414
406
|
from snovault.loadxl import load_all_gen, LoadGenWrapper
|
415
407
|
from dcicutils.captured_output import captured_output
|
@@ -431,7 +423,7 @@ def _load_data(inserts_directory: str, ini_file: str,
|
|
431
423
|
return ""
|
432
424
|
|
433
425
|
LOADXL_RESPONSE_PATTERN = re.compile(r"^([A-Z]+):\s*([a-zA-Z\/\d_-]+)\s*(\S+)\s*(\S+)?\s*(.*)$")
|
434
|
-
LOADXL_ACTION_NAME = {"POST": "Create", "PATCH": "Update", "SKIP": "
|
426
|
+
LOADXL_ACTION_NAME = {"POST": "Create", "PATCH": "Update", "SKIP": "Check",
|
435
427
|
"CHECK": "Validate", "ERROR": "Error"}
|
436
428
|
current_item_type = None
|
437
429
|
current_item_count = 0
|
@@ -472,14 +464,66 @@ def _load_data(inserts_directory: str, ini_file: str,
|
|
472
464
|
if not os.path.exists(ini_file):
|
473
465
|
_print(f"The INI file required for --load is not found: {ini_file}")
|
474
466
|
exit(1)
|
475
|
-
|
476
|
-
|
477
|
-
|
478
|
-
|
479
|
-
|
467
|
+
|
468
|
+
if not os.path.isabs(load := os.path.expanduser(load)):
|
469
|
+
load = os.path.join(os.getcwd(), load)
|
470
|
+
if not os.path.exists(load):
|
471
|
+
return False
|
472
|
+
|
473
|
+
if os.path.isdir(load):
|
474
|
+
inserts_directory = load
|
475
|
+
inserts_file = None
|
476
|
+
else:
|
477
|
+
inserts_directory = None
|
478
|
+
inserts_file = load
|
479
|
+
|
480
480
|
portal = None
|
481
481
|
with captured_output(not debug):
|
482
482
|
portal = Portal(ini_file)
|
483
|
+
|
484
|
+
if inserts_file:
|
485
|
+
with io.open(inserts_file, "r") as f:
|
486
|
+
try:
|
487
|
+
data = json.load(f)
|
488
|
+
except Exception:
|
489
|
+
_print(f"Cannot load JSON data from file: {inserts_file}")
|
490
|
+
return False
|
491
|
+
if isinstance(data, list):
|
492
|
+
if not (schema_name := explicit_schema_name):
|
493
|
+
if not (schema_name := _get_schema_name_from_schema_named_json_file_name(portal, inserts_file)):
|
494
|
+
_print("Unable to determine schema name for JSON data file: {inserts_file}")
|
495
|
+
return False
|
496
|
+
with temporary_directory() as tmpdir:
|
497
|
+
file_name = os.path.join(tmpdir, f"{to_snake_case(schema_name)}.json")
|
498
|
+
with io.open(file_name, "w") as f:
|
499
|
+
json.dump(data, f)
|
500
|
+
return _load_data(load=tmpdir, ini_file=ini_file, explicit_schema_name=explicit_schema_name,
|
501
|
+
verbose=verbose, debug=debug, noprogress=noprogress)
|
502
|
+
elif isinstance(data, dict):
|
503
|
+
_print("DICT IN FILE FOR LOAD NOT YET SUPPPORTED")
|
504
|
+
if not _is_schema_name_list(portal, schema_names := list(data.keys())):
|
505
|
+
_print(f"Unrecognized types in JSON data file: {inserts_file}")
|
506
|
+
return False
|
507
|
+
with temporary_directory() as tmpdir:
|
508
|
+
nfiles = 0
|
509
|
+
for schema_name in schema_names:
|
510
|
+
if not isinstance(schema_data := data[schema_name], list):
|
511
|
+
_print(f"Unexpected value for data type ({schema_name})"
|
512
|
+
f" in JSON data file: {inserts_file} ▶ ignoring")
|
513
|
+
continue
|
514
|
+
file_name = os.path.join(tmpdir, f"{to_snake_case(schema_name)}.json")
|
515
|
+
with io.open(file_name, "w") as f:
|
516
|
+
json.dump(schema_data, f)
|
517
|
+
nfiles += 1
|
518
|
+
if nfiles > 0:
|
519
|
+
return _load_data(load=tmpdir, ini_file=ini_file,
|
520
|
+
verbose=verbose, debug=debug, noprogress=noprogress)
|
521
|
+
# TODO
|
522
|
+
return True
|
523
|
+
else:
|
524
|
+
_print(f"Unrecognized JSON data in file: {inserts_file}")
|
525
|
+
return False
|
526
|
+
return True
|
483
527
|
if verbose:
|
484
528
|
_print(f"Loading data files into Portal (via snovault.loadxl) from: {inserts_directory}")
|
485
529
|
_print(f"Portal INI file for load is: {ini_file}")
|
@@ -511,7 +555,7 @@ def _load_data(inserts_directory: str, ini_file: str,
|
|
511
555
|
copy_to_temporary_directory = True
|
512
556
|
if not schema_names_to_load:
|
513
557
|
_print("Directory contains no valid data: {inserts_directory}")
|
514
|
-
return
|
558
|
+
return False
|
515
559
|
if copy_to_temporary_directory:
|
516
560
|
with temporary_directory() as tmpdir:
|
517
561
|
if debug:
|
@@ -526,6 +570,16 @@ def _load_data(inserts_directory: str, ini_file: str,
|
|
526
570
|
loadxl(portal=portal, inserts_directory=inserts_directory, schema_names_to_load=schema_names_to_load)
|
527
571
|
if verbose:
|
528
572
|
_print(f"Done loading data into Portal (via snovault.loadxl) files from: {inserts_directory}")
|
573
|
+
return True
|
574
|
+
|
575
|
+
|
576
|
+
def _is_schema_name_list(portal: Portal, keys: list) -> bool:
|
577
|
+
if isinstance(keys, list):
|
578
|
+
for key in keys:
|
579
|
+
if portal.get_schema(key) is None:
|
580
|
+
return False
|
581
|
+
return True
|
582
|
+
return False
|
529
583
|
|
530
584
|
|
531
585
|
def _prune_data_for_update(data: dict, noignore: bool = False, ignore: Optional[List[str]] = None) -> dict:
|
@@ -603,13 +657,15 @@ def _parse_delete_fields(value: str) -> str:
|
|
603
657
|
|
604
658
|
|
605
659
|
def _get_schema_name_from_schema_named_json_file_name(portal: Portal, value: str) -> Optional[str]:
|
606
|
-
|
607
|
-
|
608
|
-
|
609
|
-
|
610
|
-
|
611
|
-
|
612
|
-
|
660
|
+
if isinstance(value, str) and value:
|
661
|
+
try:
|
662
|
+
if value.endswith(".json"):
|
663
|
+
value = value[:-5]
|
664
|
+
_, schema_name = _get_schema(portal, os.path.basename(value))
|
665
|
+
return schema_name
|
666
|
+
except Exception:
|
667
|
+
pass
|
668
|
+
return False
|
613
669
|
|
614
670
|
|
615
671
|
@lru_cache(maxsize=1)
|
@@ -60,7 +60,7 @@ dcicutils/s3_utils.py,sha256=h2B9ftOo-kxqfiKth5ZDC_cAUFy1Pbu7BrVanFnE5Iw,28839
|
|
60
60
|
dcicutils/schema_utils.py,sha256=GmRm-XqZKJ6qine16SQF1txcby9WougDav_sYmKNs9E,12400
|
61
61
|
dcicutils/scripts/publish_to_pypi.py,sha256=sMd4WASQGlxlh7uLrt2eGkFRXYgONVmvIg8mClMS5RQ,13903
|
62
62
|
dcicutils/scripts/run_license_checker.py,sha256=z2keYnRDZsHQbTeo1XORAXSXNJK5axVzL5LjiNqZ7jE,4184
|
63
|
-
dcicutils/scripts/update_portal_object.py,sha256=
|
63
|
+
dcicutils/scripts/update_portal_object.py,sha256=fLgsPgnugKLS18A8JwP8O_UdmeFqNIjUIhrEFlIhfgE,32589
|
64
64
|
dcicutils/scripts/view_portal_object.py,sha256=lcgXWH9ooVf7tJDIRnoFGOgT0wYLGhiJlJW3a9w6A_c,36983
|
65
65
|
dcicutils/secrets_utils.py,sha256=8dppXAsiHhJzI6NmOcvJV5ldvKkQZzh3Fl-cb8Wm7MI,19745
|
66
66
|
dcicutils/sheet_utils.py,sha256=VlmzteONW5VF_Q4vo0yA5vesz1ViUah1MZ_yA1rwZ0M,33629
|
@@ -75,8 +75,8 @@ dcicutils/trace_utils.py,sha256=g8kwV4ebEy5kXW6oOrEAUsurBcCROvwtZqz9fczsGRE,1769
|
|
75
75
|
dcicutils/validation_utils.py,sha256=cMZIU2cY98FYtzK52z5WUYck7urH6JcqOuz9jkXpqzg,14797
|
76
76
|
dcicutils/variant_utils.py,sha256=2H9azNx3xAj-MySg-uZ2SFqbWs4kZvf61JnK6b-h4Qw,4343
|
77
77
|
dcicutils/zip_utils.py,sha256=_Y9EmL3D2dUZhxucxHvrtmmlbZmK4FpSsHEb7rGSJLU,3265
|
78
|
-
dcicutils-8.14.0.
|
79
|
-
dcicutils-8.14.0.
|
80
|
-
dcicutils-8.14.0.
|
81
|
-
dcicutils-8.14.0.
|
82
|
-
dcicutils-8.14.0.
|
78
|
+
dcicutils-8.14.0.1b11.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
|
79
|
+
dcicutils-8.14.0.1b11.dist-info/METADATA,sha256=HFNj87yKbAwTA98cVDJm_0r9aVJssg7zo5dIsqngVrg,3440
|
80
|
+
dcicutils-8.14.0.1b11.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
|
81
|
+
dcicutils-8.14.0.1b11.dist-info/entry_points.txt,sha256=W6kEWdUJk9tQ4myAgpehPdebcwvCAZ7UgB-wyPgDUMg,335
|
82
|
+
dcicutils-8.14.0.1b11.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|