dcicutils 8.14.0.1b10__py3-none-any.whl → 8.14.0.1b11__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -162,7 +162,7 @@ def main():
162
162
  _print("The --env is not used for the --load option (to load data via snovault.loadxl).")
163
163
  if args.schema:
164
164
  _print("The --schema is not used for the --load option (to load data via snovault.loadxl).")
165
- _load_data(inserts_directory=args.load, ini_file=args.ini,
165
+ _load_data(load=args.load, ini_file=args.ini,
166
166
  verbose=args.verbose, debug=args.debug, noprogress=args.noprogress)
167
167
  exit(0)
168
168
 
@@ -226,14 +226,6 @@ def _post_or_patch_or_upsert(portal: Portal, file_or_directory: str,
226
226
  confirm: bool = False, verbose: bool = False,
227
227
  quiet: bool = False, debug: bool = False) -> None:
228
228
 
229
- def is_schema_name_list(portal: Portal, keys: list) -> bool:
230
- if isinstance(keys, list):
231
- for key in keys:
232
- if portal.get_schema(key) is None:
233
- return False
234
- return True
235
- return False
236
-
237
229
  def post_or_patch_or_upsert(portal: Portal, file: str, schema_name: Optional[str],
238
230
  patch_delete_fields: Optional[str] = None,
239
231
  confirm: bool = False, verbose: bool = False,
@@ -251,7 +243,7 @@ def _post_or_patch_or_upsert(portal: Portal, file_or_directory: str,
251
243
  patch_delete_fields=patch_delete_fields,
252
244
  noignore=noignore, ignore=ignore,
253
245
  confirm=confirm, verbose=verbose, debug=debug)
254
- elif is_schema_name_list(portal, list(data.keys())):
246
+ elif _is_schema_name_list(portal, list(data.keys())):
255
247
  if debug:
256
248
  _print(f"DEBUG: File ({file}) contains a dictionary of schema names.")
257
249
  for schema_name in data:
@@ -408,8 +400,8 @@ def _upsert_data(portal: Portal, data: dict, schema_name: str,
408
400
  return
409
401
 
410
402
 
411
- def _load_data(inserts_directory: str, ini_file: str,
412
- verbose: bool = False, debug: bool = False, noprogress: bool = False) -> None:
403
+ def _load_data(load: str, ini_file: str, explicit_schema_name: Optional[str] = None,
404
+ verbose: bool = False, debug: bool = False, noprogress: bool = False) -> bool:
413
405
 
414
406
  from snovault.loadxl import load_all_gen, LoadGenWrapper
415
407
  from dcicutils.captured_output import captured_output
@@ -431,7 +423,7 @@ def _load_data(inserts_directory: str, ini_file: str,
431
423
  return ""
432
424
 
433
425
  LOADXL_RESPONSE_PATTERN = re.compile(r"^([A-Z]+):\s*([a-zA-Z\/\d_-]+)\s*(\S+)\s*(\S+)?\s*(.*)$")
434
- LOADXL_ACTION_NAME = {"POST": "Create", "PATCH": "Update", "SKIP": "Skip",
426
+ LOADXL_ACTION_NAME = {"POST": "Create", "PATCH": "Update", "SKIP": "Check",
435
427
  "CHECK": "Validate", "ERROR": "Error"}
436
428
  current_item_type = None
437
429
  current_item_count = 0
@@ -472,14 +464,66 @@ def _load_data(inserts_directory: str, ini_file: str,
472
464
  if not os.path.exists(ini_file):
473
465
  _print(f"The INI file required for --load is not found: {ini_file}")
474
466
  exit(1)
475
- if not os.path.isabs(inserts_directory := os.path.expanduser(inserts_directory)):
476
- inserts_directory = os.path.join(os.getcwd(), inserts_directory)
477
- if not os.path.isdir(inserts_directory := os.path.expanduser(inserts_directory)):
478
- _print(f"Load directory does not exist: {inserts_directory}")
479
- exit(1)
467
+
468
+ if not os.path.isabs(load := os.path.expanduser(load)):
469
+ load = os.path.join(os.getcwd(), load)
470
+ if not os.path.exists(load):
471
+ return False
472
+
473
+ if os.path.isdir(load):
474
+ inserts_directory = load
475
+ inserts_file = None
476
+ else:
477
+ inserts_directory = None
478
+ inserts_file = load
479
+
480
480
  portal = None
481
481
  with captured_output(not debug):
482
482
  portal = Portal(ini_file)
483
+
484
+ if inserts_file:
485
+ with io.open(inserts_file, "r") as f:
486
+ try:
487
+ data = json.load(f)
488
+ except Exception:
489
+ _print(f"Cannot load JSON data from file: {inserts_file}")
490
+ return False
491
+ if isinstance(data, list):
492
+ if not (schema_name := explicit_schema_name):
493
+ if not (schema_name := _get_schema_name_from_schema_named_json_file_name(portal, inserts_file)):
494
+ _print("Unable to determine schema name for JSON data file: {inserts_file}")
495
+ return False
496
+ with temporary_directory() as tmpdir:
497
+ file_name = os.path.join(tmpdir, f"{to_snake_case(schema_name)}.json")
498
+ with io.open(file_name, "w") as f:
499
+ json.dump(data, f)
500
+ return _load_data(load=tmpdir, ini_file=ini_file, explicit_schema_name=explicit_schema_name,
501
+ verbose=verbose, debug=debug, noprogress=noprogress)
502
+ elif isinstance(data, dict):
503
+ _print("DICT IN FILE FOR LOAD NOT YET SUPPPORTED")
504
+ if not _is_schema_name_list(portal, schema_names := list(data.keys())):
505
+ _print(f"Unrecognized types in JSON data file: {inserts_file}")
506
+ return False
507
+ with temporary_directory() as tmpdir:
508
+ nfiles = 0
509
+ for schema_name in schema_names:
510
+ if not isinstance(schema_data := data[schema_name], list):
511
+ _print(f"Unexpected value for data type ({schema_name})"
512
+ f" in JSON data file: {inserts_file} ▶ ignoring")
513
+ continue
514
+ file_name = os.path.join(tmpdir, f"{to_snake_case(schema_name)}.json")
515
+ with io.open(file_name, "w") as f:
516
+ json.dump(schema_data, f)
517
+ nfiles += 1
518
+ if nfiles > 0:
519
+ return _load_data(load=tmpdir, ini_file=ini_file,
520
+ verbose=verbose, debug=debug, noprogress=noprogress)
521
+ # TODO
522
+ return True
523
+ else:
524
+ _print(f"Unrecognized JSON data in file: {inserts_file}")
525
+ return False
526
+ return True
483
527
  if verbose:
484
528
  _print(f"Loading data files into Portal (via snovault.loadxl) from: {inserts_directory}")
485
529
  _print(f"Portal INI file for load is: {ini_file}")
@@ -511,7 +555,7 @@ def _load_data(inserts_directory: str, ini_file: str,
511
555
  copy_to_temporary_directory = True
512
556
  if not schema_names_to_load:
513
557
  _print("Directory contains no valid data: {inserts_directory}")
514
- return
558
+ return False
515
559
  if copy_to_temporary_directory:
516
560
  with temporary_directory() as tmpdir:
517
561
  if debug:
@@ -526,6 +570,16 @@ def _load_data(inserts_directory: str, ini_file: str,
526
570
  loadxl(portal=portal, inserts_directory=inserts_directory, schema_names_to_load=schema_names_to_load)
527
571
  if verbose:
528
572
  _print(f"Done loading data into Portal (via snovault.loadxl) files from: {inserts_directory}")
573
+ return True
574
+
575
+
576
+ def _is_schema_name_list(portal: Portal, keys: list) -> bool:
577
+ if isinstance(keys, list):
578
+ for key in keys:
579
+ if portal.get_schema(key) is None:
580
+ return False
581
+ return True
582
+ return False
529
583
 
530
584
 
531
585
  def _prune_data_for_update(data: dict, noignore: bool = False, ignore: Optional[List[str]] = None) -> dict:
@@ -603,13 +657,15 @@ def _parse_delete_fields(value: str) -> str:
603
657
 
604
658
 
605
659
  def _get_schema_name_from_schema_named_json_file_name(portal: Portal, value: str) -> Optional[str]:
606
- try:
607
- if not value.endswith(".json"):
608
- return None
609
- _, schema_name = _get_schema(portal, os.path.basename(value[:-5]))
610
- return schema_name
611
- except Exception:
612
- return False
660
+ if isinstance(value, str) and value:
661
+ try:
662
+ if value.endswith(".json"):
663
+ value = value[:-5]
664
+ _, schema_name = _get_schema(portal, os.path.basename(value))
665
+ return schema_name
666
+ except Exception:
667
+ pass
668
+ return False
613
669
 
614
670
 
615
671
  @lru_cache(maxsize=1)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: dcicutils
3
- Version: 8.14.0.1b10
3
+ Version: 8.14.0.1b11
4
4
  Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
5
5
  Home-page: https://github.com/4dn-dcic/utils
6
6
  License: MIT
@@ -60,7 +60,7 @@ dcicutils/s3_utils.py,sha256=h2B9ftOo-kxqfiKth5ZDC_cAUFy1Pbu7BrVanFnE5Iw,28839
60
60
  dcicutils/schema_utils.py,sha256=GmRm-XqZKJ6qine16SQF1txcby9WougDav_sYmKNs9E,12400
61
61
  dcicutils/scripts/publish_to_pypi.py,sha256=sMd4WASQGlxlh7uLrt2eGkFRXYgONVmvIg8mClMS5RQ,13903
62
62
  dcicutils/scripts/run_license_checker.py,sha256=z2keYnRDZsHQbTeo1XORAXSXNJK5axVzL5LjiNqZ7jE,4184
63
- dcicutils/scripts/update_portal_object.py,sha256=OQ1v6QRJdVVUik_4RtNbMK2w7l4t-Htrm46pgj4kTNo,30063
63
+ dcicutils/scripts/update_portal_object.py,sha256=fLgsPgnugKLS18A8JwP8O_UdmeFqNIjUIhrEFlIhfgE,32589
64
64
  dcicutils/scripts/view_portal_object.py,sha256=lcgXWH9ooVf7tJDIRnoFGOgT0wYLGhiJlJW3a9w6A_c,36983
65
65
  dcicutils/secrets_utils.py,sha256=8dppXAsiHhJzI6NmOcvJV5ldvKkQZzh3Fl-cb8Wm7MI,19745
66
66
  dcicutils/sheet_utils.py,sha256=VlmzteONW5VF_Q4vo0yA5vesz1ViUah1MZ_yA1rwZ0M,33629
@@ -75,8 +75,8 @@ dcicutils/trace_utils.py,sha256=g8kwV4ebEy5kXW6oOrEAUsurBcCROvwtZqz9fczsGRE,1769
75
75
  dcicutils/validation_utils.py,sha256=cMZIU2cY98FYtzK52z5WUYck7urH6JcqOuz9jkXpqzg,14797
76
76
  dcicutils/variant_utils.py,sha256=2H9azNx3xAj-MySg-uZ2SFqbWs4kZvf61JnK6b-h4Qw,4343
77
77
  dcicutils/zip_utils.py,sha256=_Y9EmL3D2dUZhxucxHvrtmmlbZmK4FpSsHEb7rGSJLU,3265
78
- dcicutils-8.14.0.1b10.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
79
- dcicutils-8.14.0.1b10.dist-info/METADATA,sha256=A0R06QINC7rpiYDmPXmX_lGCeK6VN7KmdgP8do7ss54,3440
80
- dcicutils-8.14.0.1b10.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
81
- dcicutils-8.14.0.1b10.dist-info/entry_points.txt,sha256=W6kEWdUJk9tQ4myAgpehPdebcwvCAZ7UgB-wyPgDUMg,335
82
- dcicutils-8.14.0.1b10.dist-info/RECORD,,
78
+ dcicutils-8.14.0.1b11.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
79
+ dcicutils-8.14.0.1b11.dist-info/METADATA,sha256=HFNj87yKbAwTA98cVDJm_0r9aVJssg7zo5dIsqngVrg,3440
80
+ dcicutils-8.14.0.1b11.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
81
+ dcicutils-8.14.0.1b11.dist-info/entry_points.txt,sha256=W6kEWdUJk9tQ4myAgpehPdebcwvCAZ7UgB-wyPgDUMg,335
82
+ dcicutils-8.14.0.1b11.dist-info/RECORD,,