dcicutils 8.14.0.1b10__py3-none-any.whl → 8.14.0.1b12__py3-none-any.whl

dcicutils/scripts/update_portal_object.py

@@ -162,7 +162,7 @@ def main():
             _print("The --env is not used for the --load option (to load data via snovault.loadxl).")
         if args.schema:
             _print("The --schema is not used for the --load option (to load data via snovault.loadxl).")
-        _load_data(inserts_directory=args.load, ini_file=args.ini,
+        _load_data(load=args.load, ini_file=args.ini,
                    verbose=args.verbose, debug=args.debug, noprogress=args.noprogress)
         exit(0)
 
@@ -226,14 +226,6 @@ def _post_or_patch_or_upsert(portal: Portal, file_or_directory: str,
                              confirm: bool = False, verbose: bool = False,
                              quiet: bool = False, debug: bool = False) -> None:
 
-    def is_schema_name_list(portal: Portal, keys: list) -> bool:
-        if isinstance(keys, list):
-            for key in keys:
-                if portal.get_schema(key) is None:
-                    return False
-                return True
-        return False
-
     def post_or_patch_or_upsert(portal: Portal, file: str, schema_name: Optional[str],
                                 patch_delete_fields: Optional[str] = None,
                                 confirm: bool = False, verbose: bool = False,
@@ -251,7 +243,7 @@ def _post_or_patch_or_upsert(portal: Portal, file_or_directory: str,
                                     patch_delete_fields=patch_delete_fields,
                                     noignore=noignore, ignore=ignore,
                                     confirm=confirm, verbose=verbose, debug=debug)
-                elif is_schema_name_list(portal, list(data.keys())):
+                elif _is_schema_name_list(portal, list(data.keys())):
                     if debug:
                         _print(f"DEBUG: File ({file}) contains a dictionary of schema names.")
                     for schema_name in data:
@@ -408,8 +400,9 @@ def _upsert_data(portal: Portal, data: dict, schema_name: str,
         return
 
 
-def _load_data(inserts_directory: str, ini_file: str,
-               verbose: bool = False, debug: bool = False, noprogress: bool = False) -> None:
+def _load_data(load: str, ini_file: str, explicit_schema_name: Optional[str] = None,
+               verbose: bool = False, debug: bool = False, noprogress: bool = False,
+               _portal: Optional[Portal] = None, _single_insert_file: Optional[str] = None) -> bool:
 
     from snovault.loadxl import load_all_gen, LoadGenWrapper
     from dcicutils.captured_output import captured_output
@@ -419,7 +412,7 @@ def _load_data(inserts_directory: str, ini_file: str,
 
         nonlocal LoadGenWrapper, load_all_gen, verbose, debug
         progress_total = sum(schema_names_to_load.values()) * 2  # loadxl does two passes
-        progress_bar = ProgressBar(progress_total) if not noprogress else None
+        progress_bar = ProgressBar(progress_total, interrupt_exit=True) if not noprogress else None
 
         def decode_bytes(str_or_bytes: Union[str, bytes], *, encoding: str = "utf-8") -> str:
             if not isinstance(encoding, str):
@@ -431,7 +424,7 @@ def _load_data(inserts_directory: str, ini_file: str,
             return ""
 
         LOADXL_RESPONSE_PATTERN = re.compile(r"^([A-Z]+):\s*([a-zA-Z\/\d_-]+)\s*(\S+)\s*(\S+)?\s*(.*)$")
-        LOADXL_ACTION_NAME = {"POST": "Create", "PATCH": "Update", "SKIP": "Skip",
+        LOADXL_ACTION_NAME = {"POST": "Create", "PATCH": "Update", "SKIP": "Check",
                               "CHECK": "Validate", "ERROR": "Error"}
         current_item_type = None
         current_item_count = 0
@@ -472,16 +465,73 @@ def _load_data(inserts_directory: str, ini_file: str,
     if not os.path.exists(ini_file):
         _print(f"The INI file required for --load is not found: {ini_file}")
         exit(1)
-    if not os.path.isabs(inserts_directory := os.path.expanduser(inserts_directory)):
-        inserts_directory = os.path.join(os.getcwd(), inserts_directory)
-    if not os.path.isdir(inserts_directory := os.path.expanduser(inserts_directory)):
-        _print(f"Load directory does not exist: {inserts_directory}")
-        exit(1)
-    portal = None
-    with captured_output(not debug):
-        portal = Portal(ini_file)
+
+    if not os.path.isabs(load := os.path.expanduser(load)):
+        load = os.path.join(os.getcwd(), load)
+    if not os.path.exists(load):
+        return False
+
+    if os.path.isdir(load):
+        inserts_directory = load
+        inserts_file = None
+    else:
+        inserts_directory = None
+        inserts_file = load
+
+    if not (portal := _portal):
+        with captured_output(not debug):
+            portal = Portal(ini_file)
+
+    if inserts_file:
+        with io.open(inserts_file, "r") as f:
+            try:
+                data = json.load(f)
+            except Exception:
+                _print(f"Cannot load JSON data from file: {inserts_file}")
+                return False
+            if isinstance(data, list):
+                if not (schema_name := explicit_schema_name):
+                    if not (schema_name := _get_schema_name_from_schema_named_json_file_name(portal, inserts_file)):
+                        _print("Unable to determine schema name for JSON data file: {inserts_file}")
+                        return False
+                with temporary_directory() as tmpdir:
+                    file_name = os.path.join(tmpdir, f"{to_snake_case(schema_name)}.json")
+                    with io.open(file_name, "w") as f:
+                        json.dump(data, f)
+                    return _load_data(load=tmpdir, ini_file=ini_file, explicit_schema_name=explicit_schema_name,
+                                      verbose=verbose, debug=debug, noprogress=noprogress,
+                                      _portal=portal, _single_insert_file=inserts_file)
+            elif isinstance(data, dict):
+                _print("DICT IN FILE FOR LOAD NOT YET SUPPPORTED")
+                if not _is_schema_name_list(portal, schema_names := list(data.keys())):
+                    _print(f"Unrecognized types in JSON data file: {inserts_file}")
+                    return False
+                with temporary_directory() as tmpdir:
+                    nfiles = 0
+                    for schema_name in schema_names:
+                        if not isinstance(schema_data := data[schema_name], list):
+                            _print(f"Unexpected value for data type ({schema_name})"
+                                   f" in JSON data file: {inserts_file} ▶ ignoring")
+                            continue
+                        file_name = os.path.join(tmpdir, f"{to_snake_case(schema_name)}.json")
+                        with io.open(file_name, "w") as f:
+                            json.dump(schema_data, f)
+                        nfiles += 1
+                    if nfiles > 0:
+                        return _load_data(load=tmpdir, ini_file=ini_file,
+                                          verbose=verbose, debug=debug, noprogress=noprogress,
+                                          _portal=portal, _single_insert_file=inserts_file)
+                # TODO
+                return True
+            else:
+                _print(f"Unrecognized JSON data in file: {inserts_file}")
+                return False
+        return True
     if verbose:
-        _print(f"Loading data files into Portal (via snovault.loadxl) from: {inserts_directory}")
+        if _single_insert_file:
+            _print(f"Loading data file into Portal (via snovault.loadxl) from: {_single_insert_file}")
+        else:
+            _print(f"Loading data files into Portal (via snovault.loadxl) from: {inserts_directory}")
         _print(f"Portal INI file for load is: {ini_file}")
 
     schema_names = list(_get_schemas(portal).keys())
@@ -511,7 +561,7 @@ def _load_data(inserts_directory: str, ini_file: str,
                 copy_to_temporary_directory = True
     if not schema_names_to_load:
         _print("Directory contains no valid data: {inserts_directory}")
-        return
+        return False
     if copy_to_temporary_directory:
         with temporary_directory() as tmpdir:
             if debug:
@@ -526,6 +576,16 @@ def _load_data(inserts_directory: str, ini_file: str,
         loadxl(portal=portal, inserts_directory=inserts_directory, schema_names_to_load=schema_names_to_load)
     if verbose:
         _print(f"Done loading data into Portal (via snovault.loadxl) files from: {inserts_directory}")
+    return True
+
+
+def _is_schema_name_list(portal: Portal, keys: list) -> bool:
+    if isinstance(keys, list):
+        for key in keys:
+            if portal.get_schema(key) is None:
+                return False
+            return True
+    return False
 
 
 def _prune_data_for_update(data: dict, noignore: bool = False, ignore: Optional[List[str]] = None) -> dict:
@@ -603,13 +663,15 @@ def _parse_delete_fields(value: str) -> str:
 
 
 def _get_schema_name_from_schema_named_json_file_name(portal: Portal, value: str) -> Optional[str]:
-    try:
-        if not value.endswith(".json"):
-            return None
-        _, schema_name = _get_schema(portal, os.path.basename(value[:-5]))
-        return schema_name
-    except Exception:
-        return False
+    if isinstance(value, str) and value:
+        try:
+            if value.endswith(".json"):
+                value = value[:-5]
+            _, schema_name = _get_schema(portal, os.path.basename(value))
+            return schema_name
+        except Exception:
+            pass
+    return False
 
 
 @lru_cache(maxsize=1)

{dcicutils-8.14.0.1b10.dist-info → dcicutils-8.14.0.1b12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcicutils
-Version: 8.14.0.1b10
+Version: 8.14.0.1b12
 Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
 Home-page: https://github.com/4dn-dcic/utils
 License: MIT

{dcicutils-8.14.0.1b10.dist-info → dcicutils-8.14.0.1b12.dist-info}/RECORD

@@ -60,7 +60,7 @@ dcicutils/s3_utils.py,sha256=h2B9ftOo-kxqfiKth5ZDC_cAUFy1Pbu7BrVanFnE5Iw,28839
 dcicutils/schema_utils.py,sha256=GmRm-XqZKJ6qine16SQF1txcby9WougDav_sYmKNs9E,12400
 dcicutils/scripts/publish_to_pypi.py,sha256=sMd4WASQGlxlh7uLrt2eGkFRXYgONVmvIg8mClMS5RQ,13903
 dcicutils/scripts/run_license_checker.py,sha256=z2keYnRDZsHQbTeo1XORAXSXNJK5axVzL5LjiNqZ7jE,4184
-dcicutils/scripts/update_portal_object.py,sha256=OQ1v6QRJdVVUik_4RtNbMK2w7l4t-Htrm46pgj4kTNo,30063
+dcicutils/scripts/update_portal_object.py,sha256=KSsbHVOaS7zGOSVYbB2Lf7FT_gYFAGk4ot8e-us2HkE,33057
 dcicutils/scripts/view_portal_object.py,sha256=lcgXWH9ooVf7tJDIRnoFGOgT0wYLGhiJlJW3a9w6A_c,36983
 dcicutils/secrets_utils.py,sha256=8dppXAsiHhJzI6NmOcvJV5ldvKkQZzh3Fl-cb8Wm7MI,19745
 dcicutils/sheet_utils.py,sha256=VlmzteONW5VF_Q4vo0yA5vesz1ViUah1MZ_yA1rwZ0M,33629
@@ -75,8 +75,8 @@ dcicutils/trace_utils.py,sha256=g8kwV4ebEy5kXW6oOrEAUsurBcCROvwtZqz9fczsGRE,1769
 dcicutils/validation_utils.py,sha256=cMZIU2cY98FYtzK52z5WUYck7urH6JcqOuz9jkXpqzg,14797
 dcicutils/variant_utils.py,sha256=2H9azNx3xAj-MySg-uZ2SFqbWs4kZvf61JnK6b-h4Qw,4343
 dcicutils/zip_utils.py,sha256=_Y9EmL3D2dUZhxucxHvrtmmlbZmK4FpSsHEb7rGSJLU,3265
-dcicutils-8.14.0.1b10.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
-dcicutils-8.14.0.1b10.dist-info/METADATA,sha256=A0R06QINC7rpiYDmPXmX_lGCeK6VN7KmdgP8do7ss54,3440
-dcicutils-8.14.0.1b10.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
-dcicutils-8.14.0.1b10.dist-info/entry_points.txt,sha256=W6kEWdUJk9tQ4myAgpehPdebcwvCAZ7UgB-wyPgDUMg,335
-dcicutils-8.14.0.1b10.dist-info/RECORD,,
+dcicutils-8.14.0.1b12.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
+dcicutils-8.14.0.1b12.dist-info/METADATA,sha256=iUlj60aXP1C2TpYySyc6ieQWkM3a11PKfLLFXEOFH9s,3440
+dcicutils-8.14.0.1b12.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+dcicutils-8.14.0.1b12.dist-info/entry_points.txt,sha256=W6kEWdUJk9tQ4myAgpehPdebcwvCAZ7UgB-wyPgDUMg,335
+dcicutils-8.14.0.1b12.dist-info/RECORD,,