dcicutils 8.14.0.1b2__tar.gz → 8.14.0.1b4__tar.gz

{dcicutils-8.14.0.1b2 → dcicutils-8.14.0.1b4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcicutils
-Version: 8.14.0.1b2
+Version: 8.14.0.1b4
 Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
 Home-page: https://github.com/4dn-dcic/utils
 License: MIT

{dcicutils-8.14.0.1b2 → dcicutils-8.14.0.1b4}/dcicutils/scripts/update_portal_object.py

@@ -123,7 +123,8 @@ def main():
     parser.add_argument("--delete", type=str, required=False, default=None, help="Delete data.")
     parser.add_argument("--purge", type=str, required=False, default=None, help="Purge data.")
     parser.add_argument("--noignore", action="store_true", required=False, default=False,
-                        help="Do not ignore standard fields on insert.")
+                        help="Do not ignore standard fields on update(s).")
+    parser.add_argument("--ignore", nargs="+", help="Ignore these additional fields.")
     parser.add_argument("--confirm", action="store_true", required=False, default=False, help="Confirm before action.")
     parser.add_argument("--verbose", action="store_true", required=False, default=False, help="Verbose output.")
     parser.add_argument("--quiet", action="store_true", required=False, default=False, help="Quiet output.")
@@ -158,7 +159,7 @@ def main():
                                  explicit_schema_name=explicit_schema_name,
                                  update_function=post_data,
                                  update_action_name="POST",
-                                 noignore=args.noignore,
+                                 noignore=args.noignore, ignore=args.ignore,
                                  confirm=args.confirm, verbose=args.verbose, quiet=args.quiet, debug=args.debug)
     if args.patch:
         _post_or_patch_or_upsert(portal=portal,
@@ -167,7 +168,7 @@ def main():
                                  update_function=patch_data,
                                  update_action_name="PATCH",
                                  patch_delete_fields=args.delete,
-                                 noignore=args.noignore,
+                                 noignore=args.noignore, ignore=args.ignore,
                                  confirm=args.confirm, verbose=args.verbose, quiet=args.quiet, debug=args.debug)
         args.delete = None
     if args.upsert:
@@ -177,7 +178,7 @@ def main():
                                  update_function=upsert_data,
                                  update_action_name="UPSERT",
                                  patch_delete_fields=args.delete,
-                                 noignore=args.noignore,
+                                 noignore=args.noignore, ignore=args.ignore,
                                  confirm=args.confirm, verbose=args.verbose, quiet=args.quiet, debug=args.debug)
         args.delete = None
 
@@ -201,7 +202,7 @@ def _post_or_patch_or_upsert(portal: Portal, file_or_directory: str,
                              explicit_schema_name: str,
                              update_function: Callable, update_action_name: str,
                              patch_delete_fields: Optional[str] = None,
-                             noignore: bool = False,
+                             noignore: bool = False, ignore: Optional[List[str]] = None,
                              confirm: bool = False, verbose: bool = False,
                              quiet: bool = False, debug: bool = False) -> None:
 
@@ -227,18 +228,21 @@ def _post_or_patch_or_upsert(portal: Portal, file_or_directory: str,
                     if debug:
                         _print(f"DEBUG: File ({file}) contains an object of type: {schema_name}")
                     update_function(portal, data, schema_name, file=file,
-                                    patch_delete_fields=patch_delete_fields, noignore=noignore,
+                                    patch_delete_fields=patch_delete_fields,
+                                    noignore=noignore, ignore=ignore,
                                     confirm=confirm, verbose=verbose, debug=debug)
                 elif is_schema_name_list(portal, list(data.keys())):
                     if debug:
                         _print(f"DEBUG: File ({file}) contains a dictionary of schema names.")
                     for schema_name in data:
                         if isinstance(schema_data := data[schema_name], list):
+                            schema_data = _impose_special_ordering(schema_data, schema_name)
                             if debug:
                                 _print(f"DEBUG: Processing {update_action_name}s for type: {schema_name}")
                             for index, item in enumerate(schema_data):
                                 update_function(portal, item, schema_name, file=file, index=index,
-                                                patch_delete_fields=patch_delete_fields, noignore=noignore,
+                                                patch_delete_fields=patch_delete_fields,
+                                                noignore=noignore, ignore=ignore,
                                                 confirm=confirm, verbose=verbose, debug=debug)
                         else:
                             _print(f"WARNING: File ({file}) contains schema item which is not a list: {schema_name}")
@@ -247,9 +251,12 @@ def _post_or_patch_or_upsert(portal: Portal, file_or_directory: str,
             elif isinstance(data, list):
                 if debug:
                     _print(f"DEBUG: File ({file}) contains a list of objects of type: {schema_name}")
+                import pdb ; pdb.set_trace()  # noqa
+                data = _impose_special_ordering(data, schema_name)
                 for index, item in enumerate(data):
                     update_function(portal, item, schema_name, file=file, index=index,
-                                    patch_delete_fields=patch_delete_fields, noignore=noignore,
+                                    patch_delete_fields=patch_delete_fields,
+                                    noignore=noignore, ignore=ignore,
                                     confirm=confirm, verbose=verbose, debug=debug)
             if debug:
                 _print(f"DEBUG: Processing {update_action_name} file done: {file}")
@@ -282,9 +289,16 @@ def _post_or_patch_or_upsert(portal: Portal, file_or_directory: str,
         _print(f"ERROR: Cannot find file or directory: {file_or_directory}")
 
 
+def _impose_special_ordering(data: List[dict], schema_name: str) -> List[dict]:
+    if schema_name == "FileFormat":
+        return sorted(data, key=lambda item: "extra_file_formats" in item)
+    return data
+
+
 def post_data(portal: Portal, data: dict, schema_name: str,
               file: Optional[str] = None, index: int = 0,
-              patch_delete_fields: Optional[str] = None, noignore: bool = False,
+              patch_delete_fields: Optional[str] = None,
+              noignore: bool = False, ignore: Optional[List[str]] = None,
               confirm: bool = False, verbose: bool = False, debug: bool = False) -> None:
     ignored(patch_delete_fields)
     if not (identifying_path := portal.get_identifying_path(data, portal_type=schema_name)):
@@ -301,7 +315,7 @@ def post_data(portal: Portal, data: dict, schema_name: str,
     if verbose:
         _print(f"POST {schema_name} item: {identifying_path}")
     try:
-        data = _prune_data_for_update(data, noignore=noignore)
+        data = _prune_data_for_update(data, noignore=noignore, ignore=ignore)
         portal.post_metadata(schema_name, data)
         if debug:
             _print(f"DEBUG: POST {schema_name} item done: {identifying_path}")
@@ -313,7 +327,8 @@ def post_data(portal: Portal, data: dict, schema_name: str,
 
 def patch_data(portal: Portal, data: dict, schema_name: str,
                file: Optional[str] = None, index: int = 0,
-               patch_delete_fields: Optional[str] = None, noignore: bool = False,
+               patch_delete_fields: Optional[str] = None,
+               noignore: bool = False, ignore: Optional[List[str]] = None,
                confirm: bool = False, verbose: bool = False, debug: bool = False) -> None:
     if not (identifying_path := portal.get_identifying_path(data, portal_type=schema_name)):
         if isinstance(file, str) and isinstance(index, int):
@@ -331,7 +346,7 @@ def patch_data(portal: Portal, data: dict, schema_name: str,
     try:
         if delete_fields := _parse_delete_fields(patch_delete_fields):
             identifying_path += f"?delete_fields={delete_fields}"
-        data = _prune_data_for_update(data, noignore=noignore)
+        data = _prune_data_for_update(data, noignore=noignore, ignore=ignore)
         portal.patch_metadata(identifying_path, data)
         if debug:
             _print(f"DEBUG: PATCH {schema_name} item OK: {identifying_path}")
@@ -343,7 +358,8 @@ def patch_data(portal: Portal, data: dict, schema_name: str,
 
 def upsert_data(portal: Portal, data: dict, schema_name: str,
                 file: Optional[str] = None, index: int = 0,
-                patch_delete_fields: Optional[str] = None, noignore: bool = False,
+                patch_delete_fields: Optional[str] = None,
+                noignore: bool = False, ignore: Optional[List[str]] = None,
                 confirm: bool = False, verbose: bool = False, debug: bool = False) -> None:
     if not (identifying_path := portal.get_identifying_path(data, portal_type=schema_name)):
         if isinstance(file, str) and isinstance(index, int):
@@ -358,12 +374,12 @@ def upsert_data(portal: Portal, data: dict, schema_name: str,
         _print(f"{'PATCH' if exists else 'POST'} {schema_name} item: {identifying_path}")
     try:
         if not exists:
-            data = _prune_data_for_update(data, noignore=noignore)
+            data = _prune_data_for_update(data, noignore=noignore, ignore=ignore)
             portal.post_metadata(schema_name, data)
         else:
             if delete_fields := _parse_delete_fields(patch_delete_fields):
                 identifying_path += f"?delete_fields={delete_fields}"
-            data = _prune_data_for_update(data, noignore=noignore)
+            data = _prune_data_for_update(data, noignore=noignore, ignore=ignore)
             portal.patch_metadata(identifying_path, data)
         if debug:
             _print(f"DEBUG: UPSERT {schema_name} item OK: {identifying_path}")
@@ -373,10 +389,13 @@ def upsert_data(portal: Portal, data: dict, schema_name: str,
         return
 
 
-def _prune_data_for_update(data: dict, noignore: bool = False) -> dict:
-    if noignore is True:
+def _prune_data_for_update(data: dict, noignore: bool = False, ignore: Optional[List[str]] = None) -> dict:
+    ignore_these_properties = [] if noignore is True else _IGNORE_PROPERTIES_ON_UPDATE
+    if isinstance(ignore, list):
+        ignore_these_properties = ignore_these_properties + ignore
+    if not ignore_these_properties:
         return data
-    return {key: value for key, value in data.items() if key not in _IGNORE_PROPERTIES_ON_UPDATE}
+    return {key: value for key, value in data.items() if key not in ignore_these_properties}
 
 
 def _create_portal(env: Optional[str] = None, app: Optional[str] = None,

{dcicutils-8.14.0.1b2 → dcicutils-8.14.0.1b4}/dcicutils/scripts/view_portal_object.py

@@ -104,6 +104,7 @@ def main():
     parser.add_argument("--raw", action="store_true", required=False, default=False, help="Raw output.")
     parser.add_argument("--inserts", action="store_true", required=False, default=False,
                         help="Format output for subsequent inserts.")
+    parser.add_argument("--ignore", nargs="+", help="Ignore these fields for --inserts.")
     parser.add_argument("--tree", action="store_true", required=False, default=False, help="Tree output for schemas.")
     parser.add_argument("--database", action="store_true", required=False, default=False,
                         help="Read from database output.")
@@ -191,7 +192,7 @@ def main():
             return
 
     data = _get_portal_object(portal=portal, uuid=args.uuid, raw=args.raw, inserts=args.inserts,
-                              database=args.database, check=args.bool, verbose=args.verbose)
+                              ignore=args.ignore, database=args.database, check=args.bool, verbose=args.verbose)
     if args.bool:
         if data:
             _print(f"{args.uuid}: found")
@@ -242,7 +243,15 @@ def _create_portal(ini: str, env: Optional[str] = None,
 
 def _get_portal_object(portal: Portal, uuid: str,
                        raw: bool = False, inserts: bool = False, database: bool = False,
+                       ignore: Optional[List[str]] = None,
                        check: bool = False, verbose: bool = False) -> dict:
+
+    def prune_data(data: dict) -> dict:
+        nonlocal ignore
+        if not isinstance(ignore, list) or not ignore:
+            return data
+        return {key: value for key, value in data.items() if key not in ignore}
+
     response = None
     try:
         if not uuid.startswith("/"):
@@ -300,6 +309,7 @@ def _get_portal_object(portal: Portal, uuid: str,
             for result in results:
                 results_index += 1
                 result.pop("schema_version", None)
+                result = prune_data(result)
                 if (subtypes and
                     (result_uuid := result.get("uuid")) and
                     (individual_result_type := get_metadata_for_individual_result_type(result_uuid))):  # noqa
@@ -313,7 +323,7 @@ def _get_portal_object(portal: Portal, uuid: str,
         # Get the result as non-raw so we can get its type.
         elif ((response_cooked := portal.get(path, database=database)) and
               (isinstance(response_type := response_cooked.json().get("@type"), list) and response_type)):
-            response = {f"{response_type[0]}": [response]}
+            response = {f"{response_type[0]}": [prune_data(response)]}
     elif raw:
         response.pop("schema_version", None)
     return response

{dcicutils-8.14.0.1b2 → dcicutils-8.14.0.1b4}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dcicutils"
-version = "8.14.0.1b2"  # TODO: To become 8.14.1
+version = "8.14.0.1b4"  # TODO: To become 8.14.1
 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources"
 authors = ["4DN-DCIC Team <support@4dnucleome.org>"]
 license = "MIT"