dcicutils 8.14.0__tar.gz → 8.14.0.1b3__tar.gz

{dcicutils-8.14.0 → dcicutils-8.14.0.1b3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcicutils
-Version: 8.14.0
+Version: 8.14.0.1b3
 Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
 Home-page: https://github.com/4dn-dcic/utils
 License: MIT

{dcicutils-8.14.0 → dcicutils-8.14.0.1b3}/dcicutils/scripts/update_portal_object.py

@@ -19,7 +19,7 @@ from typing import Callable, List, Optional, Tuple, Union
 from dcicutils.command_utils import yes_or_no
 from dcicutils.common import ORCHESTRATED_APPS, APP_SMAHT
 from dcicutils.ff_utils import delete_metadata, purge_metadata
-from dcicutils.misc_utils import get_error_message, PRINT
+from dcicutils.misc_utils import get_error_message, ignored, PRINT
 from dcicutils.portal_utils import Portal as PortalFromUtils
 
 
@@ -40,7 +40,7 @@ _DEFAULT_APP = "smaht"
 _SMAHT_ENV_ENVIRON_NAME = "SMAHT_ENV"
 
 # Schema properties to ignore (by default) for the view schema usage.
-_SCHEMAS_IGNORE_PROPERTIES = [
+_IGNORE_PROPERTIES_ON_UPDATE = [
     "date_created",
     "last_modified",
     "principals_allowed",
@@ -122,6 +122,9 @@ def main():
     parser.add_argument("--upsert", type=str, required=False, default=None, help="Upsert data.")
     parser.add_argument("--delete", type=str, required=False, default=None, help="Delete data.")
     parser.add_argument("--purge", type=str, required=False, default=None, help="Purge data.")
+    parser.add_argument("--noignore", action="store_true", required=False, default=False,
+                        help="Do not ignore standard fields on update(s).")
+    parser.add_argument("--ignore", nargs="+", help="Ignore these additional fields.")
     parser.add_argument("--confirm", action="store_true", required=False, default=False, help="Confirm before action.")
     parser.add_argument("--verbose", action="store_true", required=False, default=False, help="Verbose output.")
     parser.add_argument("--quiet", action="store_true", required=False, default=False, help="Quiet output.")
@@ -156,6 +159,7 @@ def main():
                                  explicit_schema_name=explicit_schema_name,
                                  update_function=post_data,
                                  update_action_name="POST",
+                                 noignore=args.noignore, ignore=args.ignore,
                                  confirm=args.confirm, verbose=args.verbose, quiet=args.quiet, debug=args.debug)
     if args.patch:
         _post_or_patch_or_upsert(portal=portal,
@@ -164,6 +168,7 @@ def main():
                                  update_function=patch_data,
                                  update_action_name="PATCH",
                                  patch_delete_fields=args.delete,
+                                 noignore=args.noignore, ignore=args.ignore,
                                  confirm=args.confirm, verbose=args.verbose, quiet=args.quiet, debug=args.debug)
         args.delete = None
     if args.upsert:
@@ -173,6 +178,7 @@ def main():
                                  update_function=upsert_data,
                                  update_action_name="UPSERT",
                                  patch_delete_fields=args.delete,
+                                 noignore=args.noignore, ignore=args.ignore,
                                  confirm=args.confirm, verbose=args.verbose, quiet=args.quiet, debug=args.debug)
         args.delete = None
 
@@ -196,6 +202,7 @@ def _post_or_patch_or_upsert(portal: Portal, file_or_directory: str,
                              explicit_schema_name: str,
                              update_function: Callable, update_action_name: str,
                              patch_delete_fields: Optional[str] = None,
+                             noignore: bool = False, ignore: Optional[List[str]] = None,
                              confirm: bool = False, verbose: bool = False,
                              quiet: bool = False, debug: bool = False) -> None:
 
@@ -222,6 +229,7 @@ def _post_or_patch_or_upsert(portal: Portal, file_or_directory: str,
                         _print(f"DEBUG: File ({file}) contains an object of type: {schema_name}")
                     update_function(portal, data, schema_name, file=file,
                                     patch_delete_fields=patch_delete_fields,
+                                    noignore=noignore, ignore=ignore,
                                     confirm=confirm, verbose=verbose, debug=debug)
                 elif is_schema_name_list(portal, list(data.keys())):
                     if debug:
@@ -233,6 +241,7 @@ def _post_or_patch_or_upsert(portal: Portal, file_or_directory: str,
                             for index, item in enumerate(schema_data):
                                 update_function(portal, item, schema_name, file=file, index=index,
                                                 patch_delete_fields=patch_delete_fields,
+                                                noignore=noignore, ignore=ignore,
                                                 confirm=confirm, verbose=verbose, debug=debug)
                         else:
                             _print(f"WARNING: File ({file}) contains schema item which is not a list: {schema_name}")
@@ -244,6 +253,7 @@ def _post_or_patch_or_upsert(portal: Portal, file_or_directory: str,
                 for index, item in enumerate(data):
                     update_function(portal, item, schema_name, file=file, index=index,
                                     patch_delete_fields=patch_delete_fields,
+                                    noignore=noignore, ignore=ignore,
                                     confirm=confirm, verbose=verbose, debug=debug)
             if debug:
                 _print(f"DEBUG: Processing {update_action_name} file done: {file}")
@@ -278,8 +288,10 @@ def _post_or_patch_or_upsert(portal: Portal, file_or_directory: str,
 
 def post_data(portal: Portal, data: dict, schema_name: str,
               file: Optional[str] = None, index: int = 0,
-              patch_delete_fields: Optional[str] = None,  # unused here
+              patch_delete_fields: Optional[str] = None,
+              noignore: bool = False, ignore: Optional[List[str]] = None,
               confirm: bool = False, verbose: bool = False, debug: bool = False) -> None:
+    ignored(patch_delete_fields)
     if not (identifying_path := portal.get_identifying_path(data, portal_type=schema_name)):
         if isinstance(file, str) and isinstance(index, int):
             _print(f"ERROR: Item for POST has no identifying property: {file} (#{index + 1})")
@@ -294,6 +306,7 @@ def post_data(portal: Portal, data: dict, schema_name: str,
     if verbose:
         _print(f"POST {schema_name} item: {identifying_path}")
     try:
+        data = _prune_data_for_update(data, noignore=noignore, ignore=ignore)
         portal.post_metadata(schema_name, data)
         if debug:
             _print(f"DEBUG: POST {schema_name} item done: {identifying_path}")
@@ -306,6 +319,7 @@ def post_data(portal: Portal, data: dict, schema_name: str,
 def patch_data(portal: Portal, data: dict, schema_name: str,
                file: Optional[str] = None, index: int = 0,
                patch_delete_fields: Optional[str] = None,
+               noignore: bool = False, ignore: Optional[List[str]] = None,
                confirm: bool = False, verbose: bool = False, debug: bool = False) -> None:
     if not (identifying_path := portal.get_identifying_path(data, portal_type=schema_name)):
         if isinstance(file, str) and isinstance(index, int):
@@ -323,6 +337,7 @@ def patch_data(portal: Portal, data: dict, schema_name: str,
     try:
         if delete_fields := _parse_delete_fields(patch_delete_fields):
             identifying_path += f"?delete_fields={delete_fields}"
+        data = _prune_data_for_update(data, noignore=noignore, ignore=ignore)
         portal.patch_metadata(identifying_path, data)
         if debug:
             _print(f"DEBUG: PATCH {schema_name} item OK: {identifying_path}")
@@ -335,6 +350,7 @@ def patch_data(portal: Portal, data: dict, schema_name: str,
 def upsert_data(portal: Portal, data: dict, schema_name: str,
                 file: Optional[str] = None, index: int = 0,
                 patch_delete_fields: Optional[str] = None,
+                noignore: bool = False, ignore: Optional[List[str]] = None,
                 confirm: bool = False, verbose: bool = False, debug: bool = False) -> None:
     if not (identifying_path := portal.get_identifying_path(data, portal_type=schema_name)):
         if isinstance(file, str) and isinstance(index, int):
@@ -349,10 +365,12 @@ def upsert_data(portal: Portal, data: dict, schema_name: str,
         _print(f"{'PATCH' if exists else 'POST'} {schema_name} item: {identifying_path}")
     try:
         if not exists:
+            data = _prune_data_for_update(data, noignore=noignore, ignore=ignore)
             portal.post_metadata(schema_name, data)
         else:
             if delete_fields := _parse_delete_fields(patch_delete_fields):
                 identifying_path += f"?delete_fields={delete_fields}"
+            data = _prune_data_for_update(data, noignore=noignore, ignore=ignore)
             portal.patch_metadata(identifying_path, data)
         if debug:
             _print(f"DEBUG: UPSERT {schema_name} item OK: {identifying_path}")
@@ -362,6 +380,15 @@ def upsert_data(portal: Portal, data: dict, schema_name: str,
         return
 
 
+def _prune_data_for_update(data: dict, noignore: bool = False, ignore: Optional[List[str]] = None) -> dict:
+    ignore_these_properties = [] if noignore is True else _IGNORE_PROPERTIES_ON_UPDATE
+    if isinstance(ignore, list):
+        ignore_these_properties = ignore_these_properties + ignore
+    if not ignore_these_properties:
+        return data
+    return {key: value for key, value in data.items() if key not in ignore_these_properties}
+
+
 def _create_portal(env: Optional[str] = None, app: Optional[str] = None,
                    verbose: bool = False, debug: bool = False) -> Optional[Portal]:
 

{dcicutils-8.14.0 → dcicutils-8.14.0.1b3}/dcicutils/scripts/view_portal_object.py

@@ -104,6 +104,7 @@ def main():
     parser.add_argument("--raw", action="store_true", required=False, default=False, help="Raw output.")
     parser.add_argument("--inserts", action="store_true", required=False, default=False,
                         help="Format output for subsequent inserts.")
+    parser.add_argument("--ignore", nargs="+", help="Ignore these fields for --inserts.")
     parser.add_argument("--tree", action="store_true", required=False, default=False, help="Tree output for schemas.")
     parser.add_argument("--database", action="store_true", required=False, default=False,
                         help="Read from database output.")
@@ -191,7 +192,7 @@ def main():
             return
 
     data = _get_portal_object(portal=portal, uuid=args.uuid, raw=args.raw, inserts=args.inserts,
-                              database=args.database, check=args.bool, verbose=args.verbose)
+                              ignore=args.ignore, database=args.database, check=args.bool, verbose=args.verbose)
     if args.bool:
         if data:
             _print(f"{args.uuid}: found")
@@ -242,7 +243,15 @@ def _create_portal(ini: str, env: Optional[str] = None,
 
 def _get_portal_object(portal: Portal, uuid: str,
                        raw: bool = False, inserts: bool = False, database: bool = False,
+                       ignore: Optional[List[str]] = None,
                        check: bool = False, verbose: bool = False) -> dict:
+
+    def prune_data(data: dict) -> dict:
+        nonlocal ignore
+        if not isinstance(ignore, list) or not ignore:
+            return data
+        return {key: value for key, value in data.items() if key not in ignore}
+
     response = None
     try:
         if not uuid.startswith("/"):
@@ -279,13 +288,32 @@ def _get_portal_object(portal: Portal, uuid: str,
             if not ((supertypes := portal.get_schemas_super_type_map()) and (subtypes := supertypes.get(results_type))):
                 subtypes = None
             response = {}
+            results_index = 0
+            results_total = len(results)
+            def get_metadata_for_individual_result_type(uuid: str) -> Optional[dict]:  # noqa
+                # There can be a lot of individual results for which we may need to get the actual type,
+                # so do this in a function we were can give verbose output feedback.
+                nonlocal portal, results_index, results_total, verbose
+                if verbose:
+                    _print(f"Getting actual type for {results_type} result:"
+                           f" {uuid} [{results_index} of {results_total}]", end="")
+                result = portal.get_metadata(uuid, raise_exception=False)
+                if (isinstance(result_types := result.get("@type"), list) and
+                    result_types and (result_type := result_types[0])):  # noqa
+                    if verbose:
+                        _print(f" -> {result_type}")
+                    return result_type
+                if verbose:
+                    _print()
+                return None
             for result in results:
+                results_index += 1
                 result.pop("schema_version", None)
+                result = prune_data(result)
                 if (subtypes and
                     (result_uuid := result.get("uuid")) and
-                    (individual_result := portal.get_metadata(result_uuid, raise_exception=False)) and
-                    isinstance(result_type:= individual_result.get("@type"), list) and result_type and result_type[0]):  # noqa
-                    result_type = result_type[0]
+                    (individual_result_type := get_metadata_for_individual_result_type(result_uuid))):  # noqa
+                    result_type = individual_result_type
                 else:
                     result_type = results_type
                 if response.get(result_type):
@@ -295,7 +323,7 @@ def _get_portal_object(portal: Portal, uuid: str,
         # Get the result as non-raw so we can get its type.
         elif ((response_cooked := portal.get(path, database=database)) and
               (isinstance(response_type := response_cooked.json().get("@type"), list) and response_type)):
-            response = {f"{response_type[0]}": [response]}
+            response = {f"{response_type[0]}": [prune_data(response)]}
     elif raw:
         response.pop("schema_version", None)
     return response

{dcicutils-8.14.0 → dcicutils-8.14.0.1b3}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dcicutils"
-version = "8.14.0"
+version = "8.14.0.1b3"  # TODO: To become 8.14.1
 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources"
 authors = ["4DN-DCIC Team <support@4dnucleome.org>"]
 license = "MIT"