dcicutils 8.13.3.1b30__py3-none-any.whl → 8.14.0.1b2__py3-none-any.whl

dcicutils/misc_utils.py

@@ -1027,7 +1027,7 @@ _TO_NUMBER_MULTIPLIER_SUFFIXES = {
     "GB": _TO_NUMBER_POWER_OF_TEN_FOR_G,
     "T": _TO_NUMBER_POWER_OF_TEN_FOR_T,
     "TB": _TO_NUMBER_POWER_OF_TEN_FOR_T,
-    # B means bytes or bases and BP means base pairs; needs to be last.
+    # B means bytes or bases and BP means base pairs; this needs to be last.
     "B": _TO_NUMBER_POWER_OF_TEN_FOR_NOTHING,
     "BP": _TO_NUMBER_POWER_OF_TEN_FOR_NOTHING
 }
@@ -1093,9 +1093,9 @@ def to_number(value: str,
                 if not value_fraction:
                     return fallback
                 value = "0"
-    elif (as_float is not True) and (value_dot_zero_suffix := re.search(r"\.0*$", value)):
+    elif (as_float is not True) and (value_dot_zeros_suffix := re.search(r"\.0*$", value)):
         # Allow for example "123.00" to mean 123 (int).
-        value = value[:value_dot_zero_suffix.start()]
+        value = value[:value_dot_zeros_suffix.start()]
 
     if (allow_commas is True) and ("," in value):
         # Make sure any commas are properly placed/spaced.

dcicutils/scripts/update_portal_object.py

@@ -19,7 +19,7 @@ from typing import Callable, List, Optional, Tuple, Union
 from dcicutils.command_utils import yes_or_no
 from dcicutils.common import ORCHESTRATED_APPS, APP_SMAHT
 from dcicutils.ff_utils import delete_metadata, purge_metadata
-from dcicutils.misc_utils import get_error_message, PRINT
+from dcicutils.misc_utils import get_error_message, ignored, PRINT
 from dcicutils.portal_utils import Portal as PortalFromUtils
 
 
@@ -40,7 +40,7 @@ _DEFAULT_APP = "smaht"
 _SMAHT_ENV_ENVIRON_NAME = "SMAHT_ENV"
 
 # Schema properties to ignore (by default) for the view schema usage.
-_SCHEMAS_IGNORE_PROPERTIES = [
+_IGNORE_PROPERTIES_ON_UPDATE = [
     "date_created",
     "last_modified",
     "principals_allowed",
@@ -122,6 +122,8 @@ def main():
     parser.add_argument("--upsert", type=str, required=False, default=None, help="Upsert data.")
     parser.add_argument("--delete", type=str, required=False, default=None, help="Delete data.")
     parser.add_argument("--purge", type=str, required=False, default=None, help="Purge data.")
+    parser.add_argument("--noignore", action="store_true", required=False, default=False,
+                        help="Do not ignore standard fields on insert.")
     parser.add_argument("--confirm", action="store_true", required=False, default=False, help="Confirm before action.")
     parser.add_argument("--verbose", action="store_true", required=False, default=False, help="Verbose output.")
     parser.add_argument("--quiet", action="store_true", required=False, default=False, help="Quiet output.")
@@ -156,6 +158,7 @@ def main():
                                  explicit_schema_name=explicit_schema_name,
                                  update_function=post_data,
                                  update_action_name="POST",
+                                 noignore=args.noignore,
                                  confirm=args.confirm, verbose=args.verbose, quiet=args.quiet, debug=args.debug)
     if args.patch:
         _post_or_patch_or_upsert(portal=portal,
@@ -164,6 +167,7 @@ def main():
                                  update_function=patch_data,
                                  update_action_name="PATCH",
                                  patch_delete_fields=args.delete,
+                                 noignore=args.noignore,
                                  confirm=args.confirm, verbose=args.verbose, quiet=args.quiet, debug=args.debug)
         args.delete = None
     if args.upsert:
@@ -173,6 +177,7 @@ def main():
                                  update_function=upsert_data,
                                  update_action_name="UPSERT",
                                  patch_delete_fields=args.delete,
+                                 noignore=args.noignore,
                                  confirm=args.confirm, verbose=args.verbose, quiet=args.quiet, debug=args.debug)
         args.delete = None
 
@@ -196,6 +201,7 @@ def _post_or_patch_or_upsert(portal: Portal, file_or_directory: str,
                              explicit_schema_name: str,
                              update_function: Callable, update_action_name: str,
                              patch_delete_fields: Optional[str] = None,
+                             noignore: bool = False,
                              confirm: bool = False, verbose: bool = False,
                              quiet: bool = False, debug: bool = False) -> None:
 
@@ -221,7 +227,7 @@ def _post_or_patch_or_upsert(portal: Portal, file_or_directory: str,
                     if debug:
                         _print(f"DEBUG: File ({file}) contains an object of type: {schema_name}")
                     update_function(portal, data, schema_name, file=file,
-                                    patch_delete_fields=patch_delete_fields,
+                                    patch_delete_fields=patch_delete_fields, noignore=noignore,
                                     confirm=confirm, verbose=verbose, debug=debug)
                 elif is_schema_name_list(portal, list(data.keys())):
                     if debug:
@@ -232,7 +238,7 @@ def _post_or_patch_or_upsert(portal: Portal, file_or_directory: str,
                                 _print(f"DEBUG: Processing {update_action_name}s for type: {schema_name}")
                             for index, item in enumerate(schema_data):
                                 update_function(portal, item, schema_name, file=file, index=index,
-                                                patch_delete_fields=patch_delete_fields,
+                                                patch_delete_fields=patch_delete_fields, noignore=noignore,
                                                 confirm=confirm, verbose=verbose, debug=debug)
                         else:
                             _print(f"WARNING: File ({file}) contains schema item which is not a list: {schema_name}")
@@ -243,7 +249,7 @@ def _post_or_patch_or_upsert(portal: Portal, file_or_directory: str,
                     _print(f"DEBUG: File ({file}) contains a list of objects of type: {schema_name}")
                 for index, item in enumerate(data):
                     update_function(portal, item, schema_name, file=file, index=index,
-                                    patch_delete_fields=patch_delete_fields,
+                                    patch_delete_fields=patch_delete_fields, noignore=noignore,
                                     confirm=confirm, verbose=verbose, debug=debug)
             if debug:
                 _print(f"DEBUG: Processing {update_action_name} file done: {file}")
@@ -278,8 +284,9 @@ def _post_or_patch_or_upsert(portal: Portal, file_or_directory: str,
 
 def post_data(portal: Portal, data: dict, schema_name: str,
               file: Optional[str] = None, index: int = 0,
-              patch_delete_fields: Optional[str] = None,  # unused here
+              patch_delete_fields: Optional[str] = None, noignore: bool = False,
               confirm: bool = False, verbose: bool = False, debug: bool = False) -> None:
+    ignored(patch_delete_fields)
     if not (identifying_path := portal.get_identifying_path(data, portal_type=schema_name)):
         if isinstance(file, str) and isinstance(index, int):
             _print(f"ERROR: Item for POST has no identifying property: {file} (#{index + 1})")
@@ -294,6 +301,7 @@ def post_data(portal: Portal, data: dict, schema_name: str,
     if verbose:
         _print(f"POST {schema_name} item: {identifying_path}")
     try:
+        data = _prune_data_for_update(data, noignore=noignore)
         portal.post_metadata(schema_name, data)
         if debug:
             _print(f"DEBUG: POST {schema_name} item done: {identifying_path}")
@@ -305,7 +313,7 @@ def post_data(portal: Portal, data: dict, schema_name: str,
 
 def patch_data(portal: Portal, data: dict, schema_name: str,
                file: Optional[str] = None, index: int = 0,
-               patch_delete_fields: Optional[str] = None,
+               patch_delete_fields: Optional[str] = None, noignore: bool = False,
                confirm: bool = False, verbose: bool = False, debug: bool = False) -> None:
     if not (identifying_path := portal.get_identifying_path(data, portal_type=schema_name)):
         if isinstance(file, str) and isinstance(index, int):
@@ -323,6 +331,7 @@ def patch_data(portal: Portal, data: dict, schema_name: str,
     try:
         if delete_fields := _parse_delete_fields(patch_delete_fields):
             identifying_path += f"?delete_fields={delete_fields}"
+        data = _prune_data_for_update(data, noignore=noignore)
         portal.patch_metadata(identifying_path, data)
         if debug:
             _print(f"DEBUG: PATCH {schema_name} item OK: {identifying_path}")
@@ -334,7 +343,7 @@ def patch_data(portal: Portal, data: dict, schema_name: str,
 
 def upsert_data(portal: Portal, data: dict, schema_name: str,
                 file: Optional[str] = None, index: int = 0,
-                patch_delete_fields: Optional[str] = None,
+                patch_delete_fields: Optional[str] = None, noignore: bool = False,
                 confirm: bool = False, verbose: bool = False, debug: bool = False) -> None:
     if not (identifying_path := portal.get_identifying_path(data, portal_type=schema_name)):
         if isinstance(file, str) and isinstance(index, int):
@@ -349,10 +358,12 @@ def upsert_data(portal: Portal, data: dict, schema_name: str,
         _print(f"{'PATCH' if exists else 'POST'} {schema_name} item: {identifying_path}")
     try:
         if not exists:
+            data = _prune_data_for_update(data, noignore=noignore)
             portal.post_metadata(schema_name, data)
         else:
             if delete_fields := _parse_delete_fields(patch_delete_fields):
                 identifying_path += f"?delete_fields={delete_fields}"
+            data = _prune_data_for_update(data, noignore=noignore)
             portal.patch_metadata(identifying_path, data)
         if debug:
             _print(f"DEBUG: UPSERT {schema_name} item OK: {identifying_path}")
@@ -362,6 +373,12 @@ def upsert_data(portal: Portal, data: dict, schema_name: str,
         return
 
 
+def _prune_data_for_update(data: dict, noignore: bool = False) -> dict:
+    if noignore is True:
+        return data
+    return {key: value for key, value in data.items() if key not in _IGNORE_PROPERTIES_ON_UPDATE}
+
+
 def _create_portal(env: Optional[str] = None, app: Optional[str] = None,
                    verbose: bool = False, debug: bool = False) -> Optional[Portal]:
 

dcicutils/scripts/view_portal_object.py

@@ -279,13 +279,31 @@ def _get_portal_object(portal: Portal, uuid: str,
             if not ((supertypes := portal.get_schemas_super_type_map()) and (subtypes := supertypes.get(results_type))):
                 subtypes = None
             response = {}
+            results_index = 0
+            results_total = len(results)
+            def get_metadata_for_individual_result_type(uuid: str) -> Optional[dict]:  # noqa
+                # There can be a lot of individual results for which we may need to get the actual type,
+                # so do this in a function we were can give verbose output feedback.
+                nonlocal portal, results_index, results_total, verbose
+                if verbose:
+                    _print(f"Getting actual type for {results_type} result:"
+                           f" {uuid} [{results_index} of {results_total}]", end="")
+                result = portal.get_metadata(uuid, raise_exception=False)
+                if (isinstance(result_types := result.get("@type"), list) and
+                    result_types and (result_type := result_types[0])):  # noqa
+                    if verbose:
+                        _print(f" -> {result_type}")
+                    return result_type
+                if verbose:
+                    _print()
+                return None
             for result in results:
+                results_index += 1
                 result.pop("schema_version", None)
                 if (subtypes and
                     (result_uuid := result.get("uuid")) and
-                    (individual_result := portal.get_metadata(result_uuid, raise_exception=False)) and
-                    isinstance(result_type:= individual_result.get("@type"), list) and result_type and result_type[0]):  # noqa
-                    result_type = result_type[0]
+                    (individual_result_type := get_metadata_for_individual_result_type(result_uuid))):  # noqa
+                    result_type = individual_result_type
                 else:
                     result_type = results_type
                 if response.get(result_type):

{dcicutils-8.13.3.1b30.dist-info → dcicutils-8.14.0.1b2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcicutils
-Version: 8.13.3.1b30
+Version: 8.14.0.1b2
 Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
 Home-page: https://github.com/4dn-dcic/utils
 License: MIT

{dcicutils-8.13.3.1b30.dist-info → dcicutils-8.14.0.1b2.dist-info}/RECORD

@@ -45,7 +45,7 @@ dcicutils/license_policies/park-lab-gpl-pipeline.jsonc,sha256=vLZkwm3Js-kjV44nug
 dcicutils/license_policies/park-lab-pipeline.jsonc,sha256=9qlY0ASy3iUMQlr3gorVcXrSfRHnVGbLhkS427UaRy4,283
 dcicutils/license_utils.py,sha256=2Yxnh1T1iuMe6wluwbvpFO_zYSGPxB4-STAMc-vz-YM,47202
 dcicutils/log_utils.py,sha256=7pWMc6vyrorUZQf-V-M3YC6zrPgNhuV_fzm9xqTPph0,10883
-dcicutils/misc_utils.py,sha256=yoccfuCRlHOky31ljxtMDugrNE4QHqaweGWOWjWHI94,115922
+dcicutils/misc_utils.py,sha256=mO-5i0IUZ2U-P7rnQcC9VURzQ35pz7jFsMZdq6BB_QM,115929
 dcicutils/obfuscation_utils.py,sha256=fo2jOmDRC6xWpYX49u80bVNisqRRoPskFNX3ymFAmjw,5963
 dcicutils/opensearch_utils.py,sha256=V2exmFYW8Xl2_pGFixF4I2Cc549Opwe4PhFi5twC0M8,1017
 dcicutils/portal_object_utils.py,sha256=Az3n1aL-PQkN5gOFE6ZqC2XkYsqiwKlq7-tZggs1QN4,11062
@@ -60,8 +60,8 @@ dcicutils/s3_utils.py,sha256=h2B9ftOo-kxqfiKth5ZDC_cAUFy1Pbu7BrVanFnE5Iw,28839
 dcicutils/schema_utils.py,sha256=GmRm-XqZKJ6qine16SQF1txcby9WougDav_sYmKNs9E,12400
 dcicutils/scripts/publish_to_pypi.py,sha256=sMd4WASQGlxlh7uLrt2eGkFRXYgONVmvIg8mClMS5RQ,13903
 dcicutils/scripts/run_license_checker.py,sha256=z2keYnRDZsHQbTeo1XORAXSXNJK5axVzL5LjiNqZ7jE,4184
-dcicutils/scripts/update_portal_object.py,sha256=Lrgj01JuxmQSoWikukYgwm5LKi9CgSzjAogoMIBsQC4,20373
-dcicutils/scripts/view_portal_object.py,sha256=h8COy0lcLNWF9b5spjrlQ28wfqyTTMqAeC_xpFXutus,32262
+dcicutils/scripts/update_portal_object.py,sha256=gIO0fz6_SrfttYrdhfX7BVAYdtC4VZDXomycPrNWdbI,21395
+dcicutils/scripts/view_portal_object.py,sha256=yQAGxFWBZ7UU8vrD8xbeI_oS4oU2mXZBaQPCraLbdCU,33241
 dcicutils/secrets_utils.py,sha256=8dppXAsiHhJzI6NmOcvJV5ldvKkQZzh3Fl-cb8Wm7MI,19745
 dcicutils/sheet_utils.py,sha256=VlmzteONW5VF_Q4vo0yA5vesz1ViUah1MZ_yA1rwZ0M,33629
 dcicutils/snapshot_utils.py,sha256=YDeI3vD-MhAtHwKDzfEm2q-n3l-da2yRpRR3xp0Ah1M,23021
@@ -75,8 +75,8 @@ dcicutils/trace_utils.py,sha256=g8kwV4ebEy5kXW6oOrEAUsurBcCROvwtZqz9fczsGRE,1769
 dcicutils/validation_utils.py,sha256=cMZIU2cY98FYtzK52z5WUYck7urH6JcqOuz9jkXpqzg,14797
 dcicutils/variant_utils.py,sha256=2H9azNx3xAj-MySg-uZ2SFqbWs4kZvf61JnK6b-h4Qw,4343
 dcicutils/zip_utils.py,sha256=_Y9EmL3D2dUZhxucxHvrtmmlbZmK4FpSsHEb7rGSJLU,3265
-dcicutils-8.13.3.1b30.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
-dcicutils-8.13.3.1b30.dist-info/METADATA,sha256=W41FLeNYNat2V-1BvXOE60D4QN1N-Gj7PdjYXESWib4,3440
-dcicutils-8.13.3.1b30.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
-dcicutils-8.13.3.1b30.dist-info/entry_points.txt,sha256=W6kEWdUJk9tQ4myAgpehPdebcwvCAZ7UgB-wyPgDUMg,335
-dcicutils-8.13.3.1b30.dist-info/RECORD,,
+dcicutils-8.14.0.1b2.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
+dcicutils-8.14.0.1b2.dist-info/METADATA,sha256=uiJoLtk6VKcjRjvIlCDmF_8ADNo31t_Kuf1fxtv5OBM,3439
+dcicutils-8.14.0.1b2.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+dcicutils-8.14.0.1b2.dist-info/entry_points.txt,sha256=W6kEWdUJk9tQ4myAgpehPdebcwvCAZ7UgB-wyPgDUMg,335
+dcicutils-8.14.0.1b2.dist-info/RECORD,,