dcicutils 8.14.0.1b21__py3-none-any.whl → 8.14.0.1b22__py3-none-any.whl

dcicutils/scripts/update_portal_object.py

@@ -2,9 +2,16 @@
 # Command-line utility to update (post, patch, upsert) portal objects for SMaHT/CGAP/Fourfront.
 # ------------------------------------------------------------------------------------------------------
 # Example commands:
-# update-portal-object --post file_format.json
-# update-portal-object --upsert directory-with-schema-named-dot-json-files
-# update-portal-object --patch file-not-named-for-schema-name.json --schema UnalignedReads
+#
+# update-portal-object --load {json-file | directory-with-json-files}
+# update-portal-object --post {json-file | directory-with-json-files}
+# update-portal-object --upsert {json-file | directory-with-json-files}
+# update-portal-object --patch {json-file | directory-with-json-files}
+#
+# The specified json-file or file withing directory-with-jaon-files must be JSON containing either
+# a list of objects, which which case the file name for the target schema name, or if not, then
+# the --schema option must be used to specified the target schema; or the JSON must be a dictionary
+# of schema names, where the value of each is a list of objects for that schema.
 # --------------------------------------------------------------------------------------------------
 
 import argparse
@@ -464,8 +471,7 @@ def _load_data(portal: Portal, load: str, ini_file: str, explicit_schema_name: O
                 # view_name: \\\'22813a02-906b-4b60-b2b2-4afaea24aa28\\\', subpath: (), traversed: (), root:
                 # <encoded.root.SMAHTRoot object at 0x136d41460>, vroot: <encoded.root.SMAHTRoot object at 0x136d41460>, vroot_path: ()"}\' # noqa
                 #
-                if ((item_type := re.search(r"https?://.*/(.*)\?skip_indexing=.*", item)) and
-                    (len(item_type.groups()) == 1)):  # noqa
+                if (item_type := re.search(r"https?://.*/(.*)\?skip_indexing=.*", item)) and (len(item_type.groups()) == 1):  # noqa
                     item_type = to_snake_case(item_type.group(1))
                     identifying_value = f"/{to_camel_case(item_type)}{identifying_value}"
                 unresolved_link_error_message_prefix = "Unable to resolve link:"
@@ -628,7 +634,7 @@ def _load_data(portal: Portal, load: str, ini_file: str, explicit_schema_name: O
         _print(f"Total items loaded: {loadxl_total_item_count // 2}"  # TODO: straightend out this arithmetic
                f"{f' (errors: {loadxl_total_error_count})' if loadxl_total_error_count else ''}")
         for item in sorted(loadxl_summary.keys()):
-            _print(f"▷ {to_camel_case(item)}: {loadxl_summary[item]}")
+            _print(f"▷ {to_camel_case(item)}: {loadxl_summary[item] // 2}")  # TODO: straightend out this arithmetic
     if loadxl_unresolved:
         _print("✗ Unresolved references:")
         for item in loadxl_unresolved:

dcicutils/scripts/view_portal_object.py

@@ -290,6 +290,23 @@ def _get_portal_object(portal: Portal, uuid: str,
             _print()
         return None
 
+    def get_metadata_types(path: str) -> Optional[dict]:
+        nonlocal portal, debug
+        metadata_types = {}
+        try:
+            if debug:
+                _print(f"Executing separted query to get actual  metadata types for raw/inserts query.")
+            if ((response := portal.get(path)) and (response.status_code in [200, 307]) and
+                (response := response.json()) and (results := response.get("@graph"))):  # noqa
+                for result in results:
+                    if (result_type := result.get("@type")) and (result_uuid := result.get("uuid")):
+                        if ((isinstance(result_type, list) and (result_type := result_type[0])) or
+                            isinstance(result_type, str)):  # noqa
+                            metadata_types[result_uuid] = result_type
+        except Exception:
+            return None
+        return metadata_types
+
     response = None
     try:
         if not uuid.startswith("/"):
@@ -312,6 +329,7 @@ def _get_portal_object(portal: Portal, uuid: str,
     if not response.json:
         _exit(f"Invalid JSON getting Portal object: {uuid}")
     response = response.json()
+    response_types = {}
     if inserts:
         # Format results as suitable for inserts (e.g. via update-portal-object).
         response.pop("schema_version", None)
@@ -319,10 +337,12 @@ def _get_portal_object(portal: Portal, uuid: str,
             (isinstance(results_type := response.get("@type"), list) and results_type) and
             (isinstance(results_type := results_type[0], str) and results_type.endswith("SearchResults")) and
             (results_type := results_type[0:-len("SearchResults")])):  # noqa
-            # For search results, the type (from XyzSearchResults, above) may not be precisely correct for
-            # each of the results; it may be the supertype (e.g. QualityMetric vs QualityMetricWorkflowRun);
+            # For (raw frame) search results, the type (from XyzSearchResults, above) may not be precisely correct
+            # for each of the results; it may be the supertype (e.g. QualityMetric vs QualityMetricWorkflowRun);
             # so for types which are supertypes (gotten via Portal.get_schemas_super_type_map) we actually
-            # lookup each result individually to determine its actual precise type.
+            # lookup each result individually to determine its actual precise type. Although, if we have
+            # more than (say) 5 results to do this for, then do a separate query (get_metadata_types)
+            # to get the result types all at once.
             if not ((supertypes := portal.get_schemas_super_type_map()) and (subtypes := supertypes.get(results_type))):
                 subtypes = None
             response = {}
@@ -335,9 +355,16 @@ def _get_portal_object(portal: Portal, uuid: str,
                 result.pop("schema_version", None)
                 result = prune_data(result)
                 if (subtypes and one_or_more_objects_of_types_exists(portal, subtypes, debug=debug) and
-                    (result_uuid := result.get("uuid")) and
-                    (individual_result_type := get_metadata_for_individual_result_type(result_uuid))):  # noqa
-                    result_type = individual_result_type
+                    (result_uuid := result.get("uuid"))):  # noqa
+                    # If we have more than (say) 5 results for which we need to determine that actual result type,
+                    # then get them all at once via separate query (get_metadata_types)) which is not the raw frame.
+                    if (results_total > 5) and (not response_types):
+                        response_types = get_metadata_types(path)
+                    if not (response_types and (result_type := response_types.get(result_uuid))):
+                        if individual_result_type := get_metadata_for_individual_result_type(result_uuid):
+                            result_type = individual_result_type
+                        else:
+                            result_type = results_type
                 else:
                     result_type = results_type
                 if response.get(result_type):

{dcicutils-8.14.0.1b21.dist-info → dcicutils-8.14.0.1b22.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcicutils
-Version: 8.14.0.1b21
+Version: 8.14.0.1b22
 Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
 Home-page: https://github.com/4dn-dcic/utils
 License: MIT

{dcicutils-8.14.0.1b21.dist-info → dcicutils-8.14.0.1b22.dist-info}/RECORD

@@ -60,8 +60,8 @@ dcicutils/s3_utils.py,sha256=h2B9ftOo-kxqfiKth5ZDC_cAUFy1Pbu7BrVanFnE5Iw,28839
 dcicutils/schema_utils.py,sha256=GmRm-XqZKJ6qine16SQF1txcby9WougDav_sYmKNs9E,12400
 dcicutils/scripts/publish_to_pypi.py,sha256=sMd4WASQGlxlh7uLrt2eGkFRXYgONVmvIg8mClMS5RQ,13903
 dcicutils/scripts/run_license_checker.py,sha256=z2keYnRDZsHQbTeo1XORAXSXNJK5axVzL5LjiNqZ7jE,4184
-dcicutils/scripts/update_portal_object.py,sha256=_Wk9UYcT4XQKQQoqyTmiNEEh0ttY3TG1vMATsHr8W2Y,39661
-dcicutils/scripts/view_portal_object.py,sha256=lcgXWH9ooVf7tJDIRnoFGOgT0wYLGhiJlJW3a9w6A_c,36983
+dcicutils/scripts/update_portal_object.py,sha256=pRTzbfT4Ul64KwlBJa6cqfNRDIqPh2Uyi9BmXeAjoeU,40135
+dcicutils/scripts/view_portal_object.py,sha256=7I7wEBLFoVLpP8VZNGQuxh6N1PALJ38IKZTJ6WSNRRM,38654
 dcicutils/secrets_utils.py,sha256=8dppXAsiHhJzI6NmOcvJV5ldvKkQZzh3Fl-cb8Wm7MI,19745
 dcicutils/sheet_utils.py,sha256=VlmzteONW5VF_Q4vo0yA5vesz1ViUah1MZ_yA1rwZ0M,33629
 dcicutils/snapshot_utils.py,sha256=YDeI3vD-MhAtHwKDzfEm2q-n3l-da2yRpRR3xp0Ah1M,23021
@@ -75,8 +75,8 @@ dcicutils/trace_utils.py,sha256=g8kwV4ebEy5kXW6oOrEAUsurBcCROvwtZqz9fczsGRE,1769
 dcicutils/validation_utils.py,sha256=cMZIU2cY98FYtzK52z5WUYck7urH6JcqOuz9jkXpqzg,14797
 dcicutils/variant_utils.py,sha256=2H9azNx3xAj-MySg-uZ2SFqbWs4kZvf61JnK6b-h4Qw,4343
 dcicutils/zip_utils.py,sha256=_Y9EmL3D2dUZhxucxHvrtmmlbZmK4FpSsHEb7rGSJLU,3265
-dcicutils-8.14.0.1b21.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
-dcicutils-8.14.0.1b21.dist-info/METADATA,sha256=dBs3CAErEqR7h0pLE7K9CZKVJw5FOcSk1JCuZo7ZkZg,3440
-dcicutils-8.14.0.1b21.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
-dcicutils-8.14.0.1b21.dist-info/entry_points.txt,sha256=W6kEWdUJk9tQ4myAgpehPdebcwvCAZ7UgB-wyPgDUMg,335
-dcicutils-8.14.0.1b21.dist-info/RECORD,,
+dcicutils-8.14.0.1b22.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
+dcicutils-8.14.0.1b22.dist-info/METADATA,sha256=ACXqTzYQxjKfS18CdckhbfPZZF2AeIQ88Ta25nkDMPM,3440
+dcicutils-8.14.0.1b22.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+dcicutils-8.14.0.1b22.dist-info/entry_points.txt,sha256=W6kEWdUJk9tQ4myAgpehPdebcwvCAZ7UgB-wyPgDUMg,335
+dcicutils-8.14.0.1b22.dist-info/RECORD,,