dcicutils 8.7.2.1b7__py3-none-any.whl → 8.8.0.1b1__py3-none-any.whl

dcicutils/portal_utils.py

@@ -331,15 +331,15 @@ class Portal:
         Returns the "super type map" for all of the known schemas (via /profiles).
         This is a dictionary with property names which are all known schema type names which
         have (one or more) sub-types, and the value of each such property name is an array
-        of all of those sub-types (direct and all descendents), in breadth first order.
+        of all of those sub-type names (direct and all descendents), in breadth first order.
         """
         def list_breadth_first(super_type_map: dict, super_type_name: str) -> dict:
             result = []
             queue = deque(super_type_map.get(super_type_name, []))
             while queue:
-                result.append(sub_type_name := queue.popleft())
-                if sub_type_name in super_type_map:
-                    queue.extend(super_type_map[sub_type_name])
+                result.append(subtype_name := queue.popleft())
+                if subtype_name in super_type_map:
+                    queue.extend(super_type_map[subtype_name])
             return result
         if not (schemas := self.get_schemas()):
             return {}

dcicutils/scripts/view_portal_object.py

@@ -56,15 +56,26 @@
 # --------------------------------------------------------------------------------------------------
 
 import argparse
+from functools import lru_cache
 import json
 import pyperclip
+import os
 import sys
-from typing import Optional
+from typing import Callable, List, Optional, Tuple
 import yaml
 from dcicutils.captured_output import captured_output, uncaptured_output
-from dcicutils.misc_utils import get_error_message
+from dcicutils.misc_utils import get_error_message, is_uuid, PRINT
 from dcicutils.portal_utils import Portal
-from dcicutils.structured_data import Schema
+
+
+# Schema properties to ignore (by default) for the view schema usage.
+_SCHEMAS_IGNORE_PROPERTIES = [
+    "date_created",
+    "last_modified",
+    "principals_allowed",
+    "submitted_by",
+    "schema_version"
+]
 
 
 def main():
@@ -82,26 +93,81 @@ def main():
                         help=f"Application name (one of: smaht, cgap, fourfront).")
     parser.add_argument("--schema", action="store_true", required=False, default=False,
                         help="View named schema rather than object.")
+    parser.add_argument("--all", action="store_true", required=False, default=False,
+                        help="Include all properties for schema usage.")
     parser.add_argument("--raw", action="store_true", required=False, default=False, help="Raw output.")
+    parser.add_argument("--tree", action="store_true", required=False, default=False, help="Tree output for schemas.")
     parser.add_argument("--database", action="store_true", required=False, default=False,
                         help="Read from database output.")
     parser.add_argument("--yaml", action="store_true", required=False, default=False, help="YAML output.")
     parser.add_argument("--copy", "-c", action="store_true", required=False, default=False,
                         help="Copy object data to clipboard.")
+    parser.add_argument("--details", action="store_true", required=False, default=False, help="Detailed output.")
+    parser.add_argument("--more-details", action="store_true", required=False, default=False,
+                        help="More detailed output.")
     parser.add_argument("--verbose", action="store_true", required=False, default=False, help="Verbose output.")
     parser.add_argument("--debug", action="store_true", required=False, default=False, help="Debugging output.")
     args = parser.parse_args()
 
-    portal = _create_portal(ini=args.ini, env=args.env, server=args.server, app=args.app, debug=args.debug)
-    if args.uuid == "schemas":
-        _print_all_schema_names(portal=portal, verbose=args.verbose)
+    if args.more_details:
+        args.details = True
+
+    portal = _create_portal(ini=args.ini, env=args.env or os.environ.get("SMAHT_ENV"),
+                            server=args.server, app=args.app, verbose=args.verbose, debug=args.debug)
+
+    if args.uuid.lower() == "schemas" or args.uuid.lower() == "schema":
+        _print_all_schema_names(portal=portal, details=args.details,
+                                more_details=args.more_details, all=args.all,
+                                tree=args.tree, raw=args.raw, raw_yaml=args.yaml)
         return
-    elif args.schema:
-        data = _get_schema(portal=portal, schema_name=args.uuid)
-    else:
-        data = _get_portal_object(portal=portal, uuid=args.uuid, raw=args.raw,
-                                  database=args.database, verbose=args.verbose)
+    elif args.uuid.lower() == "info":  # TODO: need word for what consortiums and submission centers are collectively
+        if consortia := portal.get_metadata("/consortia?limit=1000"):
+            _print("Known Consortia:")
+            consortia = sorted(consortia.get("@graph", []), key=lambda key: key.get("identifier"))
+            for consortium in consortia:
+                if ((consortium_name := consortium.get("identifier")) and
+                    (consortium_uuid := consortium.get("uuid"))):  # noqa
+                    _print(f"- {consortium_name}: {consortium_uuid}")
+        if submission_centers := portal.get_metadata("/submission-centers?limit=1000"):
+            _print("Known Submission Centers:")
+            submission_centers = sorted(submission_centers.get("@graph", []), key=lambda key: key.get("identifier"))
+            for submission_center in submission_centers:
+                if ((submission_center_name := submission_center.get("identifier")) and
+                    (submission_center_uuid := submission_center.get("uuid"))):  # noqa
+                    _print(f"- {submission_center_name}: {submission_center_uuid}")
+        try:
+            if file_formats := portal.get_metadata("/file-formats?limit=1000"):
+                _print("Known File Formats:")
+                file_formats = sorted(file_formats.get("@graph", []), key=lambda key: key.get("identifier"))
+                for file_format in file_formats:
+                    if ((file_format_name := file_format.get("identifier")) and
+                        (file_format_uuid := file_format.get("uuid"))):  # noqa
+                        _print(f"- {file_format_name}: {file_format_uuid}")
+        except Exception:
+            _print("Known File Formats: None")
+        return
+
+    if _is_maybe_schema_name(args.uuid):
+        args.schema = True
+
+    if args.schema:
+        schema, schema_name = _get_schema(portal, args.uuid)
+        if schema:
+            if args.copy:
+                pyperclip.copy(json.dumps(schema, indent=4))
+            if not args.raw:
+                if parent_schema_name := _get_parent_schema_name(schema):
+                    if schema.get("isAbstract") is True:
+                        _print(f"{schema_name} | parent: {parent_schema_name} | abstract")
+                    else:
+                        _print(f"{schema_name} | parent: {parent_schema_name}")
+                else:
+                    _print(schema_name)
+            _print_schema(schema, details=args.details, more_details=args.details,
+                          all=args.all, raw=args.raw, raw_yaml=args.yaml)
+            return
 
+    data = _get_portal_object(portal=portal, uuid=args.uuid, raw=args.raw, database=args.database, verbose=args.verbose)
     if args.copy:
         pyperclip.copy(json.dumps(data, indent=4))
     if args.yaml:
@@ -111,25 +177,28 @@ def main():
 
 
 def _create_portal(ini: str, env: Optional[str] = None,
-                   server: Optional[str] = None, app: Optional[str] = None, debug: bool = False) -> Portal:
+                   server: Optional[str] = None, app: Optional[str] = None,
+                   verbose: bool = False, debug: bool = False) -> Portal:
+    portal = None
     with captured_output(not debug):
-        return Portal(env, server=server, app=app) if env or app else Portal(ini)
+        portal = Portal(env, server=server, app=app) if env or app else Portal(ini)
+    if portal:
+        if verbose:
+            if portal.env:
+                _print(f"Portal environment: {portal.env}")
+            if portal.keys_file:
+                _print(f"Portal keys file: {portal.keys_file}")
+            if portal.key_id:
+                _print(f"Portal key prefix: {portal.key_id[0:2]}******")
+            if portal.ini_file:
+                _print(f"Portal ini file: {portal.ini_file}")
+            if portal.server:
+                _print(f"Portal server: {portal.server}")
+        return portal
 
 
 def _get_portal_object(portal: Portal, uuid: str,
                        raw: bool = False, database: bool = False, verbose: bool = False) -> dict:
-    if verbose:
-        _print(f"Getting object from Portal: {uuid}")
-        if portal.env:
-            _print(f"Portal environment: {portal.env}")
-        if portal.keys_file:
-            _print(f"Portal keys file: {portal.keys_file}")
-        if portal.key_id:
-            _print(f"Portal key prefix: {portal.key_id[0:2]}******")
-        if portal.ini_file:
-            _print(f"Portal ini file: {portal.ini_file}")
-        if portal.server:
-            _print(f"Portal server: {portal.server}")
     response = None
     try:
         if not uuid.startswith("/"):
@@ -139,56 +208,334 @@ def _get_portal_object(portal: Portal, uuid: str,
         response = portal.get(path, raw=raw, database=database)
     except Exception as e:
         if "404" in str(e) and "not found" in str(e).lower():
-            _print(f"Portal object not found: {uuid}")
-            _exit_without_action()
-        _exit_without_action(f"Exception getting Portal object: {uuid}\n{get_error_message(e)}")
+            _print(f"Portal object not found at {portal.server}: {uuid}")
+            _exit()
+        _exit(f"Exception getting Portal object from {portal.server}: {uuid}\n{get_error_message(e)}")
     if not response:
-        _exit_without_action(f"Null response getting Portal object: {uuid}")
+        _exit(f"Null response getting Portal object from {portal.server}: {uuid}")
     if response.status_code not in [200, 307]:
         # TODO: Understand why the /me endpoint returns HTTP status code 307, which is only why we mention it above.
-        _exit_without_action(f"Invalid status code ({response.status_code}) getting Portal object: {uuid}")
+        _exit(f"Invalid status code ({response.status_code}) getting Portal object from {portal.server}: {uuid}")
     if not response.json:
-        _exit_without_action(f"Invalid JSON getting Portal object: {uuid}")
-    if verbose:
-        _print("OK")
+        _exit(f"Invalid JSON getting Portal object: {uuid}")
     return response.json()
 
 
-def _get_schema(portal: Portal, schema_name: str) -> Optional[dict]:
-    def rummage_for_schema_name(portal: Portal, schema_name: str) -> Optional[str]:  # noqa
-        if schemas := portal.get_schemas():
-            for schema in schemas:
-                if schema.lower() == schema_name.lower():
-                    return schema
-    schema = Schema.load_by_name(schema_name, portal)
-    if not schema:
-        if schema_name := rummage_for_schema_name(portal, schema_name):
-            schema = Schema.load_by_name(schema_name, portal)
-    return schema.data if schema else None
-
-
-def _print_all_schema_names(portal: Portal, verbose: bool = False) -> None:
-    if schemas := portal.get_schemas():
-        for schema in sorted(schemas.keys()):
-            _print(schema)
-            if verbose:
-                if identifying_properties := schemas[schema].get("identifyingProperties"):
-                    _print("- identifying properties:")
-                    for identifying_property in sorted(identifying_properties):
-                        _print(f"  - {identifying_property}")
-                if required_properties := schemas[schema].get("required"):
-                    _print("- required properties:")
-                    for required_property in sorted(required_properties):
-                        _print(f"  - {required_property}")
+@lru_cache(maxsize=1)
+def _get_schemas(portal: Portal) -> Optional[dict]:
+    return portal.get_schemas()
+
+
+def _get_schema(portal: Portal, name: str) -> Tuple[Optional[dict], Optional[str]]:
+    if portal and name and (name := name.replace("_", "").replace("-", "").strip().lower()):
+        if schemas := _get_schemas(portal):
+            for schema_name in schemas:
+                if schema_name.replace("_", "").replace("-", "").strip().lower() == name:
+                    return schemas[schema_name], schema_name
+    return None, None
+
+
+def _is_maybe_schema_name(value: str) -> bool:
+    if value and not is_uuid(value) and not value.startswith("/"):
+        return True
+    return False
+
+
+def _print_schema(schema: dict, details: bool = False, more_details: bool = False, all: bool = False,
+                  raw: bool = False, raw_yaml: bool = False) -> None:
+    if raw:
+        if raw_yaml:
+            _print(yaml.dump(schema))
+        else:
+            _print(json.dumps(schema, indent=4))
+        return
+    _print_schema_info(schema, details=details, more_details=more_details, all=all)
+
+
+def _print_schema_info(schema: dict, level: int = 0,
+                       details: bool = False, more_details: bool = False, all: bool = False,
+                       required: Optional[List[str]] = None) -> None:
+    if not schema or not isinstance(schema, dict):
+        return
+    if level == 0:
+        if required_properties := schema.get("required"):
+            _print("- required properties:")
+            for required_property in sorted(list(set(required_properties))):
+                if not all and required_property in _SCHEMAS_IGNORE_PROPERTIES:
+                    continue
+                if property_type := (info := schema.get("properties", {}).get(required_property, {})).get("type"):
+                    if property_type == "array" and (array_type := info.get("items", {}).get("type")):
+                        _print(f"  - {required_property}: {property_type} of {array_type}")
+                    else:
+                        _print(f"  - {required_property}: {property_type}")
+                else:
+                    _print(f"  - {required_property}")
+            if isinstance(any_of := schema.get("anyOf"), list):
+                if ((any_of == [{"required": ["submission_centers"]}, {"required": ["consortia"]}]) or
+                    (any_of == [{"required": ["consortia"]}, {"required": ["submission_centers"]}])):  # noqa
+                    # Very very special case.
+                    _print(f"  - at least one of:")
+                    _print(f"    - consortia: array of string")
+                    _print(f"    - submission_centers: array of string")
+            required = required_properties
+        if identifying_properties := schema.get("identifyingProperties"):
+            _print("- identifying properties:")
+            for identifying_property in sorted(list(set(identifying_properties))):
+                if not all and identifying_property in _SCHEMAS_IGNORE_PROPERTIES:
+                    continue
+                if property_type := (info := schema.get("properties", {}).get(identifying_property, {})).get("type"):
+                    if property_type == "array" and (array_type := info.get("items", {}).get("type")):
+                        _print(f"  - {identifying_property}: {property_type} of {array_type}")
+                    else:
+                        _print(f"  - {identifying_property}: {property_type}")
+                else:
+                    _print(f"  - {identifying_property}")
+        if properties := schema.get("properties"):
+            reference_properties = []
+            for property_name in properties:
+                if not all and property_name in _SCHEMAS_IGNORE_PROPERTIES:
+                    continue
+                property = properties[property_name]
+                if link_to := property.get("linkTo"):
+                    reference_properties.append({"name": property_name, "ref": link_to})
+            if reference_properties:
+                _print("- reference properties:")
+                for reference_property in sorted(reference_properties, key=lambda key: key["name"]):
+                    _print(f"  - {reference_property['name']}: {reference_property['ref']}")
+        if schema.get("additionalProperties") is True:
+            _print(f"  - additional properties are allowed")
+    if not more_details:
+        return
+    if properties := (schema.get("properties") if level == 0 else schema):
+        if level == 0:
+            _print("- properties:")
+        for property_name in sorted(properties):
+            if not all and property_name in _SCHEMAS_IGNORE_PROPERTIES:
+                continue
+            if property_name.startswith("@"):
+                continue
+            spaces = f"{' ' * (level + 1) * 2}"
+            property = properties[property_name]
+            property_required = required and property_name in required
+            if property_type := property.get("type"):
+                if property_type == "object":
+                    suffix = ""
+                    if not (object_properties := property.get("properties")):
+                        if property.get("additionalProperties") is True:
+                            property_type = "any object"
+                        else:
+                            property_type = "undefined object"
+                    elif property.get("additionalProperties") is True:
+                        property_type = "open ended object"
+                    if property.get("calculatedProperty"):
+                        suffix += f" | calculated"
+                    _print(f"{spaces}- {property_name}: {property_type}{suffix}")
+                    _print_schema_info(object_properties, level=level + 1,
+                                       details=details, more_details=more_details, all=all,
+                                       required=property.get("required"))
+                elif property_type == "array":
+                    suffix = ""
+                    if property_required:
+                        suffix += f" | required"
+                    if property.get("uniqueItems"):
+                        suffix += f" | unique"
+                    if property.get("calculatedProperty"):
+                        suffix += f" | calculated"
+                    if property_items := property.get("items"):
+                        if (enumeration := property_items.get("enum")) is not None:
+                            suffix = f" | enum" + suffix
+                        if pattern := property_items.get("pattern"):
+                            suffix += f" | pattern: {pattern}"
+                        if (format := property_items.get("format")) and (format != "uuid"):
+                            suffix += f" | format: {format}"
+                        if (max_length := property_items.get("maxLength")) is not None:
+                            suffix += f" | max items: {max_length}"
+                        if property_type := property_items.get("type"):
+                            if property_type == "object":
+                                suffix = ""
+                                _print(f"{spaces}- {property_name}: array of object{suffix}")
+                                _print_schema_info(property_items.get("properties"), level=level + 1,
+                                                   details=details, more_details=more_details, all=all,
+                                                   required=property_items.get("required"))
+                            elif property_type == "array":
+                                # This (array-of-array) never happens to occur at this time (February 2024).
+                                _print(f"{spaces}- {property_name}: array of array{suffix}")
+                            else:
+                                _print(f"{spaces}- {property_name}: array of {property_type}{suffix}")
+                        else:
+                            _print(f"{spaces}- {property_name}: array{suffix}")
+                    else:
+                        _print(f"{spaces}- {property_name}: array{suffix}")
+                    if enumeration:
+                        nenums = 0
+                        maxenums = 15
+                        for enum in sorted(enumeration):
+                            if (nenums := nenums + 1) >= maxenums:
+                                if (remaining := len(enumeration) - nenums) > 0:
+                                    _print(f"{spaces}  - [{remaining} more ...]")
+                                break
+                            _print(f"{spaces}  - {enum}")
+                else:
+                    if isinstance(property_type, list):
+                        property_type = " or ".join(sorted(property_type))
+                    suffix = ""
+                    if (enumeration := property.get("enum")) is not None:
+                        suffix += f" | enum"
+                    if property_required:
+                        suffix += f" | required"
+                    if property.get("uniqueKey"):
+                        suffix += f" | unique"
+                    if pattern := property.get("pattern"):
+                        suffix += f" | pattern: {pattern}"
+                    if (format := property.get("format")) and (format != "uuid"):
+                        suffix += f" | format: {format}"
+                    if isinstance(any_of := property.get("anyOf"), list):
+                        if ((any_of == [{"format": "date"}, {"format": "date-time"}]) or
+                            (any_of == [{"format": "date-time"}, {"format": "date"}])):  # noqa
+                            # Very special case.
+                            suffix += f" | format: date or date-time"
+                    if link_to := property.get("linkTo"):
+                        suffix += f" | reference: {link_to}"
+                    if property.get("calculatedProperty"):
+                        suffix += f" | calculated"
+                    if (default := property.get("default")) is not None:
+                        suffix += f" | default:"
+                        if isinstance(default, dict):
+                            suffix += f" object"
+                        elif isinstance(default, list):
+                            suffix += f" array"
+                        else:
+                            suffix += f" {default}"
+                    if (minimum := property.get("minimum")) is not None:
+                        suffix += f" | min: {minimum}"
+                    if (maximum := property.get("maximum")) is not None:
+                        suffix += f" | max: {maximum}"
+                    if (max_length := property.get("maxLength")) is not None:
+                        suffix += f" | max length: {max_length}"
+                    if (min_length := property.get("minLength")) is not None:
+                        suffix += f" | min length: {min_length}"
+                    _print(f"{spaces}- {property_name}: {property_type}{suffix}")
+                    if enumeration:
+                        nenums = 0
+                        maxenums = 15
+                        for enum in sorted(enumeration):
+                            if (nenums := nenums + 1) >= maxenums:
+                                if (remaining := len(enumeration) - nenums) > 0:
+                                    _print(f"{spaces}  - [{remaining} more ...]")
+                                break
+                            _print(f"{spaces}  - {enum}")
+            else:
+                _print(f"{spaces}- {property_name}")
+
+
+def _print_all_schema_names(portal: Portal,
+                            details: bool = False, more_details: bool = False, all: bool = False,
+                            tree: bool = False, raw: bool = False, raw_yaml: bool = False) -> None:
+    if not (schemas := _get_schemas(portal)):
+        return
+
+    if raw:
+        if raw_yaml:
+            _print(yaml.dump(schemas))
+        else:
+            _print(json.dumps(schemas, indent=4))
+        return
+
+    if tree:
+        _print_schemas_tree(schemas)
+        return
+
+    for schema_name in sorted(schemas.keys()):
+        if parent_schema_name := _get_parent_schema_name(schemas[schema_name]):
+            if schemas[schema_name].get("isAbstract") is True:
+                _print(f"{schema_name} | parent: {parent_schema_name} | abstract")
+            else:
+                _print(f"{schema_name} | parent: {parent_schema_name}")
+        else:
+            if schemas[schema_name].get("isAbstract") is True:
+                _print(f"{schema_name} | abstract")
+            else:
+                _print(schema_name)
+        if details:
+            _print_schema(schemas[schema_name], details=details, more_details=more_details, all=all)
+
+
+def _get_parent_schema_name(schema: dict) -> Optional[str]:
+    if (isinstance(schema, dict) and
+        (parent_schema_name := schema.get("rdfs:subClassOf")) and
+        (parent_schema_name := parent_schema_name.replace("/profiles/", "").replace(".json", "")) and
+        (parent_schema_name != "Item")):  # noqa
+        return parent_schema_name
+    return None
+
+
+def _print_schemas_tree(schemas: dict) -> None:
+    def children_of(name: str) -> List[str]:
+        nonlocal schemas
+        children = []
+        if not (name is None or isinstance(name, str)):
+            return children
+        if name and name.lower() == "schemas":
+            name = None
+        for schema_name in (schemas if isinstance(schemas, dict) else {}):
+            if _get_parent_schema_name(schemas[schema_name]) == name:
+                children.append(schema_name)
+        return sorted(children)
+    def name_of(name: str) -> str:  # noqa
+        nonlocal schemas
+        if not (name is None or isinstance(name, str)):
+            return name
+        if (schema := schemas.get(name)) and schema.get("isAbstract") is True:
+            return f"{name} (abstact)"
+        return name
+    _print_tree(root_name="Schemas", children_of=children_of, name_of=name_of)
+
+
+def _print_tree(root_name: Optional[str],
+                children_of: Callable,
+                has_children: Optional[Callable] = None,
+                name_of: Optional[Callable] = None,
+                print: Callable = print) -> None:
+    """
+    Recursively prints as a tree structure the given root name and any of its
+    children (again, recursively) as specified by the given children_of callable;
+    the has_children may be specified, for efficiency, though if not specified
+    it will use the children_of function to determine this; the name_of callable
+    may be specified to modify the name before printing.
+    """
+    first = "└─ "
+    space = "    "
+    branch = "│   "
+    tee = "├── "
+    last = "└── "
+
+    if not callable(children_of):
+        return
+    if not callable(has_children):
+        has_children = lambda name: children_of(name) is not None  # noqa
+
+    # This function adapted from stackoverflow.
+    # Ref: https://stackoverflow.com/questions/9727673/list-directory-tree-structure-in-python
+    def tree_generator(name: str, prefix: str = ""):
+        contents = children_of(name)
+        pointers = [tee] * (len(contents) - 1) + [last]
+        for pointer, path in zip(pointers, contents):
+            yield prefix + pointer + (name_of(path) if callable(name_of) else path)
+            if has_children(path):
+                extension = branch if pointer == tee else space
+                yield from tree_generator(path, prefix=prefix+extension)
+    print(first + ((name_of(root_name) if callable(name_of) else root_name) or "root"))
+    for line in tree_generator(root_name, prefix="   "):
+        print(line)
 
 
 def _print(*args, **kwargs):
     with uncaptured_output():
-        print(*args, **kwargs)
+        PRINT(*args, **kwargs)
     sys.stdout.flush()
 
 
-def _exit_without_action(message: Optional[str] = None) -> None:
+def _exit(message: Optional[str] = None) -> None:
     if message:
         _print(f"ERROR: {message}")
     exit(1)

dcicutils/structured_data.py

@@ -47,11 +47,27 @@ StructuredDataSet = Type["StructuredDataSet"]
 
 class StructuredDataSet:
 
+    # Reference (linkTo) lookup strategies; on a per-reference (type/value) basis;
+    # controlled by optional ref_lookup_strategy callable; default is lookup at root path
+    # but after the named reference (linkTo) type path lookup, and then lookup all subtypes;
+    # can choose to lookup root path first, or not lookup root path at all, or not lookup
+    # subtypes at all; the ref_lookup_strategy callable if specified should take a type_name
+    # and value (string) arguements and return an integer of any of the below ORed together.
+    REF_LOOKUP_ROOT = 0x0001
+    REF_LOOKUP_ROOT_FIRST = 0x0002 | REF_LOOKUP_ROOT
+    REF_LOOKUP_SUBTYPES = 0x0004
+    REF_LOOKUP_MINIMAL = 0
+    REF_LOOKUP_DEFAULT = REF_LOOKUP_ROOT | REF_LOOKUP_SUBTYPES
+
     def __init__(self, file: Optional[str] = None, portal: Optional[Union[VirtualApp, TestApp, Portal]] = None,
                  schemas: Optional[List[dict]] = None, autoadd: Optional[dict] = None,
-                 order: Optional[List[str]] = None, prune: bool = True) -> None:
+                 order: Optional[List[str]] = None, prune: bool = True,
+                 ref_lookup_strategy: Optional[Callable] = None,
+                 ref_lookup_nocache: bool = False) -> None:
         self._data = {}
-        self._portal = Portal(portal, data=self._data, schemas=schemas) if portal else None
+        self._portal = Portal(portal, data=self._data, schemas=schemas,
+                              ref_lookup_strategy=ref_lookup_strategy,
+                              ref_lookup_nocache=ref_lookup_nocache) if portal else None
         self._order = order
         self._prune = prune
         self._warnings = {}
@@ -72,8 +88,11 @@ class StructuredDataSet:
     @staticmethod
     def load(file: str, portal: Optional[Union[VirtualApp, TestApp, Portal]] = None,
              schemas: Optional[List[dict]] = None, autoadd: Optional[dict] = None,
-             order: Optional[List[str]] = None, prune: bool = True) -> StructuredDataSet:
-        return StructuredDataSet(file=file, portal=portal, schemas=schemas, autoadd=autoadd, order=order, prune=prune)
+             order: Optional[List[str]] = None, prune: bool = True,
+             ref_lookup_strategy: Optional[Callable] = None,
+             ref_lookup_nocache: bool = False) -> StructuredDataSet:
+        return StructuredDataSet(file=file, portal=portal, schemas=schemas, autoadd=autoadd, order=order, prune=prune,
+                                 ref_lookup_strategy=ref_lookup_strategy, ref_lookup_nocache=ref_lookup_nocache)
 
     def validate(self, force: bool = False) -> None:
         def data_without_deleted_properties(data: dict) -> dict:
@@ -255,6 +274,23 @@ class StructuredDataSet:
             if name not in structured_row and (not schema or schema.data.get("properties", {}).get(name)):
                 structured_row[name] = properties[name]
 
+    def _is_ref_lookup_root(ref_lookup_flags: int) -> bool:
+        return (ref_lookup_flags & StructuredDataSet.REF_LOOKUP_ROOT) == StructuredDataSet.REF_LOOKUP_ROOT
+
+    def _is_ref_lookup_root_first(ref_lookup_flags: int) -> bool:
+        return (ref_lookup_flags & StructuredDataSet.REF_LOOKUP_ROOT_FIRST) == StructuredDataSet.REF_LOOKUP_ROOT_FIRST
+
+    def _is_ref_lookup_subtypes(ref_lookup_flags: int) -> bool:
+        return (ref_lookup_flags & StructuredDataSet.REF_LOOKUP_SUBTYPES) == StructuredDataSet.REF_LOOKUP_SUBTYPES
+
+    @property
+    def ref_cache_hit_count(self) -> int:
+        return self.portal.ref_cache_hit_count if self.portal else -1
+
+    @property
+    def ref_lookup_count(self) -> int:
+        return self.portal.ref_lookup_count if self.portal else -1
+
     def _note_warning(self, item: Optional[Union[dict, List[dict]]], group: str) -> None:
         self._note_issue(self._warnings, item, group)
 
@@ -637,6 +673,8 @@ class Portal(PortalBase):
                  env: Optional[str] = None, server: Optional[str] = None,
                  app: Optional[OrchestratedApp] = None,
                  data: Optional[dict] = None, schemas: Optional[List[dict]] = None,
+                 ref_lookup_strategy: Optional[Callable] = None,
+                 ref_lookup_nocache: bool = False,
                  raise_exception: bool = True) -> None:
         super().__init__(arg, env=env, server=server, app=app, raise_exception=raise_exception)
         if isinstance(arg, Portal):
@@ -645,10 +683,21 @@ class Portal(PortalBase):
         else:
             self._schemas = schemas
             self._data = data
+        if callable(ref_lookup_strategy):
+            self._ref_lookup_strategy = ref_lookup_strategy
+        else:
+            self._ref_lookup_strategy = lambda type_name, value: StructuredDataSet.REF_LOOKUP_DEFAULT
+        self._ref_cache = {} if not ref_lookup_nocache else None
+        self._ref_cache_hit_count = 0
+        self._ref_lookup_count = 0
 
-    @lru_cache(maxsize=256)
-    def get_metadata(self, object_name: str) -> Optional[dict]:
+    @lru_cache(maxsize=8092)
+    def get_metadata_cache(self, object_name: str) -> Optional[dict]:
+        return self.get_metadata_nocache(object_name)
+
+    def get_metadata_nocache(self, object_name: str) -> Optional[dict]:
         try:
+            self._ref_lookup_count += 1
             return super().get_metadata(object_name)
         except Exception:
             return None
@@ -675,53 +724,90 @@ class Portal(PortalBase):
                     schemas[user_specified_schema["title"]] = user_specified_schema
         return schemas
 
+    @lru_cache(maxsize=64)
+    def _get_schema_subtypes(self, type_name: str) -> Optional[List[str]]:
+        if not (schemas_super_type_map := self.get_schemas_super_type_map()):
+            return []
+        return schemas_super_type_map.get(type_name)
+
     def is_file_schema(self, schema_name: str) -> bool:
         """
         Returns True iff the given schema name isa File type, i.e. has an ancestor which is of type File.
         """
         return self.is_schema_type(schema_name, FILE_SCHEMA_NAME)
 
-    def ref_exists(self, type_name: str, value: Optional[str] = None) -> List[str]:
+    def _ref_exists_from_cache(self, type_name: str, value: str) -> Optional[List[dict]]:
+        if self._ref_cache is not None:
+            return self._ref_cache.get(f"/{type_name}/{value}", None)
+        return None
+
+    def _cache_ref(self, type_name: str, value: str, resolved: List[str],
+                   subtype_names: Optional[List[str]]) -> None:
+        if self._ref_cache is not None:
+            for type_name in [type_name] + (subtype_names or []):
+                object_path = f"/{type_name}/{value}"
+                if self._ref_cache.get(object_path, None) is None:
+                    self._ref_cache[object_path] = resolved
+
+    def ref_exists(self, type_name: str, value: Optional[str] = None) -> List[dict]:
         if not value:
             if type_name.startswith("/") and len(parts := type_name[1:].split("/")) == 2:
                 type_name = parts[0]
                 value = parts[1]
             else:
-                return []
+                return []  # Should not happen.
+        if (resolved := self._ref_exists_from_cache(type_name, value)) is not None:
+            self._ref_cache_hit_count += 1
+            return resolved
+        # Not cached here.
         resolved = []
-        is_resolved, resolved_uuid = self._ref_exists_single(type_name, value)
+        ref_lookup_strategy = self._ref_lookup_strategy(type_name, value)
+        is_ref_lookup_root = StructuredDataSet._is_ref_lookup_root(ref_lookup_strategy)
+        is_ref_lookup_root_first = StructuredDataSet._is_ref_lookup_root_first(ref_lookup_strategy)
+        is_ref_lookup_subtypes = StructuredDataSet._is_ref_lookup_subtypes(ref_lookup_strategy)
+        is_resolved = False
+        subtype_names = self._get_schema_subtypes(type_name)
+        if is_ref_lookup_root_first:
+            is_resolved, resolved_uuid = self._ref_exists_single(type_name, value, root=True)
+        if not is_resolved:
+            is_resolved, resolved_uuid = self._ref_exists_single(type_name, value)
+        if not is_resolved and is_ref_lookup_root and not is_ref_lookup_root_first:
+            is_resolved, resolved_uuid = self._ref_exists_single(type_name, value, root=True)
         if is_resolved:
             resolved.append({"type": type_name, "uuid": resolved_uuid})
-            # TODO: Added this return on 2024-01-14 (dmichaels).
-            # Why did I orginally check for multiple existing values?
-            # Why not just return right away if I find that the ref exists?
-            # Getting multiple values because, for example, we find
-            # both this /Sample/UW_CELL-CULTURE-SAMPLE_COLO-829BL_HI-C_1
-            # and /CellSample/UW_CELL-CULTURE-SAMPLE_COLO-829BL_HI-C_1
-            # Why does that matter at all? Same thing.
-            return resolved
-        # Check for the given ref in all sub-types of the given type.
-        if (schemas_super_type_map := self.get_schemas_super_type_map()):
-            if (sub_type_names := schemas_super_type_map.get(type_name)):
-                for sub_type_name in sub_type_names:
-                    is_resolved, resolved_uuid = self._ref_exists_single(sub_type_name, value)
-                    if is_resolved:
-                        resolved.append({"type": type_name, "uuid": resolved_uuid})
-                        # TODO: Added this return on 2024-01-14 (dmichaels). See above TODO.
-                        return resolved
+        # Check for the given ref in all subtypes of the given type.
+        elif subtype_names and is_ref_lookup_subtypes:
+            for subtype_name in subtype_names:
+                is_resolved, resolved_uuid = self._ref_exists_single(subtype_name, value)
+                if is_resolved:
+                    resolved.append({"type": type_name, "uuid": resolved_uuid})
+                    break
+        # Cache this ref (and all subtype versions of it); whether or not found;
+        # if not found it will be an empty array (array because caching all matches;
+        # but TODO - do not think we should do this anymore - maybe test changes needed).
+        self._cache_ref(type_name, value, resolved, subtype_names)
         return resolved
 
-    def _ref_exists_single(self, type_name: str, value: str) -> Tuple[bool, Optional[str]]:
+    def _ref_exists_single(self, type_name: str, value: str, root: bool = False) -> Tuple[bool, Optional[str]]:
+        # Check first in our own data (i.e. e.g. within the given spreadsheet).
         if self._data and (items := self._data.get(type_name)) and (schema := self.get_schema(type_name)):
             iproperties = set(schema.get("identifyingProperties", [])) | {"identifier", "uuid"}
             for item in items:
                 if (ivalue := next((item[iproperty] for iproperty in iproperties if iproperty in item), None)):
                     if isinstance(ivalue, list) and value in ivalue or ivalue == value:
                         return True, (ivalue if isinstance(ivalue, str) and is_uuid(ivalue) else None)
-        if (value := self.get_metadata(f"/{type_name}/{value}")) is None:
+        if (value := self.get_metadata(f"/{type_name}/{value}" if not root else f"/{value}")) is None:
             return False, None
         return True, value.get("uuid")
 
+    @property
+    def ref_cache_hit_count(self) -> int:
+        return self._ref_cache_hit_count
+
+    @property
+    def ref_lookup_count(self) -> int:
+        return self._ref_lookup_count
+
     @staticmethod
     def create_for_testing(arg: Optional[Union[str, bool, List[dict], dict, Callable]] = None,
                            schemas: Optional[List[dict]] = None) -> Portal:

{dcicutils-8.7.2.1b7.dist-info → dcicutils-8.8.0.1b1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcicutils
-Version: 8.7.2.1b7
+Version: 8.8.0.1b1
 Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
 Home-page: https://github.com/4dn-dcic/utils
 License: MIT

{dcicutils-8.7.2.1b7.dist-info → dcicutils-8.8.0.1b1.dist-info}/RECORD

@@ -47,7 +47,7 @@ dcicutils/misc_utils.py,sha256=zVc4urdVGgnWjQ4UQlrGH-URAzr2l_PwZWI3u_GJdFE,10221
 dcicutils/obfuscation_utils.py,sha256=fo2jOmDRC6xWpYX49u80bVNisqRRoPskFNX3ymFAmjw,5963
 dcicutils/opensearch_utils.py,sha256=V2exmFYW8Xl2_pGFixF4I2Cc549Opwe4PhFi5twC0M8,1017
 dcicutils/portal_object_utils.py,sha256=6QYVsmIVH-GUgZnws0Ob2d-THtiHERETx_OrLKNn0vA,13015
-dcicutils/portal_utils.py,sha256=WA0KfBYYLXJmG-TVUPbBiwSfI-3phU07F5Fy79cJEWs,27684
+dcicutils/portal_utils.py,sha256=OR1wJNiZVEsxcweArA4-K6yiuy7_bCqawxhZnuFsUtM,27686
 dcicutils/project_utils.py,sha256=qPdCaFmWUVBJw4rw342iUytwdQC0P-XKpK4mhyIulMM,31250
 dcicutils/qa_checkers.py,sha256=cdXjeL0jCDFDLT8VR8Px78aS10hwNISOO5G_Zv2TZ6M,20534
 dcicutils/qa_utils.py,sha256=TT0SiJWiuxYvbsIyhK9VO4uV_suxhB6CpuC4qPacCzQ,160208
@@ -57,20 +57,20 @@ dcicutils/s3_utils.py,sha256=LauLFQGvZLfpBJ81tYMikjLd3SJRz2R_FrL1n4xSlyI,28868
 dcicutils/schema_utils.py,sha256=IhtozG2jQ7bFyn54iPEdmDrHoCf3ryJXeXvPJRBXNn0,10095
 dcicutils/scripts/publish_to_pypi.py,sha256=LFzNHIQK2EXFr88YcfctyA_WKEBFc1ElnSjWrCXedPM,13889
 dcicutils/scripts/run_license_checker.py,sha256=z2keYnRDZsHQbTeo1XORAXSXNJK5axVzL5LjiNqZ7jE,4184
-dcicutils/scripts/view_portal_object.py,sha256=QZqNtXIDmgyVZEAYRNSpjCRLPwP-TUyFqcBx2La-nUo,8245
+dcicutils/scripts/view_portal_object.py,sha256=Cy-8GwGJS9EX-5RxE8mjsqNlDT0N6OCpkNffPVkTFQc,26262
 dcicutils/secrets_utils.py,sha256=8dppXAsiHhJzI6NmOcvJV5ldvKkQZzh3Fl-cb8Wm7MI,19745
 dcicutils/sheet_utils.py,sha256=VlmzteONW5VF_Q4vo0yA5vesz1ViUah1MZ_yA1rwZ0M,33629
 dcicutils/snapshot_utils.py,sha256=ymP7PXH6-yEiXAt75w0ldQFciGNqWBClNxC5gfX2FnY,22961
 dcicutils/ssl_certificate_utils.py,sha256=F0ifz_wnRRN9dfrfsz7aCp4UDLgHEY8LaK7PjnNvrAQ,9707
-dcicutils/structured_data.py,sha256=xVtidHl9d_yEfJVmwmHpYdEgaeqT0j3A_8NwqSeMaA0,38873
+dcicutils/structured_data.py,sha256=X0HJTqVjSpcivwR69EgdOoz8fKacF-KQsSw-qJQ-N7c,43338
 dcicutils/task_utils.py,sha256=MF8ujmTD6-O2AC2gRGPHyGdUrVKgtr8epT5XU8WtNjk,8082
 dcicutils/tmpfile_utils.py,sha256=n95XF8dZVbQRSXBZTGToXXfSs3JUVRyN6c3ZZ0nhAWI,1403
 dcicutils/trace_utils.py,sha256=g8kwV4ebEy5kXW6oOrEAUsurBcCROvwtZqz9fczsGRE,1769
 dcicutils/validation_utils.py,sha256=cMZIU2cY98FYtzK52z5WUYck7urH6JcqOuz9jkXpqzg,14797
 dcicutils/variant_utils.py,sha256=2H9azNx3xAj-MySg-uZ2SFqbWs4kZvf61JnK6b-h4Qw,4343
 dcicutils/zip_utils.py,sha256=rnjNv_k6L9jT2SjDSgVXp4BEJYLtz9XN6Cl2Fy-tqnM,2027
-dcicutils-8.7.2.1b7.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
-dcicutils-8.7.2.1b7.dist-info/METADATA,sha256=QPMBO-ACtYso8RGVZNhcN3t5o2mDUFvlBviU6Q6bIXk,3356
-dcicutils-8.7.2.1b7.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
-dcicutils-8.7.2.1b7.dist-info/entry_points.txt,sha256=51Q4F_2V10L0282W7HFjP4jdzW4K8lnWDARJQVFy_hw,270
-dcicutils-8.7.2.1b7.dist-info/RECORD,,
+dcicutils-8.8.0.1b1.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
+dcicutils-8.8.0.1b1.dist-info/METADATA,sha256=6rVfZfQMjB0OO2_7LSjq0S3_bLnvxBCl3c2vhPrrGgg,3356
+dcicutils-8.8.0.1b1.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+dcicutils-8.8.0.1b1.dist-info/entry_points.txt,sha256=51Q4F_2V10L0282W7HFjP4jdzW4K8lnWDARJQVFy_hw,270
+dcicutils-8.8.0.1b1.dist-info/RECORD,,