dcicutils 8.10.0.0b0__py3-none-any.whl → 8.10.0.1b2__py3-none-any.whl

dcicutils/command_utils.py

@@ -1,3 +1,4 @@
+from __future__ import annotations
 import contextlib
 import functools
 import glob
@@ -6,8 +7,9 @@ import os
 import re
 import requests
 import subprocess
+import sys
 
-from typing import Optional
+from typing import Callable, Optional
 from .exceptions import InvalidParameterError
 from .lang_utils import there_are
 from .misc_utils import INPUT, PRINT, environ_bool, print_error_message, decorator
@@ -371,7 +373,7 @@ def script_catch_errors():
         raise ScriptFailure(' '.join(message))
     try:
         yield fail
-        exit(0)
+        sys.exit(0)
     except (Exception, ScriptFailure) as e:
         if DEBUG_SCRIPT:
             # If debugging, let the error propagate, do not trap it.
@@ -383,4 +385,71 @@ def script_catch_errors():
             else:
                 message = str(e)  # Note: We ignore the type, which isn't intended to be shown.
                 PRINT(message)
-            exit(1)
+            sys.exit(1)
+
+
+class Question:
+    """
+    Supports asking the user (via stdin) a yes/no question, possibly repeatedly; and after
+    some maximum number times of the same answer in a row (consecutively), then asks them
+    if they want to automatically give that same answer to any/all subsequent questions.
+    Supports static/global list of such Question instances, hashed (only) by the question text.
+    """
+    _static_instances = {}
+
+    @staticmethod
+    def instance(question: Optional[str] = None,
+                 max: Optional[int] = None, printf: Optional[Callable] = None) -> Question:
+        question = question if isinstance(question, str) else ""
+        if not (instance := Question._static_instances.get(question)):
+            Question._static_instances[question] = (instance := Question(question, max=max, printf=printf))
+        return instance
+
+    @staticmethod
+    def yes(question: Optional[str] = None,
+            max: Optional[int] = None, printf: Optional[Callable] = None) -> bool:
+        return Question.instance(question, max=max, printf=printf).ask()
+
+    def __init__(self, question: Optional[str] = None,
+                 max: Optional[int] = None, printf: Optional[Callable] = None) -> None:
+        self._question = question if isinstance(question, str) else ""
+        self._max = max if isinstance(max, int) and max > 0 else None
+        self._print = printf if callable(printf) else print
+        self._yes_consecutive_count = 0
+        self._no_consecutive_count = 0
+        self._yes_automatic = False
+        self._no_automatic = False
+
+    def ask(self, question: Optional[str] = None) -> bool:
+
+        def question_automatic(value: str) -> bool:
+            nonlocal self
+            RARROW = "▶"
+            LARROW = "◀"
+            if yes_or_no(f"{RARROW}{RARROW}{RARROW}"
+                         f" Do you want to answer {value} to all such questions?"
+                         f" {LARROW}{LARROW}{LARROW}"):
+                return True
+            self._yes_consecutive_count = 0
+            self._no_consecutive_count = 0
+
+        if self._yes_automatic:
+            return True
+        elif self._no_automatic:
+            return False
+        elif yes_or_no((question if isinstance(question, str) else "") or self._question or "Undefined question"):
+            self._yes_consecutive_count += 1
+            self._no_consecutive_count = 0
+            if (self._no_consecutive_count == 0) and self._max and (self._yes_consecutive_count >= self._max):
+                # Have reached the maximum number of consecutive YES answers; ask if YES to all subsequent.
+                if question_automatic("YES"):
+                    self._yes_automatic = True
+            return True
+        else:
+            self._no_consecutive_count += 1
+            self._yes_consecutive_count = 0
+            if (self._yes_consecutive_count == 0) and self._max and (self._no_consecutive_count >= self._max):
+                # Have reached the maximum number of consecutive NO answers; ask if NO to all subsequent.
+                if question_automatic("NO"):
+                    self._no_automatic = True
+            return False

dcicutils/deployment_utils.py

@@ -396,9 +396,9 @@ class EBDeployer:
             packaging_was_successful = cls.build_application_version(args.repo, args.version_name, branch=args.branch)
             if packaging_was_successful:  # XXX: how to best detect?
                 time.sleep(5)  # give EB a second to catch up (it needs it)
-                exit(cls.deploy_new_version(args.env, args.repo, args.version_name))
+                sys.exit(cls.deploy_new_version(args.env, args.repo, args.version_name))
         else:
-            exit(cls.deploy_indexer(args.env, args.application_version))
+            sys.exit(cls.deploy_indexer(args.env, args.application_version))
 
 
 class IniFileManager:

dcicutils/ecr_scripts.py

@@ -3,6 +3,7 @@ import botocore.exceptions
 import boto3
 import contextlib
 import os
+import sys
 
 from typing import Optional, Union, List
 from .command_utils import yes_or_no
@@ -68,7 +69,7 @@ def ecr_command_context(account_number, ecs_repository=None, ecr_client=None):
             elif account_number != account_number_in_environ:
                 raise RuntimeError("The account number you have specified does not match your declared credentials.")
             yield ECRCommandContext(account_number=account_number, ecs_repository=ecs_repository, ecr_client=ecr_client)
-            exit(0)
+            sys.exit(0)
         except botocore.exceptions.ClientError as e:
             error_info = e.response.get('Error', {})
             message = error_info.get('Message')
@@ -77,12 +78,12 @@ def ecr_command_context(account_number, ecs_repository=None, ecr_client=None):
                 raise RuntimeError("Your security token seems to have expired.")
             elif message:
                 PRINT(f"{code}: {message}")
-                exit(1)
+                sys.exit(1)
             else:
                 raise
     except Exception as e:
         PRINT(f"{full_class_name(e)}: {e}")
-        exit(1)
+        sys.exit(1)
 
 
 class ECRCommandContext:

dcicutils/env_scripts.py

@@ -1,6 +1,7 @@
 import argparse
 import boto3
 import json
+import sys
 import yaml
 
 from .lang_utils import disjoined_list
@@ -52,7 +53,7 @@ def show_global_env_bucket(bucket, mode='json', key=None):
         else:
             PRINT(f"There is no default bucket. Please use a '--bucket' argument"
                   f" to specify one of {disjoined_list(envs_buckets)}.")
-        exit(1)
+        sys.exit(1)
 
     print_heading(bucket, style='=')
 
@@ -70,7 +71,7 @@ def show_global_env_bucket(bucket, mode='json', key=None):
         except Exception as e:
             PRINT("Bucket contents could not be downlaoded.")
             print_error_message(e)
-            exit(1)
+            sys.exit(1)
 
         object = None  # Without this, PyCharm fusses that object might not get set. -kmp 20-Jul-2022
         try:
@@ -78,7 +79,7 @@ def show_global_env_bucket(bucket, mode='json', key=None):
         except Exception as e:
             PRINT("Bucket contents could not be parsed as JSON.")
             print_error_message(e)
-            exit(1)
+            sys.exit(1)
 
         if mode == 'json':
             PRINT(json.dumps(object, indent=2, default=str))
@@ -86,7 +87,7 @@ def show_global_env_bucket(bucket, mode='json', key=None):
             PRINT(yaml.dump(object))
         else:
             PRINT(f"Unknown mode: {mode}. Try 'json' or 'yaml'.")
-            exit(1)
+            sys.exit(1)
 
 
 DEFAULT_BUCKET = get_env_bucket()

dcicutils/license_policies/park-lab-common.jsonc

@@ -248,6 +248,12 @@
             "docutils"  // Used only privately as a separate documentation-generation task for ReadTheDocs
         ],
 
+
+        "GNU General Public License v2 (GPLv2)": [
+            "pyinstaller",
+            "pyinstaller-hooks-contrib"
+        ],
+
         "MIT/X11 Derivative": [
             // The license used by libxkbcommon is complicated and involves numerous included licenses,
             // but all are permissive.

dcicutils/misc_utils.py

@@ -4,6 +4,7 @@ This file contains functions that might be generally useful.
 
 from collections import namedtuple
 import appdirs
+from copy import deepcopy
 import contextlib
 import datetime
 import functools
@@ -2199,28 +2200,58 @@ def merge_key_value_dict_lists(x, y):
     return [key_value_dict(k, v) for k, v in merged.items()]
 
 
-def merge_objects(target: Union[dict, List[Any]], source: Union[dict, List[Any]], full: bool = False) -> dict:
+def merge_objects(target: Union[dict, List[Any]], source: Union[dict, List[Any]],
+                  full: bool = False,  # deprecated
+                  expand_lists: Optional[bool] = None,
+                  primitive_lists: bool = False,
+                  copy: bool = False, _recursing: bool = False) -> Union[dict, List[Any]]:
     """
-    Merges the given source dictionary or list into the target dictionary or list.
-    This MAY well change the given target (dictionary or list) IN PLACE.
-    The the full argument is True then any target lists longer than the
-    source be will be filled out with the last element(s) of the source.
+    Merges the given source dictionary or list into the target dictionary or list and returns the
+    result. This MAY well change the given target (dictionary or list) IN PLACE ... UNLESS the copy
+    argument is True, then the given target will not change as a local copy is made (and returned).
+
+    If the expand_lists argument is True then any target lists longer than the
+    source be will be filled out with the last element(s) of the source; the full
+    argument (is deprecated and) is a synomym for this. The default is False.
+
+    If the primitive_lists argument is True then lists of primitives (i.e. lists in which
+    NONE of its elements are dictionaries, lists, or tuples) will themselves be treated
+    like primitives, meaning the whole of a source list will replace the corresponding
+    target; otherwise they  will be merged normally, meaning each element of a source list
+    will be merged, recursively, into the corresponding target list. The default is False.
     """
+    def is_primitive_list(value: Any) -> bool:  # noqa
+        if not isinstance(value, list):
+            return False
+        for item in value:
+            if isinstance(item, (dict, list, tuple)):
+                return False
+        return True
+
     if target is None:
         return source
+    if expand_lists not in (True, False):
+        expand_lists = full is True
+    if (copy is True) and (_recursing is not True):
+        target = deepcopy(target)
     if isinstance(target, dict) and isinstance(source, dict) and source:
         for key, value in source.items():
-            target[key] = merge_objects(target[key], value, full) if key in target else value
+            if ((primitive_lists is True) and
+                (key in target) and is_primitive_list(target[key]) and is_primitive_list(value)):  # noqa
+                target[key] = value
+            else:
+                target[key] = merge_objects(target[key], value,
+                                            expand_lists=expand_lists, _recursing=True) if key in target else value
     elif isinstance(target, list) and isinstance(source, list) and source:
         for i in range(max(len(source), len(target))):
             if i < len(target):
                 if i < len(source):
-                    target[i] = merge_objects(target[i], source[i], full)
-                elif full:
-                    target[i] = merge_objects(target[i], source[len(source) - 1], full)
+                    target[i] = merge_objects(target[i], source[i], expand_lists=expand_lists, _recursing=True)
+                elif expand_lists is True:
+                    target[i] = merge_objects(target[i], source[len(source) - 1], expand_lists=expand_lists)
             else:
                 target.append(source[i])
-    elif source:
+    elif source not in (None, {}, []):
         target = source
     return target
 

dcicutils/portal_object_utils.py

@@ -1,6 +1,5 @@
 from copy import deepcopy
 from functools import lru_cache
-import re
 from typing import Any, Callable, List, Optional, Tuple, Type, Union
 from dcicutils.data_readers import RowReader
 from dcicutils.misc_utils import create_readonly_object
@@ -14,11 +13,9 @@ class PortalObject:
 
     _PROPERTY_DELETION_SENTINEL = RowReader.CELL_DELETION_SENTINEL
 
-    def __init__(self, data: dict, portal: Portal = None,
-                 schema: Optional[Union[dict, Schema]] = None, type: Optional[str] = None) -> None:
+    def __init__(self, data: dict, portal: Optional[Portal] = None, type: Optional[str] = None) -> None:
         self._data = data if isinstance(data, dict) else {}
         self._portal = portal if isinstance(portal, Portal) else None
-        self._schema = schema if isinstance(schema, dict) else (schema.data if isinstance(schema, Schema) else None)
         self._type = type if isinstance(type, str) else ""
 
     @property
@@ -32,7 +29,7 @@ class PortalObject:
     @property
     @lru_cache(maxsize=1)
     def type(self) -> str:
-        return self._type or Portal.get_schema_type(self._data) or (Schema(self._schema).type if self._schema else "")
+        return self._type or Portal.get_schema_type(self._data) or ""
 
     @property
     @lru_cache(maxsize=1)
@@ -47,7 +44,7 @@ class PortalObject:
     @property
     @lru_cache(maxsize=1)
     def schema(self) -> Optional[dict]:
-        return self._schema if self._schema else (self._portal.get_schema(self.type) if self._portal else None)
+        return self._portal.get_schema(self.type) if self._portal else None
 
     def copy(self) -> PortalObject:
         return PortalObject(deepcopy(self.data), portal=self.portal, type=self.type)
@@ -59,39 +56,29 @@ class PortalObject:
         Returns the list of all identifying property names of this Portal object which actually have values.
         Implicitly include "uuid" and "identifier" properties as identifying properties if they are actually
         properties in the object schema, and favor these (first); defavor "aliases"; no other ordering defined.
+        Changed (2024-05-26) to use portal_utils.get_identifying_property_names; migrating some intricate stuff there.
         """
-        if not (schema := self.schema) or not (schema_identifying_properties := schema.get("identifyingProperties")):
-            return None
-        identifying_properties = []
-        for identifying_property in schema_identifying_properties:
-            if identifying_property not in ["uuid", "identifier", "aliases"]:
-                if self._data.get(identifying_property):
-                    identifying_properties.append(identifying_property)
-        if self._data.get("identifier"):
-            identifying_properties.insert(0, "identifier")
-        if self._data.get("uuid"):
-            identifying_properties.insert(0, "uuid")
-        if "aliases" in schema_identifying_properties and self._data.get("aliases"):
-            identifying_properties.append("aliases")
-        return identifying_properties or None
+        # Migrating to and unifying this in portal_utils.Portal.get_identifying_paths (2024-05-26).
+        return self._portal.get_identifying_property_names(self.type, portal_object=self._data) if self._portal else []
 
     @lru_cache(maxsize=8192)
     def lookup(self, raw: bool = False,
                ref_lookup_strategy: Optional[Callable] = None) -> Tuple[Optional[PortalObject], Optional[str], int]:
+        if not (identifying_paths := self._get_identifying_paths(ref_lookup_strategy=ref_lookup_strategy)):
+            return None, None, 0
         nlookups = 0
         first_identifying_path = None
         try:
-            if identifying_paths := self._get_identifying_paths(ref_lookup_strategy=ref_lookup_strategy):
-                for identifying_path in identifying_paths:
-                    if not first_identifying_path:
-                        first_identifying_path = identifying_path
-                    nlookups += 1
-                    if (value := self._portal.get(identifying_path, raw=raw)) and (value.status_code == 200):
-                        return (
-                            PortalObject(value.json(), portal=self._portal, type=self.type if raw else None),
-                            identifying_path,
-                            nlookups
-                        )
+            for identifying_path in identifying_paths:
+                if not first_identifying_path:
+                    first_identifying_path = identifying_path
+                nlookups += 1
+                if self._portal and (item := self._portal.get(identifying_path, raw=raw)) and (item.status_code == 200):
+                    return (
+                        PortalObject(item.json(), portal=self._portal, type=self.type if raw else None),
+                        identifying_path,
+                        nlookups
+                    )
         except Exception:
             pass
         return None, first_identifying_path, nlookups
@@ -159,64 +146,12 @@ class PortalObject:
 
     @lru_cache(maxsize=1)
     def _get_identifying_paths(self, ref_lookup_strategy: Optional[Callable] = None) -> Optional[List[str]]:
-        """
-        Returns a list of the possible Portal URL paths identifying this Portal object.
-        """
-        identifying_paths = []
-        if not (identifying_properties := self.identifying_properties):
-            if self.uuid:
-                if self.type:
-                    identifying_paths.append(f"/{self.type}/{self.uuid}")
-                identifying_paths.append(f"/{self.uuid}")
-            return identifying_paths
-        for identifying_property in identifying_properties:
-            if identifying_value := self._data.get(identifying_property):
-                if identifying_property == "uuid":
-                    if self.type:
-                        identifying_paths.append(f"/{self.type}/{identifying_value}")
-                    identifying_paths.append(f"/{identifying_value}")
-                # For now at least we include the path both with and without the schema type component,
-                # as for some identifying values, it works (only) with, and some, it works (only) without.
-                # For example: If we have FileSet with "accession", an identifying property, with value
-                # SMAFSFXF1RO4 then /SMAFSFXF1RO4 works but /FileSet/SMAFSFXF1RO4 does not; and
-                # conversely using "submitted_id", also an identifying property, with value
-                # UW_FILE-SET_COLO-829BL_HI-C_1 then /UW_FILE-SET_COLO-829BL_HI-C_1 does
-                # not work but /FileSet/UW_FILE-SET_COLO-829BL_HI-C_1 does work.
-                elif isinstance(identifying_value, list):
-                    for identifying_value_item in identifying_value:
-                        if self.type:
-                            identifying_paths.append(f"/{self.type}/{identifying_value_item}")
-                        identifying_paths.append(f"/{identifying_value_item}")
-                else:
-                    # TODO: Import from somewhere ...
-                    lookup_options = 0
-                    if schema := self.schema:
-                        # TODO: Hook into the ref_lookup_strategy thing in structured_data to make
-                        # sure we check accession format (since it does not have a pattern).
-                        if callable(ref_lookup_strategy):
-                            lookup_options, ref_validator = ref_lookup_strategy(
-                                self._portal, self.type, schema, identifying_value)
-                            if callable(ref_validator):
-                                if ref_validator(schema, identifying_property, identifying_value) is False:
-                                    continue
-                        if pattern := schema.get("properties", {}).get(identifying_property, {}).get("pattern"):
-                            if not re.match(pattern, identifying_value):
-                                # If this identifying value is for a (identifying) property which has a
-                                # pattern, and the value does NOT match the pattern, then do NOT include
-                                # this value as an identifying path, since it cannot possibly be found.
-                                continue
-                    if not lookup_options:
-                        lookup_options = Portal.LOOKUP_DEFAULT
-                    if Portal.is_lookup_root_first(lookup_options):
-                        identifying_paths.append(f"/{identifying_value}")
-                    if Portal.is_lookup_specified_type(lookup_options) and self.type:
-                        identifying_paths.append(f"/{self.type}/{identifying_value}")
-                    if Portal.is_lookup_root(lookup_options) and not Portal.is_lookup_root_first(lookup_options):
-                        identifying_paths.append(f"/{identifying_value}")
-                    if Portal.is_lookup_subtypes(lookup_options):
-                        for subtype_name in self._portal.get_schema_subtype_names(self.type):
-                            identifying_paths.append(f"/{subtype_name}/{identifying_value}")
-        return identifying_paths or None
+        if not self._portal and (uuid := self.uuid):
+            return [f"/{uuid}"]
+        # Migrating to and unifying this in portal_utils.Portal.get_identifying_paths (2024-05-26).
+        return self._portal.get_identifying_paths(self._data,
+                                                  portal_type=self.schema,
+                                                  lookup_strategy=ref_lookup_strategy) if self._portal else None
 
     def _normalized_refs(self, refs: List[dict]) -> Tuple[PortalObject, int]:
         """

dcicutils/portal_utils.py

@@ -1,5 +1,6 @@
 from collections import deque
 from functools import lru_cache
+from dcicutils.function_cache_decorator import function_cache
 import io
 import json
 from pyramid.config import Configurator as PyramidConfigurator
@@ -18,6 +19,7 @@ from wsgiref.simple_server import make_server as wsgi_make_server
 from dcicutils.common import APP_SMAHT, OrchestratedApp, ORCHESTRATED_APPS
 from dcicutils.ff_utils import get_metadata, get_schema, patch_metadata, post_metadata
 from dcicutils.misc_utils import to_camel_case, VirtualApp
+from dcicutils.schema_utils import get_identifying_properties
 from dcicutils.tmpfile_utils import temporary_file
 
 Portal = Type["Portal"]  # Forward type reference for type hints.
@@ -48,15 +50,16 @@ class Portal:
     FILE_TYPE_SCHEMA_NAME = "File"
 
     # Object lookup strategies; on a per-reference (type/value) basis, used currently ONLY by
-    # structured_data.py; controlled by an optional ref_lookup_strategy callable; default is
+    # structured_data.py; controlled by an optional lookup_strategy callable; default is
     # lookup at root path but after the specified type path lookup, and then lookup all subtypes;
     # can choose to lookup root path first, or not lookup root path at all, or not lookup
-    # subtypes at all; the ref_lookup_strategy callable if specified should take a type_name
+    # subtypes at all; the lookup_strategy callable if specified should take a type_name
     # and value (string) arguements and return an integer of any of the below ORed together.
     # The main purpose of this is optimization; to minimize portal lookups; since for example,
     # currently at least, /{type}/{accession} does not work but /{accession} does; so we
     # currently (smaht-portal/.../ingestion_processors) use LOOKUP_ROOT_FIRST for this.
     # And current usage NEVER has LOOKUP_SUBTYPES turned OFF; but support just in case.
+    LOOKUP_UNDEFINED = 0
     LOOKUP_SPECIFIED_TYPE = 0x0001
     LOOKUP_ROOT = 0x0002
     LOOKUP_ROOT_FIRST = 0x0004 | LOOKUP_ROOT
@@ -205,23 +208,6 @@ class Portal:
     def vapp(self) -> Optional[TestApp]:
         return self._vapp
 
-    @staticmethod
-    def is_lookup_specified_type(lookup_options: int) -> bool:
-        return (lookup_options &
-                Portal.LOOKUP_SPECIFIED_TYPE) == Portal.LOOKUP_SPECIFIED_TYPE
-
-    @staticmethod
-    def is_lookup_root(lookup_options: int) -> bool:
-        return (lookup_options & Portal.LOOKUP_ROOT) == Portal.LOOKUP_ROOT
-
-    @staticmethod
-    def is_lookup_root_first(lookup_options: int) -> bool:
-        return (lookup_options & Portal.LOOKUP_ROOT_FIRST) == Portal.LOOKUP_ROOT_FIRST
-
-    @staticmethod
-    def is_lookup_subtypes(lookup_options: int) -> bool:
-        return (lookup_options & Portal.LOOKUP_SUBTYPES) == Portal.LOOKUP_SUBTYPES
-
     def get(self, url: str, follow: bool = True,
             raw: bool = False, database: bool = False, raise_for_status: bool = False, **kwargs) -> OptionalResponse:
         url = self.url(url, raw, database)
@@ -305,7 +291,10 @@ class Portal:
 
     @lru_cache(maxsize=100)
     def get_schema(self, schema_name: str) -> Optional[dict]:
-        return get_schema(self.schema_name(schema_name), portal_vapp=self.vapp, key=self.key)
+        try:
+            return get_schema(self.schema_name(schema_name), portal_vapp=self.vapp, key=self.key)
+        except Exception:
+            return None
 
     @lru_cache(maxsize=1)
     def get_schemas(self) -> dict:
@@ -416,6 +405,218 @@ class Portal:
             return []
         return schemas_super_type_map.get(type_name, [])
 
+    @function_cache(maxsize=100, serialize_key=True)
+    def get_identifying_paths(self, portal_object: dict, portal_type: Optional[Union[str, dict]] = None,
+                              first_only: bool = False,
+                              lookup_strategy: Optional[Union[Callable, bool]] = None) -> List[str]:
+        """
+        Returns the list of the identifying Portal (URL) paths for the given Portal object. Favors any uuid
+        and identifier based paths and defavors aliases based paths (ala self.get_identifying_property_names);
+        no other ordering defined. Returns an empty list if no identifying properties or otherwise not found.
+        Note that this is a newer version of what was in portal_object_utils and just uses the ref_lookup_stratey
+        module directly, as it no longer needs to be exposed (to smaht-portal/ingester and smaht-submitr) and so
+        this is a first step toward internalizing it to structured_data/portal_utils/portal_object_utils usages.
+        """
+        def is_lookup_specified_type(lookup_options: int) -> bool:
+            return (lookup_options & Portal.LOOKUP_SPECIFIED_TYPE) == Portal.LOOKUP_SPECIFIED_TYPE
+        def is_lookup_root(lookup_options: int) -> bool:  # noqa
+            return (lookup_options & Portal.LOOKUP_ROOT) == Portal.LOOKUP_ROOT
+        def is_lookup_root_first(lookup_options: int) -> bool:  # noqa
+            return (lookup_options & Portal.LOOKUP_ROOT_FIRST) == Portal.LOOKUP_ROOT_FIRST
+        def is_lookup_subtypes(lookup_options: int) -> bool:  # noqa
+            return (lookup_options & Portal.LOOKUP_SUBTYPES) == Portal.LOOKUP_SUBTYPES
+
+        results = []
+        if not isinstance(portal_object, dict):
+            return results
+        if not (isinstance(portal_type, str) and portal_type):
+            if isinstance(portal_type, dict):
+                # It appears that the given portal_type is an actual schema dictionary.
+                portal_type = self.schema_name(portal_type.get("title"))
+            if not (isinstance(portal_type, str) and portal_type):
+                if not (portal_type := self.get_schema_type(portal_object)):
+                    return results
+        if not callable(lookup_strategy):
+            lookup_strategy = None if lookup_strategy is False else Portal._lookup_strategy
+        for identifying_property in self.get_identifying_property_names(portal_type):
+            if not (identifying_value := portal_object.get(identifying_property)):
+                continue
+            # The get_identifying_property_names call above ensures uuid is first if it is in the object.
+            # And also note that ALL schemas do in fact have identifyingProperties which do in fact have
+            # uuid, except for a couple "Test" ones, and (for some reason) SubmittedItem; otherwise we
+            # might have a special case to check the Portal object explicitly for uuid, but no need.
+            if identifying_property == "uuid":
+                #
+                # Note this idiosyncrasy with Portal paths: the only way we do NOT get a (HTTP 301) redirect
+                # is if we use the lower-case-dashed-plural based version of the path, e.g. all of these:
+                #
+                # - /d13d06c1-218e-4f61-aaf0-91f226248b3c
+                # - /d13d06c1-218e-4f61-aaf0-91f226248b3c/
+                # - /FileFormat/d13d06c1-218e-4f61-aaf0-91f226248b3c
+                # - /FileFormat/d13d06c1-218e-4f61-aaf0-91f226248b3c/
+                # - /files-formats/d13d06c1-218e-4f61-aaf0-91f226248b3c
+                #
+                # Will result in a (HTTP 301) redirect to:
+                #
+                # - /files-formats/d13d06c1-218e-4f61-aaf0-91f226248b3c/
+                #
+                # Unfortunately, this code here has no reasonable way of getting that lower-case-dashed-plural
+                # based name (e.g. file-formats) from the schema/portal type name (e.g. FileFormat); as the
+                # information is contained, for this example, in the snovault.collection decorator for the
+                # endpoint definition in smaht-portal/.../types/file_format.py. Unfortunately merely because
+                # behind-the-scenes an extra round-trip HTTP request will occur, but happens automatically.
+                # And note the disction of just using /{uuid} here rather than /{type}/{uuid} as in the else
+                # statement below is not really necessary; just here for emphasis that this is all that's needed.
+                #
+                # TODO
+                # Consider (from PR-308) writing a portal API for retrieving possible path formats.
+                #
+                if first_only is True:
+                    results.append(f"/{portal_type}/{identifying_value}")
+                else:
+                    results.append(f"/{identifying_value}")
+            elif isinstance(identifying_value, list):
+                for identifying_value_item in identifying_value:
+                    if identifying_value_item:
+                        results.append(f"/{portal_type}/{identifying_value_item}")
+            else:
+                lookup_options = Portal.LOOKUP_UNDEFINED
+                if schema := self.get_schema(portal_type):
+                    if callable(lookup_strategy):
+                        lookup_options, validator = lookup_strategy(self, portal_type, schema, identifying_value)
+                        if callable(validator):
+                            if validator(schema, identifying_property, identifying_value) is False:
+                                continue
+                    if pattern := schema.get("properties", {}).get(identifying_property, {}).get("pattern"):
+                        if not re.match(pattern, identifying_value):
+                            # If this identifying value is for a (identifying) property which has a
+                            # pattern, and the value does NOT match the pattern, then do NOT include
+                            # this value as an identifying path, since it cannot possibly be found.
+                            continue
+                if lookup_options == Portal.LOOKUP_UNDEFINED:
+                    lookup_options = Portal.LOOKUP_DEFAULT
+                if is_lookup_root_first(lookup_options):
+                    results.append(f"/{identifying_value}")
+                if is_lookup_specified_type(lookup_options) and portal_type:
+                    results.append(f"/{portal_type}/{identifying_value}")
+                if is_lookup_root(lookup_options) and not is_lookup_root_first(lookup_options):
+                    results.append(f"/{identifying_value}")
+                if is_lookup_subtypes(lookup_options):
+                    for subtype_name in self.get_schema_subtype_names(portal_type):
+                        results.append(f"/{subtype_name}/{identifying_value}")
+            if (first_only is True) and results:
+                return results
+        return results
+
+    @function_cache(maxsize=100, serialize_key=True)
+    def get_identifying_path(self, portal_object: dict, portal_type: Optional[Union[str, dict]] = None,
+                             lookup_strategy: Optional[Union[Callable, bool]] = None) -> Optional[str]:
+        if identifying_paths := self.get_identifying_paths(portal_object, portal_type, first_only=True,
+                                                           lookup_strategy=lookup_strategy):
+            return identifying_paths[0]
+        return None
+
+    @function_cache(maxsize=100, serialize_key=True)
+    def get_identifying_property_names(self, schema: Union[str, dict],
+                                       portal_object: Optional[dict] = None) -> List[str]:
+        """
+        Returns the list of identifying property names for the given Portal schema, which may be
+        either a schema name or a schema object. If a Portal object is also given then restricts this
+        set of identifying properties to those which actually have values within this Portal object.
+        Favors the uuid and identifier property names and defavors the aliases property name; no other
+        ordering imposed. Returns empty list if no identifying properties or otherwise not found.
+        """
+        results = []
+        if isinstance(schema, str):
+            if not (schema := self.get_schema(schema)):
+                return results
+        elif not isinstance(schema, dict):
+            return results
+        if not (identifying_properties := get_identifying_properties(schema)):
+            return results
+        identifying_properties = list(set(identifying_properties))  # paranoid dedup
+        identifying_properties = [*identifying_properties]  # copy so as not to change schema if given
+        favored_identifying_properties = ["uuid", "identifier"]
+        defavored_identifying_properties = ["aliases"]
+        for favored_identifying_property in reversed(favored_identifying_properties):
+            if favored_identifying_property in identifying_properties:
+                identifying_properties.remove(favored_identifying_property)
+                identifying_properties.insert(0, favored_identifying_property)
+        for defavored_identifying_property in defavored_identifying_properties:
+            if defavored_identifying_property in identifying_properties:
+                identifying_properties.remove(defavored_identifying_property)
+                identifying_properties.append(defavored_identifying_property)
+        if isinstance(portal_object, dict):
+            for identifying_property in [*identifying_properties]:
+                if portal_object.get(identifying_property) is None:
+                    identifying_properties.remove(identifying_property)
+        return identifying_properties
+
+    @staticmethod
+    def _lookup_strategy(portal: Portal, type_name: str, schema: dict, value: str) -> (int, Optional[str]):
+        #
+        # Note this slightly odd situation WRT object lookups by submitted_id and accession:
+        # -----------------------------+-----------------------------------------------+---------------+
+        # PATH                         | EXAMPLE                                       | LOOKUP RESULT |
+        # -----------------------------+-----------------------------------------------+---------------+
+        # /submitted_id                | //UW_FILE-SET_COLO-829BL_HI-C_1               | NOT FOUND     |
+        # /UnalignedReads/submitted_id | /UnalignedReads/UW_FILE-SET_COLO-829BL_HI-C_1 | FOUND         |
+        # /SubmittedFile/submitted_id  | /SubmittedFile/UW_FILE-SET_COLO-829BL_HI-C_1  | FOUND         |
+        # /File/submitted_id           | /File/UW_FILE-SET_COLO-829BL_HI-C_1           | NOT FOUND     |
+        # -----------------------------+-----------------------------------------------+---------------+
+        # /accession                   | /SMAFSFXF1RO4                                 | FOUND         |
+        # /UnalignedReads/accession    | /UnalignedReads/SMAFSFXF1RO4                  | NOT FOUND     |
+        # /SubmittedFile/accession     | /SubmittedFile/SMAFSFXF1RO4                   | NOT FOUND     |
+        # /File/accession              | /File/SMAFSFXF1RO4                            | FOUND         |
+        # -----------------------------+-----------------------------------------------+---------------+
+        #
+        def ref_validator(schema: Optional[dict],
+                          property_name: Optional[str], property_value: Optional[str]) -> Optional[bool]:
+            """
+            Returns False iff objects of type represented by the given schema, CANNOT be referenced with
+            a Portal path using the given property name and its given property value, otherwise returns None.
+
+            For example, if the schema is for UnalignedReads and the property name is accession, then we will
+            return False iff the given property value is NOT a properly formatted accession ID; otherwise, we
+            will return None, which indicates that the caller (e.g. dcicutils.structured_data.Portal.ref_exists)
+            will continue executing its default behavior, which is to check other ways in which the given type
+            CANNOT be referenced by the given value, i.e. it checks other identifying properties for the type
+            and makes sure any patterns (e.g. for submitted_id or uuid) are ahered to.
+
+            The goal (in structured_data) being to detect if a type is being referenced in such a way that
+            CANNOT possibly be allowed, i.e. because none of its identifying types are in the required form,
+            if indeed there any requirements. It is assumed/guaranteed the given property name is indeed an
+            identifying property for the given type.
+            """
+            if property_format := schema.get("properties", {}).get(property_name, {}).get("format"):
+                if (property_format == "accession") and (property_name == "accession"):
+                    if not Portal._is_accession_id(property_value):
+                        return False
+            return None
+
+        DEFAULT_RESULT = (Portal.LOOKUP_DEFAULT, ref_validator)
+        if not value:
+            return DEFAULT_RESULT
+        if not schema:
+            if not isinstance(portal, Portal) or not (schema := portal.get_schema(type_name)):
+                return DEFAULT_RESULT
+        if schema_properties := schema.get("properties"):
+            if schema_properties.get("accession") and Portal._is_accession_id(value):
+                # Case: lookup by accession (only by root).
+                return (Portal.LOOKUP_ROOT, ref_validator)
+            elif schema_property_info_submitted_id := schema_properties.get("submitted_id"):
+                if schema_property_pattern_submitted_id := schema_property_info_submitted_id.get("pattern"):
+                    if re.match(schema_property_pattern_submitted_id, value):
+                        # Case: lookup by submitted_id (only by specified type).
+                        return (Portal.LOOKUP_SPECIFIED_TYPE, ref_validator)
+        return DEFAULT_RESULT
+
+    @staticmethod
+    def _is_accession_id(value: str) -> bool:
+        # This is here for now because of problems with circular dependencies.
+        # See: smaht-portal/.../schema_formats.py/is_accession(instance) ...
+        return isinstance(value, str) and re.match(r"^SMA[1-9A-Z]{9}$", value) is not None
+
     def url(self, url: str, raw: bool = False, database: bool = False) -> str:
         if not isinstance(url, str) or not url:
             return "/"
@@ -516,6 +717,22 @@ class Portal:
             response = TestResponseWrapper(response)
         return response
 
+    @staticmethod
+    def _create_vapp(arg: Union[TestApp, VirtualApp, PyramidRouter, str] = None) -> TestApp:
+        if isinstance(arg, TestApp):
+            return arg
+        elif isinstance(arg, VirtualApp):
+            if not isinstance(arg.wrapped_app, TestApp):
+                raise Exception("Portal._create_vapp VirtualApp argument error.")
+            return arg.wrapped_app
+        if isinstance(arg, PyramidRouter):
+            router = arg
+        elif isinstance(arg, str) or not arg:
+            router = pyramid_get_app(arg or "development.ini", "app")
+        else:
+            raise Exception("Portal._create_vapp argument error.")
+        return TestApp(router, {"HTTP_ACCEPT": Portal.MIME_TYPE_JSON, "REMOTE_USER": "TEST"})
+
     @staticmethod
     def create_for_testing(arg: Optional[Union[str, bool, List[dict], dict, Callable]] = None) -> Portal:
         if isinstance(arg, list) or isinstance(arg, dict) or isinstance(arg, Callable):
@@ -547,22 +764,6 @@ class Portal:
         with temporary_file(content=minimal_ini_for_testing, suffix=".ini") as ini_file:
             return Portal(ini_file)
 
-    @staticmethod
-    def _create_vapp(arg: Union[TestApp, VirtualApp, PyramidRouter, str] = None) -> TestApp:
-        if isinstance(arg, TestApp):
-            return arg
-        elif isinstance(arg, VirtualApp):
-            if not isinstance(arg.wrapped_app, TestApp):
-                raise Exception("Portal._create_vapp VirtualApp argument error.")
-            return arg.wrapped_app
-        if isinstance(arg, PyramidRouter):
-            router = arg
-        elif isinstance(arg, str) or not arg:
-            router = pyramid_get_app(arg or "development.ini", "app")
-        else:
-            raise Exception("Portal._create_vapp argument error.")
-        return TestApp(router, {"HTTP_ACCEPT": Portal.MIME_TYPE_JSON, "REMOTE_USER": "TEST"})
-
     @staticmethod
     def _create_router_for_testing(endpoints: Optional[List[Dict[str, Union[str, Callable]]]] = None) -> PyramidRouter:
         if isinstance(endpoints, dict):

dcicutils/progress_bar.py

@@ -249,7 +249,7 @@ class ProgressBar:
                         if self._interrupt_exit_message:
                             if isinstance(interrupt_exit_message := self._interrupt_exit_message(self), str):
                                 print(interrupt_exit_message)
-                        exit(1)
+                        sys.exit(1)
                     elif interrupt_stop is False or ((interrupt_stop is None) and (self._interrupt_exit is False)):
                         set_interrupt_handler(handle_interrupt)
                         interrupt_continue = self._interrupt_continue(self) if self._interrupt_continue else None

dcicutils/schema_utils.py

@@ -1,6 +1,5 @@
 import os
 from typing import Any, Dict, List, Optional, Tuple
-
 from dcicutils.misc_utils import to_camel_case
 
 
@@ -9,7 +8,6 @@ class JsonSchemaConstants:
     ARRAY = "array"
     BOOLEAN = "boolean"
     DEFAULT = "default"
-    DEPENDENT_REQUIRED = "dependentRequired"
     ENUM = "enum"
     FORMAT = "format"
     INTEGER = "integer"
@@ -31,10 +29,6 @@ class EncodedSchemaConstants:
     LINK_TO = "linkTo"
     MERGE_REF = "$merge"
     MIXIN_PROPERTIES = "mixinProperties"
-    SUBMISSION_COMMENT = "submissionComment"
-    SUBMISSION_EXAMPLES = "submissionExamples"
-    SUBMITTER_REQUIRED = "submitterRequired"
-    SUGGESTED_ENUM = "suggested_enum"
     UNIQUE_KEY = "uniqueKey"
 
 
@@ -209,50 +203,6 @@ def get_description(schema: Dict[str, Any]) -> str:
     return schema.get(SchemaConstants.DESCRIPTION, "")
 
 
-def is_submitter_required(schema: Dict[str, Any]) -> bool:
-    """Return True if the schema is marked as required for submitters.
-
-    Specifically, required for external (i.e. non-admin) submitters.
-
-    This is typically validated within the context of a oneOf, anyOf,
-    or allOf schema on an item type which is used within the team and
-    by external submitters, and is tricky to pick up on automatically.
-    """
-    return schema.get(SchemaConstants.SUBMITTER_REQUIRED, False)
-
-
-def get_submission_comment(schema: Dict[str, Any]) -> str:
-    """Return the submission comment for a property.
-
-    Custom property that can be manually added to a schema to provide
-    additional context for submitters.
-    """
-    return schema.get(SchemaConstants.SUBMISSION_COMMENT, "")
-
-
-def get_submission_examples(schema: Dict[str, Any]) -> List[str]:
-    """Return the submission example for a property.
-
-    Custom property that can be manually added to a schema to provide
-    an example for submitters.
-    """
-    return schema.get(SchemaConstants.SUBMISSION_EXAMPLES, [])
-
-
-def get_suggested_enum(schema: Dict[str, Any]) -> List[str]:
-    """Return the suggested enum for a property.
-
-    Custom property that can be manually added to a schema to provide
-    a suggested list of values for submitters.
-    """
-    return schema.get(SchemaConstants.SUGGESTED_ENUM, [])
-
-
-def get_dependent_required(schema: Dict[str, Any]) -> Dict[str, List[str]]:
-    """Return the dependent required properties of a schema."""
-    return schema.get(SchemaConstants.DEPENDENT_REQUIRED, {})
-
-
 class Schema:
 
     def __init__(self, schema: dict, type: Optional[str] = None) -> None:

dcicutils/scripts/publish_to_pypi.py

@@ -33,6 +33,7 @@ import argparse
 import os
 import requests
 import subprocess
+import sys
 import toml
 
 from typing import Tuple, Union
@@ -335,7 +336,7 @@ def exit_with_no_action() -> None:
     first prints a message saying no action was taken.
     """
     PRINT("Exiting without taking action.")
-    exit(1)
+    sys.exit(1)
 
 
 PRINT = print

dcicutils/scripts/view_portal_object.py

@@ -201,17 +201,17 @@ def main():
             return
         else:
             _print(f"No PATCH data found in file: {args.patch}")
-            exit(1)
+            sys.exit(1)
 
     data = _get_portal_object(portal=portal, uuid=args.uuid, raw=args.raw,
                               database=args.database, check=args.bool, verbose=args.verbose)
     if args.bool:
         if data:
             _print(f"{args.uuid}: found")
-            exit(0)
+            sys.exit(0)
         else:
             _print(f"{args.uuid}: not found")
-            exit(1)
+            sys.exit(1)
     if args.copy:
         pyperclip.copy(json.dumps(data, indent=4))
     if args.yaml:
@@ -597,18 +597,18 @@ def _print_tree(root_name: Optional[str],
 def _read_json_from_file(file: str) -> Optional[dict]:
     if not os.path.exists(file):
         _print(f"Cannot find file: {file}")
-        exit(1)
+        sys.exit(1)
     try:
         with io.open(file, "r") as f:
             try:
                 return json.load(f)
             except Exception:
                 _print(f"Cannot parse JSON in file: {file}")
-                exit(1)
+                sys.exit(1)
     except Exception as e:
         print(e)
         _print(f"Cannot open file: {file}")
-        exit(1)
+        sys.exit(1)
 
 
 def _print(*args, **kwargs):
@@ -620,7 +620,7 @@ def _print(*args, **kwargs):
 def _exit(message: Optional[str] = None) -> None:
     if message:
         _print(f"ERROR: {message}")
-    exit(1)
+    sys.exit(1)
 
 
 if __name__ == "__main__":

dcicutils/structured_data.py

@@ -11,7 +11,6 @@ from webtest.app import TestApp
 from dcicutils.common import OrchestratedApp
 from dcicutils.data_readers import CsvReader, Excel, RowReader
 from dcicutils.datetime_utils import normalize_date_string, normalize_datetime_string
-from dcicutils.file_utils import search_for_file
 from dcicutils.misc_utils import (create_dict, create_readonly_object, is_uuid, load_json_if,
                                   merge_objects, remove_empty_properties, right_trim, split_string,
                                   to_boolean, to_enum, to_float, to_integer, VirtualApp)
@@ -56,7 +55,7 @@ class StructuredDataSet:
                  remove_empty_objects_from_lists: bool = True,
                  ref_lookup_strategy: Optional[Callable] = None,
                  ref_lookup_nocache: bool = False,
-                 norefs: bool = False,
+                 norefs: bool = False, merge: bool = False,
                  progress: Optional[Callable] = None,
                  debug_sleep: Optional[str] = None) -> None:
         self._progress = progress if callable(progress) else None
@@ -75,6 +74,7 @@ class StructuredDataSet:
         self._nrows = 0
         self._autoadd_properties = autoadd if isinstance(autoadd, dict) and autoadd else None
         self._norefs = True if norefs is True else False
+        self._merge = True if merge is True else False  # New merge functionality (2024-05-25)
         self._debug_sleep = None
         if debug_sleep:
             try:
@@ -98,13 +98,13 @@ class StructuredDataSet:
              remove_empty_objects_from_lists: bool = True,
              ref_lookup_strategy: Optional[Callable] = None,
              ref_lookup_nocache: bool = False,
-             norefs: bool = False,
+             norefs: bool = False, merge: bool = False,
              progress: Optional[Callable] = None,
              debug_sleep: Optional[str] = None) -> StructuredDataSet:
         return StructuredDataSet(file=file, portal=portal, schemas=schemas, autoadd=autoadd, order=order, prune=prune,
                                  remove_empty_objects_from_lists=remove_empty_objects_from_lists,
                                  ref_lookup_strategy=ref_lookup_strategy, ref_lookup_nocache=ref_lookup_nocache,
-                                 norefs=norefs, progress=progress, debug_sleep=debug_sleep)
+                                 norefs=norefs, merge=merge, progress=progress, debug_sleep=debug_sleep)
 
     def validate(self, force: bool = False) -> None:
         def data_without_deleted_properties(data: dict) -> dict:
@@ -208,14 +208,6 @@ class StructuredDataSet:
                             result.append({"type": type_name, "file": file_name})
         return result
 
-    def upload_files_located(self,
-                             location: Union[str, Optional[List[str]]] = None, recursive: bool = False) -> List[str]:
-        upload_files = copy.deepcopy(self.upload_files)
-        for upload_file in upload_files:
-            if file_path := search_for_file(upload_file["file"], location, recursive=recursive, single=True):
-                upload_file["path"] = file_path
-        return upload_files
-
     @property
     def nrows(self) -> int:
         return self._nrows
@@ -350,18 +342,23 @@ class StructuredDataSet:
 
     def _load_json_file(self, file: str) -> None:
         with open(file) as f:
-            file_json = json.load(f)
-            schema_inferred_from_file_name = Schema.type_name(file)
-            if self._portal.get_schema(schema_inferred_from_file_name) is not None:
+            data = json.load(f)
+            if ((schema_name_inferred_from_file_name := Schema.type_name(file)) and
+                (self._portal.get_schema(schema_name_inferred_from_file_name) is not None)):  # noqa
                 # If the JSON file name looks like a schema name then assume it
                 # contains an object or an array of object of that schema type.
-                self._add(Schema.type_name(file), file_json)
-            elif isinstance(file_json, dict):
+                if self._merge:  # New merge functionality (2024-05-25)
+                    data = self._merge_with_existing_portal_object(data, schema_name_inferred_from_file_name)
+                self._add(Schema.type_name(file), data)
+            elif isinstance(data, dict):
                 # Otherwise if the JSON file name does not look like a schema name then
                 # assume it a dictionary where each property is the name of a schema, and
                 # which (each property) contains a list of object of that schema type.
-                for schema_name in file_json:
-                    self._add(schema_name, file_json[schema_name])
+                for schema_name in data:
+                    item = data[schema_name]
+                    if self._merge:  # New merge functionality (2024-05-25)
+                        item = self._merge_with_existing_portal_object(item, schema_name)
+                    self._add(schema_name, item)
 
     def _load_reader(self, reader: RowReader, type_name: str) -> None:
         schema = None
@@ -383,11 +380,13 @@ class StructuredDataSet:
                 structured_row_template.set_value(structured_row, column_name, value, reader.file, reader.row_number)
                 if self._autoadd_properties:
                     self._add_properties(structured_row, self._autoadd_properties, schema)
+            if self._merge:  # New merge functionality (2024-05-25)
+                structured_row = self._merge_with_existing_portal_object(structured_row, schema_name)
             if (prune_error := self._prune_structured_row(structured_row)) is not None:
                 self._note_error({"src": create_dict(type=schema_name, row=reader.row_number),
                                   "error": prune_error}, "validation")
             else:
-                self._add(type_name, structured_row)
+                self._add(type_name, structured_row)  # TODO: why type_name and not schema_name?
             if self._progress:
                 self._progress({
                     PROGRESS.LOAD_ITEM: self._nrows,
@@ -428,6 +427,18 @@ class StructuredDataSet:
             if name not in structured_row and (not schema or schema.data.get("properties", {}).get(name)):
                 structured_row[name] = properties[name]
 
+    def _merge_with_existing_portal_object(self, portal_object: dict, portal_type: str) -> dict:
+        """
+        Given a Portal object (presumably/in-practice from the given metadata), if there is
+        an existing Portal item, identified by the identifying properties for the given object,
+        then merges the given object into the existing one and returns the result; otherwise
+        just returns the given object. Note that the given object may be CHANGED in place.
+        """
+        for identifying_path in self._portal.get_identifying_paths(portal_object, portal_type):
+            if existing_portal_object := self._portal.get_metadata(identifying_path, raw=True, raise_exception=False):
+                return merge_objects(existing_portal_object, portal_object, primitive_lists=True)
+        return portal_object
+
     def _is_ref_lookup_specified_type(ref_lookup_flags: int) -> bool:
         return (ref_lookup_flags &
                 Portal.LOOKUP_SPECIFIED_TYPE) == Portal.LOOKUP_SPECIFIED_TYPE

dcicutils/submitr/ref_lookup_strategy.py

@@ -2,39 +2,45 @@ import re
 from typing import Optional
 from dcicutils.structured_data import Portal
 
+# This function is exposed (to smaht-portal/ingester and smaht-submitr) only because previously,
+# before it was fully developed, we had differing behaviors; but this has been unified; so this
+# could now be internalized to structured_data, and portal_object_utils (TODO).
+
 
 def ref_lookup_strategy(portal: Portal, type_name: str, schema: dict, value: str) -> (int, Optional[str]):
     #
-    # FYI: Note this situation WRT object lookups ...
-    #
-    # /{submitted_id}                # NOT FOUND
-    # /UnalignedReads/{submitted_id} # OK
-    # /SubmittedFile/{submitted_id}  # OK
-    # /File/{submitted_id}           # NOT FOUND
-    #
-    # /{accession}                   # OK
-    # /UnalignedReads/{accession}    # NOT FOUND
-    # /SubmittedFile/{accession}     # NOT FOUND
-    # /File/{accession}              # OK
+    # Note this slight odd situation WRT object lookups by submitted_id and accession:
+    # -----------------------------+-----------------------------------------------+---------------+
+    # PATH                         | EXAMPLE                                       | LOOKUP RESULT |
+    # -----------------------------+-----------------------------------------------+---------------+
+    # /submitted_id                | //UW_FILE-SET_COLO-829BL_HI-C_1               | NOT FOUND     |
+    # /UnalignedReads/submitted_id | /UnalignedReads/UW_FILE-SET_COLO-829BL_HI-C_1 | FOUND         |
+    # /SubmittedFile/submitted_id  | /SubmittedFile/UW_FILE-SET_COLO-829BL_HI-C_1  | FOUND         |
+    # /File/submitted_id           | /File/UW_FILE-SET_COLO-829BL_HI-C_1           | NOT FOUND     |
+    # -----------------------------+-----------------------------------------------+---------------+
+    # /accession                   | /SMAFSFXF1RO4                                 | FOUND         |
+    # /UnalignedReads/accession    | /UnalignedReads/SMAFSFXF1RO4                  | NOT FOUND     |
+    # /SubmittedFile/accession     | /SubmittedFile/SMAFSFXF1RO4                   | NOT FOUND     |
+    # /File/accession              | /File/SMAFSFXF1RO4                            | FOUND         |
+    # -----------------------------+-----------------------------------------------+---------------+
     #
     def ref_validator(schema: Optional[dict],
                       property_name: Optional[str], property_value: Optional[str]) -> Optional[bool]:
         """
-        Returns False iff the type represented by the given schema, can NOT be referenced by
-        the given property name with the given property value, otherwise returns None.
+        Returns False iff objects of type represented by the given schema, CANNOT be referenced with
+        a Portal path using the given property name and its given property value, otherwise returns None.
 
-        For example, if the schema is for the UnalignedReads type and the property name
-        is accession, then we will return False iff the given property value is NOT a properly
-        formatted accession ID. Otherwise, we will return None, which indicates that the
-        caller (in dcicutils.structured_data.Portal.ref_exists) will continue executing
-        its default behavior, which is to check other ways in which the given type can NOT
-        be referenced by the given value, i.e. it checks other identifying properties for
-        the type and makes sure any patterns (e.g. for submitted_id or uuid) are ahered to.
+        For example, if the schema is for UnalignedReads and the property name is accession, then we will
+        return False iff the given property value is NOT a properly formatted accession ID; otherwise, we
+        will return None, which indicates that the caller (e.g. dcicutils.structured_data.Portal.ref_exists)
+        will continue executing its default behavior, which is to check other ways in which the given type
+        CANNOT be referenced by the given value, i.e. it checks other identifying properties for the type
+        and makes sure any patterns (e.g. for submitted_id or uuid) are ahered to.
 
-        The goal (in structured_data) being to detect if a type is being referenced in such
-        a way that cannot possibly be allowed, i.e. because none of its identifying types
-        are in the required form (if indeed there any requirements). Note that it is guaranteed
-        that the given property name is indeed an identifying property for the given type.
+        The goal (in structured_data) being to detect if a type is being referenced in such a way that
+        CANNOT possibly be allowed, i.e. because none of its identifying types are in the required form,
+        if indeed there any requirements. It is assumed/guaranteed the given property name is indeed an
+        identifying property for the given type.
         """
         if property_format := schema.get("properties", {}).get(property_name, {}).get("format"):
             if (property_format == "accession") and (property_name == "accession"):
@@ -62,6 +68,6 @@ def ref_lookup_strategy(portal: Portal, type_name: str, schema: dict, value: str
 
 
 # This is here for now because of problems with circular dependencies.
-# See: smaht-portal/.../schema_formats.py
+# See: smaht-portal/.../schema_formats.py/is_accession(instance) ...
 def _is_accession_id(value: str) -> bool:
     return isinstance(value, str) and re.match(r"^SMA[1-9A-Z]{9}$", value) is not None

{dcicutils-8.10.0.0b0.dist-info → dcicutils-8.10.0.1b2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcicutils
-Version: 8.10.0.0b0
+Version: 8.10.0.1b2
 Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
 Home-page: https://github.com/4dn-dcic/utils
 License: MIT

{dcicutils-8.10.0.0b0.dist-info → dcicutils-8.10.0.1b2.dist-info}/RECORD

@@ -5,7 +5,7 @@ dcicutils/bundle_utils.py,sha256=ZVQcqlt7Yly8-YbL3A-5DW859_hMWpTL6dXtknEYZIw,346
 dcicutils/captured_output.py,sha256=0hP7sPwleMaYXQAvCfJOxG8Z8T_JJYy8ADp8A5ZoblE,3295
 dcicutils/cloudformation_utils.py,sha256=MtWJrSTXyiImgbPHgRvfH9bWso20ZPLTFJAfhDQSVj4,13786
 dcicutils/codebuild_utils.py,sha256=CKpmhJ-Z8gYbkt1I2zyMlKtFdsg7T8lqrx3V5ieta-U,1155
-dcicutils/command_utils.py,sha256=JExll5TMqIcmuiGvuS8q4XDUvoEfi2oSH0E2FVF6suU,15285
+dcicutils/command_utils.py,sha256=4LkszKXWlvgaUP_7d_tRsRqpOMc9IaohD2Kc9u9bEeU,18506
 dcicutils/common.py,sha256=YE8Mt5-vaZWWz4uaChSVhqGFbFtW5QKtnIyOr4zG4vM,3955
 dcicutils/contribution_scripts.py,sha256=0k5Gw1TumcD5SAcXVkDd6-yvuMEw-jUp5Kfb7FJH6XQ,2015
 dcicutils/contribution_utils.py,sha256=vYLS1JUB3sKd24BUxZ29qUBqYeQBLK9cwo8x3k64uPg,25653
@@ -13,15 +13,15 @@ dcicutils/creds_utils.py,sha256=64BbIfS90T1eJmmQJrDyfrRa3V2F1x7T8lOrEeFfqJY,1112
 dcicutils/data_readers.py,sha256=6EMrY7TjDE8H7bA_TCWtpLQP7slJ0YTL77_dNh6e7sg,7626
 dcicutils/data_utils.py,sha256=k2OxOlsx7AJ6jF-YNlMyGus_JqSUBe4_n1s65Mv1gQQ,3098
 dcicutils/datetime_utils.py,sha256=sM653aw_1zy1qBmfAH-WetCi2Fw9cnFK7FZN_Tg4onI,13499
-dcicutils/deployment_utils.py,sha256=sKv8Jb-_Zw2aH3OAywRlsre-Cqm3a7fEGG3_1PT-r-s,69908
+dcicutils/deployment_utils.py,sha256=6Sv1cM9T944d_9CTJt8O2RPtpw9MiTIPKw_DgYLOnCU,69916
 dcicutils/diff_utils.py,sha256=sQx-yz56DHAcQWOChYbAG3clXu7TbiZKlw-GggeveO0,8118
 dcicutils/docker_utils.py,sha256=30gUiqz7X9rJwSPXTPn4ewjQibUgoSJqhP9o9vn5X-A,1747
-dcicutils/ecr_scripts.py,sha256=cxDl4BQt5W9NN1zjOHYZ7kk3yqoMnaCfNIyLaYmCqdE,19474
+dcicutils/ecr_scripts.py,sha256=puiWqjIshGWNc7ErJvkyizQRINoDToDu7DXHM9ouKMc,19497
 dcicutils/ecr_utils.py,sha256=V1Eif-6THI38W1uKXUyYs-5ciYAVkGLv0sVS4xoh70o,13079
 dcicutils/ecs_utils.py,sha256=1sIIY5IVb__rNyZ5B9iIQKHiUIMldB7JRaeIISLaujY,3590
 dcicutils/env_base.py,sha256=KQmChAvX9riF_khDlW4hmlrHO0GqFMNXrFdiuSmbZtc,6356
 dcicutils/env_manager.py,sha256=Nte1oiW_AQ-isatzsf-fiNpPWInLdfRxzPrATdNoKkI,9444
-dcicutils/env_scripts.py,sha256=0YcEKn7MLLx_St_n74Oe7RlIs7xt3GA0mE04eqrTrfg,3909
+dcicutils/env_scripts.py,sha256=gAoXOemg-zA4I_IPeRuymIfhNOYCw_WmPUa6kcSiWkg,3936
 dcicutils/env_utils.py,sha256=hJTXsn6GC55-55ja1ERBmUbeh-DjJABm74eYRGRaiSc,46970
 dcicutils/env_utils_legacy.py,sha256=J81OAtJHN69o1beHO6q1j7_J6TeblSjnAHlS8VA5KSM,29032
 dcicutils/es_utils.py,sha256=ZksLh5ei7kRUfiFltk8sd2ZSfh15twbstrMzBr8HNw4,7541
@@ -39,42 +39,42 @@ dcicutils/lang_utils.py,sha256=MI3K6bPHLUqlkx3s_9jYZfbGbahiQFlpq4rBE3OYMbg,28151
 dcicutils/license_policies/c4-infrastructure.jsonc,sha256=xEQbIN08Y2xh3gSLRtSz9EhAZox1p3kHC4r678hCpss,278
 dcicutils/license_policies/c4-python-infrastructure.jsonc,sha256=Tkq8P1mKGYlix68I82IFNmasrT4wtSdokOIM-g2B8DQ,296
 dcicutils/license_policies/park-lab-common-server.jsonc,sha256=aaK-NdFDT8f8z_gBXihZnQJ6g3CAZdGSlHOwUP8HvUQ,5790
-dcicutils/license_policies/park-lab-common.jsonc,sha256=0LIFlIQWmdHlt160ku8cTOiB59akgYzRhRN3Xb5G_Aw,18864
+dcicutils/license_policies/park-lab-common.jsonc,sha256=QyzpPveVr87RMpjrLLhnxLSp4VuEWta1gehMAqgKKig,18995
 dcicutils/license_policies/park-lab-gpl-pipeline.jsonc,sha256=vLZkwm3Js-kjV44nug3PizRGDLVnDox4CnvDKu5d2oQ,3260
 dcicutils/license_policies/park-lab-pipeline.jsonc,sha256=9qlY0ASy3iUMQlr3gorVcXrSfRHnVGbLhkS427UaRy4,283
 dcicutils/license_utils.py,sha256=d1cq6iwv5Ju-VjdoINi6q7CPNNL7Oz6rcJdLMY38RX0,46978
 dcicutils/log_utils.py,sha256=7pWMc6vyrorUZQf-V-M3YC6zrPgNhuV_fzm9xqTPph0,10883
-dcicutils/misc_utils.py,sha256=zHwsxxEn24muLBP7mDvMa8I9VdMejwW8HMuCL5xbhhw,107690
+dcicutils/misc_utils.py,sha256=-syqTAj8DESiiP_KHoyBv9VvfboFYB03QbBlmXnBZXw,109423
 dcicutils/obfuscation_utils.py,sha256=fo2jOmDRC6xWpYX49u80bVNisqRRoPskFNX3ymFAmjw,5963
 dcicutils/opensearch_utils.py,sha256=V2exmFYW8Xl2_pGFixF4I2Cc549Opwe4PhFi5twC0M8,1017
-dcicutils/portal_object_utils.py,sha256=gDXRgPsRvqCFwbC8WatsuflAxNiigOnqr0Hi93k3AgE,15422
-dcicutils/portal_utils.py,sha256=DYyE5o15GekDgzpJWas9iS7klAYbjJZUPW0G42McArk,30779
-dcicutils/progress_bar.py,sha256=UT7lxb-rVF_gp4yjY2Tg4eun1naaH__hB4_v3O85bcE,19468
+dcicutils/portal_object_utils.py,sha256=Az3n1aL-PQkN5gOFE6ZqC2XkYsqiwKlq7-tZggs1QN4,11062
+dcicutils/portal_utils.py,sha256=R7v4uQUll34mn-NxyU3qoTouAwWrVDzW6W1zBGSU-M4,44762
+dcicutils/progress_bar.py,sha256=R3bWLMYM3Xq9PY7U0JadcFT1m7QmCbrbDp9qs9_Kf_c,19472
 dcicutils/project_utils.py,sha256=qPdCaFmWUVBJw4rw342iUytwdQC0P-XKpK4mhyIulMM,31250
 dcicutils/qa_checkers.py,sha256=cdXjeL0jCDFDLT8VR8Px78aS10hwNISOO5G_Zv2TZ6M,20534
 dcicutils/qa_utils.py,sha256=TT0SiJWiuxYvbsIyhK9VO4uV_suxhB6CpuC4qPacCzQ,160208
 dcicutils/redis_tools.py,sha256=qkcSNMtvqkpvts-Cm9gWhneK523Q_oHwhNUud1be1qk,7055
 dcicutils/redis_utils.py,sha256=VJ-7g8pOZqR1ZCtdcjKz3-6as2DMUcs1b1zG6wSprH4,6462
 dcicutils/s3_utils.py,sha256=LauLFQGvZLfpBJ81tYMikjLd3SJRz2R_FrL1n4xSlyI,28868
-dcicutils/schema_utils.py,sha256=GmRm-XqZKJ6qine16SQF1txcby9WougDav_sYmKNs9E,12400
-dcicutils/scripts/publish_to_pypi.py,sha256=LFzNHIQK2EXFr88YcfctyA_WKEBFc1ElnSjWrCXedPM,13889
+dcicutils/schema_utils.py,sha256=IIteRrg-iOJOFU17n2lvKByVdWdiMfuAQ1kf_QIM96Q,10604
+dcicutils/scripts/publish_to_pypi.py,sha256=JCrAiYFF9lMkj3s2w0J2EtN8P65qEamMe3ZZYPCF4Yo,13904
 dcicutils/scripts/run_license_checker.py,sha256=z2keYnRDZsHQbTeo1XORAXSXNJK5axVzL5LjiNqZ7jE,4184
-dcicutils/scripts/view_portal_object.py,sha256=HZzM44BDcGycO9XTOTZyP-F7PRMZaZrnFfiqiT7Qvqg,29777
+dcicutils/scripts/view_portal_object.py,sha256=rnlIoblDpoLPJ-Bor7OHxLgrVZyvqoxA0jmHW8ogg3s,29805
 dcicutils/secrets_utils.py,sha256=8dppXAsiHhJzI6NmOcvJV5ldvKkQZzh3Fl-cb8Wm7MI,19745
 dcicutils/sheet_utils.py,sha256=VlmzteONW5VF_Q4vo0yA5vesz1ViUah1MZ_yA1rwZ0M,33629
 dcicutils/snapshot_utils.py,sha256=ymP7PXH6-yEiXAt75w0ldQFciGNqWBClNxC5gfX2FnY,22961
 dcicutils/ssl_certificate_utils.py,sha256=F0ifz_wnRRN9dfrfsz7aCp4UDLgHEY8LaK7PjnNvrAQ,9707
-dcicutils/structured_data.py,sha256=XOMxrmkJohdCAyCJU09uI8ivthTKrtSSYReFbC9VYMs,63058
+dcicutils/structured_data.py,sha256=HVe1ruXz0vH4nRBOwq0cNrfR4KtqocC4U940KRXM5zY,64160
 dcicutils/submitr/progress_constants.py,sha256=5bxyX77ql8qEJearfHEvsvXl7D0GuUODW0T65mbRmnE,2895
-dcicutils/submitr/ref_lookup_strategy.py,sha256=Js2cVznTmgjciLWBPLCvMiwLIHXjDn3jww-gJPjYuFw,3467
+dcicutils/submitr/ref_lookup_strategy.py,sha256=VJN-Oo0LLna6Vo2cu47eC-eU-yUC9NFlQP29xajejVU,4741
 dcicutils/task_utils.py,sha256=MF8ujmTD6-O2AC2gRGPHyGdUrVKgtr8epT5XU8WtNjk,8082
 dcicutils/tmpfile_utils.py,sha256=irmN6Otvtxyum-7qr5h9GIzDs9rtFFyUsGQyqJXd_y4,2997
 dcicutils/trace_utils.py,sha256=g8kwV4ebEy5kXW6oOrEAUsurBcCROvwtZqz9fczsGRE,1769
 dcicutils/validation_utils.py,sha256=cMZIU2cY98FYtzK52z5WUYck7urH6JcqOuz9jkXpqzg,14797
 dcicutils/variant_utils.py,sha256=2H9azNx3xAj-MySg-uZ2SFqbWs4kZvf61JnK6b-h4Qw,4343
 dcicutils/zip_utils.py,sha256=_Y9EmL3D2dUZhxucxHvrtmmlbZmK4FpSsHEb7rGSJLU,3265
-dcicutils-8.10.0.0b0.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
-dcicutils-8.10.0.0b0.dist-info/METADATA,sha256=vcIAVrY7xnIajSS-yHlYgIrl9PGr2EmjUnnG9jDctEQ,3440
-dcicutils-8.10.0.0b0.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
-dcicutils-8.10.0.0b0.dist-info/entry_points.txt,sha256=51Q4F_2V10L0282W7HFjP4jdzW4K8lnWDARJQVFy_hw,270
-dcicutils-8.10.0.0b0.dist-info/RECORD,,
+dcicutils-8.10.0.1b2.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
+dcicutils-8.10.0.1b2.dist-info/METADATA,sha256=KmlGHSWKamA0Ry44aKGV2kgQNs4f0Qxi3XrxHvcDE0I,3440
+dcicutils-8.10.0.1b2.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+dcicutils-8.10.0.1b2.dist-info/entry_points.txt,sha256=51Q4F_2V10L0282W7HFjP4jdzW4K8lnWDARJQVFy_hw,270
+dcicutils-8.10.0.1b2.dist-info/RECORD,,