dcicutils 8.9.0.1b5__tar.gz → 8.10.0.0b0__tar.gz

{dcicutils-8.9.0.1b5 → dcicutils-8.10.0.0b0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcicutils
-Version: 8.9.0.1b5
+Version: 8.10.0.0b0
 Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
 Home-page: https://github.com/4dn-dcic/utils
 License: MIT

{dcicutils-8.9.0.1b5 → dcicutils-8.10.0.0b0}/dcicutils/command_utils.py

@@ -1,4 +1,3 @@
-from __future__ import annotations
 import contextlib
 import functools
 import glob
@@ -8,7 +7,7 @@ import re
 import requests
 import subprocess
 
-from typing import Callable, Optional
+from typing import Optional
 from .exceptions import InvalidParameterError
 from .lang_utils import there_are
 from .misc_utils import INPUT, PRINT, environ_bool, print_error_message, decorator
@@ -385,70 +384,3 @@ def script_catch_errors():
                 message = str(e)  # Note: We ignore the type, which isn't intended to be shown.
                 PRINT(message)
             exit(1)
-
-
-class Question:
-    """
-    Supports asking the user (via stdin) a yes/no question, possibly repeatedly; and after
-    some maximum number times of the same answer in a row (consecutively), then asks them
-    if they want to automatically give that same answer to any/all subsequent questions.
-    Supports static/global list of such Question instances, hashed (only) by the question text.
-    """
-    _static_instances = {}
-
-    @staticmethod
-    def instance(question: Optional[str] = None,
-                 max: Optional[int] = None, printf: Optional[Callable] = None) -> Question:
-        question = question if isinstance(question, str) else ""
-        if not (instance := Question._static_instances.get(question)):
-            Question._static_instances[question] = (instance := Question(question, max=max, printf=printf))
-        return instance
-
-    @staticmethod
-    def yes(question: Optional[str] = None,
-            max: Optional[int] = None, printf: Optional[Callable] = None) -> bool:
-        return Question.instance(question, max=max, printf=printf).ask()
-
-    def __init__(self, question: Optional[str] = None,
-                 max: Optional[int] = None, printf: Optional[Callable] = None) -> None:
-        self._question = question if isinstance(question, str) else ""
-        self._max = max if isinstance(max, int) and max > 0 else None
-        self._print = printf if callable(printf) else print
-        self._yes_consecutive_count = 0
-        self._no_consecutive_count = 0
-        self._yes_automatic = False
-        self._no_automatic = False
-
-    def ask(self, question: Optional[str] = None) -> bool:
-
-        def question_automatic(value: str) -> bool:
-            nonlocal self
-            RARROW = "▶"
-            LARROW = "◀"
-            if yes_or_no(f"{RARROW}{RARROW}{RARROW}"
-                         f" Do you want to answer {value} to all such questions?"
-                         f" {LARROW}{LARROW}{LARROW}"):
-                return True
-            self._yes_consecutive_count = 0
-            self._no_consecutive_count = 0
-
-        if self._yes_automatic:
-            return True
-        elif self._no_automatic:
-            return False
-        elif yes_or_no((question if isinstance(question, str) else "") or self._question or "Undefined question"):
-            self._yes_consecutive_count += 1
-            self._no_consecutive_count = 0
-            if (self._no_consecutive_count == 0) and self._max and (self._yes_consecutive_count >= self._max):
-                # Have reached the maximum number of consecutive YES answers; ask if YES to all subsequent.
-                if question_automatic("YES"):
-                    self._yes_automatic = True
-            return True
-        else:
-            self._no_consecutive_count += 1
-            self._yes_consecutive_count = 0
-            if (self._yes_consecutive_count == 0) and self._max and (self._no_consecutive_count >= self._max):
-                # Have reached the maximum number of consecutive NO answers; ask if NO to all subsequent.
-                if question_automatic("NO"):
-                    self._no_automatic = True
-            return False

{dcicutils-8.9.0.1b5 → dcicutils-8.10.0.0b0}/dcicutils/misc_utils.py

@@ -4,7 +4,6 @@ This file contains functions that might be generally useful.
 
 from collections import namedtuple
 import appdirs
-from copy import deepcopy
 import contextlib
 import datetime
 import functools
@@ -2200,58 +2199,28 @@ def merge_key_value_dict_lists(x, y):
     return [key_value_dict(k, v) for k, v in merged.items()]
 
 
-def merge_objects(target: Union[dict, List[Any]], source: Union[dict, List[Any]],
-                  full: bool = False,  # deprecated
-                  expand_lists: Optional[bool] = None,
-                  primitive_lists: bool = False,
-                  copy: bool = False, _recursing: bool = False) -> Union[dict, List[Any]]:
+def merge_objects(target: Union[dict, List[Any]], source: Union[dict, List[Any]], full: bool = False) -> dict:
     """
-    Merges the given source dictionary or list into the target dictionary or list and returns the
-    result. This MAY well change the given target (dictionary or list) IN PLACE ... UNLESS the copy
-    argument is True, then the given target will not change as a local copy is made (and returned).
-
-    If the expand_lists argument is True then any target lists longer than the
-    source be will be filled out with the last element(s) of the source; the full
-    argument (is deprecated and) is a synomym for this. The default is False.
-
-    If the primitive_lists argument is True then lists of primitives (i.e. lists in which
-    NONE of its elements are dictionaries, lists, or tuples) will themselves be treated
-    like primitives, meaning the whole of a source list will replace the corresponding
-    target; otherwise they  will be merged normally, meaning each element of a source list
-    will be merged, recursively, into the corresponding target list. The default is False.
+    Merges the given source dictionary or list into the target dictionary or list.
+    This MAY well change the given target (dictionary or list) IN PLACE.
+    The the full argument is True then any target lists longer than the
+    source be will be filled out with the last element(s) of the source.
     """
-    def is_primitive_list(value: Any) -> bool:  # noqa
-        if not isinstance(value, list):
-            return False
-        for item in value:
-            if isinstance(item, (dict, list, tuple)):
-                return False
-        return True
-
     if target is None:
         return source
-    if expand_lists not in (True, False):
-        expand_lists = full is True
-    if (copy is True) and (_recursing is not True):
-        target = deepcopy(target)
     if isinstance(target, dict) and isinstance(source, dict) and source:
         for key, value in source.items():
-            if ((primitive_lists is True) and
-                (key in target) and is_primitive_list(target[key]) and is_primitive_list(value)):  # noqa
-                target[key] = value
-            else:
-                target[key] = merge_objects(target[key], value,
-                                            expand_lists=expand_lists, _recursing=True) if key in target else value
+            target[key] = merge_objects(target[key], value, full) if key in target else value
     elif isinstance(target, list) and isinstance(source, list) and source:
         for i in range(max(len(source), len(target))):
             if i < len(target):
                 if i < len(source):
-                    target[i] = merge_objects(target[i], source[i], expand_lists=expand_lists, _recursing=True)
-                elif expand_lists is True:
-                    target[i] = merge_objects(target[i], source[len(source) - 1], expand_lists=expand_lists)
+                    target[i] = merge_objects(target[i], source[i], full)
+                elif full:
+                    target[i] = merge_objects(target[i], source[len(source) - 1], full)
             else:
                 target.append(source[i])
-    elif source not in (None, {}, []):
+    elif source:
         target = source
     return target
 

{dcicutils-8.9.0.1b5 → dcicutils-8.10.0.0b0}/dcicutils/portal_object_utils.py

@@ -1,5 +1,6 @@
 from copy import deepcopy
 from functools import lru_cache
+import re
 from typing import Any, Callable, List, Optional, Tuple, Type, Union
 from dcicutils.data_readers import RowReader
 from dcicutils.misc_utils import create_readonly_object
@@ -13,9 +14,11 @@ class PortalObject:
 
     _PROPERTY_DELETION_SENTINEL = RowReader.CELL_DELETION_SENTINEL
 
-    def __init__(self, data: dict, portal: Optional[Portal] = None, type: Optional[str] = None) -> None:
+    def __init__(self, data: dict, portal: Portal = None,
+                 schema: Optional[Union[dict, Schema]] = None, type: Optional[str] = None) -> None:
         self._data = data if isinstance(data, dict) else {}
         self._portal = portal if isinstance(portal, Portal) else None
+        self._schema = schema if isinstance(schema, dict) else (schema.data if isinstance(schema, Schema) else None)
         self._type = type if isinstance(type, str) else ""
 
     @property
@@ -29,7 +32,7 @@ class PortalObject:
     @property
     @lru_cache(maxsize=1)
     def type(self) -> str:
-        return self._type or Portal.get_schema_type(self._data) or ""
+        return self._type or Portal.get_schema_type(self._data) or (Schema(self._schema).type if self._schema else "")
 
     @property
     @lru_cache(maxsize=1)
@@ -44,7 +47,7 @@ class PortalObject:
     @property
     @lru_cache(maxsize=1)
     def schema(self) -> Optional[dict]:
-        return self._portal.get_schema(self.type) if self._portal else None
+        return self._schema if self._schema else (self._portal.get_schema(self.type) if self._portal else None)
 
     def copy(self) -> PortalObject:
         return PortalObject(deepcopy(self.data), portal=self.portal, type=self.type)
@@ -56,29 +59,39 @@ class PortalObject:
         Returns the list of all identifying property names of this Portal object which actually have values.
         Implicitly include "uuid" and "identifier" properties as identifying properties if they are actually
         properties in the object schema, and favor these (first); defavor "aliases"; no other ordering defined.
-        Changed (2024-05-26) to use portal_utils.get_identifying_property_names; migrating some intricate stuff there.
         """
-        # Migrating to and unifying this in portal_utils.Portal.get_identifying_paths (2024-05-26).
-        return self._portal.get_identifying_property_names(self.type, portal_object=self._data) if self._portal else []
+        if not (schema := self.schema) or not (schema_identifying_properties := schema.get("identifyingProperties")):
+            return None
+        identifying_properties = []
+        for identifying_property in schema_identifying_properties:
+            if identifying_property not in ["uuid", "identifier", "aliases"]:
+                if self._data.get(identifying_property):
+                    identifying_properties.append(identifying_property)
+        if self._data.get("identifier"):
+            identifying_properties.insert(0, "identifier")
+        if self._data.get("uuid"):
+            identifying_properties.insert(0, "uuid")
+        if "aliases" in schema_identifying_properties and self._data.get("aliases"):
+            identifying_properties.append("aliases")
+        return identifying_properties or None
 
     @lru_cache(maxsize=8192)
     def lookup(self, raw: bool = False,
                ref_lookup_strategy: Optional[Callable] = None) -> Tuple[Optional[PortalObject], Optional[str], int]:
-        if not (identifying_paths := self._get_identifying_paths(ref_lookup_strategy=ref_lookup_strategy)):
-            return None, None, 0
         nlookups = 0
         first_identifying_path = None
         try:
-            for identifying_path in identifying_paths:
-                if not first_identifying_path:
-                    first_identifying_path = identifying_path
-                nlookups += 1
-                if self._portal and (item := self._portal.get(identifying_path, raw=raw)) and (item.status_code == 200):
-                    return (
-                        PortalObject(item.json(), portal=self._portal, type=self.type if raw else None),
-                        identifying_path,
-                        nlookups
-                    )
+            if identifying_paths := self._get_identifying_paths(ref_lookup_strategy=ref_lookup_strategy):
+                for identifying_path in identifying_paths:
+                    if not first_identifying_path:
+                        first_identifying_path = identifying_path
+                    nlookups += 1
+                    if (value := self._portal.get(identifying_path, raw=raw)) and (value.status_code == 200):
+                        return (
+                            PortalObject(value.json(), portal=self._portal, type=self.type if raw else None),
+                            identifying_path,
+                            nlookups
+                        )
         except Exception:
             pass
         return None, first_identifying_path, nlookups
@@ -146,12 +159,64 @@ class PortalObject:
 
     @lru_cache(maxsize=1)
     def _get_identifying_paths(self, ref_lookup_strategy: Optional[Callable] = None) -> Optional[List[str]]:
-        if not self._portal and (uuid := self.uuid):
-            return [f"/{uuid}"]
-        # Migrating to and unifying this in portal_utils.Portal.get_identifying_paths (2024-05-26).
-        return self._portal.get_identifying_paths(self._data,
-                                                  portal_type=self.schema,
-                                                  lookup_strategy=ref_lookup_strategy) if self._portal else None
+        """
+        Returns a list of the possible Portal URL paths identifying this Portal object.
+        """
+        identifying_paths = []
+        if not (identifying_properties := self.identifying_properties):
+            if self.uuid:
+                if self.type:
+                    identifying_paths.append(f"/{self.type}/{self.uuid}")
+                identifying_paths.append(f"/{self.uuid}")
+            return identifying_paths
+        for identifying_property in identifying_properties:
+            if identifying_value := self._data.get(identifying_property):
+                if identifying_property == "uuid":
+                    if self.type:
+                        identifying_paths.append(f"/{self.type}/{identifying_value}")
+                    identifying_paths.append(f"/{identifying_value}")
+                # For now at least we include the path both with and without the schema type component,
+                # as for some identifying values, it works (only) with, and some, it works (only) without.
+                # For example: If we have FileSet with "accession", an identifying property, with value
+                # SMAFSFXF1RO4 then /SMAFSFXF1RO4 works but /FileSet/SMAFSFXF1RO4 does not; and
+                # conversely using "submitted_id", also an identifying property, with value
+                # UW_FILE-SET_COLO-829BL_HI-C_1 then /UW_FILE-SET_COLO-829BL_HI-C_1 does
+                # not work but /FileSet/UW_FILE-SET_COLO-829BL_HI-C_1 does work.
+                elif isinstance(identifying_value, list):
+                    for identifying_value_item in identifying_value:
+                        if self.type:
+                            identifying_paths.append(f"/{self.type}/{identifying_value_item}")
+                        identifying_paths.append(f"/{identifying_value_item}")
+                else:
+                    # TODO: Import from somewhere ...
+                    lookup_options = 0
+                    if schema := self.schema:
+                        # TODO: Hook into the ref_lookup_strategy thing in structured_data to make
+                        # sure we check accession format (since it does not have a pattern).
+                        if callable(ref_lookup_strategy):
+                            lookup_options, ref_validator = ref_lookup_strategy(
+                                self._portal, self.type, schema, identifying_value)
+                            if callable(ref_validator):
+                                if ref_validator(schema, identifying_property, identifying_value) is False:
+                                    continue
+                        if pattern := schema.get("properties", {}).get(identifying_property, {}).get("pattern"):
+                            if not re.match(pattern, identifying_value):
+                                # If this identifying value is for a (identifying) property which has a
+                                # pattern, and the value does NOT match the pattern, then do NOT include
+                                # this value as an identifying path, since it cannot possibly be found.
+                                continue
+                    if not lookup_options:
+                        lookup_options = Portal.LOOKUP_DEFAULT
+                    if Portal.is_lookup_root_first(lookup_options):
+                        identifying_paths.append(f"/{identifying_value}")
+                    if Portal.is_lookup_specified_type(lookup_options) and self.type:
+                        identifying_paths.append(f"/{self.type}/{identifying_value}")
+                    if Portal.is_lookup_root(lookup_options) and not Portal.is_lookup_root_first(lookup_options):
+                        identifying_paths.append(f"/{identifying_value}")
+                    if Portal.is_lookup_subtypes(lookup_options):
+                        for subtype_name in self._portal.get_schema_subtype_names(self.type):
+                            identifying_paths.append(f"/{subtype_name}/{identifying_value}")
+        return identifying_paths or None
 
     def _normalized_refs(self, refs: List[dict]) -> Tuple[PortalObject, int]:
         """