dcicutils 8.10.0.0b0__tar.gz → 8.10.0.1b2__tar.gz

{dcicutils-8.10.0.0b0 → dcicutils-8.10.0.1b2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcicutils
-Version: 8.10.0.0b0
+Version: 8.10.0.1b2
 Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
 Home-page: https://github.com/4dn-dcic/utils
 License: MIT

{dcicutils-8.10.0.0b0 → dcicutils-8.10.0.1b2}/dcicutils/command_utils.py

@@ -1,3 +1,4 @@
+from __future__ import annotations
 import contextlib
 import functools
 import glob
@@ -6,8 +7,9 @@ import os
 import re
 import requests
 import subprocess
+import sys
 
-from typing import Optional
+from typing import Callable, Optional
 from .exceptions import InvalidParameterError
 from .lang_utils import there_are
 from .misc_utils import INPUT, PRINT, environ_bool, print_error_message, decorator
@@ -371,7 +373,7 @@ def script_catch_errors():
         raise ScriptFailure(' '.join(message))
     try:
         yield fail
-        exit(0)
+        sys.exit(0)
     except (Exception, ScriptFailure) as e:
         if DEBUG_SCRIPT:
             # If debugging, let the error propagate, do not trap it.
@@ -383,4 +385,71 @@ def script_catch_errors():
             else:
                 message = str(e)  # Note: We ignore the type, which isn't intended to be shown.
                 PRINT(message)
-            exit(1)
+            sys.exit(1)
+
+
+class Question:
+    """
+    Supports asking the user (via stdin) a yes/no question, possibly repeatedly; and after
+    some maximum number times of the same answer in a row (consecutively), then asks them
+    if they want to automatically give that same answer to any/all subsequent questions.
+    Supports static/global list of such Question instances, hashed (only) by the question text.
+    """
+    _static_instances = {}
+
+    @staticmethod
+    def instance(question: Optional[str] = None,
+                 max: Optional[int] = None, printf: Optional[Callable] = None) -> Question:
+        question = question if isinstance(question, str) else ""
+        if not (instance := Question._static_instances.get(question)):
+            Question._static_instances[question] = (instance := Question(question, max=max, printf=printf))
+        return instance
+
+    @staticmethod
+    def yes(question: Optional[str] = None,
+            max: Optional[int] = None, printf: Optional[Callable] = None) -> bool:
+        return Question.instance(question, max=max, printf=printf).ask()
+
+    def __init__(self, question: Optional[str] = None,
+                 max: Optional[int] = None, printf: Optional[Callable] = None) -> None:
+        self._question = question if isinstance(question, str) else ""
+        self._max = max if isinstance(max, int) and max > 0 else None
+        self._print = printf if callable(printf) else print
+        self._yes_consecutive_count = 0
+        self._no_consecutive_count = 0
+        self._yes_automatic = False
+        self._no_automatic = False
+
+    def ask(self, question: Optional[str] = None) -> bool:
+
+        def question_automatic(value: str) -> bool:
+            nonlocal self
+            RARROW = "▶"
+            LARROW = "◀"
+            if yes_or_no(f"{RARROW}{RARROW}{RARROW}"
+                         f" Do you want to answer {value} to all such questions?"
+                         f" {LARROW}{LARROW}{LARROW}"):
+                return True
+            self._yes_consecutive_count = 0
+            self._no_consecutive_count = 0
+
+        if self._yes_automatic:
+            return True
+        elif self._no_automatic:
+            return False
+        elif yes_or_no((question if isinstance(question, str) else "") or self._question or "Undefined question"):
+            self._yes_consecutive_count += 1
+            self._no_consecutive_count = 0
+            if (self._no_consecutive_count == 0) and self._max and (self._yes_consecutive_count >= self._max):
+                # Have reached the maximum number of consecutive YES answers; ask if YES to all subsequent.
+                if question_automatic("YES"):
+                    self._yes_automatic = True
+            return True
+        else:
+            self._no_consecutive_count += 1
+            self._yes_consecutive_count = 0
+            if (self._yes_consecutive_count == 0) and self._max and (self._no_consecutive_count >= self._max):
+                # Have reached the maximum number of consecutive NO answers; ask if NO to all subsequent.
+                if question_automatic("NO"):
+                    self._no_automatic = True
+            return False

{dcicutils-8.10.0.0b0 → dcicutils-8.10.0.1b2}/dcicutils/deployment_utils.py

@@ -396,9 +396,9 @@ class EBDeployer:
             packaging_was_successful = cls.build_application_version(args.repo, args.version_name, branch=args.branch)
             if packaging_was_successful:  # XXX: how to best detect?
                 time.sleep(5)  # give EB a second to catch up (it needs it)
-                exit(cls.deploy_new_version(args.env, args.repo, args.version_name))
+                sys.exit(cls.deploy_new_version(args.env, args.repo, args.version_name))
         else:
-            exit(cls.deploy_indexer(args.env, args.application_version))
+            sys.exit(cls.deploy_indexer(args.env, args.application_version))
 
 
 class IniFileManager:

{dcicutils-8.10.0.0b0 → dcicutils-8.10.0.1b2}/dcicutils/ecr_scripts.py

@@ -3,6 +3,7 @@ import botocore.exceptions
 import boto3
 import contextlib
 import os
+import sys
 
 from typing import Optional, Union, List
 from .command_utils import yes_or_no
@@ -68,7 +69,7 @@ def ecr_command_context(account_number, ecs_repository=None, ecr_client=None):
             elif account_number != account_number_in_environ:
                 raise RuntimeError("The account number you have specified does not match your declared credentials.")
             yield ECRCommandContext(account_number=account_number, ecs_repository=ecs_repository, ecr_client=ecr_client)
-            exit(0)
+            sys.exit(0)
         except botocore.exceptions.ClientError as e:
             error_info = e.response.get('Error', {})
             message = error_info.get('Message')
@@ -77,12 +78,12 @@ def ecr_command_context(account_number, ecs_repository=None, ecr_client=None):
                 raise RuntimeError("Your security token seems to have expired.")
             elif message:
                 PRINT(f"{code}: {message}")
-                exit(1)
+                sys.exit(1)
             else:
                 raise
     except Exception as e:
         PRINT(f"{full_class_name(e)}: {e}")
-        exit(1)
+        sys.exit(1)
 
 
 class ECRCommandContext:

{dcicutils-8.10.0.0b0 → dcicutils-8.10.0.1b2}/dcicutils/env_scripts.py

@@ -1,6 +1,7 @@
 import argparse
 import boto3
 import json
+import sys
 import yaml
 
 from .lang_utils import disjoined_list
@@ -52,7 +53,7 @@ def show_global_env_bucket(bucket, mode='json', key=None):
         else:
             PRINT(f"There is no default bucket. Please use a '--bucket' argument"
                   f" to specify one of {disjoined_list(envs_buckets)}.")
-        exit(1)
+        sys.exit(1)
 
     print_heading(bucket, style='=')
 
@@ -70,7 +71,7 @@ def show_global_env_bucket(bucket, mode='json', key=None):
         except Exception as e:
             PRINT("Bucket contents could not be downlaoded.")
             print_error_message(e)
-            exit(1)
+            sys.exit(1)
 
         object = None  # Without this, PyCharm fusses that object might not get set. -kmp 20-Jul-2022
         try:
@@ -78,7 +79,7 @@ def show_global_env_bucket(bucket, mode='json', key=None):
         except Exception as e:
             PRINT("Bucket contents could not be parsed as JSON.")
             print_error_message(e)
-            exit(1)
+            sys.exit(1)
 
         if mode == 'json':
             PRINT(json.dumps(object, indent=2, default=str))
@@ -86,7 +87,7 @@ def show_global_env_bucket(bucket, mode='json', key=None):
             PRINT(yaml.dump(object))
         else:
             PRINT(f"Unknown mode: {mode}. Try 'json' or 'yaml'.")
-            exit(1)
+            sys.exit(1)
 
 
 DEFAULT_BUCKET = get_env_bucket()

{dcicutils-8.10.0.0b0 → dcicutils-8.10.0.1b2}/dcicutils/license_policies/park-lab-common.jsonc

@@ -248,6 +248,12 @@
             "docutils"  // Used only privately as a separate documentation-generation task for ReadTheDocs
         ],
 
+
+        "GNU General Public License v2 (GPLv2)": [
+            "pyinstaller",
+            "pyinstaller-hooks-contrib"
+        ],
+
         "MIT/X11 Derivative": [
             // The license used by libxkbcommon is complicated and involves numerous included licenses,
             // but all are permissive.

{dcicutils-8.10.0.0b0 → dcicutils-8.10.0.1b2}/dcicutils/misc_utils.py

@@ -4,6 +4,7 @@ This file contains functions that might be generally useful.
 
 from collections import namedtuple
 import appdirs
+from copy import deepcopy
 import contextlib
 import datetime
 import functools
@@ -2199,28 +2200,58 @@ def merge_key_value_dict_lists(x, y):
     return [key_value_dict(k, v) for k, v in merged.items()]
 
 
-def merge_objects(target: Union[dict, List[Any]], source: Union[dict, List[Any]], full: bool = False) -> dict:
+def merge_objects(target: Union[dict, List[Any]], source: Union[dict, List[Any]],
+                  full: bool = False,  # deprecated
+                  expand_lists: Optional[bool] = None,
+                  primitive_lists: bool = False,
+                  copy: bool = False, _recursing: bool = False) -> Union[dict, List[Any]]:
     """
-    Merges the given source dictionary or list into the target dictionary or list.
-    This MAY well change the given target (dictionary or list) IN PLACE.
-    The the full argument is True then any target lists longer than the
-    source be will be filled out with the last element(s) of the source.
+    Merges the given source dictionary or list into the target dictionary or list and returns the
+    result. This MAY well change the given target (dictionary or list) IN PLACE ... UNLESS the copy
+    argument is True, then the given target will not change as a local copy is made (and returned).
+
+    If the expand_lists argument is True then any target lists longer than the
+    source be will be filled out with the last element(s) of the source; the full
+    argument (is deprecated and) is a synomym for this. The default is False.
+
+    If the primitive_lists argument is True then lists of primitives (i.e. lists in which
+    NONE of its elements are dictionaries, lists, or tuples) will themselves be treated
+    like primitives, meaning the whole of a source list will replace the corresponding
+    target; otherwise they  will be merged normally, meaning each element of a source list
+    will be merged, recursively, into the corresponding target list. The default is False.
     """
+    def is_primitive_list(value: Any) -> bool:  # noqa
+        if not isinstance(value, list):
+            return False
+        for item in value:
+            if isinstance(item, (dict, list, tuple)):
+                return False
+        return True
+
     if target is None:
         return source
+    if expand_lists not in (True, False):
+        expand_lists = full is True
+    if (copy is True) and (_recursing is not True):
+        target = deepcopy(target)
     if isinstance(target, dict) and isinstance(source, dict) and source:
         for key, value in source.items():
-            target[key] = merge_objects(target[key], value, full) if key in target else value
+            if ((primitive_lists is True) and
+                (key in target) and is_primitive_list(target[key]) and is_primitive_list(value)):  # noqa
+                target[key] = value
+            else:
+                target[key] = merge_objects(target[key], value,
+                                            expand_lists=expand_lists, _recursing=True) if key in target else value
     elif isinstance(target, list) and isinstance(source, list) and source:
         for i in range(max(len(source), len(target))):
             if i < len(target):
                 if i < len(source):
-                    target[i] = merge_objects(target[i], source[i], full)
-                elif full:
-                    target[i] = merge_objects(target[i], source[len(source) - 1], full)
+                    target[i] = merge_objects(target[i], source[i], expand_lists=expand_lists, _recursing=True)
+                elif expand_lists is True:
+                    target[i] = merge_objects(target[i], source[len(source) - 1], expand_lists=expand_lists)
             else:
                 target.append(source[i])
-    elif source:
+    elif source not in (None, {}, []):
         target = source
     return target
 

{dcicutils-8.10.0.0b0 → dcicutils-8.10.0.1b2}/dcicutils/portal_object_utils.py

@@ -1,6 +1,5 @@
 from copy import deepcopy
 from functools import lru_cache
-import re
 from typing import Any, Callable, List, Optional, Tuple, Type, Union
 from dcicutils.data_readers import RowReader
 from dcicutils.misc_utils import create_readonly_object
@@ -14,11 +13,9 @@ class PortalObject:
 
     _PROPERTY_DELETION_SENTINEL = RowReader.CELL_DELETION_SENTINEL
 
-    def __init__(self, data: dict, portal: Portal = None,
-                 schema: Optional[Union[dict, Schema]] = None, type: Optional[str] = None) -> None:
+    def __init__(self, data: dict, portal: Optional[Portal] = None, type: Optional[str] = None) -> None:
         self._data = data if isinstance(data, dict) else {}
         self._portal = portal if isinstance(portal, Portal) else None
-        self._schema = schema if isinstance(schema, dict) else (schema.data if isinstance(schema, Schema) else None)
         self._type = type if isinstance(type, str) else ""
 
     @property
@@ -32,7 +29,7 @@ class PortalObject:
     @property
     @lru_cache(maxsize=1)
     def type(self) -> str:
-        return self._type or Portal.get_schema_type(self._data) or (Schema(self._schema).type if self._schema else "")
+        return self._type or Portal.get_schema_type(self._data) or ""
 
     @property
     @lru_cache(maxsize=1)
@@ -47,7 +44,7 @@ class PortalObject:
     @property
     @lru_cache(maxsize=1)
     def schema(self) -> Optional[dict]:
-        return self._schema if self._schema else (self._portal.get_schema(self.type) if self._portal else None)
+        return self._portal.get_schema(self.type) if self._portal else None
 
     def copy(self) -> PortalObject:
         return PortalObject(deepcopy(self.data), portal=self.portal, type=self.type)
@@ -59,39 +56,29 @@ class PortalObject:
         Returns the list of all identifying property names of this Portal object which actually have values.
         Implicitly include "uuid" and "identifier" properties as identifying properties if they are actually
         properties in the object schema, and favor these (first); defavor "aliases"; no other ordering defined.
+        Changed (2024-05-26) to use portal_utils.get_identifying_property_names; migrating some intricate stuff there.
         """
-        if not (schema := self.schema) or not (schema_identifying_properties := schema.get("identifyingProperties")):
-            return None
-        identifying_properties = []
-        for identifying_property in schema_identifying_properties:
-            if identifying_property not in ["uuid", "identifier", "aliases"]:
-                if self._data.get(identifying_property):
-                    identifying_properties.append(identifying_property)
-        if self._data.get("identifier"):
-            identifying_properties.insert(0, "identifier")
-        if self._data.get("uuid"):
-            identifying_properties.insert(0, "uuid")
-        if "aliases" in schema_identifying_properties and self._data.get("aliases"):
-            identifying_properties.append("aliases")
-        return identifying_properties or None
+        # Migrating to and unifying this in portal_utils.Portal.get_identifying_paths (2024-05-26).
+        return self._portal.get_identifying_property_names(self.type, portal_object=self._data) if self._portal else []
 
     @lru_cache(maxsize=8192)
     def lookup(self, raw: bool = False,
                ref_lookup_strategy: Optional[Callable] = None) -> Tuple[Optional[PortalObject], Optional[str], int]:
+        if not (identifying_paths := self._get_identifying_paths(ref_lookup_strategy=ref_lookup_strategy)):
+            return None, None, 0
         nlookups = 0
         first_identifying_path = None
         try:
-            if identifying_paths := self._get_identifying_paths(ref_lookup_strategy=ref_lookup_strategy):
-                for identifying_path in identifying_paths:
-                    if not first_identifying_path:
-                        first_identifying_path = identifying_path
-                    nlookups += 1
-                    if (value := self._portal.get(identifying_path, raw=raw)) and (value.status_code == 200):
-                        return (
-                            PortalObject(value.json(), portal=self._portal, type=self.type if raw else None),
-                            identifying_path,
-                            nlookups
-                        )
+            for identifying_path in identifying_paths:
+                if not first_identifying_path:
+                    first_identifying_path = identifying_path
+                nlookups += 1
+                if self._portal and (item := self._portal.get(identifying_path, raw=raw)) and (item.status_code == 200):
+                    return (
+                        PortalObject(item.json(), portal=self._portal, type=self.type if raw else None),
+                        identifying_path,
+                        nlookups
+                    )
         except Exception:
             pass
         return None, first_identifying_path, nlookups
@@ -159,64 +146,12 @@ class PortalObject:
 
     @lru_cache(maxsize=1)
     def _get_identifying_paths(self, ref_lookup_strategy: Optional[Callable] = None) -> Optional[List[str]]:
-        """
-        Returns a list of the possible Portal URL paths identifying this Portal object.
-        """
-        identifying_paths = []
-        if not (identifying_properties := self.identifying_properties):
-            if self.uuid:
-                if self.type:
-                    identifying_paths.append(f"/{self.type}/{self.uuid}")
-                identifying_paths.append(f"/{self.uuid}")
-            return identifying_paths
-        for identifying_property in identifying_properties:
-            if identifying_value := self._data.get(identifying_property):
-                if identifying_property == "uuid":
-                    if self.type:
-                        identifying_paths.append(f"/{self.type}/{identifying_value}")
-                    identifying_paths.append(f"/{identifying_value}")
-                # For now at least we include the path both with and without the schema type component,
-                # as for some identifying values, it works (only) with, and some, it works (only) without.
-                # For example: If we have FileSet with "accession", an identifying property, with value
-                # SMAFSFXF1RO4 then /SMAFSFXF1RO4 works but /FileSet/SMAFSFXF1RO4 does not; and
-                # conversely using "submitted_id", also an identifying property, with value
-                # UW_FILE-SET_COLO-829BL_HI-C_1 then /UW_FILE-SET_COLO-829BL_HI-C_1 does
-                # not work but /FileSet/UW_FILE-SET_COLO-829BL_HI-C_1 does work.
-                elif isinstance(identifying_value, list):
-                    for identifying_value_item in identifying_value:
-                        if self.type:
-                            identifying_paths.append(f"/{self.type}/{identifying_value_item}")
-                        identifying_paths.append(f"/{identifying_value_item}")
-                else:
-                    # TODO: Import from somewhere ...
-                    lookup_options = 0
-                    if schema := self.schema:
-                        # TODO: Hook into the ref_lookup_strategy thing in structured_data to make
-                        # sure we check accession format (since it does not have a pattern).
-                        if callable(ref_lookup_strategy):
-                            lookup_options, ref_validator = ref_lookup_strategy(
-                                self._portal, self.type, schema, identifying_value)
-                            if callable(ref_validator):
-                                if ref_validator(schema, identifying_property, identifying_value) is False:
-                                    continue
-                        if pattern := schema.get("properties", {}).get(identifying_property, {}).get("pattern"):
-                            if not re.match(pattern, identifying_value):
-                                # If this identifying value is for a (identifying) property which has a
-                                # pattern, and the value does NOT match the pattern, then do NOT include
-                                # this value as an identifying path, since it cannot possibly be found.
-                                continue
-                    if not lookup_options:
-                        lookup_options = Portal.LOOKUP_DEFAULT
-                    if Portal.is_lookup_root_first(lookup_options):
-                        identifying_paths.append(f"/{identifying_value}")
-                    if Portal.is_lookup_specified_type(lookup_options) and self.type:
-                        identifying_paths.append(f"/{self.type}/{identifying_value}")
-                    if Portal.is_lookup_root(lookup_options) and not Portal.is_lookup_root_first(lookup_options):
-                        identifying_paths.append(f"/{identifying_value}")
-                    if Portal.is_lookup_subtypes(lookup_options):
-                        for subtype_name in self._portal.get_schema_subtype_names(self.type):
-                            identifying_paths.append(f"/{subtype_name}/{identifying_value}")
-        return identifying_paths or None
+        if not self._portal and (uuid := self.uuid):
+            return [f"/{uuid}"]
+        # Migrating to and unifying this in portal_utils.Portal.get_identifying_paths (2024-05-26).
+        return self._portal.get_identifying_paths(self._data,
+                                                  portal_type=self.schema,
+                                                  lookup_strategy=ref_lookup_strategy) if self._portal else None
 
     def _normalized_refs(self, refs: List[dict]) -> Tuple[PortalObject, int]:
         """