dcicutils 7.4.1__tar.gz → 7.4.2.1b0__tar.gz

{dcicutils-7.4.1 → dcicutils-7.4.2.1b0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcicutils
-Version: 7.4.1
+Version: 7.4.2.1b0
 Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
 Home-page: https://github.com/4dn-dcic/utils
 License: MIT

{dcicutils-7.4.1 → dcicutils-7.4.2.1b0}/dcicutils/common.py

@@ -94,7 +94,6 @@ AVAILABLE_S3_STORAGE_CLASSES = [
 #
 # See boto3 docs for info on possible values, but these 3 are the current ones used for
 # glacier (that require restore calls) - Will 7 Apr 2023
-
 S3_GLACIER_CLASSES = [
     'GLACIER_IR',  # Glacier Instant Retrieval
     'GLACIER',  # Glacier Flexible Retrieval
@@ -118,6 +117,10 @@ S3StorageClass = Union[
 ]
 
 
+# This constant is used in our Lifecycle management system to automatically transition objects
+ENCODED_LIFECYCLE_TAG_KEY = 'Lifecycle'
+
+
 # These numbers come from AWS and is the max size that can be copied with a single request
 # Any larger than this requires a multipart upload - Will 24 April 2023
 MAX_STANDARD_COPY_SIZE = 5368709120

{dcicutils-7.4.1 → dcicutils-7.4.2.1b0}/dcicutils/glacier_utils.py

@@ -2,7 +2,10 @@ import boto3
 from typing import Union, List, Tuple
 from concurrent.futures import ThreadPoolExecutor
 from tqdm import tqdm
-from .common import S3_GLACIER_CLASSES, S3StorageClass, MAX_MULTIPART_CHUNKS, MAX_STANDARD_COPY_SIZE
+from .common import (
+    S3_GLACIER_CLASSES, S3StorageClass, MAX_MULTIPART_CHUNKS, MAX_STANDARD_COPY_SIZE,
+    ENCODED_LIFECYCLE_TAG_KEY
+)
 from .command_utils import require_confirmation
 from .misc_utils import PRINT
 from .ff_utils import get_metadata, search_metadata, get_health_page, patch_metadata
@@ -255,8 +258,19 @@ class GlacierUtils:
             PRINT(f'Error deleting Glacier versions of object {bucket}/{key}: {str(e)}')
             return False
 
+    @staticmethod
+    def _format_tags(tags: List[dict]) -> str:
+        """ Helper method that formats tags so that they match the format expected by the boto3 API
+
+        :param tags: array of dictionaries containing Key, Value mappings to be reformatted
+        :return: String formatted tag list ie:
+            [{Key: key1, Value: value1}, Key: key2, Value: value2}] --> 'key1=value1&key2=value2'
+        """
+        return '&'.join([f'{tag["Key"]}={tag["Value"]}' for tag in tags])
+
     def _do_multipart_upload(self, bucket: str, key: str, total_size: int, part_size: int = 200,
-                             storage_class: str = 'STANDARD', version_id: Union[str, None] = None) -> Union[dict, None]:
+                             storage_class: str = 'STANDARD', tags: str = '',
+                             version_id: Union[str, None] = None) -> Union[dict, None]:
         """ Helper function for copy_object_back_to_original_location, not intended to
             be called directly, will arrange for a multipart copy of large updates
             to change storage class
@@ -266,6 +280,7 @@ class GlacierUtils:
         :param total_size: total size of object
         :param part_size: what size to divide the object into when uploading the chunks
         :param storage_class: new storage class to use
+        :param tags: string of tags to apply
         :param version_id: object version ID, if applicable
         :return: response, if successful, or else None
         """
@@ -275,7 +290,12 @@ class GlacierUtils:
             if num_parts > MAX_MULTIPART_CHUNKS:
                 raise GlacierRestoreException(f'Must user a part_size larger than {part_size}'
                                               f' that will result in fewer than {MAX_MULTIPART_CHUNKS} chunks')
-            mpu = self.s3.create_multipart_upload(Bucket=bucket, Key=key, StorageClass=storage_class)
+            cmu = {
+                'Bucket': bucket, 'Key': key, 'StorageClass': storage_class
+            }
+            if tags:
+                cmu['Tagging'] = tags
+            mpu = self.s3.create_multipart_upload(**cmu)
             mpu_upload_id = mpu['UploadId']
         except Exception as e:
             PRINT(f'Error creating multipart upload for {bucket}/{key} : {str(e)}')
@@ -327,6 +347,7 @@ class GlacierUtils:
 
     def copy_object_back_to_original_location(self, bucket: str, key: str, storage_class: str = 'STANDARD',
                                               part_size: int = 200,  # MB
+                                              preserve_lifecycle_tag: bool = False,
                                               version_id: Union[str, None] = None) -> Union[dict, None]:
         """ Reads the temporary location from the restored object and copies it back to the original location
 
@@ -334,6 +355,7 @@ class GlacierUtils:
         :param key: key within bucket where object is stored
         :param storage_class: new storage class for this object
         :param part_size: if doing a large copy, size of chunks to upload (in MB)
+        :param preserve_lifecycle_tag: whether to keep existing lifecycle tag on the object
         :param version_id: version of object, if applicable
         :return: boolean whether the copy was successful
         """
@@ -342,12 +364,20 @@ class GlacierUtils:
             response = self.s3.head_object(Bucket=bucket, Key=key)
             size = response['ContentLength']
             multipart = (size >= MAX_STANDARD_COPY_SIZE)
+            if not preserve_lifecycle_tag:  # default: preserve tags except 'Lifecycle'
+                tags = self.s3.get_object_tagging(Bucket=bucket, Key=key).get('TagSet', [])
+                tags = [tag for tag in tags if tag['Key'] != ENCODED_LIFECYCLE_TAG_KEY]
+                tags = self._format_tags(tags)
+                if not tags:
+                    self.s3.delete_object_tagging(Bucket=bucket, Key=key)
+            else:
+                tags = ''
         except Exception as e:
             PRINT(f'Could not retrieve metadata on file {bucket}/{key} : {str(e)}')
             return None
         try:
             if multipart:
-                return self._do_multipart_upload(bucket, key, size, part_size, storage_class, version_id)
+                return self._do_multipart_upload(bucket, key, size, part_size, storage_class, tags, version_id)
             else:
                 # Force copy the object into standard in a single operation
                 copy_source = {'Bucket': bucket, 'Key': key}
@@ -358,6 +388,8 @@ class GlacierUtils:
                 if version_id:
                     copy_source['VersionId'] = version_id
                     copy_target['CopySourceVersionId'] = version_id
+                if tags:
+                    copy_target['Tagging'] = tags
                 response = self.s3.copy_object(CopySource=copy_source, **copy_target)
                 PRINT(f'Response from boto3 copy:\n{response}')
                 PRINT(f'Object {bucket}/{key} copied back to its original location in S3')

dcicutils-7.4.2.1b0/dcicutils/project_utils.py

@@ -0,0 +1,257 @@
+import os
+import toml
+
+from pkg_resources import resource_filename
+from typing import Optional
+from .env_utils import EnvUtils
+from .misc_utils import classproperty
+
+
+def project_filename(filename):
+    # TODO: In fact we should do this based on the working dir so that when this is imported to another repo,
+    #       it gets the inserts out of that repo's tests, not our own.
+    return resource_filename(Project.PACKAGE_NAME, filename)
+
+
+class ProjectRegistry:
+
+    SHOW_HERALD_WHEN_INITIALIZED = True
+
+    REGISTERED_PROJECTS = {}
+
+    # All of these might never be other than None so be careful when accessing them.
+    APPLICATION_PROJECT_HOME = None
+    PYPROJECT_TOML_FILE = None
+    PYPROJECT_TOML = None
+    POETRY_DATA = None
+    # This is expected to ultimately be set properly.
+    _PYPROJECT_NAME = None
+
+    @classproperty
+    def PYPROJECT_NAME(cls) -> str:  # noQA - PyCharm thinks this should be 'self'
+        if cls._PYPROJECT_NAME is None:
+            cls.initialize_pyproject_name()
+        result: Optional[str] = cls._PYPROJECT_NAME
+        if result is None:
+            raise ValueError(f"ProjectRegistry.PROJECT_NAME not initialized properly.")
+        return result
+
+    @classmethod
+    def initialize_pyproject_name(cls, project_home=None, pyproject_toml_file=None):
+        if cls._PYPROJECT_NAME is None:
+            # This isn't the home of snovault, but the home of the snovault-based application.
+            # So in CGAP, for example, this would want to be the home of the CGAP application.
+            # If not set, it will be assumed that the current working directory is that.
+            if not project_home:
+                project_home = os.environ.get("APPLICATION_PROJECT_HOME", os.path.abspath(os.curdir))
+            cls.APPLICATION_PROJECT_HOME = project_home
+            if not pyproject_toml_file:
+                expected_pyproject_toml_file = os.path.join(project_home, "pyproject.toml")
+                pyproject_toml_file = (expected_pyproject_toml_file
+                                       if os.path.exists(expected_pyproject_toml_file)
+                                       else None)
+            cls.PYPROJECT_TOML_FILE = pyproject_toml_file
+            cls.PYPROJECT_TOML = pyproject_toml = (toml.load(cls.PYPROJECT_TOML_FILE)
+                                                   if cls.PYPROJECT_TOML_FILE
+                                                   else None)
+            cls.POETRY_DATA = (pyproject_toml['tool']['poetry']
+                               if pyproject_toml
+                               else None)
+
+            declared_pyproject_name = os.environ.get("APPLICATION_PYPROJECT_NAME")
+            inferred_pyproject_name = cls.POETRY_DATA['name'] if cls.POETRY_DATA else None
+            if (declared_pyproject_name and inferred_pyproject_name
+                    and declared_pyproject_name != inferred_pyproject_name):
+                raise RuntimeError(f"APPLICATION_PYPROJECT_NAME={declared_pyproject_name!r},"
+                                   f" but {pyproject_toml_file} says it should be {inferred_pyproject_name!r}")
+
+            cls._PYPROJECT_NAME = declared_pyproject_name or inferred_pyproject_name
+
+    @classmethod
+    def register(cls, name):
+        """
+        Registers a class to be used based on the name in the top of pyproject.toml.
+        Note that this means that cgap-portal and fourfront will both register as 'encoded',
+        as in:
+
+            @Project.register('encoded')
+            class FourfrontProject(EncodedCoreProject):
+                PRETTY_NAME = "Fourfront"
+
+        Since fourfront and cgap-portal don't occupy the same space, no confusion should result.
+        """
+        def _wrap_class(the_class):
+            the_class_name = the_class.__name__
+            if not issubclass(the_class, Project):
+                raise ValueError(f"The class {the_class_name} must inherit from Project.")
+            lower_registry_name = name.lower()
+            for x in ['cgap-portal', 'fourfront', 'smaht']:
+                if x in lower_registry_name:
+                    # It's an easy error to make, but the name of the project from which we're gaining foothold
+                    # in pyproject.toml is 'encoded', not 'cgap-portal', etc., so the name 'encoded' will be
+                    # needed for bootstrapping. So it should look like
+                    # -kmp 15-May-2023
+                    raise ValueError(f"Please use ProjectRegistry.register('encoded'),"
+                                     f" not ProjectRegistry.register({name!r})."
+                                     f" This registration is just for bootstrapping."
+                                     f" The class can still be {the_class_name}.")
+            cls.REGISTERED_PROJECTS[name] = the_class
+            return the_class
+        return _wrap_class
+
+    @classmethod
+    def _lookup(cls, name):
+        """
+        Returns the project object with the given name.
+
+        :param name: a string name that was used in a ProjectRegistry.register decorator
+
+        NOTE: There is no need for this function to be called outside of this class except for testing.
+              Really only one of these should be instantiated per running application, and that's
+              done automatically by this class.
+        """
+        project_class = cls.REGISTERED_PROJECTS.get(name)
+        return project_class
+
+    @classmethod
+    def _make_project(cls):
+        """
+        Creates and returns an instantiated project object for the current project.
+
+        The project to use can be specified by setting the environment variable APPLICATION_PROJECT_HOME
+        to a particular directory that contains the pyproject.toml file to use.
+        If no such variable is set, the current working directory is used.
+
+        NOTE: There is no need for this function to be called outside of this class except for testing.
+              Really only one of these should be instantiated per running application, and that's
+              done automatically by this class.
+        """
+        project_class = cls._lookup(cls.PYPROJECT_NAME)
+        assert issubclass(project_class, Project)
+        project: Project = project_class()
+        return project  # instantiate and return
+
+    _app_project = None
+    _initialized = False
+
+    @classmethod
+    def initialize(cls):
+        if cls._initialized:
+            raise RuntimeError(f"{cls.__name__}.initialize() was called more than once.")
+        cls._app_project = cls._make_project()
+        cls._initalized = True
+        if cls.SHOW_HERALD_WHEN_INITIALIZED:
+            cls.show_herald()
+        app_project: Project = cls.app_project
+        return app_project  # It's initialized now, so we use the proper interface
+
+    @classmethod
+    def show_herald(cls):
+        app_project = cls.app_project_maker()
+
+        print("=" * 80)
+        print(f"APPLICATION_PROJECT_HOME == {cls.APPLICATION_PROJECT_HOME!r}")
+        print(f"PYPROJECT_TOML_FILE == {cls.PYPROJECT_TOML_FILE!r}")
+        print(f"PYPROJECT_NAME == {cls.PYPROJECT_NAME!r}")
+        the_app_project = Project.app_project
+        the_app_project_class = the_app_project.__class__
+        the_app_project_class_name = the_app_project_class.__name__
+        assert (Project.app_project
+                == app_project()
+                == the_app_project_class.app_project
+                == the_app_project.app_project), (
+            "Project consistency check failed."
+        )
+        print(f"{the_app_project_class_name}.app_project == Project.app_project == app_project() == {app_project()!r}")
+        print(f"app_project().NAME == {app_project().NAME!r}")
+        print(f"app_project().PRETTY_NAME == {app_project().PRETTY_NAME!r}")
+        print(f"app_project().PACKAGE_NAME == {app_project().PACKAGE_NAME!r}")
+        print(f"app_project().APP_NAME == {app_project().APP_NAME!r}")
+        print(f"app_project().APP_PRETTY_NAME == {app_project().APP_PRETTY_NAME!r}")
+        print("=" * 80)
+
+    @classproperty
+    def app_project(cls):  # noQA - PyCharm thinks we should use 'self'
+        """
+        Once the project is initialized, ProjectRegistry.app_project returns the application object
+        that should be used to dispatch project-dependent behavior.
+        """
+        if cls._app_project is None:
+            # You need to put a call to
+            raise RuntimeError(f"Attempt to access {cls.__name__}.project before .initialize() called.")
+        return cls._app_project
+
+    @classmethod
+    def app_project_maker(cls):
+
+        def app_project(initialize=False, initialization_options: Optional[dict] = None):
+            if initialize:
+                Project.initialize_app_project(**(initialization_options or {}))
+            return ProjectRegistry.app_project
+
+        return app_project
+
+
+class Project:
+    """
+    A class that should be a superclass of all classes registered using ProjectRegistry.register
+
+    All such classes have these names:
+      .NAME - The name of the project in pyproject.toml
+      .PACKAGE_NAME - The pypi name of the project, useful for pkg_resources, for example.
+      .PRETTY_NAME - The pretty name of the package name
+      .APP_NAME - The ame of the project application (see dcicutils.common and the orchestrated app in EnvUtils)
+      .APP_PRETTY_NAME - The pretty name of the project application.
+
+    Some sample usess of pre-defined attributes of a Project that may help motivate the choice of attribute names:
+
+      registered  |
+         name     |     NAME     |  PACKAGE_NAME  |  PRETTY NAME |  APP_NAME | APP_PRETTY_NAME
+     -------------+--------------+----------------+--------------+-----------+----------------
+     snovault     | dcicsnovault |  snovault      | Snovault     | snovault  | Snovault
+     encoded-core | encoded-core |  encoded-core  | Encoded Core | core      | Core
+     encoded      | cgap-portal  |  cgap-portal   | CGAP Portal  | cgap      | CGAP
+     encoded      | fourfront    |  fourfront     | Fourfront    | fourfront | Fourfront
+     encoded      | smaht-portal |  smaht-portal  | SMaHT Portal | smaht     | SMaHT
+
+     The registered name is the one used with the ProjectRegistry.register() decorator.
+    """
+
+    NAME = 'project'
+
+    @classmethod
+    def _prettify(cls, name):
+        return name.title().replace("Cgap", "CGAP").replace("Smaht", "SMaHT").replace("-", " ")
+
+    @classproperty
+    def PACKAGE_NAME(cls):  # noQA - PyCharm wants the variable name to be self
+        return cls.NAME.replace('dcic', '')
+
+    @classproperty
+    def PRETTY_NAME(cls):  # noQA - PyCharm wants the variable name to be self
+        return cls._prettify(cls.PACKAGE_NAME)
+
+    @classproperty
+    def APP_NAME(cls):  # noQA - PyCharm wants the variable name to be self
+        return cls.PACKAGE_NAME.replace('-portal', '').replace('encoded-', '')
+
+    @classproperty
+    def APP_PRETTY_NAME(cls):  # noQA - PyCharm wants the variable name to be self
+        return cls._prettify(cls.APP_NAME)
+
+    @classproperty
+    def app_project(cls):  # noQA - PyCharm wants the variable name to be self
+        """
+        Project.app_project returns the actual instantiated project for app-specific behavior,
+        which might be of this class or one of its subclasses.
+
+        This access will fail if the project has not been initialized.
+        """
+        return ProjectRegistry.app_project
+
+    @classmethod
+    def initialize_app_project(cls, initialize_env_utils=True):
+        if initialize_env_utils:
+            EnvUtils.init()
+        project: Project = ProjectRegistry.initialize()
+        return project

{dcicutils-7.4.1 → dcicutils-7.4.2.1b0}/dcicutils/s3_utils.py

@@ -462,6 +462,8 @@ class s3Utils(s3Base):  # NOQA - This class name violates style rules, but a lot
         content_type = mimetypes.guess_type(upload_key)[0]
         if content_type is None:
             content_type = 'binary/octet-stream'
+        if isinstance(obj, dict):
+            obj = json.dumps(obj)
         if acl:
             # we use this to set some of the object as public
             return self.s3.put_object(Bucket=self.outfile_bucket,
@@ -480,6 +482,8 @@ class s3Utils(s3Base):  # NOQA - This class name violates style rules, but a lot
             bucket = self.sys_bucket
         if not secret:
             secret = os.environ["S3_ENCRYPT_KEY"]
+        if isinstance(data, dict):
+            data = json.dumps(data)
         return self.s3.put_object(Bucket=bucket,
                                   Key=keyname,
                                   Body=data,

{dcicutils-7.4.1 → dcicutils-7.4.2.1b0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dcicutils"
-version = "7.4.1"
+version = "7.4.2.1b0"  # to become 7.5.0
 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources"
 authors = ["4DN-DCIC Team <support@4dnucleome.org>"]
 license = "MIT"

{dcicutils-7.4.1 → dcicutils-7.4.2.1b0}/setup.py

@@ -38,7 +38,7 @@ entry_points = \
 
 setup_kwargs = {
     'name': 'dcicutils',
-    'version': '7.4.1',
+    'version': '7.4.2.1b0',
     'description': 'Utility package for interacting with the 4DN Data Portal and other 4DN resources',
     'long_description': '=====\nutils\n=====\n\nCheck out our full documentation `here <https://dcic-utils.readthedocs.io/en/latest/>`_\n\nThis repository contains various utility modules shared amongst several projects in the 4DN-DCIC. It is meant to be used internally by the DCIC team and externally as a Python API to `Fourfront <https://data.4dnucleome.org>`_\\ , the 4DN data portal.\n\npip installable as the ``dcicutils`` package with: ``pip install dcicutils``\n\nSee `this document <https://dcic-utils.readthedocs.io/en/latest/getting_started.html>`_ for tips on getting started. `Go here <https://dcic-utils.readthedocs.io/en/latest/examples.html>`_ for examples of some of the most useful functions.\n\n\n.. image:: https://travis-ci.org/4dn-dcic/utils.svg?branch=master\n   :target: https://travis-ci.org/4dn-dcic/utils\n   :alt: Build Status\n\n\n.. image:: https://coveralls.io/repos/github/4dn-dcic/utils/badge.svg?branch=master\n   :target: https://coveralls.io/github/4dn-dcic/utils?branch=master\n   :alt: Coverage\n\n.. image:: https://readthedocs.org/projects/dcic-utils/badge/?version=latest\n   :target: https://dcic-utils.readthedocs.io/en/latest/?badge=latest\n   :alt: Documentation Status\n',
     'author': '4DN-DCIC Team',