amapy-utils 1.0.0__tar.gz

amapy-utils-1.0.0/PKG-INFO

@@ -0,0 +1,25 @@
+Metadata-Version: 2.1
+Name: amapy-utils
+Version: 1.0.0
+Summary: A utility package that provides common functionality shared across the asset-manager ecosystem. It serves as a foundation library offering various helper functions and tools needed by other components in the system.
+Author-email: Swarup Mahanti <swarup.mahanti@roche.com>
+Maintainer-email: Swarup Mahanti <swarup.mahanti@roche.com>
+License: Copyright (c) 2024 Roche Diagnostics Computation Science & Informatics
+Requires-Python: <3.11,>=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: aiofiles==0.7.*
+Requires-Dist: cached_property==1.5.*
+Requires-Dist: colorama==0.4.*
+Requires-Dist: crcmod==1.*
+Requires-Dist: psutil==5.9.*
+Requires-Dist: pyopenssl==24.0.*
+Requires-Dist: pypager==3.0.1
+Requires-Dist: pytz==2021.3
+Requires-Dist: pyyaml==6.0.*
+Requires-Dist: requests==2.32.*
+Requires-Dist: ruamel.yaml==0.18.*
+Requires-Dist: speedtest-cli==2.1.*
+Requires-Dist: tabulate==0.9.*
+Requires-Dist: tqdm==4.62.*
+
+asset-utils

amapy-utils-1.0.0/README.md

@@ -0,0 +1 @@
+asset-utils

amapy-utils-1.0.0/amapy_utils.egg-info/PKG-INFO

@@ -0,0 +1,25 @@
+Metadata-Version: 2.1
+Name: amapy-utils
+Version: 1.0.0
+Summary: A utility package that provides common functionality shared across the asset-manager ecosystem. It serves as a foundation library offering various helper functions and tools needed by other components in the system.
+Author-email: Swarup Mahanti <swarup.mahanti@roche.com>
+Maintainer-email: Swarup Mahanti <swarup.mahanti@roche.com>
+License: Copyright (c) 2024 Roche Diagnostics Computation Science & Informatics
+Requires-Python: <3.11,>=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: aiofiles==0.7.*
+Requires-Dist: cached_property==1.5.*
+Requires-Dist: colorama==0.4.*
+Requires-Dist: crcmod==1.*
+Requires-Dist: psutil==5.9.*
+Requires-Dist: pyopenssl==24.0.*
+Requires-Dist: pypager==3.0.1
+Requires-Dist: pytz==2021.3
+Requires-Dist: pyyaml==6.0.*
+Requires-Dist: requests==2.32.*
+Requires-Dist: ruamel.yaml==0.18.*
+Requires-Dist: speedtest-cli==2.1.*
+Requires-Dist: tabulate==0.9.*
+Requires-Dist: tqdm==4.62.*
+
+asset-utils

amapy-utils-1.0.0/amapy_utils.egg-info/SOURCES.txt

@@ -0,0 +1,45 @@
+README.md
+pyproject.toml
+amapy_utils.egg-info/PKG-INFO
+amapy_utils.egg-info/SOURCES.txt
+amapy_utils.egg-info/dependency_links.txt
+amapy_utils.egg-info/entry_points.txt
+amapy_utils.egg-info/requires.txt
+amapy_utils.egg-info/top_level.txt
+asset_utils/__init__.py
+asset_utils/common/__init__.py
+asset_utils/common/better_set.py
+asset_utils/common/class_property.py
+asset_utils/common/exceptions.py
+asset_utils/common/singleton.py
+asset_utils/common/user_commands.py
+asset_utils/common/user_messages.py
+asset_utils/common/tests/__init__.py
+asset_utils/common/tests/test_better_set.py
+asset_utils/common/tests/test_class_property.py
+asset_utils/common/tests/test_singleton.py
+asset_utils/utils/__init__.py
+asset_utils/utils/aws_hash.py
+asset_utils/utils/cloud_utils.py
+asset_utils/utils/file_html_diff.py
+asset_utils/utils/file_tree.py
+asset_utils/utils/file_utils.py
+asset_utils/utils/in_memory_file.py
+asset_utils/utils/in_memory_zip.py
+asset_utils/utils/log_utils.py
+asset_utils/utils/pager.py
+asset_utils/utils/path_utils.py
+asset_utils/utils/progress.py
+asset_utils/utils/stat_utils.py
+asset_utils/utils/utils.py
+asset_utils/utils/web_utils.py
+asset_utils/utils/tests/__init__.py
+asset_utils/utils/tests/test_aws_hash.py
+asset_utils/utils/tests/test_cloud_utils.py
+asset_utils/utils/tests/test_file_tree.py
+asset_utils/utils/tests/test_file_utils.py
+asset_utils/utils/tests/test_in_memory_file.py
+asset_utils/utils/tests/test_log_utils.py
+asset_utils/utils/tests/test_path_utils.py
+asset_utils/utils/tests/test_progress.py
+asset_utils/utils/tests/test_utils.py

amapy-utils-1.0.0/amapy_utils.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

amapy-utils-1.0.0/amapy_utils.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+utils = asset_utils.utils.pager:run

amapy-utils-1.0.0/amapy_utils.egg-info/requires.txt

@@ -0,0 +1,14 @@
+aiofiles==0.7.*
+cached_property==1.5.*
+colorama==0.4.*
+crcmod==1.*
+psutil==5.9.*
+pyopenssl==24.0.*
+pypager==3.0.1
+pytz==2021.3
+pyyaml==6.0.*
+requests==2.32.*
+ruamel.yaml==0.18.*
+speedtest-cli==2.1.*
+tabulate==0.9.*
+tqdm==4.62.*

amapy-utils-1.0.0/amapy_utils.egg-info/top_level.txt

@@ -0,0 +1 @@
+asset_utils

amapy-utils-1.0.0/asset_utils/common/__init__.py

@@ -0,0 +1,7 @@
+from .better_set import BetterSet
+from .class_property import classproperty
+from .singleton import Singleton
+
+DEBUG = True
+PROFILE = True
+PRINT_ARGS = False

amapy-utils-1.0.0/asset_utils/common/better_set.py

@@ -0,0 +1,86 @@
+import copy
+
+
+class BetterSet:
+    """Custom Set for Objects"""
+
+    def __init__(self, *args):
+        # self._dict = OrderedDict()
+        self._dict = {}
+        for arg in args:
+            self.add(arg)
+
+    @property
+    def items(self):
+        """ Return a list containing all items in sorted order, if possible """
+        return list(self._dict.keys())
+
+    def extend(self, args):
+        """ Add several items at once. """
+        for arg in args:
+            self.add(arg)
+
+    def add(self, item):
+        """Add one item to set"""
+        if item in self._dict:
+            del self._dict[item]
+        self._dict[item] = item
+
+    def remove(self, item):
+        """Remove item from set
+        this will raise a KeyError if item doesn't exist
+        """
+        del self._dict[item]
+
+    def discard(self, item):
+        """Removes an item for set
+        Doesn't raise error if item doesn't exist
+        """
+        if item in self._dict:
+            self.remove(item)
+
+    def contains(self, item):
+        """Check if the set contains item"""
+        return item in self._dict
+
+    def get(self, item):
+        """returns the object matching the value of the item"""
+        return self._dict.get(item, None)
+
+    def clear(self):
+        """clears all items in the set"""
+        self._dict = {}
+
+    @property
+    def first(self):
+        return self.items[0]
+
+    @property
+    def last(self):
+        return self.items[len(self.items) - 1]
+
+    def union(self, s2):
+        result = copy.copy(self)
+        for item in s2:
+            result.add(item)
+        return result
+
+    # High-performance membership test
+    __contains__ = contains
+
+    def __repr__(self):
+        return f"{self.__class__.__name__}"
+
+    def __getitem__(self, index):
+        """ Support the 'for item in set:' protocol. """
+        return list(self._dict.keys())[index]
+
+    def __iter__(self):
+        """ convert to list """
+        return iter(self._dict)
+
+    def __len__(self):
+        return len(self._dict)
+
+    def __copy__(self):
+        return BetterSet(*self.items)

amapy-utils-1.0.0/asset_utils/common/class_property.py

@@ -0,0 +1,31 @@
+class ClassPropertyDescriptor(object):
+    """Class Property Descriptor that allows adding property to class objects
+    based on: https://stackoverflow.com/questions/5189699/how-to-make-a-class-property
+    """
+
+    def __init__(self, fget, fset=None):
+        self.fget = fget
+        self.fset = fset
+
+    def __get__(self, obj, klass=None):
+        if klass is None:
+            klass = type(obj)
+        return self.fget.__get__(obj, klass)()
+
+    def __set__(self, obj, value):
+        if not self.fset:
+            raise AttributeError("can't set attribute")
+        type_ = type(obj)
+        return self.fset.__get__(obj, type_)(value)
+
+    def setter(self, func):
+        if not isinstance(func, (classmethod, staticmethod)):
+            func = classmethod(func)
+        self.fset = func
+        return self
+
+
+def classproperty(func):
+    if not isinstance(func, (classmethod, staticmethod)):
+        func = classmethod(func)
+    return ClassPropertyDescriptor(func)

amapy-utils-1.0.0/asset_utils/common/exceptions.py

@@ -0,0 +1,286 @@
+from functools import cached_property
+
+from asset_utils.utils.log_utils import LogData, colored_string, LogColors
+
+
+class AssetException(Exception):
+    """Base class for all asset exceptions."""
+    msg: str = None
+    data: dict = None
+    fatal: bool = True
+
+    def __init__(self, msg=None, data=None, fatal=True, *args):
+        self.msg = msg or self.__class__.msg
+        if not self.msg and args:
+            self.msg = args[0]
+        self.data = data or self.__class__.data
+        self.fatal = fatal
+        if not self.msg:
+            raise Exception("required param missing: msg")
+        super().__init__(msg, data, *args)
+
+    @cached_property
+    def logs(self):
+        return LogData()
+
+    @cached_property
+    def log_colors(self):
+        return LogColors
+
+    def stop_execution(self):
+        self.__class__.stop_process()
+
+    def __str__(self):
+        msg = colored_string(f"error: {self.msg}", LogColors.ALERT)
+        extras = self.logs.print_format()
+        if extras:
+            msg += f"\n{extras}" if msg else extras
+        return msg
+
+    @classmethod
+    def stop_process(cls):
+        exit(1)
+
+
+class AssetObjectConflictException(AssetException):
+    def __init__(self, b1, b2):
+        msg = f'Input type `{b1.unit}` provided by {b1.__name__} already registered by {b2.__name__}'
+        super().__init__(msg=msg)
+
+
+class InvalidStorageURLError(AssetException):
+    msg = "Invalid storage url"
+
+
+class InvalidStorageBackendError(AssetException):
+    msg = "Invalid storage backend, this blob store is not supported"
+
+
+class InvalidStorageCredentialsError(AssetException):
+    msg = "Invalid storage credentials, please check your credentials and try again"
+
+
+class NoActiveProjectError(AssetException):
+    msg = "There is no active project set, please select an active project"
+
+
+class InvalidProjectError(AssetException):
+    msg = "project not found"
+
+
+class InvalidRemoteURLError(AssetException):
+    msg = "remote_url is missing, please login again to retry"
+
+
+class RemoteStorageError(AssetException):
+    msg = "remote storage for asset not configured yet"
+
+
+class InvalidCredentialError(AssetException):
+    msg = "Invalid application credentials, please sign-in to update your credentials"
+
+
+class InsufficientCredentialError(AssetException):
+    msg = "You don't have sufficient permission to perform this operation"
+
+
+class ContentNotAvailableError(AssetException):
+    msg = "Content not available locally"
+
+
+class ForbiddenRefError(AssetException):
+    """thrown in case of forbidden references for example,
+    self refs
+    """
+    pass
+
+
+class InvalidAliasError(AssetException):
+    pass
+
+
+class InvalidTagError(AssetException):
+    pass
+
+
+class AssetStoreCreateError(AssetException):
+    """throw if asset store is invalid"""
+    pass
+
+
+class AssetStoreInvalidError(AssetException):
+    pass
+
+
+class InvalidArgumentError(ValueError, AssetException):
+    """Thrown if arguments are invalid."""
+    pass
+
+
+class InvalidAssetNameError(AssetException):
+    """Thrown if the asset name passed by the user is invalid"""
+    msg = "invalid asset name"
+    pass
+
+
+class AssetNotFoundError(AssetException):
+    msg = "asset not found locally"
+    pass
+
+
+class InvalidObjectSourceError(AssetException):
+    msg = "file not found"
+    pass
+
+
+class UnSupportedOperation(AssetException):
+    pass
+
+
+class StagePathAsOutputError(AssetException):
+    """Thrown if directory that stage is going to be saved in is specified as
+    an output of another stage.
+
+    Args:
+        stage (Stage): a stage that is in some other stages output
+        output (str): an output covering the stage above
+    """
+
+    def __init__(self, stage, output):
+        assert isinstance(output, str)
+        super().__init__(
+            "{stage} is within an output '{output}' of another stage".format(
+                stage=stage, output=output
+            )
+        )
+
+
+class CircularDependencyError(AssetException):
+    """Thrown if a file/directory specified both as an output and as a
+    dependency.
+
+    Args:
+        dependency (str): path to the dependency.
+    """
+
+    def __init__(self, dependency):
+        assert isinstance(dependency, str)
+
+        msg = "'{}' is specified as an output and as a dependency."
+        super().__init__(msg.format(dependency))
+
+
+class ArgumentDuplicationError(AssetException):
+    """Thrown if a file/directory is specified as a dependency/output more
+    than once.
+
+    Args:
+        path (str): path to the file/directory.
+    """
+
+    def __init__(self, path):
+        assert isinstance(path, str)
+        super().__init__(f"file '{path}' is specified more than once.")
+
+
+class MoveNotDataSourceError(AssetException):
+    """Thrown when trying to move a file/directory that is not an output
+    in a data source stage.
+
+    Args:
+        path (str): path to the file/directory.
+    """
+
+    def __init__(self, path):
+        msg = (
+            "move is not permitted for stages that are not data sources. "
+            "You need to either move '{path}' to a new location and edit "
+            "it by hand, or remove '{path}' and create a new one at the "
+            "desired location."
+        )
+        super().__init__(msg.format(path=path))
+
+
+class NotAssetRepoError(AssetException):
+    """Thrown if a directory is not a BaseAsset repo"""
+    pass
+
+
+class DuplicateRepoError(AssetException):
+    """Thrown if two repos with the same id exist """
+    msg = "Multiple repos found with the same repo id"
+
+
+class RepoOverwriteError(AssetException):
+    msg = "Possible Repo override detected - existing repo id found"
+
+
+class RepoMovedError(AssetException):
+    msg = "Repo was moved from its original location"
+
+
+class NestedRepoError(AssetException):
+    msg = "Nested repo detected"
+
+
+class InvalidVersionError(AssetException):
+    pass
+
+
+class InvalidSeqIdError(AssetException):
+    pass
+
+
+class InvalidInputError(AssetException):
+    pass
+
+
+class ServerNotAvailableError(AssetException):
+    pass
+
+
+class ServerUrlNotSetError(AssetException):
+    msg = "asset-server URL not set"
+
+
+class ResourceDownloadError(AssetException):
+    msg = "failed to download resource"
+
+
+class InvalidRefError(AssetException):
+    pass
+
+
+class AssetClassNotFoundError(AssetException):
+    msg = "asset class not found"
+
+
+class ClassListNotFoundError(AssetException):
+    msg = "asset-class list not found"
+
+
+class IncorrectServerResponseError(AssetException):
+    pass
+
+
+class AssetParserError(AssetException):
+    """Base class for CLI parser errors."""
+
+    def __init__(self):
+        super().__init__("parser error")
+
+
+class BackoffRetryError(AssetException):
+    msg = "retry immediately"
+
+
+class GroupRetryError(AssetException):
+    msg = "retry in group"
+
+
+class ReadOnlyAssetError(AssetException):
+    msg = "read-only asset, change tracking and updates are disabled"
+
+
+class AssetSnapshotError(AssetException):
+    msg = "error while handling asset snapshot"

amapy-utils-1.0.0/asset_utils/common/singleton.py

@@ -0,0 +1,35 @@
+import abc
+
+
+class Singleton:
+    __metaclass__ = abc.ABCMeta
+
+    def __new__(cls):
+        raise Exception("singleton, used class.shared() instead")
+
+    @classmethod
+    def shared(cls, **kwargs):
+        if cls is Singleton:
+            raise Exception("abstract class, you must inherit")
+        if not hasattr(cls, 'instance') or not getattr(cls, 'instance'):
+            instance = super(Singleton, cls).__new__(cls)
+            instance.post_init(**kwargs)
+            cls.instance = instance
+        return cls.instance
+
+    def post_init(self, **kwargs):
+        """subclass must implement and do any custom initialization here
+        - we do this here to not let users make the mistake of implementing __init__
+        - __init__ is called everytime the user tries to instantiate
+        the class, even when we override __new__ as obove
+        - this is counterintuitive to how we perceive singletons in other languages
+        """
+        raise NotImplementedError
+
+    @classmethod
+    def de_init(cls):
+        """
+        need a de_init so that we can test different inheritances of the singleton
+        :return:
+        """
+        setattr(cls, 'instance', None)

amapy-utils-1.0.0/asset_utils/common/tests/test_better_set.py

@@ -0,0 +1,63 @@
+import pytest
+
+from asset_utils.common.better_set import BetterSet
+
+
+def test_initialization():
+    # Test initialization with no items
+    empty_set = BetterSet()
+    assert len(empty_set) == 0
+
+    # Test initialization with one item
+    single_item_set = BetterSet("item1")
+    assert "item1" in single_item_set
+
+    # Test initialization with multiple items
+    multi_item_set = BetterSet("item1", "item2")
+    assert len(multi_item_set) == 2
+    assert "item1" in multi_item_set and "item2" in multi_item_set
+
+
+def test_add_extend():
+    test_set = BetterSet()
+    test_set.add("item1")
+    assert "item1" in test_set
+    # Test extend with a duplicate item
+    test_set.extend(["item1", "item2"])
+    assert "item1" in test_set and "item2" in test_set
+    assert len(test_set) == 2
+
+
+def test_remove_discard():
+    test_set = BetterSet("item1", "item2")
+    test_set.remove("item1")
+    assert "item1" not in test_set
+    # Test discard does not raise an error if item does not exist
+    test_set.discard("item3")
+    assert "item3" not in test_set
+    # Test remove raises KeyError if item does not exist
+    with pytest.raises(KeyError):
+        test_set.remove("item3")
+
+
+def test_contains_get():
+    test_set = BetterSet("item1")
+    assert test_set.contains("item1")
+    assert not test_set.contains("item2")
+
+    assert test_set.get("item1") == "item1"
+    assert test_set.get("item2") is None
+
+
+def test_clear():
+    test_set = BetterSet("item1", "item2")
+    test_set.clear()
+    assert len(test_set) == 0
+
+
+def test_union():
+    set1 = BetterSet("item1", "item2")
+    set2 = BetterSet("item2", "item3")
+    union_set = set1.union(set2)
+    assert len(union_set) == 3
+    assert "item1" in union_set and "item2" in union_set and "item3" in union_set

amapy-utils-1.0.0/asset_utils/common/tests/test_class_property.py

@@ -0,0 +1,29 @@
+from asset_utils.common.class_property import classproperty
+
+
+def test_class_property():
+    class Bar(object):
+        _bar = 1
+
+        @classproperty
+        def bar(cls):
+            return cls._bar
+
+        @bar.setter
+        def bar(cls, value):
+            cls._bar = value
+
+    # test instance creation
+    foo = Bar()
+    assert foo.bar == 1
+
+    baz = Bar()
+    assert baz.bar == 1
+
+    # classproperty getter
+    assert Bar.bar == 1
+
+    # classproperty setter
+    Bar.bar = 50
+    assert foo.bar == 50
+    assert baz.bar == 50

amapy-utils-1.0.0/asset_utils/common/tests/test_singleton.py

@@ -0,0 +1,23 @@
+from asset_utils.common.singleton import Singleton
+
+
+class SingletonChild(Singleton):
+    def post_init(self):
+        pass
+
+
+class SingletonChild2(Singleton):
+    def post_init(self):
+        pass
+
+
+def test_singleton():
+    child = SingletonChild.shared()
+    child_copy = SingletonChild.shared()
+    child2 = SingletonChild2.shared()
+    child2_copy = SingletonChild2.shared()
+    assert child == child_copy
+    assert child2 == child2_copy
+    assert child != child2
+    assert child.__class__ is SingletonChild
+    assert child2.__class__ is SingletonChild2