idmtools 0.0.0.dev0__py3-none-any.whl → 0.0.3__py3-none-any.whl

idmtools/utils/gitrepo.py

@@ -0,0 +1,336 @@
+"""
+Utilities for getting information and examples from gitrepos.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+import os
+import sys
+import json
+from logging import getLogger
+import requests
+import urllib.request
+from click import secho
+from dataclasses import dataclass, field
+logger = getLogger(__name__)
+user_logger = getLogger('user')
+
+REPO_OWNER = 'institutefordiseasemodeling'
+REPO_NAME = 'idmtools'
+GITHUB_HOME = 'https://github.com'
+GITHUB_API_HOME = 'https://api.github.com'
+
+
+@dataclass
+class GitRepo:
+    """
+    GitRepo allows interaction with remote git repos, mainly for examples.
+    """
+    repo_owner: str = field(default=None)
+    repo_name: str = field(default=None)
+    _branch: str = field(default='main', init=False, repr=False)
+    _path: str = field(default='', init=False, repr=False)
+    _verbose: bool = field(default=False, init=False, repr=False)
+
+    def __post_init__(self):
+        """
+        Initialize GitRepo.
+
+        If repo_owner or repo_name is None, the defaults REPO_OWNER and REPO_NAME
+
+        Returns:
+            None
+        """
+        self.repo_owner = self.repo_owner or REPO_OWNER
+        self.repo_name = self.repo_name or REPO_NAME
+
+    @property
+    def path(self):
+        """
+        Path property.
+
+        Returns:
+            Return path property
+        """
+        return self._path
+
+    @property
+    def branch(self):
+        """
+        Branch property.
+
+        Returns:
+            Return branch property
+        """
+        return self._branch
+
+    @property
+    def verbose(self):
+        """
+        Return verbose property.
+
+        Returns:
+            Return verbose property
+        """
+        return self._verbose
+
+    @property
+    def repo_home_url(self):
+        """
+        Construct repo home url.
+
+        Returns: repo home url
+        """
+        return f'{GITHUB_HOME}/{self.repo_owner}/{self.repo_name}'
+
+    @property
+    def repo_example_url(self):
+        """
+        Construct repo example url.
+
+        Returns: repo example url
+        """
+        return f'{self.repo_home_url}/tree/{self._branch}/{self._path}'
+
+    @property
+    def api_example_url(self):
+        """
+        Construct api url of the examples for download.
+
+        Returns: api url
+        """
+        return f'{GITHUB_API_HOME}/repos/{self.repo_owner}/{self.repo_name}/contents/{self._path}?ref={self._branch}'
+
+    def parse_url(self, url: str, branch: str = None, update: bool = True):
+        """
+        Parse url for owner, repo, branch and example path.
+
+        Args:
+            url: example url
+            branch: user branch to replace the branch in url
+            update: True/False - update repo or not
+
+        Returns: None
+        """
+        default_branch = 'main'
+        ex_text = 'Please Verify URL Format: \nhttps://github.com/<owner>/<repo>/(tree|blob)/<branch>/<path>\nor\nhttps://github.com/<owner>/<repo>/'
+
+        example_url = url.lower().strip().rstrip('/')
+        url_chunks = example_url.replace(f'{GITHUB_HOME}/', '').split('/')
+
+        if len(url_chunks) < 2 or (len(url_chunks) >= 3 and url_chunks[2] not in ['tree', 'blob']):
+            raise Exception(f'Your Example URL: {url}\n{ex_text}')
+
+        repo_owner = url_chunks[0]
+        repo_name = url_chunks[1]
+
+        if len(url_chunks) <= 3:
+            _branch = branch if branch else default_branch
+            _path = ''
+        else:
+            _branch = branch if branch else url_chunks[3] if url_chunks[3] else default_branch
+            _path = '/'.join(url_chunks[4:])
+
+        if update:
+            self.repo_owner = repo_owner
+            self.repo_name = repo_name
+            self._branch = _branch
+            self._path = _path
+        else:
+            return {'repo_owner': repo_owner, 'repo_name': repo_name, 'branch': _branch, 'path': _path}
+
+    def list_public_repos(self, repo_owner: str = None, page: int = 1, raw: bool = False):
+        """
+        Utility method to retrieve all public repos.
+
+        Args:
+            repo_owner: the owner of the repo
+            page: pagination of results
+            raw: bool - return rwo data or simplified list
+
+        Returns: repo list
+        """
+        # build api url
+        api_url = f'{GITHUB_API_HOME}/users/{repo_owner if repo_owner else self.repo_owner}/repos'
+
+        if page:
+            api_url = f'{api_url}?page={page}'
+
+        resp = requests.get(api_url)
+        if resp.status_code != 200:
+            raise Exception(f'Failed to access: {api_url}')
+
+        # get repos as json
+        repo_list = resp.json()
+
+        if raw:
+            return repo_list
+        else:
+            return [r['full_name'] for r in repo_list]
+
+    def list_repo_releases(self, repo_owner: str = None, repo_name: str = None, raw: bool = False):
+        """
+        Utility method to retrieve all releases of the repo.
+
+        Args:
+            repo_owner: the owner of the repo
+            repo_name: the name of repo
+            raw: bool - return raw data or simplified list
+
+        Returns: the release list of the repo
+        """
+        # build api url
+        api_url = f'{GITHUB_API_HOME}/repos/{repo_owner if repo_owner else self.repo_owner}/{repo_name if repo_name else self.repo_name}/releases'
+
+        # make api call
+        resp = requests.get(api_url)
+        if resp.status_code != 200:
+            raise Exception(f'Failed to access: {api_url}')
+
+        # get repos as json
+        repo_list = resp.json()
+
+        if raw:
+            return repo_list
+        else:
+            return [f"{r['tag_name']} at {r['published_at']}" for r in repo_list]
+
+    def download(self, path: str = '', output_dir: str = "./", branch: str = 'main') -> int:
+        """
+        Download files with example url provided.
+
+        Args:
+            path: local file path to the repo
+            output_dir: user local folder to download files to
+            branch: specify branch for files download from
+
+        Returns: total file count downloaded
+        """
+        if path.startswith('https://'):
+            self.parse_url(path)
+        else:
+            self._path = path
+            self._branch = branch
+
+        if not os.path.exists(output_dir):
+            raise Exception(f"output_dir does not exist: {output_dir}")
+
+        # First time display download url and local destination info
+        if self.verbose:
+            user_logger.info(f'Download Examples From: {self.repo_example_url}')
+            user_logger.info(f'Local Destination: {os.path.abspath(output_dir)}')
+            user_logger.info('Processing...')
+            self._verbose = False
+
+        try:
+            opener = urllib.request.build_opener()
+            opener.addheaders = [('User-agent', 'Mozilla/5.0')]
+            urllib.request.install_opener(opener)
+            response = urllib.request.urlretrieve(self.api_example_url)
+        except KeyboardInterrupt:
+            # when CTRL+C is pressed during the execution of this script,
+            # bring the cursor to the beginning, erase the current line, and dont make a new line
+            user_logger.error("✘ Got interrupted")
+            sys.exit()
+        except Exception as ex:
+            secho(f'Failed to access: {self.api_example_url}', fg="yellow")
+            logger.exception(ex)
+            exit(1)
+
+        download_dir = os.path.join(output_dir, self.repo_name)
+
+        # total files count
+        total_files = 0
+        with open(response[0], "r") as f:
+            data = json.load(f)
+
+        if isinstance(data, dict) and data["type"] == "file":
+            # create folder when necessary
+            path = data["path"]
+            os.makedirs(os.path.dirname(os.path.join(download_dir, path)), exist_ok=True)
+
+            try:
+                # download the file
+                opener = urllib.request.build_opener()
+                opener.addheaders = [('User-agent', 'Mozilla/5.0')]
+                urllib.request.install_opener(opener)
+                urllib.request.urlretrieve(data["download_url"], os.path.join(download_dir, path))
+                return 1
+            except KeyboardInterrupt:
+                # when CTRL+C is pressed during the execution of this script,
+                # bring the cursor to the beginning, erase the current line, and dont make a new line
+                user_logger.error("✘ Got interrupted", )
+                sys.exit()
+            except Exception as ex:
+                secho(f'Failed to access: {self.api_example_url}', fg="yellow")
+                user_logger.error(ex)
+                exit(1)
+
+        total_files += len([f for f in data if f['type'] == 'file'])
+
+        for file in data:
+            file_url = file["download_url"]
+            path = file["path"]
+
+            # create folder when necessary
+            os.makedirs(os.path.dirname(os.path.join(download_dir, path)), exist_ok=True)
+
+            if file_url is not None:
+                try:
+                    # download the file
+                    opener = urllib.request.build_opener()
+                    opener.addheaders = [('User-agent', 'Mozilla/5.0')]
+                    urllib.request.install_opener(opener)
+                    urllib.request.urlretrieve(file_url, os.path.join(download_dir, path))
+                except KeyboardInterrupt:
+                    # when CTRL+C is pressed during the execution of this script,
+                    # bring the cursor to the beginning, erase the current line, and dont make a new line
+                    user_logger.error("✘ Got interrupted", )
+                    sys.exit()
+            else:
+                total_files += self.download(path, output_dir, branch)
+
+        return total_files
+
+    def peep(self, path: str = '', branch: str = 'main'):
+        """
+        Download files with example url provided.
+
+        Args:
+            path: local file path to the repo
+            branch: specify branch for files download from
+
+        Returns: None
+        """
+        if path.startswith('https://'):
+            repo_meta = self.parse_url(path, branch, False)
+        else:
+            self._path = path
+            self._branch = branch
+            repo_meta = {'repo_owner': self.repo_owner, 'repo_name': self.repo_name, 'branch': branch or self.branch,
+                         'path': path or self.path}
+
+        try:
+            api_example_url = f"{GITHUB_API_HOME}/repos/{repo_meta['repo_owner']}/{repo_meta['repo_name']}/contents/{repo_meta['path']}?ref={repo_meta['branch']}"
+            opener = urllib.request.build_opener()
+            opener.addheaders = [('User-agent', 'Mozilla/5.0')]
+            urllib.request.install_opener(opener)
+            response = urllib.request.urlretrieve(api_example_url)
+        except KeyboardInterrupt:
+            # when CTRL+C is pressed during the execution of this script,
+            # bring the cursor to the beginning, erase the current line, and dont make a new line
+            user_logger.error("✘ Got interrupted")
+            sys.exit()
+
+        result = []
+        with open(response[0], "r") as f:
+            data = json.load(f)
+
+        if isinstance(data, dict):
+            d = {'type': data['type'], 'name': data['path'], 'path': data['path'], 'html_url': data['html_url']}
+            result.append(d)
+        else:
+            for file in data:
+                d = {'type': file['type'], 'name': file['name'], 'path': file['path'], 'html_url': file['html_url']}
+                result.append(d)
+
+        return result

idmtools/utils/hashing.py

@@ -0,0 +1,239 @@
+"""
+Fast hash of Python objects.
+
+Copyright 2025, Gates Foundation. All rights reserved.
+"""
+from typing import Union, BinaryIO
+
+import decimal
+import hashlib
+import io
+import pickle
+import types
+from dataclasses import fields, MISSING
+from logging import getLogger, Logger
+
+logger = getLogger(__name__)
+Pickler = pickle._Pickler
+
+
+class _ConsistentSet(object):
+    """
+    Class used to ensure the hash of sets is preserved whatever the order of its items.
+    """
+
+    def __init__(self, set_sequence):
+        """
+        Force the order of elements in a set to ensure consistent hashing.
+        """
+        try:
+            # Trying first to order the set using sorted
+            self._sequence = sorted(set_sequence)
+        except (TypeError, decimal.InvalidOperation):
+            # If elements are unorderable, sort them using their hash.
+            self._sequence = sorted((hash(e) for e in set_sequence))
+
+
+class _MyHash(object):
+    """
+    A class used to hash objects that won't normally pickle.
+    """
+
+    def __init__(self, *args):
+        self.args = args
+
+
+class Hasher(Pickler):
+    """
+    A subclass of pickler to do hashing, rather than pickling.
+    """
+
+    def __init__(self, hash_name='md5'):
+        """
+        Initialize our hasher.
+
+        Args:
+            hash_name: Hash type to use. Defaults to md5
+        """
+        self.stream = io.BytesIO()
+        Pickler.__init__(self, self.stream)
+        # Initialise the hash obj
+        self._hash = hashlib.new(hash_name)
+
+    def hash(self, obj, return_digest=True):
+        """
+        Hash an object.
+
+        Args:
+            obj: Object to hash
+            return_digest: Should the digest be returned?
+
+        Returns:
+            None if return_digest is False, otherwise the hash digest is returned
+        """
+        try:
+            self.dump(obj)
+        except pickle.PicklingError as e:
+            e.args += ('PicklingError while hashing %r: %r' % (obj, e),)
+            raise
+        dumps = self.stream.getvalue()
+        self._hash.update(dumps)
+        if return_digest:
+            return self._hash.hexdigest()
+
+    def save(self, obj):
+        """
+        Save an object to hash.
+
+        Args:
+            obj: Obj to save.
+
+        Returns:
+            None
+        """
+        from idmtools.utils.collections import ExperimentParentIterator
+        import abc
+        if isinstance(obj, abc.ABCMeta):
+            pass
+        elif isinstance(obj, ExperimentParentIterator):
+            pass
+        elif isinstance(obj, Logger):
+            pass
+        else:
+            if isinstance(obj, (types.MethodType, type({}.pop))):
+                # the Pickler cannot pickle instance methods; here we decompose
+                # them into components that make them uniquely identifiable
+                if hasattr(obj, '__func__'):
+                    func_name = obj.__func__.__name__
+                else:
+                    func_name = obj.__name__
+                inst = obj.__self__
+                if isinstance(inst, pickle):
+                    obj = _MyHash(func_name, inst.__name__)
+                elif inst is None:
+                    # type(None) or type(module) do not pickle
+                    obj = _MyHash(func_name, inst)
+                else:
+                    cls = obj.__self__.__class__
+                    obj = _MyHash(func_name, inst, cls)
+            Pickler.save(self, obj)
+
+    def memoize(self, obj):
+        """
+        Disable memoization for strings so hashing happens on value and not reference.
+        """
+        if isinstance(obj, (str, bytes)):
+            return
+        Pickler.memoize(self, obj)
+
+    def _batch_setitems(self, items):
+        """
+        Force the order of keys in dictionary to ensure consistent hashing.
+        """
+        try:
+            # First try quick way of sorting keys if possible
+            Pickler._batch_setitems(self, iter(sorted(items)))
+        except TypeError:
+            # If keys are unorderable, sort them using their hash
+            Pickler._batch_setitems(self, iter(sorted((hash(k), v) for k, v in items)))
+
+    def save_set(self, set_items):
+        """
+        Save set hashing.
+
+        Args:
+            set_items: Set items
+
+        Returns:
+            None
+        """
+        # forces order of items in Set to ensure consistent hash
+        Pickler.save(self, _ConsistentSet(set_items))
+
+
+def hash_obj(obj, hash_name='md5'):
+    """
+    Quick calculation of a hash to identify uniquely Python objects.
+
+    Args:
+        obj: Object to hash
+        hash_name: The hashing algorithm to use. 'md5' is faster; 'sha1' is considered safer.
+    """
+    hasher = Hasher(hash_name=hash_name)
+    return hasher.hash(obj)
+
+
+def ignore_fields_in_dataclass_on_pickle(item):
+    """
+    Ignore certain fields for pickling on dataclasses.
+
+    Args:
+        item: Item to pickle
+
+    Returns:
+        State of item to pickle
+    """
+    state = item.__dict__.copy()
+    attrs = set(vars(item).keys())
+
+    # Retrieve fields default values
+    fds = fields(item)
+    field_default = {f.name: f.default for f in fds}
+
+    # Update default with parent's pre-populated values
+    if hasattr(item, 'pre_getstate'):
+        pre_state = item.pre_getstate()
+        pre_state = pre_state or {}
+        field_default.update(pre_state)
+
+    # Don't pickle ignore_pickle fields: set values to default
+    for field_name in attrs.intersection(item.pickle_ignore_fields):
+        if field_name in state:
+            if field_default[field_name] is MISSING:
+                state[field_name] = None
+            else:
+                state[field_name] = field_default[field_name]
+
+    return state
+
+
+def calculate_md5(filename: str, chunk_size: int = 8192) -> str:
+    """
+    Calculate MD5.
+
+    Args:
+        filename: Filename to caclulate md5 for
+        chunk_size: Chunk size
+
+    Returns:
+        md5 as string
+    """
+    with open(filename, "rb") as f:
+        return calculate_md5_stream(f, chunk_size)
+
+
+def calculate_md5_stream(stream: Union[io.BytesIO, BinaryIO], chunk_size: int = 8192, hash_type: str = 'md5', file_hash=None):
+    """
+    Calculate md5 on stream.
+
+    Args:
+        chunk_size:
+        stream:
+        hash_type: Hash function
+        file_hash: File hash
+
+    Returns:
+        md5 of stream
+    """
+    if file_hash is None:
+        if not hasattr(hashlib, hash_type):
+            raise ValueError(f"Could not find hash function {hash_type}")
+        else:
+            file_hash = getattr(hashlib, hash_type)()
+
+    while True:
+        chunk = stream.read(chunk_size)
+        if not chunk:
+            break
+        file_hash.update(chunk)
+    return file_hash.hexdigest()

idmtools/utils/info.py

@@ -0,0 +1,124 @@
+"""
+Utilities to fetch info about local system such as packages installed.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+import sys
+from logging import getLogger
+from typing import List, Optional
+
+logger = getLogger(__name__)
+
+
+def get_doc_base_url() -> str:
+    """
+    Get base url for documentation links.
+
+    Returns:
+        Doc base url
+    """
+    return "https://institutefordiseasemodeling.github.io/idmtools/"
+
+
+def get_pip_packages_10_to_6():
+    """
+    Load packages for versions 1.0 to 6 of pip.
+
+    Returns:
+        None
+
+    Raises:
+        ImportError: If the pip version is different.
+    """
+    from pip.util import get_installed_distributions
+    return get_installed_distributions()
+
+
+def get_pip_packages_6_to_9():
+    """
+    Get packages for pip versions 6 through 9.
+
+    Returns:
+        None
+
+    Raises:
+        ImportError: If the pip version is different.
+    """
+    from pip.utils import get_installed_distributions
+    return get_installed_distributions()
+
+
+def get_pip_packages_10_to_current():
+    """
+    Get packages for pip versions 10 to current.
+
+    Returns:
+        None
+
+    Raises:
+        ImportError: If the pip version is different.
+    """
+    from pip._internal.utils.misc import get_installed_distributions
+    return get_installed_distributions()
+
+
+def get_packages_from_pip():
+    """
+    Attempt to load packages from pip.
+
+    Returns:
+        (List[str]): A list of packages installed.
+    """
+    try:
+        from importlib.metadata import distributions
+    except ImportError:
+        from importlib_metadata import distributions  # for python 3.7
+    return [f'{d.metadata["Name"]} {d.version}' for d in distributions()]
+
+
+def get_packages_list() -> List[str]:
+    """
+    Return a list of installed packages in the current environment.
+
+    Currently |IT_s| depends on pip for this functionality and since it is just used for troubleshooting, errors can be ignored.
+
+    Returns:
+        (List[str]): A list of packages installed.
+    """
+    packages = get_packages_from_pip()
+    if packages is None:  # fall back to sys modules
+        packages = []
+        # for name, module in sys.modules:
+        modules = list(sys.modules.items())
+        for name, mod in modules:
+            version = ''
+            if hasattr(mod, 'version'):
+                version = mod.version
+            elif hasattr(mod, '__version__'):
+                version = mod.__version__
+            packages.append(f'{name}=={version}')
+        packages = list(sorted(packages))
+    return packages
+
+
+def get_help_version_url(help_path, url_template: str = 'https://docs.idmod.org/projects/idmtools/en/{version}/', version: Optional[str] = None) -> str:
+    """
+    Get the help url for a subject based on a version.
+
+    Args:
+        help_path: Path to config(minus base url). For example, configuration.html
+        url_template: Template for URL containing version replacement formatter
+        version: Optional version. If not provided, the version of idmtools installed will be used. For development versions, the version will always be nightly
+
+    Returns:
+        Path to url
+    """
+    from idmtools import __version__
+    from urllib import parse
+    if version is None:
+        if "nightly" in __version__:
+            version = "latest"
+        else:
+            version = f'v{__version__[0:5]}'
+
+    return parse.urljoin(url_template.format(version=version), help_path)