idmtools-platform-comps 0.0.0.dev0__py3-none-any.whl → 0.0.2__py3-none-any.whl

idmtools_platform_comps/utils/package_version.py

@@ -0,0 +1,489 @@
+"""idmtools Tools to filter versions of packages for requriements for asset collections.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+import functools
+import operator
+import json
+import os
+import re
+from abc import ABC
+from datetime import datetime
+from logging import getLogger, DEBUG
+from typing import Optional, List, Type
+from urllib import request
+import requests
+from packaging.requirements import Requirement
+from packaging.version import Version, parse
+from html.parser import HTMLParser
+
+user_logger = getLogger('user')
+
+
+PKG_PYPI = 'https://pypi.python.org/pypi/{}/json'
+PYPI_PRODUCTION_SIMPLE = 'https://packages.idmod.org/artifactory/api/pypi/pypi-production/simple'
+
+IDM_DOCKER_PROD = 'https://packages.idmod.org/artifactory/list/docker-production'
+IDMTOOLS_DOCKER_PROD = f'{IDM_DOCKER_PROD}/idmtools/'
+MANIFEST_URL = "https://hub.docker.com/v2/repositories/library/{repository}/tags/?page_size=25&page=1&name={tag}"
+
+logger = getLogger(__name__)
+
+
+class PackageHTMLParser(HTMLParser, ABC):
+    """Base Parser for our other parsers."""
+    previous_tag = None
+    pkg_version = None
+
+    def __init__(self):
+        """Constructor."""
+        super().__init__()
+        self.pkg_version = set()
+
+
+class LinkHTMLParser(PackageHTMLParser):
+    """Parse hrefs from links."""
+
+    def handle_starttag(self, tag, attrs):
+        """Parse links and extra hrefs."""
+        self.previous_tag = tag
+        if tag != 'a':
+            return
+
+        attr = dict(attrs)
+        v = attr['href']
+        v = v.rstrip('/')
+        self.pkg_version.add(v)
+
+
+class LinkNameParser(PackageHTMLParser):
+    """
+    Provides parsing of packages from pypi/arfifactory.
+
+    We parse links that match versions patterns
+    """
+    in_link = False
+    ver_pattern = re.compile(r'^[\d\.brcdev\+nightly]+$')
+
+    def handle_starttag(self, tag, attrs):
+        """Handle begin of links."""
+        self.previous_tag = tag
+        self.in_link = tag == "a"
+
+    def handle_endtag(self, tag):
+        """End link tags."""
+        if tag == "a":
+            self.in_link = False
+
+    def handle_data(self, data):
+        """Process links."""
+        if self.in_link:
+            parts = data.split("-")
+            if len(parts) >= 2:
+                if self.ver_pattern.match(parts[1]):
+                    self.pkg_version.add(parts[1])
+                elif parts[1].endswith(".zip"):
+                    self.pkg_version.add(parts[1][:-4])
+                elif parts[1].endswith(".tar.gz"):
+                    self.pkg_version.add(parts[1][:-7])
+
+
+def get_latest_package_version_from_pypi(pkg_name, display_all=False):
+    """
+    Utility to get the latest version for a given package name.
+
+    Args:
+        pkg_name: package name given
+        display_all: determine if output all package releases
+
+    Returns: the latest version of ven package
+    """
+    url = f'https://pypi.python.org/pypi/{pkg_name}/json'
+    try:
+        releases = json.loads(request.urlopen(url).read())['releases']
+    except Exception:
+        return None
+
+    all_releases = sorted(releases, key=parse, reverse=True)
+
+    if display_all:
+        print(all_releases)
+
+    release_versions = [ver for ver in all_releases if not parse(ver).is_prerelease]
+    latest_version = release_versions[0]
+
+    return latest_version
+
+
+def get_latest_pypi_package_version_from_artifactory(pkg_name, display_all=False, base_version: str = None):
+    """
+    Utility to get the latest version for a given package name.
+
+    Args:
+        pkg_name: package name given
+        display_all: determine if output all package releases
+        base_version: Base version
+
+    Returns: the latest version of ven package
+    """
+    pkg_url = "/".join([PYPI_PRODUCTION_SIMPLE, pkg_name])
+    return get_latest_version_from_site(pkg_url, display_all=display_all, base_version=base_version)
+
+
+def get_pypi_package_versions_from_artifactory(pkg_name, display_all=False, base_version: str = None,
+                                               exclude_pre_release: bool = True):
+    """
+    Utility to get versions of a package in artifactory.
+
+    Args:
+        pkg_name: package name given
+        display_all: determine if output all package releases
+        base_version: Base version
+        exclude_pre_release: Exclude any prerelease versions
+
+    Returns: the latest version of ven package
+    """
+    pkg_url = "/".join([PYPI_PRODUCTION_SIMPLE, pkg_name])
+    return get_versions_from_site(pkg_url, base_version, display_all=display_all, parser=LinkNameParser,
+                                  exclude_pre_release=exclude_pre_release)
+
+
+def get_latest_ssmt_image_version_from_artifactory(pkg_name='comps_ssmt_worker', base_version: Optional[str] = None,
+                                                   display_all=False):
+    """
+    Utility to get the latest version for a given package name.
+
+    Args:
+        pkg_name: package name given
+        base_version: Optional base version. Versions above this will not be added.
+        display_all: determine if output all package releases
+
+    Returns: the latest version of ven package
+    """
+    pkg_path = IDMTOOLS_DOCKER_PROD
+    pkg_url = "/".join([pkg_path, pkg_name])
+    base_version = ".".join(base_version.replace("+nightly", "").split(".")[:2])
+    return get_latest_version_from_site(pkg_url, base_version=base_version, display_all=display_all,
+                                        parser=LinkHTMLParser)
+
+
+def get_docker_manifest(image_path="idmtools/comps_ssmt_worker", repo_base=IDM_DOCKER_PROD):
+    """
+    Get docker manifest from IDM Artifactory. It mimics latest even when user has no latest tag defined.
+
+    Args:
+        image_path:Path of docker image we want
+        repo_base:Base of the repo
+
+    Returns:
+        None
+
+    Raises:
+        ValueError - When the manifest cannot be found
+    """
+    if ":" not in image_path:
+        image_path += ":latest"
+
+    path, tag = image_path.split(":")
+    if tag == "latest":
+        url = "/".join([IDM_DOCKER_PROD, path])
+        response = requests.get(url)
+        content = response.text
+        lines = [link.split(">") for link in content.split("\n") if "<a href" in link and "pre" not in link]
+        lines = {item_date[1].replace("/</a", ''): datetime.strptime(item_date[2].strip(" -"), '%d-%b-%Y %H:%M') for
+                 item_date in lines}
+        tag = list(sorted(lines.items(), key=operator.itemgetter(1), reverse=True))[0][0]
+        image_path = ":".join([path, tag])
+    final_path = "/".join([path, tag, "manifest.json"])
+    pkg_path = f'{repo_base}/{final_path}'
+    response = requests.get(pkg_path)
+    if response.status_code != 200:
+        raise ValueError("Could not find manifest for file")
+    return response.json(), image_path
+
+
+def get_digest_from_docker_hub(repo, tag='latest'):
+    """
+    Get the digest for image from docker.
+
+    Args:
+        repo: string, repository (e.g. 'library/fedora')
+        tag:  string, tag of the repository (e.g. 'latest')
+    """
+    response = requests.get(
+        MANIFEST_URL.format(repository=repo, tag=tag)
+    )
+    manifest = response.json()
+    if response.ok and manifest['count']:
+        images = list(filter(lambda x: x['architecture'] == "amd64", manifest['results'][0]['images']))
+        if len(images):
+            return images[0]['digest']
+
+    return None
+
+
+@functools.lru_cache(8)
+def fetch_versions_from_server(pkg_url: str, parser: Type[PackageHTMLParser] = LinkHTMLParser) -> List[str]:
+    """
+    Fetch all versions from server.
+
+    Args:
+        pkg_url: Url to fetch
+        parser: Parser tp use
+
+    Returns:
+        All the releases for a package
+    """
+    resp = requests.get(pkg_url)
+    pkg_name = pkg_url.split('/')[-1]
+    if resp.status_code != 200:
+        logger.warning(f"Error fetching package {pkg_name} information. Status code: {resp.status_code}")
+        return []
+
+    html_str = resp.text
+
+    parser = parser()
+    parser.feed(html_str)
+    releases = parser.pkg_version
+    releases = [v for v in releases if not v.startswith('.')]
+
+    all_releases = sorted(releases, key=parse, reverse=True)
+    return all_releases
+
+
+def fetch_versions_from_artifactory(pkg_name: str, parser: Type[PackageHTMLParser] = LinkHTMLParser) -> List[str]:
+    """
+    Fetch all versions from server.
+
+    Args:
+        pkg_name: Url to fetch
+        parser: Parser tp use
+
+    Returns:
+        Available releases
+    """
+    pkg_path = IDM_DOCKER_PROD
+    pkg_url = os.path.join(pkg_path, pkg_name)
+
+    resp = requests.get(pkg_url)
+    if resp.status_code != 200:
+        logger.warning('Could not fetch URL')
+        return None
+
+    html_str = resp.text
+
+    parser = parser()
+    parser.feed(html_str)
+    releases = parser.pkg_version
+    releases = [v for v in releases if not v.startswith('.')]
+
+    all_releases = sorted(releases, key=parse, reverse=True)
+    return all_releases
+
+
+@functools.lru_cache(3)
+def get_versions_from_site(pkg_url, base_version: Optional[str] = None, display_all=False,
+                           parser: Type[PackageHTMLParser] = LinkNameParser, exclude_pre_release: bool = True):
+    """
+    Utility to get the the available versions for a package.
+
+    The default properties filter out pre releases. You can also include a base version to only list items starting with a particular version
+
+    Args:
+        pkg_url: package name given
+        base_version: Optional base version. Versions above this will not be added. For example, to get versions 1.18.5, 1.18.4, 1.18.3, 1.18.2 pass 1.18
+        display_all: determine if output all package releases
+        parser: Parser needs to be a HTMLParser that returns a pkg_versions
+        exclude_pre_release: Exclude prerelease versions
+
+    Returns: the latest version of ven package
+
+    Raises:
+        ValueError - If a latest versions cannot be determined
+    """
+    all_releases = fetch_versions_from_server(pkg_url, parser=parser)
+    if all_releases is None:
+        raise ValueError(
+            f"Could not determine latest version for package {pkg_url}. You can manually specify a version to avoid this error")
+
+    if display_all:
+        print(all_releases)
+    if exclude_pre_release:
+        ver_pattern = re.compile(r'^[\d\.]+$')
+        release_versions = [ver for ver in all_releases if ver_pattern.match(ver)]
+    else:
+        release_versions = all_releases
+
+    if base_version:
+        release_versions = [ver for ver in release_versions if ver.startswith(base_version)]
+
+    # comps_ssmt_worker will store only x.x.x.x
+    if 'comps_ssmt_worker' in pkg_url.lower():
+        release_versions = [ver for ver in release_versions if len(ver.split('.')) == 4]
+    return release_versions
+
+
+@functools.lru_cache(3)
+def get_latest_version_from_site(pkg_url, base_version: Optional[str] = None, display_all=False,
+                                 parser: Type[PackageHTMLParser] = LinkNameParser, exclude_pre_release: bool = True):
+    """
+    Utility to get the latest version for a given package name.
+
+    Args:
+        pkg_url: package name given
+        base_version: Optional base version. Versions above this will not be added.
+        display_all: determine if output all package releases
+        parser: Parser needs to be a HTMLParser that returns a pkg_versions
+        exclude_pre_release: Exclude pre-release versions
+
+    Returns: the latest version of ven package
+    """
+    if logger.isEnabledFor(DEBUG):
+        logger.debug(f"Fetching version from {pkg_url} with base {base_version}")
+    release_versions = get_versions_from_site(pkg_url, base_version, display_all=display_all, parser=parser,
+                                              exclude_pre_release=exclude_pre_release)
+    if base_version:
+        # only use the longest match latest
+        version_compatible_portion = ".".join(base_version.split(".")[:2])
+        if logger.isEnabledFor(DEBUG):
+            logger.debug(
+                f"Finding latest of matches for version {base_version} from {release_versions} using {version_compatible_portion}")
+
+        for ver in release_versions:
+            if ".".join(ver.split('.')[:2]) == version_compatible_portion:
+                return ver
+        return None
+    return release_versions[0] if release_versions else None
+
+
+def fetch_package_versions_from_pypi(pkg_name):
+    """
+    Utility to get the latest version for a given package name.
+
+    Args:
+        pkg_name: package name given
+
+    Returns: the latest version of ven package
+    """
+    url = PKG_PYPI.format(pkg_name)
+    try:
+        releases = json.loads(request.urlopen(url).read())['releases']
+    except Exception:
+        return None
+
+    return releases
+
+
+def fetch_package_versions(pkg_name, is_released=True, sort=True, display_all=False):
+    """
+    Utility to get the latest version for a given package name.
+
+    Args:
+        pkg_name: package name given
+        is_released: get released version only
+        sort: make version sorted or not
+        display_all: determine if output all package releases
+
+    Returns: the latest version of ven package
+    """
+    # First fetch versions from Artifactory
+    pkg_url = "/".join([PYPI_PRODUCTION_SIMPLE, pkg_name])
+    versions = fetch_versions_from_server(pkg_url, parser=LinkNameParser)
+
+    if versions is None:
+        versions = fetch_package_versions_from_pypi(pkg_name)
+
+    if sort:
+        versions = sorted(versions, key=parse, reverse=True)
+
+    if is_released:
+        versions = [ver for ver in versions if not parse(ver).is_prerelease]
+
+    if display_all:
+        print(display_all)
+
+    return versions
+
+
+def get_highest_version(pkg_requirement: str):
+    """
+    Utility to get the latest version for a given package name.
+
+    Args:
+        pkg_requirement: package requirement given
+    Returns: the highest valid version of the package
+    """
+    req = Requirement(pkg_requirement)
+    available_versions = fetch_package_versions(req.name)
+
+    # Filter versions that satisfy the specifier
+    valid_versions = [Version(version) for version in available_versions if parse(version) in req.specifier]
+
+    if not valid_versions:
+        return None  # No valid versions found
+
+    # Return the highest valid version
+    return str(max(valid_versions))
+
+
+def get_latest_version(pkg_name):
+    """
+    Utility to get the latest version for a given package name.
+
+    Args:
+        pkg_name: package name given
+
+    Returns: the latest version of package
+    """
+    version = get_highest_version(pkg_name)
+    if version is None:
+        user_logger.info(f"No valid versions found for '{pkg_name}'")
+        exit(1)
+    return version
+
+
+def get_latest_compatible_version(pkg_name, base_version=None, versions=None, validate=True):
+    """
+    Utility to get the latest compatible version from a given version list.
+
+    Args:
+        base_version: Optional base version. Versions above this will not be added.
+        pkg_name: package name given
+        versions: user input of version list
+        validate: bool, if True, will validate base_version
+
+    Returns: the latest compatible version from versions
+
+    Raises:
+        Exception - If we cannot find version
+    Notes:
+        - TODO - Make custom exception or use ValueError
+    """
+    if versions is None:
+        versions = fetch_package_versions(pkg_name)
+
+    # Return None if given version list is None or empty
+    if not versions:
+        return None
+
+    # Return the latest version if no base_version is given
+    if base_version is None:
+        return versions[0]
+
+    # Cleanup
+    base_version = base_version.replace('+nightly', '')
+
+    # Make sure the input is valid
+    parsed_version_to_check = parse(base_version)
+
+    # Check if the version is in the list
+    is_in_list = any(parsed_version_to_check == parse(version) for version in versions)
+    if not is_in_list and validate:
+        user_logger.info(f"Could not find the version of '{base_version}' for '{pkg_name}'.")
+        return None
+
+    # Find all possible candidates
+    candidates = [version for version in versions if version.startswith(base_version)]
+
+    # Pick the latest
+    return candidates[0]

idmtools_platform_comps/utils/python_requirements_ac/__init__.py

@@ -0,0 +1,4 @@
+"""idmtools python requirement ac output.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""

idmtools_platform_comps/utils/python_requirements_ac/create_asset_collection.py

@@ -0,0 +1,155 @@
+"""idmtools create asset collection script.
+
+This is part of the RequirementsToAssetCollection tool. This is ran on the SSMT to convert installed files to a AssetCollection.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+import os
+import sys
+from COMPS import Client
+from COMPS.Data import AssetCollectionFile, QueryCriteria
+from COMPS.Data import Experiment
+from COMPS.Data.AssetCollection import AssetCollection
+from idmtools.utils.hashing import calculate_md5
+
+MD5_KEY = 'idmtools-requirements-md5-{}'
+AC_FILE = 'ac_info.txt'
+LIBRARY_ROOT_PREFIX = 'L'
+
+
+def build_asset_file_list(prefix=LIBRARY_ROOT_PREFIX):
+    """
+    Utility function to build all library files.
+
+    Args:
+        prefix: used to identify library files
+
+    Returns: file paths as a list
+    """
+    output = []
+    for root, _, filenames in os.walk(prefix):
+        for filename in filenames:
+            asset = AssetCollectionFile(file_name=os.path.basename(filename),
+                                        relative_path=os.path.join("site-packages",
+                                                                   root.replace(prefix, "").strip("/")).strip("/"),
+                                        md5_checksum=calculate_md5(os.path.join(root, filename))
+                                        )
+            output.append(asset)
+
+    return output
+
+
+def get_first_simulation_of_experiment(exp_id):
+    """
+    Retrieve the first simulation from an experiment.
+
+    Args:
+        exp_id: use input (experiment id)
+
+    Returns: list of files paths
+    """
+    comps_exp = Experiment.get(exp_id)
+    comps_sims = comps_exp.get_simulations(QueryCriteria().select_children('hpc_jobs'))
+    comps_sim = comps_sims[0]
+
+    return comps_sim
+
+
+def main():  # pragma: no cover
+    """Main entry point for our create asset collection script."""
+    print(sys.argv)
+
+    if len(sys.argv) < 3:
+        raise Exception(
+            "The script needs to be called with `python <model.py> <experiment_id> <endpoint> <os_str>'.\n{}".format(
+                " ".join(sys.argv)))
+
+    # Get the experiments
+    exp_id = sys.argv[1]
+    print('exp_id: ', exp_id)
+
+    # Get endpoint
+    endpoint = sys.argv[2]
+    print('endpoint: ', endpoint)
+
+    # Platform key
+    os_target = sys.argv[3]
+    print('os: ', os_target)
+
+    client = Client()
+    client.login(endpoint)
+
+    # Retrieve the first simulation of the experiment
+    comps_sim = get_first_simulation_of_experiment(exp_id)
+    print('sim_id: ', comps_sim.id)
+
+    # Build files metadata
+    base_path = os.path.join(comps_sim.hpc_jobs[-1].working_directory, LIBRARY_ROOT_PREFIX)
+    asset_files = build_asset_file_list(prefix=base_path)
+    print('asset files count: ', len(asset_files))
+
+    # Output files
+    max_files = 10
+    print('Display the first 10 files:\n',
+          "\n".join([f"{a.relative_path}/{a.file_name}" for a in asset_files[0:max_files]]))
+
+    # Retrieve experiment's tags
+    comps_exp = Experiment.get(exp_id, QueryCriteria().select_children('tags'))
+    exp_tags = comps_exp.tags
+
+    # Retrieve experiment's tags
+    _reserved_tag = ['idmtools', 'task_type', MD5_KEY.format(os_target)]
+    comps_exp = Experiment.get(exp_id, QueryCriteria().select_children('tags'))
+    user_tags = {key: value for key, value in comps_exp.tags.items() if key not in _reserved_tag}
+
+    # Get md5_str
+    md5_str = exp_tags.get(MD5_KEY.format(os_target), None)
+
+    # Collect ac's tags
+    ac = AssetCollection()
+    tags = {MD5_KEY.format(os_target): md5_str}
+    tags.update(user_tags)
+    ac.set_tags(tags)
+
+    # Create asset collection
+    for af in asset_files:
+        ac.add_asset(af)
+
+    sys.stdout.flush()
+    missing_files = ac.save(return_missing_files=True)
+
+    # If COMPS responds that we're missing some files, then try creating it again,
+    # uploading only the files that COMPS doesn't already have.
+    if missing_files:
+        print(f"Total of {len(ac.assets) - len(missing_files)} files currently in comps. Resolving missing files")
+        ac2 = AssetCollection()
+        ac2.set_tags(tags)
+
+        for acf in ac.assets:
+            if acf.md5_checksum in missing_files:
+                rp = acf.relative_path
+                fn = acf.file_name
+                acf2 = AssetCollectionFile(fn, rp, tags=acf.tags)
+                rfp = os.path.join(base_path, rp.replace("site-packages", "").strip(os.path.sep), fn)
+                ac2.add_asset(acf2, rfp)
+            else:
+                ac2.add_asset(acf)
+
+        print("\n\n\n=====================\nUploading files not in comps: " + "\n".join(
+            [f"{a.relative_path}/{a.file_name}" for a in ac2.assets if
+             a.md5_checksum in missing_files or a.md5_checksum is None]))
+
+        sys.stdout.flush()
+        ac2.save()
+        ac = ac2
+    # Output ac
+    print('ac_id: ', ac.id)
+
+    # write ac_id to file ac_info.txt
+    with open(AC_FILE, 'w') as outfile:
+        outfile.write(str(ac.id))
+    sys.stdout.flush()
+
+
+if __name__ == "__main__":  # pragma: no cover
+    main()