idmtools-cli 0.0.0.dev0__py3-none-any.whl → 0.0.3__py3-none-any.whl

idmtools_cli/cli/init.py

@@ -0,0 +1,113 @@
+"""Defines the init(templates) cli command."""
+import itertools
+import json
+import logging
+import os
+import urllib.request
+from logging import getLogger
+from typing import Dict, List
+from idmtools.config import IdmConfigParser
+from idmtools.registry.plugin_specification import ProjectTemplate
+from idmtools_cli.cli.entrypoint import cli
+
+logger = getLogger(__name__)
+
+
+@cli.group(help="Commands to help start or extend projects through templating.")
+def init():
+    """Base init group command."""
+    pass
+
+
+@cli.command(help="Export list of project templates.")
+def init_export():
+    """Export project templates."""
+    with open('templates.json', 'w') as o:
+        import dataclasses
+        result = get_project_list()
+        result = {x: dataclasses.asdict(v) for x, v in result.items()}
+        json.dump(result, o)
+
+
+def define_cookiecutter_project_command(project_details: ProjectTemplate):
+    """
+    Defines the specific project cookie cutter command.
+
+    Args:
+        project_details:
+
+    Returns:
+        Define a dynamic cookie cutter command around a template
+    """
+
+    @init.command(name=project_details.name, help=project_details.description)
+    def run_project():
+        from cookiecutter.main import cookiecutter
+        if not isinstance(project_details.url, list):
+            project_details.url = [project_details.url]
+        for url in project_details.url:
+            cookiecutter(url)
+
+    return run_project
+
+
+def get_project_list() -> Dict[str, ProjectTemplate]:
+    """
+    Build a list of cookie cutter options for menu.
+
+    Returns:
+        Return list of project templates
+    """
+    from idmtools.registry.experiment_specification import ExperimentPlugins
+    from idmtools.registry.platform_specification import PlatformPlugins
+
+    # fetch
+    items = list()
+    f_dir = os.path.dirname(__file__)
+    with open(os.path.join(f_dir, 'common_project_templates.json'), 'rb') as fin:
+        items.extend(ProjectTemplate.read_templates_from_json_stream(fin))
+
+    for pm in [ExperimentPlugins().get_plugins(), PlatformPlugins().get_plugins()]:
+        items.extend(list(itertools.chain(*map(lambda pl: pl.get_project_templates(), pm))))
+
+    # check for values in config
+    url = IdmConfigParser.get_option('Templating', 'url')
+    if url:
+        logger.debug(f'Loading templates from url: {url}')
+        with urllib.request.urlopen(url) as s:
+            items.extend(ProjectTemplate.read_templates_from_json_stream(s))
+    result = {x.name: x for x in items}
+
+    if logger.isEnabledFor(logging.DEBUG):
+        logger.debug(f'Cookie cutter project list: {result}')
+
+    return result
+
+
+def build_project_commands():
+    """
+    Builds the cookie cutter cli commands.
+
+    Returns:
+        None
+    """
+    result = get_project_list()
+    # Now define all the cookie cutter projects
+    for _name, details in result.items():
+        define_cookiecutter_project_command(details)
+
+
+def read_templates_from_json_stream(items: List[ProjectTemplate], s):
+    """
+    Read Project Template from stream onto the list.
+
+    Args:
+        items: List to append data to
+        s: Stream where json data resides
+
+    Returns:
+        None
+    """
+    data = json.loads(s.read().decode())
+    for item in data:
+        items.append(ProjectTemplate(**item))

idmtools_cli/cli/package.py

@@ -0,0 +1,106 @@
+"""Define our package group CLI command."""
+import os
+from logging import getLogger
+import click
+from typing import Optional, List
+from idmtools_cli.cli.entrypoint import cli
+
+user_logger = getLogger('user')
+
+
+@cli.group(short_help="Contains commands related to package versions.")
+def package():
+    """Package group cli command."""
+    pass
+
+
+@package.command()
+@click.option('--name', required=True, type=str, help="package name")
+def latest_version(name: Optional[str]):
+    """
+    Display the latest version of a package.
+
+    Args:
+        name: package name
+    """
+    from idmtools_platform_comps.utils.package_version import get_latest_version
+    v = get_latest_version(name)
+    print(v)
+
+
+@package.command()
+@click.option('--name', required=True, type=str, help="package name")
+@click.option('--base_version', required=True, default=None, type=str, help="package version")
+def compatible_version(name: Optional[str], base_version: Optional[str]):
+    """
+    Display the latest compatible version of a package.
+
+    Args:
+        name: package name
+        base_version: package version
+    """
+    from idmtools_platform_comps.utils.package_version import get_latest_compatible_version
+    v = get_latest_compatible_version(name, base_version)
+    print(v)
+
+
+@package.command()
+@click.option('--name', required=True, type=str, help="package name")
+@click.option('--all/--no-all', type=bool, default=False, help="package version")
+def list_versions(name: Optional[str], all: Optional[bool]):
+    """
+    Display all package versions.
+
+    Args:
+        name: package name
+        all: True/False - return all or only released versions
+    """
+    from idmtools_platform_comps.utils.package_version import fetch_package_versions
+    versions = fetch_package_versions(name, not all)
+    print(versions)
+
+
+@package.command()
+@click.argument('requirement', type=click.Path(exists=True), required=False)
+@click.option('--pkg', multiple=True, help="Package for override. Format: 'key==value'")
+@click.option('--wheel', multiple=True, help="Local wheel file")
+def updated_requirements(requirement: str = None, pkg: Optional[List[str]] = None, wheel: Optional[List[str]] = None):
+    """
+    Build Updated_Requirements from requirement file.
+
+    Args:
+        requirement: Path to requirement file
+        pkg: package name (along with version)
+        wheel: package wheel file
+    """
+    from idmtools_platform_comps.utils.python_requirements_ac.requirements_to_asset_collection import \
+        RequirementsToAssetCollection
+
+    pkg_list = list(pkg)
+    wheel_list = [os.path.abspath(w) for w in wheel]
+    pl = RequirementsToAssetCollection(None, requirements_path=requirement, pkg_list=pkg_list, local_wheels=wheel_list)
+    pl.save_updated_requirements()
+    req = open('requirements_updated.txt').read()
+    print(req)
+
+
+@package.command()
+@click.argument('requirement', type=click.Path(exists=True), required=False)
+@click.option('--pkg', multiple=True, help="Package for override. Format: 'key==value'")
+@click.option('--wheel', multiple=True, help="Local wheel file")
+def checksum(requirement: str = None, pkg: Optional[List[str]] = None, wheel: Optional[List[str]] = None):
+    """
+    Construct checksum from requirement file.
+
+    Args:
+        requirement: path to requirement file
+        pkg: package name (along with version)
+        wheel: package wheel file
+    """
+    from idmtools_platform_comps.utils.python_requirements_ac.requirements_to_asset_collection import \
+        RequirementsToAssetCollection
+
+    pkg_list = list(pkg)
+    wheel_list = [os.path.abspath(w) for w in wheel]
+    pl = RequirementsToAssetCollection(None, requirements_path=requirement, pkg_list=pkg_list, local_wheels=wheel_list)
+    print(pl.checksum)

idmtools_cli/cli/system_info.py

@@ -0,0 +1,158 @@
+"""
+Defines the info group cli command and version command as well.
+"""
+from collections import defaultdict
+
+import os
+from logging import getLogger, DEBUG
+import stat
+import click
+import pyperclip
+
+from idmtools.registry.master_plugin_registry import MasterPluginRegistry
+from idmtools.registry.task_specification import TaskPlugins
+from tabulate import tabulate
+from idmtools.core.system_information import get_system_information
+from idmtools.registry.platform_specification import PlatformPlugins
+from idmtools_cli.cli.entrypoint import cli
+from idmtools_cli.cli.utils import supported_platforms
+from idmtools_cli.utils.cols import columns
+
+logger = getLogger(__name__)
+
+
+@cli.command(help="List version info about idmtools and plugins.")
+@click.option('--no-plugins/--plugins', default=False, help="Control whether we display plugins with modules")
+def version(no_plugins: bool):
+    """Returns version of all idmtools components."""
+    from idmtools import __version__
+    from idmtools_cli import __version__ as cli_version
+    plugin_map = MasterPluginRegistry().get_plugin_map()
+    module_map = defaultdict(dict)
+    for name in sorted(plugin_map.keys()):
+        try:
+            spec = plugin_map[name]
+            module_name = str(spec.get_type().__module__)
+            if module_name:
+                module_name = module_name.split(".")[0]
+        except Exception as e:
+            if logger.isEnabledFor(DEBUG):
+                logger.exception(e)
+            module_name = 'Unknown Module'
+        if plugin_map[name].get_version():
+            module_map[module_name][name] = plugin_map[name].get_version()
+
+    if not module_map['idmtools']:
+        module_map['idmtools'] = __version__
+
+    if not module_map['idmtools-cli']:
+        module_map['idmtools-cli'] = cli_version
+
+    for module in sorted(module_map.keys()):
+        if isinstance(module_map[module], dict):
+            mod_version = list(module_map[module].values())[0]
+            click.echo(click.style(columns((module.replace("_", "-"), 36), (f'Version: {mod_version}', 40)), fg='green'))
+            if not no_plugins:
+                click.echo(click.style('  Plugins:', fg='yellow'))
+                for plugin_name in sorted(module_map[module].keys()):
+                    click.echo(click.style(columns(('', 3), (f'{plugin_name}', 25)), fg='blue'))
+        else:
+            click.echo(click.style(columns((module.replace("_", "-"), 36), (f'Version: {module_map[module]}', 40)), fg='green'))
+
+
+@cli.group(help="Troubleshooting and debugging information.")
+def info():
+    """Info cli subcommand."""
+    pass
+
+
+@info.command(help="Provide an output with details about your current execution platform and IDM-Tools install")
+@click.option('--copy-to-clipboard/--no-copy-to-clipboard', default=False, help="Copy output to clipboard")
+@click.option('--no-format-for-gh/--format-for-gh', default=False, help="When copying to clipboard, do we want to "
+                                                                        "formatted for Github")
+@click.option('--issue/--no-issue', default=False, help="Copy data and format for github alias")
+@click.option('--output-filename', default=None, help="Output filename")
+def system(copy_to_clipboard, no_format_for_gh, issue, output_filename):
+    """Provide info about your current install of idmtools."""
+    logger.debug("Building system info")
+    system_info = get_system_information()
+    logger.debug("Building system info output")
+    lines = [f'System Information\n{"=" * 20}']
+    ordered_fields = sorted(system_info.__dict__.keys())
+    [lines.append(f'{k}: {system_info.__dict__[k]}') for k in ordered_fields]
+    if os.name != 'nt':
+        logger.debug("Building windows info")
+        for f in ['workers', 'redis-data']:
+            fname = os.path.join(system_info.data_directory, f)
+            if os.path.exists(fname):
+                from pwd import getpwuid
+                s = os.stat(fname)
+                owned_by = f'{getpwuid(s.st_uid).pw_name} UID {s.st_uid}:{s.st_gid}'
+                lines.append(f'{fname} has permissions of {oct(stat.S_IMODE(os.lstat(fname).st_mode))} '
+                             f'and is owned by {owned_by}')
+        logger.debug("Windows info done")
+    try:
+        import docker
+        logger.debug("Building docker info")
+        lines.append(f'\nDocker Information\n{"=" * 20}')
+        client = docker.from_env()
+        lines.append(f'Version: {client.version()}')
+        docker_info = client.info()
+        ordered_fields = sorted(docker_info.keys())
+        [lines.append(f'{k}: {docker_info[k]}') for k in ordered_fields]
+        logger.debug("docker info done")
+    except ImportError:
+        pass
+    except Exception as e:
+        logger.error(e)
+        raise e
+    if copy_to_clipboard:
+        logger.debug("Copying to clipboard")
+        output = '\n'.join(lines)
+        if not no_format_for_gh or issue:
+            logger.debug('Formatting for github')
+            # let's remove system information here by rebuilding without it
+            output = '\n'.join(lines[0:])
+            output = f'```### System Information\n{output}````'
+        pyperclip.copy(output)
+    if output_filename is not None:
+        with open(output_filename, 'w') as log_out:
+            log_out.writelines(lines)
+    else:
+        logger.debug("Writing output")
+        for line in lines:
+            click.echo(line)
+
+
+@info.group(help="Commands to get information about installed IDM-Tools plugins")
+def plugins():
+    """Info about plugins installed."""
+    pass
+
+
+@plugins.command(help="List CLI plugins")
+def cli():
+    """List CLI plugins."""
+    items = list(supported_platforms.keys())
+    print(tabulate([[x] for x in items], headers=['CLI Plugins'], tablefmt='psql'))
+
+
+@plugins.command(help="List Platform plugins configuration aliases")
+def platform_aliases():
+    """List platform aliases."""
+    aliases = PlatformPlugins().get_aliases()
+    print(tabulate([[name, details[1]] for name, details in aliases.items()], headers=['Platform Plugin Aliases', "Configuration Options"], tablefmt='psql'))
+
+
+@plugins.command(help="List Platform plugins")
+def platform():
+    """List platform plugins."""
+    platforms = PlatformPlugins().get_plugin_map().keys()
+    print(tabulate([[x] for x in platforms], headers=['Platform Plugins'], tablefmt='psql'))
+
+
+@plugins.command(help="List Task plugins")
+def task():
+    """List task plugins."""
+    tasks = TaskPlugins().get_plugin_map().keys()
+    print(tabulate([[x] for x in tasks], headers=['Task Plugins'], tablefmt='psql'))

idmtools_cli/cli/utils.py

@@ -0,0 +1,49 @@
+"""Defines utilities used with the cli commands."""
+import sys
+from typing import NoReturn, Union
+import requests
+from click import UsageError
+from colorama import Fore
+from idmtools.core.platform_factory import Platform
+from idmtools_cli.iplatform_cli import PlatformCLIPlugins
+
+supported_platforms = PlatformCLIPlugins().get_plugin_map()
+
+tags_help = "Tag to filter by. This should be in the form name value. For example, if you have a tag type=PythonTask " \
+            "you would use --tags type PythonTask. In addition, you can provide multiple tags, ie --tags a 1 " \
+            "--tags b 2. This will perform an AND based query on the tags meaning only jobs contains ALL the tags " \
+            "specified will be displayed"
+
+
+def show_error(message: Union[str, requests.Response]) -> NoReturn:
+    """
+    Display an error response from API on the command line.
+
+    Args:
+        message (Union[str, requests.Response]): message to display
+
+    Returns:
+        Nothing
+    """
+    print(f'{Fore.RED}Error{Fore.RESET}: {message}')
+    sys.exit(-1)
+
+
+def get_platform_from_config_or_name(config_block, platform):
+    """
+    Attempt to find the config block or crete platform obj from name.
+
+    Args:
+        config_block: Config block
+        platform: Platform type
+
+    Returns:
+        Platform object
+    """
+    if platform is None and config_block is None:
+        raise UsageError("You must specify a platform or a configuration block")
+    if config_block:
+        platform_obj = Platform(config_block)
+    else:
+        platform_obj = supported_platforms[platform].get({})
+    return platform_obj

idmtools_cli/iplatform_cli.py

@@ -0,0 +1,141 @@
+"""Define our platform specific specifications.
+
+This is a specific specification to illuminate a command CLI between platforms for interacting with
+experiment/simulations.
+"""
+#
+from abc import ABC, abstractmethod
+from typing import NoReturn, Dict, Set, cast, Optional, List, Tuple
+import pluggy
+from idmtools.registry.plugin_specification import PLUGIN_REFERENCE_NAME, PluginSpecification
+from idmtools.registry.utils import load_plugin_map
+
+get_platform_cli_spec = pluggy.HookspecMarker(PLUGIN_REFERENCE_NAME)
+get_additional_commands_spec = pluggy.HookspecMarker(PLUGIN_REFERENCE_NAME)
+get_platform_cli_impl = pluggy.HookimplMarker(PLUGIN_REFERENCE_NAME)
+get_additional_commands_impl = pluggy.HookimplMarker(PLUGIN_REFERENCE_NAME)
+
+
+class IPlatformCLI(ABC):
+    """The Base PLatformCLI definition. It is mostly abstract classes around fetching info from a platform."""
+
+    @abstractmethod
+    def get_experiment_status(self, id: Optional[str], tags: Optional[List[Tuple[str, str]]]) -> NoReturn:
+        """
+        Get experiment status from a platform.
+
+        Args:
+            id: Optional ID to search for
+            tags: Optional tags to filter for as well
+
+        Returns:
+            None
+        """
+        pass
+
+    @abstractmethod
+    def get_simulation_status(self, id: Optional[str], experiment_id: Optional[str], status: Optional[str],
+                              tags: Optional[List[Tuple[str, str]]]) -> NoReturn:
+        """
+        Get simulation status.
+
+        Args:
+            id: Optional simulation id
+            experiment_id: Optional experiment id
+            status: Status to filter for
+            tags: Tags to filter for
+
+        Returns:
+            None
+        """
+        pass
+
+    @abstractmethod
+    def get_platform_information(self) -> dict:
+        """
+        Get any platform relevant information.
+
+        Returns:
+            Str containing platform info
+        """
+        pass
+
+
+class PlatformCLISpecification(PluginSpecification, ABC):
+    """Defines the Platform CLI Plugin Base."""
+    @classmethod
+    def get_name(cls, strip_all: bool = True) -> str:
+        """
+        Get name of plugin. By default we remove the CLISpecification portion.
+
+        Args:
+            strip_all: When true, CLISpecification is stripped from name. When false only Specification is Stripped
+
+        Returns:
+            None
+        """
+        if strip_all:
+            ret = cls.__name__.replace("CLISpecification", '')
+        else:
+            ret = cls.__name__.replace('Specification', '')
+        return ret
+
+    @get_platform_cli_spec
+    def get(self, configuration: dict) -> IPlatformCLI:
+        """
+        Factor that should return a new platform using the passed-in configuration.
+
+        Args:
+            configuration:
+
+        Returns:
+            None
+
+        Raises:
+            NotImplementedError
+        """
+        raise NotImplementedError("Plugin did not implement get")
+
+    @get_additional_commands_spec
+    def get_additional_commands(self) -> NoReturn:
+        """
+        Get any additional commands defined by the platform CLI.
+
+        Returns:
+            None
+
+        Raises:
+            NotImplementedError
+        """
+        raise NotImplementedError("Plugin did not implement get_additional_commands")
+
+
+class PlatformCLIPlugins:
+    """Platform CLI Plugin Manager. Loads all the plugins."""
+    def __init__(self, strip_all: bool = True) -> None:
+        """
+        Creates our plugin index.
+
+        Args:
+            strip_all: Formatting option for plugin names in registry
+        """
+        self._plugins = cast(Dict[str, PlatformCLISpecification],
+                             load_plugin_map('idmtools_platform_cli', PlatformCLISpecification, strip_all))
+
+    def get_plugins(self) -> Set[PlatformCLISpecification]:
+        """
+        Get plugins in the manager.
+
+        Returns:
+            List of plugin specs
+        """
+        return set(self._plugins.values())
+
+    def get_plugin_map(self) -> Dict[str, PlatformCLISpecification]:
+        """
+        Get the plugin map.
+
+        Returns:
+            Plugin Dict with name of plugin -> plugin cli spec
+        """
+        return self._plugins

idmtools_cli/main.py

@@ -0,0 +1,52 @@
+"""Entrypoint for idmtools cli."""
+import os
+from typing import NoReturn
+from colorama import init as color_init
+
+
+def main() -> NoReturn:
+    """
+    This is our main run function for the CLI.
+
+    It basically calls start(load cli including the plugins) and then run the cli.
+
+    Returns:
+        None
+    """
+    os.environ['IDMTOOLS_NO_CONFIG_WARNING'] = '1'
+    from idmtools_cli.cli.entrypoint import cli
+    start()
+    cli(auto_envvar_prefix='IDMTOOLS_CLI')
+
+
+def start() -> NoReturn:
+    """
+    Loads the different components of the CLI.
+
+    Currently this involves
+
+    1) Initializing colorama
+    2) Loading built-in commands
+    3) Loading Platform CLI items
+    4) Loading custom CLI from platform plugins
+
+    Returns:
+        None
+    """
+    from idmtools_cli.iplatform_cli import PlatformCLIPlugins
+    color_init()
+    import idmtools_cli.cli.init  # noqa: F401
+    import idmtools_cli.cli.config_file  # noqa: F401
+    import idmtools_cli.cli.system_info  # noqa: F401
+    import idmtools_cli.cli.gitrepo  # noqa: F401
+    import idmtools_cli.cli.package  # noqa: F401
+    platform_plugins = PlatformCLIPlugins()
+    from idmtools_cli.cli.init import build_project_commands
+    build_project_commands()
+    # Trigger the loading of additional cli from platforms
+    [p.get_additional_commands() for p in platform_plugins.get_plugins()]
+
+
+# our entrypoint for our cli
+if __name__ == '__main__':
+    main()

idmtools_cli/utils/__init__.py

@@ -0,0 +1 @@
+"""Module for cli utils."""