idmtools-cli 0.0.0.dev0__py3-none-any.whl → 0.0.3__py3-none-any.whl

idmtools_cli/__init__.py

@@ -1,8 +1,12 @@
-"""
-idmtools-cli - Placeholder Package
-
-This is a placeholder package to reserve the name on PyPI.
-The actual package will be published later.
-"""
-
-__version__ = "0.0.0.dev0"
+"""iidmtools_cli version definition file."""
+try:
+    from importlib.metadata import version, PackageNotFoundError
+except ImportError:
+    # Python < 3.8
+    from importlib_metadata import version, PackageNotFoundError
+
+try:
+    __version__ = version("idmtools-cli")  # Use your actual package name
+except PackageNotFoundError:
+    # Package not installed, use fallback
+    __version__ = "0.0.0+unknown"

idmtools_cli/cli/__init__.py

@@ -0,0 +1 @@
+"""Package init file."""

idmtools_cli/cli/common_project_templates.json

@@ -0,0 +1,19 @@
+[{
+  "name": "reproducible-science",
+  "url": "gh:mkrapp/cookiecutter-reproducible-science",
+  "description": "A boilerplate for reproducible and transparent science with close resemblances to the philosophy of Cookiecutter Data Science: A logical, reasonably standardized, but flexible project structure for doing and sharing data science work.",
+  "info": "https://github.com/mkrapp/cookiecutter-reproducible-science"
+  },
+  {
+  "name": "docker-science",
+  "url": "git@github.com:docker-science/cookiecutter-docker-science.git",
+  "description": "This project is a tiny template for machine learning projects developed in Docker environments. In machine learning tasks, projects glow uniquely to fit target tasks, but in the initial state, most directory structure and targets in Makefile are common. Cookiecutter Docker Science generates initial directories which fits simple machine learning tasks.",
+  "info": "https://docker-science.github.io/"
+  },
+  {
+  "name": "data-science",
+  "url": "https://github.com/drivendata/cookiecutter-data-science",
+  "description": "A logical, reasonably standardized, but flexible project structure for doing and sharing data science work.",
+  "info": "https://docker-science.github.io/"
+  }
+  ]

idmtools_cli/cli/config_file.py

@@ -0,0 +1,199 @@
+"""Defines the config cli group and commands."""
+import configparser
+import dataclasses
+import os
+import re
+import click
+from click import secho
+from colorama import Fore, Style
+from idmtools.registry.platform_specification import PlatformPlugins
+from idmtools_cli.cli.entrypoint import cli
+
+IGNORED_PLATFORMS = ["Test", "Slurm", "File", "Container", "SSMT", "TestExecute", "Process"]
+AVAILABLE_PLATFORMS = PlatformPlugins().get_plugin_map()
+for platform in IGNORED_PLATFORMS:
+    if platform in AVAILABLE_PLATFORMS:
+        del AVAILABLE_PLATFORMS[platform]
+HIDDEN_FIELD_REGEX = re.compile('^_.+$')
+FIELD_BLACKLIST = ['platform_type_map', 'supported_types', 'plugin_key', 'docker_image']
+
+
+@cli.group()
+@click.option("--config_path", prompt="Path to the idmtools.ini file",
+              help="Path to the idmtools.ini file",
+              default=os.path.join(os.getcwd(), "idmtools.ini"),
+              type=click.Path(dir_okay=False, file_okay=True, exists=False, writable=True, resolve_path=True))
+@click.option("--global-config/--no-global-config", default=False, help="Allow generating config in the platform default global location")
+@click.pass_context
+def config(ctx, config_path, global_config):
+    """
+    Contains commands related to the creation of idmtools.ini.
+
+    With the config command, you can :
+     - Generate an idmtools.ini file in the current directory
+     - Add a configuration block
+    """
+    ctx.ensure_object(dict)
+    if global_config:
+        from idmtools import IdmConfigParser
+        config_path = IdmConfigParser.get_global_configuration_name()
+
+    # Create a config parser and read the file if it exist
+    # The comment prefixes and allow_no_value is a truck to keep the comments even while editing
+    cp = configparser.ConfigParser(comment_prefixes='/', allow_no_value=True)
+    if os.path.exists(config_path):
+        cp.read_file(open(config_path))
+
+    # Store the config parser in the context
+    ctx.obj["cp"] = cp
+    ctx.obj["path"] = config_path
+
+
+def slugify(value):
+    """
+    Slugify the option.
+
+    This means upper-casing and replacing spaces with "-"
+
+    Args:
+        value: Item to slugify
+
+    Returns:
+        Slugified string
+    """
+    value = value.upper()
+    value = value.replace(" ", "_")
+    return value
+
+
+def validate_block_name(context, value):
+    """
+    Validate if a block name exists, and if so, should we overwrite it.
+
+    Args:
+        context: Context object
+        value: Value to check
+
+    Returns:
+        Slugified value name
+    """
+    cp = context.obj["cp"]
+    value = slugify(value)
+    if value in cp.sections():
+        secho(f"The {value} block already exists in the selected ini file.", fg="bright_yellow")
+        click.confirm(click.style("Do you want to continue and overwrite the existing block?", fg="bright_yellow"), default=False, abort=True)
+
+        # Remove the block from the config parser
+        del cp[value]
+
+    context.obj['cp'] = cp
+
+    return value
+
+
+@config.command()
+@click.option("--block_name", prompt="New block name",
+              help="Name of the new block in the file",
+              callback=lambda c, p, v: validate_block_name(c, v),
+              type=click.STRING)
+@click.option('--platform', default='COMPS', type=click.Choice(AVAILABLE_PLATFORMS.keys()), prompt="Platform type")
+@click.pass_context
+def block(ctx, block_name, platform):
+    """
+    Command to create/replace a block in the selected idmtools.ini.
+
+    Args:
+        ctx: Context containing the path of idmtools.ini and the associated configparser
+        block_name:  Name of the block to create/replace
+        platform:  Selected platform
+    """
+    config_path = ctx.obj['path']
+    print("\n" + Style.BRIGHT + "-" * 50)
+    print("idmtools.ini Utility")
+    print(f"- INI Location: {config_path}")
+    print(f"- Selected block: {block_name}")
+    print(f"- Selected platform: {platform}")
+    print("-" * 50 + Style.NORMAL + "\n")
+
+    # Retrieve the platform and its associated fields
+    platform_obj = AVAILABLE_PLATFORMS[platform]
+    fields = dataclasses.fields(platform_obj.get_type())
+
+    # Dictionary to store user choices and field defaults
+    # Store both to allow the fields callback functions to access the previous user choices regardless of defaults
+    values = {"type": platform}
+    defaults = {}
+
+    # Ask about each field
+    # The field needs to contain a `help` section in the metadata to be considered
+    for field in filter(lambda f: "help" in f.metadata, fields):
+
+        # Display the help message
+        print(f"{Fore.CYAN}{field.metadata['help']}{Fore.RESET}")
+
+        # Retrieve the metadata
+        md = dict(field.metadata)
+
+        # If a callback exists -> execute it
+        if "callback" in md:
+            md.update(md["callback"](values, field))
+
+        # Create the default
+        field_default = md.get("default", field.default if field.default is not None else '')
+        defaults[field.name] = field.default
+
+        # Handle the choices if any
+        prompt_type = click.Choice(md["choices"]) if "choices" in md else field.type
+
+        # Retrieve the validation function if any
+        if "validate" in md:
+            validation = md["validate"]
+        else:
+            validation = lambda v: (True, None)  # noqa: E731
+
+        # Prompt the user
+        while True:
+            user_input = click.prompt(field.name, type=prompt_type, default=field_default, prompt_suffix=f": {Fore.GREEN}")
+
+            # Call the validation
+            result, msg = validation(user_input)
+
+            # If positive, get out
+            if result:
+                break
+
+            # Else display the error message
+            secho(msg, fg="bright_red")
+
+        # Store the value
+        values[field.name] = user_input if user_input != "" else None
+        print(Fore.RESET)
+
+    # Remove the default values from the values
+    for k, d in defaults.items():
+        if values[k] == d:
+            del values[k]
+
+    # Display a validation prompt
+    print("The following block will be added to the file:\n")
+    longest_param = max(len(p) for p in values)
+    block_parameters = "\n".join(f"{param.ljust(longest_param)} = {value}" for param, value in values.items())
+    block_headers = f"[{block_name}]"
+    block = block_headers + "\n" + block_parameters
+    secho(f"{block}\n", fg="bright_blue")
+
+    # If we decide to go ahead -> write to file
+    if click.confirm("Do you want to write this block to the file?", default=True):
+        # First re-write the content of the config parser
+        cp = ctx.obj["cp"]
+        with open(config_path, 'w') as fp:
+            cp.write(fp)
+            fp.writelines("\n" + block)
+
+        secho("Block written successfully!", fg="bright_green")
+    else:
+        secho("Aborted...", fg="bright_red")
+
+
+if __name__ == '__main__':
+    config(["block", '--block_name', 'test'])

idmtools_cli/cli/entrypoint.py

@@ -0,0 +1,45 @@
+"""Base click group definition."""
+import logging
+from idmtools import IdmConfigParser
+from idmtools.core.logging import setup_logging, IdmToolsLoggingConfig
+import click
+from click_plugins import with_plugins
+
+try:
+    from importlib.metadata import entry_points
+except ImportError:
+    from importlib_metadata import entry_points  # for python 3.7
+from idmtools_cli.iplatform_cli import IPlatformCLI
+
+# Decorator for CLI functions that will require a platform object passed down to them
+pass_platform_cli = click.make_pass_decorator(IPlatformCLI)
+
+
+def get_filtered_entry_points(group):
+    """
+    Get entry points for a specific group, compatible across Python versions.
+
+    Args:
+        group (str): The entry point group to filter by.
+
+    Returns:
+        An iterable of entry point objects for the specified group.
+    """
+    user_entry_points = entry_points()
+    # For Python 3.10 and newer, use the select method if available
+    if hasattr(user_entry_points, 'select'):
+        return user_entry_points.select(group=group)
+    else:
+        # For Python 3.9 and earlier, manually filter the entry points
+        return (ep for ep in user_entry_points.get(group, []))
+
+
+@with_plugins(get_filtered_entry_points('idmtools_cli.cli_plugins'))
+@click.group()
+@click.option('--debug/--no-debug', default=False, help="When selected, enables console level logging")
+def cli(debug):
+    """Allows you to perform multiple idmtools commands."""
+    IdmConfigParser()
+    # init config by just calling config parser
+    if debug:
+        setup_logging(IdmToolsLoggingConfig(console=True, level=logging.DEBUG, force=True))

idmtools_cli/cli/gitrepo.py

@@ -0,0 +1,393 @@
+"""Define the gitrepo group cli command."""
+import os
+import json
+from logging import getLogger
+import click
+from click import secho
+from colorama import Fore
+from typing import Optional, List
+
+from idmtools.core.logging import SUCCESS
+from idmtools_cli.cli.entrypoint import cli
+from idmtools.utils.gitrepo import GitRepo, REPO_OWNER, GITHUB_HOME, REPO_NAME
+from idmtools.registry.master_plugin_registry import MasterPluginRegistry
+
+user_logger = getLogger('user')
+
+
+@cli.group(short_help="Contains commands related to examples download.")
+def gitrepo():
+    """Contains command to download examples."""
+    pass
+
+
+@gitrepo.command()
+@click.option('--raw', default=False, type=bool, help="Files in detail")
+def view(raw: Optional[bool]):
+    """
+    Display all idmtools available examples.
+
+    Args:
+        raw: True/False - display results in details or simplified format
+    """
+    list_examples(raw)
+
+
+def list_examples(raw: bool):
+    """
+    Display all idmtools available examples.
+
+    Args:
+        raw: True/False - display results in details or simplified format
+    """
+    examples = get_plugins_examples()
+    if raw:
+        user_logger.info(json.dumps(examples, indent=3))
+        exit(0)
+    for plugin, urls in examples.items():
+        user_logger.info(f'\n{plugin}')
+        urls = [urls] if isinstance(urls, str) else urls
+        url_list = [f'    - {url}' for url in urls]
+        user_logger.info('\n'.join(url_list))
+
+
+# alias under examples
+@cli.group(help="Display a list of examples organized by plugin type.")
+def examples():
+    """Examples group command. alias for gitrepo command."""
+    pass
+
+
+@examples.command(name='list', help="List examples available")
+@click.option('--raw', default=False, type=bool, help="Files in detail")
+def list_m(raw: Optional[bool]):
+    """Alternate path for gitrepo."""
+    list_examples(raw)
+
+
+@gitrepo.command()
+@click.option('--owner', default=REPO_OWNER, help="Repo owner")
+@click.option('--page', default=1, help="Pagination")
+def repos(owner: Optional[str], page: Optional[int]):
+    """
+    Display all public repos of the owner.
+
+    Args:
+        owner: Repo owner
+        page: Result pagination
+    """
+    gr = GitRepo(owner)
+    try:
+        repos = gr.list_public_repos(page=page)
+    except Exception as ex:
+        secho(str(ex), fg="yellow")
+        exit(1)
+    repos_full = [f'    - {GITHUB_HOME}/{r}' for r in repos]
+    user_logger.log(SUCCESS, f"GitHub Owner: {gr.repo_owner}")
+    user_logger.info('\n'.join(repos_full))
+
+
+@gitrepo.command()
+@click.option('--owner', default=REPO_OWNER, help="Repo owner")
+@click.option('--repo', default=REPO_NAME, help="Repo name")
+def releases(owner: Optional[str], repo: Optional[str]):
+    """
+    Display all the releases of the repo.
+
+    Args:
+        owner: Repo owner
+        repo: Repo name
+    """
+    gr = GitRepo(owner, repo)
+    try:
+        rels = gr.list_repo_releases()
+    except Exception as ex:
+        secho(str(ex), fg="yellow")
+        exit(1)
+    rels_list = [f' - {r}' for r in rels]
+    secho(f'The Repo: {gr.repo_home_url}', fg="green")
+    user_logger.info('\n'.join(rels_list))
+
+
+@gitrepo.command()
+@click.option('--url', required=True, help="Repo files url")
+@click.option('--raw', default=False, type=bool, help="Display files in detail")
+def peep(url: Optional[str], raw: Optional[bool]):
+    """
+    Display all current files/dirs of the repo folder (not recursive).
+
+    Args:
+        url: GitHub repo files url (required)
+        raw: Display details or not
+    """
+    user_logger.info(f'Peep: {url}')
+    user_logger.info('Processing...')
+    try:
+        result = GitRepo().peep(url)
+    except Exception as ex:
+        secho(f'Failed to access: {url}', fg="yellow")
+        user_logger.error(ex)
+        exit(1)
+
+    secho(f"Item Count: {len(result)}", fg="green")
+    if raw:
+        user_logger.info(json.dumps(result, indent=3))
+        exit(0)
+
+    for file in result:
+        if file['type'] == 'dir':
+            secho(f"    - {file['name']}", fg="yellow")
+        else:
+            secho(f"    - {file['name']}")
+
+
+@gitrepo.command()
+@click.option('--type', default=None, multiple=True, help="Download examples by type (COMPSPlatform, PythonTask, etc)")
+@click.option('--url', default=None, multiple=True, help="Repo files url")
+@click.option('--output', default='./', help="Files download destination")
+def download(type: Optional[str], url: Optional[str], output: Optional[str]):
+    """
+    Download files from GitHub repo to user location.
+
+    Args:
+        type: Object type (COMPSPlatform, PythonTask, etc)
+        url: GitHub repo files url
+        output: Local folder
+
+    Returns: Files download count
+    """
+    download_github_repo(output, url, example_types=type)
+
+
+@examples.command(name='download')
+@click.option('--type', default=None, multiple=True, help="Download examples by type (COMPSPlatform, PythonTask, etc)")
+@click.option('--url', default=None, multiple=True, help="Repo files url")
+@click.option('--output', default='./', help="Files download destination")
+def download_alias(type: Optional[str], url: Optional[List[str]], output: Optional[str]):
+    """
+    Download examples from specified location.
+
+    Args:
+        type: Object type (COMPSPlatform, PythonTask, etc)
+        url: GitHub repo files url
+        output: Local folder
+
+    Returns: Files download count
+    """
+    download_github_repo(output, url, example_types=list(type))
+
+
+def download_github_repo(output, urls: List[str], example_types: List[str] = None):
+    """
+    Download github repo.
+
+    Args:
+        output: Output folder
+        urls: Urls to download
+        example_types: List of example types to download
+
+    Returns:
+        None
+    """
+    total = 0
+    if example_types:
+        urls = list(urls)
+        urls.extend(get_examples_by_types(list(example_types)))
+    urls = list(filter(None, urls)) if urls else None
+    option, file_dict = choice(urls)
+    secho(f"This is your selection: {option}", fg="bright_blue")
+    # If we decide to go ahead -> write to file
+    if click.confirm("Do you want to go ahead to download files?", default=True):
+        simplified_option, duplicated = remove_duplicated_files(option, file_dict)
+        secho(f'Removed duplicated files: {duplicated}', fg="bright_red")
+        for i in simplified_option:
+            total += download_file(i, file_dict[i], output)
+
+        secho(f"Total Files: {total}", fg="yellow")
+        secho("Download successfully!", fg="bright_green")
+    else:
+        secho("Aborted...", fg="bright_red")
+
+
+def get_examples_by_types(example_types: List[str]) -> List[str]:
+    """
+    Get examples from Plugins.
+
+    Args:
+        example_types: List of types(plugins) to pull examples from
+
+    Returns:
+        List of strings to download
+    """
+    items = get_plugins_examples()
+    result = []
+    for example in example_types:
+        if example in items:
+            result.extend(items[example])
+        else:
+            user_logger.warning(f"Cannot find example type {example}")
+    return result
+
+
+def download_file(option: int, url: str, output: str):
+    """
+    Use GitRepo utility to download files.
+
+    Args:
+        option: file index
+        url: file url
+        output: local folder to save files
+
+    Returns:
+        file count
+    """
+    # Display file information
+    click.echo(f"\nDownloading Files {option if option else ''}: '{url}'")
+    click.echo(f'Local Folder: {os.path.abspath(output)}')
+    secho('Processing...')
+
+    # Start to download files
+    gr = GitRepo()
+    total = gr.download(path=url, output_dir=output)
+    return total
+
+
+def get_plugins_examples():
+    """
+    Collect all idmtools files.
+
+    Returns:
+        files urls as dict
+
+    Notes:
+        test_examples = {
+         'TestA': 'https://github.com/dustin/py-github/tree/main/github/data',
+         'TestB': 'https://github.com/dustin/py-github/tree/main/github',
+         'TestC': 'https://github.com/dustin/py-github/tree/main/github/__init__.py',
+         'TestD': ['https://github.com/dustin/py-github/tree/main/github',
+                   'https://github.com/dustin/py-github/tree/main/github/data']
+        }
+    """
+    # Collect all idmtools examples
+    plugin_map = MasterPluginRegistry().get_plugin_map()
+
+    example_plugins = {}
+    for spec_name, plugin in plugin_map.items():
+        try:
+            plugin_url_list = plugin.get_example_urls()
+            if len(plugin_url_list) > 0:
+                example_plugins[spec_name] = plugin_url_list
+        except Exception as ex:
+            user_logger.error(ex)
+
+    return example_plugins
+
+
+def choice(urls: list = None):
+    """
+    Take urls as user selection or prompt user for file selections.
+
+    Args:
+        urls: user provided files
+
+    Returns: True/False and results (List)
+    """
+    if urls is None:
+        files = get_plugins_examples()
+
+        # Collect all files and remove duplicates
+        url_list = []
+        for exp_urls in files.values():
+            exp_urls = [exp_urls] if isinstance(exp_urls, str) else exp_urls
+            url_list.extend(list(map(str.lower, exp_urls)))
+    else:
+        url_list = urls
+
+    # Remove duplicates
+    url_list = list(set(url_list))
+    # Soring urls
+    url_list = sorted(url_list, reverse=False)
+
+    # Provide index to each file
+    file_dict = {}
+    for i in range(len(url_list)):
+        file_dict[i + 1] = url_list[i]
+
+    # Pre-view files for user to select
+    file_list = [f'    {i}. {url}' for i, url in file_dict.items()]
+    user_logger.info('File List:')
+    user_logger.info('\n'.join(file_list))
+
+    if urls:
+        # Return without user prompt for selection
+        return ['all'], file_dict
+
+    # Make sure user makes correct selection
+    choice_set = set(range(1, len(url_list) + 1))
+    choice_set.add('all')
+    while True:
+        user_input = click.prompt(
+            f"\nSelect files (multiple) for download (all or 1-{len(url_list)} separated by space)", type=str,
+            default='all',
+            prompt_suffix=f": {Fore.GREEN}")
+        valid, result = validate(user_input, choice_set)
+
+        if valid:
+            user_input = sorted(result, reverse=False)
+            break
+
+        # Else display the error message
+        secho(f'This is not correct choice: {result}', fg="bright_red")
+
+    # Return user selection and indexed files
+    return user_input, file_dict
+
+
+def validate(user_input: object, choice_set: set):
+    """
+    Validate user_input against num_set.
+
+    Args:
+        user_input: user input
+        choice_set: test against this set
+
+    Returns: True/False and result (List)
+    """
+    # Normalize user selection
+    selection = user_input.lower().strip().split(' ')
+    selection = list(filter(None, selection))
+    selection = [int(a) if a.isdigit() else a for a in selection]
+
+    # Find difference
+    extra = set(selection) - choice_set
+
+    # Return True/False along with selection details
+    if len(extra) == 0 and len(selection) > 0:
+        if 'all' in selection:
+            selection = ['all']
+        return True, selection
+    else:
+        return False, list(extra)
+
+
+def remove_duplicated_files(user_selected: list, file_dict: dict):
+    """
+    Removed duplicated files.
+
+    Args:
+        user_selected: user selection
+        file_dict: all files
+
+    Returns: simplified selection, duplicated selection
+    """
+    if 'all' in user_selected:
+        user_selected = range(1, len(file_dict) + 1)
+
+    duplicated_selection = []
+    for i in range(len(user_selected)):
+        pre = [] if i == 0 else user_selected[0:i]
+        if any([file_dict[user_selected[i]].startswith(file_dict[j]) for j in pre]):
+            duplicated_selection.append(user_selected[i])
+
+    simplified_selection = [i for i in user_selected if i not in duplicated_selection]
+    return simplified_selection, duplicated_selection