idmtools 0.0.0.dev0__py3-none-any.whl → 0.0.2__py3-none-any.whl

idmtools/frozen/frozen_utils.py

@@ -0,0 +1,179 @@
+"""
+frozen_utils provided utilities for a read-only objects.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+from enum import Enum
+from idmtools.frozen.frozen_dict import ImDict
+from idmtools.frozen.frozen_list import ImList
+from idmtools.frozen.frozen_set import ImSet
+from idmtools.frozen.frozen_tuple import ImTuple
+from idmtools.frozen.ifrozen import IFrozen
+
+
+def get_frozen_item(obj):
+    """
+    Get frozen version of item.
+
+    Args:
+        obj: object to be frozen
+
+    Returns: the transformed object
+    """
+
+    class FrozenDict(ImDict):
+        def __init__(self):
+            super().__init__()
+            for key, value in obj.items():
+                self.data[key] = frozen_transform(value)
+
+            # In case inherited from dict with customer fields
+            if hasattr(obj, '__dict__'):
+                for key, value in obj.__dict__.items():
+                    setattr(self, key, frozen_transform(value))
+
+            self._frozen = True
+
+    class FrozenList(ImList):
+        def __init__(self):
+            super().__init__()
+            for value in obj:
+                self.data.append(frozen_transform(value))
+
+            # In case inherited from list with customer fields
+            if hasattr(obj, '__dict__'):
+                for key, value in obj.__dict__.items():
+                    setattr(self, key, frozen_transform(value))
+
+            self._frozen = True
+
+    class FrozenSet(ImSet):
+        def __init__(self):
+            super().__init__()
+            for value in obj:
+                self.add(frozen_transform(value))
+
+            # In case inherited from set with customer fields
+            if hasattr(obj, '__dict__'):
+                for key, value in obj.__dict__.items():
+                    setattr(self, key, frozen_transform(value))
+
+            self._frozen = True
+
+    class FrozenTuple(ImTuple):
+        def __new__(cls):
+            o = ImTuple(obj)
+
+            # In case inherited from tuple with customer fields
+            if hasattr(obj, '__dict__'):
+                for key, value in obj.__dict__.items():
+                    setattr(o, key, frozen_transform(value))
+
+            return ImTuple(obj)
+
+    class FrozenObject(IFrozen, obj.__class__):
+        __metaclass__ = obj.__class__
+
+        def __init__(self):
+            for key, value in obj.__dict__.items():
+                setattr(self, key, frozen_transform(value))
+
+    if isinstance(obj, dict):
+        return FrozenDict()
+
+    if isinstance(obj, list):
+        return FrozenList()
+
+    if isinstance(obj, (set, frozenset)):
+        return FrozenSet()
+
+    if isinstance(obj, tuple):
+        return FrozenTuple()
+
+    return FrozenObject()
+
+
+def is_builtins_single_object(obj):
+    """
+    Is builtins single object?
+
+    Args:
+        obj: Object
+
+    Returns:
+        True if single object
+    """
+    # Handle special cases
+    if isinstance(obj, (Enum, range)):
+        return True
+
+    return type(obj).__module__ == 'builtins' and not is_builtins_collection(obj)
+
+
+def is_builtins_collection(obj):
+    """
+    Is object a builtin collection?
+
+    Args:
+        obj: Object
+
+    Returns:
+        True if a built in collection
+    """
+    from collections.abc import Collection
+    return isinstance(obj, Collection) and not isinstance(obj, (str, bytes, bytearray))  # range, generator, etc,...
+
+
+def is_user_defined_object(obj):
+    """
+    Is user defined object.
+
+    Args:
+        obj: Object to check
+
+    Returns:
+        True if item is not a builtin type
+    """
+    return type(obj).__module__ != 'builtins'
+
+
+def frozen_transform(obj=None):
+    """
+    Transform item to frozen version.
+
+    Args:
+        obj:Obj to transform
+
+    Returns:
+        Frozen version of object
+    """
+    if isinstance(obj, IFrozen):
+        obj.freeze()
+        return obj
+
+    if is_builtins_single_object(obj):
+        return obj
+
+    if isinstance(obj, dict):
+        imdict = get_frozen_item(obj)
+        return imdict
+
+    if isinstance(obj, list):
+        imlist = get_frozen_item(obj)
+        return imlist
+
+    if isinstance(obj, (set, frozenset)):
+        imset = get_frozen_item(obj)
+        return imset
+
+    if isinstance(obj, tuple):
+        imtuple = get_frozen_item(obj)
+        return imtuple
+
+    if not hasattr(obj, '__dict__'):  # say, iterator, zip, file, function, method, etc
+        return obj
+
+    obj_new = get_frozen_item(obj)
+    obj_new._frozen = True
+
+    return obj_new

idmtools/frozen/ifrozen.py

@@ -0,0 +1,66 @@
+"""
+IFrozen provided utilities for a read-only objects.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+
+
+class IFrozen:
+    """
+    Allows for frozen(read-only) items.
+    """
+    _frozen = False
+
+    def freeze(self):
+        """
+        Freeze the object.
+
+        Returns:
+            None
+        """
+        from idmtools.frozen.frozen_utils import frozen_transform
+
+        # Make sure don't do it twice
+        if self._frozen:
+            return
+
+        if hasattr(self, '__dict__'):
+            for key, value in self.__dict__.items():
+                setattr(self, key, frozen_transform(value))
+
+        self._frozen = True
+
+    def __setattr__(self, key, value):
+        """
+        Attempt to set attr. For frozen objects, this throws an error.
+
+        Args:
+            key:
+            value:
+
+        Returns:
+            None
+
+        Raises:
+            ValueError - If item is frozen
+        """
+        if self._frozen:
+            raise ValueError('Frozen')
+        super().__setattr__(key, value)
+
+    def __delattr__(self, item):
+        """
+        Delete attr replace. Frozen doesn't allow deletes.
+
+        Args:
+            item: Item to delete
+
+        Returns:
+            None
+
+        Raises:
+            ValueError - Frozen
+        """
+        if self._frozen:
+            raise ValueError('Frozen')
+        super().__delattr__(item)

idmtools/plugins/__init__.py

@@ -0,0 +1,5 @@
+"""
+idmtools plugins.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""

idmtools/plugins/git_commit.py

@@ -0,0 +1,117 @@
+"""
+Git plugin to add git repo details to items.
+"""
+import functools
+import os
+from contextlib import suppress
+from logging import getLogger
+from typing import TYPE_CHECKING, Dict
+from idmtools import IdmConfigParser
+from idmtools.assets import AssetCollection
+from idmtools.core import TRUTHY_VALUES
+from idmtools.registry.hook_specs import function_hook_impl
+
+if TYPE_CHECKING:
+    from idmtools.core.interfaces.ientity import IEntity
+
+logger = getLogger(__name__)
+user_logger = getLogger('user')
+
+
+@function_hook_impl
+def idmtools_platform_pre_create_item(item: 'IEntity', kwargs):
+    """
+    Adds git information from local repo as tags to items on creation.
+
+    There following options are valid kwargs and configuration options:
+    * add_git_tags_to_all - Add git tags to everything
+    * add_to_experiments - Add git tags to experiments
+    * add_git_tags_to_simulations - Add git tags to simulations
+    * add_git_tags_to_workitems - Add git tags to workitems
+    * add_git_tags_to_suite - Add git tags to suites
+    * add_git_tags_to_asset_collection - Add git tags to asset collections
+
+    Every option expects a truthy value, meaning "True, False, t, f, 1, 0, yes, or no. Any positive value, True, yes, 1, t, y will enable the option.
+
+    When defined in the idmtools.ini, these should be added under the "git_tag" section without the "git_tags" portion. For example
+
+    [git_tag]
+    add_to_experiments = y
+
+    Also, you can do this through environment variables using IDMTOOLS_GIT_TAG_<option>. For example, experiments would be
+
+    IDMTOOLS_GIT_TAG_ADD_TO_EXPERIMENTS
+
+    Args:
+        item: Item to add tags two
+        kwargs: Optional kwargs
+
+    Returns:
+        None
+    """
+    from idmtools.entities.experiment import Experiment
+    from idmtools.entities.simulation import Simulation
+    from idmtools.entities.iworkflow_item import IWorkflowItem
+    from idmtools.entities.suite import Suite
+    add_to_all_default = kwargs.get('add_git_tags_to_all', 'f')
+    add_to_simulations_default = kwargs.get('add_git_tags_to_simulations', 'f')
+    add_to_experiments_default = kwargs.get('add_git_tags_to_experiments', 'f')
+    add_to_workitems_default = kwargs.get('add_git_tags_to_workitems', 'f')
+    add_to_suite_default = kwargs.get('add_git_tags_to_suite', 'f')
+    add_to_asset_collection_default = kwargs.get('add_git_tags_to_asset_collection', 'f')
+    if IdmConfigParser.get_option("git_tag", "add_to_all", add_to_all_default) in TRUTHY_VALUES or \
+            (isinstance(item, Experiment) and IdmConfigParser.get_option("git_tag", "add_to_experiments", add_to_experiments_default) in TRUTHY_VALUES) or \
+            (isinstance(item, Simulation) and IdmConfigParser.get_option("git_tag", "add_to_simulations", add_to_simulations_default) in TRUTHY_VALUES) or \
+            (isinstance(item, IWorkflowItem) and IdmConfigParser.get_option("git_tag", "add_to_workitems", add_to_workitems_default) in TRUTHY_VALUES) or \
+            (isinstance(item, Suite) and IdmConfigParser.get_option("git_tag", "add_to_suite", add_to_suite_default) in TRUTHY_VALUES) or \
+            (isinstance(item, AssetCollection) and IdmConfigParser.get_option("git_tag", "add_to_asset_collection", add_to_asset_collection_default) in TRUTHY_VALUES):
+        tags = add_details_using_gitpython()
+        if not tags:
+            tags = add_details_using_pygit()
+            user_logger.warning("You need gitpython or pygit2 installed to have this functionality")
+        item.tags.update(tags)
+
+
+@functools.lru_cache(1)
+def add_details_using_gitpython():
+    """
+    Support gitpython if installed.
+
+    Returns:
+        Git tags
+    """
+    result = dict()
+    with suppress(ImportError):
+        import git
+        repo = git.Repo(search_parent_directories=True)
+        if repo:
+            sha = repo.head.object.hexsha
+            result['git_hash'] = sha
+            if len(repo.remotes) > 0:
+                result['git_url'] = repo.remotes[0].url
+            if repo.head.ref.name:
+                result['git_branch'] = repo.head.ref.name
+    return result
+
+
+@functools.lru_cache(1)
+def add_details_using_pygit() -> Dict[str, str]:
+    """
+    Support pygit if installed.
+
+    Returns:
+        Git tags
+    """
+    result = dict()
+    with suppress(ImportError):
+        import pygit2
+        repo_base = pygit2.discover_repository(os.getcwd())
+        if repo_base:
+            repo = pygit2.Repository(repo_base)
+            sha = repo.head.target.hex
+            result['git_hash'] = sha
+            if len(repo.remotes) > 0:
+                result['git_url'] = repo.remotes[0].url
+            if repo.head.shorthand:
+                result['git_branch'] = repo.head.shorthand
+    return result

idmtools/registry/__init__.py

@@ -0,0 +1,4 @@
+# flake8: noqa F821
+from idmtools.registry.plugin_specification import PluginSpecification
+
+

idmtools/registry/experiment_specification.py

@@ -0,0 +1,105 @@
+"""
+ExperimentPluginSpecification provided definition for the experiment plugin specification, hooks, and plugin manager.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+# Define our model specific specifications
+import typing
+from abc import ABC
+from logging import getLogger
+
+import pluggy
+
+from idmtools.registry import PluginSpecification
+from idmtools.registry.plugin_specification import PLUGIN_REFERENCE_NAME
+from idmtools.registry.utils import load_plugin_map
+
+example_configuration_spec = pluggy.HookspecMarker(PLUGIN_REFERENCE_NAME)
+get_model_spec = pluggy.HookspecMarker(PLUGIN_REFERENCE_NAME)
+get_model_type_spec = pluggy.HookspecMarker(PLUGIN_REFERENCE_NAME)
+example_configuration_impl = pluggy.HookimplMarker(PLUGIN_REFERENCE_NAME)
+get_model_impl = pluggy.HookimplMarker(PLUGIN_REFERENCE_NAME)
+get_model_type_impl = pluggy.HookimplMarker(PLUGIN_REFERENCE_NAME)
+logger = getLogger(__name__)
+
+
+class ExperimentPluginSpecification(PluginSpecification, ABC):
+    """
+    ExperimentPluginSpecification defines the specification for Experiment plugins.
+    """
+
+    @classmethod
+    def get_name(cls, strip_all: bool = True) -> str:
+        """
+        Get name of plugin. By default we remove the PlatformSpecification portion.
+
+        Args:
+            strip_all: When true, ExperimentPluginSpecification and ExperimentPluginSpec is stripped from name.
+            When false only  Specification and Spec is Stripped
+
+        Returns:
+            Name of plugin
+        """
+        if strip_all:
+            ret = cls.__name__.replace('ExperimentPluginSpecification', '').replace('Specification', '')\
+                .replace("ExperimentPluginSpec", '').replace('Spec', '')
+        else:
+            ret = cls.__name__.replace('Specification', '').replace('Spec', '')
+        return ret
+
+    @get_model_spec
+    def get(self, configuration: dict) -> 'Experiment':  # noqa: F821
+        """
+        Return a new model using the passed in configuration.
+
+        Args:
+            configuration: The INI configuration file to use.
+
+        Returns:
+            The new model.
+        """
+        raise NotImplementedError("Plugin did not implement get")
+
+    @get_model_type_spec
+    def get_type(self) -> typing.Type['Experiment']:  # noqa: F821
+        """
+        Get Experiment type.
+
+        Returns:
+            Experiment type.
+        """
+        pass
+
+
+class ExperimentPlugins:
+    """
+    ExperimentPlugins acts as registry for Experiment plugins.
+    """
+
+    def __init__(self, strip_all: bool = True) -> None:
+        """
+        Initialize the Experiment Registry. When strip all is false, the full plugin name will be used for names in map.
+
+        Args:
+            strip_all: Whether to strip common parts of name from plugins in plugin map
+        """
+        self._plugins = typing.cast(typing.Dict[str, ExperimentPluginSpecification],
+                                    load_plugin_map('idmtools_experiment', ExperimentPluginSpecification, strip_all))
+
+    def get_plugins(self) -> typing.Set[ExperimentPluginSpecification]:
+        """
+        Get plugins.
+
+        Returns:
+            Experiment plugins.
+        """
+        return set(self._plugins.values())
+
+    def get_plugin_map(self) -> typing.Dict[str, ExperimentPluginSpecification]:
+        """
+        Get experiment plugin map.
+
+        Returns:
+            Experiment plugin map.
+        """
+        return self._plugins

idmtools/registry/functions.py

@@ -0,0 +1,28 @@
+"""
+FunctionPluginManager provided definition for the function plugin specification, hooks, and plugin manager.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+from logging import getLogger, DEBUG
+
+from pluggy import PluginManager
+import idmtools.registry.hook_specs as hookspecs
+from idmtools.utils.decorators import SingletonMixin
+
+logger = getLogger(__name__)
+
+
+class FunctionPluginManager(PluginManager, SingletonMixin):
+    """
+    FunctionPluginManager acts as registry for function based plugins.
+    """
+
+    def __init__(self):
+        """
+        Initialize function plugin manager.
+        """
+        super(FunctionPluginManager, self).__init__(hookspecs.IDMTOOLS_HOOKS)
+        self.add_hookspecs(hookspecs)
+        self.load_setuptools_entrypoints(hookspecs.IDMTOOLS_HOOKS.lower())
+        if logger.isEnabledFor(DEBUG):
+            logger.debug(self.get_plugins())

idmtools/registry/hook_specs.py

@@ -0,0 +1,132 @@
+"""
+Define a list of function only hook specs. Useful for simple plugins.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+from logging import getLogger
+from typing import TYPE_CHECKING
+from pluggy import HookspecMarker, HookimplMarker
+
+if TYPE_CHECKING:
+    from idmtools.core.interfaces.ientity import IEntity
+    from idmtools.core.interfaces.irunnable_entity import IRunnableEntity
+logger = getLogger(__name__)
+user_logger = getLogger('user')
+
+IDMTOOLS_HOOKS = 'IDMTOOLS_HOOKS'
+
+function_hook_spec = HookspecMarker(IDMTOOLS_HOOKS)
+function_hook_impl = HookimplMarker(IDMTOOLS_HOOKS)
+
+
+@function_hook_spec
+def idmtools_platform_pre_create_item(item: 'IEntity', kwargs) -> 'IEntity':
+    """
+    This callback is called by the pre_create of each object type on a platform. An item can be a suite, workitem, simulation, asset collection or an experiment.
+
+    Args:
+        item:
+        kwargs: extra args
+
+    Returns:
+        None
+    """
+    pass
+
+
+@function_hook_spec
+def idmtools_platform_post_create_item(item: 'IEntity', kwargs) -> 'IEntity':
+    """
+    This callback is called by the post_create of each object type on a platform.
+    An item can be a suite, workitem, simulation, asset collection or an experiment.
+
+    Args:
+        item:
+        kwargs: extra args
+
+    Returns:
+        None
+    """
+    pass
+
+
+@function_hook_spec
+def idmtools_platform_post_run(item: 'IEntity', kwargs) -> 'IEntity':
+    """
+    This is called when item finishes calling the run on the server.
+
+    Args:
+        item:
+        kwargs: extra args
+
+    Returns:
+        None
+    """
+    pass
+
+
+@function_hook_spec
+def idmtools_on_start():
+    """
+    Execute on startup when idmtools is first imported.
+
+    Returns:
+        None
+    """
+    pass
+
+
+@function_hook_spec
+def idmtools_generate_id(item: 'IEntity') -> str:
+    """
+    Generates an id for an IItem.
+
+    Returns:
+        None
+    """
+    pass
+
+
+@function_hook_spec
+def idmtools_runnable_on_done(item: 'IRunnableEntity', **kwargs):
+    """
+    Called when a runnable item finishes when it was being monitored.
+
+    Args:
+        item: Item that was running
+        **kwargs:
+
+    Returns:
+        None
+    """
+    pass
+
+
+@function_hook_spec
+def idmtools_runnable_on_succeeded(item: 'IRunnableEntity', **kwargs):
+    """
+    Executed when a runnable item succeeds.
+
+    Args:
+        item: Item that was running
+        **kwargs:
+
+    Returns:
+        None
+    """
+    pass
+
+
+@function_hook_spec
+def idmtools_runnable_on_failure(item: 'IRunnableEntity', **kwargs):
+    """
+    Executed when a runnable item fails.
+
+    Args:
+        item: Item that was running
+        **kwargs:
+
+    Returns:
+        None
+    """
+    pass

idmtools/registry/master_plugin_registry.py

@@ -0,0 +1,51 @@
+"""
+Registry to aggregate all plugins to one place.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+from typing import Dict, Set
+from idmtools.registry import PluginSpecification
+from idmtools.registry.experiment_specification import ExperimentPlugins
+from idmtools.utils.decorators import SingletonMixin
+
+
+class MasterPluginRegistry(SingletonMixin):
+    """
+    MasterPluginRegistry indexes all type of plugins into one class.
+
+    Notes:
+        TODO - Rename this class
+    """
+
+    def __init__(self) -> None:
+        """
+        Initialize Master registry.
+        """
+        from idmtools.registry.platform_specification import PlatformPlugins
+        from idmtools.registry.task_specification import TaskPlugins
+        self._plugin_map = PlatformPlugins(strip_all=False).get_plugin_map()
+        self._plugin_map.update(TaskPlugins(strip_all=False).get_plugin_map())
+        self._plugin_map.update(ExperimentPlugins(strip_all=False).get_plugin_map())
+        try:
+            from idmtools_cli.iplatform_cli import PlatformCLIPlugins
+            self._plugin_map.update(PlatformCLIPlugins(strip_all=False).get_plugin_map())
+        except ImportError:
+            pass
+
+    def get_plugin_map(self) -> Dict[str, PluginSpecification]:
+        """
+        Get plugin map.
+
+        Returns:
+            Plugin map
+        """
+        return self._plugin_map
+
+    def get_plugins(self) -> Set[PluginSpecification]:
+        """
+        Get Plugins map.
+
+        Returns:
+            The full plugin map
+        """
+        return set(self._plugin_map.values())