idmtools 0.0.0.dev0__py3-none-any.whl → 0.0.2__py3-none-any.whl

idmtools/registry/platform_specification.py

@@ -0,0 +1,138 @@
+"""
+PlatformSpecification provided definition for the platform plugin specification, hooks, and plugin manager.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+# Define our platform specific specifications
+import typing
+from abc import ABC
+from logging import getLogger, DEBUG
+import pluggy
+from idmtools.registry import PluginSpecification
+from idmtools.registry.plugin_specification import PLUGIN_REFERENCE_NAME
+from idmtools.registry.utils import load_plugin_map
+from idmtools.utils.decorators import SingletonMixin
+
+if typing.TYPE_CHECKING:
+    from idmtools.entities.iplatform import IPlatform
+example_configuration_spec = pluggy.HookspecMarker(PLUGIN_REFERENCE_NAME)
+get_platform_spec = pluggy.HookspecMarker(PLUGIN_REFERENCE_NAME)
+get_platform_type_spec = pluggy.HookspecMarker(PLUGIN_REFERENCE_NAME)
+example_configuration_impl = pluggy.HookimplMarker(PLUGIN_REFERENCE_NAME)
+get_platform_impl = pluggy.HookimplMarker(PLUGIN_REFERENCE_NAME)
+get_platform_type_impl = pluggy.HookimplMarker(PLUGIN_REFERENCE_NAME)
+logger = getLogger(__name__)
+
+
+class PlatformSpecification(PluginSpecification, ABC):
+    """
+    PlatformSpecification for Platform Plugins.
+    """
+
+    @classmethod
+    def get_name(cls, strip_all: bool = True) -> str:
+        """
+        Get name of plugin. By default we remove the PlatformSpecification portion.
+
+        Args:
+            strip_all: When true, PlatformSpecification is stripped from name. When false only Specification is Stripped
+
+        Returns:
+            Name of plugin
+        """
+        if strip_all:
+            ret = cls.__name__.replace("PlatformSpecification", '')
+        else:
+            ret = cls.__name__.replace('Specification', '')
+        return ret
+
+    @example_configuration_spec
+    def example_configuration(self):
+        """
+        Example configuration for the platform. This is useful in help or error messages.
+
+        Returns:
+            Example configuration
+        """
+        raise NotImplementedError("Plugin did not implement example_configuration")
+
+    @get_platform_spec
+    def get(self, configuration: dict) -> 'IPlatform':
+        """
+        Return a new platform using the passed in configuration.
+
+        Args:
+            configuration: The INI configuration file to use.
+
+        Returns:
+            The new platform.
+        """
+        raise NotImplementedError("Plugin did not implement get")
+
+    @get_platform_type_spec
+    def get_type(self) -> typing.Type['IPlatform']:
+        """
+        Get type of the Platform type.
+        """
+        pass
+
+    def get_configuration_aliases(self) -> typing.Dict[str, typing.Dict]:
+        """
+        Get a list of configuration aliases for the platform.
+
+        A configuration alias should be in the form of "name" -> "Spec, Config Options Dict" where name is the alias the user will use, and the config options is a dictionary of config options to be passed to the item
+        Returns:
+
+        """
+        return {}
+
+
+class PlatformPlugins(SingletonMixin):
+    """
+    PlatformPlugin registry.
+    """
+
+    def __init__(self, strip_all: bool = True) -> None:
+        """
+        Initialize the Platform Registry. When strip all is false, the full plugin name will be used for names in map.
+
+        Args:
+            strip_all: Whether to strip common parts of name from plugins in plugin map
+        """
+        self._plugins = typing.cast(typing.Dict[str, PlatformSpecification],
+                                    load_plugin_map('idmtools_platform', PlatformSpecification, strip_all))
+        self._aliases: typing.Dict[str, typing.Tuple[PlatformSpecification, typing.Dict]] = dict()
+        for _name, spec in self._plugins.items():
+            for alias, details in spec.get_configuration_aliases().items():
+                if alias.upper() in self._aliases or alias.upper() in self._plugins:
+                    logger.debug(f"Conflicting alias found: {alias.upper()} from {spec.get_name()}")
+                if logger.isEnabledFor(DEBUG):
+                    logger.debug(f"Found Platform Configuration Alias: {alias}")
+                self._aliases[alias.upper()] = (spec, details)
+
+    def get_plugins(self) -> typing.Set[PlatformSpecification]:
+        """
+        Get platform plugins.
+
+        Returns:
+            Platform plugins
+        """
+        return set(self._plugins.values())
+
+    def get_aliases(self) -> typing.Dict[str, typing.Tuple[PlatformSpecification, typing.Dict]]:
+        """
+        Get Platform Configuration Aliases for Platform Plugin.
+
+        Returns:
+            Platform CConfiguration Aliases
+        """
+        return self._aliases
+
+    def get_plugin_map(self) -> typing.Dict[str, PlatformSpecification]:
+        """
+        Get plugin map for Platform Plugins.
+
+        Returns:
+            Plugin map
+        """
+        return self._plugins

idmtools/registry/plugin_specification.py

@@ -0,0 +1,129 @@
+"""
+Defines our base plugin definition and specifications.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+from dataclasses import dataclass
+from logging import getLogger
+import pluggy
+from typing import List, Union, Dict
+
+PLUGIN_REFERENCE_NAME = 'idmtools_plugins'
+get_description_spec = pluggy.HookspecMarker(PLUGIN_REFERENCE_NAME)
+get_description_impl = pluggy.HookimplMarker(PLUGIN_REFERENCE_NAME)
+logger = getLogger(__name__)
+
+
+@dataclass
+class ProjectTemplate:
+    """
+    Defines a ProjectTemplate that plugins can define.
+    """
+    name: str
+    url: Union[str, List[str]]
+    description: str = None
+    info: str = None
+
+    @staticmethod
+    def read_templates_from_json_stream(s) -> List['ProjectTemplate']:
+        """
+        Read Project Template from stream.
+
+        Args:
+            s: Stream where json data resides
+
+        Returns:
+            Templates loaded from json
+        """
+        import json
+        data = json.loads(s.read().decode())
+        ret = list()
+        if isinstance(data, list):
+            for item in data:
+                ret.append(ProjectTemplate(**item))
+        else:
+            ret.append(ProjectTemplate(**data))
+        return ret
+
+
+class PluginSpecification:
+    """
+    Base class for all plugins.
+    """
+
+    @classmethod
+    def get_name(cls, strip_all: bool = True) -> str:
+        """
+        Get the name of the plugin. Although it can be overridden, the best practice is to use the class name as the plugin name.
+
+        Returns:
+            The name of the plugin as a string.
+        """
+        if strip_all:
+            return cls.__name__.replace("Specification", "")
+        else:
+            return cls.__name__
+
+    @get_description_spec
+    def get_description(self) -> str:
+        """
+        Get a brief description of the plugin and its functionality.
+
+        Returns:
+            The plugin description.
+        """
+        raise NotImplementedError("The plugin did not implement a description!")
+
+    def get_project_templates(self) -> List[ProjectTemplate]:
+        """
+        Returns a list of project templates related to the a plugin.
+
+        Returns:
+            List of project templates
+        """
+        return list()
+
+    def get_example_urls(self) -> List[str]:
+        """
+        Returns a list of URLs that a series of Examples for plugin can be downloaded from.
+
+        Returns:
+            List of urls
+        """
+        return list()
+
+    def get_help_urls(self) -> Dict[str, str]:
+        """
+        Returns a dictionary of topics and links to help.
+
+        Returns:
+            Dict of help urls
+        """
+        return dict()
+
+    @staticmethod
+    def get_version_url(version: str, extra: str = None,
+                        repo_base_url: str = 'https://github.com/InstituteforDiseaseModeling/idmtools/tree/',
+                        nightly_branch: str = 'dev'):
+        """
+        Build a url using version.
+
+        Here we assume the tag will exist for that specific version
+        Args:
+            version: Version to look up. If it contains nightly, we default to nightly_branch
+            extra: Extra parts of url pass base
+            repo_base_url: Optional url
+            nightly_branch: default to dev
+        Returns:
+            URL for item
+        """
+        return f'{repo_base_url}{nightly_branch if "nightly" in version else version[0:6]}/{extra}'
+
+    def get_version(self) -> str:
+        """
+        Returns the version of the plugin.
+
+        Returns:
+            Version for the plugin
+        """
+        return None

idmtools/registry/task_specification.py

@@ -0,0 +1,104 @@
+"""
+TaskSpecification provided definition for the experiment plugin specification, hooks, and plugin manager.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+# Define our model specific specifications
+import typing
+from abc import ABC
+from logging import getLogger
+import pluggy
+from idmtools.entities.itask import ITask
+from idmtools.registry import PluginSpecification
+from idmtools.registry.plugin_specification import PLUGIN_REFERENCE_NAME
+from idmtools.registry.utils import load_plugin_map
+from idmtools.utils.decorators import SingletonMixin
+
+example_configuration_spec = pluggy.HookspecMarker(PLUGIN_REFERENCE_NAME)
+get_task_spec = pluggy.HookspecMarker(PLUGIN_REFERENCE_NAME)
+get_task_type_spec = pluggy.HookspecMarker(PLUGIN_REFERENCE_NAME)
+example_configuration_impl = pluggy.HookimplMarker(PLUGIN_REFERENCE_NAME)
+get_task_impl = pluggy.HookimplMarker(PLUGIN_REFERENCE_NAME)
+get_task_type_impl = pluggy.HookimplMarker(PLUGIN_REFERENCE_NAME)
+logger = getLogger(__name__)
+
+
+class TaskSpecification(PluginSpecification, ABC):
+    """
+    TaskSpecification is spec for Task plugins.
+    """
+
+    @classmethod
+    def get_name(cls, strip_all: bool = True) -> str:
+        """
+        Get name of plugin. By default we remove the PlatformSpecification portion.
+
+        Args:
+            strip_all: When true, TaskSpecification and TaskSpec is stripped from name. When false only
+            Specification and Spec is Stripped
+
+        Returns:
+            Name of plugin
+        """
+        if strip_all:
+            ret = cls.__name__.replace('TaskSpecification', '').replace("TaskSpec", '').replace('Spec', '')
+        else:
+            ret = cls.__name__.replace('Specification', '').replace('Spec', '')
+        return ret
+
+    @get_task_spec
+    def get(self, configuration: dict) -> 'ITask':  # noqa: F821
+        """
+        Return a new model using the passed in configuration.
+
+        Args:
+            configuration: The INI configuration file to use.
+
+        Returns:
+            The new model.
+        """
+        raise NotImplementedError("Plugin did not implement get")
+
+    @get_task_type_spec
+    def get_type(self) -> typing.Type['ITask']:  # noqa: F821
+        """
+        Get task type.
+
+        Returns:
+            Task type
+        """
+        pass
+
+
+class TaskPlugins(SingletonMixin):
+    """
+    TaskPlugins acts as a registry for Task Plugins.
+    """
+
+    def __init__(self, strip_all: bool = True) -> None:
+        """
+        Initialize the Task Registry. When strip all is false, the full plugin name will be used for names in map.
+
+        Args:
+            strip_all: Whether to strip common parts of name from plugins in plugin map
+        """
+        self._plugins = typing.cast(typing.Dict[str, TaskSpecification],
+                                    load_plugin_map('idmtools_task', TaskSpecification, strip_all))
+
+    def get_plugins(self) -> typing.Set[TaskSpecification]:
+        """
+        Get plugins for Tasks.
+
+        Returns:
+            Plugins
+        """
+        return set(self._plugins.values())
+
+    def get_plugin_map(self) -> typing.Dict[str, TaskSpecification]:
+        """
+        Get a map of task plugins.
+
+        Returns:
+            Task plugin map
+        """
+        return self._plugins

idmtools/registry/utils.py

@@ -0,0 +1,119 @@
+"""
+Provides utilities for plugins.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+import functools
+import inspect
+import logging
+from logging import DEBUG, getLogger
+from typing import Type, List, Any, Set, Dict
+import pluggy
+from idmtools.registry import PluginSpecification
+from idmtools.registry.plugin_specification import PLUGIN_REFERENCE_NAME
+
+
+logger = getLogger(__name__)
+user_logger = getLogger('user')
+
+
+def is_a_plugin_of_type(value, plugin_specification: Type[PluginSpecification]) -> bool:
+    """
+    Determine if a value of a plugin specification is of type :class:`~idmtools.registry.plugin_specification.PluginSpecification`.
+
+    Args:
+        value: The value to inspect.
+        plugin_specification: Plugin specification to check against.
+
+    Returns:
+        A Boolean indicating True if the plugin is of a subclass of :class:`~idmtools.registry.plugin_specification.PluginSpecification`,
+        else False.
+    """
+    return inspect.isclass(value) and issubclass(value, plugin_specification) \
+        and not inspect.isabstract(value) and value is not plugin_specification
+
+
+def load_plugin_map(entrypoint: str, spec_type: Type[PluginSpecification], strip_all: bool = True) -> Dict[str, Type[PluginSpecification]]:
+    """
+    Load plugins from entry point with the indicated type of specification into a map.
+
+    .. warning::
+
+        This could cause name collisions if plugins of the same name are installed.
+
+    Args:
+        entrypoint: The name of the entry point.
+        spec_type: The type of plugin specification.
+        strip_all: Pass through for get_name from Plugins. Changes names in plugin registries
+
+    Returns:
+        (Dict[str, Type[PluginSpecification]]): Returns a dictionary of name and :class:`~idmtools.registry.plugin_specification.PluginSpecification`.
+    """
+    plugins = plugins_loader(entrypoint, spec_type)
+    # create instances of the plugins
+    _plugin_map = dict()
+    for plugin in plugins:
+        if logger.isEnabledFor(logging.DEBUG):
+            logger.debug(f"Loading {str(plugin)} as {plugin.get_name()}")
+        try:
+            _plugin_map[plugin.get_name(strip_all)] = plugin()
+        except Exception as e:
+            logger.exception(e)
+            user_logger.error(f'Problem loading plugin: {plugin.get_name()}')
+    return _plugin_map
+
+
+def plugins_loader(entry_points_name: str, plugin_specification: Type[PluginSpecification]) -> Set[PluginSpecification]:
+    """
+    Loads all the plugins of type :class:`~idmtools.registry.plugin_specification.PluginSpecification` from entry point name.
+
+    |IT_s| also supports loading plugins through a list of strings representing the paths to modules containing plugins.
+
+    Args:
+        entry_points_name: Entry point name for plugins.
+        plugin_specification: Plugin specification to load.
+
+    Returns:
+        (Set[PluginSpecification]): All the plugins of the type indicated.
+    """
+    manager = pluggy.PluginManager(PLUGIN_REFERENCE_NAME)
+    manager.add_hookspecs(plugin_specification)
+    manager.load_setuptools_entrypoints(entry_points_name)
+
+    manager.check_pending()
+    return manager.get_plugins()
+
+
+@functools.lru_cache(maxsize=32)
+def discover_plugins_from(library: Any, plugin_specification: Type[PluginSpecification]) -> \
+        List[Type[PluginSpecification]]:
+    """
+    Search a library object for plugins of type :class:`~idmtools.registry.plugin_specification.PluginSpecification`.
+
+    Currently it detects module and classes. In the future support for strings will be added.
+
+    Args:
+        library: Library object to discover plugins from.
+        plugin_specification: Specification to search for.
+
+    Returns:
+        List[Type[PluginSpecification]]: List of plugins.
+    """
+    plugins = []
+    # check if the item is a module
+    if inspect.ismodule(library):
+        if logger.isEnabledFor(DEBUG):
+            logger.debug('Attempting to load library as a module: %s', library.__name__)
+        for k, v in library.__dict__.items():
+            if k[:2] != '__' and is_a_plugin_of_type(v, plugin_specification):
+                if logger.isEnabledFor(DEBUG):
+                    logger.debug('Adding class %s from %s as a plugin', v.__name__, library.__name__)
+                plugins.append(v)
+    # or maybe a plugin object
+    elif is_a_plugin_of_type(library, plugin_specification):
+        if logger.isEnabledFor(DEBUG):
+            logger.debug('Adding class %s as a plugin', library.__name__)
+        plugins.append(library)
+    else:
+        logger.warn('Could not determine the the type of library specified by %s', str(library))
+    return plugins

idmtools/services/__init__.py

@@ -0,0 +1,5 @@
+"""
+Internal services for idmtools.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""

idmtools/services/ipersistance_service.py

@@ -0,0 +1,135 @@
+"""
+IPersistenceService allows caching of items locally into a diskcache db that does not expire upon deletion.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+import os
+import logging
+import time
+from pathlib import Path
+from multiprocessing import cpu_count
+
+import diskcache
+from abc import ABCMeta
+
+from idmtools.core import IDMTOOLS_USER_HOME
+
+logger = logging.getLogger(__name__)
+
+
+class IPersistenceService(metaclass=ABCMeta):
+    """
+    IPersistenceService provides a persistent cache. This is useful for network heavy operations.
+    """
+    cache_directory = None
+    cache_name = None
+
+    @classmethod
+    def _open_cache(cls):
+        """
+        Open cache.
+
+        Returns:
+            None
+        """
+        import sqlite3
+        from idmtools import IdmConfigParser
+        cls.cache_directory = Path(
+            IdmConfigParser.get_option(option="cache_directory", fallback=IDMTOOLS_USER_HOME.joinpath("cache")))
+
+        # the more the cpus, the more likely we are to encounter a scaling issue. Let's try to scale with that up to
+        # one second. above one second, we are introducing to much lag in processes
+        default_timeout = min(max(0.25, cpu_count() * 0.025 * 2), 2)
+        retries = 0
+        while retries < 5:
+
+            try:
+                os.makedirs(cls.cache_directory, exist_ok=True)
+                cache = diskcache.FanoutCache(os.path.join(str(cls.cache_directory), 'disk_cache', cls.cache_name),
+                                              timeout=default_timeout, shards=cpu_count() * 2)
+                return cache
+            except (sqlite3.OperationalError, FileNotFoundError):
+                retries += 1
+                time.sleep(0.1)
+
+    @classmethod
+    def retrieve(cls, uid):
+        """
+        Retrieve item with id <uid> from cache.
+
+        Args:
+            uid: Id to fetch
+
+        Returns:
+            Item loaded from cache
+        """
+        with cls._open_cache() as cache:
+            obj = cache.get(uid, retry=True)
+            return obj
+
+    @classmethod
+    def save(cls, obj):
+        """
+        Save an item to our cache.
+
+        Args:
+            obj: Object to save.
+
+        Returns:
+            Object uid
+        """
+        with cls._open_cache() as cache:
+            if logger.isEnabledFor(logging.DEBUG):
+                logging.debug('Saving %s to %s', obj.uid, cls.cache_name)
+            cache.set(obj.uid, obj, retry=True)
+
+        return obj.uid
+
+    @classmethod
+    def delete(cls, uid):
+        """
+        Delete at item from our cache with id <uid>.
+
+        Args:
+            uid: Id to delete
+
+        Returns:
+            None
+        """
+        with cls._open_cache() as cache:
+            cache.delete(uid, retry=True)
+
+    @classmethod
+    def clear(cls):
+        """
+        Clear our cache.
+
+        Returns:
+            None
+        """
+        with cls._open_cache() as cache:
+            cache.clear(retry=True)
+
+    @classmethod
+    def list(cls):
+        """
+        List items in our cache.
+
+        Returns:
+            List of items in our cache
+        """
+        with cls._open_cache() as cache:
+            _list = list(cache)
+            return _list
+
+    @classmethod
+    def length(cls):
+        """
+        Total length of our persistence cache.
+
+        Returns:
+            Count of our cache
+        """
+        with cls._open_cache() as cache:
+            _len = len(cache)
+            return _len

idmtools/services/platforms.py

@@ -0,0 +1,13 @@
+"""
+PlatformPersistService provides cache for platforms.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+from idmtools.services.ipersistance_service import IPersistenceService
+
+
+class PlatformPersistService(IPersistenceService):
+    """
+    Provide a cache for our platforms.
+    """
+    cache_name = "platforms"

idmtools/utils/__init__.py

@@ -0,0 +1,5 @@
+"""
+root of utilities for idmtools.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""

idmtools/utils/caller.py

@@ -0,0 +1,24 @@
+"""
+Defines our IAnalyzer interface used as base of all other analyzers.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+
+
+def get_caller():
+    """
+    Trace the stack and find the caller.
+
+    Returns:
+        The direct caller.
+    """
+    import inspect
+
+    try:
+        s = inspect.stack()
+    except (IndexError, RuntimeError):
+        # in some high thread environments and under heavy load, we can get environment changes before retrieving
+        # stack in those case assume we are good
+        # We can also encounter IndexError in dynamic environments like Snakemake, jinja, etc
+        return "__newobj__"
+    return s[2][3]