idmtools 0.0.0.dev0__py3-none-any.whl → 0.0.3__py3-none-any.whl

idmtools/config/idm_config_parser.py

@@ -0,0 +1,486 @@
+"""
+idmtools IdmConfig paraer, the main configuration engine for idmtools.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+import copy
+import platform
+from dataclasses import fields
+from pathlib import Path
+import json
+import os
+from configparser import ConfigParser
+from logging import getLogger, DEBUG
+from typing import Any, Dict
+from idmtools.core import TRUTHY_VALUES
+from idmtools.utils.info import get_help_version_url
+
+default_config = 'idmtools.ini'
+
+# this is the only logger that should not be defined using init_logger
+logger = getLogger(__name__)
+user_logger = getLogger('user')
+
+
+def initialization(force=False):
+    """
+    Initialization decorator for configuration methods.
+
+    Args:
+        force: Force initialization
+
+    Returns:
+        Wrapper function
+    """
+
+    def wrap(func):
+        def wrapped_f(*args, **kwargs):
+            IdmConfigParser.ensure_init(force=force)
+            value = func(*args, **kwargs)
+            return value
+
+        return wrapped_f
+
+    return wrap
+
+
+class IdmConfigParser:
+    """
+    Class that parses an INI configuration file.
+    """
+    _config = None
+    _instance = None
+    _config_path = None
+    _block = None
+
+    def __new__(cls, dir_path: str = '.', file_name: str = default_config) -> 'IdmConfigParser':
+        """
+        Make :class:`IdmConfigParser` creation a singleton.
+
+        Args:
+            dir_path: The INI configuration file directory.
+            file_name: The INI file name.
+
+        Returns:
+            An :class:`IdmConfigParser` instance.
+        """
+        if not cls._instance:
+            cls._instance = super(IdmConfigParser, cls).__new__(cls)
+            cls._instance._load_config_file(dir_path, file_name)
+            # Only error when a user overrides the filename for idmtools.ini
+            if (dir_path != "." or file_name != default_config) and not cls.found_ini():
+                raise FileNotFoundError(f"The configuration file {os.path.join(dir_path, file_name)} was not found!")
+            # Call our startup plugins
+            from idmtools.registry.functions import FunctionPluginManager
+            FunctionPluginManager.instance().hook.idmtools_on_start()
+        return cls._instance
+
+    @classmethod
+    def retrieve_dict_config_block(cls, field_type, section) -> Dict[str, Any]:
+        """
+        Retrieve dictionary config block.
+
+        Args:
+            field_type: Field type
+            section: Section to load
+
+        Returns:
+            Dictionary of the config block
+        """
+        import ast
+
+        inputs = copy.deepcopy(section)
+        fs = set(field_type.keys()).intersection(set(section.keys()))
+        for fn in fs:
+            ft = field_type[fn]
+            try:
+                if ft in (int, float, str):
+                    inputs[fn] = ft(section[fn])
+                elif ft is bool:
+                    if isinstance(section[fn], str):
+                        inputs[fn] = ast.literal_eval(section[fn])
+                    else:
+                        inputs[fn] = section[fn]
+            except ValueError as e:
+                user_logger.error(
+                    f"The field {fn} requires a value of type {ft.__name__}. You provided <{section[fn]}>")
+                raise e
+        return inputs
+
+    @classmethod
+    @initialization
+    def retrieve_settings(cls, section: str = None, field_type=None) -> Dict[str, str]:
+        """
+        Retrieve INI configuration values (to be used when updating platform fields). Call from each platform.
+
+        Args:
+            section: The INI section from which to retrieve configuration values.
+            field_type: The requested data types.
+
+        Returns:
+            The configuration values as a dictionary.
+        """
+        # retrieve THIS platform config settings
+        if field_type is None:
+            field_type = {}
+        field_config = cls.get_section(section)
+
+        # update field types
+        field_config_updated = cls.retrieve_dict_config_block(field_config, section)
+        return field_config_updated
+
+    @classmethod
+    def _find_config(cls, dir_path: str = None, file_name: str = default_config) -> None:
+        """
+        Recursively search for the INI configuration file starting from the **dir_path** provided up to the root, stopping once one is found.
+
+        Args:
+            dir_path: The directory to start looking for the INI configuration file.
+            file_name: The name of the configuration file to look for.
+
+        Returns:
+            None
+        """
+        full_dir_path = os.path.abspath(dir_path)
+        if os.path.exists(os.path.join(full_dir_path, file_name)):
+            cls._config_path = os.path.join(full_dir_path, file_name)
+            return cls._config_path
+        else:
+            dir_parent = os.path.dirname(full_dir_path)
+            if dir_parent == full_dir_path:
+                return None
+            else:
+                cls._config_path = cls._find_config(dir_parent, file_name)
+                return cls._config_path
+
+    @staticmethod
+    def get_global_configuration_name() -> str:
+        r"""
+        Get Global Configuration Name.
+
+        Returns:
+            On Windows, this returns %LOCALDATA%\\idmtools\\idmtools.ini
+            On Mac and Linux, it returns "/home/username/.idmtools.ini'
+
+        Raises:
+            Value Error on OSs not supported
+        """
+        if platform.system() in ["Linux", "Darwin"]:
+            ini_file = os.path.join(str(Path.home()), ".idmtools.ini")
+        # On Windows, c:\users\user\AppData\Local\idmtools\idmtools.ini
+        elif platform.system() in ["Windows"]:
+            ini_file = os.path.join(os.path.expandvars(r'%LOCALAPPDATA%'), "idmtools", "idmtools.ini")
+        else:
+            raise ValueError("OS global configuration cannot be detected")
+        return ini_file
+
+    @classmethod
+    def _load_config_file(cls, dir_path: str = None, file_name: str = default_config):
+        """
+        Find and then load the INI configuration file and parse it with :class:`IdmConfigParser`.
+
+        Args:
+            dir_path: The directory to start looking for the INI configuration file.
+            file_name: The name of the configuration file to look for.
+
+        Returns:
+            None
+        """
+        if dir_path is None:
+            dir_path = os.getcwd()
+
+        logger.debug(
+            f"Looking for config file in {dir_path}")  # This log will generally only happen on recreation of config after clearing config
+
+        # Look for the config file. First check environment vars
+        if "IDMTOOLS_CONFIG_FILE" in os.environ:
+            if not os.path.exists(os.environ["IDMTOOLS_CONFIG_FILE"]):
+                raise FileNotFoundError(f'Cannot for idmtools config at {os.environ["IDMTOOLS_CONFIG_FILE"]}')
+            ini_file = os.environ["IDMTOOLS_CONFIG_FILE"]
+        # Try find file
+        else:
+            ini_file = cls._find_config(dir_path, file_name)
+            # Fallback to user home directories
+            if ini_file is None:
+                global_config = cls.get_global_configuration_name()
+                if os.path.exists(global_config):
+                    ini_file = global_config
+
+        # If we didn't find a file, warn the user and init logging
+        if ini_file is None:
+            if os.getenv("NO_LOGGING_INIT", "f").lower() not in TRUTHY_VALUES:
+                cls._init_logging()
+            logger.warning(
+                f"/!\\ WARNING: File '{file_name}' Not Found! For details on how to configure idmtools, see {get_help_version_url('configuration.html')} for details on how to configure idmtools.")
+            return
+
+        # Load file
+        cls._config_path = ini_file
+        cls._config = ConfigParser()
+        cls._config.read(ini_file)
+
+        # in order to have case-insensitive section names, we add the lowercase version of all sections if not present
+        sections = cls._config.sections()
+        for section in sections:
+            lowercase_version = section.lower()
+            if not cls._config.has_section(section=lowercase_version):
+                cls._config._sections[lowercase_version] = cls._config._sections[section]
+
+        if os.getenv("NO_LOGGING_INIT", "f").lower() not in TRUTHY_VALUES:
+            # init logging here as this is our most likely entry-point into an idmtools "application"
+            cls._init_logging()
+            from idmtools.core.logging import VERBOSE
+
+            if IdmConfigParser.get_option("NO_PRINT_CONFIG_USED",
+                                          fallback="F").lower() not in TRUTHY_VALUES and IdmConfigParser.get_option(
+                    "logging", "USER_OUTPUT", fallback="t").lower() in TRUTHY_VALUES:
+                # let users know when they are using environment variable to local config
+                if "IDMTOOLS_CONFIG_FILE" in os.environ:
+                    user_logger.warning("idmtools config defined through 'IDMTOOLS_CONFIG_FILE' environment variable")
+
+                if IdmConfigParser.found_ini():
+                    user_logger.log(VERBOSE, "INI File Found: {}".format(ini_file))
+
+    @classmethod
+    def _init_logging(cls):
+        from idmtools.core.logging import setup_logging, IdmToolsLoggingConfig
+        # set up default log values
+        log_config = dict()
+        # try to fetch logging options from config file and from environment vars
+        for field in fields(IdmToolsLoggingConfig):
+            value = cls.get_option("logging", field.name, fallback=None)
+            if value is not None:
+                log_config[field.name] = value
+
+        # handle special case
+        if log_config.get('console', None) is None:
+            log_config['console'] = None
+
+        setup_logging(IdmToolsLoggingConfig(**log_config))
+
+        if platform.system() == "Darwin":
+            # see https://bugs.python.org/issue27126
+            os.environ['NO_PROXY'] = "*"
+
+        # Do import locally to prevent load error
+        from idmtools import __version__
+        if "+nightly" in __version__ and os.getenv('IDMTOOLS_HIDE_DEV_WARNING', None) is None and os.getenv(
+                "_IDMTOOLS_COMPLETE", None) is None:
+            if logger.isEnabledFor(DEBUG):
+                logger.debug(f"You are using a development version of idmtools, version {__version__}!")
+
+    @classmethod
+    @initialization()
+    def get_section(cls, section: str = None, error: bool = True) -> Dict[str, str]:
+        """
+        Retrieve INI section values (call directly from platform creation).
+
+        Args:
+            section: The INI section name where we retrieve all fields.
+            error: Should we throw error is we cannot find block
+
+        Returns:
+            All fields as a dictionary.
+
+        Raises:
+            ValueError: If the block doesn't exist
+        """
+        original_case_section = section
+        if section is None:
+            return None
+        lower_case_section = section.lower()
+        if (not cls.found_ini() or not cls.has_section(section=lower_case_section)) and error:
+            raise ValueError(f"Block '{original_case_section}' doesn't exist!")
+
+        section_item = cls._config.items(lower_case_section)
+        cls._block = lower_case_section
+        return dict(section_item)
+
+    @classmethod
+    @initialization()
+    def get_option(cls, section: str = None, option: str = None, fallback=None, environment_first: bool = True) -> str:
+        """
+        Get configuration value based on the INI section and option.
+
+        Args:
+            section: The INI section name.
+            option: The INI field name.
+            fallback: Fallback value
+            environment_first: Try to load from environment var first. Default to True. Environment variable names are in form IDMTOOLS_SECTION_OPTION
+
+        Returns:
+            A configuration value as a string.
+        """
+        if environment_first:
+            evn_name = "_".join(filter(None, ["IDMTOOLS", section, option])).upper()
+            value = os.environ.get(evn_name, None)
+            if value:
+                if logger.isEnabledFor(DEBUG):
+                    logger.debug(f"Loaded option from environment var {evn_name}")
+                return value
+        if not cls.found_ini():
+            return fallback
+
+        if cls._config is None:
+            if fallback is None:
+                user_logger.warning("No Configuration file defined. Please define a fallback value")
+            return fallback
+
+        if section:
+            return cls._config.get(section.lower(), option, fallback=fallback)
+        else:
+            return cls._config.get("COMMON", option, fallback=fallback)
+
+    @classmethod
+    def is_progress_bar_disabled(cls) -> bool:
+        """
+        Are progress bars disabled.
+
+        Returns:
+            Return is progress bars should be enabled
+        """
+        return all(
+            [x.lower() in TRUTHY_VALUES for x in [IdmConfigParser.get_option(None, "DISABLE_PROGRESS_BAR", 'f')]])
+
+    @classmethod
+    def is_output_enabled(cls) -> bool:
+        """
+        Is output enabled.
+
+        Returns:
+            Return if output should be disabled
+        """
+        return any([x.lower() in TRUTHY_VALUES for x in [IdmConfigParser.get_option('logging', "USER_OUTPUT", 'on')]])
+
+    @classmethod
+    def ensure_init(cls, dir_path: str = '.', file_name: str = default_config, force: bool = False) -> None:
+        """
+        Verify that the INI file loaded and a configparser instance is available.
+
+        Args:
+            dir_path: The directory to search for the INI configuration file.
+            file_name: The configuration file name to search for.
+            force: Force reload of everything
+
+        Returns:
+            None
+
+        Raises:
+            ValueError: If the config file is found but cannot be parsed
+        """
+        if force:
+            cls.clear_instance()
+
+        if cls._instance is None:
+            cls(dir_path, file_name)
+
+    @classmethod
+    @initialization()
+    def get_config_path(cls) -> str:
+        """
+        Check which INI configuration file is being used.
+
+        Returns:
+            The INI file full path that is loaded.
+        """
+        return cls._config_path
+
+    @classmethod
+    @initialization()
+    def display_config_path(cls) -> None:
+        """
+        Display the INI file path being used.
+
+        Returns:
+            None
+        """
+        user_logger.info(cls.get_config_path())
+
+    @classmethod
+    @initialization()
+    def view_config_file(cls) -> None:
+        """
+        Display the INI file being used.
+
+        Returns:
+            None
+        """
+        if cls._config_path is None:
+            user_logger.warning("No configuration fouind")
+        else:
+            user_logger.info("View Config INI: \n{}".format(cls._config_path))
+            user_logger.info('-' * len(cls._config_path), '\n')
+            with open(cls._config_path) as f:
+                read_data = f.read()
+                user_logger.info(read_data)
+
+    @classmethod
+    def display_config_block_details(cls, block):
+        """
+        Display the values of a config block.
+
+        Args:
+            block: Block to print
+
+        Returns:
+            None
+        """
+        if cls.found_ini():
+            from idmtools.core.logging import VERBOSE
+            block_details = cls.get_section(block)
+            user_logger.log(VERBOSE, f"\n[{block}]")
+            user_logger.log(VERBOSE, json.dumps(block_details, indent=3))
+
+    @classmethod
+    @initialization()
+    def has_section(cls, section: str) -> bool:
+        """
+        Does the config contain a section.
+
+        Args:
+            section: Section to check for
+
+        Returns:
+            True if the section exists, False otherwise
+        """
+        return cls._config.has_section(section.lower()) if cls._config else False
+
+    @classmethod
+    @initialization
+    def has_option(cls, section: str, option: str):
+        """
+        Does the config have an option in specified section?
+
+        Args:
+            section: Section
+            option: Option
+
+        Returns:
+            True if config has option
+        """
+        return cls._config.has_option(section, option, fallback=None)
+
+    @classmethod
+    def found_ini(cls) -> bool:
+        """
+        Did we find the config?
+
+        Returns:
+            True if did, False Otherwise
+        """
+        return cls._config is not None
+
+    @classmethod
+    def clear_instance(cls) -> None:
+        """
+        Uninitialize and clean the :class:`IdmConfigParser` instance.
+
+        Returns:
+            None
+        """
+        # log as verbose
+        logger.log(15, "Clearing idm config")
+        cls._config = None
+        cls._instance = None
+        cls._config_path = None
+        cls._block = None

idmtools/core/__init__.py

@@ -0,0 +1,10 @@
+"""
+Core area classes are defined in this packaged.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+# flake8: noqa F821
+from idmtools.core.cache_enabled import CacheEnabled
+from idmtools.core.interfaces.entity_container import EntityContainer
+from idmtools.core.enums import *
+from idmtools.core.exceptions import *

idmtools/core/cache_enabled.py

@@ -0,0 +1,114 @@
+"""
+CacheEnabled definition. CacheEnabled enables diskcache wrapping on an item.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+import os
+import shutil
+import tempfile
+import time
+from contextlib import suppress
+from dataclasses import dataclass, field
+from multiprocessing import current_process, cpu_count
+from logging import getLogger, DEBUG
+from diskcache import Cache, DEFAULT_SETTINGS, FanoutCache
+from typing import Union, Optional
+
+MAX_CACHE_SIZE = int(2 ** 33)  # 8GB
+DEFAULT_SETTINGS["size_limit"] = MAX_CACHE_SIZE
+DEFAULT_SETTINGS["sqlite_mmap_size"] = 2 ** 28
+DEFAULT_SETTINGS["sqlite_cache_size"] = 2 ** 15
+logger = getLogger(__name__)
+
+
+@dataclass(init=False, repr=False)
+class CacheEnabled:
+    """
+    Allows a class to leverage Diskcache and expose a cache property.
+    """
+    _cache: Union[Cache, FanoutCache] = field(default=None, init=False, compare=False,
+                                              metadata={"pickle_ignore": True})
+    _cache_directory: str = field(default=None, init=False, compare=False)
+
+    def __del__(self):
+        """
+        Deletes object. On Deletion, we destroy the cache.
+
+        Returns:
+            None
+        """
+        self.cleanup_cache()
+
+    def initialize_cache(self, shards: Optional[int] = None, eviction_policy=None):
+        """
+        Initialize cache.
+
+        Args:
+            shards (Optional[int], optional): How many shards. It is best to set this when multi-procressing Defaults to None.
+            eviction_policy ([type], optional): See Diskcache docs. Defaults to None.
+        """
+        logger.debug(f"Initializing the cache with {shards or 0} shards and {eviction_policy or 'none'} policy.")
+        if self._cache:
+            logger.warning("CacheEnabled class is calling `initialize_cache()` with a cache already initialized: "
+                           "deleting the cache and recreating a new one.")
+            self.cleanup_cache()
+
+        # Create the cache directory if does not exist
+        if not self._cache_directory or not os.path.exists(self._cache_directory):
+            self._cache_directory = tempfile.mkdtemp()
+            logger.debug(f"Cache created in {self._cache_directory}")
+        else:
+            logger.debug(f"Cache retrieved in {self._cache_directory}")
+
+        # Create different cache depending on the options
+        if shards:
+            # set default timeout to grow with cpu count. In high thread environments, user hit timeouts
+            default_timeout = max(0.1, cpu_count() * 0.0125)
+            if logger.isEnabledFor(DEBUG):
+                logger.debug(f"Setting cache timeout to {default_timeout}")
+            self._cache = FanoutCache(self._cache_directory, shards=shards, timeout=default_timeout,
+                                      eviction_policy=eviction_policy)
+        else:
+            self._cache = Cache(self._cache_directory)
+
+    def cleanup_cache(self):
+        """
+        Cleanup our cache.
+
+        Returns:
+            None
+        """
+        # Only delete and close the cache if the owner thread ends
+        # Avoid deleting and closing when a child thread ends
+        with suppress(AttributeError):
+            if current_process().name != 'MainProcess':
+                return
+
+        if self._cache is not None:
+            self._cache.close()
+            del self._cache
+
+            if self._cache_directory and os.path.exists(self._cache_directory):
+                retries = 0
+                while retries < 3:
+                    try:
+                        shutil.rmtree(self._cache_directory)
+                        time.sleep(0.15)
+                        break
+                    except IOError:
+                        # we don't use logger here because it could be destroyed
+                        print(f"Could not delete cache {self._cache_directory}")
+                        retries += 1
+
+    @property
+    def cache(self) -> Union[Cache, FanoutCache]:
+        """
+        Allows fetches of cache and ensures it is initialized.
+
+        Returns:
+            Cache
+        """
+        if self._cache is None:
+            self.initialize_cache()
+
+        return self._cache

idmtools/core/context.py

@@ -0,0 +1,68 @@
+"""
+Manages the idmtools context, mostly around the platform object.
+
+This context allows us to easily fetch what platforms we are executing on and also supported nested, multi-platform operations.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+from logging import getLogger
+from typing import TYPE_CHECKING
+
+logger = getLogger(__name__)
+if TYPE_CHECKING:  # pragma: no cover
+    # The current platform
+    from idmtools.entities.iplatform import IPlatform
+
+current_platform_stack = []
+CURRENT_PLATFORM: 'IPlatform' = None
+
+
+def set_current_platform(platform: 'IPlatform'):
+    """
+    Set the current platform that is being used to execute scripts.
+
+    Args:
+        platform: Platform to set
+
+    Returns:
+        None
+    """
+    global CURRENT_PLATFORM, current_platform_stack  # noqa: F824
+    if CURRENT_PLATFORM and CURRENT_PLATFORM != platform and platform not in current_platform_stack:
+        current_platform_stack.append(CURRENT_PLATFORM)
+    CURRENT_PLATFORM = platform
+
+
+def remove_current_platform():
+    """
+    Set CURRENT_PLATFORM to None and delete old platform object.
+
+    Returns:
+        None
+    """
+    global CURRENT_PLATFORM, current_platform_stack  # noqa: F824
+    old_current_platform = CURRENT_PLATFORM
+    if len(current_platform_stack):
+        new_platform = current_platform_stack.pop()
+    else:
+        new_platform = None
+    del old_current_platform
+    CURRENT_PLATFORM = new_platform
+
+
+def clear_context():
+    """
+    Clear all platforms from context.
+    """
+    global CURRENT_PLATFORM, current_platform_stack  # noqa: F824
+    old_current_platform = CURRENT_PLATFORM
+    CURRENT_PLATFORM = None
+    current_platform_stack.clear()
+    del old_current_platform
+
+
+def get_current_platform() -> 'IPlatform':
+    """
+    Get current platform.
+    """
+    return CURRENT_PLATFORM