idmtools 0.0.0.dev0__py3-none-any.whl → 0.0.3__py3-none-any.whl

idmtools/utils/collections.py

@@ -0,0 +1,246 @@
+"""
+utilities for collections.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+import typing
+from itertools import tee
+from typing import Tuple, List, Mapping, Union, Iterable, Generator
+from more_itertools import take
+
+
+def cut_iterable_to(obj: Iterable, to: int) -> Tuple[Union[List, Mapping], int]:
+    """
+       Cut an iterable to a certain length.
+
+    Args:
+        obj: The iterable to cut.
+        to: The number of elements to return.
+
+    Returns:
+        A list or dictionary (depending on the type of object) of elements and
+        the remaining elements in the original list or dictionary.
+    """
+    if isinstance(obj, dict):
+        slice = {k: v for (k, v) in take(to, obj.items())}
+    else:
+        slice = take(to, obj)
+
+    remaining = len(obj) - to
+    remaining = 0 if remaining < 0 else remaining
+    return slice, remaining
+
+
+class ExperimentParentIterator(typing.Iterator['Simulation']):  # noqa F821
+    """
+    Wraps a list of simulations with iterator that always provides parent experiment.
+    """
+
+    def __init__(self, lst, parent: 'IEntity'):  # noqa F821
+        """
+        Initializes the ExperimentParentIterator.
+
+        Args:
+            lst: List of items(simulations) to iterator over
+            parent: Parent of items(Experiment)
+        """
+        self.items = lst
+        self.__iter = iter(self.items) if not isinstance(self.items, (typing.Iterator, Generator)) else self.items
+        self.parent = parent
+
+    def __iter__(self):
+        """
+        Iterator method returns self.
+
+        Returns:
+            Self
+        """
+        return self
+
+    def __next__(self):
+        """
+        Fetch the next items from our list.
+
+        Returns:
+            Next item from our list
+        """
+        i = next(self.__iter)
+        i._parent = self.parent
+        if hasattr(i, 'parent_id') and self.parent.uid is not None:
+            i.parent_id = self.parent.uid
+        return i
+
+    def __getitem__(self, item):
+        """
+        Get items wrapper.
+
+        Args:
+            item: Item to fetch
+
+        Returns:
+            Item from self.items
+        """
+        return self.items[item]
+
+    def __getattr__(self, item):
+        """
+        Get attr wrapper.
+
+        Args:
+            item: Item to get
+
+        Returns:
+            Attribute from our items
+        """
+        return getattr(self.items, item)
+
+    def __len__(self):
+        """
+        Returns the total simulations.
+
+        Returns:
+            Total simulations
+
+        Raises:
+            ValueError - When the underlying object is a generator, we cannot get the length of the object
+        """
+        from idmtools.entities.templated_simulation import TemplatedSimulations
+        if isinstance(self.items, typing.Sized):
+            return len(self.items)
+        elif isinstance(self.items, TemplatedSimulations):
+            return sum([len(b) for b in self.items.builders])
+        raise ValueError("Cannot get the length of a generator object")
+
+    def append(self, item: 'Simulation'):  # noqa F821
+        """
+        Adds a simulation to an object.
+
+        Args:
+            item: Item to add
+
+        Returns:
+            None
+
+        Raises:
+            ValueError when we cannot append because the item is not a simulation or our underlying object doesn't support appending
+        """
+        from idmtools.entities.templated_simulation import TemplatedSimulations
+        from idmtools.entities.simulation import Simulation
+        if not isinstance(item, Simulation):
+            raise ValueError("You can only append simulations")
+
+        # Check possible duplicate
+        if self.parent.check_duplicate(item.id):
+            return
+
+        # Set parent
+        item.parent = self.parent
+
+        # Add to collection
+        if isinstance(self.items, (list, set)):
+            self.items.append(item)
+            return
+        elif isinstance(self.items, TemplatedSimulations):
+            self.items.add_simulation(item)
+            return
+        raise ValueError("Items doesn't support appending")
+
+    def extend(self, item: Union[List['Simulation'], 'TemplatedSimulations']):  # noqa F821
+        """
+        Extends object.
+
+        Args:
+            item: Item to extend
+
+        Returns:
+            None
+
+        Raises:
+            ValueError when the underlying data object doesn't support adding additional item
+        """
+        from idmtools.entities.templated_simulation import TemplatedSimulations
+        if isinstance(self.items, (list, set)):
+            # if it is a template, try to preserve so we can use generators
+            if isinstance(item, TemplatedSimulations):
+                self.extend(list(item))
+            else:
+                for it in item:
+                    self.append(it)
+            return
+        if isinstance(self.items, TemplatedSimulations):
+            if isinstance(item, TemplatedSimulations):
+                self.items.add_simulations(list(item))
+            else:
+                self.items.add_simulations(item)
+            return
+        raise ValueError("Items doesn't support extending")
+
+
+class ResetGenerator(typing.Iterator):
+    """Iterator that counts upward forever."""
+
+    def __init__(self, generator_init):
+        """
+        Initialize the ResetGenerator from generator_init.
+
+        Creates a copy of the generator using tee.
+
+        Args:
+            generator_init: Initialize iterator/generator to copy
+        """
+        self.generator_init = generator_init
+        self.generator = generator_init()
+        self.generator, self.__next_gen = tee(self.generator)
+
+    def __iter__(self):
+        """
+        Get iteror.
+        """
+        return self
+
+    def next_gen(self):
+        """
+        The original generator/iterator.
+
+        Returns:
+            original generator/iterator.
+        """
+        return self.__next_gen
+
+    def __next__(self):
+        """
+        Get next item.
+
+        For reset iteration, if we hit the end of our iterator/generator, we copy it and reset
+
+        Returns:
+            Next item
+
+        Raises:
+            StopIteration at the end of the iteration.
+        """
+        try:
+            result = next(self.generator)
+        except StopIteration:
+            self.generator, self.__next_gen = tee(self.__next_gen)
+            raise StopIteration
+        return result
+
+
+def duplicate_list_of_generators(lst: List[Generator]):
+    """
+    Copy a list of iterators using tee.
+
+    Args:
+        lst: List of generators
+
+    Returns:
+        Tuple with duplicate of iterators
+    """
+    new_sw = []
+    old_sw = []
+    for sw in lst:
+        o, n = tee(sw)
+        new_sw.append(n)
+        old_sw.append(o)
+    return old_sw, new_sw

idmtools/utils/command_line.py

@@ -0,0 +1,45 @@
+"""
+utilities for command line.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+import contextlib
+import logging
+import sys
+
+from io import StringIO
+from itertools import cycle
+
+animation = cycle(("|", "/", "-", "\\"))
+
+
+@contextlib.contextmanager
+def suppress_output(stdout=True, stderr=True):
+    """
+    Suppress any print/logging from a block of code.
+
+    Args:
+        stdout: If True, hide output from stdout; if False, show it.
+        stderr: If True, hide output from stderr; if False, show it.
+    """
+    # Remember existing output streams for restoration
+    original_stdout = sys.stdout
+    original_stderr = sys.stderr
+
+    # Suppress requested channel output
+    if stdout:
+        sys.stdout = StringIO()
+    if stderr:
+        sys.stderr = StringIO()
+
+    # Deactivate logging
+    previous_level = logging.root.manager.disable
+    logging.disable(logging.ERROR)
+
+    yield
+
+    # Restore original output streams
+    sys.stdout = original_stdout
+    sys.stderr = original_stderr
+
+    logging.disable(previous_level)

idmtools/utils/decorators.py

@@ -0,0 +1,300 @@
+"""
+Decorators defined for idmtools.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+import datetime
+import functools
+import importlib
+import importlib.util
+import os
+import sys
+import threading
+from concurrent.futures import Executor, as_completed
+from concurrent.futures.thread import ThreadPoolExecutor
+from functools import wraps
+from logging import getLogger, DEBUG
+from typing import Callable, Union, Optional, Type
+
+user_logger = getLogger('user')
+logger = getLogger(__name__)
+
+
+class abstractstatic(staticmethod):
+    """
+    A decorator for defining a method both as static and abstract.
+    """
+    __slots__ = ()
+
+    def __init__(self, function):
+        """
+        Initialize abstractstatic.
+
+        Args:
+            function: Function to wrap as abstract
+        """
+        super(abstractstatic, self).__init__(function)
+        function.__isabstractmethod__ = True
+
+    __isabstractmethod__ = True
+
+
+def optional_decorator(decorator: Callable, condition: Union[bool, Callable[[], bool]]):
+    """
+    A decorator that adds a decorator only if condition is true.
+
+    Args:
+        decorator: Decorator to add
+        condition: Condition to determine. Condition can be a callable as well
+
+    Returns:
+        Optionally wrapped func.
+    """
+    if callable(condition):
+        condition = condition()
+
+    def decorate_in(func):
+        if condition:
+            func = decorator(func)
+
+        @wraps
+        def wrapper(*args, **kwargs):
+            func(*args, **kwargs)
+
+        return wrapper
+
+    return decorate_in
+
+
+class SingletonMixin(object):
+    """
+    SingletonMixin defines a singleton that can be added to any class.
+
+    As a singleton, on one instance will be made per process.
+    """
+    __singleton_lock = threading.Lock()
+    __singleton_instance = None
+
+    @classmethod
+    def instance(cls):
+        """
+        Return the instance of the object.
+
+        If the instance has not been created, it will be initialized before returning.
+
+        Returns:
+            The singleton instance
+        """
+        if not cls.__singleton_instance:
+            with cls.__singleton_lock:
+                if not cls.__singleton_instance:
+                    cls.__singleton_instance = cls()
+        return cls.__singleton_instance
+
+
+def cache_for(ttl=None) -> Callable:
+    """
+    Cache a value for a certain time period.
+
+    Args:
+        ttl: Expiration of cache
+
+    Returns:
+        Wrapper Function
+    """
+    if ttl is None:
+        ttl = datetime.timedelta(minutes=1)
+
+    def wrap(func):
+        time, value = None, None
+
+        @wraps(func)
+        def wrapped(*args, **kw):
+            # if we are testing, disable caching of functions as it complicates test-all setups
+            from idmtools.core import TRUTHY_VALUES
+            if os.getenv('TESTING', '0').lower() in TRUTHY_VALUES:
+                return func(*args, **kw)
+
+            nonlocal time
+            nonlocal value
+            now = datetime.datetime.now()
+            if not time or now - time > ttl:
+                value = func(*args, **kw)
+                time = now
+            return value
+
+        return wrapped
+
+    return wrap
+
+
+def optional_yaspin_load(*yargs, **ykwargs) -> Callable:
+    """
+    Adds a CLI spinner to a function based on conditions.
+
+    The spinner will be present if
+
+    * yaspin package is present.
+    * NO_SPINNER environment variable is not defined.
+
+    Args:
+        *yargs: Arguments to pass to yaspin constructor.
+        **ykwargs: Keyword arguments to pass to yaspin constructor.
+
+    Examples:
+        ::
+
+            @optional_yaspin_load(text="Loading test", color="yellow")
+            def test():
+                time.sleep(100)
+
+    Returns:
+        A callable wrapper function.
+    """
+    has_yaspin = importlib.util.find_spec("yaspin")
+    spinner = None
+    if has_yaspin and not os.getenv('NO_SPINNER', False):
+        from yaspin import yaspin
+        spinner = yaspin(*yargs, **ykwargs)
+
+    def decorate(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            if spinner and not os.getenv('NO_SPINNER', False):
+                spinner.start()
+            try:
+                kwargs['spinner'] = spinner
+                result = func(*args, **kwargs)
+            except Exception as e:
+                if spinner:
+                    spinner.stop()
+                raise e
+            if spinner:
+                spinner.stop()
+            return result
+
+        return wrapper
+
+    return decorate
+
+
+class ParallelizeDecorator:
+    """
+    ParallelizeDecorator allows you to easily parallelize a group of code.
+
+    A simple of example would be following
+
+    Examples:
+        ::
+
+            op_queue = ParallelizeDecorator()
+
+            class Ops:
+                op_queue.parallelize
+                def heavy_op():
+                    time.sleep(10)
+
+                def do_lots_of_heavy():
+                    futures = [self.heavy_op() for i in range(100)]
+                    results = op_queue.get_results(futures)
+    """
+
+    def __init__(self, queue=None, pool_type: Optional[Type[Executor]] = ThreadPoolExecutor):
+        """
+        Initialize our ParallelizeDecorator.
+
+        Args:
+            queue: Queue to use. If not provided, one will be created.
+            pool_type: Pool type to use. Defaults to ThreadPoolExecutor.
+        """
+        if queue is None:
+            self.queue = pool_type()
+        else:
+            self.queue = queue
+
+    def parallelize(self, func):
+        """
+        Wrap a function in parallelization.
+
+        Args:
+            func: Function to wrap with parallelization
+
+        Returns:
+            Function wrapped with parallelization object
+        """
+
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            future = self.queue.submit(func, *args, **kwargs)
+            return future
+
+        return wrapper
+
+    def join(self):
+        """
+        Join our queue.
+
+        Returns:
+            Join operation from queue
+        """
+        return self.queue.join()
+
+    def get_results(self, futures, ordered=False):
+        """
+        Get Results from our decorator.
+
+        Args:
+            futures: Futures to get results from
+            ordered: Do we want results in order provided or as they complete. Default is as they complete which is False.
+
+        Returns:
+            Results from all the futures.
+        """
+        results = []
+        if ordered:
+            for f in futures:
+                results.append(f.result())
+        else:
+            for f in as_completed(futures):
+                results.append(f.result())
+
+        if logger.isEnabledFor(DEBUG):
+            logger.debug("Parallelize Total Results: " + str(results))
+        return results
+
+    def __del__(self):
+        """
+        Delete our queue before deleting ourselves.
+
+        Returns:
+            None
+        """
+        del self.queue
+
+
+def check_symlink_capabilities(func):
+    """Decorator to check symlink creation capabilities."""
+
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        if sys.platform == 'win32' and sys.version_info < (3, 8):
+            user_logger.debug('Need administrator privileges to create symbolic links on Windows.')
+        return func(*args, **kwargs)
+
+    return wrapper
+
+
+def cache_directory(func):
+    """Decorator to cache Suite/Experiment/simulation directory."""
+
+    @functools.wraps(func)
+    def wrapper(self, item, *args, **kwargs):
+        _platform_directory = getattr(item, '_platform_directory', None)
+        if _platform_directory:
+            item_dir = _platform_directory
+        else:
+            item_dir = func(self, item, *args, **kwargs)
+            setattr(item, '_platform_directory', item_dir)
+        return item_dir
+
+    return wrapper

idmtools/utils/display/__init__.py

@@ -0,0 +1,22 @@
+"""
+root of display utilities for idmtools.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+from idmtools.utils.display.settings import *  # noqa: F401, F403
+from idmtools.utils.display.displays import *  # noqa: F401, F403
+
+
+def display(obj, settings):
+    """
+    Display an object using our settings.
+
+    Args:
+        obj: Obj to display
+        settings: Display settings
+
+    Returns:
+        None
+    """
+    for setting in settings:
+        print(setting.display(obj))