idmtools 0.0.0.dev0__py3-none-any.whl → 0.0.2__py3-none-any.whl

idmtools/entities/iplatform_ops/iplatform_workflowitem_operations.py

@@ -0,0 +1,301 @@
+"""
+IPlatformWorkflowItemOperations defines workflow item operations interface.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from logging import DEBUG
+from typing import Type, Any, List, Tuple, Dict, NoReturn, TYPE_CHECKING
+from idmtools.assets import Asset
+from idmtools.core import CacheEnabled, getLogger
+from idmtools.entities.iplatform_ops.utils import batch_create_items
+from idmtools.entities.iworkflow_item import IWorkflowItem
+from idmtools.registry.functions import FunctionPluginManager
+
+if TYPE_CHECKING:  # pragma: no cover
+    from idmtools.entities.iplatform import IPlatform
+
+logger = getLogger(__name__)
+
+
+@dataclass
+class IPlatformWorkflowItemOperations(CacheEnabled, ABC):
+    """
+    IPlatformWorkflowItemOperations defines workflow item operations interface.
+    """
+    platform: 'IPlatform'  # noqa: F821
+    platform_type: Type
+
+    @abstractmethod
+    def get(self, workflow_item_id: str, **kwargs) -> Any:
+        """
+        Returns the platform representation of an WorkflowItem.
+
+        Args:
+            workflow_item_id: Item id of WorkflowItems
+            **kwargs:
+
+        Returns:
+            Platform Representation of an workflow_item
+        """
+        pass
+
+    def batch_create(self, workflow_items: List[IWorkflowItem], display_progress: bool = True, **kwargs) -> List[Any]:
+        """
+        Provides a method to batch create workflow items.
+
+        Args:
+            workflow_items: List of worfklow items to create
+            display_progress: Whether to display progress bar
+            **kwargs:
+
+        Returns:
+            List of tuples containing the create object and id of item that was created.
+        """
+        return batch_create_items(workflow_items, create_func=self.create, display_progress=display_progress,
+                                  progress_description="Creating WorkItems", unit="workitem", **kwargs)
+
+    def pre_create(self, workflow_item: IWorkflowItem, **kwargs) -> NoReturn:
+        """
+        Run the platform/workflow item post creation events.
+
+        Args:
+            workflow_item: IWorkflowItem to run post-creation events
+            **kwargs: Optional arguments mainly for extensibility
+
+        Returns:
+            NoReturn
+        """
+        if logger.isEnabledFor(DEBUG):
+            logger.debug("Calling idmtools_platform_pre_create_item")
+        FunctionPluginManager.instance().hook.idmtools_platform_pre_create_item(item=workflow_item, kwargs=kwargs)
+        if logger.isEnabledFor(DEBUG):
+            logger.debug("Calling pre_creation")
+        workflow_item.pre_creation(self.platform)
+
+    def post_create(self, workflow_item: IWorkflowItem, **kwargs) -> NoReturn:
+        """
+        Run the platform/workflow item post creation events.
+
+        Args:
+            workflow_item: IWorkflowItem to run post-creation events
+            **kwargs: Optional arguments mainly for extensibility
+
+        Returns:
+            NoReturn
+        """
+        if logger.isEnabledFor(DEBUG):
+            logger.debug("Calling idmtools_platform_post_create_item hooks")
+        FunctionPluginManager.instance().hook.idmtools_platform_post_create_item(item=workflow_item, kwargs=kwargs)
+        if logger.isEnabledFor(DEBUG):
+            logger.debug("Calling post_creation")
+        workflow_item.post_creation(self.platform)
+
+    def create(self, workflow_item: IWorkflowItem, do_pre: bool = True, do_post: bool = True, **kwargs) -> Any:
+        """
+        Creates an workflow item from an IDMTools IWorkflowItem object.
+
+        Also performs pre-creation and post-creation locally and on platform.
+
+        Args:
+            workflow_item: Suite to create
+            do_pre: Perform Pre creation events for item
+            do_post: Perform Post creation events for item
+            **kwargs: Optional arguments mainly for extensibility
+
+        Returns:
+            Created platform item and the UUID of said item
+        """
+        if workflow_item.status is not None:
+            return workflow_item._platform_object, workflow_item.uid
+        if do_pre:
+            if logger.isEnabledFor(DEBUG):
+                logger.debug("Calling pre_create")
+            self.pre_create(workflow_item, **kwargs)
+        if logger.isEnabledFor(DEBUG):
+            logger.debug("Calling platform_create")
+        ret = self.platform_create(workflow_item, **kwargs)
+        workflow_item.platform = self.platform
+        if do_post:
+            if logger.isEnabledFor(DEBUG):
+                logger.debug("Calling post_create")
+            self.post_create(workflow_item, **kwargs)
+        return ret
+
+    @abstractmethod
+    def platform_create(self, workflow_item: IWorkflowItem, **kwargs) -> Tuple[Any, str]:
+        """
+        Creates an workflow_item from an IDMTools workflow_item object.
+
+        Args:
+            workflow_item: WorkflowItem to create
+            **kwargs: Optional arguments mainly for extensibility
+
+        Returns:
+            Created platform item and the id of said item
+        """
+        pass
+
+    def pre_run_item(self, workflow_item: IWorkflowItem, **kwargs):
+        """
+        Trigger right before commissioning experiment on platform.
+
+        This ensures that the item is created. It also ensures that the children(simulations) have also been created.
+
+        Args:
+            workflow_item: Experiment to commission
+
+        Returns:
+            None
+        """
+        # ensure the item is created before running
+        # TODO what status are valid here? Create only?
+        if workflow_item.status is None:
+            if logger.isEnabledFor(DEBUG):
+                logger.debug("Calling create")
+            self.create(workflow_item)
+
+    def post_run_item(self, workflow_item: IWorkflowItem, **kwargs):
+        """
+        Trigger right after commissioning workflow item on platform.
+
+        Args:
+            workflow_item: Experiment just commissioned
+
+        Returns:
+            None
+        """
+        if logger.isEnabledFor(DEBUG):
+            logger.debug("Calling idmtools_platform_post_run hooks")
+        FunctionPluginManager.instance().hook.idmtools_platform_post_run(item=workflow_item, kwargs=kwargs)
+
+    def run_item(self, workflow_item: IWorkflowItem, **kwargs):
+        """
+        Called during commissioning of an item. This should create the remote resource.
+
+        Args:
+            workflow_item:
+
+        Returns:
+            None
+        """
+        if logger.isEnabledFor(DEBUG):
+            logger.debug("Calling pre_run_item")
+        self.pre_run_item(workflow_item, **kwargs)
+        if logger.isEnabledFor(DEBUG):
+            logger.debug("Calling platform_run_item")
+        self.platform_run_item(workflow_item, **kwargs)
+        if logger.isEnabledFor(DEBUG):
+            logger.debug("Calling post_run_item")
+        self.post_run_item(workflow_item, **kwargs)
+
+    @abstractmethod
+    def platform_run_item(self, workflow_item: IWorkflowItem, **kwargs):
+        """
+        Called during commissioning of an item. This should perform what is needed to commission job on platform.
+
+        Args:
+            workflow_item:
+
+        Returns:
+            None
+        """
+        pass
+
+    @abstractmethod
+    def get_parent(self, workflow_item: Any, **kwargs) -> Any:
+        """
+        Returns the parent of item. If the platform doesn't support parents, you should throw a TopLevelItem error.
+
+        Args:
+            workflow_item: Workflow item to get parent of
+            **kwargs:
+
+        Returns:
+            Parent of Worktflow item
+
+        Raise:
+            TopLevelItem
+        """
+        pass
+
+    @abstractmethod
+    def get_children(self, workflow_item: Any, **kwargs) -> List[Any]:
+        """
+        Returns the children of an workflow_item object.
+
+        Args:
+            workflow_item: WorkflowItem object
+            **kwargs: Optional arguments mainly for extensibility
+
+        Returns:
+            Children of workflow_item object
+        """
+        pass
+
+    def to_entity(self, workflow_item: Any, **kwargs) -> IWorkflowItem:
+        """
+        Converts the platform representation of workflow_item to idmtools representation.
+
+        Args:
+            workflow_item:Platform workflow_item object
+
+        Returns:
+            IDMTools workflow_item object
+        """
+        return workflow_item
+
+    @abstractmethod
+    def refresh_status(self, workflow_item: IWorkflowItem, **kwargs):
+        """
+        Refresh status for workflow item.
+
+        Args:
+            workflow_item: Item to refresh status for
+
+        Returns:
+            None
+        """
+        pass
+
+    @abstractmethod
+    def send_assets(self, workflow_item: Any, **kwargs):
+        """
+        Send assets for workflow item to platform.
+
+        Args:
+            workflow_item: Item to send assets for
+
+        Returns:
+            None
+        """
+        pass
+
+    @abstractmethod
+    def get_assets(self, workflow_item: IWorkflowItem, files: List[str], **kwargs) -> Dict[str, bytearray]:
+        """
+        Load assets for workflow item.
+
+        Args:
+            workflow_item: Item
+            files: List of files to load
+            **kwargs:
+
+        Returns:
+            Dictionary with filename as key and value as binary content
+        """
+        pass
+
+    @abstractmethod
+    def list_assets(self, workflow_item: IWorkflowItem, **kwargs) -> List[Asset]:
+        """
+        List available assets for a workflow item.
+
+        Args:
+            workflow_item: workflow item to list files for
+
+        Returns:
+            List of filenames
+        """
+        pass

idmtools/entities/iplatform_ops/utils.py

@@ -0,0 +1,185 @@
+"""
+Utils for platform operations.
+
+Here we have mostly utilities to handle batch operations which tend to overlap across different item types.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+from concurrent.futures import as_completed, Future
+from concurrent.futures.process import ProcessPoolExecutor
+from concurrent.futures.thread import ThreadPoolExecutor
+from functools import partial
+from logging import getLogger, DEBUG
+from os import cpu_count
+from typing import List, Union, Generator, Iterable, Callable, Any
+from more_itertools import chunked
+from idmtools.core import EntityContainer
+from idmtools.entities.templated_simulation import TemplatedSimulations
+
+logger = getLogger(__name__)
+user_logger = getLogger('user')
+# Global executor
+EXECUTOR = None
+
+
+def batch_items(items: Union[Iterable, Generator], batch_size=16):
+    """
+    Batch items.
+
+    Args:
+        items: Items to batch
+        batch_size: Size of the batch
+
+    Returns:
+        Generator
+
+    Raises:
+        StopIteration
+    """
+    for item_chunk in chunked(items, batch_size):
+        logger.info('created chunk')
+        yield item_chunk
+    raise StopIteration
+
+
+def item_batch_worker_thread(create_func: Callable, items: Union[List], **kwargs) -> List:
+    """
+    Default batch worker thread function. It just calls create on each item.
+
+    Args:
+        create_func: Create function for item
+        items: Items to create
+
+    Returns:
+        List of items created
+    """
+    if logger.isEnabledFor(DEBUG):
+        logger.debug(f'Create {len(items)}')
+
+    ret = []
+    for item in items:
+        ret.append(create_func(item, **kwargs))
+
+    return ret
+
+
+def batch_create_items(items: Union[Iterable, Generator], batch_worker_thread_func: Callable[[List], List] = None,
+                       create_func: Callable[..., Any] = None, display_progress: bool = True,
+                       progress_description: str = "Commissioning items", unit: str = None, **kwargs):
+    """
+    Batch create items. You must specify either batch_worker_thread_func or create_func.
+
+    Args:
+        items: Items to create
+        batch_worker_thread_func: Optional Function to execute. Should take a list and return a list
+        create_func: Optional Create function
+        display_progress: Enable progress bar
+        progress_description: Description to show in progress bar
+        unit: Unit for progress bar
+        **kwargs:
+
+    Returns:
+        Batches crated results
+    """
+    global EXECUTOR
+    from idmtools.config import IdmConfigParser
+    from idmtools.utils.collections import ExperimentParentIterator
+
+    max_workers = kwargs.get('max_workers', None)
+
+    # Consider values from the block that Platform uses
+    _batch_size = int(IdmConfigParser.get_option(None, "batch_size", fallback=16))
+
+    batch_size = kwargs.get('batch_size', None)
+    if batch_size is not None:
+        _batch_size = batch_size
+
+    if display_progress and not IdmConfigParser.is_progress_bar_disabled():
+        from tqdm import tqdm
+        extra_args = dict(unit=unit) if unit else dict()
+        prog = tqdm(desc="Initializing objects for creation", **extra_args)
+    else:
+        prog = None
+
+    if EXECUTOR is None:
+        _workers_per_cpu = IdmConfigParser.get_option(None, "workers_per_cpu", fallback=None)
+        if _workers_per_cpu:
+            _max_workers = int(_workers_per_cpu) * cpu_count()
+            if logger.isEnabledFor(DEBUG):
+                logger.debug(f"workers set by cpu: {_workers_per_cpu} * {cpu_count()}")
+        else:
+            _max_workers = int(IdmConfigParser.get_option(None, "max_workers", fallback=16))
+
+        if max_workers is not None:
+            _max_workers = max_workers
+
+        logger.info(f'Creating {_max_workers} Platform Workers')
+        default_pool_executor = IdmConfigParser.get_option(None, "default_pool_executor", fallback="thread").lower()
+        if default_pool_executor == "process":
+            EXECUTOR = ProcessPoolExecutor(max_workers=_max_workers)
+        else:
+            EXECUTOR = ThreadPoolExecutor(max_workers=_max_workers)
+
+    if batch_worker_thread_func is None:
+
+        if create_func is None:
+            raise ValueError("You must provide either an item create callback or a item batch worker thread callback to"
+                             " perform batches")
+        batch_worker_thread_func = partial(item_batch_worker_thread, create_func, **kwargs)
+    if logger.isEnabledFor(DEBUG):
+        logger.debug(f'Batching creation by {_batch_size}')
+
+    futures = []
+
+    total = 0
+    parent = None
+    if isinstance(items, ExperimentParentIterator) and isinstance(items.items, TemplatedSimulations):
+        parent = items.parent
+        i = items.items.simulations().generator
+    elif isinstance(items, ExperimentParentIterator) and isinstance(items.items, EntityContainer):
+        parent = items.parent
+        i = items.items
+    else:
+        i = items
+    if display_progress and not IdmConfigParser.is_progress_bar_disabled() and hasattr(items, '__len__'):
+        prog.total = len(items)
+    for chunk in chunked(i, _batch_size):
+        total += len(chunk)
+        if parent:
+            for c in chunk:
+                c.parent = parent
+        if logger.isEnabledFor(DEBUG):
+            logger.debug(f"Submitting chunk: {len(chunk)}")
+        if display_progress and not IdmConfigParser.is_progress_bar_disabled():
+            prog.update(len(chunk))
+        futures.append(EXECUTOR.submit(batch_worker_thread_func, chunk))
+
+    results = []
+    if display_progress and not IdmConfigParser.is_progress_bar_disabled():
+        prog.set_description(progress_description)
+        prog.reset(total)
+        results = show_progress_of_batch(prog, futures)
+    else:
+        for future in futures:
+            results.extend(future.result())
+
+    return results
+
+
+def show_progress_of_batch(progress_bar: 'tqdm', futures: List[Future]) -> List:  # noqa: F821
+    """
+    Show progress bar for batch.
+
+    Args:
+        progress_bar: Progress bar
+        futures: List of futures that are still running/queued
+
+    Returns:
+        Returns results
+    """
+    results = []
+    for future in as_completed(futures):
+        result = future.result()
+        progress_bar.update(len(result))
+        results.extend(future.result())
+    return results