idmtools 0.0.0.dev0__py3-none-any.whl → 0.0.3__py3-none-any.whl

idmtools/entities/simulation.py

@@ -0,0 +1,255 @@
+"""
+Defines our Simulation object.
+
+The simulation object can be thought as a metadata object. It represents a configuration of a remote job execution metadata.
+All simulations have a task. Optionally that have assets. All simulations should belong to an Experiment.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+from dataclasses import dataclass, field, fields
+from logging import getLogger, DEBUG
+from typing import List, Union, Mapping, Any, Type, TypeVar, Dict, TYPE_CHECKING
+from idmtools.assets import AssetCollection, Asset
+from idmtools.core import ItemType, NoTaskFound
+from idmtools.core.enums import EntityStatus
+from idmtools.core.interfaces.iassets_enabled import IAssetsEnabled
+from idmtools.core.interfaces.iitem import IItem
+from idmtools.core.interfaces.inamed_entity import INamedEntity
+from idmtools.entities.task_proxy import TaskProxy
+from idmtools.utils.language import get_qualified_class_name_from_obj
+
+if TYPE_CHECKING:  # pragma: no cover
+    from idmtools.entities.itask import ITask
+    from idmtools.entities.iplatform import IPlatform
+    from idmtools.entities.experiment import Experiment
+
+logger = getLogger(__name__)
+user_logger = getLogger('user')
+
+
+@dataclass
+class Simulation(IAssetsEnabled, INamedEntity):
+    """
+    Class that represents a generic simulation.
+
+    This class needs to be implemented for each model type with specifics.
+    """
+    #: Experiment ID
+    experiment_id: str = field(default=None)
+    #: Task representing the configuration of the command to be executed
+    task: 'ITask' = field(default=None)  # noqa: F821
+    #: Item Type. Should not be changed from Simulation
+    item_type: ItemType = field(default=ItemType.SIMULATION, compare=False)
+    #: Control whether we should replace the task with a proxy after creation
+    __replace_task_with_proxy: bool = field(default=True, init=False, compare=False)
+    #: Ensure we don't gather assets twice
+    __assets_gathered: bool = field(default=False)
+    #: Extra arguments to pass on creation to platform
+    _platform_kwargs: dict = field(default_factory=dict)
+
+    @property
+    def experiment(self) -> 'Experiment':  # noqa: F821
+        """
+        Get experiment parent.
+
+        Returns:
+            Parent Experiment
+        """
+        return self.parent
+
+    @experiment.setter
+    def experiment(self, experiment: 'Experiment'):  # noqa: F821
+        """
+        Set the parent experiment.
+
+        Args:
+            experiment: Experiment to set as parent.
+
+        Returns:
+            None
+        """
+        self.parent = experiment
+
+    @property
+    def parent(self):
+        """
+        Return parent object for item.
+
+        Returns:
+            Parent entity if set
+        """
+        if not self._parent:
+            self.parent_id = self.parent_id or self.experiment_id
+            if not self.parent_id:
+                return None
+            if not self.platform:
+                from idmtools.core import NoPlatformException
+                raise NoPlatformException("The object has no platform set...")
+            exp = self.platform.get_item(self.parent_id, ItemType.EXPERIMENT, force=True)
+            self._parent = exp
+            self.parent_id = self.suite_id = exp.id
+
+        return self._parent
+
+    @parent.setter
+    def parent(self, parent: 'Experiment'):
+        """
+        Sets the parent object for Entity.
+
+        Args:
+            parent: Parent object
+
+        Returns:
+            None
+        """
+        if parent is not None:
+            self._parent = parent
+            self.parent_id = self.suite_id = parent.id
+        else:
+            self._parent = self.parent_id = self.experiment_id = None
+
+    def __repr__(self):
+        """
+        String representation of simulation.
+        """
+        return f"<Simulation: {self.uid} - Exp_id: {self.parent_id}>"
+
+    def __hash__(self):
+        """
+        Hash of simulation(id).
+
+        Returns:
+            Hash of simulation
+        """
+        return id(self.uid)
+
+    def pre_creation(self, platform: 'IPlatform'):
+        """
+
+        Runs before a simulation is created server side.
+
+        Args:
+            platform: Platform the item is being executed on
+
+        Returns:
+            None
+        """
+        # skip the IItem created function
+        IItem.pre_creation(self, platform)
+        if self.task is None:
+            msg = 'Task is required for simulations'
+            user_logger.error(msg)
+            raise NoTaskFound(msg)
+
+        if logger.isEnabledFor(DEBUG):
+            logger.debug('Calling task pre creation')
+        self.task.pre_creation(self, platform)
+
+        self.gather_assets()
+
+        if self.__class__ is not Simulation:
+            # Add a tag to keep the Simulation class name
+            sn = get_qualified_class_name_from_obj(self)
+            if logger.isEnabledFor(DEBUG):
+                logger.debug(f'Setting Simulation Tag "simulation_type" to "{sn}"')
+            self.tags["simulation_type"] = sn
+
+    def post_creation(self, platform: 'IPlatform') -> None:
+        """
+        Called after a simulation is created.
+
+        Args:
+            platform: Platform simulation is being executed on
+
+        Returns:
+            None
+        """
+        if logger.isEnabledFor(DEBUG):
+            logger.debug('Calling task post creation')
+        if self.task is not None and not isinstance(self.task, TaskProxy):
+            self.task.post_creation(self, self.platform)
+
+        IItem.post_creation(self, platform)
+        if self.__replace_task_with_proxy or (self.parent and self.parent._Experiment__replace_task_with_proxy):
+            if logger.isEnabledFor(DEBUG):
+                logger.debug('Replacing task with proxy')
+            self.task = TaskProxy.from_task(self.task)
+        if self.status is None:
+            self.status = EntityStatus.CREATED
+        self.platform = platform
+
+    def pre_getstate(self):
+        """
+        Return default values for :meth:`pickle_ignore_fields`. Call before pickling.
+        """
+        from idmtools.assets import AssetCollection
+        from idmtools.core.interfaces.entity_container import EntityContainer
+        return {"assets": AssetCollection(), "simulations": EntityContainer()}
+
+    def gather_assets(self):
+        """
+        Gather all the assets for the simulation.
+        """
+        if not self.__assets_gathered:
+            self.task.gather_transient_assets()
+            self.assets.add_assets(self.task.transient_assets, fail_on_duplicate=False)
+        self.__assets_gathered = True
+
+    @classmethod
+    def from_task(cls, task: 'ITask', tags: Dict[str, Any] = None,  # noqa E821
+                  asset_collection: AssetCollection = None):
+        """
+        Create a simulation from a task.
+
+        Args:
+            task: Task to create from
+            tags: Tags to create on the simulation
+            asset_collection: Simulation Assets
+
+        Returns:
+            Simulation using the  parameters provided
+        """
+        return Simulation(task=task, tags=dict() if tags is None else tags,
+                          assets=asset_collection if asset_collection else AssetCollection())
+
+    def list_static_assets(self, platform: 'IPlatform' = None, **kwargs) -> List[Asset]:
+        """
+        List assets that have been uploaded to a server already.
+
+        Args:
+            platform: Optional platform to load assets list from
+            **kwargs:
+
+        Returns:
+            List of assets
+
+        Raises:
+            ValueError - If you try to list an assets for an simulation that hasn't been created/loaded from a remote platform.
+        """
+        if self.id is None:
+            raise ValueError("You can only list static assets on an existing experiment")
+        p = super()._check_for_platform_from_context(platform)
+        return p._simulations.list_assets(self, **kwargs)
+
+    def to_dict(self) -> Dict:
+        """
+        Do a lightweight conversation to json.
+
+        Returns:
+            Dict representing json of object
+        """
+        result = dict()
+        for f in fields(self):
+            if not f.name.startswith("_") and f.name not in ['parent']:
+                result[f.name] = getattr(self, f.name)
+        result['_uid'] = self.uid
+        result['task'] = self.task.to_dict() if self.task else None
+        return result
+
+
+# TODO Rename to T simulation once old simulation is one
+TTSimulation = TypeVar("TTSimulation", bound=Simulation)
+TTSimulationClass = Type[TTSimulation]
+TTSimulationBatch = List[TTSimulation]
+TTAllSimulationData = Mapping[TTSimulation, Any]
+TTSimulationList = List[Union[TTSimulation, str]]

idmtools/entities/suite.py

@@ -0,0 +1,188 @@
+"""
+Defines our Suite object.
+
+The Suite object can be thought as a metadata object. It represents a container object for Experiments. All Suites should have one or more experiments.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+from typing import NoReturn, Type, TYPE_CHECKING, Dict, List
+from abc import ABC
+from dataclasses import dataclass, field, fields
+from idmtools.core.interfaces.iitem import IItem
+from idmtools.core.interfaces.inamed_entity import INamedEntity
+from idmtools.core import ItemType, EntityContainer
+from idmtools.core.interfaces.irunnable_entity import IRunnableEntity
+
+if TYPE_CHECKING:  # pragma: no cover
+    from idmtools.entities.iplatform import IPlatform
+    from idmtools.entities.experiment import Experiment
+
+
+@dataclass(repr=False)
+class Suite(INamedEntity, ABC, IRunnableEntity):
+    """
+    Class that represents a generic suite (a collection of experiments).
+
+    Args:
+        experiments: The child items of this suite.
+    """
+    experiments: EntityContainer = field(
+        default_factory=lambda: EntityContainer(),
+        compare=False,
+        metadata={"pickle_ignore": True}
+    )
+
+    item_type: ItemType = field(default=ItemType.SUITE, compare=False, init=False)
+    description: str = field(default=None, compare=False)
+
+    def __post_init__(self):
+        """
+        Initialize Suite.
+        """
+        self.experiments = EntityContainer()
+
+    def add_experiment(self, experiment: 'Experiment') -> 'NoReturn':  # noqa: F821
+        """
+        Add an experiment to the suite.
+
+        Args:
+            experiment: the experiment to be linked to suite
+        """
+        # Link the suite to the experiment. Assumes the experiment suite setter adds the experiment to the suite.
+        experiment._parent = self
+        experiment.parent_id = experiment.suite_id = self.id
+
+        ids = [exp.uid for exp in self.experiments]
+        if experiment.uid in ids:
+            return
+
+        # add experiment
+        self.experiments.append(experiment)
+
+        # clear possible directory cache for Experiment and Simulation
+        experiment.clear_directory_cache()
+
+    def display(self):
+        """
+        Display workflowitem.
+
+        Returns:
+            None
+        """
+        from idmtools.utils.display import display, suite_table_display
+        display(self, suite_table_display)
+
+    def pre_creation(self, platform: 'IPlatform'):
+        """
+        Pre Creation of IWorkflowItem.
+
+        Args:
+            platform: Platform we are creating item on
+
+        Returns:
+            None
+        """
+        IItem.pre_creation(self, platform)
+
+    def post_creation(self, platform: 'IPlatform'):
+        """
+        Post Creation of IWorkflowItem.
+
+        Args:
+            platform: Platform
+
+        Returns:
+            None
+        """
+        IItem.post_creation(self, platform)
+
+    def __repr__(self):
+        """
+        String representation of suite.
+        """
+        count = len(self.experiments) if self.experiments is not None else 0
+        return f"<Suite {self.uid} - {count} experiments>"
+
+    @property
+    def done(self):
+        """
+        Return if a suite has finished executing.
+
+        Returns:
+            True if all experiments have ran, False otherwise
+        """
+        return all([s.done for s in self.experiments])
+
+    @property
+    def succeeded(self) -> bool:
+        """
+        Return if a suite has succeeded. A suite is succeeded when all experiments have succeeded.
+
+        Returns:
+            True if all experiments have succeeded, False otherwise
+        """
+        return all([s.succeeded for s in self.experiments])
+
+    def to_dict(self) -> Dict:
+        """
+        Converts suite to a dictionary.
+
+        Returns:
+            Dictionary of suite.
+        """
+        result = dict()
+        for f in fields(self):
+            if not f.name.startswith("_") and f.name not in ['parent']:
+                result[f.name] = getattr(self, f.name)
+        result['_uid'] = self.uid
+        return result
+
+    def get_experiments(self) -> EntityContainer:
+        """
+        Retrieve the experiments associated with this suite from the platform.
+        Returns:
+            EntityContainer: A container of Experiment objects belonging to this suite.
+        """
+        experiments = self.experiments
+        if not experiments:
+            experiments = self.platform.get_children(self.id, ItemType.SUITE, force=True)
+        return experiments
+
+    def get_simulations_by_tags(self, tags=None, status=None, entity_type=False, skip_sims=None, max_simulations=None,
+                                **kwargs) -> Dict[str, List[str]]:
+        """
+        Retrieve simulation ids or simulation objects with matching tags across all experiments in the suite.
+        This method filters simulations based on the provided tags, skipping specified simulations,
+        and limiting the number of results if `max_simulations` is set. The return type can be
+        either a dictionary of simulation IDs or simulation objects, depending on the `entity_type` flag.
+        Args:
+            status:
+            tags (dict, optional): A simulation's tags to filter by.
+            status (EntityStatus, Optional): Simulation status.
+            entity_type (bool, optional): If True, return simulation objects; otherwise, return simulation IDs. Defaults to False.
+            skip_sims (List[str], optional): A list of simulation IDs to exclude from the results.
+            max_simulations (int, optional): The maximum number of simulations to return per experiment.
+            **kwargs: Additional filter parameters.
+        Returns:
+            Dict[str, List[str]]: A dictionary where the keys are experiment IDs and the values are lists of
+                                  simulation IDs or simulation objects, depending on the `entity_type` flag.
+        """
+        experiments = self.get_experiments()
+        sims = {}
+        for experiment in experiments:
+            sims[experiment.id] = experiment.get_simulations_by_tags(tags=tags, status=status, entity_type=entity_type,
+                                                                     skip_sims=skip_sims,
+                                                                     max_simulations=max_simulations, **kwargs)
+        return sims
+
+    def __setstate__(self, state):
+        """
+        Add ignored fields back since they don't exist in the pickle.
+        """
+        # First call parent's __setstate__ to restore base attributes
+        super().__setstate__(state)
+        # Restore the pickle fields with values requested
+        self.experiments = EntityContainer()
+
+
+ISuiteClass = Type[Suite]

idmtools/entities/task_proxy.py

@@ -0,0 +1,37 @@
+"""
+Defines our TaskProxy object.
+
+The TaskProxy object is mean to reduce the memory requirements of large simulation sets/configurations after provisioning. Instead of keeping the full original object in memory,
+the object is replaced with a proxy object with minimal information needed to work with the task.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+from typing import Union
+from dataclasses import dataclass, field
+from idmtools.entities import CommandLine
+
+
+@dataclass
+class TaskProxy:
+    """
+    This class is used to reduce the memory footprint of tasks after a simulation has been provisioned.
+    """
+    command: Union[str, CommandLine] = field(default=None)
+    is_docker: bool = False
+    is_gpu: bool = False
+
+    @staticmethod
+    def from_task(task: 'ITask'):  # noqa: F821
+        """
+        Create a task proxy from a task.
+
+        Args:
+            task: Task to proxy
+
+        Returns:
+            TaskProxy of task
+        """
+        from idmtools.core.docker_task import DockerTask
+        is_docker = isinstance(task, DockerTask)
+        item = TaskProxy(command=task.command, is_docker=is_docker, is_gpu=is_docker and task.use_nvidia_run)
+        return item