idmtools 0.0.0.dev0__py3-none-any.whl → 0.0.2__py3-none-any.whl

idmtools/entities/templated_simulation.py

@@ -0,0 +1,325 @@
+"""
+TemplatedSimulations provides a utility to build sets of simulations from a base simulation.
+
+This is meant to be combined with builders.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+import copy
+from dataclasses import dataclass, field, fields, InitVar
+from functools import partial
+from itertools import chain
+from typing import Set, Generator, Dict, Any, List, TYPE_CHECKING, Union
+from more_itertools import grouper
+from idmtools.entities.itask import ITask
+from idmtools.entities.simulation import Simulation
+from idmtools.utils.collections import ResetGenerator
+from idmtools.utils.hashing import ignore_fields_in_dataclass_on_pickle
+from idmtools.builders import SimulationBuilder, ArmSimulationBuilder
+
+if TYPE_CHECKING:  # pragma: no cover
+    from idmtools.entities.experiment import Experiment
+
+
+def simulation_generator(builders, new_sim_func, additional_sims=None, batch_size=10):
+    """
+    Generates batches of simulations from the templated simulations.
+
+    Args:
+        builders: List of builders to build
+        new_sim_func: Build new simulation callback
+        additional_sims: Additional simulations
+        batch_size: Batch size
+
+    Returns:
+        Generator for simulations in batches
+    """
+    if additional_sims is None:
+        additional_sims = []
+    # Then the builders
+    for groups in grouper(chain(*builders), batch_size):
+        for simulation_functions in filter(None, groups):
+            simulation = new_sim_func()
+            tags = {}
+
+            for func in simulation_functions:
+                new_tags = func(simulation=simulation)
+                if new_tags:
+                    tags.update(new_tags)
+
+            simulation.tags.update(tags)
+            yield simulation
+
+    yield from additional_sims
+
+
+@dataclass(repr=False)
+class TemplatedSimulations:
+    """
+    Class for building templated simulations and commonly used with SimulationBuilder class.
+
+    Examples:
+        Add tags to all simulations via base task::
+
+            ts = TemplatedSimulations(base_task=task)
+            ts.tags = {'a': 'test', 'b': 9}
+            ts.add_builder(builder)
+
+        Add tags to a specific simulation::
+
+            experiment =  Experiment.from_builder(builder, task, name=expname)
+            experiment.simulations = list(experiment.simulations)
+            experiment.simulations[2].tags['test']=123
+    """
+    builders: Set[Union[SimulationBuilder, ArmSimulationBuilder]] = field(default_factory=set, compare=False)
+    base_simulation: Simulation = field(default=None, compare=False, metadata={"pickle_ignore": True})
+    base_task: ITask = field(default=None)
+    parent: 'Experiment' = field(default=None)
+    tags: InitVar[Dict] = None
+    __extra_simulations: List[Simulation] = field(default_factory=list)
+
+    def __post_init__(self, tags):
+        """
+        Constructor.
+
+        Args:
+            tags: Tags to set on the base simulation
+
+        Returns:
+            None
+
+        Raises:
+            ValueError - If base task is not set, and base simulations is not set or bas_simulation_task is not set.
+        """
+        if self.base_task is None and (self.base_simulation is None or self.base_simulation.task is None):
+            raise ValueError("Either a base simulation or a base_task are required")
+
+        if self.base_simulation is None:
+            self.base_simulation = Simulation(task=self.base_task)
+        else:
+            self.base_task = self.base_simulation.task
+
+        if tags and not isinstance(tags, property):
+            self.base_simulation.tags.update(tags)
+
+    @property
+    def builder(self) -> Union[SimulationBuilder, ArmSimulationBuilder]:
+        """
+        For backward-compatibility purposes.
+
+        Returns:
+            The last ``TExperimentBuilder``.
+        """
+        return list(self.builders)[-1] if self.builders and len(self.builders) > 0 else None
+
+    @builder.setter
+    def builder(self, builder: Union[SimulationBuilder, ArmSimulationBuilder]) -> None:
+        """
+        For backward-compatibility purposes.
+
+        Args:
+            builder: The new builder to be used.
+
+        Returns:
+            None
+        """
+        # Make sure we only take the last builder assignment
+        if self.builders:
+            self.builders.clear()
+
+        self.add_builder(builder)
+
+    def add_builder(self, builder: Union[SimulationBuilder, ArmSimulationBuilder]) -> None:
+        """
+        Add builder to builder collection.
+
+        Args:
+            builder: A builder to be added.
+
+        Returns:
+            None
+
+        Raises:
+            ValueError - Builder must be type of SimulationBuilder
+        """
+        # Add builder validation
+        if not isinstance(builder, (SimulationBuilder, ArmSimulationBuilder)):
+            raise ValueError("Builder ({}) must have type of ExperimentBuilder!".format(builder))
+
+        # Initialize builders the first time
+        if self.builders is None:
+            self.builders = set()
+
+        # Add new builder to the collection
+        self.builders.add(builder)
+
+    @property
+    def pickle_ignore_fields(self):
+        """
+        Fields that we should ignore on the object.
+
+        Returns:
+            Fields to ignore
+        """
+        return set(f.name for f in fields(self) if "pickle_ignore" in f.metadata and f.metadata["pickle_ignore"])
+
+    def display(self):
+        """
+        Display the templated simulation.
+
+        Returns:
+            None
+        """
+        from idmtools.utils.display import display, experiment_table_display
+        display(self, experiment_table_display)
+
+    def simulations(self) -> Generator[Simulation, None, None]:
+        """
+        Simulations iterator.
+
+        Returns:
+            Simulation iterator
+        """
+        p = partial(simulation_generator, self.builders, self.new_simulation, self.__extra_simulations)
+        return ResetGenerator(p)
+
+    def extra_simulations(self) -> List[Simulation]:
+        """
+        Returns the extra simulations defined on template.
+
+        Returns:
+            Returns the extra simulations defined
+        """
+        return self.__extra_simulations
+
+    def add_simulation(self, simulation: Simulation):
+        """
+        Add a simulation that was built outside template engine to template generator.
+
+        This is useful we you can build most simulations through a template but need a some that cannot. This is especially true
+        for large simulation sets.
+
+        Args:
+            simulation: Simulation to add
+
+        Returns:
+            None
+        """
+        self.__extra_simulations.append(simulation)
+
+    def add_simulations(self, simulations: List[Simulation]):
+        """
+        Add multiple simulations without templating. See add_simulation.
+
+        Args:
+            simulations: Simulation to add
+
+        Returns:
+            None
+        """
+        self.__extra_simulations.extend(simulations)
+
+    def new_simulation(self):
+        """
+        Return a new simulation object.
+
+        The simulation will be copied from the base simulation of the experiment.
+
+        Returns:
+            The created simulation.
+        """
+        # TODO: the experiment should be frozen when the first simulation is created
+        sim = copy.deepcopy(self.base_simulation)
+        # Set UID=none to ensure it is regenerated
+        sim._uid = None
+        sim.assets = copy.deepcopy(self.base_simulation.assets)
+        sim.parent = self.parent
+        return sim
+
+    @property
+    def tags(self):  # noqa: F811
+        """
+        Get tags for the base simulation.
+
+        Returns:
+            Tags for base simulation
+        """
+        return self.base_simulation.tags
+
+    @tags.setter
+    def tags(self, tags):
+        """
+        Set tags on the base simulation.
+
+        Args:
+            tags: Tags to set
+
+        Returns:
+            None
+        """
+        self.base_simulation.tags = tags
+
+    def __iter__(self):
+        """
+        Iterator over the simulations.
+
+        Returns:
+            Simulations itero
+        """
+        return self.simulations()
+
+    def __getstate__(self):
+        """
+        Ignore the fields in pickle_ignore_fields during pickling.
+        """
+        return ignore_fields_in_dataclass_on_pickle(self)
+
+    def __setstate__(self, state):
+        """
+        Add ignored fields back since they don't exist in the pickle.
+        """
+        self.__dict__.update(state)
+
+    def __len__(self):
+        """
+        Length of the templated simulations.
+
+        Returns:
+            Total number of simulations
+        """
+        return sum([len(b) for b in self.builders]) + len(self.__extra_simulations)
+
+    @classmethod
+    def from_task(cls, task: ITask, tags: Dict[str, Any] = None) -> 'TemplatedSimulations':
+        """
+        Creates a templated simulation from a task.
+
+        We use the task to set as base_task, and the tags are applied to the base simulation we need internally.
+
+        Args:
+            task: Task to use as base task
+            tags: Tags to add to base simulation
+
+        Returns:
+            TemplatedSimulations from the task
+        """
+        return TemplatedSimulations(base_task=task, tags=tags)
+
+    def check_duplicate(self, simulation_id: str) -> bool:
+        """
+        Check if a simulation id is already in the templated simulations.
+        Args:
+            simulation_id:
+
+        Returns:
+            True if the simulation id is already in the templated simulations, False otherwise.
+        """
+        ids = [sim.id for sim in self.__extra_simulations]
+        return simulation_id in ids
+
+    def clear_extra_simulation_directory_cache(self):
+        """
+        Clear templated extra simulations directory cache.
+        """
+        for sim in self.__extra_simulations:
+            sim._platform_directory = None

idmtools/frozen/frozen_dict.py

@@ -0,0 +1,71 @@
+"""
+Frozen_dict provided utilities for a read-only dict.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+from collections import UserDict
+from idmtools.frozen.ifrozen import IFrozen
+
+
+class ImDict(UserDict, IFrozen):
+    """
+    ImDict is a frozen wrapper for dict.
+    """
+
+    def __setitem__(self, *args, **kwargs):  # real signature unknown
+        """Set self[key] to value."""
+        if self._frozen:
+            raise Exception('Frozen')
+        super().__setitem__(*args, **kwargs)
+
+    def __delitem__(self, *args, **kwargs):  # real signature unknown
+        """Delete self[key]."""
+        if self._frozen:
+            raise Exception('Frozen')
+        super().__delitem__(*args, **kwargs)
+
+    def setdefault(self, *args, **kwargs):  # real signature unknown
+        """
+        Insert key with a value of default if key is not in the dictionary.
+
+        Return the value for key if key is in the dictionary, else default.
+        """
+        if self._frozen:
+            raise Exception('Frozen')
+        super().setdefault(*args, **kwargs)
+
+    def update(self, E=None, **F):  # known special case of dict.update
+        """
+        Update the dict.
+
+        D.update([E, ]**F) -> None.  Update D from dict/iterable E and F.
+
+        If E is present and has a .keys() method, then does:  for k in E: D[k] = E[k]
+        If E is present and lacks a .keys() method, then does:  for k, v in E: D[k] = v
+        In either case, this is followed by: for k in F:  D[k] = F[k]
+        """
+        if self._frozen:
+            raise Exception('Frozen')
+        super().update(E, **F)
+
+    def pop(self, k, d=None):  # real signature unknown; restored from __doc__
+        """
+        Pop from dictionary.
+
+        D.pop(k[,d]) -> v, remove specified key and return the corresponding value.
+
+        If key is not found, d is returned if given, otherwise KeyError is raised
+        """
+        if self._frozen:
+            raise Exception('Frozen')
+        super().pop(k, d)
+
+    def popitem(self):  # real signature unknown; restored from __doc__
+        """
+        Pop item from dictionary.
+
+        Popitem - D.popitem() -> (k, v), remove and return some (key, value) pair as a 2-tuple; but raise KeyError if D is empty.
+        """
+        if self._frozen:
+            raise Exception('Frozen')
+        super().popitem()

idmtools/frozen/frozen_list.py

@@ -0,0 +1,66 @@
+"""
+Frozen_list provided utilities for a read-only list.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+from collections import UserList
+from idmtools.frozen.ifrozen import IFrozen
+
+
+class ImList(UserList, IFrozen):
+    """FrozenList wrapper."""
+    def append(self, *args, **kwargs):  # real signature unknown
+        """Append object to the end of the list."""
+        if self._frozen:
+            raise Exception('Frozen')
+        super().append(*args, **kwargs)
+
+    def clear(self, *args, **kwargs):  # real signature unknown
+        """Remove all items from list."""
+        if self._frozen:
+            raise Exception('Frozen')
+        super().clear(*args, **kwargs)
+
+    def extend(self, *args, **kwargs):  # real signature unknown
+        """Extend list by appending elements from the iterable."""
+        if self._frozen:
+            raise Exception('Frozen')
+        super().extend(*args, **kwargs)
+
+    def insert(self, *args, **kwargs):  # real signature unknown
+        """Insert object before index."""
+        if self._frozen:
+            raise Exception('Frozen')
+        super().insert(*args, **kwargs)
+
+    def pop(self, *args, **kwargs):  # real signature unknown
+        """
+        Remove and return item at index (default last).
+
+        Raises IndexError if list is empty or index is out of range.
+        """
+        if self._frozen:
+            raise Exception('Frozen')
+        super().pop(*args, **kwargs)
+
+    def remove(self, *args, **kwargs):  # real signature unknown
+        """
+        Remove first occurrence of value.
+
+        Raises ValueError if the value is not present.
+        """
+        if self._frozen:
+            raise Exception('Frozen')
+        super().remove(*args, **kwargs)
+
+    def __setitem__(self, key, item):
+        """Set item. Raises exception if frozen."""
+        if self._frozen:
+            raise Exception('Frozen')
+        super().__setitem__(key, item)
+
+    def __delitem__(self, key):
+        """Delete item. Raises exception if frozen."""
+        if self._frozen:
+            raise Exception('Frozen')
+        super().__delitem__(key)

idmtools/frozen/frozen_set.py

@@ -0,0 +1,86 @@
+"""
+Frozen_set provided utilities for a read-only set.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+from idmtools.frozen.ifrozen import IFrozen
+
+
+class ImSet(set, IFrozen):
+    """
+    ImSet is a frozen wrapper for Sets.
+    """
+
+    def add(self, *args, **kwargs):  # real signature unknown
+        """
+        Add an element to a set.
+
+        This has no effect if the element is already present.
+        """
+        if self._frozen:
+            raise Exception('Frozen')
+        super().add(*args, **kwargs)
+
+    def clear(self, *args, **kwargs):  # real signature unknown
+        """Remove all elements from this set."""
+        if self._frozen:
+            raise Exception('Frozen')
+        super().clear(*args, **kwargs)
+
+    def difference_update(self, *args, **kwargs):  # real signature unknown
+        """Remove all elements of another set from this set."""
+        if self._frozen:
+            raise Exception('Frozen')
+        super().difference_update(*args, **kwargs)
+
+    def discard(self, *args, **kwargs):  # real signature unknown
+        """
+        Remove an element from a set if it is a member.
+
+        If the element is not a member, do nothing.
+        """
+        if self._frozen:
+            raise Exception('Frozen')
+        super().discard(*args, **kwargs)
+
+    def intersection_update(self, *args, **kwargs):  # real signature unknown
+        """Update a set with the intersection of itself and another."""
+        if self._frozen:
+            raise Exception('Frozen')
+        super().intersection_update(*args, **kwargs)
+
+    def pop(self, *args, **kwargs):  # real signature unknown
+        """
+        Remove and return an arbitrary set element.
+
+        Raises KeyError if the set is empty.
+        """
+        if self._frozen:
+            raise Exception('Frozen')
+        super().pop(*args, **kwargs)
+
+    def remove(self, *args, **kwargs):  # real signature unknown
+        """
+        Remove an element from a set; it must be a member.
+
+        If the element is not a member, raise a KeyError.
+        """
+        if self._frozen:
+            raise Exception('Frozen')
+        super().remove(*args, **kwargs)
+
+    def union(self, *args, **kwargs):  # real signature unknown
+        """
+        Return the union of sets as a new set.
+
+        (i.e. all elements that are in either set.)
+        """
+        if self._frozen:
+            raise Exception('Frozen')
+        super().union(*args, **kwargs)
+
+    def update(self, *args, **kwargs):  # real signature unknown
+        """Update a set with the union of itself and others."""
+        if self._frozen:
+            raise Exception('Frozen')
+        super().update(*args, **kwargs)

idmtools/frozen/frozen_tuple.py

@@ -0,0 +1,18 @@
+"""
+Frozen_tuple provided utilities for a read-only tuple.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+from idmtools.frozen.ifrozen import IFrozen
+
+
+class ImTuple(tuple, IFrozen):
+    """
+    Doc: Once a tuple is created, you cannot add items to it. Tuples are unchangeable.
+    """
+
+    def __add__(self, *args, **kwargs):  # real signature unknown
+        """Return self+value."""
+        if self._frozen:
+            raise Exception('Frozen')
+        super().__add__(*args, **kwargs)