idmtools 0.0.0.dev0__py3-none-any.whl → 0.0.3__py3-none-any.whl

idmtools/assets/file_list.py

@@ -0,0 +1,191 @@
+"""
+idmtools FileList classes.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+from typing import List
+import os
+import re
+from idmtools.assets import Asset, AssetCollection
+from idmtools.utils.local_os import LocalOS
+
+
+class FileList:
+    """
+    Special utility class to help handling user files.
+    """
+
+    def __init__(self, root=None, files_in_root=None, recursive=False, ignore_missing=False, relative_path=None,
+                 max_depth=3):
+        """
+        Represents a set of files that are specified RELATIVE to root.
+
+        e.g. /a/b/c.json could be : root: '/a' files_in_root: ['b/c.json']
+        :param root: The dir all files_in_root are relative to.
+        :param files_in_root: The listed files
+        """
+        self.files: List[Asset] = []
+        self.ignore_missing = ignore_missing
+        self.max_depth = max_depth
+
+        # Make sure we have correct separator
+        # os.path.normpath(f) would be best but is not working the same way on UNIX systems
+        if files_in_root is not None:
+            if LocalOS.is_window():
+                files_in_root = [os.path.normpath(f) for f in files_in_root]
+            else:
+                files_in_root = [re.sub(r"[\\/]", os.sep, os.path.normpath(f)) for f in files_in_root]
+
+        if root:
+            self.add_path(path=root, files_in_dir=files_in_root, recursive=recursive, relative_path=relative_path)
+
+    def add_asset_file(self, af):
+        """
+        Method used to add asset file.
+
+        Args:
+            af: asset file to add
+
+        Returns: None
+        """
+        self.files.append(af)
+
+    def add_file(self, path, relative_path=''):
+        """
+        Method used to add a file.
+
+        Args:
+            path: file oath
+            relative_path: file relative path
+
+        Returns: None
+        """
+        # If already present -> bypass
+        for f in self.files:
+            if f.absolute_path and os.path.abspath(f.absolute_path) == os.path.abspath(path):
+                return
+
+        if os.path.isdir(path):
+            raise ValueError("%s is a directory. add_file is expecting a file!" % path)
+
+        absolute_path = os.path.abspath(path)
+        file_name = os.path.basename(path)
+        try:
+            af = Asset(filename=file_name, relative_path=relative_path, absolute_path=absolute_path)
+            self.add_asset_file(af)
+        except Exception as e:
+            if not self.ignore_missing:
+                raise e
+
+    def add_path(self, path, files_in_dir=None, relative_path=None, recursive=False):
+        """
+        Add a path to the file list.
+
+        Args:
+            path: The path to add (needs to be a dictionary)
+            files_in_dir: If we want to only retrieve certain files in this path
+            relative_path: relative_path: The relative path prefixed to each added files
+            recursive: Do we want to browse recursively
+
+        Returns: None
+
+        """
+        # Little safety
+        if not os.path.isdir(path) and not path.startswith('\\\\'):
+            raise RuntimeError("add_path() requires a directory. '%s' is not." % path)
+
+        if not recursive:
+            if files_in_dir is None:
+                files_in_dir = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))]
+
+            for file_name in files_in_dir:
+                file_path = os.path.join(path, file_name)
+                f_relative_path = os.path.normpath(
+                    file_path.replace(path, '').strip(os.sep).replace(os.path.basename(file_path), ''))
+                if relative_path is not None:
+                    f_relative_path = os.path.join(relative_path, f_relative_path)
+                self.add_file(file_path, relative_path=f_relative_path)
+
+        else:
+            # Walk through the path
+            for root, _subdirs, files in os.walk(path):
+                # Little safety to not go too deep
+                depth = root[len(path) + len(os.path.sep):].count(os.path.sep)
+                if depth > self.max_depth:
+                    continue
+
+                # Add the files in the current dir
+                for f in files:
+                    # Find the file relative path compared to the root folder
+                    # If the relative_path is . -> change it into ''
+                    f_relative_path = os.path.normpath(os.path.relpath(root, path))
+                    if f_relative_path == '.':
+                        f_relative_path = ''
+
+                    # if files_in_dir specified -> skip the ones not included
+                    if files_in_dir is not None and f not in files_in_dir and os.path.join(f_relative_path,
+                                                                                           f) not in files_in_dir:
+                        continue
+
+                    # if we want to force a relative path -> force it
+                    if relative_path is not None:
+                        f_relative_path = os.path.join(relative_path, f_relative_path)
+
+                    # add the file
+                    self.add_file(os.path.join(root, f), relative_path=f_relative_path)
+
+    def to_asset_collection(self) -> AssetCollection:
+        """
+        Convert a file list to an asset collection.
+
+        Returns:
+            AssetCollection version of filelist
+        """
+        ac = AssetCollection()
+        ac.assets = self.files
+        return ac
+
+    @staticmethod
+    def from_asset_collection(asset_collection: AssetCollection) -> 'FileList':
+        """
+        Create a FileList from a AssetCollection.
+
+        Args:
+            asset_collection: AssetCollection to convert.
+
+        Returns:
+            FileList version of AssetCollection
+        """
+        fl = FileList()
+        fl.files = [asset_collection.assets]
+        return fl
+
+    def __len__(self):
+        """
+        Length of FileList.
+
+        Returns:
+            Length
+        """
+        return len(self.files)
+
+    def __iter__(self):
+        """
+        Iterator. Basically we proxy self.files.
+
+        Returns:
+            Iterator
+        """
+        return self.files.__iter__()
+
+    def __getitem__(self, item):
+        """
+        Get file.
+
+        Args:
+            item: Item
+
+        Returns:
+            Item
+        """
+        return self.files.__getitem__(item)

idmtools/builders/__init__.py

@@ -0,0 +1,11 @@
+"""
+idmtools builders package.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+# flake8: noqa F821
+from idmtools.builders.simulation_builder import SimulationBuilder
+from idmtools.builders.sweep_arm import SweepArm, ArmType, TSweepFunction
+from idmtools.builders.arm_simulation_builder import ArmSimulationBuilder
+from idmtools.builders.csv_simulation_builder import CsvExperimentBuilder
+from idmtools.builders.yaml_simulation_builder import YamlSimulationBuilder

idmtools/builders/arm_simulation_builder.py

@@ -0,0 +1,152 @@
+"""
+idmtools arm builder definition.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+from idmtools.builders import SweepArm
+from itertools import tee
+
+
+class ArmSimulationBuilder:
+    """
+    Class that represents an experiment builder.
+
+    This particular sweep builder build sweeps in "ARMS". This is particular useful in situations where you want to sweep
+    parameters that have branches of parameters. For Example, let's say we have a model with the following parameters:
+    * population
+    * susceptible
+    * recovered
+    * enable_births
+    * birth_rate
+
+    Enable births controls an optional feature that is controlled by the birth_rate parameter. If we want to sweep a set
+    of parameters on population, susceptible with enabled_births set to off but also want to sweep the birth_rate
+    we could do that like so
+
+    .. literalinclude:: ../../examples/builders/print_builder_values.py
+
+    This would result in the output
+
+    .. list-table:: Arm Example Values
+       :widths: 25 25 25 25
+       :header-rows: 1
+
+       * - enable_births
+         - population
+         - susceptible
+         - birth_rate
+       * - False
+         - 500
+         - 0.5
+         -
+       * - False
+         - 500
+         - 0.9
+         -
+       * - False
+         - 1000
+         - 0.5
+         -
+       * - False
+         - 1000
+         - 0.9
+         -
+       * - True
+         - 500
+         - 0.5
+         - 0.01
+       * - True
+         - 500
+         - 0.5
+         - 0.1
+       * - True
+         - 500
+         - 0.9
+         - 0.01
+       * - True
+         - 500
+         - 0.9
+         - 0.1
+       * - True
+         - 1000
+         - 0.5
+         - 0.01
+       * - True
+         - 1000
+         - 0.5
+         - 0.1
+       * - True
+         - 1000
+         - 0.9
+         - 0.01
+       * - True
+         - 1000
+         - 0.9
+         - 0.1
+
+    Examples:
+        .. literalinclude:: ../../examples/builders/arm_experiment_builder_python.py
+    """
+
+    def __init__(self):
+        """
+        Constructor.
+        """
+        super().__init__()
+        self.arms = []
+
+    @property
+    def count(self):
+        """
+        Total simulations to be built by builder.
+        Returns:
+            arm count
+        """
+        return sum([arm.count for arm in self.arms])
+
+    def add_arm(self, arm):
+        """
+        Add arm sweep definition.
+        Args:
+            arm: Arm to add
+        Returns:
+            None
+        """
+        # create a new arm
+        arm_new = SweepArm(type=arm.type)
+        old_sweeps = []
+        new_sweeps = []
+        # make two copies of each sweep
+        for sweep in arm.sweeps:
+            old_sw, new_sw = tee(sweep, 2)
+            old_sweeps.append(old_sw)
+            new_sweeps.append(new_sw)
+
+        # keep original sweeps
+        arm.sweeps = old_sweeps
+        # copy sweeps to new arm
+        arm_new.sweeps = new_sweeps
+        # update count for new arm
+        arm_new.count = arm.count
+
+        # add new arm to arms
+        self.arms.append(arm_new)
+        # update sweep functions for new arm
+        arm_new._update_sweep_functions()
+
+    def __iter__(self):
+        """
+        Iterator for the simulations defined.
+        Returns:
+            Iterator
+        """
+        for arm in self.arms:
+            yield from arm.functions
+
+    def __len__(self):
+        """
+        Total simulations to be built by builder.
+        Returns:
+            Simulation count
+        """
+        return self.count

idmtools/builders/csv_simulation_builder.py

@@ -0,0 +1,76 @@
+"""
+idmtools CsvExperimentBuilder definition.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+import pandas as pd
+import numpy as np
+from numbers import Number
+from idmtools.builders import ArmSimulationBuilder, SweepArm
+
+
+class CsvExperimentBuilder(ArmSimulationBuilder):
+    """
+    Class that represents an experiment builder.
+    Examples:
+        .. literalinclude:: ../../examples/builders/csv_builder_python.py
+    """
+
+    def add_sweeps_from_file(self, file_path, func_map=None, type_map=None, sep=","):
+        """
+        Create sweeps from a CSV file.
+        Args:
+            file_path: Path to file
+            func_map: Function map
+            type_map: Type
+            sep: CSV Seperator
+        Returns:
+            None
+        """
+        if type_map is None:
+            type_map = {}
+        if func_map is None:
+            func_map = {}
+
+        def strip_column(x):
+            """
+            Strip white spaces for Number type column.
+            """
+            y = x.strip() if not isinstance(x, Number) else x
+            return np.nan if y == '' else y
+
+        # make up our column converter
+        convert_map = {c: strip_column for c, v in type_map.items() if
+                       v in (np.int64, np.float64, np.int64, np.float64)}
+
+        # load csv with our converter
+        # df_sweeps = pd.read_csv(file_path, sep=sep)
+        df_sweeps = pd.read_csv(file_path, sep=sep, converters=convert_map)
+
+        # go through each of rows
+        row_count = df_sweeps.shape[0]
+        for k in range(row_count):
+            self.sweeps = []
+
+            # get the current row as DataFrame
+            df = df_sweeps.iloc[[k]]
+
+            # drop columns with nan
+            df = df.dropna(axis=1)
+
+            # make parameter with the correct value type
+            type_map_t = {k: v for k, v in type_map.items() if k in df.columns.tolist()}
+            df = df.astype(type_map_t)
+            # make dict like: {'a': [1], 'b': [2]}
+            sweep = df.to_dict(orient='list')
+
+            # create an arm
+            arm = SweepArm()
+
+            # go through each (key, value)
+            for param, value in sweep.items():
+                # get the mapping function
+                func = func_map[param]
+                arm.add_sweep_definition(func, value)
+
+            self.add_arm(arm)