idmtools 0.0.0.dev0__py3-none-any.whl → 0.0.3__py3-none-any.whl

idmtools/utils/display/displays.py

@@ -0,0 +1,181 @@
+"""
+Tools around displays and formatting.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+from abc import ABCMeta, abstractmethod
+from typing import Any
+from tabulate import tabulate
+
+from idmtools.utils.collections import cut_iterable_to
+
+
+class IDisplaySetting(metaclass=ABCMeta):
+    """
+    Base class for a display setting.
+
+    The child class needs to implement the :meth:`display` method.
+
+    Includes:
+
+    - header: Optional header for the display.
+    - field: If specified, the :meth:`get_object` will call :attr:`getattr` for this field on the object.
+    """
+
+    def __init__(self, header: str = None, field: str = None):
+        """
+        Initialize our IDisplaySetting.
+
+        Args:
+            header: Header for display
+            field: Optional field to display instead of object
+        """
+        self.header = header
+        self.field = field
+
+    def get_object(self, obj: Any) -> Any:
+        """
+        Get object or field depending if field is set.
+
+        Args:
+            obj: Object to get
+
+        Returns:
+            Either obj.field or obj depending if self.field is set
+        """
+        return getattr(obj, self.field) if self.field else obj
+
+    @abstractmethod
+    def display(self, obj: Any) -> str:
+        """
+        Display the object.
+
+        Note that the attribute (identified by self.field) should be handled with :meth:`get_object`.
+
+        Args:
+            obj: The object to consider for display.
+
+        Returns:
+            A string representing what to show.
+        """
+        pass
+
+
+class StringDisplaySetting(IDisplaySetting):
+    """
+    Class that displays the object as string.
+    """
+
+    def display(self, obj):
+        """
+        Display object.
+
+        Args:
+            obj: Object to display
+
+        Returns:
+            String of object
+        """
+        obj = self.get_object(obj)
+        return str(obj)
+
+
+class DictDisplaySetting(IDisplaySetting):
+    """
+    Class that displays a dictionary.
+    """
+
+    def __init__(self, header: str = None, field: str = None, max_items: int = 10, flat: bool = False):
+        """
+        DictDisplay.
+
+        Args:
+            header: Optional field header.
+            field: The field in the object to consider.
+            max_items: The maximum number of items to display.
+            flat: If False, display as a list; if True, display as a comma-separated list.
+        """
+        super().__init__(header=header, field=field)
+        self.max_items = max_items
+        self.flat = flat
+
+    def display(self, obj: Any) -> str:
+        """
+        Display a dictionary.
+
+        Args:
+            obj: Object to display
+
+        Returns:
+            String display of object
+        """
+        # Retrieve our object
+        obj = self.get_object(obj)
+
+        # Slice the dictionary depending on the `max_items`
+        slice, remaining = cut_iterable_to(obj, self.max_items)
+        printout = ""
+
+        # Different display depending on `flat`
+        if self.flat:
+            printout = ", ".join(f"{k}:{v}" for k, v in slice.items())
+        else:
+            for k, v in slice.items():
+                printout += f"- {k}:{v}\n"
+            printout = printout.strip()
+
+        # If there are items remaining, display
+        if remaining > 0:
+            printout = str(printout)
+            printout += f"\n... and {remaining} more"
+
+        return printout
+
+
+class TableDisplay(IDisplaySetting):
+    """
+    Class that displays the object as a table.
+    """
+
+    def __init__(self, columns, max_rows=5, field=None):
+        """
+        Initialize our TableDisplay.
+
+        Args:
+           columns: A list of display settings.
+           max_rows: The maximum number of rows to display.
+           field: The field of the object to consider.
+        """
+        super().__init__(field=field)
+        self.columns = columns
+        self.max_rows = max_rows
+
+    def display(self, obj) -> str:
+        """
+        Display our object as a table.
+
+        Args:
+            obj: Object to display
+
+        Returns:
+            Table represented as a string of the object
+        """
+        # Retrieve our object
+        obj = super().get_object(obj)
+
+        # Cut the rows
+        slice, remaining = cut_iterable_to(obj, self.max_rows)
+
+        # Create the table
+        rows = []
+        for child in slice:
+            rows.append([s.display(child) for s in self.columns])
+
+        printout = tabulate(rows, headers=[s.header for s in self.columns], tablefmt='psql', showindex=False)
+
+        # If there are items remaining, display
+        if remaining > 0:
+            printout = str(printout)
+            printout += f"\n... and {remaining} more"
+
+        return printout

idmtools/utils/display/settings.py

@@ -0,0 +1,25 @@
+"""
+defines views for different types if items.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+from idmtools.utils.display.displays import DictDisplaySetting, StringDisplaySetting, TableDisplay
+
+simulation_display_settings = [
+    StringDisplaySetting(header="ID", field="uid"),
+    DictDisplaySetting(header="Tags", field="tags", flat=False)]
+
+experiment_display_settings = [
+    StringDisplaySetting(header="ID", field="uid"),
+    StringDisplaySetting(header="Simulation count", field="simulation_count")
+]
+
+experiment_table_display = [
+    StringDisplaySetting(),
+    TableDisplay(field="simulations", columns=simulation_display_settings)
+]
+
+suite_table_display = [
+    StringDisplaySetting(),
+    TableDisplay(field="experiments", columns=experiment_display_settings)
+]

idmtools/utils/dropbox_location.py

@@ -0,0 +1,30 @@
+"""
+utilities for pathing of dropbox folders.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+import os
+
+
+def get_current_user():
+    """
+    Get current user name.
+
+    Returns:
+        Current username
+    """
+    # Returns current logged in user for Windows filesystem
+    import getpass
+    return getpass.getuser()
+
+
+def get_dropbox_location():
+    """
+    Get user dropbox location.
+
+    Returns:
+        User dropbox location
+    """
+    user = get_current_user()
+    dropbox_filepath = os.path.join("C:/Users/", user, "Dropbox (IDM)")
+    return dropbox_filepath

idmtools/utils/entities.py

@@ -0,0 +1,127 @@
+"""
+utilities for dataclasses.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+import ast
+import os
+import dataclasses
+import typing
+from logging import getLogger
+from pathlib import Path
+
+if typing.TYPE_CHECKING:
+    from idmtools.entities.experiment import Experiment
+    from idmtools.entities.iworkflow_item import IWorkflowItem
+    from idmtools.entities.iplatform import IPlatform
+
+user_logger = getLogger('user')
+
+
+def get_dataclass_common_fields(src, dest, exclude_none: bool = True) -> typing.Dict:
+    """
+    Extracts fields from a dataclass source object who are also defined on destination object.
+
+    Useful for situations like nested configurations of data class options.
+
+    Args:
+        src: Source dataclass object
+        dest: Dest dataclass object
+        exclude_none: When true, values of None will be excluded
+
+    Returns:
+        Dictionary of common fields between source and destination object
+    """
+    dest_fields = [f.name for f in dataclasses.fields(dest)]
+    src_fields = dataclasses.fields(src)
+    result = dict()
+    for field in src_fields:
+        if field.name in dest_fields and (not exclude_none or (exclude_none and getattr(src, field.name, None) is not None)):
+            result[field.name] = getattr(src, field.name)
+    return result
+
+
+def as_dict(src, exclude: typing.List[str] = None, exclude_private_fields: bool = True):
+    """
+    Converts a dataclass to a dict while also obeys rules for exclusion.
+
+    Args:
+        src:
+        exclude: List of fields to exclude
+        exclude_private_fields: Should fields that star
+
+    Returns:
+        Data class as dict
+    """
+    if exclude is None:
+        exclude = []
+
+    result = dict()
+    # get metadata
+    metadata = dataclasses.fields(src.__class__)
+    for field in metadata:
+        field_metadata = field.metadata
+        if (not exclude_private_fields or field.name[0] != '_') and \
+                ('exclude_from_metadata' not in field_metadata or not field_metadata['exclude_from_metadata']) and \
+                field.name not in exclude:
+            result[field.name] = getattr(src, field.name)
+    return result
+
+
+def validate_user_inputs_against_dataclass(field_type, field_value):
+    """
+    Validates user entered data against dataclass fields and types.
+
+    Args:
+        field_type: Field type
+        field_value: Fields value
+
+    Returns:
+        Validates user values
+    """
+    fs_kwargs = set(field_type.keys()).intersection(set(field_value.keys()))
+    for fn in fs_kwargs:
+        ft = field_type[fn]
+        try:
+            if ft in (int, float, str):
+                field_value[fn] = ft(field_value[fn]) if field_value[fn] is not None else field_value[fn]
+            elif ft is bool:
+                field_value[fn] = ast.literal_eval(field_value[fn]) if isinstance(field_value[fn], str) else \
+                    field_value[fn]
+        except ValueError as e:
+            user_logger.error(f"The field {fn} requires a value of type {ft.__name__}. You provided <{field_value[fn]}>")
+            raise e
+    return fs_kwargs
+
+
+def get_default_tags() -> typing.Dict[str, str]:
+    """
+    Get common default tags. Currently this is the version of idmtools.
+
+    Returns:
+        Default tags which is idmtools version
+    """
+    from idmtools import __version__
+    return dict(idmtools=__version__)
+
+
+def save_id_as_file_as_hook(item: typing.Union['Experiment', 'IWorkflowItem'], platform: 'IPlatform'):
+    """
+    Predefined hook that will save ids to files for Experiment or WorkItems.
+
+    Args:
+        item:
+        platform:
+
+    Returns:
+        None
+    """
+    # Import locally because of circular deps
+    from idmtools.entities.experiment import Experiment
+    from idmtools.entities.iworkflow_item import IWorkflowItem
+    if not isinstance(item, (Experiment, IWorkflowItem)):
+        raise NotImplementedError("Saving id is currently only support for Experiments and Workitems")
+    id_file = Path(f'{item.item_type}.{item.name}.id')
+    if os.path.exists(id_file):
+        os.remove(id_file)
+    item.to_id_file(id_file)

idmtools/utils/file.py

@@ -0,0 +1,72 @@
+"""
+utilities for files.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+import os
+from io import BytesIO
+from os import DirEntry
+from typing import Iterable, Generator, List, Union
+
+
+def scan_directory(basedir: str, recursive: bool = True, ignore_directories: List[str] = None) -> Iterable[DirEntry]:
+    """
+    Scan a directory recursively or not.
+
+    Args:
+        basedir: The root directory to start from.
+        recursive: True to search the sub-folders recursively; False to stay in the root directory.
+        ignore_directories: Ignore directories
+
+    Returns:
+        An iterator yielding all the files found.
+    """
+    for entry in os.scandir(basedir):
+        if entry.is_file():
+            yield entry
+        elif recursive:
+            if ignore_directories is None or entry.name not in ignore_directories:
+                yield from scan_directory(entry.path)
+
+
+def file_content_to_generator(absolute_path, chunk_size=128) -> Generator[bytearray, None, None]:
+    """
+    Create a generator from file contents in chunks(useful for streaming binary data and piping).
+
+    Args:
+        absolute_path: absolute path to file
+        chunk_size: chunk size
+
+    Returns:
+        Generator that return bytes in chunks of size chunk_size
+    """
+    with open(absolute_path, 'rb') as i:
+        while True:
+            res = i.read(chunk_size)
+            if res == b'':
+                break
+            else:
+                yield res
+
+
+def content_generator(content: Union[str, bytes], chunk_size=128) -> Generator[bytearray, None, None]:
+    """
+    Create a generator from file contents in chunks(useful for streaming binary data and piping).
+
+    Args:
+        content : file content
+        chunk_size : chunk size
+
+    Returns:
+        Generator that return bytes in chunks of size chunk_size
+    """
+    if isinstance(content, str):
+        content_io = BytesIO(content.encode())
+    else:
+        content_io = BytesIO(content)
+    while True:
+        chunk = content_io.read(chunk_size)
+        if chunk == b'':
+            break
+        else:
+            yield chunk

idmtools/utils/file_parser.py

@@ -0,0 +1,151 @@
+"""
+File parser utility. Used to automatically load data.
+
+Copyright 2021, Bill & Melinda Gates Foundation. All rights reserved.
+"""
+import json
+import os
+from logging import getLogger
+from typing import Dict
+
+import pandas as pd
+
+from io import StringIO, BytesIO
+
+logger = getLogger(__name__)
+
+
+class FileParser:
+    """
+    FileParser to load contents in analysis.
+    """
+
+    @classmethod
+    def parse(cls, filename, content=None):
+        """
+        Parse filename and load the content.
+
+        Args:
+            filename: Filename to load
+            content: Content to load
+
+        Returns:
+            Content loaded
+        """
+        file_extension = os.path.splitext(filename)[1][1:].lower()
+        content = BytesIO(content)
+
+        if file_extension == 'json':
+            return cls.load_json_file(filename, content)
+
+        if file_extension == 'csv':
+            return cls.load_csv_file(filename, content)
+
+        if file_extension == 'xlsx':
+            return cls.load_xlsx_file(filename, content)
+
+        if file_extension == 'txt':
+            return cls.load_txt_file(filename, content)
+
+        if file_extension == 'bin' and 'SpatialReport' in filename:
+            return cls.load_bin_file(filename, content)
+
+        return cls.load_raw_file(filename, content)
+
+    @classmethod
+    def load_json_file(cls, filename, content) -> Dict:
+        """
+        Load JSON File.
+
+        Args:
+            filename: Filename to load
+            content: Content
+
+        Returns:
+            JSOn as dict
+        """
+        return json.load(content)
+
+    @classmethod
+    def load_raw_file(self, filename, content):
+        """
+        Load content raw.
+
+        Args:
+            filename: Filename is none
+            content: Content to load
+
+        Returns:
+            Content as it was
+        """
+        return content
+
+    @classmethod
+    def load_csv_file(cls, filename, content) -> pd.DataFrame:
+        """
+        Load csv file.
+
+        Args:
+            filename: Filename to load
+            content: Content is loading
+
+        Returns:
+            Loaded csv file
+        """
+        if not isinstance(content, StringIO) and not isinstance(content, BytesIO):
+            content = StringIO(content)
+
+        csv_read = pd.read_csv(content, skipinitialspace=True)
+        return csv_read
+
+    @classmethod
+    def load_xlsx_file(cls, filename, content) -> Dict[str, pd.ExcelFile]:
+        """
+        Load excel_file.
+
+        Args:
+            filename: Filename to load
+            content: Content to load
+
+        Returns:
+            Loaded excel file
+        """
+        excel_file = pd.ExcelFile(content)
+        return {sheet_name: excel_file.parse(sheet_name) for sheet_name in excel_file.sheet_names}
+
+    @classmethod
+    def load_txt_file(cls, filename, content):
+        """
+        Load text file.
+
+        Args:
+            filename: Filename to load
+            content: Content to load
+
+        Returns:
+            Content
+        """
+        return str(content.getvalue().decode())
+
+    @classmethod
+    def load_bin_file(cls, filename, content):
+        """
+        Load a bin file.
+
+        Args:
+            filename: Filename to load
+            content: Content to load
+
+        Returns:
+            Loaded bin file
+
+        Notes:
+            We should move this to a plugin in emodpy. We need to figure out how to structure that.
+        """
+        try:
+            from idmtools_platform_comps.utils.spatial_output import SpatialOutput
+            so = SpatialOutput.from_bytes(content.read(), 'Filtered' in filename)
+            return so.to_dict()
+        except ImportError as ex:
+            logger.exception(ex)
+            logger.error("Could not import item. Most likely dtk.tools is not installed")