wxflow 0.1.0__py3-none-any.whl

wxflow/__init__.py

@@ -0,0 +1,21 @@
+import os
+
+from .attrdict import AttrDict
+from .configuration import (Configuration, cast_as_dtype,
+                            cast_strdict_as_dtypedict)
+from .exceptions import WorkflowException, msg_except_handle
+from .executable import CommandNotFoundError, Executable, which
+from .factory import Factory
+from .file_utils import FileHandler
+from .fsutils import chdir, cp, mkdir, mkdir_p, rm_p, rmdir
+from .jinja import Jinja
+from .logger import Logger, logit
+from .task import Task
+from .template import Template, TemplateConstants
+from .timetools import *
+from .yaml_file import (YAMLFile, dump_as_yaml, parse_j2yaml, parse_yaml,
+                        parse_yamltmpl, save_as_yaml, vanilla_yaml)
+
+__docformat__ = "restructuredtext"
+__version__ = "0.1.0"
+wxflow_directory = os.path.dirname(__file__)

wxflow/attrdict.py

@@ -0,0 +1,171 @@
+# attrdict is a Python module that gives you dictionaries whose values are both
+# gettable and settable using attributes, in addition to standard item-syntax.
+# https://github.com/mewwts/addict
+# addict/addict.py -> attrdict.py
+# hash: 7e8d23d
+# License: MIT
+# class Dict -> class AttrDict to prevent name collisions w/ typing.Dict
+
+import copy
+
+__all__ = ['AttrDict']
+
+
+class AttrDict(dict):
+
+    def __init__(__self, *args, **kwargs):
+        object.__setattr__(__self, '__parent', kwargs.pop('__parent', None))
+        object.__setattr__(__self, '__key', kwargs.pop('__key', None))
+        object.__setattr__(__self, '__frozen', False)
+        for arg in args:
+            if not arg:
+                continue
+            elif isinstance(arg, dict):
+                for key, val in arg.items():
+                    __self[key] = __self._hook(val)
+            elif isinstance(arg, tuple) and (not isinstance(arg[0], tuple)):
+                __self[arg[0]] = __self._hook(arg[1])
+            else:
+                for key, val in iter(arg):
+                    __self[key] = __self._hook(val)
+
+        for key, val in kwargs.items():
+            __self[key] = __self._hook(val)
+
+    def __setattr__(self, name, value):
+        if hasattr(self.__class__, name):
+            raise AttributeError("'AttrDict' object attribute "
+                                 "'{0}' is read-only".format(name))
+        else:
+            self[name] = value
+
+    def __setitem__(self, name, value):
+        isFrozen = (hasattr(self, '__frozen') and
+                    object.__getattribute__(self, '__frozen'))
+        if isFrozen and name not in super(AttrDict, self).keys():
+            raise KeyError(name)
+        if isinstance(value, dict):
+            value = AttrDict(value)
+        super(AttrDict, self).__setitem__(name, value)
+        try:
+            p = object.__getattribute__(self, '__parent')
+            key = object.__getattribute__(self, '__key')
+        except AttributeError:
+            p = None
+            key = None
+        if p is not None:
+            p[key] = self
+            object.__delattr__(self, '__parent')
+            object.__delattr__(self, '__key')
+
+    def __add__(self, other):
+        if not self.keys():
+            return other
+        else:
+            self_type = type(self).__name__
+            other_type = type(other).__name__
+            msg = "unsupported operand type(s) for +: '{}' and '{}'"
+            raise TypeError(msg.format(self_type, other_type))
+
+    @classmethod
+    def _hook(cls, item):
+        if isinstance(item, dict):
+            return cls(item)
+        elif isinstance(item, (list, tuple)):
+            return type(item)(cls._hook(elem) for elem in item)
+        return item
+
+    def __getattr__(self, item):
+        return self.__getitem__(item)
+
+    def __missing__(self, name):
+        if object.__getattribute__(self, '__frozen'):
+            raise KeyError(name)
+        return self.__class__(__parent=self, __key=name)
+
+    def __delattr__(self, name):
+        del self[name]
+
+    def to_dict(self):
+        base = {}
+        for key, value in self.items():
+            if isinstance(value, type(self)):
+                base[key] = value.to_dict()
+            elif isinstance(value, (list, tuple)):
+                base[key] = type(value)(
+                    item.to_dict() if isinstance(item, type(self)) else
+                    item for item in value)
+            else:
+                base[key] = value
+        return base
+
+    def copy(self):
+        return copy.copy(self)
+
+    def deepcopy(self):
+        return copy.deepcopy(self)
+
+    def __deepcopy__(self, memo):
+        other = self.__class__()
+        memo[id(self)] = other
+        for key, value in self.items():
+            other[copy.deepcopy(key, memo)] = copy.deepcopy(value, memo)
+        return other
+
+    def update(self, *args, **kwargs):
+        other = {}
+        if args:
+            if len(args) > 1:
+                raise TypeError()
+            other.update(args[0])
+        other.update(kwargs)
+        for k, v in other.items():
+            if ((k not in self) or
+                (not isinstance(self[k], dict)) or
+                    (not isinstance(v, dict))):
+                self[k] = v
+            else:
+                self[k].update(v)
+
+    def __getnewargs__(self):
+        return tuple(self.items())
+
+    def __getstate__(self):
+        return self
+
+    def __setstate__(self, state):
+        self.update(state)
+
+    def __or__(self, other):
+        if not isinstance(other, (AttrDict, dict)):
+            return NotImplemented
+        new = AttrDict(self)
+        new.update(other)
+        return new
+
+    def __ror__(self, other):
+        if not isinstance(other, (AttrDict, dict)):
+            return NotImplemented
+        new = AttrDict(other)
+        new.update(self)
+        return new
+
+    def __ior__(self, other):
+        self.update(other)
+        return self
+
+    def setdefault(self, key, default=None):
+        if key in self:
+            return self[key]
+        else:
+            self[key] = default
+            return default
+
+    def freeze(self, shouldFreeze=True):
+        object.__setattr__(self, '__frozen', shouldFreeze)
+        for key, val in self.items():
+            if isinstance(val, AttrDict):
+                val.freeze(shouldFreeze)
+
+    def unfreeze(self):
+        self.freeze(False)

wxflow/configuration.py

@@ -0,0 +1,179 @@
+import glob
+import os
+import random
+import subprocess
+from pathlib import Path
+from pprint import pprint
+from typing import Any, Dict, List, Union
+
+from .attrdict import AttrDict
+from .timetools import to_datetime
+
+__all__ = ['Configuration', 'cast_as_dtype', 'cast_strdict_as_dtypedict']
+
+
+class ShellScriptException(Exception):
+    def __init__(self, scripts, errors):
+        self.scripts = scripts
+        self.errors = errors
+        super(ShellScriptException, self).__init__(
+            str(errors) +
+            ': error processing' +
+            (' '.join(scripts)))
+
+
+class UnknownConfigError(Exception):
+    pass
+
+
+class Configuration:
+    """
+    Configuration parser for the global-workflow
+    (or generally for sourcing a shell script into a python dictionary)
+    """
+
+    def __init__(self, config_dir: Union[str, Path]):
+        """
+        Given a directory containing config files (config.XYZ),
+        return a list of config_files minus the ones ending with ".default"
+        """
+
+        self.config_dir = config_dir
+        self.config_files = self._get_configs
+
+    @property
+    def _get_configs(self) -> List[str]:
+        """
+        Given a directory containing config files (config.XYZ),
+        return a list of config_files minus the ones ending with ".default"
+        """
+        result = list()
+        for config in glob.glob(f'{self.config_dir}/config.*'):
+            if not config.endswith('.default'):
+                result.append(config)
+
+        return result
+
+    def find_config(self, config_name: str) -> str:
+        """
+            Given a config file name, find the full path of the config file
+        """
+
+        for config in self.config_files:
+            if config_name == os.path.basename(config):
+                return config
+
+        raise UnknownConfigError(
+            f'{config_name} does not exist (known: {repr(config_name)}), ABORT!')
+
+    def parse_config(self, files: Union[str, bytes, list]) -> Dict[str, Any]:
+        """
+        Given the name of config file(s), key-value pair of all variables in the config file(s)
+        are returned as a dictionary
+        :param files: config file or list of config files
+        :type files: list or str or unicode
+        :return: Key value pairs representing the environment variables defined
+                in the script.
+        :rtype: dict
+        """
+        if isinstance(files, (str, bytes)):
+            files = [files]
+        files = [self.find_config(file) for file in files]
+        return cast_strdict_as_dtypedict(self._get_script_env(files))
+
+    def print_config(self, files: Union[str, bytes, list]) -> None:
+        """
+        Given the name of config file(s), key-value pair of all variables in the config file(s) are printed
+        Same signature as parse_config
+        :param files: config file or list of config files
+        :type files: list or str or unicode
+        :return: None
+        """
+        config = self.parse_config(files)
+        pprint(config, width=4)
+
+    @classmethod
+    def _get_script_env(cls, scripts: List) -> Dict[str, Any]:
+        default_env = cls._get_shell_env([])
+        and_script_env = cls._get_shell_env(scripts)
+        vars_just_in_script = set(and_script_env) - set(default_env)
+        union_env = dict(default_env)
+        union_env.update(and_script_env)
+        return dict([(v, union_env[v]) for v in vars_just_in_script])
+
+    @staticmethod
+    def _get_shell_env(scripts: List) -> Dict[str, Any]:
+        varbls = dict()
+        runme = ''.join([f'source {s} ; ' for s in scripts])
+        magic = f'--- ENVIRONMENT BEGIN {random.randint(0,64**5)} ---'
+        runme += f'/bin/echo -n "{magic}" ; /usr/bin/env -0'
+        with open('/dev/null', 'w') as null:
+            env = subprocess.Popen(runme, shell=True, stdin=null.fileno(),
+                                   stdout=subprocess.PIPE)
+            (out, err) = env.communicate()
+        out = out.decode()
+        begin = out.find(magic)
+        if begin < 0:
+            raise ShellScriptException(scripts, 'Cannot find magic string; '
+                                       'at least one script failed: ' + repr(out))
+        for entry in out[begin + len(magic):].split('\x00'):
+            iequal = entry.find('=')
+            varbls[entry[0:iequal]] = entry[iequal + 1:]
+        return varbls
+
+
+def cast_strdict_as_dtypedict(ctx: Dict[str, str]) -> Dict[str, Any]:
+    """
+    Environment variables are typically stored as str
+    This method attempts to translate those into datatypes
+    Parameters
+    ----------
+    ctx : dict
+          dictionary with values as str
+    Returns
+    -------
+    varbles : dict
+              dictionary with values as datatypes
+    """
+    varbles = AttrDict()
+    for key, value in ctx.items():
+        varbles[key] = cast_as_dtype(value)
+    return varbles
+
+
+def cast_as_dtype(string: str) -> Union[str, int, float, bool, Any]:
+    """
+    Cast a value into known datatype
+    Parameters
+    ----------
+    string: str
+    Returns
+    -------
+    value : str or int or float or datetime
+            default: str
+    """
+    TRUTHS = ['y', 'yes', 't', 'true', '.t.', '.true.']
+    BOOLS = ['n', 'no', 'f', 'false', '.f.', '.false.'] + TRUTHS
+    BOOLS = [x.upper() for x in BOOLS] + BOOLS + ['Yes', 'No', 'True', 'False']
+
+    def _cast_or_not(type: Any, string: str):
+        try:
+            return type(string)
+        except ValueError:
+            return string
+
+    def _true_or_not(string: str):
+        try:
+            return string.lower() in TRUTHS
+        except AttributeError:
+            return string
+
+    try:
+        return to_datetime(string)  # Try as a datetime
+    except Exception as exc:
+        if string in BOOLS:  # Likely a boolean, convert to True/False
+            return _true_or_not(string)
+        elif '.' in string:  # Likely a number and that too a float
+            return _cast_or_not(float, string)
+        else:  # Still could be a number, may be an integer
+            return _cast_or_not(int, string)

wxflow/exceptions.py

@@ -0,0 +1,87 @@
+# pylint: disable=unused-argument
+
+# ----
+
+from collections.abc import Callable
+
+from .logger import Logger, logit
+
+logger = Logger(level="error", colored_log=True)
+
+__all__ = ["WorkflowException", "msg_except_handle"]
+
+
+class WorkflowException(Exception):
+    """
+    Description
+    -----------
+
+    This is the base-class for all exceptions; it is a sub-class of
+    Exceptions.
+
+    Parameters
+    ----------
+
+    msg: str
+
+        A Python string containing a message to accompany the
+        exception.
+
+    """
+
+    @logit(logger)
+    def __init__(self: Exception, msg: str):
+        """
+        Description
+        -----------
+
+        Creates a new WorkflowException object.
+
+        """
+
+        # Define the base-class attributes.
+        logger.error(msg=msg)
+        super().__init__()
+
+
+# ----
+
+
+def msg_except_handle(err_cls: object) -> Callable:
+    """
+    Description
+    -----------
+
+    This function provides a decorator to be used to raise specified
+    exceptions.
+
+    Parameters
+    ----------
+
+    err_cls: object
+
+        A Python object containing the WorkflowException subclass to
+        be used for exception raises.
+
+    Parameters
+    ----------
+
+    decorator: Callable
+
+        A Python decorator.
+
+    """
+
+    # Define the decorator function.
+    def decorator(func: Callable):
+
+        # Execute the caller function; proceed accordingly.
+        def call_function(msg: str) -> None:
+
+            # If an exception is encountered, raise the respective
+            # exception.
+            raise err_cls(msg=msg)
+
+        return call_function
+
+    return decorator