sier2 1.0.1__py2.py3-none-any.whl

sier2/__init__.py

@@ -0,0 +1,6 @@
+from ._block import Block, BlockError, BlockValidateError
+from ._config import Config
+from ._dag import Connection, Dag, BlockState
+from ._library import Library, Info
+from ._version import __version__
+from ._util import get_block_config

sier2/__main__.py

@@ -0,0 +1,111 @@
+import argparse
+from importlib.metadata import version
+
+from sier2 import Library, Config
+from ._library import _find_blocks, _find_dags, run_dag
+from ._util import block_doc_text, dag_doc_text
+
+BOLD = '' # '\x1b[1;37m'
+NORM = '' # '\x1b[0m'
+
+def _pkg(module):
+    return module.split('.')[0]
+
+def blocks_cmd(args):
+    """Display the blocks found via plugin entry points."""
+
+    seen = set()
+    curr_ep = None
+    for entry_point, gi in _find_blocks():
+        show = not args.block or gi.key.endswith(args.block)
+        if curr_ep is None or entry_point!=curr_ep:
+            if show:
+                pkg = _pkg(entry_point.module)
+                s = f'In {pkg} v{version(pkg)}'
+                u = '' # '\n' + '#' * len(s)
+                print(f'\n{BOLD}{s}{u}{NORM}')
+                # print(f'\x1b[1mIn {entry_point.module} v{version(entry_point.module)}:\x1b[0m')
+                curr_ep = entry_point
+
+        if show:
+            dup = f' (DUPLICATE)' if gi.key in seen else ''
+            print(f'  {BOLD}{gi.key}: {gi.doc}{NORM}{dup}')
+
+            if args.verbose:
+                block = Library.get_block(gi.key)
+                print(block_doc_text(block))
+                print()
+
+            seen.add(gi.key)
+
+def dags_cmd(args):
+    """Display the dags found via plugin entry points."""
+
+    seen = set()
+    curr_ep = None
+    for entry_point, gi in _find_dags():
+        show = not args.dag or gi.key.endswith(args.dag)
+        if curr_ep is None or entry_point!=curr_ep:
+            if show:
+                pkg = _pkg(entry_point.module)
+                s = f'In {pkg} v{version(pkg)}'
+                u = '' # '\n' + '#' * len(s)
+                print(f'\n{BOLD}{s}{u}{NORM}')
+                curr_ep = entry_point
+
+        if show:
+            dup = f' (DUPLICATE)' if gi.key in seen else ''
+            print(f'  {BOLD}{gi.key}: {gi.doc}{NORM}{dup}')
+
+            if args.verbose:
+                # We have to instantiate the dag to get the documentation.
+                #
+                dag = Library.get_dag(gi.key)
+                print(dag_doc_text(dag))
+
+            seen.add(gi.key)
+
+def run_cmd(args):
+    if args.update_config:
+        args = args.update_config.split(',')
+        block_name = args[0]
+        update_arg = ','.join(args[1:])
+        write_to_file = True
+    else:
+        block_name = None
+        update_arg = None
+        write_to_file = False
+
+    if args.config or args.update_config:
+        Config.update(location=args.config, config_block=block_name, update_arg=update_arg, write_to_file=write_to_file)
+
+    run_dag(args.dag)
+
+def main():
+    parser = argparse.ArgumentParser()
+    subparsers = parser.add_subparsers(help='sub-command help')
+
+    run = subparsers.add_parser('run', help='Run a dag')
+    run.add_argument('dag', type=str, help='A dag to run')
+    run.add_argument('-C', '--config', default=None, help='Use this config file (default to personal config file)')
+    run.add_argument('-U', '--update_config', default=None, help='A block that provides a config that updates the personal config file')
+    run.set_defaults(func=run_cmd)
+
+    blocks = subparsers.add_parser('blocks', help='Show available blocks')
+    blocks.add_argument('-v', '--verbose', action='store_true', help='Show help')
+    blocks.add_argument('block', nargs='?', help='Show all blocks ending with this string')
+    blocks.set_defaults(func=blocks_cmd)
+
+    dags = subparsers.add_parser('dags', help='Show available dags')
+    dags.add_argument('-v', '--verbose', action='store_true', help='Show help')
+    dags.add_argument('dag', nargs='?', help='Show all dags ending with this string')
+    dags.set_defaults(func=dags_cmd)
+
+    args = parser.parse_args()
+    if 'func' in args:
+        args.func(args)
+    else:
+        parser.print_help()
+
+if __name__=='__main__':
+    main()

sier2/_block.py

@@ -0,0 +1,273 @@
+from enum import StrEnum
+import inspect
+import param
+from typing import Any
+
+from . import _logger
+from ._config import Config
+
+class BlockError(Exception):
+    """Raised if a Block configuration is invalid.
+
+    If this exception is raised, the executing dag sets its stop
+    flag (which must be manually reset), and displays a stacktrace.
+    """
+
+    pass
+
+class BlockState(StrEnum):
+    """The current state of a block; also used for logging."""
+
+    DAG = 'DAG'                 # Dag logging.
+    BLOCK = 'BLOCK'             # Block logging.
+    INPUT = 'INPUT'
+    READY = 'READY'
+    EXECUTING = 'EXECUTING'
+    WAITING = 'WAITING'
+    SUCCESSFUL = 'SUCCESSFUL'
+    INTERRUPTED = 'INTERRUPTED'
+    ERROR = 'ERROR'
+
+_PAUSE_EXECUTION_DOC = '''If True, a block executes in two steps.
+
+When the block is executed by a dag, the dag first sets the input
+params, then calls ``prepare()``. Execution of the dag then stops.
+
+The dag is then restarted using ``dag.execute_after_input(input_block)``.
+(An input block must be specified because it is not required that the
+same input block be used immediately.) This causes the block's
+``execute()`` method to be called without resetting the input params.
+
+Dag execution then continues as normal.
+'''
+
+_VISIBLE_DOC = '''If True, the block will be visible in a GUI.
+
+A block may not need to be visible in a dag with a GUI. For example,
+it may be applying a pre-defined filter, or running an algorithm that
+takes an indeterminate amount of time. Setting this parameter to False
+tells the GUI not display this block. Dag execution will otherwise
+proceed as normal.
+
+This is also useful if a GUI application only requires a single block.
+A dag requires at least two blocks, because blocks can only be added
+by connecting them to another block. By making one block a "dummy"
+that is not visible, the GUI effectivly has a single block.
+'''
+
+class Block(param.Parameterized):
+    """The base class for blocks.
+
+    A block is implemented as:
+
+    .. code-block:: python
+
+        class MyBlock(Block):
+            ...
+
+    The ``Block`` class inherits from ``param.Parameterized``, and uses parameters
+    as described at https://param.holoviz.org/user_guide/Parameters.html.
+    There are three kinds of parameters:
+    * Input parameters start with ``in_``. These parameters are set before a block is executed.
+    * Output parameters start with ``out_``. The block sets these in its ``execute()`` method.
+    * Block parameters start with ``block_``. These are reserved for use by blocks.
+
+    A typical block will have at least one input parameter, and an ``execute()``
+    method that is called when an input parameter value changes.
+
+    .. code-block:: python
+
+        class MyBlock(Block):
+            in_value = param.String(label='Input Value')
+            out_upper = param.String(label='Output value)
+
+            def execute(self):
+                self.out_value = self.in_value.upper()
+                print(f'New value is {self.out_value}')
+
+    The block parameter ``block_pause_execution`` allows a block to act as an "input" block,
+    particularly when the block hsa a GUI interface. When set to True and dag execution
+    reaches this block, the block's ``prepare()`` method is called, then the dag stops executing.
+    This allows the user to interact with a user interface.
+
+    The dag is then restarted using ``dag.execute_after_input(input_block)`` (typically by
+    a "Continue" button in the GUI.) When the dag is continued at this block,
+    the block's ``execute()`` method is called, and dag execution continues.
+    """
+
+    block_pause_execution = param.Boolean(default=False, label='Pause execution', doc=_PAUSE_EXECUTION_DOC)
+    block_visible = param.Boolean(default=True, label='Display block', doc=_VISIBLE_DOC)
+
+    _block_state = param.String(default=BlockState.READY)
+
+    SIER2_KEY = '_sier2__key'
+
+    def __init__(self, *args, block_pause_execution: bool=False, block_visible: bool=True, block_doc: str|None=None, continue_label='Continue', **kwargs):
+        """
+        Parameters
+        ----------
+        block_pause_execution: bool
+            If True, ``prepare()`` is called and dag execution stops.
+        block_visible: bool
+            If True (the default), the block will be visible in a GUI.
+        block_doc: str|None
+            Markdown documentation that may displayed in the user interface.
+        """
+        super().__init__(*args, **kwargs)
+
+        if not self.__doc__:
+            raise BlockError(f'Class {self.__class__} must have a docstring')
+
+        self.block_pause_execution = block_pause_execution
+        self.block_visible = block_visible
+        self.block_doc = block_doc
+        self.continue_label = continue_label
+        # self._block_state = BlockState.READY
+        self.logger = _logger.get_logger(self.name)
+
+        # Maintain a map of "block+output parameter being watched" -> "input parameter".
+        # This is used by _block_event() to set the correct input parameter.
+        #
+        self._block_name_map: dict[tuple[str, str], str] = {}
+
+        # Record this block's output parameters.
+        # If this is an input block, we need to trigger
+        # the output values before executing the next block,
+        # in case the user didn't change anything.
+        #
+        self._block_out_params = []
+
+    @classmethod
+    def block_key(cls):
+        """The unique key of this block class.
+
+        Blocks require a unique key so they can be identified in the block library.
+        The default implementation should be sufficient, but can be overridden
+        in case of refactoring or name clashes.
+        """
+
+        im = inspect.getmodule(cls)
+
+        if hasattr(cls, Block.SIER2_KEY):
+            return getattr(cls, Block.SIER2_KEY)
+
+        return f'{im.__name__}.{cls.__qualname__}'
+
+    def get_config(self, *, block: 'Block'=None):
+        """Return a dictionary containing keys and values from the section specified by
+        the block in the sier2 config file.
+
+        The config file has the format described by the Python ``configparser`` module,
+        with the added feature that values are evaluated using :func:`ast.literal_eval`,
+        and therefore must be syntactically correct Python literals.
+
+        They keys and values are read from the section ``[block.name]``, where ``name`` is
+        this block's unique key as specified by :func:`sier2.Block.block_key`.
+        If the ``block`` parameter is unspecified, the calling block is used by default.
+
+        If the section is not present in the config file, an empty dictionary is returned.
+
+        The default config file is looked for at
+        (the default user config directory) / 'sier2sier2.ini'.
+        On Windows, the config directory is ``$ENV:APPDATA``; on Linux, ``$XDG_CONFIG_HOME``
+        or ``$HOME/.config``.
+
+        An alternative config file can be specified by setting ``Config.location`` before
+        any dag or block is executed. THis can be done from a command line using
+        the ``--config`` option.
+
+        Parameters
+        ----------
+        block: Block
+            The specified block's config section will be returned. Defaults to ``self``.
+
+        Returns
+        -------
+        A dictionary containing the section's keys and values.
+        """
+
+        b = block if block is not None else self
+        name = f'block.{b.block_key()}'
+
+        return Config[name]
+
+    def get_config_value(self, key: str, default: Any=None, *, block: 'Block'=None):
+        """Return an individual value from the section specified by
+        the block in the sier2 config file.
+
+        See :func:`sier2.Block.get_config` for more details.
+
+        Parameters
+        ----------
+        key: str
+            The key of the value to be returned.
+        default: Any
+            The default value to return if the section or key are not present in the config file.
+        block: Block
+            The specified block's config section will be returned. Defaults to ``self``.
+        """
+
+        b = block if block is not None else self
+        name = f'block.{b.block_key()}'
+        value = Config[name, key]
+
+        return value if value is not None else default
+
+    def prepare(self):
+        """If blockpause_execution is True, called by a dag before calling ``execute()```.
+
+        This gives the block author an opportunity to validate the
+        input params and set up a user inteface.
+
+        After the dag restarts on this block, ``execute()`` will be called.
+        """
+
+        pass
+
+    def execute(self, *_, **__):
+        """This method is called when one or more of the input parameters causes an event.
+
+        Override this method in a Block subclass.
+
+        The ``execute()`` method can have arguments. The arguments can be specified
+        in any order. It is not necessary to specify all, or any, arguments.
+        Arguments will not be passed via ``*args`` or ``**kwargs``.
+
+        * ``stopper`` - an indicator that the dag has been stopped. This may be
+            set while the block is executing, in which case the block should
+            stop executing as soon as possible.
+        * ``events`` - the param events that caused execute() to be called.
+        """
+
+        # print(f'** EXECUTE {self.__class__=}')
+        pass
+
+    def __call__(self, **kwargs) -> dict[str, Any]:
+        """Allow a block to be called directly."""
+
+        in_names = [name for name in self.__class__.param if name.startswith('in_')]
+        if len(kwargs)!=len(in_names) or any(name not in in_names for name in kwargs):
+            names = ', '.join(in_names)
+            raise BlockError(f'All input params must be specified: {names}')
+
+        for name, value in kwargs.items():
+            setattr(self, name, value)
+
+        self.execute()
+
+        out_names = [name for name in self.__class__.param if name.startswith('out_')]
+        result = {name: getattr(self, name) for name in out_names}
+
+        return result
+
+class BlockValidateError(BlockError):
+    """Raised if ``Block.prepare()`` or ``Block.execute()`` determines that input data is invalid.
+
+    If this exception is raised, it will be caught by the executing dag.
+    The dag will not set its stop flag, no stacktrace will be displayed,
+    and the error message will be displayed.
+    """
+
+    def __init__(self, *, block_name: str, error: str):
+        super().__init__(error)
+        self.block_name = block_name

sier2/_config.py

@@ -0,0 +1,251 @@
+# A configuration module.
+#
+
+import ast
+import configparser
+import os
+from pathlib import Path
+from typing import Any
+
+# If this key exists in a section and has the value False,
+# this section will not be updated.
+#
+CONFIG_UPDATE = 'config_update'
+
+def _default_config_file():
+    """Determine the location of the config file sier2.ini.
+
+    If the environment variable ``SIER2_INI`` is set,
+    it specifies the path of the config file.
+
+    Otherwise, if Windows, use ``$env:APPDATA/sier2/sier2.ini``.
+
+    Otherwise, if ``XDG_CONFIG_HOME`` is set, use ``$XDG_CONFIG_HOME/sier2/sier2.ini``.
+
+    Otherwise, use ``$HOME/.config/sier2/sier2.ini``.
+
+    If not using ``SIER2_INI``, the ``sier2`` directory will be created
+    if it does not exist.
+
+    TODO don't create the sier2 directory until the ini file is written.
+    """
+
+    # If a config file has been explicitly set in an environment variable,
+    # use it.
+    #
+    ini = os.environ.get('SIER2_INI', None)
+    if ini:
+        return Path(ini)
+
+    if os.name=='nt':
+        # Windows.
+        #
+        prdir = Path(os.environ['APPDATA']) / 'sier2'
+    else:
+        # Linux.
+        #
+        prdir = os.environ.get('XDG_CONFIG_HOME', None)
+        if prdir:
+            prdir = Path(prdir)
+        else:
+            prdir = Path.home() / '.config'
+
+        prdir = prdir / 'sier2'
+
+    if not prdir.exists():
+        prdir.mkdir(parents=True)
+
+    return prdir / 'sier2.ini'
+
+class _Config:
+    """This class is for internal use.
+
+    A single instance of this class is exposed publicly as ``Config``.
+    """
+
+    def __init__(self):
+        self._clear()
+
+    @staticmethod
+    def update(*, location: Path|str|None=None, config_block: str|None=None, update_arg: str|None=None, write_to_file: bool=False):
+        """Update the config.
+
+        If ``location`` has a value, that config file will be used instead of the default.
+
+        If config_block has a value, it must be the name of a block that has
+        an ``out_config`` param. The ``out_config`` param must contain a string that is
+        the content of a sier2 ini file. If ``update_arg`` is specified and the block has
+        an ``in_arg`` param, ``in_arg`` is set to ``update_arg``.
+
+        Parameters
+        ----------
+        location: Path|str|None
+            The location of a config file that will be loaded when a config is read.
+        config_block: str|None
+            A sier2 block that returns the contents of a config file in ``out_config``.
+        update_arg: str|None
+            A string that is passed to the ``config_block`` block is it has an ``in_arg`` param.
+        write_to_file: bool
+            If True, the config file at ``location`` is overwritten with the merged config.
+        """
+
+        if location:
+            Config.location = location
+
+        if config_block:
+            # Import here, otherwise there's a circular dependency Library -> Config -> Library.
+            # Config blocks better not have any config.
+            #
+            from sier2 import Library
+
+            block = Library.get_block(config_block)()
+
+            if not hasattr(block, 'out_config'):
+                raise ValueError('config block does not have out param "out_config"')
+
+            if hasattr(block, 'in_arg'):
+                block.in_arg = update_arg
+
+            block.execute()
+            Config._update(block.out_config, write_to_file=write_to_file)
+
+    def _clear(self):
+        self._location = _default_config_file()
+        self._config = {}
+        self._loaded = False
+
+    @property
+    def location(self) -> Path:
+        """Get or set the config file location.
+
+        By default, this is `$ENV:APPDATA/sier2/sier2.ini` on Window,
+        and `$XDG_CONFIG_HOME/sier2/sier2.ini` on Linux.
+
+        The location cannot be set if the config file has already been loaded.
+        """
+
+        return self._location
+
+    @location.setter
+    def location(self, config_file: Path|str):
+        if self._loaded:
+            raise ValueError('Config is already loaded')
+
+        if not isinstance(config_file, Path):
+            config_file = Path(config_file)
+
+        self._location = config_file
+
+    def _update(self, ini: str, write_to_file: bool=False):
+        """Update the config file using ini, which contains the contents of another config file.
+
+        The current config is also updated.
+
+        The config file cannot be updated if it has already been loaded.
+        """
+
+        if self._loaded:
+            raise ValueError('Config is already loaded')
+
+        new_config = configparser.ConfigParser()
+        new_config.read_string(ini)
+
+        config = configparser.ConfigParser()
+        if self._location.is_file():
+            config.read(self._location)
+
+        for section_name in new_config.sections():
+            if not config.has_section(section_name):
+                config.add_section(section_name)
+                config[section_name].update(new_config[section_name])
+            else:
+                update_section = True
+                if CONFIG_UPDATE in config[section_name]:
+                    update_section = ast.literal_eval(config[section_name][CONFIG_UPDATE])
+                    if not isinstance(update_section, bool):
+                        raise ValueError(f'Value of [{section_name}].{CONFIG_UPDATE} is not a bool')
+
+                if update_section:
+                    config[section_name].update(new_config[section_name])
+                    # for k, v in new_config[section_name].items():
+                    #     config[section_name][k] = new_config[section_name][k]
+
+        if write_to_file:
+            with open(self._location, 'w', encoding='utf-8') as f:
+                config.write(f)
+
+        self._config = config
+        self._loaded = True
+
+    def _load(self):
+        """Load the config.
+
+        Overwrites any previous config. If the location does not exist, the config will be empty.
+        """
+
+        self._config = configparser.ConfigParser()
+        if self._location.is_file():
+            self._config.read(self._location)
+
+        # The config has been loaded, even if the file didn't exist.
+        #
+        self._loaded = True
+
+    def _load_string(self, sconfig):
+        """For testing."""
+
+        self._config = configparser.ConfigParser()
+        self._config.read_string(sconfig)
+        self._loaded = True
+
+    def __getitem__(self, section_name: str|tuple[str, str]) -> Any|dict[str, Any]:
+        """If section_name is a string, get the config values for the given section name.
+
+        The config file is lazily loaded. Non-existence of the file is normal.
+        The keys and values for the specified section are loaded into a new dictionary,
+        which is returned.
+
+        Since configparser always returns values as strings, the values are evaluated
+        using :func:`ast.literal_eval` to be correctly typed. This means that strings in the
+        .ini file must be surrounded by quotes.
+
+        A section name for a block is of the form 'block.block_key_name'.
+
+        The name need not to exist; if it doesn't, an empty dictionary is returned.
+
+        If section_name is a tuple, it is interpreted as (section_name, key). If the
+        section_name and key exist, the value of the key is returned, else None is returned.
+        Only that single value is evaluated.
+        """
+
+        if not self._loaded:
+            self._load()
+
+        if isinstance(section_name, tuple):
+            section_name, key = section_name
+            if section_name not in self._config:
+                return None
+
+            section = self._config[section_name]
+            if key not in section:
+                return None
+
+            try:
+                v = section[key]
+                return ast.literal_eval(v)
+            except ValueError:
+                raise ValueError(f'Cannot eval section [{section_name}], key {key}, value {v}')
+
+        if section_name not in self._config:
+            return {}
+
+        c = {}
+        for key, v in self._config[section_name].items():
+            try:
+                c[key] = ast.literal_eval(v)
+            except ValueError:
+                raise ValueError(f'Cannot eval section [{section_name}], key {key}, value {v}')
+
+        return c
+
+Config = _Config()