sier2 0.17__py3-none-any.whl

sier2/_library.py

@@ -0,0 +1,226 @@
+from collections.abc import Iterable
+from dataclasses import dataclass
+import importlib
+from importlib.metadata import entry_points, EntryPoint
+from typing import Any, cast
+import warnings
+
+from sier2 import Block, Dag, Connection, BlockError
+from sier2.panel import PanelDag
+
+# Store a mapping from a unique key to a Block class.
+# When plugins are initially scanned, the classes are not loaded.
+#
+_block_library: dict[str, type[Block]|None] = {}
+_dag_library: set[str] = set()
+
+@dataclass
+class Info:
+    key: str
+    doc: str
+
+def docstring(func) -> str:
+    doc = func.__doc__.strip()
+
+    return doc.split('\n')[0].strip()
+
+def _find_blocks():
+    yield from _find('blocks')
+
+def _find_dags():
+    yield from _find('dags')
+
+def run_dag(dag_name):
+    """Run the named dag."""
+
+    ix = dag_name.rfind('.')
+    if ix==-1:
+        found_dag = None
+        for _, d in _find_dags():
+            dparts = d.key.split('.')
+            if dparts[-1]==dag_name:
+                if found_dag:
+                    raise BlockError(f'Found duplicate: {dag_name}, d')
+
+                found_dag = d
+
+        if found_dag is None:
+            raise BlockError('No such dag')
+
+        dag_name = found_dag.key
+        ix = dag_name.rfind('.')
+
+    m = importlib.import_module(dag_name[:ix])
+    func = getattr(m, dag_name[ix+1:])
+    # if not issubclass(cls, Block):
+    #     raise BlockError(f'{key} is not a block')
+
+    dag = func()
+    if not hasattr(dag, 'show'):
+        raise BlockError(f'Dag {dag_name} does not have a user interface')
+
+    dag.show()
+
+def _find(func_name: str) -> Iterable[tuple[EntryPoint, Info]]:
+    """Use ``importlib.metadata.entry_points`` to look up entry points named ``sier2.library``.
+
+    For each entry point, call ``load()`` to get a module,
+    then call ``getattr(module, func_name)()`` to get a list of
+    ``BlockInfo`` instances.
+
+    Parameters
+    ----------
+    func_name: str
+        The name of the function that will be called to get a list[Info].
+        Either ``'blocks'`` or ``'dags'``.
+    """
+
+    library = entry_points(group='sier2.library')
+
+    for entry_point in library:
+        try:
+            lib = entry_point.load()
+            func = getattr(lib, func_name, None)
+            if func is not None:
+                if not callable(func):
+                    warnings.warn(f'In {entry_point.module}, {func} is not a function')
+                else:
+                    info_list: list[Info] = func()
+                    if not isinstance(info_list, list) or any(not isinstance(s, Info) for s in info_list):
+                        warnings.warn(f'In {entry_point.module}, {func} does not return a list of {Info.__name__} instances')
+                    else:
+                        for gi in info_list:
+                            yield entry_point, gi
+        except Exception as e:
+            raise BlockError(f'While loading {entry_point}: {e}') from e
+
+class Library:
+    @staticmethod
+    def collect_blocks():
+        """Collect block information.
+
+        Use ``_find_blocks()`` to yield ``BlockInfo`` instances.
+
+        Note that we don't load the blocks here. We don't want to import
+        any modules: this would cause every block module to be imported,
+        which would cause a lot of imports to happen. Therefore, we just
+        create the keys in the dictionary, and let ``get_block()`` import
+        block modules as required.
+        """
+
+        for entry_point, gi in _find_blocks():
+            if gi.key in _block_library:
+                warnings.warn(f'Block plugin {entry_point}: key {gi.key} already in library')
+            else:
+                _block_library[gi.key] = None
+
+    @staticmethod
+    def collect_dags():
+        for entry_point, gi in _find_dags():
+            if gi.key in _dag_library:
+                warnings.warn(f'Dag plugin {entry_point}: key {gi.key} already in library')
+            else:
+                _dag_library.add(gi.key)
+
+    @staticmethod
+    def add_block(block_class: type[Block], key: str|None=None):
+        """Add a local block class to the library.
+
+        The library initially loads block classes using Python's entry_points() mechanism.
+        This method allows local Blocks to be added to the library.
+
+        This is useful for testing, for example.
+
+        Parameters
+        ----------
+        block_class: type[Block]
+            The Block's class.
+        key: str
+            The Block's unique key string. By default, the block's block_key()
+            class method will be used to obtain the key.
+        """
+
+        if not issubclass(block_class, Block):
+            print(f'{key} is not a Block')
+
+        # if not key:
+        #     key = block_class.block_key()
+        key_ = key if key else block_class.block_key()
+
+        if key_ in _block_library:
+            raise BlockError(f'Block {key_} is already in the library')
+
+        _block_library[key_] = block_class
+
+    @staticmethod
+    def get_block(key: str) -> type[Block]:
+        if not _block_library:
+            Library.collect_blocks()
+
+        if key not in _block_library:
+            raise BlockError(f'Block name {key} is not in the library')
+
+        if _block_library[key] is None:
+            ix = key.rfind('.')
+            m = importlib.import_module(key[:ix])
+            cls = getattr(m, key[ix+1:])
+            if not issubclass(cls, Block):
+                raise BlockError(f'{key} is not a block')
+
+            # The fully qualified name of the class is probably not the same as
+            # the library key string. This matters when the dag is dumped and loaded.
+            # Therefore we tell the class what its key is so the key can be dumped,
+            # and when the dag is loaded, the block can be found using
+            # Library.get_block().
+            #
+            setattr(cls, Block.SIER2_KEY, key)
+
+            _block_library[key] = cls
+
+        return cast(type[Block], _block_library[key])
+
+    @staticmethod
+    def get_dag(key: str) -> type[Dag]:
+        if not _dag_library:
+            Library.collect_dags()
+
+        if key not in _dag_library:
+            raise BlockError(f'Dag name {key} is not in the library')
+
+        if key in _dag_library:
+            ix = key.rfind('.')
+            m = importlib.import_module(key[:ix])
+            func = getattr(m, key[ix+1:])
+            dag = func()
+            if not isinstance(dag, Dag):
+                raise BlockError(f'{key} is not a dag')
+
+        return cast(type[Dag], dag)
+
+    @staticmethod
+    def load_dag(dump: dict[str, Any]) -> Dag:
+        """Load a dag from a serialised structure produced by Block.dump()."""
+
+        # Create new instances of the specified blocks.
+        #
+        instances = {}
+        for g in dump['blocks']:
+            block_key = g['block']
+            instance = g['instance']
+            if instance not in instances:
+                gclass = Library.get_block(block_key)
+                instances[instance] = gclass(**g['args'])
+            else:
+                raise BlockError(f'Instance {instance} ({block_key}) already exists')
+
+        # Connect the blocks.
+        #
+        DagType = PanelDag if dump['dag']['type']=='PanelDag' else Dag
+        dag = DagType(doc=dump['dag']['doc'], site=dump['dag']['site'], title=dump['dag']['title'])
+        for conn in dump['connections']:
+            conns = [Connection(**kwargs) for kwargs in conn['conn_args']]
+            dag.connect(instances[conn['src']], instances[conn['dst']], *conns)
+
+        return dag
+
+# Library.collect()

sier2/_logger.py

@@ -0,0 +1,65 @@
+import logging
+
+_BLOCK_FORMATTER = logging.Formatter('%(asctime)s %(levelname)s [%(block_name)s] %(message)s', datefmt='%H:%M:%S')
+# formatter = logging.Formatter('%(asctime)s %(levelname)s [%(block_name)s] - %(levelname)s - %(message)s', datefmt='%H:%M:%S')
+
+# class BlockHandler(logging.StreamHandler):
+#     def format(self, record):
+#         fmt = info_formatter# if record.levelno==logging.INFO else formatter
+
+#         return fmt.format(record)
+
+class BlockAdapter(logging.LoggerAdapter):
+    """An adapter that log messages from blocks.
+
+    Each block has its own adapter, so the log automatically includes the block name.
+    """
+
+    def __init__(self, logger, block_name: str, block_state):
+        super().__init__(logger)
+        self.block_name = block_name
+        self.block_state = block_state
+
+    def debug(self, msg, *args):
+        super().debug(msg, *args, extra={'block_name': self.block_name, 'block_state': self.block_state})
+
+    def info(self, msg, *args):
+        super().info(msg, *args, extra={'block_name': self.block_name, 'block_state': self.block_state})
+
+    def warning(self, msg, *args):
+        super().warning(msg, *args, extra={'block_name': self.block_name, 'block_state': self.block_state})
+
+    def error(self, msg, *args):
+        super().error(msg, *args, extra={'block_name': self.block_name, 'block_state': self.block_state})
+
+    def exception(self, msg, *args, exc_info=True):
+        super().error(msg, *args, exc_info=exc_info, extra={'block_name': self.block_name, 'block_state': self.block_state})
+
+    def critical(self, msg, *args):
+        super().critical(msg, *args, extra={'block_name': self.block_name, 'block_state': self.block_state})
+
+    def process(self, msg, kwargs):
+        # print(f'BLOCKADAPTER {msg=} {kwargs=} {self.extra=}')
+        if 'block_state' not in kwargs['extra']:
+            kwargs['extra']['block_state'] = '?'
+        if 'block_name' not in kwargs['extra']:
+            kwargs['extra']['block_name'] = 'g'
+
+        return msg, kwargs
+
+_logger = logging.getLogger('block.stream')
+_logger.setLevel(logging.INFO)
+
+# _ph = BlockHandler()
+# _ph.setLevel(logging.DEBUG)
+
+_ph = logging.StreamHandler()
+_ph.setFormatter(_BLOCK_FORMATTER)
+_ph.setLevel(logging.DEBUG)
+
+_logger.addHandler(_ph)
+
+def get_logger(block_name: str):
+    adapter = BlockAdapter(_logger, block_name, None)
+
+    return adapter

sier2/_util.py

@@ -0,0 +1,65 @@
+import sys
+
+########
+# Documentation utilities
+########
+
+def trim(docstring):
+    """From PEP-257: Fix docstring indentation"""
+
+    if not docstring:
+        return ''
+    # Convert tabs to spaces (following the normal Python rules)
+    # and split into a list of lines:
+    lines = docstring.expandtabs().splitlines()
+    # Determine minimum indentation (first line doesn't count):
+    indent = sys.maxsize
+    for line in lines[1:]:
+        stripped = line.lstrip()
+        if stripped:
+            indent = min(indent, len(line) - len(stripped))
+    # Remove indentation (first line is special):
+    trimmed = [lines[0].strip()]
+    if indent < sys.maxsize:
+        for line in lines[1:]:
+            trimmed.append(line[indent:].rstrip())
+    # Strip off trailing and leading blank lines:
+    while trimmed and not trimmed[-1]:
+        trimmed.pop()
+    while trimmed and not trimmed[0]:
+        trimmed.pop(0)
+    # Return a single string:
+    return '\n'.join(trimmed)
+
+def block_doc_text(block):
+    """Generate text documentation for a block.
+
+    The documentation is taken from the docstring of the block class
+    and the doc of each 'in_' and 'out_' param.
+    """
+
+    # Force the first line of the block docstring to have a level 2 header.
+    #
+    b_doc = '## ' + trim(block.__doc__).lstrip(' #')
+
+    params = []
+    for name, p in block.param.objects().items():
+        if name.startswith(('in_', 'out_')):
+            doc = p.doc if p.doc else ''
+            params.append((name, doc.strip()))
+
+    params.sort()
+    text = []
+    for name, doc in params:
+        text.append(f'- {name}:  {doc}\n')
+
+    return '---\n' + b_doc + '\n### Params\n' + '\n'.join(text)
+
+def dag_doc_text(dag):
+    """Generate text documentation for a dag."""
+
+    # Force the first line of the dag doc to have a level 1 header.
+    #
+    dag_text =f'# {dag.site} - {dag.title}\n\n# ' + trim(dag.doc).lstrip(' #')
+
+    return dag_text

sier2/_version.py

@@ -0,0 +1,3 @@
+from importlib.metadata import version
+
+__version__ = version('sier2')

sier2/panel/__init__.py

@@ -0,0 +1 @@
+from ._panel import PanelDag

sier2/panel/_feedlogger.py

@@ -0,0 +1,153 @@
+
+"""A logger that logs to a panel.widget.Feed."""
+
+from datetime import datetime
+import html
+import logging
+import panel as pn
+
+from .._block import BlockState
+from ._panel_util import _get_state_color
+
+_INFO_FORMATTER = logging.Formatter('%(asctime)s %(block_state)s %(block_name)s %(message)s', datefmt='%H:%M:%S')
+_FORMATTER = logging.Formatter('%(asctime)s %(block_state)s %(block_name)s - %(levelname)s - %(message)s', datefmt='%H:%M:%S')
+
+class PanelHandler(logging.Handler):
+    """A handler that emits log strings to a panel template sidebar Feed pane."""
+
+    def __init__(self, log_feed):
+        super().__init__()
+        self.log_feed = log_feed
+
+    def format(self, record):
+        # TODO override logging.Formatter.formatException to <pre> the exception string.
+
+        color = _get_state_color(record.block_state)
+
+        record.block_name = f'[{html.escape(record.block_name)}]' if record.block_name else ''
+        record.block_state = f'<span style="color:{color};">■</span>'
+        record.msg = html.escape(record.msg)
+        fmt = _INFO_FORMATTER if record.levelno==logging.INFO else _FORMATTER
+
+        return fmt.format(record)
+
+    def emit(self, record):
+        if record.block_state is None:
+            self.log_feed.clear()
+            return
+
+        try:
+            msg = self.format(record)
+            self.log_feed.append(pn.pane.HTML(msg))
+        except RecursionError:  # See issue 36272
+            raise
+        except Exception:
+            self.handleError(record)
+
+class DagPanelAdapter(logging.LoggerAdapter):
+    """An adapter that logs messages from a dag.
+
+    Each message also specifies a block name and state.
+    """
+
+    def debug(self, msg, *args, block_name, block_state):
+        super().debug(msg, *args, extra={'block_name': block_name, 'block_state': block_state})
+
+    def info(self, msg, *args, block_name, block_state):
+        super().info(msg, *args, extra={'block_name': block_name, 'block_state': block_state})
+
+    def warning(self, msg, *args, block_name, block_state):
+        super().warning(msg, *args, extra={'block_name': block_name, 'block_state': block_state})
+
+    def error(self, msg, *args, block_name, block_state):
+        super().error(msg, *args, extra={'block_name': block_name, 'block_state': block_state})
+
+    def exception(self, msg, *args, block_name, block_state):
+        super().exception(msg, *args, extra={'block_name': block_name, 'block_state': block_state})
+
+    def critical(self, msg, *args, block_name, block_state):
+        super().critical(msg, *args, extra={'block_name': block_name, 'block_state': block_state})
+
+    def process(self, msg, kwargs):
+        # print(f'ADAPTER {msg=} {kwargs=} {self.extra=}')
+        if 'block_state' not in kwargs['extra']:
+            kwargs['extra']['block_state'] = '?'
+        if 'block_name' not in kwargs['extra']:
+            kwargs['extra']['block_name'] = 'g'
+
+        return msg, kwargs
+
+_logger = logging.getLogger('block.panel')
+_logger.setLevel(logging.INFO)
+
+# ph = PanelHandler(log_feed)
+# ph.log_feed = log_feed
+# ph.setLevel(logging.INFO)
+
+# _logger.addHandler(ph)
+
+def getDagPanelLogger(log_feed):
+    # _logger = logging.getLogger('block.panel')
+    # _logger.setLevel(logging.INFO)
+
+    ph = PanelHandler(log_feed)
+    ph.log_feed = log_feed
+    ph.setLevel(logging.INFO)
+
+    _logger.addHandler(ph)
+
+    adapter = DagPanelAdapter(_logger)
+
+    return adapter
+
+####
+
+class BlockPanelAdapter(logging.LoggerAdapter):
+    """An adapter that logs messages from a block.
+
+    A state isn't required, because if a block is logging something,
+    it's executing by definition.
+
+    A name isn't required in the logging methods, because the name is
+    implicit.
+    """
+
+    def __init__(self, logger, block_name, extra=None):
+        super().__init__(logger, extra)
+        self.block_name = block_name
+
+    def debug(self, msg, *args):
+        super().debug(msg, *args, extra={'block_name': self.block_name, 'block_state': BlockState.BLOCK})
+
+    def info(self, msg, *args):
+        super().info(msg, *args, extra={'block_name': self.block_name, 'block_state': BlockState.BLOCK})
+
+    def warning(self, msg, *args):
+        super().warning(msg, *args, extra={'block_name': self.block_name, 'block_state': BlockState.BLOCK})
+
+    def error(self, msg, *args):
+        super().error(msg, *args, extra={'block_name': self.block_name, 'block_state': BlockState.BLOCK})
+
+    def exception(self, msg, *args):
+        super().exception(msg, *args, extra={'block_name': self.block_name, 'block_state': BlockState.BLOCK})
+
+    def critical(self, msg, *args):
+        super().critical(msg, *args, extra={'block_name': self.block_name, 'block_state': BlockState.BLOCK})
+
+    def process(self, msg, kwargs):
+        # print(f'GP ADAPTER {msg=} {kwargs=} {self.extra=}')
+        if 'block_state' not in kwargs['extra']:
+            kwargs['extra']['block_state'] = BlockState.BLOCK
+        if 'block_name' not in kwargs['extra']:
+            kwargs['extra']['block_name'] = self.block_name
+
+        return msg, kwargs
+
+def getBlockPanelLogger(block_name: str):
+    """A logger for blocks.
+
+    The dag gets its logger first, so we can reuse _logger."""
+
+    adapter = BlockPanelAdapter(_logger, block_name)
+
+    return adapter