ezmsg 3.3.4__tar.gz → 3.4.0__tar.gz

{ezmsg-3.3.4 → ezmsg-3.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ezmsg
-Version: 3.3.4
+Version: 3.4.0
 Summary: A simple DAG-based computation model
 License: MIT
 Author: Milsap, Griffin
@@ -61,7 +61,7 @@ $ source env/bin/activate
 (env) $ python -m pytest tests # Optionally, Perform tests
 ```
 
-Note that it is generally recommended to install poetry into it's own standalone venv via the `pipx` cli tool.
+Note that it is generally recommended to install poetry into its own standalone venv via the `pipx` cli tool.
 
 ## Documentation
 
@@ -78,7 +78,7 @@ pip install "ezmsg[all_ext]"
 ```
 
 This will install all the available public extension packages for `ezmsg` that are listed in `pyproject.toml`.
-If you prefer to install the extension packages individually, you can use the following command:
+If you prefer to install a subset of extension packages, you can use the following command:
 
 ```bash
 pip install "ezmsg[zmq,sigproc,...]"
@@ -86,17 +86,14 @@ pip install "ezmsg[zmq,sigproc,...]"
 
 Please note that the `ezmsg` package itself can still be installed without any additional extensions using `pip install ezmsg`.
 
-See the extension directory for more details
-
-- `ezmsg-sigproc` -- Timeseries signal processing modules
-- `ezmsg-websocket` -- Websocket server and client nodes for `ezmsg` graphs
-- `ezmsg-zmq` -- ZeroMQ pub and sub nodes for `ezmsg` graphs
-- ... More to come!
-
-Additionally, the following extensions are contained in external repositories:
+Extensions can be managed manually as well. Here are some of the extensions we manage or are aware of:
 
+- [ezmsg-sigproc](https://github.com/ezmsg-org/ezmsg-sigproc) -- Timeseries signal processing modules
+- [ezmsg-websocket](https://github.com/ezmsg-org/ezmsg-websocket) -- Websocket server and client nodes for `ezmsg` graphs
+- [ezmsg-zmq](https://github.com/ezmsg-org/ezmsg-zmq) -- ZeroMQ pub and sub nodes for `ezmsg` graphs
 - [ezmsg-panel](https://github.com/griffinmilsap/ezmsg-panel) -- Plotting tools for `ezmsg` that use [panel](https://github.com/holoviz/panel)
 - [ezmsg-blackrock](https://github.com/griffinmilsap/ezmsg-blackrock) -- Interface for Blackrock Cerebus ecosystem (incl. Neuroport) using `pycbsdk`
+- [ezmsg-lsl](https://github.com/ezmsg-org/ezmsg-lsl) -- Source unit for LSL Inlet and sink unit for LSL Outlet
 - [ezmsg-unicorn](https://github.com/griffinmilsap/ezmsg-unicorn) -- g.tec Unicorn Hybrid Black integration for `ezmsg`
 - [ezmsg-gadget](https://github.com/griffinmilsap/ezmsg-gadget) -- USB-gadget with HID control integration for Raspberry Pi (Zero/W/2W, 4, CM4)
 - [ezmsg-openbci](https://github.com/griffinmilsap/ezmsg-openbci) -- OpenBCI Cyton serial interface for `ezmsg`

{ezmsg-3.3.4 → ezmsg-3.4.0}/README.md

@@ -33,7 +33,7 @@ $ source env/bin/activate
 (env) $ python -m pytest tests # Optionally, Perform tests
 ```
 
-Note that it is generally recommended to install poetry into it's own standalone venv via the `pipx` cli tool.
+Note that it is generally recommended to install poetry into its own standalone venv via the `pipx` cli tool.
 
 ## Documentation
 
@@ -50,7 +50,7 @@ pip install "ezmsg[all_ext]"
 ```
 
 This will install all the available public extension packages for `ezmsg` that are listed in `pyproject.toml`.
-If you prefer to install the extension packages individually, you can use the following command:
+If you prefer to install a subset of extension packages, you can use the following command:
 
 ```bash
 pip install "ezmsg[zmq,sigproc,...]"
@@ -58,17 +58,14 @@ pip install "ezmsg[zmq,sigproc,...]"
 
 Please note that the `ezmsg` package itself can still be installed without any additional extensions using `pip install ezmsg`.
 
-See the extension directory for more details
-
-- `ezmsg-sigproc` -- Timeseries signal processing modules
-- `ezmsg-websocket` -- Websocket server and client nodes for `ezmsg` graphs
-- `ezmsg-zmq` -- ZeroMQ pub and sub nodes for `ezmsg` graphs
-- ... More to come!
-
-Additionally, the following extensions are contained in external repositories:
+Extensions can be managed manually as well. Here are some of the extensions we manage or are aware of:
 
+- [ezmsg-sigproc](https://github.com/ezmsg-org/ezmsg-sigproc) -- Timeseries signal processing modules
+- [ezmsg-websocket](https://github.com/ezmsg-org/ezmsg-websocket) -- Websocket server and client nodes for `ezmsg` graphs
+- [ezmsg-zmq](https://github.com/ezmsg-org/ezmsg-zmq) -- ZeroMQ pub and sub nodes for `ezmsg` graphs
 - [ezmsg-panel](https://github.com/griffinmilsap/ezmsg-panel) -- Plotting tools for `ezmsg` that use [panel](https://github.com/holoviz/panel)
 - [ezmsg-blackrock](https://github.com/griffinmilsap/ezmsg-blackrock) -- Interface for Blackrock Cerebus ecosystem (incl. Neuroport) using `pycbsdk`
+- [ezmsg-lsl](https://github.com/ezmsg-org/ezmsg-lsl) -- Source unit for LSL Inlet and sink unit for LSL Outlet
 - [ezmsg-unicorn](https://github.com/griffinmilsap/ezmsg-unicorn) -- g.tec Unicorn Hybrid Black integration for `ezmsg`
 - [ezmsg-gadget](https://github.com/griffinmilsap/ezmsg-gadget) -- USB-gadget with HID control integration for Raspberry Pi (Zero/W/2W, 4, CM4)
 - [ezmsg-openbci](https://github.com/griffinmilsap/ezmsg-openbci) -- OpenBCI Cyton serial interface for `ezmsg`

{ezmsg-3.3.4 → ezmsg-3.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "ezmsg"
-version = "3.3.4"
+version = "3.4.0"
 description = "A simple DAG-based computation model"
 authors = [
   "Milsap, Griffin <griffin.milsap@gmail.com>",
@@ -39,6 +39,12 @@ pytest-cov = "*"
 numpy = "^1.24.2"
 flake8 = "*"
 
+
+[tool.poetry.group.docs.dependencies]
+sphinx = "<7.2"
+sphinx-rtd-theme = "^2.0.0"
+ezmsg-sigproc = { version = "*", source = "pypi" }
+
 [tool.poetry.extras]
 sigproc = ["ezmsg-sigproc"]
 websocket = ["ezmsg-websocket"]

{ezmsg-3.3.4 → ezmsg-3.4.0}/src/ezmsg/core/backend.py

@@ -167,6 +167,27 @@ def run(
     force_single_process: bool = False,
     **components_kwargs: Component,
 ) -> None:
+    """
+    Begin execution of a set of :obj:`Component` s.
+
+    `The old method` :obj:`run_system` `has been deprecated and uses` ``run()`` `instead.`
+
+    Args:
+        components: represents the nodes in the directed acyclic graph. It is a dictionary which contains the
+            ``Components`` to be run mapped to string names. On initialization, ``ezmsg`` will call ``initialize()``
+            for each :obj:`Unit` and ``configure()`` for each :obj:`Collection`, if defined.
+        root_name:
+        connections: represents the edges is a ``NetworkDefinition`` which connects
+            ``OutputStreams`` to ``InputStreams``. On initialization, ``ezmsg`` will create a directed acyclic graph
+            using the contents of this parameter.
+        process_components: a list of ``Components`` which should live in their own process.
+        backend_process: is currently under development.
+        graph_address: the hostname and port of the graph server which ``ezmsg`` should connect to.
+            If not defined, ``ezmsg`` will start a new graph server at 127.0.0.1:25978.
+        force_single_process: run all ``Components`` in one process.
+            This is necessary when running ``ezmsg`` in a notebook.
+        components_kwargs:
+    """
     # FIXME: This function is the last major re-implementation needed to make this
     # codebase more maintainable.
     graph_service = GraphService(graph_address)

{ezmsg-3.3.4 → ezmsg-3.4.0}/src/ezmsg/core/backendprocess.py

@@ -41,10 +41,17 @@ logger = logging.getLogger("ezmsg")
 
 
 class Complete(Exception):
+    """
+    A type of ``Exception`` which signals to ``ezmsg`` that the function can be shut down gracefully.
+    If all functions in all :obj:`Units` raise ``Complete``, the entire pipeline will terminate execution.
+    """
     pass
 
 
 class NormalTermination(Exception):
+    """
+    A type of ``Exception`` which signals to ``ezmsg`` that the pipeline can be shut down gracefully.
+    """
     pass
 
 
@@ -294,32 +301,28 @@ class DefaultBackendProcess(BackendProcess):
             )
 
         pub_fn = perf_publish if hasattr(task, TIMEIT_ATTR) else publish
+        
+        call_fn = lambda _: task(unit)
+        signature = inspect.signature(task)
+        if len(signature.parameters) == 1:
+            call_fn = lambda _: task(unit)
+        elif len(signature.parameters) == 2:
+            call_fn = lambda msg: task(unit, msg)
+        else:
+            logger.error(f'Incompatible call signature on task: {task.__name__}')
 
         @wraps(task)
         async def wrapped_task(msg: Any = None) -> None:
             try:
-                # If we don't sub or pub anything, we are a simple task
-                if not hasattr(task, SUBSCRIBES_ATTR) and not hasattr(
-                    task, PUBLISHES_ATTR
-                ):
-                    await task(unit)
-
-                # No subscriptions; only publications...
-                elif not hasattr(task, SUBSCRIBES_ATTR):
-                    async for stream, obj in task(unit):
+                result = call_fn(msg)
+                if inspect.isasyncgen(result):
+                    async for stream, obj in result:
+                        if obj and getattr(task, ZERO_COPY_ATTR, False) and obj is msg:
+                            obj = deepcopy(obj)
                         await pub_fn(stream, obj)
 
-                # Subscribers need to be called with a message
-                else:
-                    if not getattr(task, ZERO_COPY_ATTR):
-                        msg = deepcopy(msg)
-                    if hasattr(task, PUBLISHES_ATTR):
-                        async for stream, obj in task(unit, msg):
-                            if getattr(task, ZERO_COPY_ATTR) and obj is msg:
-                                obj = deepcopy(obj)
-                            await pub_fn(stream, obj)
-                    else:
-                        await task(unit, msg)
+                elif asyncio.iscoroutine(result):
+                    await result
 
             except Complete:
                 logger.info(f"{task_address} Complete")

{ezmsg-3.3.4 → ezmsg-3.4.0}/src/ezmsg/core/collection.py

@@ -34,7 +34,9 @@ class CollectionMeta(ComponentMeta):
 
 
 class Collection(Component, metaclass=CollectionMeta):
-    """Collections can contain subunits and connect them together"""
+    """
+    Connects :obj:`Unit` s together by defining a graph which connects :obj:`OutputStream` s to :obj:`InputStream` s.
+    """
 
     def __init__(self, *args, settings: typing.Optional[Settings] = None, **kwargs):
         super(Collection, self).__init__(*args, settings=settings, **kwargs)
@@ -44,11 +46,24 @@ class Collection(Component, metaclass=CollectionMeta):
             setattr(self, comp_name, comp)
 
     def configure(self) -> None:
-        """This is where to percolate apply_settings to subnodes"""
+        """
+        A lifecycle hook that runs when the :obj:`Collection` is instantiated.
+        This is the best place to call ``Unit.apply_settings()`` on each member :obj:`Unit` of the :obj:`Collection`.
+        """
         ...
 
     def network(self) -> NetworkDefinition:
+        """
+        Override this method and have the definition return a :obj:`NetworkDefinition` which defines how
+        :obj:`InputStream` and :obj:`OutputStream` from member :obj:`Unit` s will be connected.
+        """
         return ()
 
     def process_components(self) -> typing.Collection[Component]:
+        """
+        Override this method and have the definition return a tuple which contains :obj:`Unit` and :obj:`Collection`
+        which should run in their own processes.
+
+        Return: the :obj:`Collection`.
+        """
         return (self,)

{ezmsg-3.3.4 → ezmsg-3.4.0}/src/ezmsg/core/command.py

@@ -1,4 +1,5 @@
 import os
+import sys
 import base64
 import asyncio
 import argparse
@@ -97,7 +98,7 @@ async def run_command(cmd: str, graph_address: Address, shm_address: Address) ->
 
     elif cmd == "start":
         popen = subprocess.Popen(
-            ["python", "-m", "ezmsg.core", "serve", f"--address={graph_address}"]
+            [sys.executable, "-m", "ezmsg.core", "serve", f"--address={graph_address}"]
         )
 
         while True:

{ezmsg-3.3.4 → ezmsg-3.4.0}/src/ezmsg/core/component.py

@@ -72,6 +72,10 @@ class ComponentMeta(ABCMeta):
 
 
 class Component(Addressable, metaclass=ComponentMeta):
+    """
+    Metaclass which :obj:`Unit` and :obj:`Collection` inherit from.
+    """
+
     SETTINGS: Settings
     STATE: State
 
@@ -96,21 +100,21 @@ class Component(Addressable, metaclass=ComponentMeta):
         for stream_name, stream in self.streams.items():
             setattr(self, stream_name, stream)
 
-        try:
-            if settings is None:
-                # settings not supplied as a kwarg. Try to build it.
-                if len(args) > 0 and type(args[0]) == self.__class__.__settings_type__:
-                    settings = args[0]
-                elif len(args) > 0 or len(kwargs) > 0:
-                    settings = self.__class__.__settings_type__(*args, **kwargs)
-                else:
+        if settings is None:
+            # settings not supplied as a kwarg. Try to build it.
+            if len(args) > 0 and type(args[0]) == self.__class__.__settings_type__:
+                settings = args[0]
+            elif len(args) > 0 or len(kwargs) > 0:
+                settings = self.__class__.__settings_type__(*args, **kwargs)
+            else:
+                try:
                     # If we weren't supplied settings, we will try to
                     # instantiate the settings type from annotations
                     settings = self.__class__.__settings_type__()
-        except TypeError:
-            # We couldn't instantiate settings with default value
-            # We will rely on late configuration via apply_settings
-            pass
+                except TypeError:
+                    # We couldn't instantiate settings with default value
+                    # We will rely on late configuration via apply_settings
+                    pass
 
         if settings is not None:
             self.apply_settings(settings)
@@ -132,6 +136,13 @@ class Component(Addressable, metaclass=ComponentMeta):
                 )
 
     def apply_settings(self, settings: Settings) -> None:
+        """
+        Update the ``Component``‘s ``Settings`` object.
+
+        Args:
+            settings: An instance of the class-specific ``Settings``.
+
+        """
         self.SETTINGS = settings
         self._settings_applied = True
 

ezmsg-3.4.0/src/ezmsg/core/settings.py

@@ -0,0 +1,65 @@
+import sys
+import typing
+
+from abc import ABC, ABCMeta
+from dataclasses import dataclass
+
+if sys.version_info < (3, 12):
+    from typing_extensions import dataclass_transform
+else:
+    from typing import dataclass_transform
+
+# All settings classes are dataclasses
+# https://rednafi.github.io/digressions/python/2020/06/26/python-metaclasses.html
+#  see -- #avoiding-dataclass-decorator-with-metaclasses
+
+
+@dataclass_transform()
+class SettingsMeta(ABCMeta):
+    def __new__(
+        cls,
+        name: str,
+        bases: typing.Tuple[type, ...],
+        classdict: typing.Dict[str, typing.Any],
+        **kwargs: typing.Any
+    ) -> typing.Type["Settings"]:
+        new_cls = super().__new__(cls, name, bases, classdict)
+        return dataclass(frozen=True)(new_cls)  # type: ignore
+
+
+class Settings(ABC, metaclass=SettingsMeta):
+    """
+    To pass parameters into a :obj:`Component`, inherit from ``Settings``.
+
+    .. code-block:: python
+
+       class YourSettings(Settings):
+          setting1: int
+          setting2: float
+
+    To use, declare the ``Settings`` object for a ``Component`` as a member variable called (all-caps!) ``SETTINGS``. ``ezmsg`` will monitor the variable called ``SETTINGS`` in the background, so it is important to name it correctly.
+
+    .. code-block:: python
+
+       class YourUnit(Unit):
+
+          SETTINGS: YourSettings
+
+    A ``Unit`` can accept a ``Settings`` object as a parameter on instantiation.
+
+    .. code-block:: python
+
+       class YourCollection(Collection):
+
+          YOUR_UNIT = YourUnit(
+             YourSettings(
+                setting1: int,
+                setting2: float
+             )
+          )
+
+    .. note::
+       ``Settings`` uses type hints to define member variables, but does not enforce type checking.
+
+    """
+    ...

ezmsg-3.4.0/src/ezmsg/core/state.py

@@ -0,0 +1,56 @@
+from abc import ABC, ABCMeta
+from dataclasses import dataclass
+
+from typing import (
+    Dict,
+    Tuple,
+    Any,
+    Type,
+)
+
+
+class StateMeta(ABCMeta):
+    def __new__(
+        cls,
+        name: str,
+        bases: Tuple[type, ...],
+        classdict: Dict[str, Any],
+        **kwargs: Any
+    ) -> Type["State"]:
+        new_cls = super().__new__(cls, name, bases, classdict)
+        return dataclass(unsafe_hash=True, frozen=False, init=False)(new_cls)  # type: ignore
+
+
+class State(ABC, metaclass=StateMeta):
+    """
+    States are mutable dataclasses that are instantiated by the Unit in its home process.
+
+    To track a mutable state for a ``Component``, inherit from ``State``.
+
+    .. code-block:: python
+
+       class YourState(State):
+          state1: int
+          state2: float
+
+    To use, declare the ``State`` object for a ``Component`` as a member variable called (all-caps!) ``STATE``.
+    ``ezmsg`` will monitor the variable called ``STATE`` in the background, so it is important to name it correctly.
+
+    Member functions can then access and mutate ``STATE`` as needed during function execution.
+    It is recommended to initialize state values inside the ``initialize()`` or ``configure()`` lifecycle hooks if
+    defaults are not defined.
+
+    .. code-block:: python
+
+       class YourUnit(Unit):
+
+          STATE: YourState
+
+          def initialize(self):
+             this.STATE.state1 = 0
+             this.STATE.state2 = 0.0
+
+    .. note::
+       ``State`` uses type hints to define member variables, but does not enforce type checking.
+    """
+    ...

{ezmsg-3.3.4 → ezmsg-3.4.0}/src/ezmsg/core/stream.py

@@ -5,6 +5,10 @@ from .addressable import Addressable
 
 
 class Stream(Addressable):
+    """
+
+    """
+
     msg_type: Type
 
     def __init__(self, msg_type: Type):
@@ -17,11 +21,17 @@ class Stream(Addressable):
 
 
 class InputStream(Stream):
+    """
+    Can be added to any ``Component`` as a member variable. Methods may subscribe to it.
+    """
     def __repr__(self) -> str:
         return f"Input{super().__repr__()}()"
 
 
 class OutputStream(Stream):
+    """
+    Can be added to any ``Component`` as a member variable. Methods may publish to it.
+    """
     host: Optional[str]
     port: Optional[int]
     num_buffers: int

{ezmsg-3.3.4 → ezmsg-3.4.0}/src/ezmsg/core/unit.py

@@ -59,7 +59,11 @@ class UnitMeta(ComponentMeta):
 
 
 class Unit(Component, metaclass=UnitMeta):
-    """Units can subscribe, publish, and have tasks"""
+    """
+    Represents a single step in the graph.
+    Units can subscribe, publish, and have tasks.
+    To create a ``Unit``, inherit from the ``Unit`` class.
+    """
 
     def __init__(self, *args, settings: Optional[Settings] = None, **kwargs):
         super(Unit, self).__init__(*args, settings=settings, **kwargs)
@@ -85,16 +89,42 @@ class Unit(Component, metaclass=UnitMeta):
         self._check_state()
 
     async def initialize(self) -> None:
-        """This is called from within the same process this unit will live"""
+        """
+        Runs when the ``Unit`` is instantiated.
+        This is called from within the same process this unit will live.
+        This lifecycle hook can be overridden. It can be run as ``async`` functions by simply adding the
+        ``async`` keyword when overriding.
+        """
         pass
 
     async def shutdown(self) -> None:
-        """This is called from within the same process this unit will live"""
+        """
+        Runs when the ``Unit`` terminates.
+        This is called from within the same process this unit will live.
+        This lifecycle hook can be overridden. It can be run as ``async`` functions by simply adding the
+        ``async`` keyword when overriding.
+        """
         pass
 
 
 def publisher(stream: OutputStream):
-    """A decorator for a method that publishes to a stream in the task/messaging thread"""
+    """
+    A decorator for a method that publishes to a stream in the task/messaging thread.
+    An async function will yield messages on the designated :obj:`OutputStream`.
+
+    .. code-block:: python
+
+      from typing import AsyncGenerator
+
+      OUTPUT = OutputStream(ez.Message)
+
+      @publisher(OUTPUT)
+      async def send_message(self) -> AsyncGenerator:
+         message = Message()
+         yield(OUTPUT, message)
+
+    A function can have both ``@subscriber`` and ``@publisher`` decorators.
+    """
 
     if not isinstance(stream, OutputStream):
         raise ValueError(f"Cannot publish to object of type {type(stream)}")
@@ -109,7 +139,22 @@ def publisher(stream: OutputStream):
 
 
 def subscriber(stream: InputStream, zero_copy: bool = False):
-    """A decorator for a method that subscribes to a stream in the task/messaging thread"""
+    """
+    A decorator for a method that subscribes to a stream in the task/messaging thread.
+    An async function will run once per message received from the :obj:`InputStream` it subscribes to.
+
+    Example:
+
+    .. code-block:: python
+
+      INPUT = ez.InputStream(Message)
+
+      @subscriber(INPUT)
+      async def print_message(self, message: Message) -> None:
+         print(message)
+
+    A function can have both ``@subscriber`` and ``@publisher`` decorators.
+    """
 
     if not isinstance(stream, InputStream):
         raise ValueError(f"Cannot subscribe to object of type {type(stream)}")
@@ -126,12 +171,18 @@ def subscriber(stream: InputStream, zero_copy: bool = False):
 
 
 def main(func: Callable):
-    """A decorator for a function that runs as the main thread.  A Unit may only have one of these."""
+    """
+    A decorator which designates this function to run as the main thread for this :obj:`Unit`.
+    A :obj:`Unit` may only have one of these.
+    """
     setattr(func, MAIN_ATTR, True)
     return func
 
 
 def timeit(func: Callable):
+    """
+    ``ezmsg`` will log the amount of time this function takes to execute.
+    """
     setattr(func, TIMEIT_ATTR, True)
 
     @functools.wraps(func)
@@ -148,17 +199,24 @@ def timeit(func: Callable):
 
 
 def thread(func: Callable):
-    """A decorator for a function that runs in a background thread"""
+    """
+    A decorator which designates this function to run as a background thread for this `:obj:`Unit`.
+    """
     setattr(func, THREAD_ATTR, True)
     return func
 
 
 def task(func: Callable):
-    """A decorator for a function that runs as a task in the task/messaging thread"""
+    """
+    A decorator which designates this function to run as a task in the task/messaging thread.
+    """
     setattr(func, TASK_ATTR, True)
     return func
 
 
 def process(func: Callable):
+    """
+    A decorator which designates this function to run in its own process.
+    """
     setattr(func, PROCESS_ATTR, True)
     return func

{ezmsg-3.3.4 → ezmsg-3.4.0}/src/ezmsg/util/debuglog.py

@@ -4,15 +4,31 @@ from typing import AsyncGenerator, Optional, Any
 
 
 class DebugLogSettings(ez.Settings):
-    name: str = "DEBUG"  # Useful name for logger
-    max_length: Optional[int] = 400  # No limit if `None``
+    """
+    ``Settings`` class associated with :obj:`DebugLog`
+
+    Args:
+        name: Useful name for the logger. The name is included in the logstring so that if multiple DebugLogs
+            are used in one pipeline, their messages can be differentiated.
+        max_length: Sets a maximum number of chars which will be printed from the message.
+            If the message is longer, the log message will be truncated.
+    """
+    name: str = "DEBUG"
+    max_length: Optional[int] = 400
 
 
 class DebugLog(ez.Unit):
+    """
+    Logs messages that pass through.
+    """
+
     SETTINGS: DebugLogSettings
 
     INPUT = ez.InputStream(Any)
+    """Send messages to log here."""
+
     OUTPUT = ez.OutputStream(Any)
+    """Send messages back out to continue through the graph."""
 
     @ez.subscriber(INPUT, zero_copy=True)
     @ez.publisher(OUTPUT)

{ezmsg-3.3.4 → ezmsg-3.4.0}/src/ezmsg/util/generator.py

@@ -94,6 +94,7 @@ class GenAxisArray(ez.Unit):
 
     INPUT_SIGNAL = ez.InputStream(AxisArray)
     OUTPUT_SIGNAL = ez.OutputStream(AxisArray)
+    INPUT_SETTINGS = ez.InputStream(ez.Settings)
 
     def initialize(self) -> None:
         self.construct_generator()
@@ -102,6 +103,11 @@ class GenAxisArray(ez.Unit):
     def construct_generator(self):
         raise NotImplementedError
 
+    @ez.subscriber(INPUT_SETTINGS)
+    async def on_settings(self, msg: ez.Settings) -> None:
+        self.apply_settings(msg)
+        self.construct_generator()
+
     @ez.subscriber(INPUT_SIGNAL)
     @ez.publisher(OUTPUT_SIGNAL)
     async def on_message(self, message: AxisArray) -> AsyncGenerator:

{ezmsg-3.3.4 → ezmsg-3.4.0}/src/ezmsg/util/messagegate.py

@@ -6,14 +6,23 @@ import ezmsg.core as ez
 
 @dataclass
 class GateMessage:
+    """Send this message to ``INPUT_GATE`` to open or close the gate."""
     open: bool
 
 
 class MessageGateSettings(ez.Settings):
+    """
+    Settings for :obj:`MessageGate` unit.
+
+    Args:
+        start_open: sets the gate's initial state to allow messages to flow through or be discarded. ``True`` will
+            allow messages to flow through initially, ``False`` will discard messages initially.
+        default_open: sets the gate's behavior after the `default_after` number of messages have flowed through.
+            ``True`` will allow messages to flow through, ``False`` will discard messages.
+        default_after: sets the number of messages after which the `default_open` state will be applied.
+    """
     start_open: bool = False
     default_open: bool = False
-
-    # Automatically change back to default state after X messages
     default_after: typing.Optional[int] = None
 
 
@@ -23,13 +32,25 @@ class MessageGateState(ez.State):
 
 
 class MessageGate(ez.Unit):
+    """
+    Blocks ``Messages`` from continuing through the system.
+    Can be set as open, closed, open after n messages, or closed after n messages.
+    """
+
     SETTINGS: MessageGateSettings
     STATE: MessageGateState
 
     INPUT_GATE = ez.InputStream(GateMessage)
+    """
+    Stop or start message flow. If ``GateMessage.open == True``, messages will flow through.
+    If ``GateMessage.open == False``, messages will be discarded.
+    """
 
     INPUT = ez.InputStream(typing.Any)
+    """Messages which will flow through or be discarded, depending on gate status."""
+
     OUTPUT = ez.OutputStream(typing.Any)
+    """Publishes messages which flow through."""
 
     def initialize(self) -> None:
         self.STATE.gate_open = self.SETTINGS.start_open

{ezmsg-3.3.4 → ezmsg-3.4.0}/src/ezmsg/util/messagelogger.py

@@ -17,6 +17,13 @@ def log_object(obj: Any) -> str:
 
 
 class MessageLoggerSettings(ez.Settings):
+    """
+    Settings for :obj:`MessageLogger` Unit.
+
+    Args:
+        output: :py:class:`pathlib.Path` for a file where the messages will be logged.
+            If the file path already exists, the existing file will be truncated to 0 length.
+    """
     output: Optional[Path] = None
 
 
@@ -25,15 +32,41 @@ class MessageLoggerState(ez.State):
 
 
 class MessageLogger(ez.Unit):
+    """
+    Logs all messages it receives to a file.
+    File path can be set in ``SETTINGS`` or set dynamically by passing a
+    :py:class:`pathlib.Path` to ``INPUT_START``.
+    """
+
     SETTINGS: MessageLoggerSettings
     STATE: MessageLoggerState
 
     INPUT_START = ez.InputStream(Path)
+    """
+    Pass a :py:class:`pathlib.Path` 
+    to begin logging messages to that path. If the file path already exists, the existing 
+    file will be truncated to 0 length. If the file is already open, nothing will happen.
+    """
+
     INPUT_STOP = ez.InputStream(Path)
+    """
+    Pass a :py:class:`pathlib.Path` 
+    to stop logging messages to that path.
+    """
+
     INPUT_MESSAGE = ez.InputStream(Any)
+    """Pass a piece of data to log it to every open file which the ``MessageLogger`` is using."""
+
     OUTPUT_MESSAGE = ez.OutputStream(Any)
+    """Messages which are sent to ``INPUT_MESSAGE`` will pass through and be published on ``OUTPUT_MESSAGE``."""
+
     OUTPUT_START = ez.OutputStream(Path)
+    """If a file passed to ``INPUT_START`` is successfully opened, its path will be published to
+    ``OUTPUT_START``, otherwise ``None``."""
+
     OUTPUT_STOP = ez.OutputStream(Path)
+    """If a file passed to ``INPUT_STOP`` is successfully closed, its path will be published to
+    ``OUTPUT_STOP``, otherwise ``None``."""
 
     def open_file(self, filepath: Path) -> Optional[Path]:
         """Returns file path if file successfully opened, otherwise None"""

{ezmsg-3.3.4 → ezmsg-3.4.0}/src/ezmsg/util/messagequeue.py

@@ -5,6 +5,13 @@ from ezmsg.util.rate import Rate
 
 
 class MessageQueueSettings(ez.Settings):
+    """
+    Settings for :obj:`MessageQueue` class.
+
+    Args:
+        maxsize: The maximum number of items which the queue will hold.
+        leaky: Whether the queue will drop new messages when it reaches its maxsize, or whether it will wait for space to open for them.
+    """
     maxsize: int = 0
     leaky: bool = False
     log_above_n: Optional[int] = None
@@ -17,11 +24,18 @@ class MessageQueueState(ez.State):
 
 
 class MessageQueue(ez.Unit):
+    """
+    Place between two other ``Units`` to induce backpressure.
+    """
+
     SETTINGS: MessageQueueSettings
     STATE: MessageQueueState
 
     INPUT = ez.InputStream(Any)
+    """Send messages to queue here."""
+
     OUTPUT = ez.OutputStream(Any)
+    """Subscribe to pull messages out of the queue."""
 
     def initialize(self):
         self.STATE.leaky = self.SETTINGS.leaky

{ezmsg-3.3.4 → ezmsg-3.4.0}/src/ezmsg/util/messagereplay.py

@@ -13,6 +13,15 @@ from .messagecodec import MessageDecoder, LogStart
 
 @dataclass
 class ReplayStatusMessage:
+    """
+    Message which gives the status of a file replay.
+
+    Args:
+        filename: The name of the file currently being replayed.
+        idx: The line number of the message that was just published.
+        total: Number of messages in the file.
+        done: Whether the file has finished replaying.
+    """
     filename: Path
     idx: int
     total: int
@@ -21,13 +30,26 @@ class ReplayStatusMessage:
 
 @dataclass
 class FileReplayMessage:
+    """
+    Add a file to the queue.
+
+    Args:
+        filename: The path of the file to replay.
+        rate: in Hertz at which the messages will be published.
+            0 = realtime (if timestamps in file)
+            If not specified, messages will publish as fast as possible.
+    """
     filename: typing.Optional[Path] = None
-
-    # 0 = realtime (if timestamps in file), None = as fast as possible
     rate: typing.Optional[float] = None  # Hz
 
 
 class MessageReplaySettings(ez.Settings, FileReplayMessage):
+    """
+    Settings for :obj:`MesssageReplay` Unit.
+
+    Args:
+        progress: will use tqdm to indicate progress through the file. tqdm must be installed.
+    """
     progress: bool = False
 
 
@@ -38,17 +60,37 @@ class MessageReplayState(ez.State):
 
 
 class MessageReplay(ez.Unit):
+    """
+    Stream messages from files created by :obj:`MessageLogger`.
+    Stores a queue of files to stream and streams from them in order.
+    """
+
     SETTINGS: MessageReplaySettings
     STATE: MessageReplayState
 
     INPUT_FILE = ez.InputStream(FileReplayMessage)
-    INPUT_PAUSED = ez.InputStream(bool)  # Pause state; True = paused, False = running
-    INPUT_STOP = ez.InputStream(bool)  # True = clear queue
+    """Add a new file to the queue."""
+
+    INPUT_PAUSED = ez.InputStream(bool)
+    """Send ``True`` to pause the stream, ``False`` to restart the stream."""
+
+    INPUT_STOP = ez.InputStream(bool)
+    """
+    Stop the stream. Send ``True`` to also clear the queue.
+    Send ``False`` to reset to the beginning of the current file.
+    """
 
     OUTPUT_MESSAGE = ez.OutputStream(typing.Any)
+    """The output on which the messages from the files will be streamed."""
+
     OUTPUT_TOTAL = ez.OutputStream(int)
+    """
+    Publishes an integer total of messages which have been published on OUTPUT_MESSAGE from a single file.
+    Resets when a file completes.
+    """
 
     OUTPUT_REPLAY_STATUS = ez.OutputStream(ReplayStatusMessage)
+    """Publishes status messages."""
 
     async def initialize(self) -> None:
         self.STATE.replay_files = asyncio.Queue()
@@ -167,10 +209,17 @@ class MessageCollectorState(ez.State):
 
 
 class MessageCollector(ez.Unit):
+    """
+    Collects ``Messages`` into a local list.
+    """
+
     STATE: MessageCollectorState
 
     INPUT_MESSAGE = ez.InputStream(typing.Any)
+    """Send messages here to be collected."""
+
     OUTPUT_MESSAGE = ez.OutputStream(typing.Any)
+    """Messages will pass straight through after being recorded and be published here."""
 
     @ez.subscriber(INPUT_MESSAGE)
     @ez.publisher(OUTPUT_MESSAGE)
@@ -180,4 +229,9 @@ class MessageCollector(ez.Unit):
 
     @property
     def messages(self) -> typing.List[typing.Any]:
+        """
+        Access the list of messages.
+
+        :return: A list of messages which have been collected.
+        """
         return self.STATE.messages

{ezmsg-3.3.4 → ezmsg-3.4.0}/src/ezmsg/util/messages/axisarray.py

@@ -12,10 +12,14 @@ import numpy.lib.stride_tricks as nps
 from ezmsg.core.util import either_dict_or_kwargs
 
 # TODO: Typehinting is all wrong in this and 
-# concatenate/transpose should probably not be staticmethods
+#  concatenate/transpose should probably not be staticmethods
+
 
 @dataclass
 class AxisArray:
+    """
+    A lightweight message class comprising a numpy ndarray and its metadata.
+    """
     data: npt.NDArray
     dims: typing.List[str]
     axes: typing.Dict[str, "AxisArray.Axis"] = field(default_factory=dict)
@@ -206,22 +210,24 @@ class AxisArray:
 def slice_along_axis(in_arr: npt.NDArray, sl: typing.Union[slice, int], axis: int) -> npt.NDArray:
     """
     Slice the input array along a specified axis using the given slice object or integer index.
+     Integer arguments to `sl` will cause the sliced dimension to be dropped.
+     Use `slice(my_int, my_int+1, None)` to keep the sliced dimension.
 
     Parameters:
-        in_arr (npt.NDArray): The input array to be sliced.
-        sl (Union[slice, int]): The slice object or integer index to use for slicing.
-        axis (int): The axis along which to slice the array.
+        in_arr: The input array to be sliced.
+        sl: The slice object or integer index to use for slicing.
+        axis: The axis along which to slice the array.
 
     Returns:
-        npt.NDArray: The sliced array (view).
+        The sliced array (view).
 
     Raises:
         ValueError: If the axis value is invalid for the input array.
     """
     if axis < -in_arr.ndim or axis >= in_arr.ndim:
         raise ValueError(f"Invalid axis value {axis} for input array with {in_arr.ndim} dimensions.")
-    if axis < 0:
-        axis = in_arr.ndim  + axis
+    if -in_arr.ndim <= axis < 0:
+        axis = in_arr.ndim + axis
     all_slice = (slice(None),) * axis + (sl,) + (slice(None),) * (in_arr.ndim - axis - 1)
     return in_arr[all_slice]
 
@@ -229,23 +235,36 @@ def slice_along_axis(in_arr: npt.NDArray, sl: typing.Union[slice, int], axis: in
 def sliding_win_oneaxis(in_arr: npt.NDArray, nwin: int, axis: int) -> npt.NDArray:
     """
     Generates a view of an array using a sliding window of specified length along a specified axis of the input array.
-    This is a slightly optimized version of nps.sliding_window_view with a few important differences.
-    Because we only accept a single nwin and a single axis, we can skip some checks.
-    The new `win` axis precedes immediately the original target axis, unlike sliding_window_view where the
-     target axis is moved to the end of the output.
+    This is a slightly optimized version of nps.sliding_window_view with a few important differences:
 
-    Parameters:
-        in_arr (npt.NDArray): The input array.
-        nwin (int): The size of the sliding window.
-        axis (int): The axis along which the sliding window will be applied.
+    - This only accepts a single nwin and a single axis, thus we can skip some checks.
+    - The new `win` axis precedes immediately the original target axis, unlike sliding_window_view where the
+        target axis is moved to the end of the output.
+
+    Combine this with slice_along_axis(..., sl=slice(None, None, step), axis=axis) to step the window
+        by more than 1 sample at a time.
+
+    Args:
+        in_arr: The input array.
+        nwin: The size of the sliding window.
+        axis: The axis along which the sliding window will be applied.
 
     Returns:
-        npt.NDArray: A view to the input array with the sliding window applied.
+        A view to the input array with the sliding window applied.
+
+    Note: There is a known edge case when nwin == shape[axis] + 1. While this should raise
+        an error because the window is larger than the input, the implementation ends up
+        returning a 0-length window. We could check for this but this function is intended
+        to have minimal latency so we have decided to skip the checks and deal with the
+        support issues as they arise.
     """
+    if -in_arr.ndim <= axis < 0:
+        axis = in_arr.ndim + axis
     out_strides = in_arr.strides[:axis] + (in_arr.strides[axis],) * 2 + in_arr.strides[axis+1:]
     out_shape = in_arr.shape[:axis] + (in_arr.shape[axis]-(nwin-1),) + (nwin,) + in_arr.shape[axis+1:]
     return nps.as_strided(in_arr, strides=out_strides, shape=out_shape, writeable=False)
 
+
 def _as2d(
     in_arr: npt.NDArray, axis: int = 0
 ) -> typing.Tuple[npt.NDArray, typing.Tuple[int]]:

{ezmsg-3.3.4 → ezmsg-3.4.0}/src/ezmsg/util/terminate.py

@@ -9,8 +9,15 @@ from typing import Optional, Any
 
 
 class TerminateOnTimeoutSettings(ez.Settings):
-    time: float = 2.0  # Terminate if no message has been received in this time (sec)
-    poll_rate: float = 4.0  # Probably no good reason to mess with this (Hz)
+    """
+    Settings for :obj:`TerminateOnTimeout` Unit.
+
+    Args:
+        time: Terminate if no message has been received in this time (sec)
+        poll_rate: Hz.
+    """
+    time: float = 2.0
+    poll_rate: float = 4.0
 
 
 class TerminateOnTimeoutState(ez.State):
@@ -18,10 +25,15 @@ class TerminateOnTimeoutState(ez.State):
 
 
 class TerminateOnTimeout(ez.Unit):
+    """
+    End a pipeline execution when a certain amount of time has passed without receiving a message.
+    """
+
     SETTINGS: TerminateOnTimeoutSettings
     STATE: TerminateOnTimeoutState
 
     INPUT = ez.InputStream(Any)
+    """Send messages here."""
 
     @ez.subscriber(INPUT)
     async def keepalive(self, _: Any) -> None:
@@ -40,6 +52,12 @@ class TerminateOnTimeout(ez.Unit):
 
 
 class TerminateOnTotalSettings(ez.Settings):
+    """
+    Settings for :obj:`TerminateOnTotal` Unit.
+
+    Args:
+        total: The total number of messages to terminate after.
+    """
     total: Optional[int] = None
 
 
@@ -49,11 +67,21 @@ class TerminateOnTotalState(ez.State):
 
 
 class TerminateOnTotal(ez.Unit):
+    """
+    End a pipeline execution once a certain number of messages have been received.
+    """
+
     SETTINGS: TerminateOnTotalSettings
     STATE: TerminateOnTotalState
 
     INPUT_MESSAGE = ez.InputStream(Any)
+    """Send messages here."""
+
     INPUT_TOTAL = ez.InputStream(int)
+    """
+    Change the total number of messages to terminate after.
+    If this number has already been reached, termination will occur immediately.
+    """
 
     def initialize(self) -> None:
         self.STATE.total = self.SETTINGS.total

ezmsg-3.3.4/src/ezmsg/core/settings.py

@@ -1,31 +0,0 @@
-import sys
-import typing
-
-from abc import ABC, ABCMeta
-from dataclasses import dataclass
-
-if sys.version_info < (3, 12):
-    from typing_extensions import dataclass_transform
-else:
-    from typing import dataclass_transform
-
-# All settings classes are dataclasses
-# https://rednafi.github.io/digressions/python/2020/06/26/python-metaclasses.html
-#  see -- #avoiding-dataclass-decorator-with-metaclasses
-
-
-@dataclass_transform()
-class SettingsMeta(ABCMeta):
-    def __new__(
-        cls,
-        name: str,
-        bases: typing.Tuple[type, ...],
-        classdict: typing.Dict[str, typing.Any],
-        **kwargs: typing.Any
-    ) -> typing.Type["Settings"]:
-        new_cls = super().__new__(cls, name, bases, classdict)
-        return dataclass(frozen=True)(new_cls)  # type: ignore
-
-
-class Settings(ABC, metaclass=SettingsMeta):
-    ...

ezmsg-3.3.4/src/ezmsg/core/state.py

@@ -1,29 +0,0 @@
-from abc import ABC, ABCMeta
-from dataclasses import dataclass
-
-from typing import (
-    Dict,
-    Tuple,
-    Any,
-    Type,
-)
-
-
-class StateMeta(ABCMeta):
-    def __new__(
-        cls,
-        name: str,
-        bases: Tuple[type, ...],
-        classdict: Dict[str, Any],
-        **kwargs: Any
-    ) -> Type["State"]:
-        new_cls = super().__new__(cls, name, bases, classdict)
-        return dataclass(unsafe_hash=True, frozen=False, init=False)(new_cls)  # type: ignore
-
-
-class State(ABC, metaclass=StateMeta):
-    """
-    States are mutable dataclasses that are instantiated by the Unit in its home process.
-    """
-
-    ...