arroyopy 0.1.0__py3-none-any.whl

arroyopy/listener.py

@@ -0,0 +1,14 @@
+import asyncio
+from abc import ABC, abstractmethod
+
+
+class Listener(ABC):
+    message_queue: asyncio.Queue
+
+    @abstractmethod
+    async def start(self, message_queue) -> None:
+        self.message_queue = message_queue
+
+    @abstractmethod
+    async def stop(self) -> None:
+        pass

arroyopy/operator.py

@@ -0,0 +1,52 @@
+import asyncio
+import logging
+from abc import ABC, abstractmethod
+from typing import List
+
+from .listener import Listener
+from .publisher import Publisher
+from .schemas import Message
+
+logger = logging.getLogger(__name__)
+
+
+class Operator(ABC):
+    listeners: List[Listener] = []
+    publishers: List[Publisher] = []
+    stop_requested: bool = False
+
+    def __init__(self):
+        self.listener_queue = asyncio.Queue()
+
+    @abstractmethod
+    async def process(self, message: Message) -> None:
+        pass
+
+    async def add_listener(self, listener: Listener) -> None:  # noqa
+        self.listeners.append(listener)
+        await listener.start(self.listener_queue)
+
+    def remove_listener(self, listener: Listener) -> None:  # noqa
+        self.listeners.remove(listener)
+
+    def add_publisher(self, publisher: Publisher) -> None:
+        self.publishers.append(publisher)
+
+    def remove_publisher(self, publisher: Publisher) -> None:
+        self.publishers.remove(publisher)
+
+    async def publish(self, message: Message) -> None:
+        for publisher in self.publishers:
+            await publisher.publish(message)
+
+    async def start(self):
+        # Process messages from the queue
+        while True:
+            if self.stop_requested:
+                logger.info("Stopping operator...")
+                for listener in self.listeners:
+                    await listener.stop()
+                break
+            message = await self.queue.get()
+            processed_message = await self.process(message)
+            await self.publish(processed_message)

arroyopy/publisher.py

@@ -0,0 +1,12 @@
+from abc import ABC, abstractmethod
+from typing import Generic, TypeVar
+
+from .schemas import Message
+
+T = TypeVar("T", bound=Message)
+
+
+class Publisher(ABC, Generic[T]):
+    @abstractmethod
+    async def publish(self, message: Message) -> None:
+        pass

arroyopy/redis.py

@@ -0,0 +1,62 @@
+import logging
+
+from redis.asyncio.client import Redis
+
+from .listener import Listener
+from .operator import Operator
+
+logger = logging.getLogger("arroyo.zmq")
+
+
+class RedisListener(Listener):
+    def __init__(
+        self, redis_client: Redis, redis_channel_name: str, operator: Operator
+    ):
+        self.stop_requested = False
+        self.redis_client: Redis = redis_client
+        self.redis_channel_name = redis_channel_name
+        self.operator = operator
+
+    @classmethod
+    async def from_client(
+        cls, redis_client: Redis, redis_channel_name: str, operator: Operator
+    ):
+        return RedisListener(redis_client, redis_channel_name, operator)
+
+    async def start(self):
+        logger.info("Listener started")
+        pubsub = self.redis_client.pubsub()
+        await pubsub.subscribe(self.redis_channel_name)
+        # Listen for messages in the subscribed channel
+        while True:
+            if self.stop_requested:
+                return
+            # get_message blocks until timeout, returning None if no message in that time
+            raw_msg = await pubsub.get_message(
+                ignore_subscribe_messages=True, timeout=1.0
+            )
+            if raw_msg is None:
+                continue
+            msg = raw_msg["data"]
+            if logger.getEffectiveLevel() == logging.DEBUG:
+                logger.debug(f"{msg=}")
+            await self.operator.process(msg)
+
+    async def stop(self):
+        self.stop_requested = True
+        await self.redis_client.aclose()
+
+
+class RedisPublisher:
+    def __init__(self, redis_client: Redis, redis_channel_name: str):
+        self.redis_client: Redis = redis_client
+        self.redis_channel_name = redis_channel_name
+
+    @classmethod
+    async def from_client(cls, redis_client: Redis, redis_channel_name: str):
+        return RedisPublisher(redis_client, redis_channel_name)
+
+    async def publish(self, message):
+        await self.redis_client.publish(self.redis_channel_name, message)
+        if logger.getEffectiveLevel() == logging.DEBUG:
+            logger.debug(f"Published {message=}")

arroyopy/schemas.py

@@ -0,0 +1,67 @@
+import numpy
+import numpy.typing
+import pandas
+from pydantic import BaseModel, field_validator
+
+
+class Message:
+    """ "
+    Base class for messages. Is not a pydantic model
+    in case implementations choose not to use pydantic
+    as a validation and (de)serialization system but still
+    want to indicate that they pass arroyo Messages.
+    """
+
+    pass
+
+
+class PydanticMessage(Message, BaseModel):
+    pass
+
+
+class Start(PydanticMessage):
+    pass
+
+
+class Stop(PydanticMessage):
+    pass
+
+
+class Event(PydanticMessage):
+    pass
+
+
+class DataFrameModel(BaseModel):
+    """
+    A Pydantic model for validating pd.DataFrame objects.
+    Does not parse array, merely validates that is a pd.DataFrame
+    """
+
+    df: pandas.DataFrame
+
+    @field_validator("df", mode="before")
+    def validate_is_numpy_array(cls, v):
+        if not isinstance(v, pandas.DataFrame):
+            raise TypeError(f"Expected pd.DataFrame, got {type(v)} instead.")
+        return v  # Do not modify or parse the array
+
+    class Config:
+        arbitrary_types_allowed = True  # Allow numpy.ndarray type
+
+
+class NumpyArrayModel(BaseModel):
+    """
+    A Pydantic model for validating numpy.ndarray objects.
+    Does not parse array, merely validates that is a np.ndarray
+    """
+
+    array: numpy.ndarray
+
+    @field_validator("array", mode="before")
+    def validate_is_numpy_array(cls, v):
+        if not isinstance(v, numpy.ndarray):
+            raise TypeError(f"Expected numpy.ndarray, got {type(v)} instead.")
+        return v  # Do not modify or parse the array
+
+    class Config:
+        arbitrary_types_allowed = True  # Allow numpy.ndarray type

arroyopy/timing.py

@@ -0,0 +1,54 @@
+import functools
+import logging
+import time
+
+import pandas as pd
+
+logger = logging.getLogger(__name__)
+
+effective_level = logger.getEffectiveLevel()
+
+
+class EventTimingDecorator:
+    """
+    Decorator to time functions and output the results in a pandas DataFrame.
+
+    This assumes within a single Event, there will be multiple calls to different functions.
+    When a new event starts, call `end_event` to store the timings and reset the timings for the next event.
+    When all events are done the timings can be accessed as a DataFrame using the `timing_dataframe` property.
+    After all events are done, call `reset` to clear all timings for the next event.
+    """
+
+    def __init__(self):
+        self.current_event_times = {}
+        self.events = []
+
+    def __call__(self, func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            start_time = time.time()
+            result = func(*args, **kwargs)
+            end_time = time.time()
+            duration = end_time - start_time
+            self.current_event_times[func.__name__] = duration
+            if effective_level == logging.DEBUG:
+                (f"{func.__name__} took {duration:.4f} seconds")
+            return result
+
+        return wrapper
+
+    def end_event(self):
+        if self.current_event_times:
+            self.events.append(self.current_event_times)
+        self.current_event_times = {}
+
+    @property
+    def timing_dataframe(self):
+        return pd.DataFrame(self.current_event_times)
+
+    def reset(self):
+        self.current_event_times = {}
+        self.events = []
+
+
+timer = EventTimingDecorator()

arroyopy/zmq.py

@@ -0,0 +1,60 @@
+import asyncio
+import logging
+
+import zmq
+import zmq.asyncio
+
+from .listener import Listener
+from .operator import Operator
+
+logger = logging.getLogger("arroyo.zmq")
+
+
+class ZMQListener(Listener):
+    stop_signal: bool = False
+
+    def __init__(self, operator: Operator, zmq_socket: zmq.Socket):
+        self.stop_requested = False
+        self.operator = operator
+        self.zmq_socket = zmq_socket
+
+    @classmethod
+    def from_socket(cls, zmq_socket: zmq.Socket):
+        """Construct a ZMQListenr using a provided socket. Gives
+        callers the ability to customize the ZMQ soket
+
+        Parameters
+        ----------
+        zmq_socket : zmq.Socket
+           provided socket
+
+        Returns
+        -------
+        ZMQListner
+            new ZMQListner
+        """
+        return ZMQListener(zmq_socket)
+
+    async def start(self):
+        logger.info("Listener started")
+        # timeout after 100 milliseconds so we can be stopped if requested
+        self.zmq_socket.setsockopt(zmq.RCVTIMEO, 100)
+        while True:
+            if self.stop_requested:
+                return
+            try:
+                msg = await self.zmq_socket.recv()
+                if logger.getEffectiveLevel() == logging.DEBUG:
+                    logger.debug(f"{msg=}")
+                await self.operator.process(msg)
+            except zmq.Again:
+                # no message occured within the timeout period
+                pass
+            except asyncio.exceptions.CancelledError:
+                # in case this is being done in a asyncio.create_task call
+                pass
+
+    async def stop(self):
+        self.stop_requested = True
+        self.zmq_socket.close()
+        self.zmq_socket.context.term()

arroyopy-0.1.0.dist-info/METADATA

@@ -0,0 +1,255 @@
+Metadata-Version: 2.4
+Name: arroyopy
+Version: 0.1.0
+Summary: A library to simplify processing streams of data
+Project-URL: Homepage, https://github.com/als-computing/arroyopy
+Project-URL: Issues, https://github.com/als-computing/arroyopy/issues
+Author-email: Dylan McReynolds <dmcreynolds@lbl.gov>
+License: BSD-3
+License-File: LICENSE
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Requires-Python: <3.13,>=3.11
+Requires-Dist: numpy
+Requires-Dist: pandas
+Requires-Dist: pydantic>=2.0
+Requires-Dist: python-dotenv
+Requires-Dist: typer
+Provides-Extra: dev
+Requires-Dist: fakeredis; extra == 'dev'
+Requires-Dist: flake8; extra == 'dev'
+Requires-Dist: pre-commit; extra == 'dev'
+Requires-Dist: pytest-asyncio; extra == 'dev'
+Requires-Dist: pytest-mock; extra == 'dev'
+Requires-Dist: pyzmq; extra == 'dev'
+Requires-Dist: redis; extra == 'dev'
+Requires-Dist: tiled[minimal-server]; extra == 'dev'
+Provides-Extra: redis
+Requires-Dist: redis; extra == 'redis'
+Provides-Extra: tiled
+Requires-Dist: tiled[client]; extra == 'tiled'
+Provides-Extra: zmq
+Requires-Dist: pyzmq; extra == 'zmq'
+Description-Content-Type: text/markdown
+
+# Arroyo Stream Processing Toolset
+
+Processing event or streaming data presents several technological challenges. A variety of technologies are often used by scientific user facilities. ZMQ is used to stream data and messages in a peer-to-peer fashion. Message brokers like Kafka, Redis and RabbitMQ are often employed to route and pass messages from instruments to processing workflows. Arroyo provides an API and structure to flexibly integrate with these tools and incorporate arbitrarily complex processing workflows, letting the hooks to the workflow code be independent of the connection code and hence reusable at a variety of instruments.
+
+The basic structure of building an arroyo implementation is to implement groups of several  classes:
+-
+- `Operator` - receives `Messages` from a listener and can optionally send `Messages` to one or more `Publisher` instances
+- `Listener` - receives `Messages` from the external world, parse them into arroyo `Message` and sends them to an `Operator`
+- `Publisher` - receives `Messages` from a `Listener` and publishes them to the outside world
+
+
+
+
+Arroyo is un-opinionated about deployment decsions. It is intended support listener-operator-publisher groups in:
+- Single process
+- Chain of processes where listening, processing and publishing can linked together through a protocol like ZMQ. One process's publisher can communicate with another process's listener, etc.
+
+This library is intended to provide  classes, and will also include more specific common subclasses, like those that communicate over ZMQ or Redis.
+
+
+
+```mermaid
+
+---
+title: Some sweet classes
+
+note: I guess we use "None" instead of "void"
+---
+
+classDiagram
+    namespace listener{
+
+        class Listener{
+            operator: Operator
+
+            *start(): None
+            *stop(): None
+        }
+
+
+    }
+
+    namespace operator{
+        class Operator{
+            publisher: List[Publisher]
+            *process(Message): None
+            add_publisher(Publisher): None
+            remove_publisher(Publisher): None
+
+        }
+    }
+
+    namespace publisher{
+        class Publisher{
+            *publish(Message): None
+        }
+
+    }
+
+    namespace message{
+
+        class Message{
+
+        }
+
+        class Start{
+            data: Dict
+        }
+
+        class Stop{
+            data: Dict
+        }
+
+        class Event{
+            metadata: Dict
+            payload: bytes
+        }
+    }
+
+    namespace zmq{
+        class ZMQListener{
+            operator: Operator
+            socket: zmq.Socket
+        }
+
+        class ZMQPublisher{
+            host: str
+            port: int
+        }
+
+    }
+
+    namespace redis{
+
+        class RedisListener{
+            operator: Redis.client
+            pubsub: Redis.pubsub
+        }
+
+        class RedisPublisher{
+            pubsub: Redis.pubsub
+        }
+
+    }
+
+
+
+    Listener <|-- ZMQListener
+    ZMQListener <|-- ZMQPubSubListener
+    Listener o-- Operator
+
+    Publisher <|-- ZMQPublisher
+    ZMQPublisher <|-- ZMQPubSubPublisher
+
+    Publisher <|-- RedisPublisher
+    Listener <|-- RedisListener
+    Operator o-- Publisher
+    Message <|-- Start
+    Message <|-- Stop
+    Message <|-- Event
+
+
+```
+##
+In-process, listening for ZMQ
+
+Note that this leaves Concrete classes undefined as placeholders
+
+TODO: parent class labels
+
+```mermaid
+
+sequenceDiagram
+    autonumber
+    ExternalPublisher ->> ZMQPubSubListener: publish(bytes)
+    loop receiving thread
+        activate ZMQPubSubListener
+            ZMQPubSubListener ->> ConcreteMessageParser: parse(bytes)
+            ZMQPubSubListener ->> MessageQueue: put(bytes)
+        deactivate ZMQPubSubListener
+
+
+        ZMQPubSubListener ->> MessageQueue: message(Message)
+    end
+    activate ConcreteOperator
+        loop polling thread
+            ConcreteOperator ->> MessageQueue: get(bytes)
+        end
+        loop processing thread
+            ConcreteOperator ->> ConcreteOperator: calculate()
+
+            ConcreteOperator ->> ConcretePublisher: publish()
+        end
+    deactivate ConcreteOperator
+```
+
+# Devloper installation
+
+## Conda environment
+We use pixi to be forward thinking tio help with CI. We like it because it helps you easily test that dependencies for a variety of architects can resolve.
+
+However, at the time of writing we can't figure out how to get it to be a good developer experience. So, we create a conda environment like (note that at this time, we are using python 3.11 because of numpy and wheel availability):
+
+```
+conda create -n arroyo python=3.11
+conda activate arroyo
+pip install -e '.[dev]'
+```
+
+## pre-commit
+We use `pre-commit` in CI so you want to use it before commiting.
+To test that your branches changes are all good, type:
+
+```
+pre-commit run --all-files
+```
+
+Since our configuration of `pre-commit` uses `black`, it's possible that it will change files. If you like the changes, you can add them to your `git` commit with
+
+```
+git add .
+```
+
+Then you can run `pre-commit run --all-files` again.
+
+## pixi
+We use `pixi` for CI in github action. It's great for that but can't get our favorite developr tools to use the python environments that `pixi` creaetes in the `.pixi` folder. If you want to play with `pixi`, here are some tips:
+
+To setup a development environment:
+
+* Git clone this repo and CD into the directory
+* Install [pixi](https://pixi.sh/v0.33.0/#installation)
+* Install dependencies with
+'''
+pixi install
+'''
+* run pre-commit on the files
+'''
+pixi r pre-commit
+'''
+
+
+* Run pytest with
+'''
+pixi r test
+'''
+
+# Copyright
+Arroyo Stream Processing Toolset (arroyopy) Copyright (c) 2025, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Dept. of Energy).
+All rights reserved.
+
+If you have questions about your rights to use or distribute this software,
+please contact Berkeley Lab's Intellectual Property Office at
+IPO@lbl.gov.
+
+NOTICE.  This Software was developed under funding from the U.S. Department
+of Energy and the U.S. Government consequently retains certain rights.  As
+such, the U.S. Government has been granted for itself and others acting on
+its behalf a paid-up, nonexclusive, irrevocable, worldwide license in the
+Software to reproduce, distribute copies to the public, prepare derivative
+works, and perform publicly and display publicly, and to permit others to do so.

arroyopy-0.1.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+arroyopy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+arroyopy/listener.py,sha256=Cy_3UWkguFI__BUzk7d4DNbA52mK_ch8t1KaRFyr7dU,289
+arroyopy/operator.py,sha256=-NvqKMbXt5O2ikJ9FhQIlnQilkQT0bbZ-1nRNmvsTx8,1574
+arroyopy/publisher.py,sha256=a3bR5BI-B9pduxrTeR70jP1BIUyZNpu1CwYFvHvTCQA,259
+arroyopy/redis.py,sha256=7OJSuKhDlHTDNtGvRmCApU0FKZ648KVs8bczySZz_sM,2098
+arroyopy/schemas.py,sha256=ONARC_pormPXg7SNqlx1FPeU0A2knNMlspU3YEclxyk,1637
+arroyopy/timing.py,sha256=HXhwQmpLq87JT5ZFZTh2vAQbO196whD6_Yw1yGjZdzI,1624
+arroyopy/zmq.py,sha256=w8WPFYbwbLlmTPiIAc-pdNRpmUdbpkiyktBz1_7bXRA,1716
+arroyopy-0.1.0.dist-info/METADATA,sha256=tgKvc43MUqIiElxXtbUoJd3UAu-myQhQXB38xvo_ZXY,7548
+arroyopy-0.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+arroyopy-0.1.0.dist-info/licenses/LICENSE,sha256=jCk8QHKtWh2hrT0X2XXNZZDfreHb9v0Th5kZ3rDCN2g,2434
+arroyopy-0.1.0.dist-info/RECORD,,

arroyopy-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.27.0
+Root-Is-Purelib: true
+Tag: py3-none-any

arroyopy-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,33 @@
+Arroyo Stream Processing Toolset (arroyopy) Copyright (c) 2025, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Dept. of Energy).
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+(1) Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+(2) Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+(3) Neither the name of the University of California, Lawrence Berkeley
+National Laboratory, U.S. Dept. of Energy nor the names of its contributors
+may be used to endorse or promote products derived from this software
+without specific prior written permission.
+
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+You are under no obligation whatsoever to provide any bug fixes, patches,
+or upgrades to the features, functionality or performance of the source
+code ("Enhancements") to anyone; however, if you choose to make your
+Enhancements available either publicly, or directly to Lawrence Berkeley
+National Laboratory, without imposing a separate written license agreement
+for such Enhancements, then you hereby grant the following license: a
+non-exclusive, royalty-free perpetual license to install, use, modify,
+prepare derivative works, incorporate into other computer software,
+distribute, and sublicense such enhancements or derivative works thereof,
+in binary and source code form.