arroyopy 0.1.0__tar.gz

arroyopy-0.1.0/.flake8

@@ -0,0 +1,8 @@
+[flake8]
+exclude =
+    .git,
+    __pycache__,
+    build,
+    dist
+max-line-length = 115
+extend-ignore = E203, W503

arroyopy-0.1.0/.gitattributes

@@ -0,0 +1,2 @@
+# GitHub syntax highlighting
+pixi.lock linguist-language=YAML linguist-generated=true

arroyopy-0.1.0/.github/dependabot.yaml

@@ -0,0 +1,12 @@
+# recommended dependabot configuration for github actions
+# recommandation from the pixi site https://pixi.sh/latest/advanced/github_actions/#usage
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: monthly
+    groups:
+      dependencies:
+        patterns:
+          - "*"

arroyopy-0.1.0/.github/workflows/ci.yaml

@@ -0,0 +1,54 @@
+name: Run Tests with Pixi
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  pre-commit:
+      name: Static Code Checks
+      runs-on: ubuntu-latest
+      steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.x"
+          cache: 'pip'
+      - uses: pre-commit/action@v3.0.0
+  test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest] # windows and pyzm are  painful, windows-latest]
+        python-version: ["py310", "py311", "py312"]
+    steps:
+      # Step 1: Checkout the repository
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      # Step 2: Install Python
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.12"  # Use a version compatible with your project
+
+      # Step 3: Install dependencies
+      - name: Run tests with pixi
+        uses: prefix-dev/setup-pixi@v0.7.0
+        with:
+          pixi-version: v0.33.0
+          cache: true
+      - run: pixi run test
+
+      # Step 6: Cache dependencies for future runs for faster execution
+      - name: Cache dependencies
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('pyproject.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-

arroyopy-0.1.0/.github/workflows/publish.yaml

@@ -0,0 +1,50 @@
+name: Publish to PyPI
+
+on:
+  push:
+    branches:
+      - main
+  release:
+    types:
+      - created
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+
+      - name: Install build dependencies
+        run: pip install build
+
+      - name: Build package
+        run: python -m build
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: python-package
+          path: dist/
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write  # Required for trusted publishing
+    steps:
+      - name: Download artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: python-package
+          path: dist/
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN }}

arroyopy-0.1.0/.gitignore

@@ -0,0 +1,186 @@
+.DS_Store
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+EC-lab
+lab book
+LTspice files
+TRS
+proc
+ESpectrum Stream 00012.bin
+Frame Stream 00012.bin
+Metadata 00012.json
+test.zarr
+misc
+test_fast.zarr
+zarr_utils.py
+
+
+data/
+
+.vscode/
+.code-workspace
+
+.pixi/
+
+# pixi environments
+.pixi
+*.egg-info

arroyopy-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,35 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+    -   id: trailing-whitespace
+    -   id: end-of-file-fixer
+    -   id: check-ast
+    -   id: check-case-conflict
+    -   id: check-merge-conflict
+    -   id: check-symlinks
+    -   id: check-yaml
+    -   id: debug-statements
+
+-   repo: https://github.com/pycqa/flake8
+    rev: 6.1.0
+    hooks:
+    -   id: flake8
+
+-   repo: https://github.com/timothycrosley/isort
+    rev: 5.12.0
+    hooks:
+    -   id: isort
+
+-   repo: https://github.com/psf/black
+    rev: 23.10.1
+    hooks:
+    -   id: black
+
+# -   repo: https://github.com/pre-commit/mirrors-mypy
+#     rev: v1.9.0
+#     hooks:
+#     -    id: mypy
+#          args: [--strict]

arroyopy-0.1.0/LICENSE

@@ -0,0 +1,33 @@
+Arroyo Stream Processing Toolset (arroyopy) Copyright (c) 2025, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Dept. of Energy).
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+(1) Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+(2) Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+(3) Neither the name of the University of California, Lawrence Berkeley
+National Laboratory, U.S. Dept. of Energy nor the names of its contributors
+may be used to endorse or promote products derived from this software
+without specific prior written permission.
+
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+You are under no obligation whatsoever to provide any bug fixes, patches,
+or upgrades to the features, functionality or performance of the source
+code ("Enhancements") to anyone; however, if you choose to make your
+Enhancements available either publicly, or directly to Lawrence Berkeley
+National Laboratory, without imposing a separate written license agreement
+for such Enhancements, then you hereby grant the following license: a
+non-exclusive, royalty-free perpetual license to install, use, modify,
+prepare derivative works, incorporate into other computer software,
+distribute, and sublicense such enhancements or derivative works thereof,
+in binary and source code form.

arroyopy-0.1.0/PKG-INFO

@@ -0,0 +1,255 @@
+Metadata-Version: 2.4
+Name: arroyopy
+Version: 0.1.0
+Summary: A library to simplify processing streams of data
+Project-URL: Homepage, https://github.com/als-computing/arroyopy
+Project-URL: Issues, https://github.com/als-computing/arroyopy/issues
+Author-email: Dylan McReynolds <dmcreynolds@lbl.gov>
+License: BSD-3
+License-File: LICENSE
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Requires-Python: <3.13,>=3.11
+Requires-Dist: numpy
+Requires-Dist: pandas
+Requires-Dist: pydantic>=2.0
+Requires-Dist: python-dotenv
+Requires-Dist: typer
+Provides-Extra: dev
+Requires-Dist: fakeredis; extra == 'dev'
+Requires-Dist: flake8; extra == 'dev'
+Requires-Dist: pre-commit; extra == 'dev'
+Requires-Dist: pytest-asyncio; extra == 'dev'
+Requires-Dist: pytest-mock; extra == 'dev'
+Requires-Dist: pyzmq; extra == 'dev'
+Requires-Dist: redis; extra == 'dev'
+Requires-Dist: tiled[minimal-server]; extra == 'dev'
+Provides-Extra: redis
+Requires-Dist: redis; extra == 'redis'
+Provides-Extra: tiled
+Requires-Dist: tiled[client]; extra == 'tiled'
+Provides-Extra: zmq
+Requires-Dist: pyzmq; extra == 'zmq'
+Description-Content-Type: text/markdown
+
+# Arroyo Stream Processing Toolset
+
+Processing event or streaming data presents several technological challenges. A variety of technologies are often used by scientific user facilities. ZMQ is used to stream data and messages in a peer-to-peer fashion. Message brokers like Kafka, Redis and RabbitMQ are often employed to route and pass messages from instruments to processing workflows. Arroyo provides an API and structure to flexibly integrate with these tools and incorporate arbitrarily complex processing workflows, letting the hooks to the workflow code be independent of the connection code and hence reusable at a variety of instruments.
+
+The basic structure of building an arroyo implementation is to implement groups of several  classes:
+-
+- `Operator` - receives `Messages` from a listener and can optionally send `Messages` to one or more `Publisher` instances
+- `Listener` - receives `Messages` from the external world, parse them into arroyo `Message` and sends them to an `Operator`
+- `Publisher` - receives `Messages` from a `Listener` and publishes them to the outside world
+
+
+
+
+Arroyo is un-opinionated about deployment decsions. It is intended support listener-operator-publisher groups in:
+- Single process
+- Chain of processes where listening, processing and publishing can linked together through a protocol like ZMQ. One process's publisher can communicate with another process's listener, etc.
+
+This library is intended to provide  classes, and will also include more specific common subclasses, like those that communicate over ZMQ or Redis.
+
+
+
+```mermaid
+
+---
+title: Some sweet classes
+
+note: I guess we use "None" instead of "void"
+---
+
+classDiagram
+    namespace listener{
+
+        class Listener{
+            operator: Operator
+
+            *start(): None
+            *stop(): None
+        }
+
+
+    }
+
+    namespace operator{
+        class Operator{
+            publisher: List[Publisher]
+            *process(Message): None
+            add_publisher(Publisher): None
+            remove_publisher(Publisher): None
+
+        }
+    }
+
+    namespace publisher{
+        class Publisher{
+            *publish(Message): None
+        }
+
+    }
+
+    namespace message{
+
+        class Message{
+
+        }
+
+        class Start{
+            data: Dict
+        }
+
+        class Stop{
+            data: Dict
+        }
+
+        class Event{
+            metadata: Dict
+            payload: bytes
+        }
+    }
+
+    namespace zmq{
+        class ZMQListener{
+            operator: Operator
+            socket: zmq.Socket
+        }
+
+        class ZMQPublisher{
+            host: str
+            port: int
+        }
+
+    }
+
+    namespace redis{
+
+        class RedisListener{
+            operator: Redis.client
+            pubsub: Redis.pubsub
+        }
+
+        class RedisPublisher{
+            pubsub: Redis.pubsub
+        }
+
+    }
+
+
+
+    Listener <|-- ZMQListener
+    ZMQListener <|-- ZMQPubSubListener
+    Listener o-- Operator
+
+    Publisher <|-- ZMQPublisher
+    ZMQPublisher <|-- ZMQPubSubPublisher
+
+    Publisher <|-- RedisPublisher
+    Listener <|-- RedisListener
+    Operator o-- Publisher
+    Message <|-- Start
+    Message <|-- Stop
+    Message <|-- Event
+
+
+```
+##
+In-process, listening for ZMQ
+
+Note that this leaves Concrete classes undefined as placeholders
+
+TODO: parent class labels
+
+```mermaid
+
+sequenceDiagram
+    autonumber
+    ExternalPublisher ->> ZMQPubSubListener: publish(bytes)
+    loop receiving thread
+        activate ZMQPubSubListener
+            ZMQPubSubListener ->> ConcreteMessageParser: parse(bytes)
+            ZMQPubSubListener ->> MessageQueue: put(bytes)
+        deactivate ZMQPubSubListener
+
+
+        ZMQPubSubListener ->> MessageQueue: message(Message)
+    end
+    activate ConcreteOperator
+        loop polling thread
+            ConcreteOperator ->> MessageQueue: get(bytes)
+        end
+        loop processing thread
+            ConcreteOperator ->> ConcreteOperator: calculate()
+
+            ConcreteOperator ->> ConcretePublisher: publish()
+        end
+    deactivate ConcreteOperator
+```
+
+# Devloper installation
+
+## Conda environment
+We use pixi to be forward thinking tio help with CI. We like it because it helps you easily test that dependencies for a variety of architects can resolve.
+
+However, at the time of writing we can't figure out how to get it to be a good developer experience. So, we create a conda environment like (note that at this time, we are using python 3.11 because of numpy and wheel availability):
+
+```
+conda create -n arroyo python=3.11
+conda activate arroyo
+pip install -e '.[dev]'
+```
+
+## pre-commit
+We use `pre-commit` in CI so you want to use it before commiting.
+To test that your branches changes are all good, type:
+
+```
+pre-commit run --all-files
+```
+
+Since our configuration of `pre-commit` uses `black`, it's possible that it will change files. If you like the changes, you can add them to your `git` commit with
+
+```
+git add .
+```
+
+Then you can run `pre-commit run --all-files` again.
+
+## pixi
+We use `pixi` for CI in github action. It's great for that but can't get our favorite developr tools to use the python environments that `pixi` creaetes in the `.pixi` folder. If you want to play with `pixi`, here are some tips:
+
+To setup a development environment:
+
+* Git clone this repo and CD into the directory
+* Install [pixi](https://pixi.sh/v0.33.0/#installation)
+* Install dependencies with
+'''
+pixi install
+'''
+* run pre-commit on the files
+'''
+pixi r pre-commit
+'''
+
+
+* Run pytest with
+'''
+pixi r test
+'''
+
+# Copyright
+Arroyo Stream Processing Toolset (arroyopy) Copyright (c) 2025, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Dept. of Energy).
+All rights reserved.
+
+If you have questions about your rights to use or distribute this software,
+please contact Berkeley Lab's Intellectual Property Office at
+IPO@lbl.gov.
+
+NOTICE.  This Software was developed under funding from the U.S. Department
+of Energy and the U.S. Government consequently retains certain rights.  As
+such, the U.S. Government has been granted for itself and others acting on
+its behalf a paid-up, nonexclusive, irrevocable, worldwide license in the
+Software to reproduce, distribute copies to the public, prepare derivative
+works, and perform publicly and display publicly, and to permit others to do so.