apppy-queues 0.1.0__tar.gz

apppy_queues-0.1.0/.gitignore

@@ -0,0 +1,28 @@
+__generated__/
+dist/
+*.egg-info
+.env
+.env.*
+*.env
+!.env.ci
+.file_store/
+*.pid
+.python-version
+*.secrets
+.secrets
+*.tar.gz
+*.test_output/
+.test_output/
+uv.lock
+*.whl
+
+# System files
+__pycache__
+.DS_Store
+
+# Editor files
+*.sublime-project
+*.sublime-workspace
+.vscode/*
+!.vscode/settings.json
+

apppy_queues-0.1.0/PKG-INFO

@@ -0,0 +1,11 @@
+Metadata-Version: 2.4
+Name: apppy-queues
+Version: 0.1.0
+Summary: Python definitions for async processing over a queue for server development
+Project-URL: Homepage, https://github.com/spals/apppy
+Author: Tim Kral
+License: MIT
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.11
+Requires-Dist: fastapi-lifespan-manager==0.1.4

apppy_queues-0.1.0/pyproject.toml

@@ -0,0 +1,25 @@
+[build-system]
+requires = ["hatchling>=1.25"]
+build-backend = "hatchling.build"
+
+[project]
+name = "apppy-queues"
+version = "0.1.0"
+description = "Python definitions for async processing over a queue for server development"
+readme = "README.md"
+requires-python = ">=3.11"
+license = {text = "MIT"}
+authors = [{ name = "Tim Kral" }]
+classifiers = [
+  "Programming Language :: Python :: 3",
+  "License :: OSI Approved :: MIT License",
+]
+dependencies = [
+    "fastapi-lifespan-manager==0.1.4"
+]
+
+[project.urls]
+Homepage = "https://github.com/spals/apppy"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/apppy"]

apppy_queues-0.1.0/queues.mk

@@ -0,0 +1,23 @@
+ifndef APPPY_QUEUES_MK_INCLUDED
+APPPY_QUEUES_MK_INCLUDED := 1
+QUEUES_PKG_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
+
+.PHONY: queues queues-dev queues/build queues/clean queues/install queues/install-dev
+
+queues: queues/clean queues/install
+
+queues-dev: queues/clean queues/install-dev
+
+queues/build:
+	cd $(QUEUES_PKG_DIR) && uvx --from build pyproject-build
+
+queues/clean:
+	cd $(QUEUES_PKG_DIR) && rm -rf dist/ *.egg-info .venv
+
+queues/install: queues/build
+	cd $(QUEUES_PKG_DIR) && uv pip install dist/*.whl
+
+queues/install-dev:
+	cd $(QUEUES_PKG_DIR) && uv pip install -e .
+
+endif # APPPY_QUEUES_MK_INCLUDED

apppy_queues-0.1.0/src/apppy/queues/__init__.py

@@ -0,0 +1,35 @@
+import abc
+
+
+class QueueHandler(metaclass=abc.ABCMeta):
+    @abc.abstractmethod
+    async def handle_exception(self, message, exception):
+        pass
+
+    @abc.abstractmethod
+    async def handle_message(self, message):
+        pass
+
+
+class Queue(metaclass=abc.ABCMeta):  # noqa: B024
+    # @abstractmethod
+    async def publish(self, message):
+        raise NotImplementedError
+
+    # @abstractmethod
+    def register_handler(self, handler: QueueHandler):
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def startup(self):
+        """
+        Startup a queue implementation.
+        """
+        pass
+
+    @abc.abstractmethod
+    def shutdown(self):
+        """
+        Allow graceful shutdown logic for a queue implementation.
+        """
+        pass

apppy_queues-0.1.0/src/apppy/queues/asyncio.py

@@ -0,0 +1,92 @@
+import asyncio
+import queue
+from threading import Thread
+
+from fastapi_lifespan_manager import LifespanManager
+
+from apppy.logger import WithLogger
+from apppy.queues import Queue, QueueHandler
+
+
+class AsyncioQueue(Queue, WithLogger):
+    def __init__(
+        self,
+        lifespan: LifespanManager | None,
+        name: str,
+        maxsize: int = -1,
+        shutdown_timeout: int = 2,
+    ) -> None:
+        ## Queue configuration
+        self._queue_name = name
+        self._handlers: list[QueueHandler] = []
+        self._maxsize = maxsize
+        self._shutdown_timeout = shutdown_timeout
+
+        ## Queue internals
+        self._native_queue: queue.Queue = queue.Queue(maxsize=self._maxsize)
+        # Start a background thread to consume the queue
+        self._worker_thread = Thread(target=self._worker_loop, daemon=True)
+        self._logger.info("Created AsyncioQueue", extra={"queue_name": self._queue_name})
+
+        if lifespan is not None:
+            lifespan.add(self._manage_queue)
+
+    async def _manage_queue(self):
+        self.startup()
+        yield
+        self.shutdown()
+
+    async def _process_queue(self):
+        self._logger.info("Started AsyncioQueue processor", extra={"queue_name": self._queue_name})
+
+        while True:
+            try:
+                message = self._native_queue.get()
+                if message is not None:
+                    for h in self._handlers:
+                        try:
+                            # self._logger.debug(
+                            #     "Dispatching to handler", extra={"handler": h.__class__.__name__}
+                            # )
+                            await h.handle_message(message)
+                        except BaseException as e:
+                            await h.handle_exception(message, e)
+            except BaseException as e:
+                self._logger.error("Unhandled exception in queue processor", exc_info=e)
+
+    def _worker_loop(self):
+        asyncio.set_event_loop(asyncio.new_event_loop())
+        loop = asyncio.get_event_loop()
+        loop.run_until_complete(self._process_queue())
+
+    def publish(self, message):
+        try:
+            self._native_queue.put_nowait(message)
+        except queue.Full:
+            self._logger.warning(
+                "Queue is full — dropping message", extra={"queue_name": self._queue_name}
+            )
+
+    def register_handler(self, handler: QueueHandler):
+        self._handlers.append(handler)
+        self._logger.info(
+            "Registered handler in asyncio queue",
+            extra={"queue_name": self._queue_name, "handler": handler.__class__.__name__},
+        )
+
+    def startup(self):
+        if self._worker_thread.is_alive():
+            self._logger.debug(
+                "AsyncioQueue already started", extra={"queue_name": self._queue_name}
+            )
+            return
+
+        self._worker_thread.start()
+
+    def shutdown(self):
+        self._logger.info("Shutting down AsyncioQueue", extra={"queue_name": self._queue_name})
+        if not hasattr(self, "_native_queue"):
+            return
+
+        self._native_queue.put(None)
+        self._worker_thread.join(timeout=self._shutdown_timeout)

apppy_queues-0.1.0/src/apppy/queues/asyncio_unit_test.py

@@ -0,0 +1,50 @@
+import asyncio
+
+from . import QueueHandler
+from .asyncio import AsyncioQueue
+
+
+class AsyncioQueueTestQueueHandler(QueueHandler):
+    def __init__(self) -> None:
+        self._exception_counter = 0
+        self._message_counter = 0
+
+    @property
+    def exception_counter(self):
+        return self._exception_counter
+
+    @property
+    def message_counter(self):
+        return self._message_counter
+
+    async def handle_exception(self, message, exception):
+        self._exception_counter = self._exception_counter + 1
+
+    async def handle_message(self, message):
+        self._message_counter = self._message_counter + 1
+        # Raise an exception on the second call
+        # to this handler
+        if self._message_counter == 2:
+            raise Exception("Mimicked QueueHandler exception")
+
+
+async def test_async_queue():
+    queue = AsyncioQueue(lifespan=None, name="test_direct_queue_handler_error")
+    queue_handler = AsyncioQueueTestQueueHandler()
+    queue.register_handler(queue_handler)
+    queue.startup()
+
+    queue.publish(message={})
+    await asyncio.sleep(0.1)
+    assert queue_handler.exception_counter == 0
+    assert queue_handler.message_counter == 1
+
+    queue.publish(message={})
+    await asyncio.sleep(0.1)
+    assert queue_handler.exception_counter == 1
+    assert queue_handler.message_counter == 2
+
+    queue.publish(message={})
+    await asyncio.sleep(0.1)
+    assert queue_handler.exception_counter == 1
+    assert queue_handler.message_counter == 3