queueio 0.3.0__tar.gz → 0.5.0__tar.gz

queueio-0.5.0/.claude/settings.local.json

@@ -0,0 +1,9 @@
+{
+  "permissions": {
+    "allow": [
+      "mcp__acp__Bash",
+      "mcp__acp__Edit",
+      "mcp__acp__Write"
+    ]
+  }
+}

queueio-0.5.0/.devcontainer/devcontainer.json

@@ -0,0 +1,7 @@
+{
+  "dockerComposeFile": ["docker-compose.yml"],
+  "service": "dev",
+  "features": {
+    "ghcr.io/va-h/devcontainers-features/uv:1": {}
+  }
+}

queueio-0.5.0/.devcontainer/docker-compose.yml

@@ -0,0 +1,10 @@
+services:
+  dev:
+    image: mcr.microsoft.com/devcontainers/python
+    volumes:
+      - ../..:/workspaces
+    command: sleep infinity
+  rabbitmq:
+    image: rabbitmq:management
+    ports:
+      - 15672:15672

queueio-0.5.0/.github/workflows/build.yml

@@ -0,0 +1,42 @@
+name: Build
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    services:
+      rabbitmq:
+        image: rabbitmq
+        env:
+          RABBITMQ_DEFAULT_USER: guest
+          RABBITMQ_DEFAULT_PASS: guest
+        ports:
+          - 5672:5672
+        options: >-
+          --health-cmd "rabbitmq-diagnostics -q ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+      - name: Set up Python
+        run: uv python install 3.14t
+      - name: Install dependencies
+        run: uv sync --all-groups
+      - name: Check formatting
+        run: uv run ruff format --check
+      - name: Run ruff linting
+        run: uv run ruff check
+      - name: Run basedpyright
+        run: uv run basedpyright
+      - name: Run tests
+        run: uv run pytest
+        timeout-minutes: 2
+      - name: Verify built package
+        run: |
+          uv build
+          uvx --from dist/queueio-*.whl queueio --help

queueio-0.5.0/.github/workflows/publish.yml

@@ -0,0 +1,17 @@
+name: Publish to PyPI
+on:
+  release:
+    types:
+      - published
+
+jobs:
+  publish:
+    permissions:
+      id-token: write
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+      - run: uv build
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

queueio-0.5.0/CHANGELOG.md

@@ -0,0 +1,105 @@
+Changelog
+=========
+
+All notable changes to this project will be documented in this file.
+
+The format is loosely based on [Keep a Changelog](https://keepachangelog.com).
+
+[Unreleased]
+------------
+
+[0.5.0] - 2026-02-08
+--------------------
+
+### Breaking Changes
+
+- `invocation.priority` removed;
+  use `invocation.context[priority]` instead.
+- `Invocation.Submitted.priority` replaced with `context`.
+- Serialized invocations from previous versions are incompatible.
+
+### Added
+
+- `QueueVar` for user-defined serializable context variables
+  that propagate across queue boundaries.
+- `QueueContext` mapping to access captured context on invocations.
+- `priority` is now a `QueueVar` instance
+  (same context manager syntax as before).
+
+[0.4.0] - 2026-02-07
+--------------------
+
+### Breaking Changes
+
+- Serialized invocations from previous versions are incompatible.
+- Queues must be recreated with `sync --recreate`
+  to support priority ordering.
+
+### Added
+
+- Priority support for invocations (0-9, default 4).
+  Use the `priority()` context manager to set the priority
+  for invocations created within its scope.
+  Priority propagates to descendent invocations.
+- `sync --recreate` flag to delete and recreate all queues.
+- `Invocation.Submitted` events now include `priority`.
+
+### Fixed
+
+- The `queueio` CLI entry point was broken
+  by a build configuration that excluded the package source.
+
+[0.3.0] - 2026-02-06
+--------------------
+
+### Added
+
+- The journal now uses a threadsafe pika connection,
+  rather than guarding a blocking connection.
+- The integration tests and examples have better queue names.
+
+[0.2.2] - 2025-11-23
+--------------------
+
+### Fixed
+
+- Use the queueio logo from GitHub so it will show on PyPI.
+
+[0.2.1] - 2025-11-22
+--------------------
+
+### Fixed
+
+- Include the queueio logo in the distribution so it will show on PyPI.
+
+[0.2.0] - 2025-11-22
+--------------------
+
+### Acknowledgement
+
+Thank you to Nick Anderegg for allowing me to use the queueio name for this project.
+
+### Added
+
+- `routine` decorator to declare sync or async functions as background routines.
+- `activate` context manager to activate the queueio system.
+- `pause` to coordinate a pause of a routine to queueio.
+- `gather` to run multiple routines concurrently and gather the results.
+- `Routine.submit()` method to submit a routine invocation to the queue.
+- Configuration in the `tool.queueio` section of `pyproject.toml`.
+  - `pika` configures the pika library to connect to the AMQP broker.
+  - `register` configures the modules that declare routines.
+- `QUEUEIO_PIKA` environment variable
+  to override the `pika` configuration in `pyproject.toml`.
+- `queueio sync` command to synchronize queues to the broker.
+- `queueio run` command to run the queueio worker.
+- The queuespec syntax to `queue run` to consume multiple queues with shared capacity.
+- `queueio monitor` command to monitor activity in the queueio system.
+
+[Unreleased]: https://github.com/ryanhiebert/queueio/compare/tag/0.5.0...HEAD
+[0.5.0]: https://github.com/ryanhiebert/queueio/compare/tag/0.4.0...tag/0.5.0
+[0.4.0]: https://github.com/ryanhiebert/queueio/compare/tag/0.3.0...tag/0.4.0
+[0.3.0]: https://github.com/ryanhiebert/queueio/compare/tag/0.2.2...tag/0.3.0
+[0.2.2]: https://github.com/ryanhiebert/queueio/compare/tag/0.2.1...tag/0.2.2
+[0.2.1]: https://github.com/ryanhiebert/queueio/compare/tag/0.2.0...tag/0.2.1
+[0.2.0]: https://github.com/ryanhiebert/queueio/releases/tag/0.2.0

{queueio-0.3.0 → queueio-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: queueio
-Version: 0.3.0
+Version: 0.5.0
 Summary: Python background queues with an async twist
 Author-email: Ryan Hiebert <ryan@ryanhiebert.com>
 License-Expression: MIT

queueio-0.5.0/dill.pyi

@@ -0,0 +1,4 @@
+from typing import Any
+
+def dumps(obj: Any) -> bytes: ...
+def loads(data: bytes) -> Any: ...

queueio-0.5.0/invocation.md

@@ -0,0 +1,32 @@
+Invocations
+===========
+
+When you call a routine, it returns an `Invocation`
+that can be submitted or awaited.
+
+```python
+from basic import yielding
+
+# Submit directly
+yielding(7).submit()
+
+# Or await from an async routine
+await yielding(7)
+```
+
+Priority
+--------
+
+Invocations have a priority from 0 (lowest) to 9 (highest),
+defaulting to 4.
+Set priority using the `priority()` context manager:
+
+```python
+from queueio import priority
+
+with priority(7):
+    yielding(7).submit()
+    await yielding(7)
+```
+
+Priority propagates to descendent invocations.

queueio-0.5.0/principles.md

@@ -0,0 +1,28 @@
+Principles
+==========
+
+There's been a great deal of momentum to embrace async functions for IO,
+but they split the world and are confusing, especially to new developers.
+Threads or green threads are often a better choice for concurrency
+because it doesn't require learning about coroutines
+before you even need concurrency.
+
+Still, async functions are a powerful way to implement state machines.
+Complex workflows can be represented as async functions
+to allow thinking about things in a pull-based fashion,
+while also scaling well for processes that must take a long time.
+Ideally, running coroutines could be serialized and resumed,
+perhaps even on a different machine.
+
+This vision motivates queueio.
+Routines need to coordinate with the worker
+because queue-level concurrency must be carefully managed.
+Instead of complex construction of pipelines to make this possible,
+queueio allows you to write complex workflows
+in a traditional imperative style.
+
+queueio encourages you to use synchronous IO in routines.
+The yield points in async functions are only needed
+when a routine needs to coordinate with the worker,
+such as to indicate that the worker can use
+the capacity reserved for it to do other work.

{queueio-0.3.0 → queueio-0.5.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "queueio"
-version = "0.3.0"
+version = "0.5.0"
 description = "Python background queues with an async twist"
 readme = "README.md"
 license = "MIT"
@@ -51,6 +51,8 @@ force-single-line = true
 register = [
     "queueio.samples.basic",
     "queueio.samples.expanded",
+    "queueio.samples.priority",
+    "queueio.samples.queuevar",
 ]
 pika = "amqp://localhost:5672"
 
@@ -61,9 +63,6 @@ timeout = 30
 branch = true
 source = ["queueio"]
 
-[tool.hatch.build]
-include = ["logo.svg"]
-
 [build-system]
 requires = ["hatchling"]
 build-backend = "hatchling.build"

queueio-0.5.0/pyrightconfig.json

@@ -0,0 +1,4 @@
+{
+  "pythonVersion": "3.14",
+  "typeCheckingMode": "standard"
+}

queueio-0.5.0/queueio/__init__.py

@@ -0,0 +1,14 @@
+from contextlib import contextmanager
+
+from .gather import gather as gather
+from .pause import pause as pause
+from .queueio import QueueIO as RealQueueIO
+from .queueio import priority as priority
+from .queuevar import QueueVar as QueueVar
+from .registry import routine as routine
+
+
+@contextmanager
+def activate():
+    with RealQueueIO().activate():
+        yield

queueio-0.5.0/queueio/__main__.py

@@ -0,0 +1,184 @@
+from contextlib import suppress
+from typing import Annotated
+
+import typer
+from typer import Argument
+from typer import Typer
+
+from .monitor import Monitor
+from .queueio import QueueIO
+from .queuespec import QueueSpec
+from .worker import Worker
+
+app = Typer()
+
+routine_app = Typer()
+queue_app = Typer()
+
+app.add_typer(
+    routine_app,
+    name="routine",
+    help="A function to coordinate background execution.",
+    rich_help_panel="Entities",
+)
+app.add_typer(
+    queue_app,
+    name="queue",
+    help="An ordered collection of work items to process.",
+    rich_help_panel="Entities",
+)
+
+
+@routine_app.command("list")
+def routine_list():
+    """Show all registered routines."""
+    queueio = QueueIO()
+    try:
+        routines = queueio.routines()
+
+        if not routines:
+            print("No routines registered.")
+            return
+
+        # Calculate column widths
+        name_width = max(len("Name"), max(len(routine.name) for routine in routines))
+        function_paths = []
+        for routine in routines:
+            module = routine.fn.__module__
+            qualname = routine.fn.__qualname__
+            function_paths.append(f"{module}.{qualname}")
+        path_width = max(len("Path"), max(len(path) for path in function_paths))
+
+        print(f"{'Name':<{name_width}} | {'Path':<{path_width}}")
+        print(f"{'-' * name_width}-+-{'-' * path_width}")
+        for routine, path in zip(routines, function_paths, strict=False):
+            print(f"{routine.name:<{name_width}} | {path:<{path_width}}")
+    finally:
+        queueio.shutdown()
+
+
+@app.command(rich_help_panel="Commands")
+def monitor(raw: bool = False):
+    """Monitor queueio events.
+
+    Show a live view of queueio activity. Use --raw for detailed event output.
+    """
+    if raw:
+        queueio = QueueIO()
+        events = queueio.subscribe({object})
+        try:
+            while True:
+                print(events.get())
+        except KeyboardInterrupt:
+            print("Shutting down gracefully.")
+        finally:
+            queueio.shutdown()
+    else:
+        Monitor().run()
+
+
+@app.command(rich_help_panel="Commands")
+def run(
+    queuespec: Annotated[
+        QueueSpec,
+        Argument(
+            parser=QueueSpec.parse,
+            help="Queue configuration in format 'queue=concurrency'. "
+            "Examples: 'production=10', 'api,background=5'",
+            metavar="QUEUE[,QUEUE2,...]=CONCURRENCY",
+        ),
+    ],
+):
+    """Run a worker to process from a queue.
+
+    The worker will process invocations from the specified queue,
+    as many at a time as specified by the concurrency.
+    """
+    queueio = QueueIO()
+    Worker(queueio, queuespec)()
+
+
+@app.command(rich_help_panel="Commands")
+def sync(
+    recreate: Annotated[
+        bool,
+        typer.Option(
+            help="Delete and recreate queues that have incompatible arguments. "
+            "WARNING: This will lose any pending messages in those queues.",
+        ),
+    ] = False,
+):
+    """Sync known queues to the broker."""
+    queueio = QueueIO()
+    try:
+        routines = queueio.routines()
+
+        if not routines:
+            print("No routines registered.")
+            return
+
+        queues = sorted({routine.queue for routine in routines})
+
+        print(f"Syncing queues for {len(routines)} routine(s):")
+        if recreate:
+            for queue in queues:
+                print(f"  Recreating queue: {queue}")
+                with suppress(Exception):
+                    queueio.delete(queue=queue)
+
+        failed = []
+        for queue in queues:
+            print(f"  Ensuring queue exists: {queue}")
+            try:
+                queueio.create(queue=queue)
+            except Exception:
+                failed.append(queue)
+
+        if failed:
+            print(
+                f"\nError: {len(failed)} queue(s) have incompatible arguments: "
+                f"{', '.join(failed)}\n"
+                f"Re-run with --recreate to delete and recreate them.\n"
+                f"WARNING: This will lose any pending messages in those queues."
+            )
+            raise typer.Exit(1)
+
+        print(f"Successfully synced {len(queues)} queue(s)")
+    finally:
+        queueio.shutdown()
+
+
+@queue_app.command("purge")
+def queue_purge(
+    queues: Annotated[
+        str,
+        Argument(
+            help="Comma-separated list of queues to purge. "
+            "Examples: 'queueio', 'production,background'",
+            metavar="QUEUE[,QUEUE2,...]",
+        ),
+    ],
+):
+    """Purge all messages from some queues.
+
+    This will remove all pending messages from the given queues.
+    Use with caution as this operation cannot be undone.
+    """
+    queueio = QueueIO()
+    try:
+        queue_list = [q.strip() for q in queues.split(",") if q.strip()]
+        if not queue_list:
+            print("Error: No valid queue names provided")
+            return
+
+        for queue in queue_list:
+            print(f"Purging queue: {queue}")
+            queueio.purge(queue=queue)
+
+        print(f"Successfully purged {len(queue_list)} queue(s)")
+    finally:
+        queueio.shutdown()
+
+
+if __name__ == "__main__":
+    app()

queueio-0.5.0/queueio/broker.py

@@ -0,0 +1,50 @@
+from abc import ABC
+from abc import abstractmethod
+
+from .queuespec import QueueSpec
+from .receiver import Receiver
+
+
+class Broker(ABC):
+    """A broker enables sending and receiving messages with a queue.
+
+    Messages sent by a broker are assumed to be idempotent, and may be
+    delivered multiple times in some conditions in order to ensure
+    at-least-once delivery.
+    """
+
+    @classmethod
+    @abstractmethod
+    def from_uri(cls, uri: str, /):
+        """Create a broker instance from a URI."""
+        raise NotImplementedError("Subclasses must implement this method.")
+
+    @abstractmethod
+    def enqueue(self, body: bytes, /, *, queue: str, priority: int):
+        """Enqueue a message."""
+        raise NotImplementedError("Subclasses must implement this method.")
+
+    @abstractmethod
+    def create(self, *, queue: str):
+        """Create a queue if it doesn't exist."""
+        raise NotImplementedError("Subclasses must implement this method.")
+
+    @abstractmethod
+    def delete(self, *, queue: str):
+        """Delete a queue."""
+        raise NotImplementedError("Subclasses must implement this method.")
+
+    @abstractmethod
+    def purge(self, *, queue: str):
+        """Purge all messages from the queue."""
+        raise NotImplementedError("Subclasses must implement this method.")
+
+    @abstractmethod
+    def receive(self, queuespec: QueueSpec, /) -> Receiver:
+        """Receive messages as specified by the QueueSpec."""
+        raise NotImplementedError("Subclasses must implement this method.")
+
+    @abstractmethod
+    def shutdown(self):
+        """Signal the final shutdown of the broker."""
+        raise NotImplementedError("Subclasses must implement this method.")