queueio 0.4.0__tar.gz → 0.5.0__tar.gz

{queueio-0.4.0 → queueio-0.5.0}/CHANGELOG.md

@@ -8,6 +8,24 @@ The format is loosely based on [Keep a Changelog](https://keepachangelog.com).
 [Unreleased]
 ------------
 
+[0.5.0] - 2026-02-08
+--------------------
+
+### Breaking Changes
+
+- `invocation.priority` removed;
+  use `invocation.context[priority]` instead.
+- `Invocation.Submitted.priority` replaced with `context`.
+- Serialized invocations from previous versions are incompatible.
+
+### Added
+
+- `QueueVar` for user-defined serializable context variables
+  that propagate across queue boundaries.
+- `QueueContext` mapping to access captured context on invocations.
+- `priority` is now a `QueueVar` instance
+  (same context manager syntax as before).
+
 [0.4.0] - 2026-02-07
 --------------------
 
@@ -78,7 +96,8 @@ Thank you to Nick Anderegg for allowing me to use the queueio name for this proj
 - The queuespec syntax to `queue run` to consume multiple queues with shared capacity.
 - `queueio monitor` command to monitor activity in the queueio system.
 
-[Unreleased]: https://github.com/ryanhiebert/queueio/compare/tag/0.4.0...HEAD
+[Unreleased]: https://github.com/ryanhiebert/queueio/compare/tag/0.5.0...HEAD
+[0.5.0]: https://github.com/ryanhiebert/queueio/compare/tag/0.4.0...tag/0.5.0
 [0.4.0]: https://github.com/ryanhiebert/queueio/compare/tag/0.3.0...tag/0.4.0
 [0.3.0]: https://github.com/ryanhiebert/queueio/compare/tag/0.2.2...tag/0.3.0
 [0.2.2]: https://github.com/ryanhiebert/queueio/compare/tag/0.2.1...tag/0.2.2

{queueio-0.4.0 → queueio-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: queueio
-Version: 0.4.0
+Version: 0.5.0
 Summary: Python background queues with an async twist
 Author-email: Ryan Hiebert <ryan@ryanhiebert.com>
 License-Expression: MIT

{queueio-0.4.0 → queueio-0.5.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "queueio"
-version = "0.4.0"
+version = "0.5.0"
 description = "Python background queues with an async twist"
 readme = "README.md"
 license = "MIT"
@@ -52,6 +52,7 @@ register = [
     "queueio.samples.basic",
     "queueio.samples.expanded",
     "queueio.samples.priority",
+    "queueio.samples.queuevar",
 ]
 pika = "amqp://localhost:5672"
 

{queueio-0.4.0 → queueio-0.5.0}/queueio/__init__.py

@@ -1,9 +1,10 @@
 from contextlib import contextmanager
 
 from .gather import gather as gather
-from .invocation import priority as priority
 from .pause import pause as pause
 from .queueio import QueueIO as RealQueueIO
+from .queueio import priority as priority
+from .queuevar import QueueVar as QueueVar
 from .registry import routine as routine
 
 

{queueio-0.4.0 → queueio-0.5.0}/queueio/consumer.py

@@ -2,6 +2,7 @@ from collections.abc import Callable
 from collections.abc import Generator
 from collections.abc import Iterable
 from collections.abc import Iterator
+from contextvars import Context
 from typing import Any
 
 from .invocation import Invocation
@@ -41,6 +42,7 @@ class Consumer(Iterable[Invocation]):
         invocation: Invocation,
         generator: Generator,
         suspension: Suspension | None,
+        context: Context,
     ):
         """Signal that the invocation has suspended."""
         if suspension:
@@ -51,6 +53,7 @@ class Consumer(Iterable[Invocation]):
                     suspension=suspension,
                     invocation=invocation,
                     generator=generator,
+                    context=context,
                 )
             )
         self.__receiver.pause(self.__invocations[invocation])

{queueio-0.4.0 → queueio-0.5.0}/queueio/continuation.py

@@ -1,5 +1,6 @@
 from collections.abc import Callable
 from collections.abc import Generator
+from contextvars import Context
 from dataclasses import dataclass
 from dataclasses import field
 from typing import Any
@@ -17,6 +18,7 @@ class Continuation[T: Callable[..., Any] = Callable[..., Any]]:
     invocation: Invocation
     generator: Generator[Invocation, Any, Any]
     result: Result[Any, BaseException]
+    context: Context
 
     def resume(self) -> Any:
         match self.result:

{queueio-0.4.0 → queueio-0.5.0}/queueio/invocation.py

@@ -3,6 +3,7 @@ from collections.abc import Callable
 from collections.abc import Generator
 from concurrent.futures import Future
 from contextlib import contextmanager
+from contextvars import Context
 from contextvars import ContextVar
 from dataclasses import dataclass
 from dataclasses import field
@@ -11,19 +12,9 @@ from typing import Self
 
 from .event import Event
 from .id import random_id
+from .queuevar import QueueContext
 from .suspension import Suspension
 
-_priority = ContextVar[int]("priority", default=4)
-
-
-@contextmanager
-def priority(level: int, /):
-    token = _priority.set(level)
-    try:
-        yield
-    finally:
-        _priority.reset(token)
-
 
 @dataclass(eq=False, kw_only=True)
 class Invocation[R](Suspension[R]):
@@ -31,7 +22,7 @@ class Invocation[R](Suspension[R]):
     routine: str
     args: tuple[Any, ...]
     kwargs: dict[str, Any]
-    priority: int = field(default_factory=_priority.get)
+    context: QueueContext = field(default_factory=QueueContext.capture)
 
     __handler = ContextVar[Callable[[Self], Future] | None](
         "Invocation.handler", default=None
@@ -68,7 +59,7 @@ class Invocation[R](Suspension[R]):
                 "routine": self.routine,
                 "args": self.args,
                 "kwargs": self.kwargs,
-                "priority": self.priority,
+                "context": self.context.serialize(),
             }
         ).encode()
 
@@ -80,7 +71,7 @@ class Invocation[R](Suspension[R]):
             routine=data["routine"],
             args=data["args"],
             kwargs=data["kwargs"],
-            priority=data["priority"],
+            context=QueueContext.deserialize(data.get("context", {})),
         )
 
     @dataclass(eq=False, kw_only=True, repr=False)
@@ -88,7 +79,7 @@ class Invocation[R](Suspension[R]):
         routine: str
         args: tuple[Any]
         kwargs: dict[str, Any]
-        priority: int
+        context: QueueContext
 
     @dataclass(eq=False, kw_only=True)
     class Started(Event): ...
@@ -104,6 +95,7 @@ class Invocation[R](Suspension[R]):
         suspension: Suspension = field(repr=False)
         generator: Generator[Invocation, Any, Any] = field(repr=False)
         invocation: Invocation = field(repr=False)
+        context: Context = field(repr=False)
 
     @dataclass(eq=False, kw_only=True)
     class BaseContinued(Event):

{queueio-0.4.0 → queueio-0.5.0}/queueio/queueio.py

@@ -17,6 +17,7 @@ from .message import Message
 from .queue import Queue
 from .queue import ShutDown
 from .queuespec import QueueSpec
+from .queuevar import QueueVar
 from .registry import ROUTINE_REGISTRY
 from .result import Err
 from .result import Ok
@@ -24,6 +25,8 @@ from .routine import Routine
 from .stream import Stream
 from .thread import Thread
 
+priority: QueueVar[int] = QueueVar("priority", default=4)
+
 
 class QueueIO:
     __active = ContextVar[Self | None]("active", default=None)
@@ -181,12 +184,14 @@ class QueueIO:
                 routine=invocation.routine,
                 args=invocation.args,
                 kwargs=invocation.kwargs,
-                priority=invocation.priority,
+                context=invocation.context,
             )
         )
         queue = routine.queue
         self.__broker.enqueue(
-            invocation.serialize(), queue=queue, priority=invocation.priority
+            invocation.serialize(),
+            queue=queue,
+            priority=invocation.context.get(priority, priority.get()),
         )
 
     def consume(self, queuespec: QueueSpec, /) -> Consumer:

queueio-0.5.0/queueio/queuevar.md

@@ -0,0 +1,102 @@
+# QueueVar
+
+Queue variables propagate context across queue boundaries.
+Like Python's `contextvars.ContextVar`,
+but values automatically serialize with invocations
+and are restored when the worker executes them.
+
+## Defining a QueueVar
+
+```python
+from queueio import QueueVar
+
+tenant_id = QueueVar("tenant_id", default=None)
+```
+
+Creating a `QueueVar` registers it globally.
+All registered queue variables are captured
+when an `Invocation` is created,
+and restored when the worker runs it.
+
+## Setting and reading values
+
+Set a value with the context manager:
+
+```python
+with tenant_id("acme"):
+    # tenant_id.get() returns "acme" here
+    invocation = my_routine()
+    # tenant_id is captured in invocation.context
+```
+
+Read the current value with `.get()`:
+
+```python
+@routine(name="greet", queue="default")
+def greet():
+    tid = tenant_id.get()
+    print(f"Hello from tenant {tid}")
+```
+
+## Propagation
+
+When an invocation is created,
+the current values of all registered queue variables are captured
+into `invocation.context`.
+The worker restores these values before executing the routine,
+so any invocations created during execution inherit the parent's context.
+
+```python
+@routine(name="inner", queue="work")
+def inner():
+    return tenant_id.get()
+
+@routine(name="outer", queue="work")
+async def outer():
+    with tenant_id("acme"):
+        result = await inner()
+        assert result == "acme"
+```
+
+## Accessing context on an invocation
+
+The `QueueVar` instance is a key,
+and `QueueContext` behaves like a mapping:
+
+```python
+with tenant_id("acme"):
+    invocation = my_routine()
+    print(invocation.context[tenant_id])  # "acme"
+    print(tenant_id in invocation.context)  # True
+
+invocation = my_routine()
+# invocation.context[tenant_id] would raise KeyError
+print(tenant_id in invocation.context)  # False
+print(invocation.context.get(tenant_id, "unknown"))  # "unknown"
+```
+
+## Priority
+
+Priority is a built-in queue variable:
+
+```python
+from queueio import priority
+
+with priority(2):
+    invocation = my_routine()
+    print(invocation.context[priority])  # 2
+    print(invocation.context.get(priority, priority.default))  # 2
+```
+
+## Context variables
+
+Regular `contextvars.ContextVar` values
+do not propagate across queue boundaries.
+Use `QueueVar` for any context that needs to travel
+with invocations.
+
+## Serialization
+
+Values must be JSON-serializable.
+The context is included in the serialized invocation
+and restored on deserialization.

queueio-0.5.0/queueio/queuevar.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+from contextlib import contextmanager
+from contextvars import Context
+from contextvars import ContextVar
+from contextvars import copy_context
+from typing import Any
+
+_registry: dict[ContextVar, QueueVar] = {}
+
+
+class QueueVar[T]:
+    def __init__(self, name: str, *, default: T):
+        self.__default = default
+        self.__var: ContextVar[T] = ContextVar(name, default=default)
+        _registry[self.__var] = self
+
+    @property
+    def name(self) -> str:
+        return self.__var.name
+
+    def get(self) -> T:
+        return self.__var.get()
+
+    @contextmanager
+    def __call__(self, value: T, /):
+        token = self.__var.set(value)
+        try:
+            yield
+        finally:
+            self.__var.reset(token)
+
+
+class QueueContext:
+    def __init__(self, data: dict[str, Any]):
+        self.__data = data
+
+    def __getitem__[T](self, var: QueueVar[T]) -> T:
+        return self.__data[var.name]
+
+    def get[T, Default](self, var: QueueVar[T], default: Default = None) -> T | Default:
+        return self.__data.get(var.name, default)
+
+    def __contains__(self, var: object) -> bool:
+        if not isinstance(var, QueueVar):
+            return False
+        return var.name in self.__data
+
+    def load(self, ctx: Context) -> None:
+        def initialize():
+            for contextvar, queuevar in _registry.items():
+                if queuevar.name in self.__data:
+                    contextvar.set(self.__data[queuevar.name])
+
+        ctx.run(initialize)
+
+    def serialize(self) -> dict[str, Any]:
+        return dict(self.__data)
+
+    @classmethod
+    def deserialize(cls, data: dict[str, Any]) -> QueueContext:
+        return cls(data)
+
+    @classmethod
+    def capture(cls) -> QueueContext:
+        data: dict[str, Any] = {}
+        ctx = copy_context()
+        for contextvar in ctx:
+            if contextvar in _registry:
+                data[_registry[contextvar].name] = ctx[contextvar]
+        return cls(data)

{queueio-0.4.0 → queueio-0.5.0}/queueio/samples/priority.py

@@ -4,16 +4,16 @@ from queueio import routine
 
 @routine(name="inherited_priority", queue="priority")
 def inherited_priority():
-    return inherited_priority().priority
+    return priority.get()
 
 
 @routine(name="demonstrate_priority_inheritance", queue="priority")
 async def demonstrate_priorities():
     invocation = inherited_priority()
-    assert invocation.priority == 4
+    assert priority not in invocation.context
     assert await invocation == 4
 
     with priority(2):
         invocation = inherited_priority()
-        assert invocation.priority == 2
+        assert invocation.context[priority] == 2
         assert await invocation == 2

queueio-0.5.0/queueio/samples/queuevar.py

@@ -0,0 +1,21 @@
+from queueio import QueueVar
+from queueio import routine
+
+tenant_id = QueueVar[str | None]("tenant_id", default=None)
+
+
+@routine(name="get_tenant", queue="queuevar")
+def get_tenant():
+    return tenant_id.get()
+
+
+@routine(name="demonstrate_queuevar", queue="queuevar")
+async def demonstrate_queuevar():
+    invocation = get_tenant()
+    assert tenant_id not in invocation.context
+    assert await invocation is None
+
+    with tenant_id("acme"):
+        invocation = get_tenant()
+        assert invocation.context[tenant_id] == "acme"
+        assert await invocation == "acme"

queueio-0.5.0/queueio/samples/queuevar_test.py

@@ -0,0 +1,42 @@
+import subprocess
+import sys
+
+import pytest
+
+from queueio.invocation import Invocation
+from queueio.queueio import QueueIO
+from queueio.result import Ok
+
+from .queuevar import demonstrate_queuevar
+
+
+@pytest.mark.timeout(10)
+def test_queuevar_propagation():
+    """QueueVar values propagate through the full invocation chain."""
+    queueio = QueueIO()
+
+    with queueio.activate():
+        queueio.create(queue="queuevar")
+        queueio.purge(queue="queuevar")
+        events = queueio.subscribe({Invocation.Completed})
+        invocation = demonstrate_queuevar()
+        queueio.submit(invocation)
+
+        proc = subprocess.Popen(
+            [sys.executable, "-m", "queueio", "run", "queuevar=2"],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+        try:
+            while event := events.get():
+                if event.id == invocation.id:
+                    assert isinstance(event.result, Ok)
+                    break
+        finally:
+            if proc.poll() is None:
+                proc.terminate()
+                try:
+                    proc.wait(timeout=5)
+                except subprocess.TimeoutExpired:
+                    proc.kill()
+                    proc.wait()

{queueio-0.4.0 → queueio-0.5.0}/queueio/worker.py

@@ -3,11 +3,11 @@ from concurrent.futures import FIRST_COMPLETED
 from concurrent.futures import Future
 from concurrent.futures import wait
 from contextlib import suppress
+from contextvars import copy_context
 from threading import Timer
 
 from .continuation import Continuation
 from .invocation import Invocation
-from .invocation import priority
 from .queue import Queue
 from .queue import ShutDown
 from .queueio import QueueIO
@@ -105,6 +105,7 @@ class Worker:
                         invocation=event.invocation,
                         generator=event.generator,
                         result=Ok(None),
+                        context=event.context,
                     )
                     # Replace ``new`` before setting the result
                     # to avoid short busy wait loops.
@@ -144,6 +145,7 @@ class Worker:
                                 invocation=continuation.invocation,
                                 generator=continuation.generator,
                                 result=Err(exception),
+                                context=continuation.context,
                             )
                         )
                 else:
@@ -156,6 +158,7 @@ class Worker:
                                 invocation=continuation.invocation,
                                 generator=continuation.generator,
                                 result=Ok(value),
+                                context=continuation.context,
                             ),
                         )
 
@@ -190,9 +193,10 @@ class Worker:
     def __run_invocation(self, invocation: Invocation):
         """Process an invocation task."""
         routine = self.__queueio.routine(invocation.routine)
+        ctx = copy_context()
+        invocation.context.load(ctx)
         try:
-            with priority(invocation.priority):
-                result = routine.fn(*invocation.args, **invocation.kwargs)
+            result = ctx.run(routine.fn, *invocation.args, **invocation.kwargs)
         except Exception as exception:
             self.__consumer.error(invocation, exception)
         else:
@@ -203,6 +207,7 @@ class Worker:
                         invocation=invocation,
                         generator=generator,
                         result=Ok(None),
+                        context=ctx,
                     )
                 )
             else:
@@ -213,15 +218,17 @@ class Worker:
         method = continuation.resume
 
         try:
-            with priority(continuation.invocation.priority):
-                suspension = method()
+            suspension = continuation.context.run(method)
         except StopIteration as stop:
             self.__consumer.succeed(continuation.invocation, stop.value)
         except Exception as exception:
             self.__consumer.error(continuation.invocation, exception)
         else:
             self.__consumer.suspend(
-                continuation.invocation, continuation.generator, suspension
+                continuation.invocation,
+                continuation.generator,
+                suspension,
+                continuation.context,
             )
 
     def stop(self):