queueio 0.2.0__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {queueio-0.2.0 → queueio-0.2.2}/PKG-INFO +3 -2
- {queueio-0.2.0 → queueio-0.2.2}/README.md +2 -1
- {queueio-0.2.0 → queueio-0.2.2}/pyproject.toml +4 -1
- queueio-0.2.0/.devcontainer/devcontainer.json +0 -7
- queueio-0.2.0/.devcontainer/docker-compose.yml +0 -10
- queueio-0.2.0/.github/workflows/build.yml +0 -38
- queueio-0.2.0/.github/workflows/publish.yml +0 -17
- queueio-0.2.0/CHANGELOG.md +0 -32
- queueio-0.2.0/dill.pyi +0 -4
- queueio-0.2.0/principles.md +0 -28
- queueio-0.2.0/pyrightconfig.json +0 -4
- queueio-0.2.0/queueio/__init__.py +0 -12
- queueio-0.2.0/queueio/__main__.py +0 -155
- queueio-0.2.0/queueio/broker.py +0 -45
- queueio-0.2.0/queueio/broker_test.py +0 -499
- queueio-0.2.0/queueio/conftest.py +0 -32
- queueio-0.2.0/queueio/consumer.py +0 -92
- queueio-0.2.0/queueio/continuation.py +0 -26
- queueio-0.2.0/queueio/event.py +0 -15
- queueio-0.2.0/queueio/freethreading_test.py +0 -28
- queueio-0.2.0/queueio/gather.py +0 -77
- queueio-0.2.0/queueio/id.py +0 -8
- queueio-0.2.0/queueio/invocation.py +0 -119
- queueio-0.2.0/queueio/journal.py +0 -23
- queueio-0.2.0/queueio/journal_test.py +0 -155
- queueio-0.2.0/queueio/message.py +0 -8
- queueio-0.2.0/queueio/monitor.py +0 -120
- queueio-0.2.0/queueio/pause.py +0 -21
- queueio-0.2.0/queueio/pika/__init__.py +0 -0
- queueio-0.2.0/queueio/pika/broker.py +0 -53
- queueio-0.2.0/queueio/pika/broker_test.py +0 -23
- queueio-0.2.0/queueio/pika/journal.py +0 -82
- queueio-0.2.0/queueio/pika/journal_test.py +0 -20
- queueio-0.2.0/queueio/pika/receiver.py +0 -72
- queueio-0.2.0/queueio/pika/threadsafe.py +0 -199
- queueio-0.2.0/queueio/queue.py +0 -183
- queueio-0.2.0/queueio/queue_test.py +0 -729
- queueio-0.2.0/queueio/queueio.py +0 -196
- queueio-0.2.0/queueio/queueio_test.py +0 -558
- queueio-0.2.0/queueio/queuespec.md +0 -31
- queueio-0.2.0/queueio/queuespec.py +0 -59
- queueio-0.2.0/queueio/queuespec_test.py +0 -67
- queueio-0.2.0/queueio/receiver.py +0 -47
- queueio-0.2.0/queueio/registry.py +0 -19
- queueio-0.2.0/queueio/result.py +0 -14
- queueio-0.2.0/queueio/routine.py +0 -16
- queueio-0.2.0/queueio/samples/__init__.py +0 -0
- queueio-0.2.0/queueio/samples/basic.py +0 -22
- queueio-0.2.0/queueio/samples/basic_test.py +0 -39
- queueio-0.2.0/queueio/samples/expanded.py +0 -45
- queueio-0.2.0/queueio/samples/expanded_test.py +0 -42
- queueio-0.2.0/queueio/select.py +0 -170
- queueio-0.2.0/queueio/stream.py +0 -76
- queueio-0.2.0/queueio/stub/__init__.py +0 -0
- queueio-0.2.0/queueio/stub/broker.py +0 -69
- queueio-0.2.0/queueio/stub/broker_test.py +0 -22
- queueio-0.2.0/queueio/stub/journal.py +0 -36
- queueio-0.2.0/queueio/stub/journal_test.py +0 -19
- queueio-0.2.0/queueio/stub/receiver.py +0 -70
- queueio-0.2.0/queueio/suspension.py +0 -33
- queueio-0.2.0/queueio/thread.py +0 -50
- queueio-0.2.0/queueio/worker.py +0 -239
- {queueio-0.2.0 → queueio-0.2.2}/.gitignore +0 -0
- {queueio-0.2.0 → queueio-0.2.2}/LICENSE +0 -0
- {queueio-0.2.0 → queueio-0.2.2}/logo.svg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: queueio
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: Python background queues with an async twist
|
|
5
5
|
Author-email: Ryan Hiebert <ryan@ryanhiebert.com>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -20,7 +20,8 @@ Requires-Dist: textual>=3.2.0
|
|
|
20
20
|
Requires-Dist: typer>=0.15.4
|
|
21
21
|
Description-Content-Type: text/markdown
|
|
22
22
|
|
|
23
|
-

|
|
23
|
+

|
|
24
|
+
|
|
24
25
|
|
|
25
26
|
Python background queues with an async twist
|
|
26
27
|
============================================
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "queueio"
|
|
3
|
-
version = "0.2.
|
|
3
|
+
version = "0.2.2"
|
|
4
4
|
description = "Python background queues with an async twist"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "MIT"
|
|
@@ -61,6 +61,9 @@ timeout = 30
|
|
|
61
61
|
branch = true
|
|
62
62
|
source = ["queueio"]
|
|
63
63
|
|
|
64
|
+
[tool.hatch.build]
|
|
65
|
+
include = ["logo.svg"]
|
|
66
|
+
|
|
64
67
|
[build-system]
|
|
65
68
|
requires = ["hatchling"]
|
|
66
69
|
build-backend = "hatchling.build"
|
|
@@ -1,38 +0,0 @@
|
|
|
1
|
-
name: Build
|
|
2
|
-
|
|
3
|
-
on: [push, pull_request]
|
|
4
|
-
|
|
5
|
-
jobs:
|
|
6
|
-
build:
|
|
7
|
-
runs-on: ubuntu-latest
|
|
8
|
-
timeout-minutes: 5
|
|
9
|
-
services:
|
|
10
|
-
rabbitmq:
|
|
11
|
-
image: rabbitmq
|
|
12
|
-
env:
|
|
13
|
-
RABBITMQ_DEFAULT_USER: guest
|
|
14
|
-
RABBITMQ_DEFAULT_PASS: guest
|
|
15
|
-
ports:
|
|
16
|
-
- 5672:5672
|
|
17
|
-
options: >-
|
|
18
|
-
--health-cmd "rabbitmq-diagnostics -q ping"
|
|
19
|
-
--health-interval 10s
|
|
20
|
-
--health-timeout 5s
|
|
21
|
-
--health-retries 5
|
|
22
|
-
steps:
|
|
23
|
-
- uses: actions/checkout@v4
|
|
24
|
-
- name: Install uv
|
|
25
|
-
uses: astral-sh/setup-uv@v3
|
|
26
|
-
- name: Set up Python
|
|
27
|
-
run: uv python install 3.14t
|
|
28
|
-
- name: Install dependencies
|
|
29
|
-
run: uv sync --all-groups
|
|
30
|
-
- name: Check formatting
|
|
31
|
-
run: uv run ruff format --check
|
|
32
|
-
- name: Run ruff linting
|
|
33
|
-
run: uv run ruff check
|
|
34
|
-
- name: Run basedpyright
|
|
35
|
-
run: uv run basedpyright
|
|
36
|
-
- name: Run tests
|
|
37
|
-
run: uv run pytest
|
|
38
|
-
timeout-minutes: 2
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
name: Publish to PyPI
|
|
2
|
-
on:
|
|
3
|
-
release:
|
|
4
|
-
types:
|
|
5
|
-
- published
|
|
6
|
-
|
|
7
|
-
jobs:
|
|
8
|
-
publish:
|
|
9
|
-
permissions:
|
|
10
|
-
id-token: write
|
|
11
|
-
runs-on: ubuntu-latest
|
|
12
|
-
steps:
|
|
13
|
-
- uses: actions/checkout@v4
|
|
14
|
-
- uses: astral-sh/setup-uv@v5
|
|
15
|
-
- run: uv build
|
|
16
|
-
- name: Publish package distributions to PyPI
|
|
17
|
-
uses: pypa/gh-action-pypi-publish@release/v1
|
queueio-0.2.0/CHANGELOG.md
DELETED
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
Changelog
|
|
2
|
-
=========
|
|
3
|
-
|
|
4
|
-
All notable changes to this project will be documented in this file.
|
|
5
|
-
|
|
6
|
-
The format is loosely based on [Keep a Changelog](https://keepachangelog.com).
|
|
7
|
-
|
|
8
|
-
[0.2.0] - 2025-11-22
|
|
9
|
-
--------------------
|
|
10
|
-
|
|
11
|
-
### Acknowledgement
|
|
12
|
-
|
|
13
|
-
Thank you to Nick Anderegg for allowing me to use the queueio name for this project.
|
|
14
|
-
|
|
15
|
-
### Added
|
|
16
|
-
|
|
17
|
-
- `routine` decorator to declare sync or async functions as background routines.
|
|
18
|
-
- `activate` context manager to activate the queueio system.
|
|
19
|
-
- `pause` to coordinate a pause of a routine to queueio.
|
|
20
|
-
- `gather` to run multiple routines concurrently and gather the results.
|
|
21
|
-
- `Routine.submit()` method to submit a routine invocation to the queue.
|
|
22
|
-
- Configuration in the `tool.queueio` section of `pyproject.toml`.
|
|
23
|
-
- `pika` configures the pika library to connect to the AMQP broker.
|
|
24
|
-
- `register` configures the modules that declare routines.
|
|
25
|
-
- `QUEUEIO_PIKA` environment variable
|
|
26
|
-
to override the `pika` configuration in `pyproject.toml`.
|
|
27
|
-
- `queueio sync` command to synchronize queues to the broker.
|
|
28
|
-
- `queueio run` command to run the queueio worker.
|
|
29
|
-
- The queuespec syntax to `queue run` to consume multiple queues with shared capacity.
|
|
30
|
-
- `queueio monitor` command to monitor activity in the queueio system.
|
|
31
|
-
|
|
32
|
-
[0.2.0]: https://github.com/ryanhiebert/queueio/releases/tag/0.2
|
queueio-0.2.0/dill.pyi
DELETED
queueio-0.2.0/principles.md
DELETED
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
Principles
|
|
2
|
-
==========
|
|
3
|
-
|
|
4
|
-
There's been a great deal of momentum to embrace async functions for IO,
|
|
5
|
-
but they split the world and are confusing, especially to new developers.
|
|
6
|
-
Threads or green threads are often a better choice for concurrency
|
|
7
|
-
because it doesn't require learning about coroutines
|
|
8
|
-
before you even need concurrency.
|
|
9
|
-
|
|
10
|
-
Still, async functions are a powerful way to implement state machines.
|
|
11
|
-
Complex workflows can be represented as async functions
|
|
12
|
-
to allow thinking about things in a pull-based fashion,
|
|
13
|
-
while also scaling well for processes that must take a long time.
|
|
14
|
-
Ideally, running coroutines could be serialized and resumed,
|
|
15
|
-
perhaps even on a different machine.
|
|
16
|
-
|
|
17
|
-
This vision motivates queueio.
|
|
18
|
-
Routines need to coordinate with the worker
|
|
19
|
-
because queue-level concurrency must be carefully managed.
|
|
20
|
-
Instead of complex construction of pipelines to make this possible,
|
|
21
|
-
queueio allows you to write complex workflows
|
|
22
|
-
in a traditional imperative style.
|
|
23
|
-
|
|
24
|
-
queueio encourages you to use synchronous IO in routines.
|
|
25
|
-
The yield points in async functions are only needed
|
|
26
|
-
when a routine needs to coordinate with the worker,
|
|
27
|
-
such as to indicate that the worker can use
|
|
28
|
-
the capacity reserved for it to do other work.
|
queueio-0.2.0/pyrightconfig.json
DELETED
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
from contextlib import contextmanager
|
|
2
|
-
|
|
3
|
-
from .gather import gather as gather
|
|
4
|
-
from .pause import pause as pause
|
|
5
|
-
from .queueio import QueueIO as RealQueueIO
|
|
6
|
-
from .registry import routine as routine
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
@contextmanager
|
|
10
|
-
def activate():
|
|
11
|
-
with RealQueueIO().activate():
|
|
12
|
-
yield
|
|
@@ -1,155 +0,0 @@
|
|
|
1
|
-
from typing import Annotated
|
|
2
|
-
|
|
3
|
-
from typer import Argument
|
|
4
|
-
from typer import Typer
|
|
5
|
-
|
|
6
|
-
from .monitor import Monitor
|
|
7
|
-
from .queueio import QueueIO
|
|
8
|
-
from .queuespec import QueueSpec
|
|
9
|
-
from .worker import Worker
|
|
10
|
-
|
|
11
|
-
app = Typer()
|
|
12
|
-
|
|
13
|
-
routine_app = Typer()
|
|
14
|
-
queue_app = Typer()
|
|
15
|
-
|
|
16
|
-
app.add_typer(
|
|
17
|
-
routine_app,
|
|
18
|
-
name="routine",
|
|
19
|
-
help="A function to coordinate background execution.",
|
|
20
|
-
rich_help_panel="Entities",
|
|
21
|
-
)
|
|
22
|
-
app.add_typer(
|
|
23
|
-
queue_app,
|
|
24
|
-
name="queue",
|
|
25
|
-
help="An ordered collection of work items to process.",
|
|
26
|
-
rich_help_panel="Entities",
|
|
27
|
-
)
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
@routine_app.command("list")
|
|
31
|
-
def routine_list():
|
|
32
|
-
"""Show all registered routines."""
|
|
33
|
-
queueio = QueueIO()
|
|
34
|
-
try:
|
|
35
|
-
routines = queueio.routines()
|
|
36
|
-
|
|
37
|
-
if not routines:
|
|
38
|
-
print("No routines registered.")
|
|
39
|
-
return
|
|
40
|
-
|
|
41
|
-
# Calculate column widths
|
|
42
|
-
name_width = max(len("Name"), max(len(routine.name) for routine in routines))
|
|
43
|
-
function_paths = []
|
|
44
|
-
for routine in routines:
|
|
45
|
-
module = routine.fn.__module__
|
|
46
|
-
qualname = routine.fn.__qualname__
|
|
47
|
-
function_paths.append(f"{module}.{qualname}")
|
|
48
|
-
path_width = max(len("Path"), max(len(path) for path in function_paths))
|
|
49
|
-
|
|
50
|
-
print(f"{'Name':<{name_width}} | {'Path':<{path_width}}")
|
|
51
|
-
print(f"{'-' * name_width}-+-{'-' * path_width}")
|
|
52
|
-
for routine, path in zip(routines, function_paths, strict=False):
|
|
53
|
-
print(f"{routine.name:<{name_width}} | {path:<{path_width}}")
|
|
54
|
-
finally:
|
|
55
|
-
queueio.shutdown()
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
@app.command(rich_help_panel="Commands")
|
|
59
|
-
def monitor(raw: bool = False):
|
|
60
|
-
"""Monitor queueio events.
|
|
61
|
-
|
|
62
|
-
Show a live view of queueio activity. Use --raw for detailed event output.
|
|
63
|
-
"""
|
|
64
|
-
if raw:
|
|
65
|
-
queueio = QueueIO()
|
|
66
|
-
events = queueio.subscribe({object})
|
|
67
|
-
try:
|
|
68
|
-
while True:
|
|
69
|
-
print(events.get())
|
|
70
|
-
except KeyboardInterrupt:
|
|
71
|
-
print("Shutting down gracefully.")
|
|
72
|
-
finally:
|
|
73
|
-
queueio.shutdown()
|
|
74
|
-
else:
|
|
75
|
-
Monitor().run()
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
@app.command(rich_help_panel="Commands")
|
|
79
|
-
def run(
|
|
80
|
-
queuespec: Annotated[
|
|
81
|
-
QueueSpec,
|
|
82
|
-
Argument(
|
|
83
|
-
parser=QueueSpec.parse,
|
|
84
|
-
help="Queue configuration in format 'queue=concurrency'. "
|
|
85
|
-
"Examples: 'production=10', 'api,background=5'",
|
|
86
|
-
metavar="QUEUE[,QUEUE2,...]=CONCURRENCY",
|
|
87
|
-
),
|
|
88
|
-
],
|
|
89
|
-
):
|
|
90
|
-
"""Run a worker to process from a queue.
|
|
91
|
-
|
|
92
|
-
The worker will process invocations from the specified queue,
|
|
93
|
-
as many at a time as specified by the concurrency.
|
|
94
|
-
"""
|
|
95
|
-
queueio = QueueIO()
|
|
96
|
-
Worker(queueio, queuespec)()
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
@app.command(rich_help_panel="Commands")
|
|
100
|
-
def sync():
|
|
101
|
-
"""Sync known queues to the broker."""
|
|
102
|
-
queueio = QueueIO()
|
|
103
|
-
try:
|
|
104
|
-
routines = queueio.routines()
|
|
105
|
-
|
|
106
|
-
if not routines:
|
|
107
|
-
print("No routines registered.")
|
|
108
|
-
return
|
|
109
|
-
|
|
110
|
-
queues = sorted({routine.queue for routine in routines})
|
|
111
|
-
|
|
112
|
-
print(f"Syncing queues for {len(routines)} routine(s):")
|
|
113
|
-
for queue in queues:
|
|
114
|
-
print(f" Ensuring queue exists: {queue}")
|
|
115
|
-
queueio.create(queue=queue)
|
|
116
|
-
|
|
117
|
-
print(f"Successfully synced {len(queues)} queue(s)")
|
|
118
|
-
finally:
|
|
119
|
-
queueio.shutdown()
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
@queue_app.command("purge")
|
|
123
|
-
def queue_purge(
|
|
124
|
-
queues: Annotated[
|
|
125
|
-
str,
|
|
126
|
-
Argument(
|
|
127
|
-
help="Comma-separated list of queues to purge. "
|
|
128
|
-
"Examples: 'queueio', 'production,background'",
|
|
129
|
-
metavar="QUEUE[,QUEUE2,...]",
|
|
130
|
-
),
|
|
131
|
-
],
|
|
132
|
-
):
|
|
133
|
-
"""Purge all messages from some queues.
|
|
134
|
-
|
|
135
|
-
This will remove all pending messages from the given queues.
|
|
136
|
-
Use with caution as this operation cannot be undone.
|
|
137
|
-
"""
|
|
138
|
-
queueio = QueueIO()
|
|
139
|
-
try:
|
|
140
|
-
queue_list = [q.strip() for q in queues.split(",") if q.strip()]
|
|
141
|
-
if not queue_list:
|
|
142
|
-
print("Error: No valid queue names provided")
|
|
143
|
-
return
|
|
144
|
-
|
|
145
|
-
for queue in queue_list:
|
|
146
|
-
print(f"Purging queue: {queue}")
|
|
147
|
-
queueio.purge(queue=queue)
|
|
148
|
-
|
|
149
|
-
print(f"Successfully purged {len(queue_list)} queue(s)")
|
|
150
|
-
finally:
|
|
151
|
-
queueio.shutdown()
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
if __name__ == "__main__":
|
|
155
|
-
app()
|
queueio-0.2.0/queueio/broker.py
DELETED
|
@@ -1,45 +0,0 @@
|
|
|
1
|
-
from abc import ABC
|
|
2
|
-
from abc import abstractmethod
|
|
3
|
-
|
|
4
|
-
from .queuespec import QueueSpec
|
|
5
|
-
from .receiver import Receiver
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
class Broker(ABC):
|
|
9
|
-
"""A broker enables sending and receiving messages with a queue.
|
|
10
|
-
|
|
11
|
-
Messages sent by a broker are assumed to be idempotent, and may be
|
|
12
|
-
delivered multiple times in some conditions in order to ensure
|
|
13
|
-
at-least-once delivery.
|
|
14
|
-
"""
|
|
15
|
-
|
|
16
|
-
@classmethod
|
|
17
|
-
@abstractmethod
|
|
18
|
-
def from_uri(cls, uri: str, /):
|
|
19
|
-
"""Create a broker instance from a URI."""
|
|
20
|
-
raise NotImplementedError("Subclasses must implement this method.")
|
|
21
|
-
|
|
22
|
-
@abstractmethod
|
|
23
|
-
def enqueue(self, body: bytes, /, *, queue: str):
|
|
24
|
-
"""Enqueue a message."""
|
|
25
|
-
raise NotImplementedError("Subclasses must implement this method.")
|
|
26
|
-
|
|
27
|
-
@abstractmethod
|
|
28
|
-
def create(self, *, queue: str):
|
|
29
|
-
"""Create a queue if it doesn't exist."""
|
|
30
|
-
raise NotImplementedError("Subclasses must implement this method.")
|
|
31
|
-
|
|
32
|
-
@abstractmethod
|
|
33
|
-
def purge(self, *, queue: str):
|
|
34
|
-
"""Purge all messages from the queue."""
|
|
35
|
-
raise NotImplementedError("Subclasses must implement this method.")
|
|
36
|
-
|
|
37
|
-
@abstractmethod
|
|
38
|
-
def receive(self, queuespec: QueueSpec, /) -> Receiver:
|
|
39
|
-
"""Receive messages as specified by the QueueSpec."""
|
|
40
|
-
raise NotImplementedError("Subclasses must implement this method.")
|
|
41
|
-
|
|
42
|
-
@abstractmethod
|
|
43
|
-
def shutdown(self):
|
|
44
|
-
"""Signal the final shutdown of the broker."""
|
|
45
|
-
raise NotImplementedError("Subclasses must implement this method.")
|