PgQueuer 0.2.1__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pgqueuer-0.2.1 → pgqueuer-0.2.2}/.github/workflows/ci.yml +31 -0
- {pgqueuer-0.2.1 → pgqueuer-0.2.2}/PKG-INFO +2 -1
- {pgqueuer-0.2.1 → pgqueuer-0.2.2}/pyproject.toml +1 -0
- pgqueuer-0.2.2/src/PgQueuer/qm.py +163 -0
- {pgqueuer-0.2.1 → pgqueuer-0.2.2}/src/PgQueuer/queries.py +63 -21
- {pgqueuer-0.2.1 → pgqueuer-0.2.2}/src/PgQueuer/tm.py +8 -0
- {pgqueuer-0.2.1 → pgqueuer-0.2.2}/src/PgQueuer.egg-info/PKG-INFO +2 -1
- {pgqueuer-0.2.1 → pgqueuer-0.2.2}/src/PgQueuer.egg-info/SOURCES.txt +2 -2
- {pgqueuer-0.2.1 → pgqueuer-0.2.2}/src/PgQueuer.egg-info/requires.txt +1 -0
- {pgqueuer-0.2.1 → pgqueuer-0.2.2}/src/_version.py +2 -2
- {pgqueuer-0.2.1 → pgqueuer-0.2.2}/test/test_qm.py +39 -2
- {pgqueuer-0.2.1 → pgqueuer-0.2.2}/test/test_queries.py +3 -3
- pgqueuer-0.2.2/test/test_tm.py +44 -0
- pgqueuer-0.2.2/tools/benchmark.py +99 -0
- pgqueuer-0.2.1/src/PgQueuer/qm.py +0 -119
- pgqueuer-0.2.1/test/test_benchmark.py +0 -46
- pgqueuer-0.2.1/test/test_tm.py +0 -24
- {pgqueuer-0.2.1 → pgqueuer-0.2.2}/.github/workflows/linting.yml +0 -0
- {pgqueuer-0.2.1 → pgqueuer-0.2.2}/.github/workflows/release.yml +0 -0
- {pgqueuer-0.2.1 → pgqueuer-0.2.2}/.gitignore +0 -0
- {pgqueuer-0.2.1 → pgqueuer-0.2.2}/LICENSE +0 -0
- {pgqueuer-0.2.1 → pgqueuer-0.2.2}/README.md +0 -0
- {pgqueuer-0.2.1 → pgqueuer-0.2.2}/docker-compose.yml +0 -0
- {pgqueuer-0.2.1 → pgqueuer-0.2.2}/setup.cfg +0 -0
- {pgqueuer-0.2.1 → pgqueuer-0.2.2}/src/PgQueuer/__init__.py +0 -0
- {pgqueuer-0.2.1 → pgqueuer-0.2.2}/src/PgQueuer/__main__.py +0 -0
- {pgqueuer-0.2.1 → pgqueuer-0.2.2}/src/PgQueuer/_version.py +0 -0
- {pgqueuer-0.2.1 → pgqueuer-0.2.2}/src/PgQueuer/cli.py +0 -0
- {pgqueuer-0.2.1 → pgqueuer-0.2.2}/src/PgQueuer/logconfig.py +0 -0
- {pgqueuer-0.2.1 → pgqueuer-0.2.2}/src/PgQueuer/models.py +0 -0
- {pgqueuer-0.2.1 → pgqueuer-0.2.2}/src/PgQueuer/py.typed +0 -0
- {pgqueuer-0.2.1 → pgqueuer-0.2.2}/src/PgQueuer.egg-info/dependency_links.txt +0 -0
- {pgqueuer-0.2.1 → pgqueuer-0.2.2}/src/PgQueuer.egg-info/top_level.txt +0 -0
- {pgqueuer-0.2.1 → pgqueuer-0.2.2}/test/conftest.py +0 -0
- {pgqueuer-0.2.1 → pgqueuer-0.2.2}/test/db/Dockerfile +0 -0
- {pgqueuer-0.2.1 → pgqueuer-0.2.2}/test/db/init_db.sh +0 -0
|
@@ -7,6 +7,37 @@ on:
|
|
|
7
7
|
branches: [main]
|
|
8
8
|
|
|
9
9
|
jobs:
|
|
10
|
+
benchmark:
|
|
11
|
+
|
|
12
|
+
strategy:
|
|
13
|
+
fail-fast: false
|
|
14
|
+
matrix:
|
|
15
|
+
python-version: ["3.11"]
|
|
16
|
+
os: [ubuntu-latest]
|
|
17
|
+
|
|
18
|
+
name: PgQueuer Benchmark
|
|
19
|
+
runs-on: ${{ matrix.os }}
|
|
20
|
+
|
|
21
|
+
steps:
|
|
22
|
+
- uses: actions/checkout@v4
|
|
23
|
+
|
|
24
|
+
- name: Set up Python
|
|
25
|
+
uses: actions/setup-python@v4
|
|
26
|
+
with:
|
|
27
|
+
python-version: ${{ matrix.python-version }}
|
|
28
|
+
|
|
29
|
+
- name: Build custom PostgreSQL Docker image
|
|
30
|
+
run: |
|
|
31
|
+
docker-compose up --build -d pgq
|
|
32
|
+
|
|
33
|
+
- name: Install PgQueuer
|
|
34
|
+
run: |
|
|
35
|
+
pip install pip -U
|
|
36
|
+
pip install .[dev]
|
|
37
|
+
|
|
38
|
+
- name: Benchmark
|
|
39
|
+
run: PGUSER=testuser PGPASSWORD=testpassword PGDATABASE=testdb python3 tools/benchmark.py
|
|
40
|
+
|
|
10
41
|
ci:
|
|
11
42
|
strategy:
|
|
12
43
|
fail-fast: false
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: PgQueuer
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: PgQueuer is a Python library leveraging PostgreSQL for efficient job queuing.
|
|
5
5
|
Author: janbjorge
|
|
6
6
|
License: MIT License
|
|
@@ -26,6 +26,7 @@ Classifier: Topic :: System :: Distributed Computing
|
|
|
26
26
|
Requires-Python: >=3.10
|
|
27
27
|
Description-Content-Type: text/markdown
|
|
28
28
|
License-File: LICENSE
|
|
29
|
+
Requires-Dist: anyio>=4.0
|
|
29
30
|
Requires-Dist: asyncpg>=0.27.0
|
|
30
31
|
Requires-Dist: pgcachewatch>=0.4
|
|
31
32
|
Requires-Dist: pydantic>=2.0.0
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import dataclasses
|
|
5
|
+
import functools
|
|
6
|
+
from datetime import timedelta
|
|
7
|
+
from typing import (
|
|
8
|
+
TYPE_CHECKING,
|
|
9
|
+
Awaitable,
|
|
10
|
+
Callable,
|
|
11
|
+
TypeAlias,
|
|
12
|
+
TypeGuard,
|
|
13
|
+
TypeVar,
|
|
14
|
+
overload,
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
import anyio
|
|
18
|
+
import anyio.to_thread
|
|
19
|
+
import asyncpg
|
|
20
|
+
from pgcachewatch.listeners import PGEventQueue
|
|
21
|
+
from pgcachewatch.models import PGChannel
|
|
22
|
+
|
|
23
|
+
from .logconfig import logger
|
|
24
|
+
from .models import Job
|
|
25
|
+
from .queries import DBSettings, Queries
|
|
26
|
+
from .tm import TaskManager
|
|
27
|
+
|
|
28
|
+
if TYPE_CHECKING:
|
|
29
|
+
AsyncEntrypoint: TypeAlias = Callable[[Job], Awaitable[None]]
|
|
30
|
+
SyncEntrypoint: TypeAlias = Callable[[Job], None]
|
|
31
|
+
Entrypoint: TypeAlias = AsyncEntrypoint | SyncEntrypoint
|
|
32
|
+
T = TypeVar("T", bound=Entrypoint)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@overload
|
|
36
|
+
def is_async_callable(obj: AsyncEntrypoint) -> TypeGuard[AsyncEntrypoint]: ...
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@overload
|
|
40
|
+
def is_async_callable(obj: SyncEntrypoint) -> TypeGuard[SyncEntrypoint]: ...
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def is_async_callable(obj: object) -> bool:
|
|
44
|
+
# Inspired by:
|
|
45
|
+
# https://github.com/encode/starlette/blob/9f16bf5c25e126200701f6e04330864f4a91a898/starlette/_utils.py#L38
|
|
46
|
+
while isinstance(obj, functools.partial):
|
|
47
|
+
obj = obj.func
|
|
48
|
+
|
|
49
|
+
return asyncio.iscoroutinefunction(obj) or (
|
|
50
|
+
callable(obj) and asyncio.iscoroutinefunction(obj.__call__)
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@dataclasses.dataclass
|
|
55
|
+
class QueueManager:
|
|
56
|
+
"""
|
|
57
|
+
Manages job queues and dispatches jobs to registered entry points,
|
|
58
|
+
handling database connections and events.
|
|
59
|
+
"""
|
|
60
|
+
|
|
61
|
+
pool: asyncpg.Pool
|
|
62
|
+
queries: Queries = dataclasses.field(init=False)
|
|
63
|
+
channel: PGChannel = dataclasses.field(
|
|
64
|
+
default=PGChannel(DBSettings().channel),
|
|
65
|
+
)
|
|
66
|
+
alive: bool = dataclasses.field(
|
|
67
|
+
init=False,
|
|
68
|
+
default=True,
|
|
69
|
+
)
|
|
70
|
+
# Should registry be a weakref?
|
|
71
|
+
registry: dict[str, Entrypoint] = dataclasses.field(
|
|
72
|
+
init=False,
|
|
73
|
+
default_factory=dict,
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
def __post_init__(self) -> None:
|
|
77
|
+
"""
|
|
78
|
+
Initializes database query handlers and validates pool size upon
|
|
79
|
+
instance creation.
|
|
80
|
+
"""
|
|
81
|
+
if self.pool.get_min_size() < 1:
|
|
82
|
+
raise ValueError("... min size must be gt 1.")
|
|
83
|
+
self.queries = Queries(self.pool)
|
|
84
|
+
|
|
85
|
+
def entrypoint(self, name: str) -> Callable[[T], T]:
|
|
86
|
+
"""
|
|
87
|
+
Decorator to register a function as an entrypoint for
|
|
88
|
+
handling specific job types.
|
|
89
|
+
"""
|
|
90
|
+
|
|
91
|
+
if name in self.registry:
|
|
92
|
+
raise RuntimeError(f"{name} already in registry, name must be unique.")
|
|
93
|
+
|
|
94
|
+
def register(func: T) -> T:
|
|
95
|
+
self.registry[name] = func
|
|
96
|
+
return func
|
|
97
|
+
|
|
98
|
+
return register
|
|
99
|
+
|
|
100
|
+
async def run(
|
|
101
|
+
self,
|
|
102
|
+
dequeue_timeout: timedelta = timedelta(seconds=30),
|
|
103
|
+
) -> None:
|
|
104
|
+
"""
|
|
105
|
+
Starts the event listener and continuously dispatches jobs to
|
|
106
|
+
registered entry points until stopped.
|
|
107
|
+
"""
|
|
108
|
+
async with (
|
|
109
|
+
self.pool.acquire() as conn,
|
|
110
|
+
TaskManager() as tm,
|
|
111
|
+
):
|
|
112
|
+
listener = PGEventQueue()
|
|
113
|
+
await listener.connect(conn, self.channel)
|
|
114
|
+
|
|
115
|
+
while self.alive:
|
|
116
|
+
while self.alive and (job := await self.queries.dequeue()):
|
|
117
|
+
tm.add(asyncio.create_task(self._dispatch(job)))
|
|
118
|
+
|
|
119
|
+
try:
|
|
120
|
+
await asyncio.wait_for(
|
|
121
|
+
listener.get(),
|
|
122
|
+
timeout=dequeue_timeout.total_seconds(),
|
|
123
|
+
)
|
|
124
|
+
except asyncio.TimeoutError:
|
|
125
|
+
logger.debug(
|
|
126
|
+
"Timeout after %r without receiving an event.",
|
|
127
|
+
dequeue_timeout,
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
await conn.reset()
|
|
131
|
+
|
|
132
|
+
async def _dispatch(self, job: Job) -> None:
|
|
133
|
+
"""
|
|
134
|
+
Internal method to asynchronously handle job dispatch,
|
|
135
|
+
including exception logging and job status updates.
|
|
136
|
+
"""
|
|
137
|
+
|
|
138
|
+
logger.debug(
|
|
139
|
+
"Dispatching entrypoint/id: %s/%s",
|
|
140
|
+
job.entrypoint,
|
|
141
|
+
job.id,
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
try:
|
|
145
|
+
fn = self.registry[job.entrypoint]
|
|
146
|
+
if is_async_callable(fn):
|
|
147
|
+
await fn(job)
|
|
148
|
+
else:
|
|
149
|
+
await anyio.to_thread.run_sync(fn, job)
|
|
150
|
+
except Exception:
|
|
151
|
+
logger.exception(
|
|
152
|
+
"Exception while processing entrypoint/id: %s/%s",
|
|
153
|
+
job.entrypoint,
|
|
154
|
+
job.id,
|
|
155
|
+
)
|
|
156
|
+
await self.queries.log_job(job, "exception")
|
|
157
|
+
else:
|
|
158
|
+
logger.debug(
|
|
159
|
+
"Dispatching entrypoint/id: %s/%s - successful",
|
|
160
|
+
job.entrypoint,
|
|
161
|
+
job.id,
|
|
162
|
+
)
|
|
163
|
+
await self.queries.log_job(job, "successful")
|
|
@@ -79,7 +79,8 @@ class Queries:
|
|
|
79
79
|
entrypoint TEXT NOT NULL,
|
|
80
80
|
payload BYTEA
|
|
81
81
|
);
|
|
82
|
-
CREATE
|
|
82
|
+
CREATE INDEX ON {self.settings.queue_table} (priority ASC, id DESC)
|
|
83
|
+
INCLUDE (id) WHERE status = 'queued';
|
|
83
84
|
|
|
84
85
|
CREATE TYPE {self.settings.log_table_status_type} AS ENUM ('exception', 'successful');
|
|
85
86
|
CREATE TABLE {self.settings.log_table} (
|
|
@@ -92,16 +93,42 @@ class Queries:
|
|
|
92
93
|
);
|
|
93
94
|
|
|
94
95
|
CREATE FUNCTION {self.settings.function}() RETURNS TRIGGER AS $$
|
|
96
|
+
DECLARE
|
|
97
|
+
to_emit BOOLEAN := false; -- Flag to decide whether to emit a notification
|
|
95
98
|
BEGIN
|
|
96
|
-
|
|
97
|
-
'
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
99
|
+
-- Check operation type and set the emit flag accordingly
|
|
100
|
+
IF TG_OP = 'UPDATE' AND OLD IS DISTINCT FROM NEW THEN
|
|
101
|
+
to_emit := true;
|
|
102
|
+
ELSIF TG_OP = 'DELETE' THEN
|
|
103
|
+
to_emit := true;
|
|
104
|
+
ELSIF TG_OP = 'INSERT' THEN
|
|
105
|
+
to_emit := true;
|
|
106
|
+
ELSIF TG_OP = 'TRUNCATE' THEN
|
|
107
|
+
to_emit := true;
|
|
108
|
+
END IF;
|
|
109
|
+
|
|
110
|
+
-- Perform notification if the emit flag is set
|
|
111
|
+
IF to_emit THEN
|
|
112
|
+
PERFORM pg_notify(
|
|
113
|
+
'{self.settings.channel}',
|
|
114
|
+
json_build_object(
|
|
115
|
+
'channel', '{self.settings.channel}',
|
|
116
|
+
'operation', lower(TG_OP),
|
|
117
|
+
'sent_at', NOW(),
|
|
118
|
+
'table', TG_TABLE_NAME
|
|
119
|
+
)::text
|
|
120
|
+
);
|
|
121
|
+
END IF;
|
|
122
|
+
|
|
123
|
+
-- Return appropriate value based on the operation
|
|
124
|
+
IF TG_OP IN ('INSERT', 'UPDATE') THEN
|
|
125
|
+
RETURN NEW;
|
|
126
|
+
ELSIF TG_OP = 'DELETE' THEN
|
|
127
|
+
RETURN OLD;
|
|
128
|
+
ELSE
|
|
129
|
+
RETURN NULL; -- For TRUNCATE and other non-row-specific contexts
|
|
130
|
+
END IF;
|
|
131
|
+
|
|
105
132
|
END;
|
|
106
133
|
$$ LANGUAGE plpgsql;
|
|
107
134
|
|
|
@@ -135,13 +162,9 @@ class Queries:
|
|
|
135
162
|
"""
|
|
136
163
|
query = f"""
|
|
137
164
|
WITH next_job AS (
|
|
138
|
-
SELECT
|
|
139
|
-
FROM {self.settings.queue_table}
|
|
140
|
-
WHERE
|
|
141
|
-
SELECT FROM
|
|
142
|
-
{self.settings.queue_table} p2
|
|
143
|
-
WHERE p1.entrypoint = p2.entrypoint AND p2.status = 'picked'
|
|
144
|
-
)
|
|
165
|
+
SELECT id
|
|
166
|
+
FROM {self.settings.queue_table}
|
|
167
|
+
WHERE status = 'queued'
|
|
145
168
|
ORDER BY priority DESC, id ASC
|
|
146
169
|
FOR UPDATE SKIP LOCKED
|
|
147
170
|
LIMIT 1
|
|
@@ -158,20 +181,39 @@ class Queries:
|
|
|
158
181
|
|
|
159
182
|
async def enqueue(
|
|
160
183
|
self,
|
|
161
|
-
entrypoint: str,
|
|
162
|
-
payload: bytes | None,
|
|
163
|
-
priority: int = 0,
|
|
184
|
+
entrypoint: str | list[str],
|
|
185
|
+
payload: bytes | None | list[bytes | None],
|
|
186
|
+
priority: int | list[int] = 0,
|
|
164
187
|
) -> None:
|
|
165
188
|
"""
|
|
166
189
|
Inserts a new job into the queue with the specified
|
|
167
190
|
entrypoint, payload, and priority, marking it as 'queued'.
|
|
191
|
+
This method ensures that if any of entrypoint, payload, or priority is a list,
|
|
192
|
+
all list arguments must be of the same length.
|
|
193
|
+
|
|
194
|
+
If two or more arguments are lists, they must be the same length;
|
|
195
|
+
otherwise, an ValueError is raised.
|
|
168
196
|
"""
|
|
197
|
+
|
|
198
|
+
# If they are not lists, create single-item lists for uniform processing
|
|
199
|
+
normed_entrypoint = entrypoint if isinstance(entrypoint, list) else [entrypoint]
|
|
200
|
+
normed_payload = payload if isinstance(payload, list) else [payload]
|
|
201
|
+
normed_priority = priority if isinstance(priority, list) else [priority]
|
|
202
|
+
|
|
169
203
|
query = f"""
|
|
170
204
|
INSERT INTO {self.settings.queue_table} (priority, status, entrypoint, payload)
|
|
171
205
|
VALUES ($1, 'queued', $2, $3)
|
|
172
206
|
""" # noqa: E501
|
|
173
207
|
|
|
174
|
-
await self.pool.
|
|
208
|
+
await self.pool.executemany(
|
|
209
|
+
query,
|
|
210
|
+
zip(
|
|
211
|
+
normed_priority,
|
|
212
|
+
normed_entrypoint,
|
|
213
|
+
normed_payload,
|
|
214
|
+
strict=True,
|
|
215
|
+
),
|
|
216
|
+
)
|
|
175
217
|
|
|
176
218
|
async def clear_queue(self, entrypoint: str | list[str] | None = None) -> None:
|
|
177
219
|
"""
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
1
3
|
import asyncio
|
|
2
4
|
import dataclasses
|
|
3
5
|
|
|
@@ -21,3 +23,9 @@ class TaskManager:
|
|
|
21
23
|
"""
|
|
22
24
|
self.tasks.add(task)
|
|
23
25
|
task.add_done_callback(self.tasks.remove)
|
|
26
|
+
|
|
27
|
+
async def __aenter__(self) -> TaskManager:
|
|
28
|
+
return self
|
|
29
|
+
|
|
30
|
+
async def __aexit__(self, *_: object) -> None:
|
|
31
|
+
await asyncio.gather(*self.tasks)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: PgQueuer
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: PgQueuer is a Python library leveraging PostgreSQL for efficient job queuing.
|
|
5
5
|
Author: janbjorge
|
|
6
6
|
License: MIT License
|
|
@@ -26,6 +26,7 @@ Classifier: Topic :: System :: Distributed Computing
|
|
|
26
26
|
Requires-Python: >=3.10
|
|
27
27
|
Description-Content-Type: text/markdown
|
|
28
28
|
License-File: LICENSE
|
|
29
|
+
Requires-Dist: anyio>=4.0
|
|
29
30
|
Requires-Dist: asyncpg>=0.27.0
|
|
30
31
|
Requires-Dist: pgcachewatch>=0.4
|
|
31
32
|
Requires-Dist: pydantic>=2.0.0
|
|
@@ -23,9 +23,9 @@ src/PgQueuer.egg-info/dependency_links.txt
|
|
|
23
23
|
src/PgQueuer.egg-info/requires.txt
|
|
24
24
|
src/PgQueuer.egg-info/top_level.txt
|
|
25
25
|
test/conftest.py
|
|
26
|
-
test/test_benchmark.py
|
|
27
26
|
test/test_qm.py
|
|
28
27
|
test/test_queries.py
|
|
29
28
|
test/test_tm.py
|
|
30
29
|
test/db/Dockerfile
|
|
31
|
-
test/db/init_db.sh
|
|
30
|
+
test/db/init_db.sh
|
|
31
|
+
tools/benchmark.py
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import asyncio
|
|
2
|
+
import time
|
|
2
3
|
|
|
3
4
|
import asyncpg
|
|
4
5
|
import pytest
|
|
@@ -7,7 +8,7 @@ from PgQueuer.qm import QueueManager
|
|
|
7
8
|
from PgQueuer.queries import Queries
|
|
8
9
|
|
|
9
10
|
|
|
10
|
-
@pytest.mark.parametrize("N", (1, 2, 32
|
|
11
|
+
@pytest.mark.parametrize("N", (1, 2, 32))
|
|
11
12
|
async def test_job_queing(
|
|
12
13
|
pgpool: asyncpg.Pool,
|
|
13
14
|
N: int,
|
|
@@ -33,7 +34,7 @@ async def test_job_queing(
|
|
|
33
34
|
assert seen == list(range(N))
|
|
34
35
|
|
|
35
36
|
|
|
36
|
-
@pytest.mark.parametrize("N", (1, 2, 32
|
|
37
|
+
@pytest.mark.parametrize("N", (1, 2, 32))
|
|
37
38
|
@pytest.mark.parametrize("concurrency", (1, 2, 3, 4))
|
|
38
39
|
async def test_job_fetch(
|
|
39
40
|
pgpool: asyncpg.Pool,
|
|
@@ -66,3 +67,39 @@ async def test_job_fetch(
|
|
|
66
67
|
timeout=10,
|
|
67
68
|
)
|
|
68
69
|
assert sorted(seen) == list(range(N))
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
@pytest.mark.parametrize("N", (1, 2, 32))
|
|
73
|
+
@pytest.mark.parametrize("concurrency", (1, 2, 3, 4))
|
|
74
|
+
async def test_sync_entrypoint(
|
|
75
|
+
pgpool: asyncpg.Pool,
|
|
76
|
+
N: int,
|
|
77
|
+
concurrency: int,
|
|
78
|
+
) -> None:
|
|
79
|
+
q = Queries(pgpool)
|
|
80
|
+
qmpool = [QueueManager(pgpool) for _ in range(concurrency)]
|
|
81
|
+
seen = list[int]()
|
|
82
|
+
|
|
83
|
+
for qm in qmpool:
|
|
84
|
+
|
|
85
|
+
@qm.entrypoint("fetch")
|
|
86
|
+
def fetch(context: Job) -> None:
|
|
87
|
+
time.sleep(2) # Sim. heavy CPU/IO.
|
|
88
|
+
if context.payload is None:
|
|
89
|
+
for qm in qmpool:
|
|
90
|
+
qm.alive = False
|
|
91
|
+
return
|
|
92
|
+
assert context
|
|
93
|
+
seen.append(int(context.payload))
|
|
94
|
+
|
|
95
|
+
for n in range(N):
|
|
96
|
+
await q.enqueue("fetch", f"{n}".encode())
|
|
97
|
+
|
|
98
|
+
# Stop flag
|
|
99
|
+
await q.enqueue("fetch", None)
|
|
100
|
+
|
|
101
|
+
await asyncio.wait_for(
|
|
102
|
+
asyncio.gather(*[qm.run() for qm in qmpool]),
|
|
103
|
+
timeout=10,
|
|
104
|
+
)
|
|
105
|
+
assert sorted(seen) == list(range(N))
|
|
@@ -37,7 +37,7 @@ async def test_queries_next_jobs(
|
|
|
37
37
|
assert seen == list(range(N))
|
|
38
38
|
|
|
39
39
|
|
|
40
|
-
@pytest.mark.parametrize("N", (1, 2, 64
|
|
40
|
+
@pytest.mark.parametrize("N", (1, 2, 64))
|
|
41
41
|
@pytest.mark.parametrize("concurrency", (1, 2, 4, 16))
|
|
42
42
|
async def test_queries_next_jobs_concurrent(
|
|
43
43
|
pgpool: asyncpg.Pool,
|
|
@@ -79,7 +79,7 @@ async def test_queries_clear(pgpool: asyncpg.Pool) -> None:
|
|
|
79
79
|
assert sum(x.count for x in await q.queue_size()) == 0
|
|
80
80
|
|
|
81
81
|
|
|
82
|
-
@pytest.mark.parametrize("N", (1, 2, 64
|
|
82
|
+
@pytest.mark.parametrize("N", (1, 2, 64))
|
|
83
83
|
async def test_move_job_log(
|
|
84
84
|
pgpool: asyncpg.Pool,
|
|
85
85
|
N: int,
|
|
@@ -130,7 +130,7 @@ async def test_clear_queue(
|
|
|
130
130
|
assert sum(x.count for x in await q.queue_size()) == N - 1
|
|
131
131
|
|
|
132
132
|
|
|
133
|
-
@pytest.mark.parametrize("N", (1, 2, 64
|
|
133
|
+
@pytest.mark.parametrize("N", (1, 2, 64))
|
|
134
134
|
async def test_queue_priority(
|
|
135
135
|
pgpool: asyncpg.Pool,
|
|
136
136
|
N: int,
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
from PgQueuer import qm
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@pytest.mark.parametrize("N", (1, 2, 3, 5, 64))
|
|
8
|
+
async def test_task_manager(N: int) -> None:
|
|
9
|
+
future = asyncio.Future[None]()
|
|
10
|
+
|
|
11
|
+
tm = qm.TaskManager()
|
|
12
|
+
assert len(tm.tasks) == 0
|
|
13
|
+
|
|
14
|
+
async def waiter(future: asyncio.Future) -> None:
|
|
15
|
+
return await future
|
|
16
|
+
|
|
17
|
+
for _ in range(N):
|
|
18
|
+
tm.add(asyncio.create_task(waiter(future)))
|
|
19
|
+
|
|
20
|
+
assert len(tm.tasks) == N
|
|
21
|
+
|
|
22
|
+
future.set_result(None)
|
|
23
|
+
await asyncio.gather(*tm.tasks)
|
|
24
|
+
assert len(tm.tasks) == 0
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@pytest.mark.parametrize("N", (1, 2, 3, 5, 64))
|
|
28
|
+
async def test_task_manager_ctx_mngr(N: int) -> None:
|
|
29
|
+
async def waiter(future: asyncio.Future) -> None:
|
|
30
|
+
return await future
|
|
31
|
+
|
|
32
|
+
future = asyncio.Future[None]()
|
|
33
|
+
|
|
34
|
+
async with qm.TaskManager() as tm:
|
|
35
|
+
assert len(tm.tasks) == 0
|
|
36
|
+
|
|
37
|
+
for _ in range(N):
|
|
38
|
+
tm.add(asyncio.create_task(waiter(future)))
|
|
39
|
+
|
|
40
|
+
assert len(tm.tasks) == N
|
|
41
|
+
|
|
42
|
+
future.set_result(None)
|
|
43
|
+
|
|
44
|
+
assert len(tm.tasks) == 0
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import contextlib
|
|
3
|
+
import time
|
|
4
|
+
from datetime import timedelta
|
|
5
|
+
from typing import Callable, Generator
|
|
6
|
+
|
|
7
|
+
import asyncpg
|
|
8
|
+
from PgQueuer.models import Job
|
|
9
|
+
from PgQueuer.qm import QueueManager
|
|
10
|
+
from PgQueuer.queries import Queries
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@contextlib.contextmanager
|
|
14
|
+
def timer() -> (
|
|
15
|
+
Generator[
|
|
16
|
+
Callable[[], float],
|
|
17
|
+
None,
|
|
18
|
+
None,
|
|
19
|
+
]
|
|
20
|
+
):
|
|
21
|
+
enter = time.perf_counter()
|
|
22
|
+
done: float | None = None
|
|
23
|
+
try:
|
|
24
|
+
yield lambda: done - enter if done else time.perf_counter() - enter
|
|
25
|
+
finally:
|
|
26
|
+
done = time.perf_counter()
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
async def jobs_per_second(pool: asyncpg.Pool) -> float:
|
|
30
|
+
qm = QueueManager(pool)
|
|
31
|
+
seen = 0
|
|
32
|
+
|
|
33
|
+
@qm.entrypoint("async")
|
|
34
|
+
async def asyncfetch(job: Job) -> None:
|
|
35
|
+
nonlocal qm, seen
|
|
36
|
+
seen += 1
|
|
37
|
+
if job.payload is None:
|
|
38
|
+
qm.alive = False
|
|
39
|
+
|
|
40
|
+
@qm.entrypoint("sync")
|
|
41
|
+
def syncfetch(job: Job) -> None:
|
|
42
|
+
nonlocal qm, seen
|
|
43
|
+
seen += 1
|
|
44
|
+
if job.payload is None:
|
|
45
|
+
qm.alive = False
|
|
46
|
+
|
|
47
|
+
with timer() as elapsed:
|
|
48
|
+
await qm.run(timedelta(seconds=0))
|
|
49
|
+
|
|
50
|
+
return seen / elapsed()
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
async def benchmark(
|
|
54
|
+
entrypoint: str,
|
|
55
|
+
concurrecy: int,
|
|
56
|
+
N: int,
|
|
57
|
+
pool: asyncpg.Pool,
|
|
58
|
+
) -> None:
|
|
59
|
+
queries = Queries(pool)
|
|
60
|
+
|
|
61
|
+
await queries.clear_log()
|
|
62
|
+
await queries.clear_queue()
|
|
63
|
+
|
|
64
|
+
await queries.enqueue(
|
|
65
|
+
[entrypoint] * N,
|
|
66
|
+
[f"{n}".encode() for n in range(N)],
|
|
67
|
+
[0] * N,
|
|
68
|
+
)
|
|
69
|
+
assert sum(x.count for x in await queries.queue_size()) == N
|
|
70
|
+
|
|
71
|
+
await queries.enqueue(
|
|
72
|
+
[entrypoint] * concurrecy**2,
|
|
73
|
+
[None] * concurrecy**2,
|
|
74
|
+
[0] * concurrecy**2,
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
jobs = [jobs_per_second(pool) for _ in range(concurrecy)]
|
|
78
|
+
results = await asyncio.gather(*jobs)
|
|
79
|
+
|
|
80
|
+
print(
|
|
81
|
+
f"Entrypoint: {entrypoint:<5} ",
|
|
82
|
+
f"Concurrecy: {concurrecy:<2} "
|
|
83
|
+
f"Jobs: {N:<5} "
|
|
84
|
+
f"JPS: {sum(results)/1_000:.1f}k",
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
async def main() -> None:
|
|
89
|
+
async with asyncpg.create_pool(
|
|
90
|
+
min_size=20,
|
|
91
|
+
max_size=99,
|
|
92
|
+
) as pool:
|
|
93
|
+
for entrypoint in ("sync", "async"):
|
|
94
|
+
for concurrecy in range(1, 6):
|
|
95
|
+
await benchmark(entrypoint, concurrecy, 1_000 * concurrecy, pool)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
if __name__ == "__main__":
|
|
99
|
+
asyncio.run(main())
|
|
@@ -1,119 +0,0 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
import asyncio
|
|
4
|
-
import dataclasses
|
|
5
|
-
from typing import TYPE_CHECKING, Awaitable, Callable, TypeAlias, TypeVar
|
|
6
|
-
|
|
7
|
-
import asyncpg
|
|
8
|
-
from pgcachewatch.listeners import PGEventQueue
|
|
9
|
-
from pgcachewatch.models import PGChannel
|
|
10
|
-
|
|
11
|
-
from .logconfig import logger
|
|
12
|
-
from .models import Job
|
|
13
|
-
from .queries import DBSettings, Queries
|
|
14
|
-
from .tm import TaskManager
|
|
15
|
-
|
|
16
|
-
if TYPE_CHECKING:
|
|
17
|
-
EntrypointFn: TypeAlias = Callable[[Job], Awaitable[None]]
|
|
18
|
-
T = TypeVar("T", bound=EntrypointFn)
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
@dataclasses.dataclass
|
|
22
|
-
class QueueManager:
|
|
23
|
-
"""
|
|
24
|
-
Manages job queues and dispatches jobs to registered entry points,
|
|
25
|
-
handling database connections and events.
|
|
26
|
-
"""
|
|
27
|
-
|
|
28
|
-
pool: asyncpg.Pool
|
|
29
|
-
queries: Queries = dataclasses.field(init=False)
|
|
30
|
-
|
|
31
|
-
channel: PGChannel = dataclasses.field(
|
|
32
|
-
default=PGChannel(DBSettings().channel),
|
|
33
|
-
init=False,
|
|
34
|
-
)
|
|
35
|
-
alive: bool = dataclasses.field(
|
|
36
|
-
init=False,
|
|
37
|
-
default=True,
|
|
38
|
-
)
|
|
39
|
-
# Should registry be a weakref?
|
|
40
|
-
registry: dict[str, EntrypointFn] = dataclasses.field(
|
|
41
|
-
init=False,
|
|
42
|
-
default_factory=dict,
|
|
43
|
-
)
|
|
44
|
-
tm: TaskManager = dataclasses.field(
|
|
45
|
-
init=False,
|
|
46
|
-
default_factory=TaskManager,
|
|
47
|
-
)
|
|
48
|
-
|
|
49
|
-
def __post_init__(self) -> None:
|
|
50
|
-
"""
|
|
51
|
-
Initializes database query handlers and validates pool size upon
|
|
52
|
-
instance creation.
|
|
53
|
-
"""
|
|
54
|
-
if self.pool.get_min_size() < 1:
|
|
55
|
-
raise ValueError("... min size must be gt 1.")
|
|
56
|
-
self.queries = Queries(self.pool)
|
|
57
|
-
|
|
58
|
-
def entrypoint(self, name: str) -> Callable[[T], T]:
|
|
59
|
-
"""
|
|
60
|
-
Decorator to register a function as an entrypoint for
|
|
61
|
-
handling specific job types.
|
|
62
|
-
"""
|
|
63
|
-
|
|
64
|
-
if name in self.registry:
|
|
65
|
-
raise RuntimeError(f"{name} already in registry, name must be unique.")
|
|
66
|
-
|
|
67
|
-
def register(func: T) -> T:
|
|
68
|
-
self.registry[name] = func
|
|
69
|
-
return func
|
|
70
|
-
|
|
71
|
-
return register
|
|
72
|
-
|
|
73
|
-
async def run(self) -> None:
|
|
74
|
-
"""
|
|
75
|
-
Starts the event listener and continuously dispatches jobs to
|
|
76
|
-
registered entry points until stopped.
|
|
77
|
-
"""
|
|
78
|
-
async with self.pool.acquire() as conn:
|
|
79
|
-
listener = PGEventQueue()
|
|
80
|
-
await listener.connect(conn, self.channel)
|
|
81
|
-
|
|
82
|
-
while self.alive:
|
|
83
|
-
while job := await self.queries.dequeue():
|
|
84
|
-
self._dispatch(job)
|
|
85
|
-
await listener.get()
|
|
86
|
-
|
|
87
|
-
await asyncio.gather(*self.tm.tasks)
|
|
88
|
-
await conn.reset()
|
|
89
|
-
|
|
90
|
-
def _dispatch(self, job: Job) -> None:
|
|
91
|
-
"""
|
|
92
|
-
Internal method to asynchronously handle job dispatch,
|
|
93
|
-
including exception logging and job status updates.
|
|
94
|
-
"""
|
|
95
|
-
|
|
96
|
-
async def runit() -> None:
|
|
97
|
-
logger.debug(
|
|
98
|
-
"Dispatching entrypoint/id: %s/%s",
|
|
99
|
-
job.entrypoint,
|
|
100
|
-
job.id,
|
|
101
|
-
)
|
|
102
|
-
try:
|
|
103
|
-
await self.registry[job.entrypoint](job)
|
|
104
|
-
except Exception:
|
|
105
|
-
logger.exception(
|
|
106
|
-
"Exception while processing entrypoint/id: %s/%s",
|
|
107
|
-
job.entrypoint,
|
|
108
|
-
job.id,
|
|
109
|
-
)
|
|
110
|
-
await self.queries.log_job(job, "exception")
|
|
111
|
-
else:
|
|
112
|
-
logger.debug(
|
|
113
|
-
"Dispatching entrypoint/id: %s/%s - successful",
|
|
114
|
-
job.entrypoint,
|
|
115
|
-
job.id,
|
|
116
|
-
)
|
|
117
|
-
await self.queries.log_job(job, "successful")
|
|
118
|
-
|
|
119
|
-
self.tm.add(asyncio.create_task(runit()))
|
|
@@ -1,46 +0,0 @@
|
|
|
1
|
-
import asyncio
|
|
2
|
-
import time
|
|
3
|
-
|
|
4
|
-
import asyncpg
|
|
5
|
-
import pytest
|
|
6
|
-
from PgQueuer.models import Job
|
|
7
|
-
from PgQueuer.qm import QueueManager
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
async def jobs_per_second(
|
|
11
|
-
N: int,
|
|
12
|
-
pool: asyncpg.Pool,
|
|
13
|
-
) -> float:
|
|
14
|
-
qm = QueueManager(pool)
|
|
15
|
-
|
|
16
|
-
@qm.entrypoint("fetch")
|
|
17
|
-
async def process_message(job: Job) -> None:
|
|
18
|
-
if job.payload is None:
|
|
19
|
-
qm.alive = False
|
|
20
|
-
|
|
21
|
-
enter = time.perf_counter()
|
|
22
|
-
await qm.run()
|
|
23
|
-
done = time.perf_counter()
|
|
24
|
-
return N / (done - enter)
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
@pytest.mark.parametrize("N", (100, 500, 1_000))
|
|
28
|
-
@pytest.mark.parametrize("concurrecy", (1, 2, 3, 4, 5))
|
|
29
|
-
async def test_benchmark(
|
|
30
|
-
concurrecy: int,
|
|
31
|
-
N: int,
|
|
32
|
-
pgpool: asyncpg.Pool,
|
|
33
|
-
) -> None:
|
|
34
|
-
qm = QueueManager(pgpool)
|
|
35
|
-
|
|
36
|
-
for n in range(N):
|
|
37
|
-
await qm.queries.enqueue("fetch", f"{n}".encode())
|
|
38
|
-
|
|
39
|
-
# Hacky, must fine a bether way to shutdown.
|
|
40
|
-
for n in range(concurrecy**2):
|
|
41
|
-
await qm.queries.enqueue("fetch", None)
|
|
42
|
-
|
|
43
|
-
jobs = [jobs_per_second(N, pgpool) for _ in range(concurrecy)]
|
|
44
|
-
jps = sum(await asyncio.wait_for(asyncio.gather(*jobs), timeout=30))
|
|
45
|
-
|
|
46
|
-
print(f"Concurrecy: {concurrecy} Jobs: {N}, Jobs per second: {jps:.1f}")
|
pgqueuer-0.2.1/test/test_tm.py
DELETED
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
import asyncio
|
|
2
|
-
|
|
3
|
-
import pytest
|
|
4
|
-
from PgQueuer import qm
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
@pytest.mark.parametrize("N", (1, 2, 64, 1024))
|
|
8
|
-
async def test_task_manager(N: int) -> None:
|
|
9
|
-
future = asyncio.Future[None]()
|
|
10
|
-
|
|
11
|
-
tm = qm.TaskManager()
|
|
12
|
-
assert len(tm.tasks) == 0
|
|
13
|
-
|
|
14
|
-
async def waiter(future: asyncio.Future) -> None:
|
|
15
|
-
return await future
|
|
16
|
-
|
|
17
|
-
for _ in range(N):
|
|
18
|
-
tm.add(asyncio.create_task(waiter(future)))
|
|
19
|
-
|
|
20
|
-
assert len(tm.tasks) == N
|
|
21
|
-
|
|
22
|
-
future.set_result(None)
|
|
23
|
-
await asyncio.gather(*tm.tasks)
|
|
24
|
-
assert len(tm.tasks) == 0
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|