PgQueuer 0.2.3__tar.gz → 0.2.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pgqueuer-0.2.3 → pgqueuer-0.2.4}/PKG-INFO +1 -1
- {pgqueuer-0.2.3 → pgqueuer-0.2.4}/src/PgQueuer/cli.py +20 -4
- {pgqueuer-0.2.3 → pgqueuer-0.2.4}/src/PgQueuer/qm.py +1 -2
- {pgqueuer-0.2.3 → pgqueuer-0.2.4}/src/PgQueuer/queries.py +1 -1
- {pgqueuer-0.2.3 → pgqueuer-0.2.4}/src/PgQueuer.egg-info/PKG-INFO +1 -1
- {pgqueuer-0.2.3 → pgqueuer-0.2.4}/src/_version.py +2 -2
- pgqueuer-0.2.4/tools/benchmark.py +100 -0
- pgqueuer-0.2.3/tools/benchmark.py +0 -101
- {pgqueuer-0.2.3 → pgqueuer-0.2.4}/.github/workflows/ci.yml +0 -0
- {pgqueuer-0.2.3 → pgqueuer-0.2.4}/.github/workflows/linting.yml +0 -0
- {pgqueuer-0.2.3 → pgqueuer-0.2.4}/.github/workflows/release.yml +0 -0
- {pgqueuer-0.2.3 → pgqueuer-0.2.4}/.gitignore +0 -0
- {pgqueuer-0.2.3 → pgqueuer-0.2.4}/LICENSE +0 -0
- {pgqueuer-0.2.3 → pgqueuer-0.2.4}/README.md +0 -0
- {pgqueuer-0.2.3 → pgqueuer-0.2.4}/docker-compose.yml +0 -0
- {pgqueuer-0.2.3 → pgqueuer-0.2.4}/pyproject.toml +0 -0
- {pgqueuer-0.2.3 → pgqueuer-0.2.4}/setup.cfg +0 -0
- {pgqueuer-0.2.3 → pgqueuer-0.2.4}/src/PgQueuer/__init__.py +0 -0
- {pgqueuer-0.2.3 → pgqueuer-0.2.4}/src/PgQueuer/__main__.py +0 -0
- {pgqueuer-0.2.3 → pgqueuer-0.2.4}/src/PgQueuer/_version.py +0 -0
- {pgqueuer-0.2.3 → pgqueuer-0.2.4}/src/PgQueuer/logconfig.py +0 -0
- {pgqueuer-0.2.3 → pgqueuer-0.2.4}/src/PgQueuer/models.py +0 -0
- {pgqueuer-0.2.3 → pgqueuer-0.2.4}/src/PgQueuer/py.typed +0 -0
- {pgqueuer-0.2.3 → pgqueuer-0.2.4}/src/PgQueuer/tm.py +0 -0
- {pgqueuer-0.2.3 → pgqueuer-0.2.4}/src/PgQueuer.egg-info/SOURCES.txt +0 -0
- {pgqueuer-0.2.3 → pgqueuer-0.2.4}/src/PgQueuer.egg-info/dependency_links.txt +0 -0
- {pgqueuer-0.2.3 → pgqueuer-0.2.4}/src/PgQueuer.egg-info/requires.txt +0 -0
- {pgqueuer-0.2.3 → pgqueuer-0.2.4}/src/PgQueuer.egg-info/top_level.txt +0 -0
- {pgqueuer-0.2.3 → pgqueuer-0.2.4}/test/conftest.py +0 -0
- {pgqueuer-0.2.3 → pgqueuer-0.2.4}/test/db/Dockerfile +0 -0
- {pgqueuer-0.2.3 → pgqueuer-0.2.4}/test/db/init_db.sh +0 -0
- {pgqueuer-0.2.3 → pgqueuer-0.2.4}/test/test_qm.py +0 -0
- {pgqueuer-0.2.3 → pgqueuer-0.2.4}/test/test_queries.py +0 -0
- {pgqueuer-0.2.3 → pgqueuer-0.2.4}/test/test_tm.py +0 -0
|
@@ -4,13 +4,16 @@ import os
|
|
|
4
4
|
from datetime import timedelta
|
|
5
5
|
|
|
6
6
|
import asyncpg
|
|
7
|
-
from tabulate import tabulate
|
|
7
|
+
from tabulate import tabulate, tabulate_formats
|
|
8
8
|
|
|
9
9
|
from PgQueuer.models import LogStatistics
|
|
10
10
|
from PgQueuer.queries import Queries
|
|
11
11
|
|
|
12
12
|
|
|
13
|
-
async def display_stats(
|
|
13
|
+
async def display_stats(
|
|
14
|
+
log_stats: list[LogStatistics],
|
|
15
|
+
tablefmt: str,
|
|
16
|
+
) -> None:
|
|
14
17
|
print(
|
|
15
18
|
tabulate(
|
|
16
19
|
[
|
|
@@ -32,7 +35,7 @@ async def display_stats(log_stats: list[LogStatistics]) -> None:
|
|
|
32
35
|
"Status",
|
|
33
36
|
"Priority",
|
|
34
37
|
],
|
|
35
|
-
tablefmt=
|
|
38
|
+
tablefmt=tablefmt,
|
|
36
39
|
)
|
|
37
40
|
)
|
|
38
41
|
|
|
@@ -41,12 +44,13 @@ async def fetch_and_dispay(
|
|
|
41
44
|
queries: Queries,
|
|
42
45
|
interval: timedelta | None,
|
|
43
46
|
tail: int,
|
|
47
|
+
tablefmt: str,
|
|
44
48
|
) -> None:
|
|
45
49
|
clear_and_home = "\033[2J\033[H"
|
|
46
50
|
while True:
|
|
47
51
|
print(clear_and_home, end="")
|
|
48
52
|
stats = await queries.log_statistics(tail) # Fetch stats once and reuse
|
|
49
|
-
await display_stats(stats)
|
|
53
|
+
await display_stats(stats, tablefmt)
|
|
50
54
|
if interval is None:
|
|
51
55
|
return
|
|
52
56
|
await asyncio.sleep(interval.total_seconds())
|
|
@@ -165,6 +169,15 @@ def cliparser() -> argparse.Namespace:
|
|
|
165
169
|
type=int,
|
|
166
170
|
default=25,
|
|
167
171
|
)
|
|
172
|
+
dashboardparser.add_argument(
|
|
173
|
+
"--table-format",
|
|
174
|
+
default="pretty",
|
|
175
|
+
help=(
|
|
176
|
+
"Specify the format of the table used to display statistics. "
|
|
177
|
+
"Options are provided by the tabulate library."
|
|
178
|
+
),
|
|
179
|
+
choices=tabulate_formats,
|
|
180
|
+
)
|
|
168
181
|
|
|
169
182
|
return parser.parse_args()
|
|
170
183
|
|
|
@@ -186,6 +199,8 @@ async def main() -> None:
|
|
|
186
199
|
port=parsed.pg_port,
|
|
187
200
|
user=parsed.pg_user,
|
|
188
201
|
host=parsed.pg_host,
|
|
202
|
+
max_size=1,
|
|
203
|
+
min_size=0,
|
|
189
204
|
) as pool:
|
|
190
205
|
queries = Queries(pool)
|
|
191
206
|
match parsed.command:
|
|
@@ -198,4 +213,5 @@ async def main() -> None:
|
|
|
198
213
|
queries,
|
|
199
214
|
parsed.interval,
|
|
200
215
|
parsed.tail,
|
|
216
|
+
parsed.table_format,
|
|
201
217
|
)
|
|
@@ -111,7 +111,6 @@ class QueueManager:
|
|
|
111
111
|
Continuously listens for events and dispatches jobs. Manages connections and
|
|
112
112
|
tasks, logs timeouts, and resets connections upon termination.
|
|
113
113
|
"""
|
|
114
|
-
|
|
115
114
|
async with (
|
|
116
115
|
self.pool.acquire() as conn,
|
|
117
116
|
TaskManager() as tm,
|
|
@@ -120,7 +119,7 @@ class QueueManager:
|
|
|
120
119
|
await listener.connect(conn, self.channel)
|
|
121
120
|
|
|
122
121
|
while self.alive:
|
|
123
|
-
while job := await self.queries.dequeue():
|
|
122
|
+
while self.alive and (job := await self.queries.dequeue()):
|
|
124
123
|
tm.add(asyncio.create_task(self._dispatch(job)))
|
|
125
124
|
|
|
126
125
|
try:
|
|
@@ -189,7 +189,7 @@ class Queries:
|
|
|
189
189
|
async def enqueue(
|
|
190
190
|
self,
|
|
191
191
|
entrypoint: str | list[str],
|
|
192
|
-
payload: bytes | None | list[bytes | None],
|
|
192
|
+
payload: bytes | None | list[bytes] | list[None] | list[bytes | None],
|
|
193
193
|
priority: int | list[int] = 0,
|
|
194
194
|
) -> None:
|
|
195
195
|
"""
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import asyncio
|
|
3
|
+
import contextlib
|
|
4
|
+
import random
|
|
5
|
+
import time
|
|
6
|
+
from datetime import datetime, timedelta
|
|
7
|
+
from typing import Callable, Generator
|
|
8
|
+
|
|
9
|
+
import asyncpg
|
|
10
|
+
from PgQueuer.models import Job
|
|
11
|
+
from PgQueuer.qm import QueueManager
|
|
12
|
+
from PgQueuer.queries import Queries
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@contextlib.contextmanager
|
|
16
|
+
def execution_timer() -> (
|
|
17
|
+
Generator[
|
|
18
|
+
Callable[[], timedelta],
|
|
19
|
+
None,
|
|
20
|
+
None,
|
|
21
|
+
]
|
|
22
|
+
):
|
|
23
|
+
enter = time.perf_counter()
|
|
24
|
+
done: float | None = None
|
|
25
|
+
try:
|
|
26
|
+
yield lambda: timedelta(seconds=((done or time.perf_counter()) - enter))
|
|
27
|
+
finally:
|
|
28
|
+
done = time.perf_counter()
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
async def run_qm(pool: asyncpg.Pool) -> None:
|
|
32
|
+
qm = QueueManager(pool)
|
|
33
|
+
|
|
34
|
+
@qm.entrypoint("async")
|
|
35
|
+
async def asyncfetch(job: Job) -> None:
|
|
36
|
+
if job.payload is None:
|
|
37
|
+
qm.alive = False
|
|
38
|
+
|
|
39
|
+
@qm.entrypoint("sync")
|
|
40
|
+
def syncfetch(job: Job) -> None:
|
|
41
|
+
if job.payload is None:
|
|
42
|
+
qm.alive = False
|
|
43
|
+
|
|
44
|
+
await qm.run(timedelta(seconds=0))
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
async def main() -> None:
|
|
48
|
+
parser = argparse.ArgumentParser(
|
|
49
|
+
description="Run the job processing benchmark.",
|
|
50
|
+
)
|
|
51
|
+
parser.add_argument(
|
|
52
|
+
"-t",
|
|
53
|
+
"--time-limit",
|
|
54
|
+
type=lambda x: timedelta(seconds=float(x)),
|
|
55
|
+
default=timedelta(seconds=10),
|
|
56
|
+
help="Run the benchmark for a limited number of seconds. Default is 10",
|
|
57
|
+
)
|
|
58
|
+
args = parser.parse_args()
|
|
59
|
+
start = datetime.now()
|
|
60
|
+
|
|
61
|
+
async with asyncpg.create_pool() as pool:
|
|
62
|
+
queries = Queries(pool)
|
|
63
|
+
await queries.clear_log()
|
|
64
|
+
while True:
|
|
65
|
+
for concurrecy in range(1, 6):
|
|
66
|
+
N = concurrecy * 1_000
|
|
67
|
+
|
|
68
|
+
await queries.clear_queue()
|
|
69
|
+
|
|
70
|
+
entrypoints = ["sync"] * (N // 2) + ["async"] * (N // 2)
|
|
71
|
+
random.shuffle(entrypoints)
|
|
72
|
+
payloads = [f"{n}".encode() for n in range(N)]
|
|
73
|
+
priorities = [0] * N
|
|
74
|
+
await queries.enqueue(entrypoints, payloads, priorities)
|
|
75
|
+
await queries.enqueue(
|
|
76
|
+
["async"] * concurrecy**2,
|
|
77
|
+
[None] * concurrecy**2,
|
|
78
|
+
[0] * concurrecy**2,
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
with execution_timer() as elapsed:
|
|
82
|
+
await asyncio.gather(
|
|
83
|
+
*[run_qm(pool) for _ in range(concurrecy)],
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
elapsed_total_seconds = elapsed().total_seconds()
|
|
87
|
+
jps = (N - len(await queries.queue_size())) / elapsed_total_seconds
|
|
88
|
+
|
|
89
|
+
print(
|
|
90
|
+
f"Concurrecy: {concurrecy:<2} ",
|
|
91
|
+
f"Jobs: {N:<5} ",
|
|
92
|
+
f"JPS: {(jps)/1_000:.1f}k",
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
if timedelta(seconds=0) < args.time_limit < datetime.now() - start:
|
|
96
|
+
return
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
if __name__ == "__main__":
|
|
100
|
+
asyncio.run(main())
|
|
@@ -1,101 +0,0 @@
|
|
|
1
|
-
import asyncio
|
|
2
|
-
import contextlib
|
|
3
|
-
import time
|
|
4
|
-
from datetime import timedelta
|
|
5
|
-
from typing import Callable, Generator
|
|
6
|
-
|
|
7
|
-
import asyncpg
|
|
8
|
-
from PgQueuer.models import Job
|
|
9
|
-
from PgQueuer.qm import QueueManager
|
|
10
|
-
from PgQueuer.queries import Queries
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
@contextlib.contextmanager
|
|
14
|
-
def timer() -> (
|
|
15
|
-
Generator[
|
|
16
|
-
Callable[[], float],
|
|
17
|
-
None,
|
|
18
|
-
None,
|
|
19
|
-
]
|
|
20
|
-
):
|
|
21
|
-
enter = time.perf_counter()
|
|
22
|
-
done: float | None = None
|
|
23
|
-
try:
|
|
24
|
-
yield lambda: done - enter if done else time.perf_counter() - enter
|
|
25
|
-
finally:
|
|
26
|
-
done = time.perf_counter()
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
async def jobs_per_second(pool: asyncpg.Pool) -> float:
|
|
30
|
-
qm = QueueManager(pool)
|
|
31
|
-
seen = 0
|
|
32
|
-
|
|
33
|
-
@qm.entrypoint("async")
|
|
34
|
-
async def asyncfetch(job: Job) -> None:
|
|
35
|
-
nonlocal seen
|
|
36
|
-
seen += 1
|
|
37
|
-
if job.payload is None:
|
|
38
|
-
qm.alive = False
|
|
39
|
-
|
|
40
|
-
@qm.entrypoint("sync")
|
|
41
|
-
def syncfetch(job: Job) -> None:
|
|
42
|
-
nonlocal seen
|
|
43
|
-
seen += 1
|
|
44
|
-
if job.payload is None:
|
|
45
|
-
qm.alive = False
|
|
46
|
-
|
|
47
|
-
with timer() as elapsed:
|
|
48
|
-
await qm.run(timedelta(seconds=0))
|
|
49
|
-
|
|
50
|
-
return seen / elapsed()
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
async def benchmark(
|
|
54
|
-
entrypoint: str,
|
|
55
|
-
concurrecy: int,
|
|
56
|
-
N: int,
|
|
57
|
-
pool: asyncpg.Pool,
|
|
58
|
-
) -> None:
|
|
59
|
-
queries = Queries(pool)
|
|
60
|
-
await queries.enqueue(
|
|
61
|
-
[entrypoint] * N,
|
|
62
|
-
[f"{n}".encode() for n in range(N)],
|
|
63
|
-
[0] * N,
|
|
64
|
-
)
|
|
65
|
-
|
|
66
|
-
await queries.enqueue(
|
|
67
|
-
[entrypoint] * concurrecy**2,
|
|
68
|
-
[None] * concurrecy**2,
|
|
69
|
-
[0] * concurrecy**2,
|
|
70
|
-
)
|
|
71
|
-
|
|
72
|
-
jps = sum(
|
|
73
|
-
await asyncio.gather(
|
|
74
|
-
*[jobs_per_second(pool) for _ in range(concurrecy)],
|
|
75
|
-
)
|
|
76
|
-
)
|
|
77
|
-
|
|
78
|
-
print(
|
|
79
|
-
f"Entrypoint: {entrypoint:<5} ",
|
|
80
|
-
f"Concurrecy: {concurrecy:<2} ",
|
|
81
|
-
f"Jobs: {N:<5} ",
|
|
82
|
-
f"JPS: {jps/1_000:.1f}k",
|
|
83
|
-
)
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
async def main() -> None:
|
|
87
|
-
async with asyncpg.create_pool(
|
|
88
|
-
min_size=20,
|
|
89
|
-
max_size=99,
|
|
90
|
-
) as pool:
|
|
91
|
-
queries = Queries(pool)
|
|
92
|
-
await queries.clear_log()
|
|
93
|
-
await queries.clear_queue()
|
|
94
|
-
for entrypoint in ("sync", "async"):
|
|
95
|
-
for concurrecy in range(1, 6):
|
|
96
|
-
await benchmark(entrypoint, concurrecy, concurrecy * 1_000, pool)
|
|
97
|
-
await asyncio.sleep(0)
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
if __name__ == "__main__":
|
|
101
|
-
asyncio.run(main())
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|