PgQueuer 0.2.3__tar.gz → 0.2.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. {pgqueuer-0.2.3 → pgqueuer-0.2.4}/PKG-INFO +1 -1
  2. {pgqueuer-0.2.3 → pgqueuer-0.2.4}/src/PgQueuer/cli.py +20 -4
  3. {pgqueuer-0.2.3 → pgqueuer-0.2.4}/src/PgQueuer/qm.py +1 -2
  4. {pgqueuer-0.2.3 → pgqueuer-0.2.4}/src/PgQueuer/queries.py +1 -1
  5. {pgqueuer-0.2.3 → pgqueuer-0.2.4}/src/PgQueuer.egg-info/PKG-INFO +1 -1
  6. {pgqueuer-0.2.3 → pgqueuer-0.2.4}/src/_version.py +2 -2
  7. pgqueuer-0.2.4/tools/benchmark.py +100 -0
  8. pgqueuer-0.2.3/tools/benchmark.py +0 -101
  9. {pgqueuer-0.2.3 → pgqueuer-0.2.4}/.github/workflows/ci.yml +0 -0
  10. {pgqueuer-0.2.3 → pgqueuer-0.2.4}/.github/workflows/linting.yml +0 -0
  11. {pgqueuer-0.2.3 → pgqueuer-0.2.4}/.github/workflows/release.yml +0 -0
  12. {pgqueuer-0.2.3 → pgqueuer-0.2.4}/.gitignore +0 -0
  13. {pgqueuer-0.2.3 → pgqueuer-0.2.4}/LICENSE +0 -0
  14. {pgqueuer-0.2.3 → pgqueuer-0.2.4}/README.md +0 -0
  15. {pgqueuer-0.2.3 → pgqueuer-0.2.4}/docker-compose.yml +0 -0
  16. {pgqueuer-0.2.3 → pgqueuer-0.2.4}/pyproject.toml +0 -0
  17. {pgqueuer-0.2.3 → pgqueuer-0.2.4}/setup.cfg +0 -0
  18. {pgqueuer-0.2.3 → pgqueuer-0.2.4}/src/PgQueuer/__init__.py +0 -0
  19. {pgqueuer-0.2.3 → pgqueuer-0.2.4}/src/PgQueuer/__main__.py +0 -0
  20. {pgqueuer-0.2.3 → pgqueuer-0.2.4}/src/PgQueuer/_version.py +0 -0
  21. {pgqueuer-0.2.3 → pgqueuer-0.2.4}/src/PgQueuer/logconfig.py +0 -0
  22. {pgqueuer-0.2.3 → pgqueuer-0.2.4}/src/PgQueuer/models.py +0 -0
  23. {pgqueuer-0.2.3 → pgqueuer-0.2.4}/src/PgQueuer/py.typed +0 -0
  24. {pgqueuer-0.2.3 → pgqueuer-0.2.4}/src/PgQueuer/tm.py +0 -0
  25. {pgqueuer-0.2.3 → pgqueuer-0.2.4}/src/PgQueuer.egg-info/SOURCES.txt +0 -0
  26. {pgqueuer-0.2.3 → pgqueuer-0.2.4}/src/PgQueuer.egg-info/dependency_links.txt +0 -0
  27. {pgqueuer-0.2.3 → pgqueuer-0.2.4}/src/PgQueuer.egg-info/requires.txt +0 -0
  28. {pgqueuer-0.2.3 → pgqueuer-0.2.4}/src/PgQueuer.egg-info/top_level.txt +0 -0
  29. {pgqueuer-0.2.3 → pgqueuer-0.2.4}/test/conftest.py +0 -0
  30. {pgqueuer-0.2.3 → pgqueuer-0.2.4}/test/db/Dockerfile +0 -0
  31. {pgqueuer-0.2.3 → pgqueuer-0.2.4}/test/db/init_db.sh +0 -0
  32. {pgqueuer-0.2.3 → pgqueuer-0.2.4}/test/test_qm.py +0 -0
  33. {pgqueuer-0.2.3 → pgqueuer-0.2.4}/test/test_queries.py +0 -0
  34. {pgqueuer-0.2.3 → pgqueuer-0.2.4}/test/test_tm.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: PgQueuer
3
- Version: 0.2.3
3
+ Version: 0.2.4
4
4
  Summary: PgQueuer is a Python library leveraging PostgreSQL for efficient job queuing.
5
5
  Author: janbjorge
6
6
  License: MIT License
@@ -4,13 +4,16 @@ import os
4
4
  from datetime import timedelta
5
5
 
6
6
  import asyncpg
7
- from tabulate import tabulate
7
+ from tabulate import tabulate, tabulate_formats
8
8
 
9
9
  from PgQueuer.models import LogStatistics
10
10
  from PgQueuer.queries import Queries
11
11
 
12
12
 
13
- async def display_stats(log_stats: list[LogStatistics]) -> None:
13
+ async def display_stats(
14
+ log_stats: list[LogStatistics],
15
+ tablefmt: str,
16
+ ) -> None:
14
17
  print(
15
18
  tabulate(
16
19
  [
@@ -32,7 +35,7 @@ async def display_stats(log_stats: list[LogStatistics]) -> None:
32
35
  "Status",
33
36
  "Priority",
34
37
  ],
35
- tablefmt="pretty",
38
+ tablefmt=tablefmt,
36
39
  )
37
40
  )
38
41
 
@@ -41,12 +44,13 @@ async def fetch_and_dispay(
41
44
  queries: Queries,
42
45
  interval: timedelta | None,
43
46
  tail: int,
47
+ tablefmt: str,
44
48
  ) -> None:
45
49
  clear_and_home = "\033[2J\033[H"
46
50
  while True:
47
51
  print(clear_and_home, end="")
48
52
  stats = await queries.log_statistics(tail) # Fetch stats once and reuse
49
- await display_stats(stats)
53
+ await display_stats(stats, tablefmt)
50
54
  if interval is None:
51
55
  return
52
56
  await asyncio.sleep(interval.total_seconds())
@@ -165,6 +169,15 @@ def cliparser() -> argparse.Namespace:
165
169
  type=int,
166
170
  default=25,
167
171
  )
172
+ dashboardparser.add_argument(
173
+ "--table-format",
174
+ default="pretty",
175
+ help=(
176
+ "Specify the format of the table used to display statistics. "
177
+ "Options are provided by the tabulate library."
178
+ ),
179
+ choices=tabulate_formats,
180
+ )
168
181
 
169
182
  return parser.parse_args()
170
183
 
@@ -186,6 +199,8 @@ async def main() -> None:
186
199
  port=parsed.pg_port,
187
200
  user=parsed.pg_user,
188
201
  host=parsed.pg_host,
202
+ max_size=1,
203
+ min_size=0,
189
204
  ) as pool:
190
205
  queries = Queries(pool)
191
206
  match parsed.command:
@@ -198,4 +213,5 @@ async def main() -> None:
198
213
  queries,
199
214
  parsed.interval,
200
215
  parsed.tail,
216
+ parsed.table_format,
201
217
  )
@@ -111,7 +111,6 @@ class QueueManager:
111
111
  Continuously listens for events and dispatches jobs. Manages connections and
112
112
  tasks, logs timeouts, and resets connections upon termination.
113
113
  """
114
-
115
114
  async with (
116
115
  self.pool.acquire() as conn,
117
116
  TaskManager() as tm,
@@ -120,7 +119,7 @@ class QueueManager:
120
119
  await listener.connect(conn, self.channel)
121
120
 
122
121
  while self.alive:
123
- while job := await self.queries.dequeue():
122
+ while self.alive and (job := await self.queries.dequeue()):
124
123
  tm.add(asyncio.create_task(self._dispatch(job)))
125
124
 
126
125
  try:
@@ -189,7 +189,7 @@ class Queries:
189
189
  async def enqueue(
190
190
  self,
191
191
  entrypoint: str | list[str],
192
- payload: bytes | None | list[bytes | None],
192
+ payload: bytes | None | list[bytes] | list[None] | list[bytes | None],
193
193
  priority: int | list[int] = 0,
194
194
  ) -> None:
195
195
  """
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: PgQueuer
3
- Version: 0.2.3
3
+ Version: 0.2.4
4
4
  Summary: PgQueuer is a Python library leveraging PostgreSQL for efficient job queuing.
5
5
  Author: janbjorge
6
6
  License: MIT License
@@ -12,5 +12,5 @@ __version__: str
12
12
  __version_tuple__: VERSION_TUPLE
13
13
  version_tuple: VERSION_TUPLE
14
14
 
15
- __version__ = version = '0.2.3'
16
- __version_tuple__ = version_tuple = (0, 2, 3)
15
+ __version__ = version = '0.2.4'
16
+ __version_tuple__ = version_tuple = (0, 2, 4)
@@ -0,0 +1,100 @@
1
+ import argparse
2
+ import asyncio
3
+ import contextlib
4
+ import random
5
+ import time
6
+ from datetime import datetime, timedelta
7
+ from typing import Callable, Generator
8
+
9
+ import asyncpg
10
+ from PgQueuer.models import Job
11
+ from PgQueuer.qm import QueueManager
12
+ from PgQueuer.queries import Queries
13
+
14
+
15
+ @contextlib.contextmanager
16
+ def execution_timer() -> (
17
+ Generator[
18
+ Callable[[], timedelta],
19
+ None,
20
+ None,
21
+ ]
22
+ ):
23
+ enter = time.perf_counter()
24
+ done: float | None = None
25
+ try:
26
+ yield lambda: timedelta(seconds=((done or time.perf_counter()) - enter))
27
+ finally:
28
+ done = time.perf_counter()
29
+
30
+
31
+ async def run_qm(pool: asyncpg.Pool) -> None:
32
+ qm = QueueManager(pool)
33
+
34
+ @qm.entrypoint("async")
35
+ async def asyncfetch(job: Job) -> None:
36
+ if job.payload is None:
37
+ qm.alive = False
38
+
39
+ @qm.entrypoint("sync")
40
+ def syncfetch(job: Job) -> None:
41
+ if job.payload is None:
42
+ qm.alive = False
43
+
44
+ await qm.run(timedelta(seconds=0))
45
+
46
+
47
+ async def main() -> None:
48
+ parser = argparse.ArgumentParser(
49
+ description="Run the job processing benchmark.",
50
+ )
51
+ parser.add_argument(
52
+ "-t",
53
+ "--time-limit",
54
+ type=lambda x: timedelta(seconds=float(x)),
55
+ default=timedelta(seconds=10),
56
+ help="Run the benchmark for a limited number of seconds. Default is 10",
57
+ )
58
+ args = parser.parse_args()
59
+ start = datetime.now()
60
+
61
+ async with asyncpg.create_pool() as pool:
62
+ queries = Queries(pool)
63
+ await queries.clear_log()
64
+ while True:
65
+ for concurrecy in range(1, 6):
66
+ N = concurrecy * 1_000
67
+
68
+ await queries.clear_queue()
69
+
70
+ entrypoints = ["sync"] * (N // 2) + ["async"] * (N // 2)
71
+ random.shuffle(entrypoints)
72
+ payloads = [f"{n}".encode() for n in range(N)]
73
+ priorities = [0] * N
74
+ await queries.enqueue(entrypoints, payloads, priorities)
75
+ await queries.enqueue(
76
+ ["async"] * concurrecy**2,
77
+ [None] * concurrecy**2,
78
+ [0] * concurrecy**2,
79
+ )
80
+
81
+ with execution_timer() as elapsed:
82
+ await asyncio.gather(
83
+ *[run_qm(pool) for _ in range(concurrecy)],
84
+ )
85
+
86
+ elapsed_total_seconds = elapsed().total_seconds()
87
+ jps = (N - len(await queries.queue_size())) / elapsed_total_seconds
88
+
89
+ print(
90
+ f"Concurrecy: {concurrecy:<2} ",
91
+ f"Jobs: {N:<5} ",
92
+ f"JPS: {(jps)/1_000:.1f}k",
93
+ )
94
+
95
+ if timedelta(seconds=0) < args.time_limit < datetime.now() - start:
96
+ return
97
+
98
+
99
+ if __name__ == "__main__":
100
+ asyncio.run(main())
@@ -1,101 +0,0 @@
1
- import asyncio
2
- import contextlib
3
- import time
4
- from datetime import timedelta
5
- from typing import Callable, Generator
6
-
7
- import asyncpg
8
- from PgQueuer.models import Job
9
- from PgQueuer.qm import QueueManager
10
- from PgQueuer.queries import Queries
11
-
12
-
13
- @contextlib.contextmanager
14
- def timer() -> (
15
- Generator[
16
- Callable[[], float],
17
- None,
18
- None,
19
- ]
20
- ):
21
- enter = time.perf_counter()
22
- done: float | None = None
23
- try:
24
- yield lambda: done - enter if done else time.perf_counter() - enter
25
- finally:
26
- done = time.perf_counter()
27
-
28
-
29
- async def jobs_per_second(pool: asyncpg.Pool) -> float:
30
- qm = QueueManager(pool)
31
- seen = 0
32
-
33
- @qm.entrypoint("async")
34
- async def asyncfetch(job: Job) -> None:
35
- nonlocal seen
36
- seen += 1
37
- if job.payload is None:
38
- qm.alive = False
39
-
40
- @qm.entrypoint("sync")
41
- def syncfetch(job: Job) -> None:
42
- nonlocal seen
43
- seen += 1
44
- if job.payload is None:
45
- qm.alive = False
46
-
47
- with timer() as elapsed:
48
- await qm.run(timedelta(seconds=0))
49
-
50
- return seen / elapsed()
51
-
52
-
53
- async def benchmark(
54
- entrypoint: str,
55
- concurrecy: int,
56
- N: int,
57
- pool: asyncpg.Pool,
58
- ) -> None:
59
- queries = Queries(pool)
60
- await queries.enqueue(
61
- [entrypoint] * N,
62
- [f"{n}".encode() for n in range(N)],
63
- [0] * N,
64
- )
65
-
66
- await queries.enqueue(
67
- [entrypoint] * concurrecy**2,
68
- [None] * concurrecy**2,
69
- [0] * concurrecy**2,
70
- )
71
-
72
- jps = sum(
73
- await asyncio.gather(
74
- *[jobs_per_second(pool) for _ in range(concurrecy)],
75
- )
76
- )
77
-
78
- print(
79
- f"Entrypoint: {entrypoint:<5} ",
80
- f"Concurrecy: {concurrecy:<2} ",
81
- f"Jobs: {N:<5} ",
82
- f"JPS: {jps/1_000:.1f}k",
83
- )
84
-
85
-
86
- async def main() -> None:
87
- async with asyncpg.create_pool(
88
- min_size=20,
89
- max_size=99,
90
- ) as pool:
91
- queries = Queries(pool)
92
- await queries.clear_log()
93
- await queries.clear_queue()
94
- for entrypoint in ("sync", "async"):
95
- for concurrecy in range(1, 6):
96
- await benchmark(entrypoint, concurrecy, concurrecy * 1_000, pool)
97
- await asyncio.sleep(0)
98
-
99
-
100
- if __name__ == "__main__":
101
- asyncio.run(main())
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes