PgQueuer 0.2.1__tar.gz → 0.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. {pgqueuer-0.2.1 → pgqueuer-0.2.2}/.github/workflows/ci.yml +31 -0
  2. {pgqueuer-0.2.1 → pgqueuer-0.2.2}/PKG-INFO +2 -1
  3. {pgqueuer-0.2.1 → pgqueuer-0.2.2}/pyproject.toml +1 -0
  4. pgqueuer-0.2.2/src/PgQueuer/qm.py +163 -0
  5. {pgqueuer-0.2.1 → pgqueuer-0.2.2}/src/PgQueuer/queries.py +63 -21
  6. {pgqueuer-0.2.1 → pgqueuer-0.2.2}/src/PgQueuer/tm.py +8 -0
  7. {pgqueuer-0.2.1 → pgqueuer-0.2.2}/src/PgQueuer.egg-info/PKG-INFO +2 -1
  8. {pgqueuer-0.2.1 → pgqueuer-0.2.2}/src/PgQueuer.egg-info/SOURCES.txt +2 -2
  9. {pgqueuer-0.2.1 → pgqueuer-0.2.2}/src/PgQueuer.egg-info/requires.txt +1 -0
  10. {pgqueuer-0.2.1 → pgqueuer-0.2.2}/src/_version.py +2 -2
  11. {pgqueuer-0.2.1 → pgqueuer-0.2.2}/test/test_qm.py +39 -2
  12. {pgqueuer-0.2.1 → pgqueuer-0.2.2}/test/test_queries.py +3 -3
  13. pgqueuer-0.2.2/test/test_tm.py +44 -0
  14. pgqueuer-0.2.2/tools/benchmark.py +99 -0
  15. pgqueuer-0.2.1/src/PgQueuer/qm.py +0 -119
  16. pgqueuer-0.2.1/test/test_benchmark.py +0 -46
  17. pgqueuer-0.2.1/test/test_tm.py +0 -24
  18. {pgqueuer-0.2.1 → pgqueuer-0.2.2}/.github/workflows/linting.yml +0 -0
  19. {pgqueuer-0.2.1 → pgqueuer-0.2.2}/.github/workflows/release.yml +0 -0
  20. {pgqueuer-0.2.1 → pgqueuer-0.2.2}/.gitignore +0 -0
  21. {pgqueuer-0.2.1 → pgqueuer-0.2.2}/LICENSE +0 -0
  22. {pgqueuer-0.2.1 → pgqueuer-0.2.2}/README.md +0 -0
  23. {pgqueuer-0.2.1 → pgqueuer-0.2.2}/docker-compose.yml +0 -0
  24. {pgqueuer-0.2.1 → pgqueuer-0.2.2}/setup.cfg +0 -0
  25. {pgqueuer-0.2.1 → pgqueuer-0.2.2}/src/PgQueuer/__init__.py +0 -0
  26. {pgqueuer-0.2.1 → pgqueuer-0.2.2}/src/PgQueuer/__main__.py +0 -0
  27. {pgqueuer-0.2.1 → pgqueuer-0.2.2}/src/PgQueuer/_version.py +0 -0
  28. {pgqueuer-0.2.1 → pgqueuer-0.2.2}/src/PgQueuer/cli.py +0 -0
  29. {pgqueuer-0.2.1 → pgqueuer-0.2.2}/src/PgQueuer/logconfig.py +0 -0
  30. {pgqueuer-0.2.1 → pgqueuer-0.2.2}/src/PgQueuer/models.py +0 -0
  31. {pgqueuer-0.2.1 → pgqueuer-0.2.2}/src/PgQueuer/py.typed +0 -0
  32. {pgqueuer-0.2.1 → pgqueuer-0.2.2}/src/PgQueuer.egg-info/dependency_links.txt +0 -0
  33. {pgqueuer-0.2.1 → pgqueuer-0.2.2}/src/PgQueuer.egg-info/top_level.txt +0 -0
  34. {pgqueuer-0.2.1 → pgqueuer-0.2.2}/test/conftest.py +0 -0
  35. {pgqueuer-0.2.1 → pgqueuer-0.2.2}/test/db/Dockerfile +0 -0
  36. {pgqueuer-0.2.1 → pgqueuer-0.2.2}/test/db/init_db.sh +0 -0
@@ -7,6 +7,37 @@ on:
7
7
  branches: [main]
8
8
 
9
9
  jobs:
10
+ benchmark:
11
+
12
+ strategy:
13
+ fail-fast: false
14
+ matrix:
15
+ python-version: ["3.11"]
16
+ os: [ubuntu-latest]
17
+
18
+ name: PgQueuer Benchmark
19
+ runs-on: ${{ matrix.os }}
20
+
21
+ steps:
22
+ - uses: actions/checkout@v4
23
+
24
+ - name: Set up Python
25
+ uses: actions/setup-python@v4
26
+ with:
27
+ python-version: ${{ matrix.python-version }}
28
+
29
+ - name: Build custom PostgreSQL Docker image
30
+ run: |
31
+ docker-compose up --build -d pgq
32
+
33
+ - name: Install PgQueuer
34
+ run: |
35
+ pip install pip -U
36
+ pip install .[dev]
37
+
38
+ - name: Benchmark
39
+ run: PGUSER=testuser PGPASSWORD=testpassword PGDATABASE=testdb python3 tools/benchmark.py
40
+
10
41
  ci:
11
42
  strategy:
12
43
  fail-fast: false
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: PgQueuer
3
- Version: 0.2.1
3
+ Version: 0.2.2
4
4
  Summary: PgQueuer is a Python library leveraging PostgreSQL for efficient job queuing.
5
5
  Author: janbjorge
6
6
  License: MIT License
@@ -26,6 +26,7 @@ Classifier: Topic :: System :: Distributed Computing
26
26
  Requires-Python: >=3.10
27
27
  Description-Content-Type: text/markdown
28
28
  License-File: LICENSE
29
+ Requires-Dist: anyio>=4.0
29
30
  Requires-Dist: asyncpg>=0.27.0
30
31
  Requires-Dist: pgcachewatch>=0.4
31
32
  Requires-Dist: pydantic>=2.0.0
@@ -30,6 +30,7 @@ classifiers = [
30
30
  ]
31
31
 
32
32
  dependencies = [
33
+ "anyio>=4.0",
33
34
  "asyncpg>=0.27.0",
34
35
  "pgcachewatch>=0.4",
35
36
  "pydantic>=2.0.0",
@@ -0,0 +1,163 @@
1
+ from __future__ import annotations
2
+
3
+ import asyncio
4
+ import dataclasses
5
+ import functools
6
+ from datetime import timedelta
7
+ from typing import (
8
+ TYPE_CHECKING,
9
+ Awaitable,
10
+ Callable,
11
+ TypeAlias,
12
+ TypeGuard,
13
+ TypeVar,
14
+ overload,
15
+ )
16
+
17
+ import anyio
18
+ import anyio.to_thread
19
+ import asyncpg
20
+ from pgcachewatch.listeners import PGEventQueue
21
+ from pgcachewatch.models import PGChannel
22
+
23
+ from .logconfig import logger
24
+ from .models import Job
25
+ from .queries import DBSettings, Queries
26
+ from .tm import TaskManager
27
+
28
+ if TYPE_CHECKING:
29
+ AsyncEntrypoint: TypeAlias = Callable[[Job], Awaitable[None]]
30
+ SyncEntrypoint: TypeAlias = Callable[[Job], None]
31
+ Entrypoint: TypeAlias = AsyncEntrypoint | SyncEntrypoint
32
+ T = TypeVar("T", bound=Entrypoint)
33
+
34
+
35
+ @overload
36
+ def is_async_callable(obj: AsyncEntrypoint) -> TypeGuard[AsyncEntrypoint]: ...
37
+
38
+
39
+ @overload
40
+ def is_async_callable(obj: SyncEntrypoint) -> TypeGuard[SyncEntrypoint]: ...
41
+
42
+
43
+ def is_async_callable(obj: object) -> bool:
44
+ # Inspired by:
45
+ # https://github.com/encode/starlette/blob/9f16bf5c25e126200701f6e04330864f4a91a898/starlette/_utils.py#L38
46
+ while isinstance(obj, functools.partial):
47
+ obj = obj.func
48
+
49
+ return asyncio.iscoroutinefunction(obj) or (
50
+ callable(obj) and asyncio.iscoroutinefunction(obj.__call__)
51
+ )
52
+
53
+
54
+ @dataclasses.dataclass
55
+ class QueueManager:
56
+ """
57
+ Manages job queues and dispatches jobs to registered entry points,
58
+ handling database connections and events.
59
+ """
60
+
61
+ pool: asyncpg.Pool
62
+ queries: Queries = dataclasses.field(init=False)
63
+ channel: PGChannel = dataclasses.field(
64
+ default=PGChannel(DBSettings().channel),
65
+ )
66
+ alive: bool = dataclasses.field(
67
+ init=False,
68
+ default=True,
69
+ )
70
+ # Should registry be a weakref?
71
+ registry: dict[str, Entrypoint] = dataclasses.field(
72
+ init=False,
73
+ default_factory=dict,
74
+ )
75
+
76
+ def __post_init__(self) -> None:
77
+ """
78
+ Initializes database query handlers and validates pool size upon
79
+ instance creation.
80
+ """
81
+ if self.pool.get_min_size() < 1:
82
+ raise ValueError("... min size must be gt 1.")
83
+ self.queries = Queries(self.pool)
84
+
85
+ def entrypoint(self, name: str) -> Callable[[T], T]:
86
+ """
87
+ Decorator to register a function as an entrypoint for
88
+ handling specific job types.
89
+ """
90
+
91
+ if name in self.registry:
92
+ raise RuntimeError(f"{name} already in registry, name must be unique.")
93
+
94
+ def register(func: T) -> T:
95
+ self.registry[name] = func
96
+ return func
97
+
98
+ return register
99
+
100
+ async def run(
101
+ self,
102
+ dequeue_timeout: timedelta = timedelta(seconds=30),
103
+ ) -> None:
104
+ """
105
+ Starts the event listener and continuously dispatches jobs to
106
+ registered entry points until stopped.
107
+ """
108
+ async with (
109
+ self.pool.acquire() as conn,
110
+ TaskManager() as tm,
111
+ ):
112
+ listener = PGEventQueue()
113
+ await listener.connect(conn, self.channel)
114
+
115
+ while self.alive:
116
+ while self.alive and (job := await self.queries.dequeue()):
117
+ tm.add(asyncio.create_task(self._dispatch(job)))
118
+
119
+ try:
120
+ await asyncio.wait_for(
121
+ listener.get(),
122
+ timeout=dequeue_timeout.total_seconds(),
123
+ )
124
+ except asyncio.TimeoutError:
125
+ logger.debug(
126
+ "Timeout after %r without receiving an event.",
127
+ dequeue_timeout,
128
+ )
129
+
130
+ await conn.reset()
131
+
132
+ async def _dispatch(self, job: Job) -> None:
133
+ """
134
+ Internal method to asynchronously handle job dispatch,
135
+ including exception logging and job status updates.
136
+ """
137
+
138
+ logger.debug(
139
+ "Dispatching entrypoint/id: %s/%s",
140
+ job.entrypoint,
141
+ job.id,
142
+ )
143
+
144
+ try:
145
+ fn = self.registry[job.entrypoint]
146
+ if is_async_callable(fn):
147
+ await fn(job)
148
+ else:
149
+ await anyio.to_thread.run_sync(fn, job)
150
+ except Exception:
151
+ logger.exception(
152
+ "Exception while processing entrypoint/id: %s/%s",
153
+ job.entrypoint,
154
+ job.id,
155
+ )
156
+ await self.queries.log_job(job, "exception")
157
+ else:
158
+ logger.debug(
159
+ "Dispatching entrypoint/id: %s/%s - successful",
160
+ job.entrypoint,
161
+ job.id,
162
+ )
163
+ await self.queries.log_job(job, "successful")
@@ -79,7 +79,8 @@ class Queries:
79
79
  entrypoint TEXT NOT NULL,
80
80
  payload BYTEA
81
81
  );
82
- CREATE UNIQUE INDEX ON {self.settings.queue_table} (priority, id) WHERE status != 'picked';
82
+ CREATE INDEX ON {self.settings.queue_table} (priority ASC, id DESC)
83
+ INCLUDE (id) WHERE status = 'queued';
83
84
 
84
85
  CREATE TYPE {self.settings.log_table_status_type} AS ENUM ('exception', 'successful');
85
86
  CREATE TABLE {self.settings.log_table} (
@@ -92,16 +93,42 @@ class Queries:
92
93
  );
93
94
 
94
95
  CREATE FUNCTION {self.settings.function}() RETURNS TRIGGER AS $$
96
+ DECLARE
97
+ to_emit BOOLEAN := false; -- Flag to decide whether to emit a notification
95
98
  BEGIN
96
- PERFORM pg_notify(
97
- '{self.settings.channel}',
98
- json_build_object(
99
- 'channel', '{self.settings.channel}',
100
- 'operation', lower(TG_OP),
101
- 'sent_at', NOW(),
102
- 'table', TG_TABLE_NAME
103
- )::text);
104
- RETURN NEW;
99
+ -- Check operation type and set the emit flag accordingly
100
+ IF TG_OP = 'UPDATE' AND OLD IS DISTINCT FROM NEW THEN
101
+ to_emit := true;
102
+ ELSIF TG_OP = 'DELETE' THEN
103
+ to_emit := true;
104
+ ELSIF TG_OP = 'INSERT' THEN
105
+ to_emit := true;
106
+ ELSIF TG_OP = 'TRUNCATE' THEN
107
+ to_emit := true;
108
+ END IF;
109
+
110
+ -- Perform notification if the emit flag is set
111
+ IF to_emit THEN
112
+ PERFORM pg_notify(
113
+ '{self.settings.channel}',
114
+ json_build_object(
115
+ 'channel', '{self.settings.channel}',
116
+ 'operation', lower(TG_OP),
117
+ 'sent_at', NOW(),
118
+ 'table', TG_TABLE_NAME
119
+ )::text
120
+ );
121
+ END IF;
122
+
123
+ -- Return appropriate value based on the operation
124
+ IF TG_OP IN ('INSERT', 'UPDATE') THEN
125
+ RETURN NEW;
126
+ ELSIF TG_OP = 'DELETE' THEN
127
+ RETURN OLD;
128
+ ELSE
129
+ RETURN NULL; -- For TRUNCATE and other non-row-specific contexts
130
+ END IF;
131
+
105
132
  END;
106
133
  $$ LANGUAGE plpgsql;
107
134
 
@@ -135,13 +162,9 @@ class Queries:
135
162
  """
136
163
  query = f"""
137
164
  WITH next_job AS (
138
- SELECT p1.id
139
- FROM {self.settings.queue_table} p1
140
- WHERE p1.status = 'queued' AND NOT EXISTS (
141
- SELECT FROM
142
- {self.settings.queue_table} p2
143
- WHERE p1.entrypoint = p2.entrypoint AND p2.status = 'picked'
144
- )
165
+ SELECT id
166
+ FROM {self.settings.queue_table}
167
+ WHERE status = 'queued'
145
168
  ORDER BY priority DESC, id ASC
146
169
  FOR UPDATE SKIP LOCKED
147
170
  LIMIT 1
@@ -158,20 +181,39 @@ class Queries:
158
181
 
159
182
  async def enqueue(
160
183
  self,
161
- entrypoint: str,
162
- payload: bytes | None,
163
- priority: int = 0,
184
+ entrypoint: str | list[str],
185
+ payload: bytes | None | list[bytes | None],
186
+ priority: int | list[int] = 0,
164
187
  ) -> None:
165
188
  """
166
189
  Inserts a new job into the queue with the specified
167
190
  entrypoint, payload, and priority, marking it as 'queued'.
191
+ This method ensures that if any of entrypoint, payload, or priority is a list,
192
+ all list arguments must be of the same length.
193
+
194
+ If two or more arguments are lists, they must be the same length;
195
+ otherwise, an ValueError is raised.
168
196
  """
197
+
198
+ # If they are not lists, create single-item lists for uniform processing
199
+ normed_entrypoint = entrypoint if isinstance(entrypoint, list) else [entrypoint]
200
+ normed_payload = payload if isinstance(payload, list) else [payload]
201
+ normed_priority = priority if isinstance(priority, list) else [priority]
202
+
169
203
  query = f"""
170
204
  INSERT INTO {self.settings.queue_table} (priority, status, entrypoint, payload)
171
205
  VALUES ($1, 'queued', $2, $3)
172
206
  """ # noqa: E501
173
207
 
174
- await self.pool.execute(query, priority, entrypoint, payload)
208
+ await self.pool.executemany(
209
+ query,
210
+ zip(
211
+ normed_priority,
212
+ normed_entrypoint,
213
+ normed_payload,
214
+ strict=True,
215
+ ),
216
+ )
175
217
 
176
218
  async def clear_queue(self, entrypoint: str | list[str] | None = None) -> None:
177
219
  """
@@ -1,3 +1,5 @@
1
+ from __future__ import annotations
2
+
1
3
  import asyncio
2
4
  import dataclasses
3
5
 
@@ -21,3 +23,9 @@ class TaskManager:
21
23
  """
22
24
  self.tasks.add(task)
23
25
  task.add_done_callback(self.tasks.remove)
26
+
27
+ async def __aenter__(self) -> TaskManager:
28
+ return self
29
+
30
+ async def __aexit__(self, *_: object) -> None:
31
+ await asyncio.gather(*self.tasks)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: PgQueuer
3
- Version: 0.2.1
3
+ Version: 0.2.2
4
4
  Summary: PgQueuer is a Python library leveraging PostgreSQL for efficient job queuing.
5
5
  Author: janbjorge
6
6
  License: MIT License
@@ -26,6 +26,7 @@ Classifier: Topic :: System :: Distributed Computing
26
26
  Requires-Python: >=3.10
27
27
  Description-Content-Type: text/markdown
28
28
  License-File: LICENSE
29
+ Requires-Dist: anyio>=4.0
29
30
  Requires-Dist: asyncpg>=0.27.0
30
31
  Requires-Dist: pgcachewatch>=0.4
31
32
  Requires-Dist: pydantic>=2.0.0
@@ -23,9 +23,9 @@ src/PgQueuer.egg-info/dependency_links.txt
23
23
  src/PgQueuer.egg-info/requires.txt
24
24
  src/PgQueuer.egg-info/top_level.txt
25
25
  test/conftest.py
26
- test/test_benchmark.py
27
26
  test/test_qm.py
28
27
  test/test_queries.py
29
28
  test/test_tm.py
30
29
  test/db/Dockerfile
31
- test/db/init_db.sh
30
+ test/db/init_db.sh
31
+ tools/benchmark.py
@@ -1,3 +1,4 @@
1
+ anyio>=4.0
1
2
  asyncpg>=0.27.0
2
3
  pgcachewatch>=0.4
3
4
  pydantic>=2.0.0
@@ -12,5 +12,5 @@ __version__: str
12
12
  __version_tuple__: VERSION_TUPLE
13
13
  version_tuple: VERSION_TUPLE
14
14
 
15
- __version__ = version = '0.2.1'
16
- __version_tuple__ = version_tuple = (0, 2, 1)
15
+ __version__ = version = '0.2.2'
16
+ __version_tuple__ = version_tuple = (0, 2, 2)
@@ -1,4 +1,5 @@
1
1
  import asyncio
2
+ import time
2
3
 
3
4
  import asyncpg
4
5
  import pytest
@@ -7,7 +8,7 @@ from PgQueuer.qm import QueueManager
7
8
  from PgQueuer.queries import Queries
8
9
 
9
10
 
10
- @pytest.mark.parametrize("N", (1, 2, 32, 500))
11
+ @pytest.mark.parametrize("N", (1, 2, 32))
11
12
  async def test_job_queing(
12
13
  pgpool: asyncpg.Pool,
13
14
  N: int,
@@ -33,7 +34,7 @@ async def test_job_queing(
33
34
  assert seen == list(range(N))
34
35
 
35
36
 
36
- @pytest.mark.parametrize("N", (1, 2, 32, 512))
37
+ @pytest.mark.parametrize("N", (1, 2, 32))
37
38
  @pytest.mark.parametrize("concurrency", (1, 2, 3, 4))
38
39
  async def test_job_fetch(
39
40
  pgpool: asyncpg.Pool,
@@ -66,3 +67,39 @@ async def test_job_fetch(
66
67
  timeout=10,
67
68
  )
68
69
  assert sorted(seen) == list(range(N))
70
+
71
+
72
+ @pytest.mark.parametrize("N", (1, 2, 32))
73
+ @pytest.mark.parametrize("concurrency", (1, 2, 3, 4))
74
+ async def test_sync_entrypoint(
75
+ pgpool: asyncpg.Pool,
76
+ N: int,
77
+ concurrency: int,
78
+ ) -> None:
79
+ q = Queries(pgpool)
80
+ qmpool = [QueueManager(pgpool) for _ in range(concurrency)]
81
+ seen = list[int]()
82
+
83
+ for qm in qmpool:
84
+
85
+ @qm.entrypoint("fetch")
86
+ def fetch(context: Job) -> None:
87
+ time.sleep(2) # Sim. heavy CPU/IO.
88
+ if context.payload is None:
89
+ for qm in qmpool:
90
+ qm.alive = False
91
+ return
92
+ assert context
93
+ seen.append(int(context.payload))
94
+
95
+ for n in range(N):
96
+ await q.enqueue("fetch", f"{n}".encode())
97
+
98
+ # Stop flag
99
+ await q.enqueue("fetch", None)
100
+
101
+ await asyncio.wait_for(
102
+ asyncio.gather(*[qm.run() for qm in qmpool]),
103
+ timeout=10,
104
+ )
105
+ assert sorted(seen) == list(range(N))
@@ -37,7 +37,7 @@ async def test_queries_next_jobs(
37
37
  assert seen == list(range(N))
38
38
 
39
39
 
40
- @pytest.mark.parametrize("N", (1, 2, 64, 256))
40
+ @pytest.mark.parametrize("N", (1, 2, 64))
41
41
  @pytest.mark.parametrize("concurrency", (1, 2, 4, 16))
42
42
  async def test_queries_next_jobs_concurrent(
43
43
  pgpool: asyncpg.Pool,
@@ -79,7 +79,7 @@ async def test_queries_clear(pgpool: asyncpg.Pool) -> None:
79
79
  assert sum(x.count for x in await q.queue_size()) == 0
80
80
 
81
81
 
82
- @pytest.mark.parametrize("N", (1, 2, 64, 256))
82
+ @pytest.mark.parametrize("N", (1, 2, 64))
83
83
  async def test_move_job_log(
84
84
  pgpool: asyncpg.Pool,
85
85
  N: int,
@@ -130,7 +130,7 @@ async def test_clear_queue(
130
130
  assert sum(x.count for x in await q.queue_size()) == N - 1
131
131
 
132
132
 
133
- @pytest.mark.parametrize("N", (1, 2, 64, 256))
133
+ @pytest.mark.parametrize("N", (1, 2, 64))
134
134
  async def test_queue_priority(
135
135
  pgpool: asyncpg.Pool,
136
136
  N: int,
@@ -0,0 +1,44 @@
1
+ import asyncio
2
+
3
+ import pytest
4
+ from PgQueuer import qm
5
+
6
+
7
+ @pytest.mark.parametrize("N", (1, 2, 3, 5, 64))
8
+ async def test_task_manager(N: int) -> None:
9
+ future = asyncio.Future[None]()
10
+
11
+ tm = qm.TaskManager()
12
+ assert len(tm.tasks) == 0
13
+
14
+ async def waiter(future: asyncio.Future) -> None:
15
+ return await future
16
+
17
+ for _ in range(N):
18
+ tm.add(asyncio.create_task(waiter(future)))
19
+
20
+ assert len(tm.tasks) == N
21
+
22
+ future.set_result(None)
23
+ await asyncio.gather(*tm.tasks)
24
+ assert len(tm.tasks) == 0
25
+
26
+
27
+ @pytest.mark.parametrize("N", (1, 2, 3, 5, 64))
28
+ async def test_task_manager_ctx_mngr(N: int) -> None:
29
+ async def waiter(future: asyncio.Future) -> None:
30
+ return await future
31
+
32
+ future = asyncio.Future[None]()
33
+
34
+ async with qm.TaskManager() as tm:
35
+ assert len(tm.tasks) == 0
36
+
37
+ for _ in range(N):
38
+ tm.add(asyncio.create_task(waiter(future)))
39
+
40
+ assert len(tm.tasks) == N
41
+
42
+ future.set_result(None)
43
+
44
+ assert len(tm.tasks) == 0
@@ -0,0 +1,99 @@
1
+ import asyncio
2
+ import contextlib
3
+ import time
4
+ from datetime import timedelta
5
+ from typing import Callable, Generator
6
+
7
+ import asyncpg
8
+ from PgQueuer.models import Job
9
+ from PgQueuer.qm import QueueManager
10
+ from PgQueuer.queries import Queries
11
+
12
+
13
+ @contextlib.contextmanager
14
+ def timer() -> (
15
+ Generator[
16
+ Callable[[], float],
17
+ None,
18
+ None,
19
+ ]
20
+ ):
21
+ enter = time.perf_counter()
22
+ done: float | None = None
23
+ try:
24
+ yield lambda: done - enter if done else time.perf_counter() - enter
25
+ finally:
26
+ done = time.perf_counter()
27
+
28
+
29
+ async def jobs_per_second(pool: asyncpg.Pool) -> float:
30
+ qm = QueueManager(pool)
31
+ seen = 0
32
+
33
+ @qm.entrypoint("async")
34
+ async def asyncfetch(job: Job) -> None:
35
+ nonlocal qm, seen
36
+ seen += 1
37
+ if job.payload is None:
38
+ qm.alive = False
39
+
40
+ @qm.entrypoint("sync")
41
+ def syncfetch(job: Job) -> None:
42
+ nonlocal qm, seen
43
+ seen += 1
44
+ if job.payload is None:
45
+ qm.alive = False
46
+
47
+ with timer() as elapsed:
48
+ await qm.run(timedelta(seconds=0))
49
+
50
+ return seen / elapsed()
51
+
52
+
53
+ async def benchmark(
54
+ entrypoint: str,
55
+ concurrecy: int,
56
+ N: int,
57
+ pool: asyncpg.Pool,
58
+ ) -> None:
59
+ queries = Queries(pool)
60
+
61
+ await queries.clear_log()
62
+ await queries.clear_queue()
63
+
64
+ await queries.enqueue(
65
+ [entrypoint] * N,
66
+ [f"{n}".encode() for n in range(N)],
67
+ [0] * N,
68
+ )
69
+ assert sum(x.count for x in await queries.queue_size()) == N
70
+
71
+ await queries.enqueue(
72
+ [entrypoint] * concurrecy**2,
73
+ [None] * concurrecy**2,
74
+ [0] * concurrecy**2,
75
+ )
76
+
77
+ jobs = [jobs_per_second(pool) for _ in range(concurrecy)]
78
+ results = await asyncio.gather(*jobs)
79
+
80
+ print(
81
+ f"Entrypoint: {entrypoint:<5} ",
82
+ f"Concurrecy: {concurrecy:<2} "
83
+ f"Jobs: {N:<5} "
84
+ f"JPS: {sum(results)/1_000:.1f}k",
85
+ )
86
+
87
+
88
+ async def main() -> None:
89
+ async with asyncpg.create_pool(
90
+ min_size=20,
91
+ max_size=99,
92
+ ) as pool:
93
+ for entrypoint in ("sync", "async"):
94
+ for concurrecy in range(1, 6):
95
+ await benchmark(entrypoint, concurrecy, 1_000 * concurrecy, pool)
96
+
97
+
98
+ if __name__ == "__main__":
99
+ asyncio.run(main())
@@ -1,119 +0,0 @@
1
- from __future__ import annotations
2
-
3
- import asyncio
4
- import dataclasses
5
- from typing import TYPE_CHECKING, Awaitable, Callable, TypeAlias, TypeVar
6
-
7
- import asyncpg
8
- from pgcachewatch.listeners import PGEventQueue
9
- from pgcachewatch.models import PGChannel
10
-
11
- from .logconfig import logger
12
- from .models import Job
13
- from .queries import DBSettings, Queries
14
- from .tm import TaskManager
15
-
16
- if TYPE_CHECKING:
17
- EntrypointFn: TypeAlias = Callable[[Job], Awaitable[None]]
18
- T = TypeVar("T", bound=EntrypointFn)
19
-
20
-
21
- @dataclasses.dataclass
22
- class QueueManager:
23
- """
24
- Manages job queues and dispatches jobs to registered entry points,
25
- handling database connections and events.
26
- """
27
-
28
- pool: asyncpg.Pool
29
- queries: Queries = dataclasses.field(init=False)
30
-
31
- channel: PGChannel = dataclasses.field(
32
- default=PGChannel(DBSettings().channel),
33
- init=False,
34
- )
35
- alive: bool = dataclasses.field(
36
- init=False,
37
- default=True,
38
- )
39
- # Should registry be a weakref?
40
- registry: dict[str, EntrypointFn] = dataclasses.field(
41
- init=False,
42
- default_factory=dict,
43
- )
44
- tm: TaskManager = dataclasses.field(
45
- init=False,
46
- default_factory=TaskManager,
47
- )
48
-
49
- def __post_init__(self) -> None:
50
- """
51
- Initializes database query handlers and validates pool size upon
52
- instance creation.
53
- """
54
- if self.pool.get_min_size() < 1:
55
- raise ValueError("... min size must be gt 1.")
56
- self.queries = Queries(self.pool)
57
-
58
- def entrypoint(self, name: str) -> Callable[[T], T]:
59
- """
60
- Decorator to register a function as an entrypoint for
61
- handling specific job types.
62
- """
63
-
64
- if name in self.registry:
65
- raise RuntimeError(f"{name} already in registry, name must be unique.")
66
-
67
- def register(func: T) -> T:
68
- self.registry[name] = func
69
- return func
70
-
71
- return register
72
-
73
- async def run(self) -> None:
74
- """
75
- Starts the event listener and continuously dispatches jobs to
76
- registered entry points until stopped.
77
- """
78
- async with self.pool.acquire() as conn:
79
- listener = PGEventQueue()
80
- await listener.connect(conn, self.channel)
81
-
82
- while self.alive:
83
- while job := await self.queries.dequeue():
84
- self._dispatch(job)
85
- await listener.get()
86
-
87
- await asyncio.gather(*self.tm.tasks)
88
- await conn.reset()
89
-
90
- def _dispatch(self, job: Job) -> None:
91
- """
92
- Internal method to asynchronously handle job dispatch,
93
- including exception logging and job status updates.
94
- """
95
-
96
- async def runit() -> None:
97
- logger.debug(
98
- "Dispatching entrypoint/id: %s/%s",
99
- job.entrypoint,
100
- job.id,
101
- )
102
- try:
103
- await self.registry[job.entrypoint](job)
104
- except Exception:
105
- logger.exception(
106
- "Exception while processing entrypoint/id: %s/%s",
107
- job.entrypoint,
108
- job.id,
109
- )
110
- await self.queries.log_job(job, "exception")
111
- else:
112
- logger.debug(
113
- "Dispatching entrypoint/id: %s/%s - successful",
114
- job.entrypoint,
115
- job.id,
116
- )
117
- await self.queries.log_job(job, "successful")
118
-
119
- self.tm.add(asyncio.create_task(runit()))
@@ -1,46 +0,0 @@
1
- import asyncio
2
- import time
3
-
4
- import asyncpg
5
- import pytest
6
- from PgQueuer.models import Job
7
- from PgQueuer.qm import QueueManager
8
-
9
-
10
- async def jobs_per_second(
11
- N: int,
12
- pool: asyncpg.Pool,
13
- ) -> float:
14
- qm = QueueManager(pool)
15
-
16
- @qm.entrypoint("fetch")
17
- async def process_message(job: Job) -> None:
18
- if job.payload is None:
19
- qm.alive = False
20
-
21
- enter = time.perf_counter()
22
- await qm.run()
23
- done = time.perf_counter()
24
- return N / (done - enter)
25
-
26
-
27
- @pytest.mark.parametrize("N", (100, 500, 1_000))
28
- @pytest.mark.parametrize("concurrecy", (1, 2, 3, 4, 5))
29
- async def test_benchmark(
30
- concurrecy: int,
31
- N: int,
32
- pgpool: asyncpg.Pool,
33
- ) -> None:
34
- qm = QueueManager(pgpool)
35
-
36
- for n in range(N):
37
- await qm.queries.enqueue("fetch", f"{n}".encode())
38
-
39
- # Hacky, must fine a bether way to shutdown.
40
- for n in range(concurrecy**2):
41
- await qm.queries.enqueue("fetch", None)
42
-
43
- jobs = [jobs_per_second(N, pgpool) for _ in range(concurrecy)]
44
- jps = sum(await asyncio.wait_for(asyncio.gather(*jobs), timeout=30))
45
-
46
- print(f"Concurrecy: {concurrecy} Jobs: {N}, Jobs per second: {jps:.1f}")
@@ -1,24 +0,0 @@
1
- import asyncio
2
-
3
- import pytest
4
- from PgQueuer import qm
5
-
6
-
7
- @pytest.mark.parametrize("N", (1, 2, 64, 1024))
8
- async def test_task_manager(N: int) -> None:
9
- future = asyncio.Future[None]()
10
-
11
- tm = qm.TaskManager()
12
- assert len(tm.tasks) == 0
13
-
14
- async def waiter(future: asyncio.Future) -> None:
15
- return await future
16
-
17
- for _ in range(N):
18
- tm.add(asyncio.create_task(waiter(future)))
19
-
20
- assert len(tm.tasks) == N
21
-
22
- future.set_result(None)
23
- await asyncio.gather(*tm.tasks)
24
- assert len(tm.tasks) == 0
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes