akadze 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- akadze-0.1.0/LICENSE +21 -0
- akadze-0.1.0/PKG-INFO +150 -0
- akadze-0.1.0/README.md +126 -0
- akadze-0.1.0/akadze/__about__.py +3 -0
- akadze-0.1.0/akadze/__init__.py +28 -0
- akadze-0.1.0/akadze/app.py +103 -0
- akadze-0.1.0/akadze/args.py +58 -0
- akadze-0.1.0/akadze/cli.py +117 -0
- akadze-0.1.0/akadze/context.py +67 -0
- akadze-0.1.0/akadze/enqueue.py +114 -0
- akadze-0.1.0/akadze/exc.py +47 -0
- akadze-0.1.0/akadze/hooks.py +51 -0
- akadze-0.1.0/akadze/job.py +139 -0
- akadze-0.1.0/akadze/maintenance.py +112 -0
- akadze-0.1.0/akadze/outcome.py +110 -0
- akadze-0.1.0/akadze/periodic.py +85 -0
- akadze-0.1.0/akadze/py.typed +0 -0
- akadze-0.1.0/akadze/queue.py +61 -0
- akadze-0.1.0/akadze/schema.py +154 -0
- akadze-0.1.0/akadze/sql/001_initial.sql +77 -0
- akadze-0.1.0/akadze/task.py +100 -0
- akadze-0.1.0/akadze/testing.py +92 -0
- akadze-0.1.0/akadze/worker.py +493 -0
- akadze-0.1.0/pyproject.toml +55 -0
akadze-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 g4st3r
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
akadze-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: akadze
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Postgres-backed task queue. The broker is PostgreSQL.
|
|
5
|
+
License: MIT
|
|
6
|
+
License-File: LICENSE
|
|
7
|
+
Keywords: postgres,queue,worker,tasks
|
|
8
|
+
Author: g4st3r
|
|
9
|
+
Requires-Python: >=3.13,<3.14
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
15
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
16
|
+
Requires-Dist: asyncpg (>=0.30.0,<0.31.0)
|
|
17
|
+
Requires-Dist: croniter (>=6.0.0,<7.0.0)
|
|
18
|
+
Requires-Dist: pydantic (>=2.10.0,<3.0.0)
|
|
19
|
+
Requires-Dist: sqlalchemy[asyncio] (>=2.0.40,<3.0.0)
|
|
20
|
+
Project-URL: Homepage, https://github.com/g4st3r-d3v/akadze
|
|
21
|
+
Project-URL: Repository, https://github.com/g4st3r-d3v/akadze
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
|
|
24
|
+
# akadze
|
|
25
|
+
|
|
26
|
+
[](https://github.com/g4st3r-d3v/akadze/actions/workflows/ci.yml)
|
|
27
|
+
|
|
28
|
+
Postgres-backed task queue: enqueue work and run it in a worker. Periodic schedules run in that same process.
|
|
29
|
+
|
|
30
|
+
Shape is familiar if you know Celery. The broker is PostgreSQL (`FOR UPDATE SKIP LOCKED` + lease), not Redis or RabbitMQ.
|
|
31
|
+
|
|
32
|
+
**Status:** alpha. Enqueue, the worker, periodic schedules, retry, and queue maintenance run on Postgres.
|
|
33
|
+
|
|
34
|
+
## Goals
|
|
35
|
+
|
|
36
|
+
- Small library, no product domain
|
|
37
|
+
- Tasks, retries, lease recovery
|
|
38
|
+
- Periodic cron and interval schedules run inside the worker
|
|
39
|
+
- Clear Python call surface; CLI for migrate and worker
|
|
40
|
+
|
|
41
|
+
## Non-goals (for now)
|
|
42
|
+
|
|
43
|
+
- HTTP / REST / web UI (another library or app owns that)
|
|
44
|
+
- Replacing any product service queue
|
|
45
|
+
- Multi-broker adapters
|
|
46
|
+
- Distributed tracing / full observability suite
|
|
47
|
+
|
|
48
|
+
## First task
|
|
49
|
+
|
|
50
|
+
Install the library, point it at the application database, and create the `akadze` schema. The tables live in that schema; they are not part of the application's own migrations.
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
poetry install
|
|
54
|
+
export AKADZE_DATABASE_URL=postgresql://akadze:akadze@localhost:5432/akadze
|
|
55
|
+
akadze migrate
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
Save this as `demo.py`. `enqueue` runs in the caller's transaction, so a rollback removes the job.
|
|
59
|
+
|
|
60
|
+
```python
|
|
61
|
+
import asyncio
|
|
62
|
+
|
|
63
|
+
from akadze import Akadze
|
|
64
|
+
|
|
65
|
+
app = Akadze(database_url="postgresql://akadze:akadze@localhost:5432/akadze")
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@app.task("hello")
|
|
69
|
+
async def hello(name: str) -> str:
|
|
70
|
+
return f"hello {name}"
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
async def main() -> None:
|
|
74
|
+
async with app.engine.begin() as connection:
|
|
75
|
+
await hello.using(session=connection).enqueue(name="ada")
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
if __name__ == "__main__":
|
|
79
|
+
asyncio.run(main())
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
Run the worker from the directory that contains `demo.py`. It loads `demo:app` from there, claims the job, runs `hello`, and sets the row to `succeeded`. Stop it with Ctrl-C: the worker takes no new jobs and waits up to 10 seconds (`shutdown_timeout`) for the ones already running. Whatever is still running goes back to the queue, and an attempt is not spent. Periodic tasks run in this same process; there is no separate beat.
|
|
83
|
+
|
|
84
|
+
If a heartbeat does not succeed within 30 seconds, the worker stops and puts unfinished jobs back on the queue. An attempt is not spent. A synchronous task runs in a thread. Stopping the worker does not stop that thread.
|
|
85
|
+
|
|
86
|
+
A claim that fills every free slot is followed by another claim immediately. Otherwise the worker waits up to `poll_interval` (1 second). The wait is at least half of that, so idle workers do not poll in step or spin. A job can run more than once if its worker disappears.
|
|
87
|
+
|
|
88
|
+
```bash
|
|
89
|
+
python demo.py
|
|
90
|
+
akadze worker demo:app
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
`akadze.jobs.state` is then `succeeded`. A higher `priority` is claimed first. `result` is optional. Pass enqueue options (`delay`, `run_at`, `priority`, `unique_key`, `expires_at`) through `using(...)`, not as task arguments. `expires_at` must be timezone-aware. A job that has already expired is cancelled before it runs. A second enqueue with the same active `unique_key` raises `DuplicateJob` and does not abort the rest of the caller's transaction.
|
|
94
|
+
|
|
95
|
+
Enqueue the follow-up job on the `complete_tx` connection so it commits only if this job succeeds:
|
|
96
|
+
|
|
97
|
+
```python
|
|
98
|
+
from akadze import current
|
|
99
|
+
|
|
100
|
+
async with current().complete_tx() as connection:
|
|
101
|
+
await next_step.using(session=connection).enqueue()
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
To commit your own writes together with the transition to `succeeded`, take the connection from `current().complete_tx()`. If this run no longer owns the job, or the job was cancelled or has expired, the block raises and those writes roll back. An error inside the block rolls the writes back and spends an attempt.
|
|
105
|
+
|
|
106
|
+
```python
|
|
107
|
+
from akadze import current
|
|
108
|
+
|
|
109
|
+
async with current().complete_tx() as connection:
|
|
110
|
+
await connection.execute(...)
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
`akadze.testing.drain(app)` runs every job that is ready now. `assert_enqueued(app, "hello", name="ada")` checks that a queued or running job has those arguments.
|
|
114
|
+
|
|
115
|
+
`queue_snapshot(engine)` reads committed rows: how many jobs are in each queue and state, how long the oldest ready job has waited, and how many are running. Count outcomes in an `on_transition` hook. Do not log task arguments or results.
|
|
116
|
+
|
|
117
|
+
Do not copy this package's SQL into the application's Alembic history. Deploy runs `akadze migrate`, or `await migrate(database_url)`, against the same database. The engine disables asyncpg's statement cache so a transaction-mode pool such as PgBouncer can sit in front.
|
|
118
|
+
|
|
119
|
+
Tests use `AKADZE_DATABASE_URL` and drop schema `akadze` in that database. The database name must be `akadze` or end with `_test`.
|
|
120
|
+
|
|
121
|
+
## Release
|
|
122
|
+
|
|
123
|
+
Create a tag `vX.Y.Z` that matches the version in `pyproject.toml` and push it. GitHub Actions runs the tests, publishes the wheel to PyPI, and opens a GitHub release.
|
|
124
|
+
|
|
125
|
+
PyPI trusted publishing, set once on the project `akadze`:
|
|
126
|
+
|
|
127
|
+
- Owner: `g4st3r-d3v`
|
|
128
|
+
- Repository: `akadze`
|
|
129
|
+
- Workflow: `release.yml`
|
|
130
|
+
- Environment: `pypi`
|
|
131
|
+
|
|
132
|
+
After that publish succeeds, an application installs the package from PyPI:
|
|
133
|
+
|
|
134
|
+
```bash
|
|
135
|
+
pip install akadze
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
## Layout
|
|
139
|
+
|
|
140
|
+
```text
|
|
141
|
+
akadze/ # library package
|
|
142
|
+
tests/
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
Design notes: [`g4st3r-d3v/akadze-harness`](https://github.com/g4st3r-d3v/akadze-harness) (`HUB.md`). This repo is code-only.
|
|
146
|
+
|
|
147
|
+
## License
|
|
148
|
+
|
|
149
|
+
MIT
|
|
150
|
+
|
akadze-0.1.0/README.md
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
# akadze
|
|
2
|
+
|
|
3
|
+
[](https://github.com/g4st3r-d3v/akadze/actions/workflows/ci.yml)
|
|
4
|
+
|
|
5
|
+
Postgres-backed task queue: enqueue work and run it in a worker. Periodic schedules run in that same process.
|
|
6
|
+
|
|
7
|
+
Shape is familiar if you know Celery. The broker is PostgreSQL (`FOR UPDATE SKIP LOCKED` + lease), not Redis or RabbitMQ.
|
|
8
|
+
|
|
9
|
+
**Status:** alpha. Enqueue, the worker, periodic schedules, retry, and queue maintenance run on Postgres.
|
|
10
|
+
|
|
11
|
+
## Goals
|
|
12
|
+
|
|
13
|
+
- Small library, no product domain
|
|
14
|
+
- Tasks, retries, lease recovery
|
|
15
|
+
- Periodic cron and interval schedules run inside the worker
|
|
16
|
+
- Clear Python call surface; CLI for migrate and worker
|
|
17
|
+
|
|
18
|
+
## Non-goals (for now)
|
|
19
|
+
|
|
20
|
+
- HTTP / REST / web UI (another library or app owns that)
|
|
21
|
+
- Replacing any product service queue
|
|
22
|
+
- Multi-broker adapters
|
|
23
|
+
- Distributed tracing / full observability suite
|
|
24
|
+
|
|
25
|
+
## First task
|
|
26
|
+
|
|
27
|
+
Install the library, point it at the application database, and create the `akadze` schema. The tables live in that schema; they are not part of the application's own migrations.
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
poetry install
|
|
31
|
+
export AKADZE_DATABASE_URL=postgresql://akadze:akadze@localhost:5432/akadze
|
|
32
|
+
akadze migrate
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
Save this as `demo.py`. `enqueue` runs in the caller's transaction, so a rollback removes the job.
|
|
36
|
+
|
|
37
|
+
```python
|
|
38
|
+
import asyncio
|
|
39
|
+
|
|
40
|
+
from akadze import Akadze
|
|
41
|
+
|
|
42
|
+
app = Akadze(database_url="postgresql://akadze:akadze@localhost:5432/akadze")
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@app.task("hello")
|
|
46
|
+
async def hello(name: str) -> str:
|
|
47
|
+
return f"hello {name}"
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
async def main() -> None:
|
|
51
|
+
async with app.engine.begin() as connection:
|
|
52
|
+
await hello.using(session=connection).enqueue(name="ada")
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
if __name__ == "__main__":
|
|
56
|
+
asyncio.run(main())
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
Run the worker from the directory that contains `demo.py`. It loads `demo:app` from there, claims the job, runs `hello`, and sets the row to `succeeded`. Stop it with Ctrl-C: the worker takes no new jobs and waits up to 10 seconds (`shutdown_timeout`) for the ones already running. Whatever is still running goes back to the queue, and an attempt is not spent. Periodic tasks run in this same process; there is no separate beat.
|
|
60
|
+
|
|
61
|
+
If a heartbeat does not succeed within 30 seconds, the worker stops and puts unfinished jobs back on the queue. An attempt is not spent. A synchronous task runs in a thread. Stopping the worker does not stop that thread.
|
|
62
|
+
|
|
63
|
+
A claim that fills every free slot is followed by another claim immediately. Otherwise the worker waits up to `poll_interval` (1 second). The wait is at least half of that, so idle workers do not poll in step or spin. A job can run more than once if its worker disappears.
|
|
64
|
+
|
|
65
|
+
```bash
|
|
66
|
+
python demo.py
|
|
67
|
+
akadze worker demo:app
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
`akadze.jobs.state` is then `succeeded`. A higher `priority` is claimed first. `result` is optional. Pass enqueue options (`delay`, `run_at`, `priority`, `unique_key`, `expires_at`) through `using(...)`, not as task arguments. `expires_at` must be timezone-aware. A job that has already expired is cancelled before it runs. A second enqueue with the same active `unique_key` raises `DuplicateJob` and does not abort the rest of the caller's transaction.
|
|
71
|
+
|
|
72
|
+
Enqueue the follow-up job on the `complete_tx` connection so it commits only if this job succeeds:
|
|
73
|
+
|
|
74
|
+
```python
|
|
75
|
+
from akadze import current
|
|
76
|
+
|
|
77
|
+
async with current().complete_tx() as connection:
|
|
78
|
+
await next_step.using(session=connection).enqueue()
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
To commit your own writes together with the transition to `succeeded`, take the connection from `current().complete_tx()`. If this run no longer owns the job, or the job was cancelled or has expired, the block raises and those writes roll back. An error inside the block rolls the writes back and spends an attempt.
|
|
82
|
+
|
|
83
|
+
```python
|
|
84
|
+
from akadze import current
|
|
85
|
+
|
|
86
|
+
async with current().complete_tx() as connection:
|
|
87
|
+
await connection.execute(...)
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
`akadze.testing.drain(app)` runs every job that is ready now. `assert_enqueued(app, "hello", name="ada")` checks that a queued or running job has those arguments.
|
|
91
|
+
|
|
92
|
+
`queue_snapshot(engine)` reads committed rows: how many jobs are in each queue and state, how long the oldest ready job has waited, and how many are running. Count outcomes in an `on_transition` hook. Do not log task arguments or results.
|
|
93
|
+
|
|
94
|
+
Do not copy this package's SQL into the application's Alembic history. Deploy runs `akadze migrate`, or `await migrate(database_url)`, against the same database. The engine disables asyncpg's statement cache so a transaction-mode pool such as PgBouncer can sit in front.
|
|
95
|
+
|
|
96
|
+
Tests use `AKADZE_DATABASE_URL` and drop schema `akadze` in that database. The database name must be `akadze` or end with `_test`.
|
|
97
|
+
|
|
98
|
+
## Release
|
|
99
|
+
|
|
100
|
+
Create a tag `vX.Y.Z` that matches the version in `pyproject.toml` and push it. GitHub Actions runs the tests, publishes the wheel to PyPI, and opens a GitHub release.
|
|
101
|
+
|
|
102
|
+
PyPI trusted publishing, set once on the project `akadze`:
|
|
103
|
+
|
|
104
|
+
- Owner: `g4st3r-d3v`
|
|
105
|
+
- Repository: `akadze`
|
|
106
|
+
- Workflow: `release.yml`
|
|
107
|
+
- Environment: `pypi`
|
|
108
|
+
|
|
109
|
+
After that publish succeeds, an application installs the package from PyPI:
|
|
110
|
+
|
|
111
|
+
```bash
|
|
112
|
+
pip install akadze
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
## Layout
|
|
116
|
+
|
|
117
|
+
```text
|
|
118
|
+
akadze/ # library package
|
|
119
|
+
tests/
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
Design notes: [`g4st3r-d3v/akadze-harness`](https://github.com/g4st3r-d3v/akadze-harness) (`HUB.md`). This repo is code-only.
|
|
123
|
+
|
|
124
|
+
## License
|
|
125
|
+
|
|
126
|
+
MIT
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""Postgres task queue. The broker is PostgreSQL."""
|
|
2
|
+
|
|
3
|
+
from akadze.app import Akadze
|
|
4
|
+
from akadze.context import current
|
|
5
|
+
from akadze.enqueue import request_cancel
|
|
6
|
+
from akadze.exc import AkadzeError, Cancel, DuplicateJob, EnqueueError, Fail, Retry, Snooze
|
|
7
|
+
from akadze.queue import QueueSnapshot, StateCount, queue_snapshot
|
|
8
|
+
from akadze.schema import applied_versions, migrate
|
|
9
|
+
from akadze.worker import Worker
|
|
10
|
+
|
|
11
|
+
__all__ = [
|
|
12
|
+
"Akadze",
|
|
13
|
+
"AkadzeError",
|
|
14
|
+
"Cancel",
|
|
15
|
+
"DuplicateJob",
|
|
16
|
+
"EnqueueError",
|
|
17
|
+
"Fail",
|
|
18
|
+
"QueueSnapshot",
|
|
19
|
+
"Retry",
|
|
20
|
+
"Snooze",
|
|
21
|
+
"StateCount",
|
|
22
|
+
"Worker",
|
|
23
|
+
"applied_versions",
|
|
24
|
+
"current",
|
|
25
|
+
"migrate",
|
|
26
|
+
"queue_snapshot",
|
|
27
|
+
"request_cancel",
|
|
28
|
+
]
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
"""Registry of tasks. The engine belongs to the worker, not to enqueue."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Callable
|
|
6
|
+
from datetime import timedelta
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
|
|
10
|
+
|
|
11
|
+
from akadze.exc import AkadzeError
|
|
12
|
+
from akadze.hooks import Hooks
|
|
13
|
+
from akadze.periodic import Schedule
|
|
14
|
+
from akadze.schema import async_database_url
|
|
15
|
+
from akadze.task import Task
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class Akadze:
|
|
19
|
+
def __init__(
|
|
20
|
+
self,
|
|
21
|
+
*,
|
|
22
|
+
engine: AsyncEngine | None = None,
|
|
23
|
+
database_url: str | None = None,
|
|
24
|
+
grace: timedelta = timedelta(minutes=1),
|
|
25
|
+
retention: timedelta = timedelta(days=7),
|
|
26
|
+
) -> None:
|
|
27
|
+
if engine is None and database_url is None:
|
|
28
|
+
raise AkadzeError("Akadze needs an engine or a database_url")
|
|
29
|
+
self.engine = engine or create_async_engine(
|
|
30
|
+
async_database_url(database_url or ""),
|
|
31
|
+
connect_args={"statement_cache_size": 0},
|
|
32
|
+
)
|
|
33
|
+
self._owns_engine = engine is None
|
|
34
|
+
self.grace = grace
|
|
35
|
+
self.retention = retention
|
|
36
|
+
self.hooks = Hooks()
|
|
37
|
+
self.tasks: dict[str, Task[Any]] = {}
|
|
38
|
+
self.schedules: list[Schedule] = []
|
|
39
|
+
|
|
40
|
+
def task[**P](
|
|
41
|
+
self,
|
|
42
|
+
name: str,
|
|
43
|
+
*,
|
|
44
|
+
queue: str = "default",
|
|
45
|
+
max_attempts: int = 3,
|
|
46
|
+
timeout: timedelta | None = None,
|
|
47
|
+
priority: int = 0,
|
|
48
|
+
) -> Callable[[Callable[P, Any]], Task[P]]:
|
|
49
|
+
if max_attempts < 1:
|
|
50
|
+
raise AkadzeError("max_attempts must be at least 1")
|
|
51
|
+
if timeout is not None and timeout <= timedelta(0):
|
|
52
|
+
raise AkadzeError("timeout must be positive")
|
|
53
|
+
|
|
54
|
+
def decorate(fn: Callable[P, Any]) -> Task[P]:
|
|
55
|
+
if name in self.tasks:
|
|
56
|
+
raise AkadzeError(f"task {name} is already registered")
|
|
57
|
+
registered: Task[P] = Task(
|
|
58
|
+
self,
|
|
59
|
+
name,
|
|
60
|
+
fn,
|
|
61
|
+
queue=queue,
|
|
62
|
+
max_attempts=max_attempts,
|
|
63
|
+
timeout=timeout,
|
|
64
|
+
priority=priority,
|
|
65
|
+
)
|
|
66
|
+
self.tasks[name] = registered
|
|
67
|
+
return registered
|
|
68
|
+
|
|
69
|
+
return decorate
|
|
70
|
+
|
|
71
|
+
def periodic(
|
|
72
|
+
self,
|
|
73
|
+
name: str,
|
|
74
|
+
*,
|
|
75
|
+
cron: str | None = None,
|
|
76
|
+
every: timedelta | None = None,
|
|
77
|
+
overlap: bool = True,
|
|
78
|
+
queue: str = "default",
|
|
79
|
+
max_attempts: int = 3,
|
|
80
|
+
timeout: timedelta | None = None,
|
|
81
|
+
) -> Callable[[Callable[..., Any]], Task[Any]]:
|
|
82
|
+
if (cron is None) == (every is None):
|
|
83
|
+
raise AkadzeError("periodic needs cron or every")
|
|
84
|
+
if every is not None and every <= timedelta(0):
|
|
85
|
+
raise AkadzeError("every must be positive")
|
|
86
|
+
|
|
87
|
+
def decorate(fn: Callable[..., Any]) -> Task[Any]:
|
|
88
|
+
registered = self.task(
|
|
89
|
+
name,
|
|
90
|
+
queue=queue,
|
|
91
|
+
max_attempts=max_attempts,
|
|
92
|
+
timeout=timeout,
|
|
93
|
+
)(fn)
|
|
94
|
+
self.schedules.append(
|
|
95
|
+
Schedule(name=name, cron=cron, every=every, overlap=overlap, queue=queue)
|
|
96
|
+
)
|
|
97
|
+
return registered
|
|
98
|
+
|
|
99
|
+
return decorate
|
|
100
|
+
|
|
101
|
+
async def aclose(self) -> None:
|
|
102
|
+
if self._owns_engine:
|
|
103
|
+
await self.engine.dispose()
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"""Validate task arguments against the function signature."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import inspect
|
|
6
|
+
from collections.abc import Callable
|
|
7
|
+
from typing import Any, cast, get_type_hints
|
|
8
|
+
|
|
9
|
+
from pydantic import BaseModel, ConfigDict, ValidationError, create_model
|
|
10
|
+
|
|
11
|
+
from akadze.exc import EnqueueError
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def argument_model(fn: Callable[..., Any]) -> type[BaseModel]:
|
|
15
|
+
signature = inspect.signature(fn)
|
|
16
|
+
hints = get_type_hints(fn)
|
|
17
|
+
fields: dict[str, Any] = {}
|
|
18
|
+
for name, param in signature.parameters.items():
|
|
19
|
+
if param.kind in (param.VAR_POSITIONAL, param.VAR_KEYWORD):
|
|
20
|
+
raise EnqueueError(f"task {fn.__name__} cannot take *args or **kwargs")
|
|
21
|
+
if param.kind is param.POSITIONAL_ONLY:
|
|
22
|
+
raise EnqueueError(f"task {fn.__name__} cannot take positional-only arguments")
|
|
23
|
+
annotation = hints.get(name, Any)
|
|
24
|
+
default: Any = ... if param.default is inspect.Parameter.empty else param.default
|
|
25
|
+
fields[name] = (annotation, default)
|
|
26
|
+
model = create_model(
|
|
27
|
+
f"{fn.__name__}Args",
|
|
28
|
+
__config__=ConfigDict(extra="forbid"),
|
|
29
|
+
**fields,
|
|
30
|
+
)
|
|
31
|
+
return cast(type[BaseModel], model)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def dump_arguments(
|
|
35
|
+
model: type[BaseModel],
|
|
36
|
+
arguments: dict[str, Any],
|
|
37
|
+
*,
|
|
38
|
+
task: str,
|
|
39
|
+
) -> dict[str, Any]:
|
|
40
|
+
try:
|
|
41
|
+
parsed = model.model_validate(arguments)
|
|
42
|
+
except ValidationError as exc:
|
|
43
|
+
raise EnqueueError(f"invalid arguments for task {task}") from exc
|
|
44
|
+
dumped = parsed.model_dump(mode="json")
|
|
45
|
+
if not isinstance(dumped, dict):
|
|
46
|
+
raise EnqueueError(f"invalid arguments for task {task}")
|
|
47
|
+
return dumped
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def load_arguments(model: type[BaseModel], payload: dict[str, Any]) -> dict[str, Any]:
|
|
51
|
+
try:
|
|
52
|
+
parsed = model.model_validate(payload)
|
|
53
|
+
except ValidationError:
|
|
54
|
+
raise RuntimeError("invalid arguments") from None
|
|
55
|
+
dumped = parsed.model_dump()
|
|
56
|
+
if not isinstance(dumped, dict):
|
|
57
|
+
raise RuntimeError("invalid arguments")
|
|
58
|
+
return dumped
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
"""CLI entrypoints: worker, beat, migrate."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import asyncio
|
|
7
|
+
import importlib
|
|
8
|
+
import os
|
|
9
|
+
import signal
|
|
10
|
+
import sys
|
|
11
|
+
|
|
12
|
+
from akadze.__about__ import __version__
|
|
13
|
+
from akadze.app import Akadze
|
|
14
|
+
from akadze.schema import MigrateError, applied_versions, migrate, redact_database_url
|
|
15
|
+
from akadze.worker import Worker
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def main(argv: list[str] | None = None) -> int:
|
|
19
|
+
parser = argparse.ArgumentParser(prog="akadze", description="Postgres-backed task queue")
|
|
20
|
+
parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
|
|
21
|
+
sub = parser.add_subparsers(dest="command")
|
|
22
|
+
|
|
23
|
+
worker = sub.add_parser("worker", help="Claim and run registered tasks")
|
|
24
|
+
worker.add_argument("app", help="Import path of an Akadze app, for example demo:app")
|
|
25
|
+
worker.add_argument("--slots", type=int, default=1)
|
|
26
|
+
worker.add_argument("--queue", action="append", dest="queues")
|
|
27
|
+
sub.add_parser("beat", help="Unused: periodic tasks run inside the worker")
|
|
28
|
+
sub.add_parser("migrate", help="Apply schema migrations")
|
|
29
|
+
|
|
30
|
+
args = parser.parse_args(argv)
|
|
31
|
+
if args.command is None:
|
|
32
|
+
parser.print_help()
|
|
33
|
+
return 0
|
|
34
|
+
if args.command == "migrate":
|
|
35
|
+
return _run_migrate()
|
|
36
|
+
if args.command == "beat":
|
|
37
|
+
print(
|
|
38
|
+
"periodic tasks run inside `akadze worker`; there is no beat process",
|
|
39
|
+
file=sys.stderr,
|
|
40
|
+
)
|
|
41
|
+
return 1
|
|
42
|
+
if args.command == "worker":
|
|
43
|
+
return _run_worker(args.app, slots=args.slots, queues=args.queues)
|
|
44
|
+
|
|
45
|
+
print(f"akadze {__version__}: `{args.command}` is not implemented yet.", file=sys.stderr)
|
|
46
|
+
return 1
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
async def _newly_applied(database_url: str) -> list[str]:
|
|
50
|
+
before = set(await applied_versions(database_url))
|
|
51
|
+
await migrate(database_url)
|
|
52
|
+
after = await applied_versions(database_url)
|
|
53
|
+
return [version for version in after if version not in before]
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _run_migrate() -> int:
|
|
57
|
+
database_url = os.environ.get("AKADZE_DATABASE_URL", "").strip()
|
|
58
|
+
if not database_url:
|
|
59
|
+
print("AKADZE_DATABASE_URL is not set", file=sys.stderr)
|
|
60
|
+
return 2
|
|
61
|
+
try:
|
|
62
|
+
applied = asyncio.run(_newly_applied(database_url))
|
|
63
|
+
except (MigrateError, ValueError, OSError) as exc:
|
|
64
|
+
print(f"migrate failed: {redact_database_url(str(exc), database_url)}", file=sys.stderr)
|
|
65
|
+
return 1
|
|
66
|
+
if applied:
|
|
67
|
+
for version in applied:
|
|
68
|
+
print(f"applied {version}")
|
|
69
|
+
else:
|
|
70
|
+
print("schema up to date")
|
|
71
|
+
return 0
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _run_worker(spec: str, *, slots: int, queues: list[str] | None) -> int:
|
|
75
|
+
app = _load_app(spec)
|
|
76
|
+
worker = Worker(app, slots=slots, queues=queues)
|
|
77
|
+
|
|
78
|
+
async def _run() -> None:
|
|
79
|
+
try:
|
|
80
|
+
await _serve(worker)
|
|
81
|
+
finally:
|
|
82
|
+
await app.aclose()
|
|
83
|
+
|
|
84
|
+
try:
|
|
85
|
+
asyncio.run(_run())
|
|
86
|
+
except (ValueError, OSError) as exc:
|
|
87
|
+
print(f"worker failed: {exc}", file=sys.stderr)
|
|
88
|
+
return 1
|
|
89
|
+
return 0
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
async def _serve(worker: Worker) -> None:
|
|
93
|
+
stop = asyncio.Event()
|
|
94
|
+
loop = asyncio.get_running_loop()
|
|
95
|
+
for sig in (signal.SIGINT, signal.SIGTERM):
|
|
96
|
+
loop.add_signal_handler(sig, stop.set)
|
|
97
|
+
await worker.serve(stop)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _load_app(spec: str) -> Akadze:
|
|
101
|
+
module_name, separator, attribute = spec.partition(":")
|
|
102
|
+
if separator != ":" or not module_name or not attribute:
|
|
103
|
+
print("app must look like package.module:app", file=sys.stderr)
|
|
104
|
+
raise SystemExit(2)
|
|
105
|
+
cwd = os.getcwd()
|
|
106
|
+
if cwd not in sys.path:
|
|
107
|
+
sys.path.insert(0, cwd)
|
|
108
|
+
module = importlib.import_module(module_name)
|
|
109
|
+
app = getattr(module, attribute, None)
|
|
110
|
+
if not isinstance(app, Akadze):
|
|
111
|
+
print(f"{spec} is not an Akadze app", file=sys.stderr)
|
|
112
|
+
raise SystemExit(2)
|
|
113
|
+
return app
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
if __name__ == "__main__":
|
|
117
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""The job that is running on this task, and a transaction that finishes it."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import AsyncIterator, Awaitable, Callable
|
|
6
|
+
from contextlib import asynccontextmanager
|
|
7
|
+
from contextvars import ContextVar, Token
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
|
|
10
|
+
from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine
|
|
11
|
+
|
|
12
|
+
from akadze.exc import AkadzeError
|
|
13
|
+
|
|
14
|
+
_Finish = Callable[[AsyncConnection], Awaitable[None]]
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass
|
|
18
|
+
class _Run:
|
|
19
|
+
engine: AsyncEngine
|
|
20
|
+
finish: _Finish
|
|
21
|
+
done: bool = False
|
|
22
|
+
busy: bool = False
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
_current: ContextVar[_Run | None] = ContextVar("akadze_run", default=None)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class Context:
|
|
29
|
+
"""Handle for the job executing on this asyncio task."""
|
|
30
|
+
|
|
31
|
+
def __init__(self, run: _Run) -> None:
|
|
32
|
+
self._run = run
|
|
33
|
+
|
|
34
|
+
@asynccontextmanager
|
|
35
|
+
async def complete_tx(self) -> AsyncIterator[AsyncConnection]:
|
|
36
|
+
"""Yield a connection. On a clean exit the job is finished in that transaction."""
|
|
37
|
+
|
|
38
|
+
if self._run.done:
|
|
39
|
+
raise AkadzeError("job is already complete")
|
|
40
|
+
if self._run.busy:
|
|
41
|
+
raise AkadzeError("complete_tx() is already open")
|
|
42
|
+
self._run.busy = True
|
|
43
|
+
try:
|
|
44
|
+
async with self._run.engine.begin() as connection:
|
|
45
|
+
yield connection
|
|
46
|
+
await self._run.finish(connection)
|
|
47
|
+
self._run.done = True
|
|
48
|
+
finally:
|
|
49
|
+
self._run.busy = False
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def current() -> Context:
|
|
53
|
+
"""Return the job running on this asyncio task."""
|
|
54
|
+
|
|
55
|
+
run = _current.get()
|
|
56
|
+
if run is None:
|
|
57
|
+
raise AkadzeError("current() is only valid inside a running job")
|
|
58
|
+
return Context(run)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def bind_run(engine: AsyncEngine, finish: _Finish) -> tuple[_Run, Token[_Run | None]]:
|
|
62
|
+
run = _Run(engine=engine, finish=finish)
|
|
63
|
+
return run, _current.set(run)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def reset_run(token: Token[_Run | None]) -> None:
|
|
67
|
+
_current.reset(token)
|