django-absurd 0.1.0a1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- django_absurd-0.1.0a1/LICENSE +21 -0
- django_absurd-0.1.0a1/MANIFEST.in +10 -0
- django_absurd-0.1.0a1/PKG-INFO +63 -0
- django_absurd-0.1.0a1/README.md +32 -0
- django_absurd-0.1.0a1/django_absurd/AGENTS.md +43 -0
- django_absurd-0.1.0a1/django_absurd/__init__.py +1 -0
- django_absurd-0.1.0a1/django_absurd/apps.py +9 -0
- django_absurd-0.1.0a1/django_absurd/backends.py +210 -0
- django_absurd-0.1.0a1/django_absurd/checks.py +217 -0
- django_absurd-0.1.0a1/django_absurd/connection.py +34 -0
- django_absurd-0.1.0a1/django_absurd/management/__init__.py +0 -0
- django_absurd-0.1.0a1/django_absurd/management/commands/__init__.py +0 -0
- django_absurd-0.1.0a1/django_absurd/management/commands/absurd_sync_queues.py +26 -0
- django_absurd-0.1.0a1/django_absurd/management/commands/absurd_worker.py +95 -0
- django_absurd-0.1.0a1/django_absurd/migrations/0001_initial_0_4_0.py +76 -0
- django_absurd-0.1.0a1/django_absurd/migrations/0001_initial_0_4_0.sql +3091 -0
- django_absurd-0.1.0a1/django_absurd/migrations/__init__.py +0 -0
- django_absurd-0.1.0a1/django_absurd/models.py +50 -0
- django_absurd-0.1.0a1/django_absurd/params.py +47 -0
- django_absurd-0.1.0a1/django_absurd/py.typed +0 -0
- django_absurd-0.1.0a1/django_absurd/queues.py +80 -0
- django_absurd-0.1.0a1/django_absurd/routers.py +26 -0
- django_absurd-0.1.0a1/django_absurd/worker.py +235 -0
- django_absurd-0.1.0a1/django_absurd.egg-info/PKG-INFO +63 -0
- django_absurd-0.1.0a1/django_absurd.egg-info/SOURCES.txt +28 -0
- django_absurd-0.1.0a1/django_absurd.egg-info/dependency_links.txt +1 -0
- django_absurd-0.1.0a1/django_absurd.egg-info/requires.txt +2 -0
- django_absurd-0.1.0a1/django_absurd.egg-info/top_level.txt +1 -0
- django_absurd-0.1.0a1/pyproject.toml +161 -0
- django_absurd-0.1.0a1/setup.cfg +4 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Lincoln Loop
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# setuptools-scm's git file-finder sweeps every tracked file into the sdist.
|
|
2
|
+
# Prune dev/internal files so only django_absurd (plus the standard packaging
|
|
3
|
+
# files: pyproject.toml, README, LICENSE) ships.
|
|
4
|
+
prune docs
|
|
5
|
+
prune examples
|
|
6
|
+
prune tests
|
|
7
|
+
prune .github
|
|
8
|
+
exclude CLAUDE.md compose.yaml renovate.json tox.ini uv.lock
|
|
9
|
+
exclude .dockerignore .editorconfig .envrc .gitignore .python-version
|
|
10
|
+
exclude .pre-commit-config.yaml .prettierrc.toml .yamllint.yml
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: django-absurd
|
|
3
|
+
Version: 0.1.0a1
|
|
4
|
+
Summary: Django integration for Absurd, the Postgres-native workflow engine.
|
|
5
|
+
Author-email: Marc Gibbons <marc@lincolnloop.com>, Lincoln Loop <info@lincolnloop.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/lincolnloop/django-absurd
|
|
8
|
+
Project-URL: Issues, https://github.com/lincolnloop/django-absurd/issues
|
|
9
|
+
Project-URL: Repository, https://github.com/lincolnloop/django-absurd
|
|
10
|
+
Keywords: absurd,background-jobs,django,postgres,queue,tasks,workflow,workflow-engine
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Environment :: Web Environment
|
|
13
|
+
Classifier: Framework :: Django
|
|
14
|
+
Classifier: Framework :: Django :: 6.0
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python
|
|
18
|
+
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
22
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
23
|
+
Classifier: Topic :: System :: Distributed Computing
|
|
24
|
+
Classifier: Typing :: Typed
|
|
25
|
+
Requires-Python: >=3.12
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
License-File: LICENSE
|
|
28
|
+
Requires-Dist: absurd-sdk<0.5.0,>=0.4.0
|
|
29
|
+
Requires-Dist: Django>=6.0
|
|
30
|
+
Dynamic: license-file
|
|
31
|
+
|
|
32
|
+
# django-absurd
|
|
33
|
+
|
|
34
|
+
Django integration for [Absurd](https://earendil-works.github.io/absurd/), the
|
|
35
|
+
Postgres-native workflow engine. Wraps Absurd's SDK so it reuses Django's database
|
|
36
|
+
connection, ships its schema as Django migrations, and exposes its queues and tasks
|
|
37
|
+
through Django settings, management commands, and system checks.
|
|
38
|
+
|
|
39
|
+
> **Alpha.** APIs and behavior may change between releases.
|
|
40
|
+
|
|
41
|
+
## Requirements
|
|
42
|
+
|
|
43
|
+
- Python 3.12+
|
|
44
|
+
- Django 6.0+
|
|
45
|
+
- PostgreSQL with the **psycopg (v3)** Django backend (the Absurd SDK reuses Django's
|
|
46
|
+
connection and requires psycopg3)
|
|
47
|
+
|
|
48
|
+
## Installation
|
|
49
|
+
|
|
50
|
+
```console
|
|
51
|
+
pip install django-absurd
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
Pre-release tags (e.g. `v0.1.0a1`) upload as PyPI pre-releases, which `pip install`
|
|
55
|
+
skips unless you pass `--pre`:
|
|
56
|
+
|
|
57
|
+
```console
|
|
58
|
+
pip install --pre django-absurd
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
## License
|
|
62
|
+
|
|
63
|
+
MIT — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# django-absurd
|
|
2
|
+
|
|
3
|
+
Django integration for [Absurd](https://earendil-works.github.io/absurd/), the
|
|
4
|
+
Postgres-native workflow engine. Wraps Absurd's SDK so it reuses Django's database
|
|
5
|
+
connection, ships its schema as Django migrations, and exposes its queues and tasks
|
|
6
|
+
through Django settings, management commands, and system checks.
|
|
7
|
+
|
|
8
|
+
> **Alpha.** APIs and behavior may change between releases.
|
|
9
|
+
|
|
10
|
+
## Requirements
|
|
11
|
+
|
|
12
|
+
- Python 3.12+
|
|
13
|
+
- Django 6.0+
|
|
14
|
+
- PostgreSQL with the **psycopg (v3)** Django backend (the Absurd SDK reuses Django's
|
|
15
|
+
connection and requires psycopg3)
|
|
16
|
+
|
|
17
|
+
## Installation
|
|
18
|
+
|
|
19
|
+
```console
|
|
20
|
+
pip install django-absurd
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
Pre-release tags (e.g. `v0.1.0a1`) upload as PyPI pre-releases, which `pip install`
|
|
24
|
+
skips unless you pass `--pre`:
|
|
25
|
+
|
|
26
|
+
```console
|
|
27
|
+
pip install --pre django-absurd
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
## License
|
|
31
|
+
|
|
32
|
+
MIT — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# django-absurd — agent guide
|
|
2
|
+
|
|
3
|
+
Guidance for coding agents integrating **django-absurd** into a Django project. This
|
|
4
|
+
file ships inside the installed package (`site-packages/django_absurd/AGENTS.md`) so it
|
|
5
|
+
is discoverable from a project's virtualenv.
|
|
6
|
+
|
|
7
|
+
django-absurd is a Django integration for
|
|
8
|
+
[Absurd](https://earendil-works.github.io/absurd/), a Postgres-native workflow engine.
|
|
9
|
+
It reuses Django's database connection, ships Absurd's schema as Django migrations, and
|
|
10
|
+
surfaces queues/tasks through Django settings, management commands, and system checks.
|
|
11
|
+
|
|
12
|
+
## Hard requirements
|
|
13
|
+
|
|
14
|
+
- **Python 3.12+**, **Django 6.0+**.
|
|
15
|
+
- **PostgreSQL via the psycopg (v3) Django backend.** The Absurd SDK reuses Django's
|
|
16
|
+
connection and requires psycopg3 — `django.db.backends.postgresql` with psycopg3
|
|
17
|
+
installed. A psycopg2 setup will not work. The app asserts this; do not work around
|
|
18
|
+
it.
|
|
19
|
+
- Targets `DATABASES['default']`.
|
|
20
|
+
|
|
21
|
+
## Setup checklist
|
|
22
|
+
|
|
23
|
+
1. Install: `pip install django-absurd` (pre-releases: add `--pre`).
|
|
24
|
+
2. Add `django_absurd` to `INSTALLED_APPS`.
|
|
25
|
+
3. Configure Django's `TASKS` setting to use the Absurd backend (see the project README
|
|
26
|
+
and the upstream Absurd docs for backend path and `OPTIONS`).
|
|
27
|
+
4. Run `python manage.py migrate` — Absurd's schema is applied via shipped SQL
|
|
28
|
+
migrations (no network access needed at migrate time).
|
|
29
|
+
5. Sync declared queues: `python manage.py absurd_sync_queues`.
|
|
30
|
+
6. Run a worker: `python manage.py absurd_worker`.
|
|
31
|
+
|
|
32
|
+
## Validation
|
|
33
|
+
|
|
34
|
+
- Run `python manage.py check django_absurd` — system checks flag misconfiguration
|
|
35
|
+
(wrong DB backend, queues declared but not synced, missing router, etc.). Treat check
|
|
36
|
+
failures as blocking; fix configuration rather than silencing them.
|
|
37
|
+
|
|
38
|
+
## Notes
|
|
39
|
+
|
|
40
|
+
- Migrations are offline and sourced only from the pinned Absurd schema — never fetch at
|
|
41
|
+
migrate time.
|
|
42
|
+
- This is alpha software; confirm behavior against the installed version's README rather
|
|
43
|
+
than assuming API stability.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
ABSURD_SCHEMA_VERSION = "0.4.0"
|
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
import typing as t
|
|
2
|
+
|
|
3
|
+
import psycopg.errors
|
|
4
|
+
import psycopg.sql
|
|
5
|
+
from absurd_sdk import CreateQueueOptions
|
|
6
|
+
from django.core.exceptions import ImproperlyConfigured
|
|
7
|
+
from django.db import connections, transaction
|
|
8
|
+
from django.db.utils import ProgrammingError
|
|
9
|
+
from django.tasks import TaskResult, TaskResultStatus
|
|
10
|
+
from django.tasks.backends.base import BaseTaskBackend
|
|
11
|
+
from django.tasks.base import TaskError
|
|
12
|
+
from django.tasks.exceptions import TaskResultDoesNotExist
|
|
13
|
+
from django.utils import timezone
|
|
14
|
+
from django.utils.module_loading import import_string
|
|
15
|
+
|
|
16
|
+
from django_absurd.connection import build_absurd_client, register_jsonb_loader
|
|
17
|
+
|
|
18
|
+
if t.TYPE_CHECKING:
|
|
19
|
+
from django.tasks.base import Task
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class AbsurdBackendOptions(t.TypedDict, total=False):
|
|
23
|
+
DATABASE: str
|
|
24
|
+
DEFAULT_MAX_ATTEMPTS: int
|
|
25
|
+
QUEUES: dict[str, CreateQueueOptions]
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class AbsurdBackend(BaseTaskBackend):
|
|
29
|
+
supports_get_result = True
|
|
30
|
+
supports_async_task = False
|
|
31
|
+
supports_defer = False
|
|
32
|
+
supports_priority = False
|
|
33
|
+
|
|
34
|
+
def __init__(self, alias: str, params: dict[str, t.Any]) -> None:
|
|
35
|
+
self.has_top_level_queues: bool = "QUEUES" in params
|
|
36
|
+
super().__init__(alias, params)
|
|
37
|
+
if "QUEUES" in self.options:
|
|
38
|
+
self.queues = set(self.options["QUEUES"]) # type: ignore[assignment]
|
|
39
|
+
self.database: str = self.options.get("DATABASE", "default")
|
|
40
|
+
self.default_max_attempts: int = self.options.get("DEFAULT_MAX_ATTEMPTS", 5)
|
|
41
|
+
|
|
42
|
+
def enqueue(
|
|
43
|
+
self, task: "Task", args: list[t.Any], kwargs: dict[str, t.Any]
|
|
44
|
+
) -> TaskResult:
|
|
45
|
+
self.validate_task(task)
|
|
46
|
+
client = build_absurd_client(self.database)
|
|
47
|
+
spawn_params = kwargs.pop("absurd_spawn_params", None)
|
|
48
|
+
defaults = getattr(task.func, "absurd_default_params", None)
|
|
49
|
+
merged = build_merged_spawn_options(defaults, spawn_params)
|
|
50
|
+
max_attempts: int = merged.pop("max_attempts", self.default_max_attempts)
|
|
51
|
+
try:
|
|
52
|
+
# Savepoint so a misconfig DB error (below) rolls back only the spawn,
|
|
53
|
+
# leaving an enclosing transaction.atomic() block usable.
|
|
54
|
+
with transaction.atomic(using=self.database, savepoint=True):
|
|
55
|
+
spawn_result = client.spawn(
|
|
56
|
+
task.module_path,
|
|
57
|
+
{"args": list(args), "kwargs": dict(kwargs)},
|
|
58
|
+
queue=task.queue_name,
|
|
59
|
+
max_attempts=max_attempts,
|
|
60
|
+
**merged,
|
|
61
|
+
)
|
|
62
|
+
except (
|
|
63
|
+
psycopg.errors.UndefinedTable,
|
|
64
|
+
psycopg.errors.UndefinedFunction,
|
|
65
|
+
psycopg.errors.InvalidSchemaName,
|
|
66
|
+
):
|
|
67
|
+
msg = (
|
|
68
|
+
f"Queue '{task.queue_name}' is not provisioned in Absurd. "
|
|
69
|
+
"Run manage.py absurd_sync_queues (and manage.py migrate if the "
|
|
70
|
+
"absurd schema is absent)."
|
|
71
|
+
)
|
|
72
|
+
raise ImproperlyConfigured(msg) from None
|
|
73
|
+
return TaskResult(
|
|
74
|
+
task=task,
|
|
75
|
+
id=f"{task.queue_name}:{spawn_result['task_id']}",
|
|
76
|
+
status=TaskResultStatus.READY,
|
|
77
|
+
enqueued_at=timezone.now(),
|
|
78
|
+
started_at=None,
|
|
79
|
+
finished_at=None,
|
|
80
|
+
last_attempted_at=None,
|
|
81
|
+
args=list(args),
|
|
82
|
+
kwargs=dict(kwargs),
|
|
83
|
+
backend=self.alias,
|
|
84
|
+
errors=[],
|
|
85
|
+
worker_ids=[],
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
def get_result(self, result_id: str) -> TaskResult:
|
|
89
|
+
queue, task_id = decode_result_id(result_id)
|
|
90
|
+
if queue not in self.queues:
|
|
91
|
+
raise TaskResultDoesNotExist(result_id)
|
|
92
|
+
connection = connections[self.database]
|
|
93
|
+
connection.ensure_connection()
|
|
94
|
+
sql = psycopg.sql.SQL(
|
|
95
|
+
"SELECT t.task_name, t.params, t.enqueue_at, t.first_started_at,"
|
|
96
|
+
" t.state, t.completed_payload, t.cancelled_at,"
|
|
97
|
+
" lr.started_at AS run_started, lr.completed_at, lr.failed_at,"
|
|
98
|
+
" lr.failure_reason,"
|
|
99
|
+
" (SELECT array_agg(r.claimed_by ORDER BY r.attempt)"
|
|
100
|
+
" FROM {r} r"
|
|
101
|
+
" WHERE r.task_id = t.task_id AND r.claimed_by IS NOT NULL) AS worker_ids"
|
|
102
|
+
" FROM {t} t"
|
|
103
|
+
" LEFT JOIN {r} lr ON lr.run_id = t.last_attempt_run"
|
|
104
|
+
" WHERE t.task_id = %s"
|
|
105
|
+
).format(
|
|
106
|
+
t=psycopg.sql.Identifier("absurd", f"t_{queue}"),
|
|
107
|
+
r=psycopg.sql.Identifier("absurd", f"r_{queue}"),
|
|
108
|
+
)
|
|
109
|
+
rendered = sql.as_string(connection.connection)
|
|
110
|
+
try:
|
|
111
|
+
with (
|
|
112
|
+
transaction.atomic(using=self.database, savepoint=True),
|
|
113
|
+
connection.cursor() as cursor,
|
|
114
|
+
):
|
|
115
|
+
register_jsonb_loader(cursor)
|
|
116
|
+
cursor.execute(rendered, [task_id])
|
|
117
|
+
row = cursor.fetchone()
|
|
118
|
+
except ProgrammingError:
|
|
119
|
+
raise TaskResultDoesNotExist(result_id) from None
|
|
120
|
+
if row is None:
|
|
121
|
+
raise TaskResultDoesNotExist(result_id)
|
|
122
|
+
return build_task_result(self, result_id, queue, row)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def decode_result_id(result_id: str) -> tuple[str, str]:
|
|
126
|
+
parts = result_id.rsplit(":", 1)
|
|
127
|
+
if len(parts) != 2:
|
|
128
|
+
raise TaskResultDoesNotExist(result_id)
|
|
129
|
+
return parts[0], parts[1]
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
STATE_TO_STATUS: dict[str, TaskResultStatus] = {
|
|
133
|
+
"pending": TaskResultStatus.READY,
|
|
134
|
+
"running": TaskResultStatus.RUNNING,
|
|
135
|
+
"sleeping": TaskResultStatus.RUNNING,
|
|
136
|
+
"completed": TaskResultStatus.SUCCESSFUL,
|
|
137
|
+
"failed": TaskResultStatus.FAILED,
|
|
138
|
+
"cancelled": TaskResultStatus.FAILED,
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def map_state_to_status(state: str) -> TaskResultStatus:
|
|
143
|
+
return STATE_TO_STATUS.get(state, TaskResultStatus.READY)
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def build_task_result(
|
|
147
|
+
backend: "AbsurdBackend",
|
|
148
|
+
result_id: str,
|
|
149
|
+
queue: str,
|
|
150
|
+
row: t.Any,
|
|
151
|
+
) -> TaskResult:
|
|
152
|
+
(
|
|
153
|
+
task_name,
|
|
154
|
+
params,
|
|
155
|
+
enqueue_at,
|
|
156
|
+
first_started_at,
|
|
157
|
+
state,
|
|
158
|
+
completed_payload,
|
|
159
|
+
cancelled_at,
|
|
160
|
+
run_started,
|
|
161
|
+
completed_at,
|
|
162
|
+
failed_at,
|
|
163
|
+
failure_reason,
|
|
164
|
+
worker_ids_array,
|
|
165
|
+
) = row
|
|
166
|
+
try:
|
|
167
|
+
task_obj = import_string(task_name)
|
|
168
|
+
except ImportError:
|
|
169
|
+
msg = f"task '{task_name}' is no longer importable"
|
|
170
|
+
raise ImproperlyConfigured(msg) from None
|
|
171
|
+
if task_obj.queue_name != queue:
|
|
172
|
+
task_obj = task_obj.using(queue_name=queue)
|
|
173
|
+
status = map_state_to_status(state)
|
|
174
|
+
errors: list[TaskError] = []
|
|
175
|
+
if state == "failed" and failure_reason:
|
|
176
|
+
errors = [
|
|
177
|
+
TaskError(
|
|
178
|
+
exception_class_path=failure_reason.get("name", ""),
|
|
179
|
+
traceback=failure_reason.get("traceback")
|
|
180
|
+
or failure_reason.get("message", ""),
|
|
181
|
+
)
|
|
182
|
+
]
|
|
183
|
+
finished_at = completed_at or failed_at or cancelled_at
|
|
184
|
+
worker_ids: list[str] = worker_ids_array or []
|
|
185
|
+
result: TaskResult = TaskResult(
|
|
186
|
+
task=task_obj,
|
|
187
|
+
id=result_id,
|
|
188
|
+
status=status,
|
|
189
|
+
enqueued_at=enqueue_at,
|
|
190
|
+
started_at=first_started_at,
|
|
191
|
+
finished_at=finished_at,
|
|
192
|
+
last_attempted_at=run_started,
|
|
193
|
+
args=params["args"],
|
|
194
|
+
kwargs=params["kwargs"],
|
|
195
|
+
backend=backend.alias,
|
|
196
|
+
errors=errors,
|
|
197
|
+
worker_ids=worker_ids,
|
|
198
|
+
)
|
|
199
|
+
if state == "completed":
|
|
200
|
+
object.__setattr__(result, "_return_value", completed_payload)
|
|
201
|
+
return result
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def build_merged_spawn_options(defaults: t.Any, per_call: t.Any) -> dict[str, t.Any]:
|
|
205
|
+
merged: dict[str, t.Any] = {}
|
|
206
|
+
if defaults is not None:
|
|
207
|
+
merged.update(defaults.to_kwargs())
|
|
208
|
+
if per_call is not None:
|
|
209
|
+
merged.update(per_call.to_kwargs())
|
|
210
|
+
return merged
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
import typing as t
|
|
2
|
+
from collections.abc import Sequence
|
|
3
|
+
from datetime import timedelta
|
|
4
|
+
|
|
5
|
+
from absurd_sdk import CreateQueueOptions, QueueDetachMode, QueueStorageMode
|
|
6
|
+
from django.apps import AppConfig
|
|
7
|
+
from django.conf import settings
|
|
8
|
+
from django.core.checks import CheckMessage, Error, Tags, register
|
|
9
|
+
from django.core.checks import Warning as DjangoWarning
|
|
10
|
+
from django.core.exceptions import ImproperlyConfigured
|
|
11
|
+
from django.db import connections
|
|
12
|
+
from django.db.utils import OperationalError, ProgrammingError
|
|
13
|
+
from django.utils.connection import ConnectionDoesNotExist
|
|
14
|
+
|
|
15
|
+
from django_absurd.connection import BACKEND_ERROR_MESSAGE, validate_backend
|
|
16
|
+
from django_absurd.models import Queue
|
|
17
|
+
from django_absurd.queues import (
|
|
18
|
+
get_absurd_backends,
|
|
19
|
+
get_absurd_database,
|
|
20
|
+
get_declared_queues,
|
|
21
|
+
)
|
|
22
|
+
from django_absurd.routers import AbsurdRouter
|
|
23
|
+
|
|
24
|
+
W001_MSG = (
|
|
25
|
+
"django-absurd: the absurd schema is not migrated;"
|
|
26
|
+
" declared queues cannot be provisioned."
|
|
27
|
+
)
|
|
28
|
+
W001_HINT = "Run 'manage.py migrate', then 'manage.py absurd_sync_queues'."
|
|
29
|
+
W002_MSG = "django-absurd: declared queues are out of sync with the database."
|
|
30
|
+
W002_HINT = "Run 'manage.py absurd_sync_queues'."
|
|
31
|
+
E005_MSG = (
|
|
32
|
+
"django-absurd: a non-default DATABASE is configured but AbsurdRouter is not in"
|
|
33
|
+
" DATABASE_ROUTERS."
|
|
34
|
+
)
|
|
35
|
+
E005_HINT = "Add 'django_absurd.routers.AbsurdRouter' to settings.DATABASE_ROUTERS."
|
|
36
|
+
E001_MSG = BACKEND_ERROR_MESSAGE
|
|
37
|
+
E002_MSG = (
|
|
38
|
+
"django-absurd: both top-level QUEUES and OPTIONS['QUEUES'] are set"
|
|
39
|
+
" on the same backend."
|
|
40
|
+
)
|
|
41
|
+
E002_HINT = "Remove either the top-level QUEUES key or OPTIONS['QUEUES'] — not both."
|
|
42
|
+
E003_MSG = "django-absurd: invalid per-queue policy options."
|
|
43
|
+
E003_HINT = (
|
|
44
|
+
"Remove unknown keys and ensure storage_mode/detach_mode values"
|
|
45
|
+
" are valid SDK literals."
|
|
46
|
+
)
|
|
47
|
+
E004_MSG = (
|
|
48
|
+
"django-absurd: multiple Absurd backends are configured with distinct"
|
|
49
|
+
" DATABASE values."
|
|
50
|
+
)
|
|
51
|
+
E004_HINT = "Use a single DATABASE across all Absurd backends."
|
|
52
|
+
|
|
53
|
+
DURATION_OPTION_KEYS = frozenset(
|
|
54
|
+
("partition_lookahead", "partition_lookback", "cleanup_ttl", "detach_min_age")
|
|
55
|
+
)
|
|
56
|
+
SCALAR_OPTION_KEYS = frozenset(("cleanup_limit", "detach_mode", "storage_mode"))
|
|
57
|
+
|
|
58
|
+
VALID_QUEUE_OPTION_KEYS = set(CreateQueueOptions.__annotations__)
|
|
59
|
+
VALID_STORAGE_MODES = set(t.get_args(QueueStorageMode))
|
|
60
|
+
VALID_DETACH_MODES = set(t.get_args(QueueDetachMode))
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@register("absurd")
|
|
64
|
+
def check_absurd_config(
|
|
65
|
+
*,
|
|
66
|
+
app_configs: Sequence[AppConfig] | None,
|
|
67
|
+
**kwargs: t.Any,
|
|
68
|
+
) -> list[CheckMessage]:
|
|
69
|
+
backends = get_absurd_backends()
|
|
70
|
+
if not backends:
|
|
71
|
+
return []
|
|
72
|
+
|
|
73
|
+
errors: list[CheckMessage] = []
|
|
74
|
+
databases: set[str] = set()
|
|
75
|
+
e005_emitted = False
|
|
76
|
+
|
|
77
|
+
for backend in backends.values():
|
|
78
|
+
db = get_absurd_database(backend)
|
|
79
|
+
databases.add(db)
|
|
80
|
+
|
|
81
|
+
if backend.has_top_level_queues and "QUEUES" in backend.options:
|
|
82
|
+
errors.append(Error(E002_MSG, hint=E002_HINT, id="absurd.E002"))
|
|
83
|
+
|
|
84
|
+
declared = get_declared_queues(backend)
|
|
85
|
+
for queue_name, policy in declared.items():
|
|
86
|
+
errors.extend(validate_queue_policy(queue_name, policy))
|
|
87
|
+
|
|
88
|
+
if db != "default" and not router_installed() and not e005_emitted:
|
|
89
|
+
errors.append(Error(E005_MSG, hint=E005_HINT, id="absurd.E005"))
|
|
90
|
+
e005_emitted = True
|
|
91
|
+
|
|
92
|
+
try:
|
|
93
|
+
validate_backend(db)
|
|
94
|
+
except (OperationalError, ConnectionDoesNotExist):
|
|
95
|
+
pass
|
|
96
|
+
except ImproperlyConfigured:
|
|
97
|
+
errors.append(Error(E001_MSG, id="absurd.E001"))
|
|
98
|
+
|
|
99
|
+
if len(databases) > 1:
|
|
100
|
+
errors.append(Error(E004_MSG, hint=E004_HINT, id="absurd.E004"))
|
|
101
|
+
|
|
102
|
+
return errors
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
@register(Tags.database, "absurd")
|
|
106
|
+
def check_absurd_queue_state(
|
|
107
|
+
*,
|
|
108
|
+
app_configs: Sequence[AppConfig] | None,
|
|
109
|
+
databases: Sequence[str] | None,
|
|
110
|
+
**kwargs: t.Any,
|
|
111
|
+
) -> list[CheckMessage]:
|
|
112
|
+
if not databases:
|
|
113
|
+
return []
|
|
114
|
+
|
|
115
|
+
backends = get_absurd_backends()
|
|
116
|
+
if not backends:
|
|
117
|
+
return []
|
|
118
|
+
|
|
119
|
+
errors: list[CheckMessage] = []
|
|
120
|
+
for backend in backends.values():
|
|
121
|
+
db = get_absurd_database(backend)
|
|
122
|
+
if db not in databases:
|
|
123
|
+
continue
|
|
124
|
+
declared = get_declared_queues(backend)
|
|
125
|
+
if not declared:
|
|
126
|
+
continue
|
|
127
|
+
errors.extend(query_queue_state(db, declared))
|
|
128
|
+
|
|
129
|
+
return errors
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def validate_queue_policy(
|
|
133
|
+
queue_name: str, policy: dict[str, t.Any]
|
|
134
|
+
) -> list[CheckMessage]:
|
|
135
|
+
errors: list[CheckMessage] = [
|
|
136
|
+
Error(
|
|
137
|
+
f"{E003_MSG} Queue '{queue_name}': unknown key '{key}'.",
|
|
138
|
+
hint=E003_HINT,
|
|
139
|
+
id="absurd.E003",
|
|
140
|
+
)
|
|
141
|
+
for key in policy
|
|
142
|
+
if key not in VALID_QUEUE_OPTION_KEYS
|
|
143
|
+
]
|
|
144
|
+
if "storage_mode" in policy and policy["storage_mode"] not in VALID_STORAGE_MODES:
|
|
145
|
+
mode = policy["storage_mode"]
|
|
146
|
+
errors.append(
|
|
147
|
+
Error(
|
|
148
|
+
f"{E003_MSG} Queue '{queue_name}': invalid storage_mode '{mode}'.",
|
|
149
|
+
hint=E003_HINT,
|
|
150
|
+
id="absurd.E003",
|
|
151
|
+
)
|
|
152
|
+
)
|
|
153
|
+
if "detach_mode" in policy and policy["detach_mode"] not in VALID_DETACH_MODES:
|
|
154
|
+
mode = policy["detach_mode"]
|
|
155
|
+
errors.append(
|
|
156
|
+
Error(
|
|
157
|
+
f"{E003_MSG} Queue '{queue_name}': invalid detach_mode '{mode}'.",
|
|
158
|
+
hint=E003_HINT,
|
|
159
|
+
id="absurd.E003",
|
|
160
|
+
)
|
|
161
|
+
)
|
|
162
|
+
return errors
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def router_installed() -> bool:
|
|
166
|
+
for router in settings.DATABASE_ROUTERS:
|
|
167
|
+
if router == "django_absurd.routers.AbsurdRouter":
|
|
168
|
+
return True
|
|
169
|
+
if isinstance(router, AbsurdRouter):
|
|
170
|
+
return True
|
|
171
|
+
return False
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def query_queue_state(alias: str, declared: dict[str, dict]) -> list[CheckMessage]:
|
|
175
|
+
try:
|
|
176
|
+
actual = {
|
|
177
|
+
q.queue_name: q
|
|
178
|
+
for q in Queue.objects.using(alias).filter(queue_name__in=declared)
|
|
179
|
+
}
|
|
180
|
+
except OperationalError:
|
|
181
|
+
return []
|
|
182
|
+
except ProgrammingError:
|
|
183
|
+
return [DjangoWarning(W001_MSG, hint=W001_HINT, id="absurd.W001")]
|
|
184
|
+
|
|
185
|
+
drift = [
|
|
186
|
+
name
|
|
187
|
+
for name, opts in declared.items()
|
|
188
|
+
if name not in actual or has_option_drifted(alias, opts, actual[name])
|
|
189
|
+
]
|
|
190
|
+
if drift:
|
|
191
|
+
return [
|
|
192
|
+
DjangoWarning(
|
|
193
|
+
W002_MSG,
|
|
194
|
+
hint=f"{W002_HINT} Out of sync: {', '.join(drift)}",
|
|
195
|
+
id="absurd.W002",
|
|
196
|
+
)
|
|
197
|
+
]
|
|
198
|
+
return []
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def has_option_drifted(alias: str, opts: dict, queue: Queue) -> bool:
|
|
202
|
+
for key, declared_val in opts.items():
|
|
203
|
+
if key in DURATION_OPTION_KEYS:
|
|
204
|
+
actual_val = getattr(queue, key)
|
|
205
|
+
if parse_interval(alias, declared_val) != actual_val:
|
|
206
|
+
return True
|
|
207
|
+
elif key in SCALAR_OPTION_KEYS:
|
|
208
|
+
actual_val = getattr(queue, key)
|
|
209
|
+
if declared_val != actual_val:
|
|
210
|
+
return True
|
|
211
|
+
return False
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def parse_interval(alias: str, interval_str: str) -> timedelta:
|
|
215
|
+
with connections[alias].cursor() as cur:
|
|
216
|
+
cur.execute("SELECT %s::interval", [interval_str])
|
|
217
|
+
return cur.fetchone()[0]
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import json
|
|
2
|
+
|
|
3
|
+
import psycopg
|
|
4
|
+
import psycopg.abc
|
|
5
|
+
from absurd_sdk import Absurd
|
|
6
|
+
from django.core.exceptions import ImproperlyConfigured
|
|
7
|
+
from django.db import connections
|
|
8
|
+
from psycopg.types.json import set_json_loads
|
|
9
|
+
|
|
10
|
+
BACKEND_ERROR_MESSAGE = (
|
|
11
|
+
"django-absurd requires the psycopg (v3) PostgreSQL backend. "
|
|
12
|
+
"See https://www.psycopg.org/psycopg3/docs/"
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def validate_backend(using: str) -> None:
|
|
17
|
+
conn = connections[using]
|
|
18
|
+
conn.ensure_connection()
|
|
19
|
+
if not isinstance(conn.connection, psycopg.Connection):
|
|
20
|
+
raise ImproperlyConfigured(BACKEND_ERROR_MESSAGE)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def register_jsonb_loader(context: psycopg.abc.AdaptContext) -> None:
|
|
24
|
+
# absurd-sdk returns jsonb columns as raw strings unless we register a loader;
|
|
25
|
+
# psycopg3's built-in loader is jsonb-type-OID only and doesn't cover the
|
|
26
|
+
# un-typed bytea path the SDK's claim_tasks cursor uses.
|
|
27
|
+
# Accepts either a Connection (for the SDK path) or a Cursor (for get_result,
|
|
28
|
+
# where cursor-scope avoids poisoning Django's shared connection adapters).
|
|
29
|
+
set_json_loads(json.loads, context)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def build_absurd_client(using: str) -> Absurd:
|
|
33
|
+
validate_backend(using)
|
|
34
|
+
return Absurd(connections[using].connection)
|
|
File without changes
|
|
File without changes
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
from django.core.management.base import BaseCommand
|
|
2
|
+
|
|
3
|
+
from django_absurd.queues import SyncResult, get_absurd_backends, sync_queues
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class Command(BaseCommand):
|
|
7
|
+
help = "Create and reconcile queues declared on each Absurd task backend."
|
|
8
|
+
|
|
9
|
+
def handle(self, *args: object, **options: object) -> None:
|
|
10
|
+
backends = get_absurd_backends()
|
|
11
|
+
if not backends:
|
|
12
|
+
self.stdout.write("No Absurd task backends configured.")
|
|
13
|
+
return
|
|
14
|
+
for alias, backend in backends.items():
|
|
15
|
+
prefix = f"[{alias}] " if len(backends) > 1 else ""
|
|
16
|
+
self.report_result(prefix, sync_queues(backend))
|
|
17
|
+
|
|
18
|
+
def report_result(self, prefix: str, result: SyncResult) -> None:
|
|
19
|
+
if result.created:
|
|
20
|
+
self.stdout.write(f"{prefix}Created: {', '.join(result.created)}")
|
|
21
|
+
if result.reconciled:
|
|
22
|
+
self.stdout.write(f"{prefix}Reconciled: {', '.join(result.reconciled)}")
|
|
23
|
+
if not result.created and not result.reconciled:
|
|
24
|
+
self.stdout.write(f"{prefix}No queues to sync.")
|
|
25
|
+
for warning in result.storage_warnings:
|
|
26
|
+
self.stderr.write(self.style.WARNING(f"{prefix}{warning}"))
|