django-lambda-tasks 0.1.1__tar.gz → 0.1.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/.kiro/specs/deferred-task-enqueue/tasks.md +1 -1
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/.kiro/specs/eager-mode-example-app/design.md +0 -3
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/.kiro/steering/product.md +5 -5
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/.kiro/steering/structure.md +1 -1
- django_lambda_tasks-0.1.1/README.md → django_lambda_tasks-0.1.3/PKG-INFO +14 -8
- django_lambda_tasks-0.1.1/PKG-INFO → django_lambda_tasks-0.1.3/README.md +4 -18
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/lambda_tasks/admin.py +2 -1
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/lambda_tasks/decorators.py +1 -2
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/lambda_tasks/handler.py +1 -1
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/lambda_tasks/logging.py +5 -5
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/lambda_tasks/migrations/0001_initial.py +2 -11
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/lambda_tasks/models.py +14 -10
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/pyproject.toml +1 -1
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/tests/test_admin.py +1 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/tests/test_deferred_enqueue.py +0 -9
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/tests/test_handler.py +4 -9
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/tests/test_logging.py +26 -25
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/tests/test_models.py +118 -121
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/tests/test_serializer.py +1 -47
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/tests/test_timeouts.py +3 -2
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/.github/workflows/ci.yml +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/.github/workflows/release.yml +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/.gitignore +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/.kiro/specs/deferred-task-enqueue/.config.kiro +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/.kiro/specs/deferred-task-enqueue/design.md +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/.kiro/specs/deferred-task-enqueue/requirements.md +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/.kiro/specs/eager-mode-example-app/.config.kiro +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/.kiro/specs/eager-mode-example-app/requirements.md +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/.kiro/specs/eager-mode-example-app/tasks.md +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/.kiro/specs/ignore-errors-decorator-option/.config.kiro +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/.kiro/specs/ignore-errors-decorator-option/design.md +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/.kiro/specs/ignore-errors-decorator-option/requirements.md +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/.kiro/specs/ignore-errors-decorator-option/tasks.md +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/.kiro/specs/import-string-task-resolution/.config.kiro +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/.kiro/specs/import-string-task-resolution/design.md +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/.kiro/specs/import-string-task-resolution/requirements.md +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/.kiro/specs/import-string-task-resolution/tasks.md +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/.kiro/specs/rse-background-tasks/.config.kiro +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/.kiro/specs/rse-background-tasks/design.md +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/.kiro/specs/rse-background-tasks/requirements.md +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/.kiro/specs/rse-background-tasks/tasks.md +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/.kiro/specs/rse-background-tasks-bugfix/.config.kiro +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/.kiro/specs/rse-background-tasks-bugfix/bugfix.md +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/.kiro/specs/rse-background-tasks-bugfix/design.md +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/.kiro/specs/rse-background-tasks-bugfix/tasks.md +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/.kiro/specs/task-retry/.config.kiro +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/.kiro/specs/task-retry/design.md +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/.kiro/specs/task-retry/requirements.md +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/.kiro/specs/task-retry/tasks.md +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/.kiro/steering/tech.md +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/.pre-commit-config.yaml +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/.vscode/settings.json +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/example/README.md +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/example/example_app/__init__.py +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/example/example_app/apps.py +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/example/example_app/tasks.py +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/example/example_app/urls.py +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/example/example_app/views.py +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/example/example_project/__init__.py +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/example/example_project/settings.py +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/example/example_project/urls.py +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/example/example_project/wsgi.py +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/example/manage.py +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/lambda_tasks/__init__.py +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/lambda_tasks/apps.py +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/lambda_tasks/migrations/__init__.py +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/lambda_tasks/secret_loader.py +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/lambda_tasks/settings.py +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/lambda_tasks/timeouts.py +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/tests/conftest.py +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/tests/settings.py +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/tests/test_decorator.py +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/tests/test_decorators.py +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/tests/test_kwargs_only.py +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/tests/test_secret_loader.py +0 -0
- {django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/tests/test_settings.py +0 -0
{django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/.kiro/specs/deferred-task-enqueue/tasks.md
RENAMED
|
@@ -94,7 +94,7 @@ All new functions and methods use kwargs-only signatures (enforced by `*`).
|
|
|
94
94
|
- Add `to_json(self, **kwargs: Any) -> dict`:
|
|
95
95
|
- Pop `_delay` / `_queue` overrides (same resolution logic as `on_commit`)
|
|
96
96
|
- Validate remaining kwargs via `self._kwargs_model.model_validate(kwargs)`
|
|
97
|
-
- Build `SQSLambdaTaskMessage(task_name=...,
|
|
97
|
+
- Build `SQSLambdaTaskMessage(task_name=..., kwargs=...)`
|
|
98
98
|
- Return `SQSLambdaSQSLambdaTaskMessage(message=task_message, delay=delay, queue=queue).model_dump()`
|
|
99
99
|
- Add `enqueue_from_json(self, *, data: dict) -> None`:
|
|
100
100
|
- Validate via `SQSLambdaSQSLambdaTaskMessage.model_validate(data)`
|
|
@@ -64,7 +64,7 @@ def sync_user(*, user_id: int) -> None:
|
|
|
64
64
|
|
|
65
65
|
## retry_on
|
|
66
66
|
|
|
67
|
-
Pass a tuple of exception types to `retry_on` on `@lambda_task`. If the task raises an instance of any of those types (or a subclass), the executor automatically re-enqueues the task via `execute_on_commit` with the same kwargs,
|
|
67
|
+
Pass a tuple of exception types to `retry_on` on `@lambda_task`. If the task raises an instance of any of those types (or a subclass), the executor automatically re-enqueues the task via `execute_on_commit` with the same kwargs, and an incremented `_n_retries` counter.
|
|
68
68
|
|
|
69
69
|
- `TaskRecord.status` is set to `RETRYING` and the traceback is recorded
|
|
70
70
|
- The retry is a new invocation — the current record is terminal at `RETRYING`
|
|
@@ -98,7 +98,7 @@ class SQSLambdaTaskMessage(BaseModel):
|
|
|
98
98
|
## Execution
|
|
99
99
|
|
|
100
100
|
`SQSLambdaTaskMessage.execute_immediately()` in `models.py`:
|
|
101
|
-
1. Checks for an existing `TaskRecord` with the same `
|
|
101
|
+
1. Checks for an existing `TaskRecord` with the same `pk` via `get_or_create`
|
|
102
102
|
2. If a record already exists (any status), logs and returns immediately — duplicate deliveries are silently skipped
|
|
103
103
|
3. Resolves timeouts: decorator default → settings defaults (soft=270s, hard=300s)
|
|
104
104
|
4. Runs task inside `transaction.atomic()` + `TimeoutContext`
|
|
@@ -141,7 +141,7 @@ Behaviour:
|
|
|
141
141
|
|
|
142
142
|
## TaskRecord Model
|
|
143
143
|
|
|
144
|
-
Fields: `task_name`, `
|
|
144
|
+
Fields: `task_name`, `pk` (unique UUID), `kwargs`, `status`, `start_time`, `end_time`, `result`, `traceback`
|
|
145
145
|
|
|
146
146
|
Statuses: `RUNNING`, `SUCCESS`, `FAILED`, `RETRYING`
|
|
147
147
|
|
|
@@ -167,7 +167,7 @@ Set `LAMBDA_TASKS_EAGER = True` to run tasks synchronously in-process (no SQS).
|
|
|
167
167
|
|
|
168
168
|
## Logging
|
|
169
169
|
|
|
170
|
-
Import `task_logger` to emit log records that are automatically prefixed with the active `
|
|
170
|
+
Import `task_logger` to emit log records that are automatically prefixed with the active `message_id`:
|
|
171
171
|
|
|
172
172
|
```python
|
|
173
173
|
from lambda_tasks.logging import task_logger
|
|
@@ -179,7 +179,7 @@ def my_task(*, user_id: int) -> None:
|
|
|
179
179
|
# → "[abc-123] processing user 42"
|
|
180
180
|
```
|
|
181
181
|
|
|
182
|
-
`task_logger` is a `LoggerAdapter` wrapping the `lambda_tasks.task` logger. The executor sets the `
|
|
182
|
+
`task_logger` is a `LoggerAdapter` wrapping the `lambda_tasks.task` logger. The executor sets the `message_id` before each task runs and clears it in a `finally` block. Using your own `logging.getLogger(__name__)` is fine — those records just won't carry the prefix.
|
|
183
183
|
|
|
184
184
|
## Conventions
|
|
185
185
|
|
|
@@ -40,7 +40,7 @@ django-lambda-tasks/
|
|
|
40
40
|
- `models.py` — `TaskRecord` (Django ORM), `SQSLambdaTaskMessage` (Pydantic, SQS schema + execution logic), `SQSLambdaTask` (Pydantic, holds message + routing; `_execute()` publishes to SQS or executes eagerly; `execute_on_commit()` registers `_execute` with `transaction.on_commit`)
|
|
41
41
|
- `handler.py` — Lambda entry point; calls `resolve_secrets_into_env()` then `django.setup()` at cold start; processes SQS records independently; returns `batchItemFailures`
|
|
42
42
|
- `secret_loader.py` — resolves `LAMBDA_TASKS_SECRET_*` env vars from Secrets Manager before Django starts; validates format, detects conflicts, batches API calls, caches results in-process
|
|
43
|
-
- `logging.py` — `task_logger` singleton; `
|
|
43
|
+
- `logging.py` — `task_logger` singleton; `message_id` set/cleared around each task execution
|
|
44
44
|
- `settings.py` — `LambdaTasksSettings` instantiated fresh per use (reads live Django settings)
|
|
45
45
|
- `admin.py` — Django admin registration for `TaskRecord`
|
|
46
46
|
|
|
@@ -1,3 +1,13 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: django-lambda-tasks
|
|
3
|
+
Version: 0.1.3
|
|
4
|
+
Summary: Run async tasks in a lambda function
|
|
5
|
+
Requires-Python: >=3.10
|
|
6
|
+
Requires-Dist: boto3
|
|
7
|
+
Requires-Dist: django
|
|
8
|
+
Requires-Dist: pydantic
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
|
|
1
11
|
# Django Lambda Tasks
|
|
2
12
|
|
|
3
13
|
A Django library for offloading work to AWS Lambda outside of the request-response cycle. Tasks are defined with a decorator, enqueued to SQS on transaction commit, and executed by a Lambda handler that AWS invokes with SQS message batches. Task results, status, and metadata are persisted in the Django database.
|
|
@@ -180,7 +190,6 @@ payload = send_welcome_email.serialize(user_id=42, template="welcome")
|
|
|
180
190
|
# {
|
|
181
191
|
# "message": {
|
|
182
192
|
# "task_name": "myapp.tasks.send_welcome_email",
|
|
183
|
-
# "invocation_id": "<uuid4>",
|
|
184
193
|
# "kwargs": {"user_id": 42, "template": "welcome"}
|
|
185
194
|
# },
|
|
186
195
|
# "delay": 0,
|
|
@@ -199,8 +208,6 @@ task = SQSLambdaTask.model_validate(payload)
|
|
|
199
208
|
task.execute_on_commit()
|
|
200
209
|
```
|
|
201
210
|
|
|
202
|
-
> **Note:** `serialize()` generates a fresh `invocation_id` on every call. Capture the result once if you need a stable reference to a specific invocation.
|
|
203
|
-
|
|
204
211
|
---
|
|
205
212
|
|
|
206
213
|
## Timeouts
|
|
@@ -265,7 +272,7 @@ TaskRecord.objects.all()
|
|
|
265
272
|
TaskRecord.objects.filter(status=TaskRecord.TaskStatus.FAILED)
|
|
266
273
|
|
|
267
274
|
# Look up a specific invocation
|
|
268
|
-
TaskRecord.objects.get(
|
|
275
|
+
TaskRecord.objects.get(pk="<uuid>")
|
|
269
276
|
```
|
|
270
277
|
|
|
271
278
|
### `TaskRecord` fields
|
|
@@ -273,7 +280,6 @@ TaskRecord.objects.get(invocation_id="<uuid>")
|
|
|
273
280
|
| Field | Type | Description |
|
|
274
281
|
|---|---|---|
|
|
275
282
|
| `task_name` | `str` | Fully-qualified function name (e.g. `myapp.tasks.send_welcome_email`). |
|
|
276
|
-
| `invocation_id` | `UUID` | Unique ID generated at enqueue time. |
|
|
277
283
|
| `kwargs` | `dict` | Serialized task arguments. |
|
|
278
284
|
| `status` | `str` | One of `RUNNING`, `SUCCESS`, `FAILED`. |
|
|
279
285
|
| `start_time` | `datetime \| None` | When the worker began executing the task. |
|
|
@@ -285,7 +291,7 @@ TaskRecord.objects.get(invocation_id="<uuid>")
|
|
|
285
291
|
|
|
286
292
|
## Logging
|
|
287
293
|
|
|
288
|
-
Import `task_logger` to emit log records that are automatically prefixed with the active `
|
|
294
|
+
Import `task_logger` to emit log records that are automatically prefixed with the active `message_id`. This makes it straightforward to filter all logs for a specific task invocation in CloudWatch Logs Insights.
|
|
289
295
|
|
|
290
296
|
```python
|
|
291
297
|
from lambda_tasks.logging import task_logger
|
|
@@ -298,9 +304,9 @@ def send_welcome_email(*, user_id: int, template: str) -> str:
|
|
|
298
304
|
return "sent"
|
|
299
305
|
```
|
|
300
306
|
|
|
301
|
-
`task_logger` is a `LoggerAdapter` wrapping the `lambda_tasks.task` logger. `SQSLambdaTaskMessage.execute_immediately()` sets the `
|
|
307
|
+
`task_logger` is a `LoggerAdapter` wrapping the `lambda_tasks.task` logger. `SQSLambdaTaskMessage.execute_immediately()` sets the `message_id` before each task runs and clears it afterwards — you don't need to manage it yourself.
|
|
302
308
|
|
|
303
|
-
Using your own `logging.getLogger(__name__)` is fine too; those records just won't carry the `
|
|
309
|
+
Using your own `logging.getLogger(__name__)` is fine too; those records just won't carry the `message_id` prefix.
|
|
304
310
|
|
|
305
311
|
To filter by invocation in CloudWatch Logs Insights:
|
|
306
312
|
|
|
@@ -1,13 +1,3 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: django-lambda-tasks
|
|
3
|
-
Version: 0.1.1
|
|
4
|
-
Summary: Run async tasks in a lambda function
|
|
5
|
-
Requires-Python: >=3.10
|
|
6
|
-
Requires-Dist: boto3
|
|
7
|
-
Requires-Dist: django
|
|
8
|
-
Requires-Dist: pydantic
|
|
9
|
-
Description-Content-Type: text/markdown
|
|
10
|
-
|
|
11
1
|
# Django Lambda Tasks
|
|
12
2
|
|
|
13
3
|
A Django library for offloading work to AWS Lambda outside of the request-response cycle. Tasks are defined with a decorator, enqueued to SQS on transaction commit, and executed by a Lambda handler that AWS invokes with SQS message batches. Task results, status, and metadata are persisted in the Django database.
|
|
@@ -190,7 +180,6 @@ payload = send_welcome_email.serialize(user_id=42, template="welcome")
|
|
|
190
180
|
# {
|
|
191
181
|
# "message": {
|
|
192
182
|
# "task_name": "myapp.tasks.send_welcome_email",
|
|
193
|
-
# "invocation_id": "<uuid4>",
|
|
194
183
|
# "kwargs": {"user_id": 42, "template": "welcome"}
|
|
195
184
|
# },
|
|
196
185
|
# "delay": 0,
|
|
@@ -209,8 +198,6 @@ task = SQSLambdaTask.model_validate(payload)
|
|
|
209
198
|
task.execute_on_commit()
|
|
210
199
|
```
|
|
211
200
|
|
|
212
|
-
> **Note:** `serialize()` generates a fresh `invocation_id` on every call. Capture the result once if you need a stable reference to a specific invocation.
|
|
213
|
-
|
|
214
201
|
---
|
|
215
202
|
|
|
216
203
|
## Timeouts
|
|
@@ -275,7 +262,7 @@ TaskRecord.objects.all()
|
|
|
275
262
|
TaskRecord.objects.filter(status=TaskRecord.TaskStatus.FAILED)
|
|
276
263
|
|
|
277
264
|
# Look up a specific invocation
|
|
278
|
-
TaskRecord.objects.get(
|
|
265
|
+
TaskRecord.objects.get(pk="<uuid>")
|
|
279
266
|
```
|
|
280
267
|
|
|
281
268
|
### `TaskRecord` fields
|
|
@@ -283,7 +270,6 @@ TaskRecord.objects.get(invocation_id="<uuid>")
|
|
|
283
270
|
| Field | Type | Description |
|
|
284
271
|
|---|---|---|
|
|
285
272
|
| `task_name` | `str` | Fully-qualified function name (e.g. `myapp.tasks.send_welcome_email`). |
|
|
286
|
-
| `invocation_id` | `UUID` | Unique ID generated at enqueue time. |
|
|
287
273
|
| `kwargs` | `dict` | Serialized task arguments. |
|
|
288
274
|
| `status` | `str` | One of `RUNNING`, `SUCCESS`, `FAILED`. |
|
|
289
275
|
| `start_time` | `datetime \| None` | When the worker began executing the task. |
|
|
@@ -295,7 +281,7 @@ TaskRecord.objects.get(invocation_id="<uuid>")
|
|
|
295
281
|
|
|
296
282
|
## Logging
|
|
297
283
|
|
|
298
|
-
Import `task_logger` to emit log records that are automatically prefixed with the active `
|
|
284
|
+
Import `task_logger` to emit log records that are automatically prefixed with the active `message_id`. This makes it straightforward to filter all logs for a specific task invocation in CloudWatch Logs Insights.
|
|
299
285
|
|
|
300
286
|
```python
|
|
301
287
|
from lambda_tasks.logging import task_logger
|
|
@@ -308,9 +294,9 @@ def send_welcome_email(*, user_id: int, template: str) -> str:
|
|
|
308
294
|
return "sent"
|
|
309
295
|
```
|
|
310
296
|
|
|
311
|
-
`task_logger` is a `LoggerAdapter` wrapping the `lambda_tasks.task` logger. `SQSLambdaTaskMessage.execute_immediately()` sets the `
|
|
297
|
+
`task_logger` is a `LoggerAdapter` wrapping the `lambda_tasks.task` logger. `SQSLambdaTaskMessage.execute_immediately()` sets the `message_id` before each task runs and clears it afterwards — you don't need to manage it yourself.
|
|
312
298
|
|
|
313
|
-
Using your own `logging.getLogger(__name__)` is fine too; those records just won't carry the `
|
|
299
|
+
Using your own `logging.getLogger(__name__)` is fine too; those records just won't carry the `message_id` prefix.
|
|
314
300
|
|
|
315
301
|
To filter by invocation in CloudWatch Logs Insights:
|
|
316
302
|
|
|
@@ -10,6 +10,7 @@ from lambda_tasks.models import TaskRecord
|
|
|
10
10
|
@admin.register(TaskRecord)
|
|
11
11
|
class TaskRecordAdmin(admin.ModelAdmin):
|
|
12
12
|
list_display = (
|
|
13
|
+
"pk",
|
|
13
14
|
"task_name",
|
|
14
15
|
"status",
|
|
15
16
|
"start_time",
|
|
@@ -20,7 +21,7 @@ class TaskRecordAdmin(admin.ModelAdmin):
|
|
|
20
21
|
)
|
|
21
22
|
list_filter = ("status", "task_name")
|
|
22
23
|
date_hierarchy = "start_time"
|
|
23
|
-
search_fields = ("
|
|
24
|
+
search_fields = ("pk", "kwargs")
|
|
24
25
|
|
|
25
26
|
def get_queryset(self, request: HttpRequest) -> QuerySet:
|
|
26
27
|
return (
|
|
@@ -153,7 +153,6 @@ class LambdaTaskWrapper:
|
|
|
153
153
|
|
|
154
154
|
message = SQSLambdaTaskMessage(
|
|
155
155
|
task_name=f"{self._func.__module__}.{self._func.__qualname__}",
|
|
156
|
-
invocation_id=str(uuid.uuid4()),
|
|
157
156
|
kwargs=dict(kwargs),
|
|
158
157
|
n_retries=n_retries,
|
|
159
158
|
)
|
|
@@ -170,7 +169,7 @@ class LambdaTaskWrapper:
|
|
|
170
169
|
Accepts task kwargs plus reserved override kwargs:
|
|
171
170
|
_delay: int
|
|
172
171
|
|
|
173
|
-
Returns a dict matching SQSLambdaTaskMessage schema
|
|
172
|
+
Returns a dict matching SQSLambdaTaskMessage schema
|
|
174
173
|
|
|
175
174
|
Raises:
|
|
176
175
|
pydantic.ValidationError: if kwargs fail the task's declared type annotations.
|
|
@@ -40,7 +40,7 @@ def handler(*, event: dict, context: object) -> dict:
|
|
|
40
40
|
try:
|
|
41
41
|
SQSLambdaTaskMessage.model_validate_json(
|
|
42
42
|
record["body"]
|
|
43
|
-
).execute_immediately()
|
|
43
|
+
).execute_immediately(message_id=record["messageId"])
|
|
44
44
|
except Exception:
|
|
45
45
|
logger.error(
|
|
46
46
|
"Failed to process SQS record %s",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"""
|
|
2
2
|
Provides task_logger — a LoggerAdapter that automatically includes the
|
|
3
|
-
current
|
|
3
|
+
current message_id on every log record while a task is executing.
|
|
4
4
|
|
|
5
5
|
Usage in task functions::
|
|
6
6
|
|
|
@@ -9,7 +9,7 @@ Usage in task functions::
|
|
|
9
9
|
@lambda_task(...)
|
|
10
10
|
def my_task(*, user_id: int) -> None:
|
|
11
11
|
task_logger.info("processing user %s", user_id)
|
|
12
|
-
# → "processing user 42" tagged with the active
|
|
12
|
+
# → "processing user 42" tagged with the active message_id
|
|
13
13
|
"""
|
|
14
14
|
|
|
15
15
|
import logging
|
|
@@ -18,16 +18,16 @@ from typing import Any
|
|
|
18
18
|
|
|
19
19
|
|
|
20
20
|
class _TaskLogger(logging.LoggerAdapter):
|
|
21
|
-
"""LoggerAdapter that prepends [
|
|
21
|
+
"""LoggerAdapter that prepends [message_id] to every message."""
|
|
22
22
|
|
|
23
23
|
def __init__(self) -> None:
|
|
24
24
|
super().__init__(logging.getLogger("lambda_tasks.task"), extra={})
|
|
25
|
-
self.
|
|
25
|
+
self.message_id: str | None = None
|
|
26
26
|
|
|
27
27
|
def process(
|
|
28
28
|
self, msg: Any, kwargs: MutableMapping[str, Any]
|
|
29
29
|
) -> tuple[Any, MutableMapping[str, Any]]:
|
|
30
|
-
prefix = f"[{self.
|
|
30
|
+
prefix = f"[{self.message_id}] " if self.message_id else ""
|
|
31
31
|
return f"{prefix}{msg}", kwargs
|
|
32
32
|
|
|
33
33
|
|
{django_lambda_tasks-0.1.1 → django_lambda_tasks-0.1.3}/lambda_tasks/migrations/0001_initial.py
RENAMED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Generated by Django 6.0.3 on 2026-03-26
|
|
1
|
+
# Generated by Django 6.0.3 on 2026-03-26 14:06
|
|
2
2
|
|
|
3
3
|
from django.db import migrations, models
|
|
4
4
|
|
|
@@ -15,15 +15,9 @@ class Migration(migrations.Migration):
|
|
|
15
15
|
fields=[
|
|
16
16
|
(
|
|
17
17
|
"id",
|
|
18
|
-
models.
|
|
19
|
-
auto_created=True,
|
|
20
|
-
primary_key=True,
|
|
21
|
-
serialize=False,
|
|
22
|
-
verbose_name="ID",
|
|
23
|
-
),
|
|
18
|
+
models.UUIDField(editable=False, primary_key=True, serialize=False),
|
|
24
19
|
),
|
|
25
20
|
("task_name", models.CharField(editable=False, max_length=255)),
|
|
26
|
-
("invocation_id", models.UUIDField(editable=False, unique=True)),
|
|
27
21
|
("kwargs", models.JSONField(editable=False)),
|
|
28
22
|
("n_retries", models.PositiveSmallIntegerField(editable=False)),
|
|
29
23
|
(
|
|
@@ -50,9 +44,6 @@ class Migration(migrations.Migration):
|
|
|
50
44
|
models.Index(
|
|
51
45
|
fields=["task_name"], name="lambda_task_task_na_f00cb7_idx"
|
|
52
46
|
),
|
|
53
|
-
models.Index(
|
|
54
|
-
fields=["invocation_id"], name="lambda_task_invocat_3d5a22_idx"
|
|
55
|
-
),
|
|
56
47
|
models.Index(
|
|
57
48
|
fields=["status"], name="lambda_task_status_6901ee_idx"
|
|
58
49
|
),
|
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
import random
|
|
4
4
|
import traceback
|
|
5
|
+
import uuid
|
|
6
|
+
from typing import Any
|
|
5
7
|
|
|
6
8
|
import boto3
|
|
7
9
|
from django.core.exceptions import ImproperlyConfigured
|
|
@@ -9,7 +11,7 @@ from django.db import models, transaction
|
|
|
9
11
|
from django.db.models import Q
|
|
10
12
|
from django.utils.module_loading import import_string
|
|
11
13
|
from django.utils.timezone import now
|
|
12
|
-
from pydantic import BaseModel, ConfigDict, Field
|
|
14
|
+
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter
|
|
13
15
|
|
|
14
16
|
from lambda_tasks.logging import task_logger
|
|
15
17
|
from lambda_tasks.settings import LambdaTasksSettings
|
|
@@ -33,8 +35,8 @@ class TaskStatus(models.TextChoices):
|
|
|
33
35
|
class TaskRecord(models.Model):
|
|
34
36
|
TaskStatus = TaskStatus
|
|
35
37
|
|
|
38
|
+
id = models.UUIDField(primary_key=True, editable=False)
|
|
36
39
|
task_name = models.CharField(max_length=255, editable=False)
|
|
37
|
-
invocation_id = models.UUIDField(unique=True, editable=False)
|
|
38
40
|
kwargs = models.JSONField(editable=False)
|
|
39
41
|
n_retries = models.PositiveSmallIntegerField(editable=False)
|
|
40
42
|
status = models.CharField(
|
|
@@ -55,7 +57,6 @@ class TaskRecord(models.Model):
|
|
|
55
57
|
ordering = ["-start_time"]
|
|
56
58
|
indexes = [
|
|
57
59
|
models.Index(fields=["task_name"]),
|
|
58
|
-
models.Index(fields=["invocation_id"]),
|
|
59
60
|
models.Index(fields=["status"]),
|
|
60
61
|
models.Index(fields=["-start_time"]),
|
|
61
62
|
]
|
|
@@ -71,18 +72,21 @@ class SQSLambdaTaskMessage(BaseModel):
|
|
|
71
72
|
model_config = ConfigDict(extra="forbid")
|
|
72
73
|
|
|
73
74
|
task_name: str
|
|
74
|
-
invocation_id: str
|
|
75
75
|
kwargs: dict
|
|
76
76
|
n_retries: int = Field(default=0, ge=0)
|
|
77
77
|
|
|
78
|
-
|
|
78
|
+
@classmethod
|
|
79
|
+
def _dump_python(cls, *, data: Any) -> Any:
|
|
80
|
+
return TypeAdapter(Any).dump_python(data, mode="json")
|
|
81
|
+
|
|
82
|
+
def execute_immediately(self, *, message_id: str) -> None:
|
|
79
83
|
"""Execute a background task described.
|
|
80
84
|
|
|
81
85
|
Creates a TaskRecord, resolves timeouts, validates configuration,
|
|
82
86
|
runs the task inside an atomic block with timeout enforcement, and
|
|
83
87
|
persists the outcome.
|
|
84
88
|
"""
|
|
85
|
-
task_logger.
|
|
89
|
+
task_logger.message_id = message_id
|
|
86
90
|
|
|
87
91
|
try:
|
|
88
92
|
task_logger.info(f"Received {self.task_name}")
|
|
@@ -104,10 +108,10 @@ class SQSLambdaTaskMessage(BaseModel):
|
|
|
104
108
|
|
|
105
109
|
record, created = (
|
|
106
110
|
TaskRecord.objects.get_or_create( # ty: ignore[unresolved-attribute]
|
|
107
|
-
|
|
111
|
+
pk=message_id,
|
|
108
112
|
defaults={
|
|
109
113
|
"task_name": self.task_name,
|
|
110
|
-
"kwargs": self.kwargs,
|
|
114
|
+
"kwargs": self._dump_python(data=self.kwargs),
|
|
111
115
|
"n_retries": self.n_retries,
|
|
112
116
|
"status": TaskRecord.TaskStatus.RUNNING,
|
|
113
117
|
"start_time": now(),
|
|
@@ -200,7 +204,7 @@ class SQSLambdaTaskMessage(BaseModel):
|
|
|
200
204
|
)
|
|
201
205
|
|
|
202
206
|
finally:
|
|
203
|
-
task_logger.
|
|
207
|
+
task_logger.message_id = None
|
|
204
208
|
|
|
205
209
|
|
|
206
210
|
class SQSLambdaTask(BaseModel):
|
|
@@ -224,7 +228,7 @@ class SQSLambdaTask(BaseModel):
|
|
|
224
228
|
conf = LambdaTasksSettings()
|
|
225
229
|
|
|
226
230
|
if conf.EAGER:
|
|
227
|
-
self.message.execute_immediately()
|
|
231
|
+
self.message.execute_immediately(message_id=str(uuid.uuid4()))
|
|
228
232
|
else:
|
|
229
233
|
try:
|
|
230
234
|
queue_url = conf.QUEUES[self.queue]
|
|
@@ -58,13 +58,6 @@ def test_to_json_task_name_matches_module_qualname():
|
|
|
58
58
|
assert result["message"]["task_name"] == expected
|
|
59
59
|
|
|
60
60
|
|
|
61
|
-
def test_to_json_invocation_id_is_valid_uuid4():
|
|
62
|
-
"""result['message']['invocation_id'] is a valid UUID4 string."""
|
|
63
|
-
result = _wrapper.serialize(x=1)
|
|
64
|
-
parsed = uuid.UUID(result["message"]["invocation_id"])
|
|
65
|
-
assert parsed.version == 4
|
|
66
|
-
|
|
67
|
-
|
|
68
61
|
def test_to_json_raises_validation_error_for_wrong_type_kwargs():
|
|
69
62
|
"""to_json(x='not_an_int') raises ValidationError when x is annotated as int."""
|
|
70
63
|
with pytest.raises(ValidationError):
|
|
@@ -95,7 +88,6 @@ _valid_deferred_dict_st = st.fixed_dictionaries(
|
|
|
95
88
|
"message": st.fixed_dictionaries(
|
|
96
89
|
{
|
|
97
90
|
"task_name": st.just("tests.test_deferred_enqueue._task"),
|
|
98
|
-
"invocation_id": st.uuids().map(str),
|
|
99
91
|
"kwargs": st.fixed_dictionaries({"x": st.integers()}),
|
|
100
92
|
}
|
|
101
93
|
),
|
|
@@ -120,7 +112,6 @@ def test_p1_to_json_structural_invariant(x: int) -> None:
|
|
|
120
112
|
result = _wrapper.serialize(x=x)
|
|
121
113
|
SQSLambdaTask.model_validate(result)
|
|
122
114
|
assert result["message"]["task_name"] == f"{_task.__module__}.{_task.__qualname__}"
|
|
123
|
-
assert uuid.UUID(result["message"]["invocation_id"]).version == 4
|
|
124
115
|
assert result["message"]["kwargs"] == {"x": x}
|
|
125
116
|
assert result["delay"] == 0
|
|
126
117
|
assert result["queue"] == "default"
|
|
@@ -37,7 +37,6 @@ def _valid_body(task_name: str = "my_module.my_task", **kwargs) -> str:
|
|
|
37
37
|
return json.dumps(
|
|
38
38
|
{
|
|
39
39
|
"task_name": task_name,
|
|
40
|
-
"invocation_id": str(uuid.uuid4()),
|
|
41
40
|
"kwargs": kwargs,
|
|
42
41
|
}
|
|
43
42
|
)
|
|
@@ -78,8 +77,7 @@ class TestHandlerAllSucceed:
|
|
|
78
77
|
class TestHandlerPartialFailure:
|
|
79
78
|
def test_one_record_fails_only_that_id_in_failures(self):
|
|
80
79
|
"""One record fails → only that messageId in batchItemFailures."""
|
|
81
|
-
fail_body = _valid_body()
|
|
82
|
-
ok_body = _valid_body()
|
|
80
|
+
ok_body, fail_body = (f"{_valid_body()}-{i}" for i in range(2))
|
|
83
81
|
records = [
|
|
84
82
|
_make_record("msg-ok", ok_body),
|
|
85
83
|
_make_record("msg-fail", fail_body),
|
|
@@ -146,9 +144,7 @@ class TestHandlerIndependentProcessing:
|
|
|
146
144
|
def test_failure_does_not_prevent_subsequent_records(self):
|
|
147
145
|
"""A failure in one record does not prevent processing of subsequent records."""
|
|
148
146
|
processed = []
|
|
149
|
-
fail_body = _valid_body()
|
|
150
|
-
before_body = _valid_body()
|
|
151
|
-
after_body = _valid_body()
|
|
147
|
+
before_body, fail_body, after_body = (f"{_valid_body()}-{i}" for i in range(3))
|
|
152
148
|
|
|
153
149
|
records = [
|
|
154
150
|
_make_record("msg-before", before_body),
|
|
@@ -169,7 +165,7 @@ class TestHandlerIndependentProcessing:
|
|
|
169
165
|
msg.execute_immediately.side_effect = RuntimeError("boom")
|
|
170
166
|
else:
|
|
171
167
|
|
|
172
|
-
def _execute(mid=msg_id):
|
|
168
|
+
def _execute(mid=msg_id, **kwargs):
|
|
173
169
|
processed.append(mid)
|
|
174
170
|
|
|
175
171
|
msg.execute_immediately.side_effect = _execute
|
|
@@ -212,7 +208,7 @@ class TestHandlerIndependentProcessing:
|
|
|
212
208
|
@settings(max_examples=100)
|
|
213
209
|
def test_property_11_batch_records_processed_independently(flags):
|
|
214
210
|
"""Property 11: Every record is attempted; batchItemFailures contains exactly the failed IDs."""
|
|
215
|
-
bodies = [_valid_body() for
|
|
211
|
+
bodies = [f"{_valid_body()}-{i}" for i in range(len(flags))]
|
|
216
212
|
records = [_make_record(f"msg-{i}", bodies[i]) for i in range(len(flags))]
|
|
217
213
|
expected_failures = {f"msg-{i}" for i, ok in enumerate(flags) if not ok}
|
|
218
214
|
attempted_bodies = []
|
|
@@ -280,7 +276,6 @@ def test_property_4_django_setup_before_execute_task(monkeypatch):
|
|
|
280
276
|
body = json.dumps(
|
|
281
277
|
{
|
|
282
278
|
"task_name": "some.task",
|
|
283
|
-
"invocation_id": str(uuid.uuid4()),
|
|
284
279
|
"kwargs": {},
|
|
285
280
|
}
|
|
286
281
|
)
|
|
@@ -2,10 +2,10 @@
|
|
|
2
2
|
Tests for lambda_tasks.logging (task_logger).
|
|
3
3
|
|
|
4
4
|
Covers:
|
|
5
|
-
- task_logger prefixes messages with [
|
|
6
|
-
- task_logger emits undecorated messages when
|
|
7
|
-
-
|
|
8
|
-
- log messages emitted during task execution carry the
|
|
5
|
+
- task_logger prefixes messages with [message_id] when set
|
|
6
|
+
- task_logger emits undecorated messages when message_id is None
|
|
7
|
+
- message_id is cleared after execute_task completes (success and failure)
|
|
8
|
+
- log messages emitted during task execution carry the message_id prefix
|
|
9
9
|
"""
|
|
10
10
|
|
|
11
11
|
import logging
|
|
@@ -43,7 +43,6 @@ def _task_name(wrapper) -> str:
|
|
|
43
43
|
def _make_message(task_name: str, kwargs: dict) -> SQSLambdaTaskMessage:
|
|
44
44
|
return SQSLambdaTaskMessage(
|
|
45
45
|
task_name=task_name,
|
|
46
|
-
invocation_id=str(uuid.uuid4()),
|
|
47
46
|
kwargs=kwargs,
|
|
48
47
|
)
|
|
49
48
|
|
|
@@ -55,70 +54,72 @@ def _make_message(task_name: str, kwargs: dict) -> SQSLambdaTaskMessage:
|
|
|
55
54
|
|
|
56
55
|
class TestTaskLoggerProcess:
|
|
57
56
|
def setup_method(self):
|
|
58
|
-
task_logger.
|
|
57
|
+
task_logger.message_id = None
|
|
59
58
|
|
|
60
59
|
def teardown_method(self):
|
|
61
|
-
task_logger.
|
|
60
|
+
task_logger.message_id = None
|
|
62
61
|
|
|
63
|
-
def
|
|
64
|
-
task_logger.
|
|
62
|
+
def test_prefixes_message_when_message_id_set(self):
|
|
63
|
+
task_logger.message_id = "abc-123"
|
|
65
64
|
msg, _ = task_logger.process("hello", {})
|
|
66
65
|
assert msg == "[abc-123] hello"
|
|
67
66
|
|
|
68
|
-
def
|
|
69
|
-
task_logger.
|
|
67
|
+
def test_no_prefix_when_message_id_is_none(self):
|
|
68
|
+
task_logger.message_id = None
|
|
70
69
|
msg, _ = task_logger.process("hello", {})
|
|
71
70
|
assert msg == "hello"
|
|
72
71
|
|
|
73
72
|
def test_kwargs_passed_through_unchanged(self):
|
|
74
|
-
task_logger.
|
|
73
|
+
task_logger.message_id = "x"
|
|
75
74
|
extra = {"exc_info": True}
|
|
76
75
|
_, returned_kwargs = task_logger.process("msg", extra)
|
|
77
76
|
assert returned_kwargs is extra
|
|
78
77
|
|
|
79
78
|
|
|
80
79
|
# ---------------------------------------------------------------------------
|
|
81
|
-
# Tests:
|
|
80
|
+
# Tests: message_id lifecycle in execute_task
|
|
82
81
|
# ---------------------------------------------------------------------------
|
|
83
82
|
|
|
84
83
|
|
|
85
84
|
@pytest.mark.django_db(transaction=True)
|
|
86
85
|
class TestTaskLoggerLifecycle:
|
|
87
|
-
def
|
|
86
|
+
def test_message_id_cleared_after_success(self):
|
|
88
87
|
msg = _make_message(_task_name(_logging_task_success), {"value": 1})
|
|
89
88
|
with patch("lambda_tasks.models.TimeoutContext"):
|
|
90
|
-
msg.execute_immediately()
|
|
91
|
-
assert task_logger.
|
|
89
|
+
msg.execute_immediately(message_id=str(uuid.uuid4()))
|
|
90
|
+
assert task_logger.message_id is None
|
|
92
91
|
|
|
93
|
-
def
|
|
92
|
+
def test_message_id_cleared_after_failure(self):
|
|
94
93
|
|
|
95
94
|
msg = _make_message(_task_name(_logging_task_failure), {"value": 2})
|
|
96
95
|
with patch("lambda_tasks.models.TimeoutContext"):
|
|
97
|
-
msg.execute_immediately()
|
|
98
|
-
assert task_logger.
|
|
96
|
+
msg.execute_immediately(message_id=str(uuid.uuid4()))
|
|
97
|
+
assert task_logger.message_id is None
|
|
99
98
|
|
|
100
|
-
def
|
|
99
|
+
def test_log_records_during_task_carry_message_id(self, caplog):
|
|
101
100
|
|
|
102
101
|
msg = _make_message(_task_name(_logging_task_success), {"value": 7})
|
|
102
|
+
message_id = str(uuid.uuid4())
|
|
103
103
|
with patch("lambda_tasks.models.TimeoutContext"):
|
|
104
104
|
with caplog.at_level(logging.INFO, logger="lambda_tasks.task"):
|
|
105
|
-
msg.execute_immediately()
|
|
105
|
+
msg.execute_immediately(message_id=message_id)
|
|
106
106
|
|
|
107
107
|
task_messages = [
|
|
108
108
|
r.message for r in caplog.records if "inside task" in r.message
|
|
109
109
|
]
|
|
110
110
|
assert len(task_messages) == 1
|
|
111
|
-
assert f"[{
|
|
111
|
+
assert f"[{message_id}]" in task_messages[0]
|
|
112
112
|
|
|
113
|
-
def
|
|
113
|
+
def test_log_records_on_failure_carry_message_id(self, caplog):
|
|
114
114
|
|
|
115
115
|
msg = _make_message(_task_name(_logging_task_failure), {"value": 3})
|
|
116
|
+
message_id = str(uuid.uuid4())
|
|
116
117
|
with patch("lambda_tasks.models.TimeoutContext"):
|
|
117
118
|
with caplog.at_level(logging.INFO, logger="lambda_tasks.task"):
|
|
118
|
-
msg.execute_immediately()
|
|
119
|
+
msg.execute_immediately(message_id=message_id)
|
|
119
120
|
|
|
120
121
|
task_messages = [
|
|
121
122
|
r.message for r in caplog.records if "about to fail" in r.message
|
|
122
123
|
]
|
|
123
124
|
assert len(task_messages) == 1
|
|
124
|
-
assert f"[{
|
|
125
|
+
assert f"[{message_id}]" in task_messages[0]
|