django-lambda-tasks 0.6.5__tar.gz → 0.6.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/steering/product.md +33 -4
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/steering/structure.md +3 -1
- django_lambda_tasks-0.6.5/README.md → django_lambda_tasks-0.6.7/PKG-INFO +45 -3
- django_lambda_tasks-0.6.5/PKG-INFO → django_lambda_tasks-0.6.7/README.md +33 -15
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/lambda_tasks/apps.py +1 -0
- django_lambda_tasks-0.6.7/lambda_tasks/checks.py +45 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/lambda_tasks/handler.py +12 -17
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/lambda_tasks/models.py +10 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/lambda_tasks/settings.py +13 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/pyproject.toml +1 -1
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/tests/test_memory_limit.py +43 -59
- django_lambda_tasks-0.6.7/tests/test_noop_execution.py +165 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.github/workflows/ci.yml +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.github/workflows/release.yml +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.gitignore +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/async-local-execution/.config.kiro +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/async-local-execution/design.md +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/async-local-execution/requirements.md +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/async-local-execution/tasks.md +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/deferred-task-enqueue/.config.kiro +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/deferred-task-enqueue/design.md +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/deferred-task-enqueue/requirements.md +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/deferred-task-enqueue/tasks.md +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/eager-mode-example-app/.config.kiro +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/eager-mode-example-app/design.md +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/eager-mode-example-app/requirements.md +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/eager-mode-example-app/tasks.md +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/ignore-errors-decorator-option/.config.kiro +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/ignore-errors-decorator-option/design.md +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/ignore-errors-decorator-option/requirements.md +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/ignore-errors-decorator-option/tasks.md +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/import-string-task-resolution/.config.kiro +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/import-string-task-resolution/design.md +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/import-string-task-resolution/requirements.md +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/import-string-task-resolution/tasks.md +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/retry-delay/.config.kiro +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/retry-delay/design.md +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/retry-delay/requirements.md +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/retry-delay/tasks.md +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/rse-background-tasks/.config.kiro +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/rse-background-tasks/design.md +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/rse-background-tasks/requirements.md +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/rse-background-tasks/tasks.md +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/rse-background-tasks-bugfix/.config.kiro +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/rse-background-tasks-bugfix/bugfix.md +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/rse-background-tasks-bugfix/design.md +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/rse-background-tasks-bugfix/tasks.md +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/singleton-task/.config.kiro +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/singleton-task/design.md +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/singleton-task/requirements.md +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/singleton-task/tasks.md +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/ssm-environment-loader/.config.kiro +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/ssm-environment-loader/design.md +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/ssm-environment-loader/requirements.md +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/ssm-environment-loader/tasks.md +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/task-retry/.config.kiro +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/task-retry/design.md +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/task-retry/requirements.md +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/task-retry/tasks.md +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/steering/tech.md +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.pre-commit-config.yaml +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.vscode/settings.json +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/example/README.md +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/example/example_app/__init__.py +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/example/example_app/apps.py +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/example/example_app/tasks.py +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/example/example_app/urls.py +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/example/example_app/views.py +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/example/example_project/__init__.py +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/example/example_project/settings.py +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/example/example_project/urls.py +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/example/example_project/wsgi.py +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/example/manage.py +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/lambda_tasks/__init__.py +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/lambda_tasks/admin.py +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/lambda_tasks/decorators.py +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/lambda_tasks/environment_loader.py +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/lambda_tasks/local_executor.py +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/lambda_tasks/logging.py +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/lambda_tasks/migrations/0001_initial.py +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/lambda_tasks/migrations/0002_alter_taskrecord_status.py +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/lambda_tasks/migrations/__init__.py +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/lambda_tasks/secret_loader.py +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/lambda_tasks/tasks.py +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/lambda_tasks/timeouts.py +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/tests/conftest.py +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/tests/settings.py +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/tests/test_admin.py +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/tests/test_batch.py +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/tests/test_batch_worker.py +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/tests/test_decorator.py +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/tests/test_decorators.py +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/tests/test_deferred_enqueue.py +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/tests/test_environment_loader.py +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/tests/test_handler.py +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/tests/test_kwargs_only.py +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/tests/test_local_executor.py +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/tests/test_logging.py +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/tests/test_models.py +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/tests/test_secret_loader.py +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/tests/test_serializer.py +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/tests/test_settings.py +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/tests/test_tasks.py +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/tests/test_timeout_validation.py +0 -0
- {django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/tests/test_timeouts.py +0 -0
|
@@ -151,7 +151,7 @@ class SQSLambdaTaskMessage(BaseModel):
|
|
|
151
151
|
- A module-level `_cold_start_done` sentinel ensures the sequence runs only once; subsequent warm invocations skip it
|
|
152
152
|
- Both loaders run unconditionally (outside the `DJANGO_SETTINGS_MODULE` check) — the environment secret may provide that var, and individual secrets may depend on environment-loaded vars
|
|
153
153
|
- A temporary `StreamHandler` is attached to the `lambda_tasks` logger for the duration of the loaders so their log output is visible before Django's `LOGGING` dictConfig has run; it is removed immediately after so that Django's configuration is the sole authority on logging from that point on
|
|
154
|
-
- `resource.setrlimit(
|
|
154
|
+
- `resource.setrlimit(RLIMIT_AS)` is set from `AWS_LAMBDA_FUNCTION_MEMORY_SIZE` (Lambda) or the ECS task metadata endpoint (Fargate/Batch) so that excessive allocation raises `MemoryError` instead of triggering the OOM killer
|
|
155
155
|
|
|
156
156
|
`main()` in `handler.py` — AWS Batch container entry point (`python -m lambda_tasks.handler`):
|
|
157
157
|
- Reads `LAMBDA_TASKS_MESSAGE` env var (the serialized task JSON, set by `submit_batch_job` via container overrides)
|
|
@@ -215,11 +215,25 @@ Statuses: `RUNNING`, `SUCCESS`, `FAILED`, `RETRYING`
|
|
|
215
215
|
| `LAMBDA_TASKS_DEFAULT_HARD_TIMEOUT` | `300` | Hard timeout in seconds |
|
|
216
216
|
| `LAMBDA_TASKS_EAGER` | `False` | Run tasks synchronously in-process (no SQS) |
|
|
217
217
|
| `LAMBDA_TASKS_LOCAL_WORKERS` | `0` | Number of worker processes for async local execution (development only; mutually exclusive with `EAGER`) |
|
|
218
|
+
| `LAMBDA_TASKS_NOOP_EXECUTION` | `False` | Drop all tasks silently (for local tests; mutually exclusive with `EAGER` and `LOCAL_WORKERS`) |
|
|
218
219
|
| `LAMBDA_TASKS_MAX_RETRIES` | `2880` | Maximum retry attempts before `MaxRetriesExceededError` is raised (60 × 24 × 2) |
|
|
219
220
|
| `LAMBDA_TASKS_SINGLETON_CACHE` | `"default"` | Django cache backend used for singleton task locks |
|
|
220
221
|
|
|
221
222
|
`LAMBDA_TASKS_QUEUES` must be set and include a `"default"` key. The `"default"` queue must be an SQS queue. Both timeout values must be greater than zero and `soft_timeout` must always be strictly less than `hard_timeout`. The maximum allowed timeout depends on the queue type: 900 for SQS, 3600 for Batch.
|
|
222
223
|
|
|
224
|
+
## Noop Execution Mode
|
|
225
|
+
|
|
226
|
+
Set `LAMBDA_TASKS_NOOP_EXECUTION = True` to silently drop all tasks. Useful for local tests where task execution is irrelevant. When a task is dispatched, a warning is logged with the task name, kwargs, and queue, and execution returns immediately — no SQS message is sent, no task is executed.
|
|
227
|
+
|
|
228
|
+
```python
|
|
229
|
+
# settings/test.py
|
|
230
|
+
LAMBDA_TASKS_NOOP_EXECUTION = True
|
|
231
|
+
```
|
|
232
|
+
|
|
233
|
+
`LAMBDA_TASKS_NOOP_EXECUTION` can only be set when `LAMBDA_TASKS_EAGER` is `False` and `LAMBDA_TASKS_LOCAL_WORKERS` is `0`. Setting it alongside either raises `ImproperlyConfigured`.
|
|
234
|
+
|
|
235
|
+
A Django deployment check (`lambda_tasks.W001`) warns if this setting is enabled, ensuring it is not accidentally left on in production.
|
|
236
|
+
|
|
223
237
|
## Eager Mode
|
|
224
238
|
|
|
225
239
|
Set `LAMBDA_TASKS_EAGER = True` to run tasks synchronously in-process (no SQS). Useful for local development and tests.
|
|
@@ -238,12 +252,15 @@ LAMBDA_TASKS_LOCAL_WORKERS = 4
|
|
|
238
252
|
```
|
|
239
253
|
|
|
240
254
|
The execution mode hierarchy is:
|
|
241
|
-
1. **
|
|
242
|
-
2. **
|
|
243
|
-
3. **
|
|
255
|
+
1. **Noop mode** (`LAMBDA_TASKS_NOOP_EXECUTION=True`) — tasks are dropped with a warning log
|
|
256
|
+
2. **Eager mode** (`LAMBDA_TASKS_EAGER=True`) — synchronous, in-process, no timeouts
|
|
257
|
+
3. **Async local mode** (`LOCAL_WORKERS > 0`) — async, separate processes, timeouts enforced
|
|
258
|
+
4. **SQS mode** (default) — async, Lambda workers, timeouts enforced
|
|
244
259
|
|
|
245
260
|
`LAMBDA_TASKS_LOCAL_WORKERS` and `LAMBDA_TASKS_EAGER` are mutually exclusive — setting both raises `ImproperlyConfigured`. A negative value also raises `ImproperlyConfigured`.
|
|
246
261
|
|
|
262
|
+
`LAMBDA_TASKS_NOOP_EXECUTION` is mutually exclusive with both `LAMBDA_TASKS_EAGER` and `LAMBDA_TASKS_LOCAL_WORKERS`.
|
|
263
|
+
|
|
247
264
|
Key behaviours:
|
|
248
265
|
- The process pool is created lazily on first task submission and reused for the server lifetime
|
|
249
266
|
- Each worker process calls `django.setup()` once via the pool initializer
|
|
@@ -267,6 +284,18 @@ Relying on `atexit` alone loses this race in applications with a heavy shutdown
|
|
|
267
284
|
|
|
268
285
|
`_shutdown_pool()` does **not** use `pool.shutdown(wait=True)`. `wait=True` blocks on joining the worker processes, and a worker that ran a heavy `django.setup()` is slow to exit — `concurrent.futures` does not unlink the pool's queue semaphores until the workers actually die, so the parent's near-instant `SIGKILL` still wins and the semaphores leak. Instead `_shutdown_pool()` **terminates the worker processes first** (a near-instant signal to children we own), then calls `pool.shutdown(wait=False, cancel_futures=True)`. With the workers already gone, the queue semaphores are unlinked in milliseconds — fast enough to beat the parent's `SIGKILL`.
|
|
269
286
|
|
|
287
|
+
## Deployment Checks
|
|
288
|
+
|
|
289
|
+
`lambda_tasks.checks` registers a Django deployment check (runs with `manage.py check --deploy`) that warns if any non-production execution mode is enabled:
|
|
290
|
+
|
|
291
|
+
| ID | Condition | Message |
|
|
292
|
+
|---|---|---|
|
|
293
|
+
| `lambda_tasks.W001` | `LAMBDA_TASKS_NOOP_EXECUTION = True` | Tasks will be silently dropped |
|
|
294
|
+
| `lambda_tasks.W002` | `LAMBDA_TASKS_EAGER = True` | Tasks will run synchronously in-process |
|
|
295
|
+
| `lambda_tasks.W003` | `LAMBDA_TASKS_LOCAL_WORKERS > 0` | Tasks will run in a local process pool |
|
|
296
|
+
|
|
297
|
+
The checks are registered via `AppConfig.ready()` and tagged with `deploy=True` so they only fire during `--deploy` checks.
|
|
298
|
+
|
|
270
299
|
## Logging
|
|
271
300
|
|
|
272
301
|
Import `task_logger` to emit log records that are automatically prefixed with the active `message_id`:
|
|
@@ -12,6 +12,7 @@ django-lambda-tasks/
|
|
|
12
12
|
│ ├── __init__.py
|
|
13
13
|
│ ├── admin.py # Django admin for TaskRecord
|
|
14
14
|
│ ├── apps.py # Django AppConfig
|
|
15
|
+
│ ├── checks.py # Django deployment checks (noop, eager, local_workers)
|
|
15
16
|
│ ├── decorators.py # LambdaTaskWrapper, @lambda_task
|
|
16
17
|
│ ├── handler.py # AWS Lambda entry point + AWS Batch container entry point (main)
|
|
17
18
|
│ ├── logging.py # task_logger — invocation-scoped LoggerAdapter
|
|
@@ -48,7 +49,8 @@ django-lambda-tasks/
|
|
|
48
49
|
- `logging.py` — `task_logger` singleton; `message_id` set/cleared around each task execution
|
|
49
50
|
- `settings.py` — constants (`MAX_DELAY`, `MAX_TIMEOUT`, `MAX_BATCH_TIMEOUT`); `_is_sqs_queue()` and `_is_batch_queue()` helpers; `LambdaTasksSettings` instantiated fresh per use (reads live Django settings); `queue_max_timeout()` method returns 900 or 3600 based on queue type
|
|
50
51
|
- `admin.py` — Django admin registration for `TaskRecord`
|
|
51
|
-
- `apps.py` — Django `AppConfig`; `ready()` installs the local-executor shutdown signal handlers when `LOCAL_WORKERS > 0`
|
|
52
|
+
- `apps.py` — Django `AppConfig`; `ready()` imports `checks` to register deployment checks and installs the local-executor shutdown signal handlers when `LOCAL_WORKERS > 0`
|
|
53
|
+
- `checks.py` — Django deployment checks; `check_noop_not_in_production` warns if `NOOP_EXECUTION`, `EAGER`, or `LOCAL_WORKERS > 0` are set (runs with `manage.py check --deploy`)
|
|
52
54
|
- `tasks.py` — built-in tasks; `cleanup_task_records` deletes old `TaskRecord` rows; `submit_batch_job` receives a serialized task message and submits it to AWS Batch via `batch.submit_job()`
|
|
53
55
|
|
|
54
56
|
## Conventions
|
|
@@ -1,3 +1,15 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: django-lambda-tasks
|
|
3
|
+
Version: 0.6.7
|
|
4
|
+
Summary: Run async tasks in a lambda function
|
|
5
|
+
Requires-Python: >=3.10
|
|
6
|
+
Requires-Dist: awslambdaric
|
|
7
|
+
Requires-Dist: boto3
|
|
8
|
+
Requires-Dist: django
|
|
9
|
+
Requires-Dist: pydantic
|
|
10
|
+
Requires-Dist: redis
|
|
11
|
+
Description-Content-Type: text/markdown
|
|
12
|
+
|
|
1
13
|
# Django Lambda Tasks
|
|
2
14
|
|
|
3
15
|
A Django library for offloading work to AWS Lambda or AWS Batch outside of the request-response cycle. Tasks are defined with a decorator, enqueued to SQS on transaction commit, and executed by a Lambda handler or Batch container. Task results, status, and metadata are persisted in the Django database.
|
|
@@ -181,9 +193,10 @@ LAMBDA_TASKS_LOCAL_WORKERS = 4
|
|
|
181
193
|
Async local mode bridges the gap between eager mode and full SQS/Lambda deployment. Tasks execute in background worker processes with true parallelism and full timeout enforcement via `SIGALRM`, without requiring AWS infrastructure.
|
|
182
194
|
|
|
183
195
|
The execution mode hierarchy is:
|
|
184
|
-
1. **
|
|
185
|
-
2. **
|
|
186
|
-
3. **
|
|
196
|
+
1. **Noop** (`LAMBDA_TASKS_NOOP_EXECUTION=True`) — tasks are dropped with a warning log
|
|
197
|
+
2. **Eager** (`LAMBDA_TASKS_EAGER=True`) — synchronous, in-process, no timeouts
|
|
198
|
+
3. **Async local** (`LAMBDA_TASKS_LOCAL_WORKERS > 0`) — async, separate processes, timeouts enforced
|
|
199
|
+
4. **SQS** (default) — async, Lambda workers, timeouts enforced
|
|
187
200
|
|
|
188
201
|
Key characteristics:
|
|
189
202
|
- `LAMBDA_TASKS_LOCAL_WORKERS` and `LAMBDA_TASKS_EAGER` are mutually exclusive — setting both raises `ImproperlyConfigured`
|
|
@@ -196,6 +209,35 @@ Key characteristics:
|
|
|
196
209
|
|
|
197
210
|
---
|
|
198
211
|
|
|
212
|
+
### Noop execution (testing)
|
|
213
|
+
|
|
214
|
+
| Setting | Type | Default | Description |
|
|
215
|
+
|---|---|---|---|
|
|
216
|
+
| `LAMBDA_TASKS_NOOP_EXECUTION` | `bool` | `False` | When `True`, all tasks are silently dropped. A warning is logged with the task name, kwargs, and queue. |
|
|
217
|
+
|
|
218
|
+
```python
|
|
219
|
+
# settings/test.py
|
|
220
|
+
LAMBDA_TASKS_NOOP_EXECUTION = True
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
Noop mode is useful for test suites where task execution is irrelevant and you want to avoid mocking SQS or running tasks at all. When a task is dispatched via `execute_on_commit()`, it logs a warning and returns immediately — no SQS message is sent, no task is executed, no `TaskRecord` is created.
|
|
224
|
+
|
|
225
|
+
`LAMBDA_TASKS_NOOP_EXECUTION` is mutually exclusive with both `LAMBDA_TASKS_EAGER` and `LAMBDA_TASKS_LOCAL_WORKERS` — setting it alongside either raises `ImproperlyConfigured`.
|
|
226
|
+
|
|
227
|
+
### Deployment checks
|
|
228
|
+
|
|
229
|
+
A Django deployment check (runs with `manage.py check --deploy`) warns if any non-production execution mode is enabled:
|
|
230
|
+
|
|
231
|
+
| Check ID | Condition | Message |
|
|
232
|
+
|---|---|---|
|
|
233
|
+
| `lambda_tasks.W001` | `LAMBDA_TASKS_NOOP_EXECUTION = True` | Tasks will be silently dropped |
|
|
234
|
+
| `lambda_tasks.W002` | `LAMBDA_TASKS_EAGER = True` | Tasks will run synchronously in-process |
|
|
235
|
+
| `lambda_tasks.W003` | `LAMBDA_TASKS_LOCAL_WORKERS > 0` | Tasks will run in a local process pool |
|
|
236
|
+
|
|
237
|
+
Add `manage.py check --deploy` to your CI pipeline to catch these misconfigurations before they reach production.
|
|
238
|
+
|
|
239
|
+
---
|
|
240
|
+
|
|
199
241
|
## Decorator options
|
|
200
242
|
|
|
201
243
|
```python
|
|
@@ -1,15 +1,3 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: django-lambda-tasks
|
|
3
|
-
Version: 0.6.5
|
|
4
|
-
Summary: Run async tasks in a lambda function
|
|
5
|
-
Requires-Python: >=3.10
|
|
6
|
-
Requires-Dist: awslambdaric
|
|
7
|
-
Requires-Dist: boto3
|
|
8
|
-
Requires-Dist: django
|
|
9
|
-
Requires-Dist: pydantic
|
|
10
|
-
Requires-Dist: redis
|
|
11
|
-
Description-Content-Type: text/markdown
|
|
12
|
-
|
|
13
1
|
# Django Lambda Tasks
|
|
14
2
|
|
|
15
3
|
A Django library for offloading work to AWS Lambda or AWS Batch outside of the request-response cycle. Tasks are defined with a decorator, enqueued to SQS on transaction commit, and executed by a Lambda handler or Batch container. Task results, status, and metadata are persisted in the Django database.
|
|
@@ -193,9 +181,10 @@ LAMBDA_TASKS_LOCAL_WORKERS = 4
|
|
|
193
181
|
Async local mode bridges the gap between eager mode and full SQS/Lambda deployment. Tasks execute in background worker processes with true parallelism and full timeout enforcement via `SIGALRM`, without requiring AWS infrastructure.
|
|
194
182
|
|
|
195
183
|
The execution mode hierarchy is:
|
|
196
|
-
1. **
|
|
197
|
-
2. **
|
|
198
|
-
3. **
|
|
184
|
+
1. **Noop** (`LAMBDA_TASKS_NOOP_EXECUTION=True`) — tasks are dropped with a warning log
|
|
185
|
+
2. **Eager** (`LAMBDA_TASKS_EAGER=True`) — synchronous, in-process, no timeouts
|
|
186
|
+
3. **Async local** (`LAMBDA_TASKS_LOCAL_WORKERS > 0`) — async, separate processes, timeouts enforced
|
|
187
|
+
4. **SQS** (default) — async, Lambda workers, timeouts enforced
|
|
199
188
|
|
|
200
189
|
Key characteristics:
|
|
201
190
|
- `LAMBDA_TASKS_LOCAL_WORKERS` and `LAMBDA_TASKS_EAGER` are mutually exclusive — setting both raises `ImproperlyConfigured`
|
|
@@ -208,6 +197,35 @@ Key characteristics:
|
|
|
208
197
|
|
|
209
198
|
---
|
|
210
199
|
|
|
200
|
+
### Noop execution (testing)
|
|
201
|
+
|
|
202
|
+
| Setting | Type | Default | Description |
|
|
203
|
+
|---|---|---|---|
|
|
204
|
+
| `LAMBDA_TASKS_NOOP_EXECUTION` | `bool` | `False` | When `True`, all tasks are silently dropped. A warning is logged with the task name, kwargs, and queue. |
|
|
205
|
+
|
|
206
|
+
```python
|
|
207
|
+
# settings/test.py
|
|
208
|
+
LAMBDA_TASKS_NOOP_EXECUTION = True
|
|
209
|
+
```
|
|
210
|
+
|
|
211
|
+
Noop mode is useful for test suites where task execution is irrelevant and you want to avoid mocking SQS or running tasks at all. When a task is dispatched via `execute_on_commit()`, it logs a warning and returns immediately — no SQS message is sent, no task is executed, no `TaskRecord` is created.
|
|
212
|
+
|
|
213
|
+
`LAMBDA_TASKS_NOOP_EXECUTION` is mutually exclusive with both `LAMBDA_TASKS_EAGER` and `LAMBDA_TASKS_LOCAL_WORKERS` — setting it alongside either raises `ImproperlyConfigured`.
|
|
214
|
+
|
|
215
|
+
### Deployment checks
|
|
216
|
+
|
|
217
|
+
A Django deployment check (runs with `manage.py check --deploy`) warns if any non-production execution mode is enabled:
|
|
218
|
+
|
|
219
|
+
| Check ID | Condition | Message |
|
|
220
|
+
|---|---|---|
|
|
221
|
+
| `lambda_tasks.W001` | `LAMBDA_TASKS_NOOP_EXECUTION = True` | Tasks will be silently dropped |
|
|
222
|
+
| `lambda_tasks.W002` | `LAMBDA_TASKS_EAGER = True` | Tasks will run synchronously in-process |
|
|
223
|
+
| `lambda_tasks.W003` | `LAMBDA_TASKS_LOCAL_WORKERS > 0` | Tasks will run in a local process pool |
|
|
224
|
+
|
|
225
|
+
Add `manage.py check --deploy` to your CI pipeline to catch these misconfigurations before they reach production.
|
|
226
|
+
|
|
227
|
+
---
|
|
228
|
+
|
|
211
229
|
## Decorator options
|
|
212
230
|
|
|
213
231
|
```python
|
|
@@ -15,6 +15,7 @@ class LambdaTasksConfig(AppConfig):
|
|
|
15
15
|
on Ctrl+C, before Django's autoreloader parent SIGKILLs this child. See
|
|
16
16
|
``local_executor._install_shutdown_handlers`` for the full rationale.
|
|
17
17
|
"""
|
|
18
|
+
import lambda_tasks.checks # noqa: F401 — register deployment checks
|
|
18
19
|
from lambda_tasks.local_executor import _install_shutdown_handlers
|
|
19
20
|
from lambda_tasks.settings import LambdaTasksSettings
|
|
20
21
|
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"""Django deployment checks for lambda_tasks."""
|
|
2
|
+
|
|
3
|
+
from django.conf import settings as django_settings
|
|
4
|
+
from django.core.checks import Warning, register
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@register("lambda_tasks", deploy=True)
|
|
8
|
+
def check_noop_not_in_production(
|
|
9
|
+
*, app_configs: object, **kwargs: object
|
|
10
|
+
) -> list[Warning]:
|
|
11
|
+
"""Warn if any non-production execution mode is enabled."""
|
|
12
|
+
warnings: list[Warning] = []
|
|
13
|
+
|
|
14
|
+
if bool(getattr(django_settings, "LAMBDA_TASKS_NOOP_EXECUTION", False)):
|
|
15
|
+
warnings.append(
|
|
16
|
+
Warning(
|
|
17
|
+
"LAMBDA_TASKS_NOOP_EXECUTION is enabled. "
|
|
18
|
+
"Tasks will be silently dropped.",
|
|
19
|
+
hint="Set LAMBDA_TASKS_NOOP_EXECUTION = False for production.",
|
|
20
|
+
id="lambda_tasks.W001",
|
|
21
|
+
)
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
if bool(getattr(django_settings, "LAMBDA_TASKS_EAGER", False)):
|
|
25
|
+
warnings.append(
|
|
26
|
+
Warning(
|
|
27
|
+
"LAMBDA_TASKS_EAGER is enabled. "
|
|
28
|
+
"Tasks will run synchronously in-process.",
|
|
29
|
+
hint="Set LAMBDA_TASKS_EAGER = False for production.",
|
|
30
|
+
id="lambda_tasks.W002",
|
|
31
|
+
)
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
local_workers = int(getattr(django_settings, "LAMBDA_TASKS_LOCAL_WORKERS", 0))
|
|
35
|
+
if local_workers > 0:
|
|
36
|
+
warnings.append(
|
|
37
|
+
Warning(
|
|
38
|
+
"LAMBDA_TASKS_LOCAL_WORKERS is set to a positive value. "
|
|
39
|
+
"Tasks will run in a local process pool.",
|
|
40
|
+
hint="Set LAMBDA_TASKS_LOCAL_WORKERS = 0 for production.",
|
|
41
|
+
id="lambda_tasks.W003",
|
|
42
|
+
)
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
return warnings
|
|
@@ -12,12 +12,13 @@ timeouts. The sequence is guarded by a module-level sentinel so subsequent
|
|
|
12
12
|
warm invocations skip it.
|
|
13
13
|
"""
|
|
14
14
|
|
|
15
|
+
import json
|
|
15
16
|
import logging
|
|
16
17
|
import os
|
|
17
18
|
import resource
|
|
18
19
|
import sys
|
|
20
|
+
import urllib.request
|
|
19
21
|
import uuid
|
|
20
|
-
from pathlib import Path
|
|
21
22
|
|
|
22
23
|
import django
|
|
23
24
|
from django.apps import apps
|
|
@@ -32,37 +33,31 @@ _cold_start_done: bool = False
|
|
|
32
33
|
|
|
33
34
|
_MEMORY_RESERVED_MB = 128
|
|
34
35
|
_MEMORY_MINIMUM_LIMIT_MB = 64
|
|
35
|
-
_CGROUP_V2_MEMORY_MAX_PATH = Path("/sys/fs/cgroup/memory.max")
|
|
36
|
-
_CGROUP_V1_MEMORY_LIMIT_PATH = Path("/sys/fs/cgroup/memory/memory.limit_in_bytes")
|
|
37
36
|
|
|
38
37
|
|
|
39
38
|
def _get_memory_mb() -> int | None:
|
|
40
|
-
"""Get container memory limit from Lambda env var or
|
|
39
|
+
"""Get container memory limit from Lambda env var or ECS metadata endpoint."""
|
|
41
40
|
memory_mb_str = os.environ.get("AWS_LAMBDA_FUNCTION_MEMORY_SIZE")
|
|
42
41
|
if memory_mb_str is not None:
|
|
43
42
|
return int(memory_mb_str)
|
|
44
43
|
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
44
|
+
metadata_uri = os.environ.get("ECS_CONTAINER_METADATA_URI_V4")
|
|
45
|
+
if metadata_uri is not None:
|
|
46
|
+
try:
|
|
47
|
+
with urllib.request.urlopen(f"{metadata_uri}/task", timeout=2) as response:
|
|
48
|
+
data = json.loads(response.read())
|
|
49
|
+
return int(data["Limits"]["Memory"])
|
|
50
|
+
except Exception:
|
|
48
51
|
return None
|
|
49
|
-
else:
|
|
50
|
-
return int(value) // (1024 * 1024)
|
|
51
|
-
except (FileNotFoundError, ValueError):
|
|
52
|
-
pass
|
|
53
52
|
|
|
54
|
-
|
|
55
|
-
value = _CGROUP_V1_MEMORY_LIMIT_PATH.read_text().strip()
|
|
56
|
-
return int(value) // (1024 * 1024)
|
|
57
|
-
except (FileNotFoundError, ValueError):
|
|
58
|
-
return None
|
|
53
|
+
return None
|
|
59
54
|
|
|
60
55
|
|
|
61
56
|
def _set_memory_limit() -> None:
|
|
62
57
|
"""Set RLIMIT_AS from the container memory limit.
|
|
63
58
|
|
|
64
59
|
On Lambda, reads AWS_LAMBDA_FUNCTION_MEMORY_SIZE. On Fargate/Batch,
|
|
65
|
-
|
|
60
|
+
queries the ECS task metadata endpoint. This causes Python to raise
|
|
66
61
|
MemoryError on excessive allocation instead of being killed by the
|
|
67
62
|
OOM killer.
|
|
68
63
|
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
import contextlib
|
|
4
4
|
import datetime
|
|
5
|
+
import logging
|
|
5
6
|
import random
|
|
6
7
|
import traceback
|
|
7
8
|
import uuid
|
|
@@ -26,6 +27,8 @@ from lambda_tasks.settings import (
|
|
|
26
27
|
)
|
|
27
28
|
from lambda_tasks.timeouts import TimeoutContext
|
|
28
29
|
|
|
30
|
+
logger = logging.getLogger("lambda_tasks")
|
|
31
|
+
|
|
29
32
|
|
|
30
33
|
class MaxRetriesExceededError(Exception):
|
|
31
34
|
def __init__(self, *, task_name: str, n_retries: int) -> None:
|
|
@@ -260,6 +263,13 @@ class SQSLambdaTask(BaseModel):
|
|
|
260
263
|
"""
|
|
261
264
|
conf = LambdaTasksSettings()
|
|
262
265
|
|
|
266
|
+
if conf.NOOP_EXECUTION:
|
|
267
|
+
logger.warning(
|
|
268
|
+
f"Task {self.message.task_name} dropped (noop execution mode): "
|
|
269
|
+
f"kwargs={self.message.kwargs!r}, queue={self.queue!r}"
|
|
270
|
+
)
|
|
271
|
+
return
|
|
272
|
+
|
|
263
273
|
if conf.EAGER:
|
|
264
274
|
self.message.execute_immediately(message_id=str(uuid.uuid4()))
|
|
265
275
|
elif conf.LOCAL_WORKERS > 0:
|
|
@@ -108,3 +108,16 @@ class LambdaTasksSettings:
|
|
|
108
108
|
)
|
|
109
109
|
else:
|
|
110
110
|
return value
|
|
111
|
+
|
|
112
|
+
@property
|
|
113
|
+
def NOOP_EXECUTION(self) -> bool:
|
|
114
|
+
value = bool(getattr(django_settings, "LAMBDA_TASKS_NOOP_EXECUTION", False))
|
|
115
|
+
if value and self.EAGER:
|
|
116
|
+
raise ImproperlyConfigured(
|
|
117
|
+
"LAMBDA_TASKS_NOOP_EXECUTION and LAMBDA_TASKS_EAGER are mutually exclusive."
|
|
118
|
+
)
|
|
119
|
+
if value and int(getattr(django_settings, "LAMBDA_TASKS_LOCAL_WORKERS", 0)) > 0:
|
|
120
|
+
raise ImproperlyConfigured(
|
|
121
|
+
"LAMBDA_TASKS_NOOP_EXECUTION and LAMBDA_TASKS_LOCAL_WORKERS are mutually exclusive."
|
|
122
|
+
)
|
|
123
|
+
return value
|
|
@@ -6,30 +6,18 @@ memory allocation raises MemoryError instead of triggering the Lambda OOM
|
|
|
6
6
|
killer.
|
|
7
7
|
"""
|
|
8
8
|
|
|
9
|
+
import json
|
|
9
10
|
import resource
|
|
10
11
|
|
|
11
12
|
import lambda_tasks.handler as handler_module
|
|
12
13
|
|
|
13
14
|
|
|
14
|
-
class _FakePath:
|
|
15
|
-
def __init__(self, *, content: str | None) -> None:
|
|
16
|
-
self._content = content
|
|
17
|
-
|
|
18
|
-
def read_text(self) -> str:
|
|
19
|
-
if self._content is None:
|
|
20
|
-
raise FileNotFoundError
|
|
21
|
-
return self._content
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
def fake_path(content: str | None) -> _FakePath:
|
|
25
|
-
return _FakePath(content=content)
|
|
26
|
-
|
|
27
|
-
|
|
28
15
|
class TestMemoryLimitSetDuringColdStart:
|
|
29
16
|
def test_rlimit_as_set_from_env_var(self, monkeypatch):
|
|
30
17
|
"""When AWS_LAMBDA_FUNCTION_MEMORY_SIZE is set, RLIMIT_AS is configured with 128 MB reserved."""
|
|
31
18
|
monkeypatch.delenv("DJANGO_SETTINGS_MODULE", raising=False)
|
|
32
19
|
monkeypatch.setenv("AWS_LAMBDA_FUNCTION_MEMORY_SIZE", "512")
|
|
20
|
+
monkeypatch.delenv("ECS_CONTAINER_METADATA_URI_V4", raising=False)
|
|
33
21
|
monkeypatch.setattr("lambda_tasks.handler.resolve_environment", lambda: None)
|
|
34
22
|
monkeypatch.setattr(
|
|
35
23
|
"lambda_tasks.handler.resolve_secrets_into_env", lambda: None
|
|
@@ -48,20 +36,15 @@ class TestMemoryLimitSetDuringColdStart:
|
|
|
48
36
|
expected = (512 - 128) * 1024 * 1024
|
|
49
37
|
assert calls == [(resource.RLIMIT_AS, (expected, expected))]
|
|
50
38
|
|
|
51
|
-
def
|
|
52
|
-
"""When
|
|
39
|
+
def test_rlimit_as_not_set_when_no_memory_source(self, monkeypatch):
|
|
40
|
+
"""When neither Lambda env var nor ECS metadata is available, RLIMIT_AS is unchanged."""
|
|
53
41
|
monkeypatch.delenv("DJANGO_SETTINGS_MODULE", raising=False)
|
|
54
42
|
monkeypatch.delenv("AWS_LAMBDA_FUNCTION_MEMORY_SIZE", raising=False)
|
|
43
|
+
monkeypatch.delenv("ECS_CONTAINER_METADATA_URI_V4", raising=False)
|
|
55
44
|
monkeypatch.setattr("lambda_tasks.handler.resolve_environment", lambda: None)
|
|
56
45
|
monkeypatch.setattr(
|
|
57
46
|
"lambda_tasks.handler.resolve_secrets_into_env", lambda: None
|
|
58
47
|
)
|
|
59
|
-
monkeypatch.setattr(
|
|
60
|
-
handler_module, "_CGROUP_V2_MEMORY_MAX_PATH", fake_path(None)
|
|
61
|
-
)
|
|
62
|
-
monkeypatch.setattr(
|
|
63
|
-
handler_module, "_CGROUP_V1_MEMORY_LIMIT_PATH", fake_path(None)
|
|
64
|
-
)
|
|
65
48
|
|
|
66
49
|
calls: list[tuple] = []
|
|
67
50
|
|
|
@@ -75,25 +58,33 @@ class TestMemoryLimitSetDuringColdStart:
|
|
|
75
58
|
|
|
76
59
|
assert calls == []
|
|
77
60
|
|
|
78
|
-
def
|
|
79
|
-
"""When
|
|
61
|
+
def test_rlimit_as_set_from_ecs_metadata(self, monkeypatch):
|
|
62
|
+
"""When ECS_CONTAINER_METADATA_URI_V4 is set, queries the task endpoint for memory."""
|
|
80
63
|
monkeypatch.delenv("DJANGO_SETTINGS_MODULE", raising=False)
|
|
81
64
|
monkeypatch.delenv("AWS_LAMBDA_FUNCTION_MEMORY_SIZE", raising=False)
|
|
65
|
+
monkeypatch.setenv("ECS_CONTAINER_METADATA_URI_V4", "http://169.254.170.2/v4")
|
|
82
66
|
monkeypatch.setattr("lambda_tasks.handler.resolve_environment", lambda: None)
|
|
83
67
|
monkeypatch.setattr(
|
|
84
68
|
"lambda_tasks.handler.resolve_secrets_into_env", lambda: None
|
|
85
69
|
)
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
70
|
+
|
|
71
|
+
task_metadata = json.dumps({"Limits": {"CPU": 2, "Memory": 8192}}).encode()
|
|
72
|
+
|
|
73
|
+
class FakeResponse:
|
|
74
|
+
def read(self):
|
|
75
|
+
return task_metadata
|
|
76
|
+
|
|
77
|
+
def __enter__(self):
|
|
78
|
+
return self
|
|
79
|
+
|
|
80
|
+
def __exit__(self, *args):
|
|
81
|
+
pass
|
|
82
|
+
|
|
83
|
+
def fake_urlopen(url, *, timeout=None):
|
|
84
|
+
assert url == "http://169.254.170.2/v4/task"
|
|
85
|
+
return FakeResponse()
|
|
86
|
+
|
|
87
|
+
monkeypatch.setattr("urllib.request.urlopen", fake_urlopen)
|
|
97
88
|
|
|
98
89
|
calls: list[tuple] = []
|
|
99
90
|
|
|
@@ -105,23 +96,23 @@ class TestMemoryLimitSetDuringColdStart:
|
|
|
105
96
|
handler_module._cold_start_done = False
|
|
106
97
|
handler_module.handler(event={"Records": []}, context=None)
|
|
107
98
|
|
|
108
|
-
expected = (
|
|
99
|
+
expected = (8192 - 128) * 1024 * 1024
|
|
109
100
|
assert calls == [(resource.RLIMIT_AS, (expected, expected))]
|
|
110
101
|
|
|
111
|
-
def
|
|
112
|
-
"""When
|
|
102
|
+
def test_rlimit_as_not_set_when_ecs_metadata_fails(self, monkeypatch):
|
|
103
|
+
"""When ECS metadata endpoint fails, RLIMIT_AS is unchanged."""
|
|
113
104
|
monkeypatch.delenv("DJANGO_SETTINGS_MODULE", raising=False)
|
|
114
105
|
monkeypatch.delenv("AWS_LAMBDA_FUNCTION_MEMORY_SIZE", raising=False)
|
|
106
|
+
monkeypatch.setenv("ECS_CONTAINER_METADATA_URI_V4", "http://169.254.170.2/v4")
|
|
115
107
|
monkeypatch.setattr("lambda_tasks.handler.resolve_environment", lambda: None)
|
|
116
108
|
monkeypatch.setattr(
|
|
117
109
|
"lambda_tasks.handler.resolve_secrets_into_env", lambda: None
|
|
118
110
|
)
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
)
|
|
111
|
+
|
|
112
|
+
def fake_urlopen(url, *, timeout=None):
|
|
113
|
+
raise OSError("connection refused")
|
|
114
|
+
|
|
115
|
+
monkeypatch.setattr("urllib.request.urlopen", fake_urlopen)
|
|
125
116
|
|
|
126
117
|
calls: list[tuple] = []
|
|
127
118
|
|
|
@@ -135,25 +126,15 @@ class TestMemoryLimitSetDuringColdStart:
|
|
|
135
126
|
|
|
136
127
|
assert calls == []
|
|
137
128
|
|
|
138
|
-
def
|
|
139
|
-
"""
|
|
129
|
+
def test_lambda_env_var_takes_precedence_over_ecs_metadata(self, monkeypatch):
|
|
130
|
+
"""AWS_LAMBDA_FUNCTION_MEMORY_SIZE is preferred over ECS metadata."""
|
|
140
131
|
monkeypatch.delenv("DJANGO_SETTINGS_MODULE", raising=False)
|
|
141
|
-
monkeypatch.
|
|
132
|
+
monkeypatch.setenv("AWS_LAMBDA_FUNCTION_MEMORY_SIZE", "512")
|
|
133
|
+
monkeypatch.setenv("ECS_CONTAINER_METADATA_URI_V4", "http://169.254.170.2/v4")
|
|
142
134
|
monkeypatch.setattr("lambda_tasks.handler.resolve_environment", lambda: None)
|
|
143
135
|
monkeypatch.setattr(
|
|
144
136
|
"lambda_tasks.handler.resolve_secrets_into_env", lambda: None
|
|
145
137
|
)
|
|
146
|
-
monkeypatch.setattr(
|
|
147
|
-
handler_module,
|
|
148
|
-
"_CGROUP_V2_MEMORY_MAX_PATH",
|
|
149
|
-
fake_path(None),
|
|
150
|
-
)
|
|
151
|
-
# 4 GB in bytes
|
|
152
|
-
monkeypatch.setattr(
|
|
153
|
-
handler_module,
|
|
154
|
-
"_CGROUP_V1_MEMORY_LIMIT_PATH",
|
|
155
|
-
fake_path(str(4096 * 1024 * 1024)),
|
|
156
|
-
)
|
|
157
138
|
|
|
158
139
|
calls: list[tuple] = []
|
|
159
140
|
|
|
@@ -165,13 +146,14 @@ class TestMemoryLimitSetDuringColdStart:
|
|
|
165
146
|
handler_module._cold_start_done = False
|
|
166
147
|
handler_module.handler(event={"Records": []}, context=None)
|
|
167
148
|
|
|
168
|
-
expected = (
|
|
149
|
+
expected = (512 - 128) * 1024 * 1024
|
|
169
150
|
assert calls == [(resource.RLIMIT_AS, (expected, expected))]
|
|
170
151
|
|
|
171
152
|
def test_memory_limit_set_before_loaders(self, monkeypatch):
|
|
172
153
|
"""The memory limit is set before resolve_environment runs."""
|
|
173
154
|
monkeypatch.delenv("DJANGO_SETTINGS_MODULE", raising=False)
|
|
174
155
|
monkeypatch.setenv("AWS_LAMBDA_FUNCTION_MEMORY_SIZE", "256")
|
|
156
|
+
monkeypatch.delenv("ECS_CONTAINER_METADATA_URI_V4", raising=False)
|
|
175
157
|
|
|
176
158
|
call_order: list[str] = []
|
|
177
159
|
|
|
@@ -207,6 +189,7 @@ class TestMemoryLimitSetDuringColdStart:
|
|
|
207
189
|
|
|
208
190
|
monkeypatch.delenv("DJANGO_SETTINGS_MODULE", raising=False)
|
|
209
191
|
monkeypatch.setenv("AWS_LAMBDA_FUNCTION_MEMORY_SIZE", "1024")
|
|
192
|
+
monkeypatch.delenv("ECS_CONTAINER_METADATA_URI_V4", raising=False)
|
|
210
193
|
monkeypatch.setattr("lambda_tasks.handler.resolve_environment", lambda: None)
|
|
211
194
|
monkeypatch.setattr(
|
|
212
195
|
"lambda_tasks.handler.resolve_secrets_into_env", lambda: None
|
|
@@ -224,6 +207,7 @@ class TestMemoryLimitSetDuringColdStart:
|
|
|
224
207
|
"""When AWS_LAMBDA_FUNCTION_MEMORY_SIZE <= reserved, RLIMIT_AS is set to the 64 MB minimum."""
|
|
225
208
|
monkeypatch.delenv("DJANGO_SETTINGS_MODULE", raising=False)
|
|
226
209
|
monkeypatch.setenv("AWS_LAMBDA_FUNCTION_MEMORY_SIZE", "128")
|
|
210
|
+
monkeypatch.delenv("ECS_CONTAINER_METADATA_URI_V4", raising=False)
|
|
227
211
|
monkeypatch.setattr("lambda_tasks.handler.resolve_environment", lambda: None)
|
|
228
212
|
monkeypatch.setattr(
|
|
229
213
|
"lambda_tasks.handler.resolve_secrets_into_env", lambda: None
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
"""Tests for LAMBDA_TASKS_NOOP_EXECUTION setting and deployment check."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
from unittest.mock import patch
|
|
5
|
+
|
|
6
|
+
import pytest
|
|
7
|
+
from django.core.exceptions import ImproperlyConfigured
|
|
8
|
+
|
|
9
|
+
DEFAULT_QUEUE = {
|
|
10
|
+
"queue_url": "https://sqs.us-east-1.amazonaws.com/000000000000/default"
|
|
11
|
+
}
|
|
12
|
+
QUEUES = {"default": DEFAULT_QUEUE}
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
# ---------------------------------------------------------------------------
|
|
16
|
+
# NOOP_EXECUTION setting property
|
|
17
|
+
# ---------------------------------------------------------------------------
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def test_noop_execution_defaults_to_false():
|
|
21
|
+
from lambda_tasks.settings import LambdaTasksSettings
|
|
22
|
+
|
|
23
|
+
conf = LambdaTasksSettings()
|
|
24
|
+
assert conf.NOOP_EXECUTION is False
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def test_noop_execution_true(settings):
|
|
28
|
+
from lambda_tasks.settings import LambdaTasksSettings
|
|
29
|
+
|
|
30
|
+
settings.LAMBDA_TASKS_NOOP_EXECUTION = True
|
|
31
|
+
settings.LAMBDA_TASKS_EAGER = False
|
|
32
|
+
settings.LAMBDA_TASKS_LOCAL_WORKERS = 0
|
|
33
|
+
conf = LambdaTasksSettings()
|
|
34
|
+
assert conf.NOOP_EXECUTION is True
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def test_noop_execution_with_eager_raises(settings):
|
|
38
|
+
from lambda_tasks.settings import LambdaTasksSettings
|
|
39
|
+
|
|
40
|
+
settings.LAMBDA_TASKS_NOOP_EXECUTION = True
|
|
41
|
+
settings.LAMBDA_TASKS_EAGER = True
|
|
42
|
+
settings.LAMBDA_TASKS_LOCAL_WORKERS = 0
|
|
43
|
+
conf = LambdaTasksSettings()
|
|
44
|
+
with pytest.raises(ImproperlyConfigured):
|
|
45
|
+
_ = conf.NOOP_EXECUTION
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def test_noop_execution_with_local_workers_raises(settings):
|
|
49
|
+
from lambda_tasks.settings import LambdaTasksSettings
|
|
50
|
+
|
|
51
|
+
settings.LAMBDA_TASKS_NOOP_EXECUTION = True
|
|
52
|
+
settings.LAMBDA_TASKS_EAGER = False
|
|
53
|
+
settings.LAMBDA_TASKS_LOCAL_WORKERS = 2
|
|
54
|
+
conf = LambdaTasksSettings()
|
|
55
|
+
with pytest.raises(ImproperlyConfigured):
|
|
56
|
+
_ = conf.NOOP_EXECUTION
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
# ---------------------------------------------------------------------------
|
|
60
|
+
# _execute() noop behaviour
|
|
61
|
+
# ---------------------------------------------------------------------------
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def test_execute_noop_does_not_call_sqs(settings):
|
|
65
|
+
from lambda_tasks.models import SQSLambdaTask, SQSLambdaTaskMessage
|
|
66
|
+
|
|
67
|
+
settings.LAMBDA_TASKS_NOOP_EXECUTION = True
|
|
68
|
+
settings.LAMBDA_TASKS_EAGER = False
|
|
69
|
+
settings.LAMBDA_TASKS_LOCAL_WORKERS = 0
|
|
70
|
+
settings.LAMBDA_TASKS_QUEUES = QUEUES
|
|
71
|
+
|
|
72
|
+
message = SQSLambdaTaskMessage(
|
|
73
|
+
task_name="myapp.tasks.my_task",
|
|
74
|
+
kwargs={"user_id": 1},
|
|
75
|
+
n_retries=0,
|
|
76
|
+
)
|
|
77
|
+
task = SQSLambdaTask(message=message, delay=0, queue="default")
|
|
78
|
+
|
|
79
|
+
with patch("lambda_tasks.models.boto3") as mock_boto3:
|
|
80
|
+
task._execute()
|
|
81
|
+
mock_boto3.client.assert_not_called()
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def test_execute_noop_logs_warning(settings, caplog):
|
|
85
|
+
from lambda_tasks.models import SQSLambdaTask, SQSLambdaTaskMessage
|
|
86
|
+
|
|
87
|
+
settings.LAMBDA_TASKS_NOOP_EXECUTION = True
|
|
88
|
+
settings.LAMBDA_TASKS_EAGER = False
|
|
89
|
+
settings.LAMBDA_TASKS_LOCAL_WORKERS = 0
|
|
90
|
+
settings.LAMBDA_TASKS_QUEUES = QUEUES
|
|
91
|
+
|
|
92
|
+
message = SQSLambdaTaskMessage(
|
|
93
|
+
task_name="myapp.tasks.my_task",
|
|
94
|
+
kwargs={"user_id": 42},
|
|
95
|
+
n_retries=0,
|
|
96
|
+
)
|
|
97
|
+
task = SQSLambdaTask(message=message, delay=0, queue="default")
|
|
98
|
+
|
|
99
|
+
with caplog.at_level(logging.WARNING, logger="lambda_tasks"):
|
|
100
|
+
task._execute()
|
|
101
|
+
|
|
102
|
+
assert len(caplog.records) == 1
|
|
103
|
+
record = caplog.records[0]
|
|
104
|
+
assert record.levelname == "WARNING"
|
|
105
|
+
assert "myapp.tasks.my_task" in record.message
|
|
106
|
+
assert "noop" in record.message.lower()
|
|
107
|
+
assert "{'user_id': 42}" in record.message
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
# ---------------------------------------------------------------------------
|
|
111
|
+
# Django deployment check
|
|
112
|
+
# ---------------------------------------------------------------------------
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def test_deployment_check_passes_when_all_false(settings):
|
|
116
|
+
from lambda_tasks.checks import check_noop_not_in_production
|
|
117
|
+
|
|
118
|
+
settings.LAMBDA_TASKS_NOOP_EXECUTION = False
|
|
119
|
+
settings.LAMBDA_TASKS_EAGER = False
|
|
120
|
+
settings.LAMBDA_TASKS_LOCAL_WORKERS = 0
|
|
121
|
+
errors = check_noop_not_in_production(app_configs=None)
|
|
122
|
+
assert errors == []
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def test_deployment_check_warns_noop(settings):
|
|
126
|
+
from lambda_tasks.checks import check_noop_not_in_production
|
|
127
|
+
|
|
128
|
+
settings.LAMBDA_TASKS_NOOP_EXECUTION = True
|
|
129
|
+
settings.LAMBDA_TASKS_EAGER = False
|
|
130
|
+
settings.LAMBDA_TASKS_LOCAL_WORKERS = 0
|
|
131
|
+
errors = check_noop_not_in_production(app_configs=None)
|
|
132
|
+
assert len(errors) == 1
|
|
133
|
+
assert errors[0].id == "lambda_tasks.W001"
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def test_deployment_check_warns_eager(settings):
|
|
137
|
+
from lambda_tasks.checks import check_noop_not_in_production
|
|
138
|
+
|
|
139
|
+
settings.LAMBDA_TASKS_NOOP_EXECUTION = False
|
|
140
|
+
settings.LAMBDA_TASKS_EAGER = True
|
|
141
|
+
settings.LAMBDA_TASKS_LOCAL_WORKERS = 0
|
|
142
|
+
errors = check_noop_not_in_production(app_configs=None)
|
|
143
|
+
assert len(errors) == 1
|
|
144
|
+
assert errors[0].id == "lambda_tasks.W002"
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def test_deployment_check_warns_local_workers(settings):
|
|
148
|
+
from lambda_tasks.checks import check_noop_not_in_production
|
|
149
|
+
|
|
150
|
+
settings.LAMBDA_TASKS_NOOP_EXECUTION = False
|
|
151
|
+
settings.LAMBDA_TASKS_EAGER = False
|
|
152
|
+
settings.LAMBDA_TASKS_LOCAL_WORKERS = 2
|
|
153
|
+
errors = check_noop_not_in_production(app_configs=None)
|
|
154
|
+
assert len(errors) == 1
|
|
155
|
+
assert errors[0].id == "lambda_tasks.W003"
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def test_deployment_check_warns_multiple(settings):
|
|
159
|
+
from lambda_tasks.checks import check_noop_not_in_production
|
|
160
|
+
|
|
161
|
+
settings.LAMBDA_TASKS_NOOP_EXECUTION = True
|
|
162
|
+
settings.LAMBDA_TASKS_EAGER = True
|
|
163
|
+
settings.LAMBDA_TASKS_LOCAL_WORKERS = 2
|
|
164
|
+
errors = check_noop_not_in_production(app_configs=None)
|
|
165
|
+
assert len(errors) == 3
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/async-local-execution/design.md
RENAMED
|
File without changes
|
|
File without changes
|
{django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/async-local-execution/tasks.md
RENAMED
|
File without changes
|
|
File without changes
|
{django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/deferred-task-enqueue/design.md
RENAMED
|
File without changes
|
|
File without changes
|
{django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/deferred-task-enqueue/tasks.md
RENAMED
|
File without changes
|
|
File without changes
|
{django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/eager-mode-example-app/design.md
RENAMED
|
File without changes
|
|
File without changes
|
{django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/eager-mode-example-app/tasks.md
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/retry-delay/.config.kiro
RENAMED
|
File without changes
|
|
File without changes
|
{django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/retry-delay/requirements.md
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/rse-background-tasks/design.md
RENAMED
|
File without changes
|
|
File without changes
|
{django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/rse-background-tasks/tasks.md
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/singleton-task/.config.kiro
RENAMED
|
File without changes
|
{django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/singleton-task/design.md
RENAMED
|
File without changes
|
{django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/singleton-task/requirements.md
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/ssm-environment-loader/design.md
RENAMED
|
File without changes
|
|
File without changes
|
{django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/ssm-environment-loader/tasks.md
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/.kiro/specs/task-retry/requirements.md
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{django_lambda_tasks-0.6.5 → django_lambda_tasks-0.6.7}/lambda_tasks/migrations/0001_initial.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|