django-lambda-tasks 0.9.1__tar.gz → 0.10.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/steering/product.md +0 -20
  2. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/steering/structure.md +1 -3
  3. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/steering/tech.md +0 -13
  4. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/PKG-INFO +1 -13
  5. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/lambda_tasks/handler.py +0 -16
  6. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/lambda_tasks/tasks.py +5 -1
  7. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/pyproject.toml +1 -18
  8. django_lambda_tasks-0.10.0/tests/test_batch_worker.py +77 -0
  9. django_lambda_tasks-0.9.1/lambda_tasks/tracing.py +0 -318
  10. django_lambda_tasks-0.9.1/tests/test_batch_worker.py +0 -138
  11. django_lambda_tasks-0.9.1/tests/test_tracing.py +0 -385
  12. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.github/dependabot.yml +0 -0
  13. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.github/workflows/ci.yml +0 -0
  14. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.github/workflows/release.yml +0 -0
  15. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.gitignore +0 -0
  16. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/specs/async-local-execution/.config.kiro +0 -0
  17. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/specs/async-local-execution/design.md +0 -0
  18. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/specs/async-local-execution/requirements.md +0 -0
  19. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/specs/async-local-execution/tasks.md +0 -0
  20. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/specs/deferred-task-enqueue/.config.kiro +0 -0
  21. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/specs/deferred-task-enqueue/design.md +0 -0
  22. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/specs/deferred-task-enqueue/requirements.md +0 -0
  23. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/specs/deferred-task-enqueue/tasks.md +0 -0
  24. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/specs/eager-mode-example-app/.config.kiro +0 -0
  25. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/specs/eager-mode-example-app/design.md +0 -0
  26. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/specs/eager-mode-example-app/requirements.md +0 -0
  27. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/specs/eager-mode-example-app/tasks.md +0 -0
  28. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/specs/ignore-errors-decorator-option/.config.kiro +0 -0
  29. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/specs/ignore-errors-decorator-option/design.md +0 -0
  30. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/specs/ignore-errors-decorator-option/requirements.md +0 -0
  31. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/specs/ignore-errors-decorator-option/tasks.md +0 -0
  32. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/specs/import-string-task-resolution/.config.kiro +0 -0
  33. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/specs/import-string-task-resolution/design.md +0 -0
  34. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/specs/import-string-task-resolution/requirements.md +0 -0
  35. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/specs/import-string-task-resolution/tasks.md +0 -0
  36. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/specs/retry-delay/.config.kiro +0 -0
  37. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/specs/retry-delay/design.md +0 -0
  38. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/specs/retry-delay/requirements.md +0 -0
  39. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/specs/retry-delay/tasks.md +0 -0
  40. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/specs/rse-background-tasks/.config.kiro +0 -0
  41. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/specs/rse-background-tasks/design.md +0 -0
  42. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/specs/rse-background-tasks/requirements.md +0 -0
  43. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/specs/rse-background-tasks/tasks.md +0 -0
  44. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/specs/rse-background-tasks-bugfix/.config.kiro +0 -0
  45. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/specs/rse-background-tasks-bugfix/bugfix.md +0 -0
  46. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/specs/rse-background-tasks-bugfix/design.md +0 -0
  47. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/specs/rse-background-tasks-bugfix/tasks.md +0 -0
  48. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/specs/singleton-task/.config.kiro +0 -0
  49. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/specs/singleton-task/design.md +0 -0
  50. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/specs/singleton-task/requirements.md +0 -0
  51. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/specs/singleton-task/tasks.md +0 -0
  52. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/specs/ssm-environment-loader/.config.kiro +0 -0
  53. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/specs/ssm-environment-loader/design.md +0 -0
  54. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/specs/ssm-environment-loader/requirements.md +0 -0
  55. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/specs/ssm-environment-loader/tasks.md +0 -0
  56. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/specs/task-retry/.config.kiro +0 -0
  57. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/specs/task-retry/design.md +0 -0
  58. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/specs/task-retry/requirements.md +0 -0
  59. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.kiro/specs/task-retry/tasks.md +0 -0
  60. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.pre-commit-config.yaml +0 -0
  61. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/.vscode/settings.json +0 -0
  62. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/README.md +0 -0
  63. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/docker-compose.yml +0 -0
  64. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/example/README.md +0 -0
  65. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/example/docker-compose.yml +0 -0
  66. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/example/example_app/__init__.py +0 -0
  67. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/example/example_app/apps.py +0 -0
  68. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/example/example_app/tasks.py +0 -0
  69. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/example/example_app/urls.py +0 -0
  70. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/example/example_app/views.py +0 -0
  71. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/example/example_project/__init__.py +0 -0
  72. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/example/example_project/settings.py +0 -0
  73. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/example/example_project/urls.py +0 -0
  74. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/example/example_project/wsgi.py +0 -0
  75. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/example/manage.py +0 -0
  76. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/lambda_tasks/__init__.py +0 -0
  77. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/lambda_tasks/admin.py +0 -0
  78. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/lambda_tasks/apps.py +0 -0
  79. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/lambda_tasks/checks.py +0 -0
  80. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/lambda_tasks/decorators.py +0 -0
  81. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/lambda_tasks/environment_loader.py +0 -0
  82. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/lambda_tasks/local_executor.py +0 -0
  83. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/lambda_tasks/logging.py +0 -0
  84. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/lambda_tasks/migrations/0001_initial.py +0 -0
  85. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/lambda_tasks/migrations/0002_alter_taskrecord_status.py +0 -0
  86. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/lambda_tasks/migrations/0003_taskrecord_taskrecord_kwargs_trgm.py +0 -0
  87. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/lambda_tasks/migrations/0004_remove_taskrecord_taskrecord_kwargs_trgm_and_more.py +0 -0
  88. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/lambda_tasks/migrations/__init__.py +0 -0
  89. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/lambda_tasks/models.py +0 -0
  90. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/lambda_tasks/secret_loader.py +0 -0
  91. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/lambda_tasks/settings.py +0 -0
  92. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/lambda_tasks/timeouts.py +0 -0
  93. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/tests/conftest.py +0 -0
  94. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/tests/settings.py +0 -0
  95. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/tests/test_admin.py +0 -0
  96. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/tests/test_batch.py +0 -0
  97. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/tests/test_checks.py +0 -0
  98. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/tests/test_decorator.py +0 -0
  99. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/tests/test_decorators.py +0 -0
  100. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/tests/test_deferred_enqueue.py +0 -0
  101. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/tests/test_environment_loader.py +0 -0
  102. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/tests/test_handler.py +0 -0
  103. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/tests/test_kwargs_only.py +0 -0
  104. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/tests/test_local_executor.py +0 -0
  105. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/tests/test_logging.py +0 -0
  106. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/tests/test_memory_limit.py +0 -0
  107. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/tests/test_models.py +0 -0
  108. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/tests/test_noop_execution.py +0 -0
  109. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/tests/test_secret_loader.py +0 -0
  110. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/tests/test_serializer.py +0 -0
  111. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/tests/test_settings.py +0 -0
  112. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/tests/test_tasks.py +0 -0
  113. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/tests/test_timeout_validation.py +0 -0
  114. {django_lambda_tasks-0.9.1 → django_lambda_tasks-0.10.0}/tests/test_timeouts.py +0 -0
@@ -31,7 +31,6 @@ Key modules:
31
31
  - `models.py` — `TaskRecord` (Django ORM), `SQSLambdaTaskMessage` (SQS schema + execution), `SQSLambdaTask` (routing + SQS publish, Batch submit, or local pool submit)
32
32
  - `local_executor.py` — `ProcessPoolExecutor`-based async local execution for development
33
33
  - `handler.py` — AWS Lambda entry point + AWS Batch container entry point (`main()`)
34
- - `tracing.py` — optional OpenTelemetry tracing (import-time setup, direct X-Ray OTLP export, SigV4-signed)
35
34
  - `logging.py` — `task_logger` for invocation-scoped log output
36
35
  - `settings.py` — lazy `LambdaTasksSettings` reading from Django settings, queue type helpers
37
36
 
@@ -159,7 +158,6 @@ class SQSLambdaTaskMessage(BaseModel):
159
158
  - Reads `LAMBDA_TASKS_MESSAGE` env var (the serialized task JSON, set by `submit_batch_job` via container overrides)
160
159
  - Uses `AWS_BATCH_JOB_ID` as the `message_id` (falls back to UUID4 if not set)
161
160
  - Constructs a synthetic SQS event and delegates to `handler()` — reuses the full cold-start and execution path
162
- - Before returning, force-flushes and shuts down the global `TracerProvider` (if OTEL tracing is configured) so short-lived Batch jobs do not drop buffered spans
163
161
  - Returns exit code 0 on success, 1 on failure
164
162
 
165
163
  ## Environment Loader
@@ -299,24 +297,6 @@ Relying on `atexit` alone loses this race in applications with a heavy shutdown
299
297
 
300
298
  The checks are registered via `AppConfig.ready()` and tagged with `deploy=True` so they only fire during `--deploy` checks.
301
299
 
302
- ## OpenTelemetry Tracing (optional)
303
-
304
- `lambda_tasks/tracing.py` adds OpenTelemetry tracing to the Lambda and Batch execution paths without the managed ADOT Lambda layer — the OTEL packages installed in the application image are the single source of instrumentation (reproducible-build friendly).
305
-
306
- Setup lives in the handler module, **not** Django settings. `AwsLambdaInstrumentor` must wrap the handler named by `_HANDLER`/`ORIG_HANDLER` **before** the runtime invokes it, which is earlier than the cold-start `django.setup()` inside `handler()`. So `lambda_tasks.handler` calls `configure_tracing()` and `instrument_lambda_handler()` at **import time**, and the setup is driven purely by standard OTEL environment variables (settings are not configured yet at import time).
307
-
308
- Behaviour:
309
- - Gated on `OTEL_EXPORTER_OTLP_ENDPOINT` (or the per-signal `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`). Unset → everything is a no-op (local dev, tests, un-opted-in deployments).
310
- - Builds a `TracerProvider` with an OTLP/HTTP span exporter via a `BatchSpanProcessor`. Spans go **directly** to the AWS X-Ray OTLP traces endpoint (`https://xray.<region>.amazonaws.com/v1/traces`) — no collector, no daemon, no Lambda layer.
311
- - The exporter's `requests` session SigV4-signs each export with the ambient AWS credentials (task / execution role) via `botocore` — no API key to manage. Unsigned when no credentials are resolvable.
312
- - Uses the AWS X-Ray id generator (`AwsXRayIdGenerator`) so the X-Ray backend accepts the trace IDs, and the X-Ray propagator (`AwsXRayLambdaPropagator` on Lambda, else `AwsXRayPropagator`). The X-Ray packages (`opentelemetry-sdk-extension-aws`, `opentelemetry-propagator-aws-xray`) are part of the `otel` extra.
313
- - The OpenTelemetry / `requests` imports are **deferred into the functions** (module scope imports only `os`), so the OTEL packages remain a genuinely optional extra: importing `lambda_tasks.handler` with them uninstalled is a no-op, not an `ImportError`. The disabled path (endpoint unset) never reaches any OTEL import.
314
- - Lambda path: also calls `AwsLambdaInstrumentor().instrument()` (only when `AWS_LAMBDA_FUNCTION_NAME` is set). Batch path (`main()`): the instrumentor is skipped (no handler to wrap); the provider alone is configured.
315
- - Flush on shutdown: `main()` calls `TracerProvider.force_flush()` / `shutdown()` before the one-shot process exits so short-lived Batch jobs do not drop buffered spans. The Lambda path needs no manual flush — `AwsLambdaInstrumentor` force-flushes per invocation.
316
- - Best-effort and observable: tracing must never take down the worker. `configure_tracing()` and `instrument_lambda_handler()` catch setup failures (missing OTEL packages, provider build errors) and log a warning instead of raising — important because they run at the handler module's import time. The `lambda_tasks.tracing` logger also surfaces otherwise-silent conditions at `INFO`/`WARNING`: tracing configured / already configured / disabled, handler instrumented or skipped (not on Lambda), no resolvable AWS region, and no resolvable AWS credentials (which would make exports unsigned and rejected by X-Ray).
317
-
318
- `configure_tracing()` returns the installed provider (or `None` when disabled or already configured — it is idempotent). Install the OTEL packages via the `otel` extra; the consuming application sets `OTEL_EXPORTER_OTLP_ENDPOINT` and declares the X-Ray packages it wants.
319
-
320
300
  ## Logging
321
301
 
322
302
  Import `task_logger` to emit log records that are automatically prefixed with the active `message_id`:
@@ -23,7 +23,6 @@ django-lambda-tasks/
23
23
  │ ├── local_executor.py # ProcessPoolExecutor for async local task execution
24
24
  │ ├── tasks.py # Built-in tasks (cleanup_task_records, submit_batch_job)
25
25
  │ ├── timeouts.py # TimeoutContext implementation
26
- │ ├── tracing.py # Optional OpenTelemetry tracing (import-time setup, X-Ray OTLP export)
27
26
  │ └── migrations/ # Django migrations for TaskRecord
28
27
  ├── tests/ # pytest test suite
29
28
  │ ├── conftest.py
@@ -44,8 +43,7 @@ django-lambda-tasks/
44
43
  - `decorators.py` — defines `LambdaTaskWrapper` and `@lambda_task` decorator factory; enforces kwargs-only at decoration time; validates timeouts (> 0, soft < hard) at decoration time; upper-bound timeout validation deferred to execution time based on queue type
45
44
  - `models.py` — `TaskRecord` (Django ORM), `SQSLambdaTaskMessage` (Pydantic, SQS schema + execution logic), `SQSLambdaTask` (Pydantic, holds message + routing; `_execute()` inspects queue config to publish to SQS, submit to Batch via `submit_batch_job`, execute eagerly, or submit to the local process pool; `execute_on_commit()` registers `_execute` with `transaction.on_commit`)
46
45
  - `local_executor.py` — `ProcessPoolExecutor`-based async local execution; `get_pool()` lazily creates a module-level pool; `submit_task()` fire-and-forget submission; `_execute_in_worker()` deserializes and runs the task in a child process; `_pool_initializer()` calls `django.setup()` per worker (and sets `SIGINT` to `SIG_IGN` so workers ignore Ctrl+C); `_install_shutdown_handlers()` installs main-thread `SIGINT`/`SIGTERM` handlers that release the pool before chaining to the previous handler
47
- - `handler.py` — Lambda entry point (`handler(event, context)`); cold-start init (memory limit → temporary log handler → `resolve_environment()` → `resolve_secrets_into_env()` → handler removed → conditional `django.setup()`) runs inside the handler on first invocation, guarded by `_cold_start_done` sentinel; calls `db.close_old_connections()` before each SQS record to handle stale connections on warm starts; processes SQS records independently; returns `batchItemFailures`. At **import time** (before any invocation) it calls `configure_tracing()` and `instrument_lambda_handler()` from `tracing.py` so the Lambda instrumentor can wrap the handler before the runtime invokes it. Also provides `main()` as the AWS Batch container entry point (`python -m lambda_tasks.handler`) — reads `LAMBDA_TASKS_MESSAGE` env var, constructs a synthetic SQS event, delegates to `handler()`, then force-flushes and shuts down the global `TracerProvider` (if any) before the one-shot process exits
48
- - `tracing.py` — optional OpenTelemetry tracing, driven purely by standard OTEL env vars (no Django settings) so the package stays generic; no-op unless `OTEL_EXPORTER_OTLP_ENDPOINT` (or `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`) is set. All OpenTelemetry / `requests` imports are **deferred into the functions** (only `os` is imported at module scope) so the OTEL packages are a genuinely optional extra — importing `lambda_tasks.handler` without them is a no-op, not an `ImportError`. `configure_tracing()` installs a global `TracerProvider` using the X-Ray id generator (`AwsXRayIdGenerator`) with an OTLP/HTTP exporter whose `requests` session SigV4-signs each export to the AWS X-Ray traces endpoint (unsigned when no credentials resolve), and sets the X-Ray propagator (`AwsXRayLambdaPropagator` on Lambda, else `AwsXRayPropagator`). `instrument_lambda_handler()` calls `AwsLambdaInstrumentor().instrument()` only when tracing is enabled and `AWS_LAMBDA_FUNCTION_NAME` is set (Lambda path); skipped on the Batch path. `flush_tracing()` force-flushes and shuts down the provider (gated; used by the Batch entry point)
46
+ - `handler.py` — Lambda entry point (`handler(event, context)`); cold-start init (memory limit → temporary log handler → `resolve_environment()` → `resolve_secrets_into_env()` → handler removed → conditional `django.setup()`) runs inside the handler on first invocation, guarded by `_cold_start_done` sentinel; calls `db.close_old_connections()` before each SQS record to handle stale connections on warm starts; processes SQS records independently; returns `batchItemFailures`. Also provides `main()` as the AWS Batch container entry point (`python -m lambda_tasks.handler`) — reads `LAMBDA_TASKS_MESSAGE` env var, constructs a synthetic SQS event, and delegates to `handler()`
49
47
  - `environment_loader.py` — loads env vars from a Secrets Manager secret at cold start; validates flat JSON format; idempotent via `_loaded` sentinel
50
48
  - `secret_loader.py` — resolves `LAMBDA_TASKS_SECRET_*` env vars from Secrets Manager before Django starts; validates format, detects conflicts, batches API calls; idempotent via `_loaded` sentinel
51
49
  - `logging.py` — `task_logger` singleton; `message_id` set/cleared around each task execution
@@ -17,19 +17,6 @@ inclusion: always
17
17
  | `boto3` | `>= 1.42.69` | AWS SDK — SQS publishing, Lambda integration |
18
18
  | `pydantic` | `>= 2.12.5` | Task argument serialization via `SQSLambdaTaskMessage` model |
19
19
 
20
- ## Optional Dependencies
21
-
22
- Installed via the `otel` extra (`pip install django-lambda-tasks[otel]`). Enables OpenTelemetry tracing in `lambda_tasks/tracing.py`; the library stays generic without them. SigV4 signing reuses `botocore` (already a boto3 transitive dependency) — no extra signing package.
23
-
24
- | Package | Purpose |
25
- |---|---|
26
- | `opentelemetry-sdk` | `TracerProvider`, `BatchSpanProcessor` |
27
- | `opentelemetry-exporter-otlp-proto-http` | OTLP/HTTP span exporter (no gRPC, no meta-package) |
28
- | `opentelemetry-instrumentation-aws-lambda` | Wraps the Lambda handler (`AwsLambdaInstrumentor`) |
29
- | `opentelemetry-sdk-extension-aws` | X-Ray id generator (`AwsXRayIdGenerator`) |
30
- | `opentelemetry-propagator-aws-xray` | X-Ray trace propagator |
31
- | `requests` | OTLP/HTTP transport session (SigV4-signed via `botocore`) |
32
-
33
20
  ## Dev Dependencies
34
21
 
35
22
  | Package | Version | Purpose |
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: django-lambda-tasks
3
- Version: 0.9.1
3
+ Version: 0.10.0
4
4
  Summary: Run async tasks in a lambda function
5
5
  Requires-Python: >=3.11
6
6
  Requires-Dist: awslambdaric
@@ -8,18 +8,6 @@ Requires-Dist: boto3
8
8
  Requires-Dist: django
9
9
  Requires-Dist: pydantic
10
10
  Requires-Dist: redis
11
- Provides-Extra: otel
12
- Requires-Dist: opentelemetry-exporter-otlp-proto-http; extra == 'otel'
13
- Requires-Dist: opentelemetry-instrumentation-aws-lambda; extra == 'otel'
14
- Requires-Dist: opentelemetry-instrumentation-botocore; extra == 'otel'
15
- Requires-Dist: opentelemetry-instrumentation-httpx; extra == 'otel'
16
- Requires-Dist: opentelemetry-instrumentation-psycopg; extra == 'otel'
17
- Requires-Dist: opentelemetry-instrumentation-redis; extra == 'otel'
18
- Requires-Dist: opentelemetry-instrumentation-requests; extra == 'otel'
19
- Requires-Dist: opentelemetry-propagator-aws-xray; extra == 'otel'
20
- Requires-Dist: opentelemetry-sdk; extra == 'otel'
21
- Requires-Dist: opentelemetry-sdk-extension-aws; extra == 'otel'
22
- Requires-Dist: requests; extra == 'otel'
23
11
  Description-Content-Type: text/markdown
24
12
 
25
13
  # Django Lambda Tasks
@@ -26,21 +26,9 @@ from django.apps import apps
26
26
 
27
27
  from lambda_tasks.environment_loader import resolve_environment
28
28
  from lambda_tasks.secret_loader import resolve_secrets_into_env
29
- from lambda_tasks.tracing import (
30
- configure_tracing,
31
- flush_tracing,
32
- instrument_lambda_handler,
33
- )
34
29
 
35
30
  logger = logging.getLogger(__name__)
36
31
 
37
- # Initialise tracing at import time so the Lambda instrumentor can wrap the
38
- # handler before the first invocation (which is earlier than the cold-start
39
- # django.setup() inside handler()). No-op unless OTEL_EXPORTER_OTLP_ENDPOINT
40
- # is set.
41
- configure_tracing()
42
- instrument_lambda_handler()
43
-
44
32
  _cold_start_done: bool = False
45
33
 
46
34
 
@@ -181,10 +169,6 @@ def main() -> int:
181
169
  context=None,
182
170
  )
183
171
 
184
- # Flush any buffered spans before the one-shot process exits (no-op unless
185
- # tracing is enabled — this never imports OTEL otherwise).
186
- flush_tracing()
187
-
188
172
  if result["batchItemFailures"]:
189
173
  return 1
190
174
  else:
@@ -11,7 +11,11 @@ from lambda_tasks.models import SQSLambdaTaskMessage, TaskRecord
11
11
  from lambda_tasks.settings import BatchQueueConfig, LambdaTasksSettings
12
12
 
13
13
 
14
- @lambda_task
14
+ @lambda_task(
15
+ singleton=True,
16
+ # No need to retry here as this should be retried periodically
17
+ retry_singleton=False,
18
+ )
15
19
  def cleanup_task_records(*, retention_days: int = 7) -> int:
16
20
  """Delete TaskRecord rows older than ``retention_days``.
17
21
 
@@ -7,7 +7,7 @@ packages = ["lambda_tasks"]
7
7
 
8
8
  [project]
9
9
  name = "django-lambda-tasks"
10
- version = "0.9.1"
10
+ version = "0.10.0"
11
11
  description = "Run async tasks in a lambda function"
12
12
  readme = "README.md"
13
13
  requires-python = ">=3.11"
@@ -19,21 +19,6 @@ dependencies = [
19
19
  "awslambdaric",
20
20
  ]
21
21
 
22
- [project.optional-dependencies]
23
- otel = [
24
- "requests",
25
- "opentelemetry-sdk",
26
- "opentelemetry-exporter-otlp-proto-http",
27
- "opentelemetry-instrumentation-aws-lambda",
28
- "opentelemetry-sdk-extension-aws",
29
- "opentelemetry-propagator-aws-xray",
30
- "opentelemetry-instrumentation-psycopg",
31
- "opentelemetry-instrumentation-botocore",
32
- "opentelemetry-instrumentation-requests",
33
- "opentelemetry-instrumentation-httpx",
34
- "opentelemetry-instrumentation-redis",
35
- ]
36
-
37
22
  [dependency-groups]
38
23
  dev = [
39
24
  "hypothesis",
@@ -46,8 +31,6 @@ dev = [
46
31
  "pytest-cov",
47
32
  "ty",
48
33
  "psycopg",
49
- "httpx",
50
- "django-lambda-tasks[otel]",
51
34
  ]
52
35
 
53
36
  [tool.ty.rules]
@@ -0,0 +1,77 @@
1
+ """Tests for lambda_tasks.batch_worker entry point."""
2
+
3
+ import uuid
4
+ from unittest.mock import patch
5
+
6
+ import pytest
7
+
8
+ from lambda_tasks.decorators import lambda_task
9
+ from lambda_tasks.handler import main
10
+ from lambda_tasks.models import SQSLambdaTaskMessage, TaskRecord
11
+
12
+
13
+ @pytest.fixture(autouse=True)
14
+ def _patch_close_old_connections():
15
+ """Prevent close_old_connections from closing the test DB connection."""
16
+ with patch("lambda_tasks.handler.db.close_old_connections"):
17
+ yield
18
+
19
+
20
+ QUEUES = {
21
+ "default": {"queue_url": "https://sqs.us-east-1.amazonaws.com/000000000000/default"}
22
+ }
23
+
24
+
25
+ # Module-level tasks for import_string resolution
26
+
27
+
28
+ @lambda_task
29
+ def _worker_task(*, value: int) -> str:
30
+ return "ok"
31
+
32
+
33
+ def test_missing_env_var_returns_1(monkeypatch):
34
+ monkeypatch.delenv("LAMBDA_TASKS_MESSAGE", raising=False)
35
+ assert main() == 1
36
+
37
+
38
+ @pytest.mark.django_db
39
+ def test_valid_message_executes_task(monkeypatch, settings):
40
+ settings.LAMBDA_TASKS_EAGER = True
41
+ settings.LAMBDA_TASKS_QUEUES = QUEUES
42
+
43
+ task_name = f"{_worker_task.__module__}.{_worker_task.__wrapped__.__qualname__}"
44
+ message = SQSLambdaTaskMessage(task_name=task_name, kwargs={"value": 1})
45
+ job_id = str(uuid.uuid4())
46
+
47
+ monkeypatch.setenv("LAMBDA_TASKS_MESSAGE", message.model_dump_json())
48
+ monkeypatch.setenv("AWS_BATCH_JOB_ID", job_id)
49
+
50
+ result = main()
51
+
52
+ assert result == 0
53
+ record = TaskRecord.objects.get(pk=job_id)
54
+ assert record.status == TaskRecord.TaskStatus.SUCCEEDED
55
+
56
+
57
+ def test_invalid_json_returns_1(monkeypatch):
58
+ monkeypatch.setenv("LAMBDA_TASKS_MESSAGE", "not-valid-json")
59
+ monkeypatch.delenv("AWS_BATCH_JOB_ID", raising=False)
60
+ assert main() == 1
61
+
62
+
63
+ @pytest.mark.django_db
64
+ def test_uses_uuid_when_no_batch_job_id(monkeypatch, settings):
65
+ settings.LAMBDA_TASKS_EAGER = True
66
+ settings.LAMBDA_TASKS_QUEUES = QUEUES
67
+
68
+ task_name = f"{_worker_task.__module__}.{_worker_task.__wrapped__.__qualname__}"
69
+ message = SQSLambdaTaskMessage(task_name=task_name, kwargs={"value": 5})
70
+
71
+ monkeypatch.setenv("LAMBDA_TASKS_MESSAGE", message.model_dump_json())
72
+ monkeypatch.delenv("AWS_BATCH_JOB_ID", raising=False)
73
+
74
+ result = main()
75
+
76
+ assert result == 0
77
+ assert TaskRecord.objects.count() == 1
@@ -1,318 +0,0 @@
1
- """Optional OpenTelemetry tracing for the Lambda and Batch execution paths.
2
-
3
- Initialised at import time of ``lambda_tasks.handler`` (before the Lambda
4
- runtime invokes the handler, which is earlier than ``django.setup()``). Driven
5
- entirely by standard OTEL environment variables so this package stays generic:
6
- the consuming application declares the OTEL dependencies and sets
7
- ``OTEL_EXPORTER_OTLP_ENDPOINT`` to the regional AWS X-Ray OTLP traces endpoint
8
- (``https://xray.<region>.amazonaws.com/v1/traces``).
9
-
10
- Spans are exported directly to the X-Ray endpoint — no collector, no daemon,
11
- no Lambda layer. Requests are SigV4-signed with the ambient AWS credentials
12
- (the task / execution role), so there is no API key to manage. When no
13
- credentials are resolvable (local dev / tests) the request is left unsigned.
14
-
15
- The OpenTelemetry and ``requests`` imports are deferred into the functions
16
- below so they are only reached when tracing is enabled (a non-empty
17
- ``OTEL_EXPORTER_OTLP_ENDPOINT`` / ``OTEL_EXPORTER_OTLP_TRACES_ENDPOINT``).
18
- This keeps the OTEL packages a genuinely optional extra: importing
19
- ``lambda_tasks.handler`` without them installed is a no-op, not an
20
- ``ImportError``.
21
- """
22
-
23
- import logging
24
- import os
25
- from typing import TYPE_CHECKING
26
-
27
- if TYPE_CHECKING:
28
- from collections.abc import Callable
29
-
30
- import requests
31
- from opentelemetry.sdk.trace import TracerProvider
32
-
33
- logger = logging.getLogger(__name__)
34
-
35
- _XRAY_SIGV4_SERVICE_NAME = "xray"
36
-
37
-
38
- def tracing_is_enabled() -> bool:
39
- """Whether an OTLP traces endpoint has been configured.
40
-
41
- Either the base ``OTEL_EXPORTER_OTLP_ENDPOINT`` or the per-signal
42
- ``OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`` turns it on.
43
- """
44
- return bool(
45
- os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "").strip()
46
- or os.environ.get("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "").strip()
47
- )
48
-
49
-
50
- def _running_on_lambda() -> bool:
51
- return bool(os.environ.get("AWS_LAMBDA_FUNCTION_NAME"))
52
-
53
-
54
- def _build_signing_session() -> "requests.Session":
55
- import requests
56
- import requests.auth
57
- from botocore.auth import SigV4Auth
58
- from botocore.awsrequest import AWSRequest
59
- from botocore.session import Session as BotocoreSession
60
-
61
- class _SigV4Auth(requests.auth.AuthBase):
62
- """SigV4-signs OTLP exports to the X-Ray endpoint."""
63
-
64
- def __init__(
65
- self, *, service_name: str, region: str, session: BotocoreSession
66
- ) -> None:
67
- self._service_name = service_name
68
- self._region = region
69
- self._session = session
70
-
71
- def __call__(self, r: "requests.PreparedRequest") -> "requests.PreparedRequest":
72
- credentials = self._session.get_credentials()
73
-
74
- if credentials is None:
75
- logger.warning(
76
- "No AWS credentials resolvable; sending OTLP span export to "
77
- "%s unsigned. The X-Ray endpoint will reject unsigned "
78
- "requests, so spans will be dropped.",
79
- r.url,
80
- )
81
- return r
82
-
83
- aws_request = AWSRequest(
84
- method=r.method,
85
- url=r.url,
86
- data=r.body,
87
- headers={"Content-Type": "application/x-protobuf"},
88
- )
89
- SigV4Auth(credentials, self._service_name, self._region).add_auth(
90
- aws_request
91
- )
92
-
93
- r.headers.update(dict(aws_request.headers))
94
-
95
- return r
96
-
97
- region = os.environ.get("AWS_REGION", os.environ.get("AWS_DEFAULT_REGION", ""))
98
-
99
- if not region:
100
- logger.warning(
101
- "No AWS region resolvable from AWS_REGION / AWS_DEFAULT_REGION; "
102
- "SigV4 signing of OTLP span exports may fail."
103
- )
104
-
105
- session = requests.Session()
106
- session.auth = _SigV4Auth(
107
- service_name=_XRAY_SIGV4_SERVICE_NAME,
108
- region=region,
109
- session=BotocoreSession(),
110
- )
111
-
112
- return session
113
-
114
-
115
- def _build_tracer_provider() -> "TracerProvider":
116
- from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
117
- OTLPSpanExporter,
118
- )
119
- from opentelemetry.sdk.extension.aws.trace import AwsXRayIdGenerator
120
- from opentelemetry.sdk.resources import Resource
121
- from opentelemetry.sdk.trace import TracerProvider
122
- from opentelemetry.sdk.trace.export import BatchSpanProcessor
123
-
124
- service_name = os.environ.get(
125
- "OTEL_SERVICE_NAME",
126
- os.environ.get("AWS_LAMBDA_FUNCTION_NAME", "lambda-tasks"),
127
- )
128
-
129
- tracer_provider = TracerProvider(
130
- resource=Resource.create(attributes={"service.name": service_name}),
131
- id_generator=AwsXRayIdGenerator(),
132
- )
133
-
134
- tracer_provider.add_span_processor(
135
- BatchSpanProcessor(
136
- span_exporter=OTLPSpanExporter(session=_build_signing_session())
137
- )
138
- )
139
- return tracer_provider
140
-
141
-
142
- def _set_xray_propagator() -> None:
143
- from opentelemetry import propagate
144
- from opentelemetry.propagators.aws import (
145
- AwsXRayLambdaPropagator,
146
- AwsXRayPropagator,
147
- )
148
-
149
- propagate.set_global_textmap(
150
- AwsXRayLambdaPropagator() if _running_on_lambda() else AwsXRayPropagator()
151
- )
152
-
153
-
154
- def _instrument_libraries() -> None:
155
- """Instrument the libraries whose calls we want as spans.
156
-
157
- SQL (psycopg 3), AWS SDK (botocore → boto3/aioboto3), outbound HTTP
158
- (requests and httpx), and cache (redis). Each is wrapped independently and
159
- is best-effort: a missing or failing instrumentor is logged, not raised, so
160
- it cannot take down the worker. All are process-global and idempotent.
161
-
162
- The OTLP span export itself goes out over ``requests`` to the X-Ray
163
- endpoint; that URL is excluded from the requests instrumentor so the
164
- exporter's own calls are not traced (which would be self-referential noise).
165
- """
166
-
167
- def _try(name: str, instrument: "Callable[[], None]") -> None:
168
- try:
169
- instrument()
170
- except Exception:
171
- logger.warning(
172
- "Failed to instrument %s for tracing; continuing without it.",
173
- name,
174
- exc_info=True,
175
- )
176
- else:
177
- logger.info("%s instrumentation installed.", name)
178
-
179
- def _psycopg() -> None:
180
- from opentelemetry.instrumentation.psycopg import PsycopgInstrumentor
181
-
182
- PsycopgInstrumentor().instrument()
183
-
184
- def _botocore() -> None:
185
- from opentelemetry.instrumentation.botocore import BotocoreInstrumentor
186
-
187
- BotocoreInstrumentor().instrument()
188
-
189
- def _requests() -> None:
190
- from opentelemetry.instrumentation.requests import RequestsInstrumentor
191
-
192
- # Do not trace the exporter's own POST to the X-Ray OTLP endpoint.
193
- RequestsInstrumentor().instrument(excluded_urls="xray\\..*\\.amazonaws\\.com")
194
-
195
- def _httpx() -> None:
196
- from opentelemetry.instrumentation.httpx import (
197
- HTTPXClientInstrumentor,
198
- )
199
-
200
- HTTPXClientInstrumentor().instrument()
201
-
202
- def _redis() -> None:
203
- from opentelemetry.instrumentation.redis import RedisInstrumentor
204
-
205
- RedisInstrumentor().instrument()
206
-
207
- _try("psycopg", _psycopg)
208
- _try("botocore", _botocore)
209
- _try("requests", _requests)
210
- _try("httpx", _httpx)
211
- _try("redis", _redis)
212
-
213
-
214
- def configure_tracing() -> "TracerProvider | None":
215
- """Install the global tracer provider. No-op unless enabled.
216
-
217
- Returns the provider so callers (e.g. the Batch entry point) can flush it
218
- on shutdown, or ``None`` when tracing is disabled or already configured.
219
-
220
- Tracing is best-effort: if it is enabled but the OTEL packages are missing
221
- or the provider cannot be built, the failure is logged and ``None`` is
222
- returned rather than propagated — a broken observability setup must not
223
- take down the worker (this runs at import time of the handler module).
224
- """
225
- if not tracing_is_enabled():
226
- logger.debug("OTEL tracing disabled (no OTLP endpoint configured).")
227
- return None
228
-
229
- try:
230
- from opentelemetry import trace
231
- from opentelemetry.sdk.trace import TracerProvider
232
-
233
- if isinstance(trace.get_tracer_provider(), TracerProvider):
234
- logger.info(
235
- "A TracerProvider is already installed; skipping configuration."
236
- )
237
- return None
238
-
239
- tracer_provider = _build_tracer_provider()
240
- trace.set_tracer_provider(tracer_provider)
241
- _set_xray_propagator()
242
- _instrument_libraries()
243
- except ImportError:
244
- logger.warning(
245
- "OTEL tracing is enabled but the OpenTelemetry packages are not "
246
- "installed; tracing is disabled. Install the 'otel' extra to "
247
- "enable it.",
248
- exc_info=True,
249
- )
250
- return None
251
- except Exception:
252
- logger.warning(
253
- "Failed to configure OTEL tracing; continuing without it.",
254
- exc_info=True,
255
- )
256
- return None
257
-
258
- logger.info("OTEL tracing configured (exporting spans to the OTLP endpoint).")
259
- return tracer_provider
260
-
261
-
262
- def instrument_lambda_handler() -> None:
263
- """Wrap the Lambda handler for tracing. No-op unless enabled.
264
-
265
- Must be called at import time of the handler module, before the runtime
266
- invokes the handler.
267
- """
268
- if not tracing_is_enabled():
269
- return
270
-
271
- if not _running_on_lambda():
272
- logger.debug(
273
- "Not running on Lambda (AWS_LAMBDA_FUNCTION_NAME unset); "
274
- "skipping AwsLambdaInstrumentor."
275
- )
276
- return
277
-
278
- try:
279
- from opentelemetry.instrumentation.aws_lambda import (
280
- AwsLambdaInstrumentor,
281
- )
282
-
283
- AwsLambdaInstrumentor().instrument()
284
- except Exception:
285
- logger.warning(
286
- "Failed to instrument the Lambda handler for tracing; "
287
- "continuing without handler spans.",
288
- exc_info=True,
289
- )
290
- return
291
-
292
- logger.info("AwsLambdaInstrumentor installed; Lambda handler is traced.")
293
-
294
-
295
- def flush_tracing() -> None:
296
- """Force-flush and shut down the global tracer provider. No-op unless enabled.
297
-
298
- Called at the end of the one-shot Batch entry point so short-lived jobs do
299
- not drop buffered spans. Gated on ``tracing_is_enabled()`` so the OTEL
300
- imports are never reached when tracing is disabled (or the packages are not
301
- installed).
302
- """
303
- if not tracing_is_enabled():
304
- return
305
-
306
- from opentelemetry import trace
307
- from opentelemetry.sdk.trace import TracerProvider
308
-
309
- provider = trace.get_tracer_provider()
310
- if isinstance(provider, TracerProvider):
311
- logger.debug("Flushing and shutting down the TracerProvider before exit.")
312
- provider.force_flush()
313
- provider.shutdown()
314
- else:
315
- logger.debug(
316
- "Tracing is enabled but no TracerProvider is installed; "
317
- "nothing to flush."
318
- )