dj-queue 0.9.2__tar.gz → 0.10.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. {dj_queue-0.9.2 → dj_queue-0.10.0}/PKG-INFO +5 -2
  2. {dj_queue-0.9.2 → dj_queue-0.10.0}/README.md +4 -1
  3. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/admin.py +3 -0
  4. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/api.py +15 -12
  5. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/backend.py +2 -0
  6. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/config.py +94 -28
  7. dj_queue-0.10.0/dj_queue/contrib/asgi.py +148 -0
  8. dj_queue-0.10.0/dj_queue/contrib/gunicorn.py +154 -0
  9. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/dashboard.py +43 -1
  10. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/db.py +6 -2
  11. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/log.py +1 -1
  12. dj_queue-0.10.0/dj_queue/migrations/0009_remove_process_dj_queue_processes_name_supervisor_unique_and_more.py +36 -0
  13. dj_queue-0.10.0/dj_queue/migrations/0010_remove_process_djq_pr_name_parent_uniq_and_more.py +23 -0
  14. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/models/recurring.py +0 -12
  15. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/models/runtime.py +13 -2
  16. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/operations/_helpers.py +82 -1
  17. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/operations/cleanup.py +24 -12
  18. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/operations/concurrency.py +117 -25
  19. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/operations/jobs.py +225 -58
  20. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/operations/recurring.py +100 -29
  21. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/runtime/base.py +52 -20
  22. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/runtime/connection_budget.py +3 -2
  23. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/runtime/dispatcher.py +2 -0
  24. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/runtime/notify.py +1 -0
  25. dj_queue-0.10.0/dj_queue/runtime/pidfile.py +68 -0
  26. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/runtime/scheduler.py +2 -0
  27. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/runtime/supervisor.py +56 -18
  28. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/runtime/worker.py +4 -1
  29. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/task_results.py +23 -2
  30. dj_queue-0.10.0/dj_queue/wakeup.py +24 -0
  31. {dj_queue-0.9.2 → dj_queue-0.10.0}/pyproject.toml +2 -1
  32. dj_queue-0.9.2/dj_queue/contrib/asgi.py +0 -47
  33. dj_queue-0.9.2/dj_queue/contrib/gunicorn.py +0 -45
  34. dj_queue-0.9.2/dj_queue/runtime/pidfile.py +0 -39
  35. {dj_queue-0.9.2 → dj_queue-0.10.0}/LICENSE +0 -0
  36. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/__init__.py +0 -0
  37. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/apps.py +0 -0
  38. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/contrib/__init__.py +0 -0
  39. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/contrib/prometheus.py +0 -0
  40. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/cron.py +0 -0
  41. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/exceptions.py +0 -0
  42. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/hooks.py +0 -0
  43. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/management/__init__.py +0 -0
  44. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/management/commands/__init__.py +0 -0
  45. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/management/commands/dj_queue.py +0 -0
  46. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/management/commands/dj_queue_health.py +0 -0
  47. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/management/commands/dj_queue_prune.py +0 -0
  48. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/metrics.py +0 -0
  49. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/migrations/0001_initial.py +0 -0
  50. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/migrations/0002_pause_semaphore.py +0 -0
  51. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/migrations/0003_recurringtask_recurringexecution.py +0 -0
  52. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/migrations/0004_dashboard.py +0 -0
  53. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/migrations/0005_remove_recurringexecution_dj_queue_recurring_executions_task_key_run_at_unique_and_more.py +0 -0
  54. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/migrations/0006_blockedexecution_dj_queue_bl_concurr_2d8393_idx_and_more.py +0 -0
  55. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/migrations/0007_recurringtask_next_run_at.py +0 -0
  56. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/migrations/0008_remove_blockedexecution_dj_queue_bl_concurr_1ce730_idx_and_more.py +0 -0
  57. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/migrations/__init__.py +0 -0
  58. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/models/__init__.py +0 -0
  59. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/models/jobs.py +0 -0
  60. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/observability.py +0 -0
  61. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/operations/__init__.py +0 -0
  62. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/operations/_insert.py +0 -0
  63. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/operations/queues.py +0 -0
  64. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/queue_selectors.py +0 -0
  65. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/queue_state.py +0 -0
  66. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/routers.py +0 -0
  67. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/runtime/__init__.py +0 -0
  68. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/runtime/errors.py +0 -0
  69. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/runtime/interruptible.py +0 -0
  70. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/runtime/pool.py +0 -0
  71. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/runtime/procline.py +0 -0
  72. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/runtime/topology.py +0 -0
  73. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/templates/admin/dj_queue/_dashboard_process_rows.html +0 -0
  74. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/templates/admin/dj_queue/_dashboard_recurring_rows.html +0 -0
  75. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/templates/admin/dj_queue/_dashboard_section_table.html +0 -0
  76. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/templates/admin/dj_queue/_dashboard_semaphore_rows.html +0 -0
  77. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/templates/admin/dj_queue/_paginator.html +0 -0
  78. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/templates/admin/dj_queue/_queue_controls.html +0 -0
  79. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/templates/admin/dj_queue/_sortable_header_cells.html +0 -0
  80. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/templates/admin/dj_queue/change_form.html +0 -0
  81. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/templates/admin/dj_queue/change_list.html +0 -0
  82. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/templates/admin/dj_queue/dashboard.html +0 -0
  83. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/templates/admin/dj_queue/includes/fieldset.html +0 -0
  84. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/templates/admin/dj_queue/queue_jobs.html +0 -0
  85. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/templatetags/__init__.py +0 -0
  86. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/templatetags/dj_queue_admin.py +0 -0
  87. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/urls.py +0 -0
  88. {dj_queue-0.9.2 → dj_queue-0.10.0}/dj_queue/views.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dj-queue
3
- Version: 0.9.2
3
+ Version: 0.10.0
4
4
  Summary: Database-backed task queue backend for Django’s Tasks framework.
5
5
  License-Expression: MIT
6
6
  License-File: LICENSE
@@ -713,9 +713,12 @@ from django.core.asgi import get_asgi_application
713
713
  from dj_queue.contrib.asgi import DjQueueLifespan
714
714
 
715
715
  django_application = get_asgi_application()
716
- application = DjQueueLifespan(django_application)
716
+ application = DjQueueLifespan(django_application, forward_wrapped_lifespan=False)
717
717
  ```
718
718
 
719
+ Set `forward_wrapped_lifespan=False` when the wrapped app does not implement the
720
+ ASGI lifespan protocol itself, such as Django's plain `get_asgi_application()`.
721
+
719
722
  ### Gunicorn
720
723
 
721
724
  Import the provided hooks in your Gunicorn config:
@@ -687,9 +687,12 @@ from django.core.asgi import get_asgi_application
687
687
  from dj_queue.contrib.asgi import DjQueueLifespan
688
688
 
689
689
  django_application = get_asgi_application()
690
- application = DjQueueLifespan(django_application)
690
+ application = DjQueueLifespan(django_application, forward_wrapped_lifespan=False)
691
691
  ```
692
692
 
693
+ Set `forward_wrapped_lifespan=False` when the wrapped app does not implement the
694
+ ASGI lifespan protocol itself, such as Django's plain `get_asgi_application()`.
695
+
693
696
  ### Gunicorn
694
697
 
695
698
  Import the provided hooks in your Gunicorn config:
@@ -389,8 +389,10 @@ class JobConcurrencyKeyListFilter(admin.SimpleListFilter):
389
389
 
390
390
  def lookups(self, request, model_admin):
391
391
  alias = model_admin._backend_database_alias(request)
392
+ backend_alias = model_admin._backend_alias(request)
392
393
  return tuple(
393
394
  Job.objects.using(alias)
395
+ .filter(backend_alias=backend_alias)
394
396
  .exclude(concurrency_key__isnull=True)
395
397
  .exclude(concurrency_key="")
396
398
  .order_by("concurrency_key")
@@ -713,6 +715,7 @@ class FailedExecutionAdmin(HiddenSidebarAdminMixin, admin.ModelAdmin):
713
715
 
714
716
  @admin.register(Process)
715
717
  class ProcessAdmin(HiddenSidebarAdminMixin, admin.ModelAdmin):
718
+ backend_filter_field = "backend_alias"
716
719
  list_display = (
717
720
  "name",
718
721
  "backend_alias",
@@ -1,9 +1,12 @@
1
+ from datetime import timedelta
1
2
  from functools import partial
2
3
 
3
4
  from django.db.models.functions import Coalesce
4
5
  from django.db import transaction
5
6
  from django.utils import timezone
6
7
 
8
+ from dj_queue import observability
9
+ from dj_queue.config import load_backend_config
7
10
  from dj_queue.db import get_database_alias
8
11
  from dj_queue.models import Pause, ReadyExecution
9
12
  from dj_queue.operations.jobs import (
@@ -50,6 +53,9 @@ class QueueInfo:
50
53
 
51
54
  @property
52
55
  def latency(self):
56
+ if self.paused:
57
+ return None
58
+
53
59
  oldest = (
54
60
  self._ready_queryset()
55
61
  .annotate(latency_at=Coalesce("latency_started_at", "created_at"))
@@ -59,7 +65,7 @@ class QueueInfo:
59
65
  )
60
66
  if oldest is None:
61
67
  return 0.0
62
- return (timezone.now() - oldest).total_seconds()
68
+ return max((timezone.now() - oldest).total_seconds(), 0.0)
63
69
 
64
70
  @property
65
71
  def paused(self):
@@ -93,18 +99,15 @@ class QueueInfo:
93
99
 
94
100
  @classmethod
95
101
  def all(cls, *, backend_alias="default"):
96
- alias = get_database_alias(backend_alias)
97
- queue_names = (
98
- ReadyExecution.objects.using(alias)
99
- .filter(backend_alias=backend_alias)
100
- .order_by("queue_name")
101
- .values_list(
102
- "queue_name",
103
- flat=True,
104
- )
105
- .distinct()
102
+ now = timezone.now()
103
+ config = load_backend_config(backend_alias)
104
+ process_cutoff = now - timedelta(seconds=config.process_alive_threshold)
105
+ queue_rows = observability.queue_rows(
106
+ backend_alias=backend_alias,
107
+ now=now,
108
+ process_cutoff=process_cutoff,
106
109
  )
107
- return [cls(queue_name, backend_alias=backend_alias) for queue_name in queue_names]
110
+ return [cls(row["name"], backend_alias=backend_alias) for row in queue_rows]
108
111
 
109
112
  def _ready_queryset(self):
110
113
  alias = get_database_alias(self.backend_alias)
@@ -9,6 +9,7 @@ from dj_queue.operations.jobs import (
9
9
  DispatchOutcome,
10
10
  enqueue_job_with_dispatch,
11
11
  enqueue_jobs_bulk,
12
+ validate_priority,
12
13
  validate_queue_allowed,
13
14
  )
14
15
  from dj_queue.task_results import task_result_from_enqueued_job, task_result_from_job
@@ -22,6 +23,7 @@ class DjQueueBackend(BaseTaskBackend):
22
23
 
23
24
  def validate_task(self, task):
24
25
  validate_queue_allowed(task.queue_name, backend_alias=self.alias)
26
+ validate_priority(task.priority)
25
27
  return super().validate_task(task)
26
28
 
27
29
  def enqueue(self, task, args, kwargs):
@@ -5,7 +5,6 @@ import tomllib
5
5
  import warnings
6
6
  from collections.abc import Mapping, Sequence
7
7
  from dataclasses import asdict, dataclass, field, replace
8
- from functools import lru_cache
9
8
  from pathlib import Path
10
9
  from typing import Any
11
10
 
@@ -61,6 +60,7 @@ DJ_QUEUE_BACKEND_PATH = "dj_queue.backend.DjQueueBackend"
61
60
  TRUTHY_ENV_VALUES = {"1", "true", "yes", "on"}
62
61
  FALSY_ENV_VALUES = {"0", "false", "no", "off"}
63
62
  CONFIG_ENV_KEYS = ("DJ_QUEUE_CONFIG", "DJ_QUEUE_MODE", "DJ_QUEUE_SKIP_RECURRING")
63
+ _BACKEND_CONFIG_CACHE = {}
64
64
 
65
65
 
66
66
  @dataclass(frozen=True, slots=True)
@@ -149,24 +149,29 @@ def load_backend_config(
149
149
  if tasks_settings is None:
150
150
  tasks_settings = getattr(settings, "TASKS", {})
151
151
 
152
- return _load_backend_config_cached(
152
+ env_values = {key: env.get(key) for key in CONFIG_ENV_KEYS if env.get(key) is not None}
153
+ cache_key = (
153
154
  backend_alias,
154
155
  _cache_key(cli_overrides),
155
- _cache_key({key: env.get(key) for key in CONFIG_ENV_KEYS if env.get(key) is not None}),
156
+ _cache_key(env_values),
156
157
  _cache_key(tasks_settings),
157
158
  )
159
+ if cache_key not in _BACKEND_CONFIG_CACHE:
160
+ _BACKEND_CONFIG_CACHE[cache_key] = _load_backend_config_uncached(
161
+ backend_alias,
162
+ cli_overrides,
163
+ env_values,
164
+ tasks_settings,
165
+ )
166
+ return _BACKEND_CONFIG_CACHE[cache_key]
158
167
 
159
168
 
160
- @lru_cache(maxsize=None)
161
- def _load_backend_config_cached(
169
+ def _load_backend_config_uncached(
162
170
  backend_alias: str,
163
- cli_overrides_key: str,
164
- env_key: str,
165
- tasks_settings_key: str,
171
+ cli_overrides: Mapping[str, Any],
172
+ env: Mapping[str, str],
173
+ tasks_settings: Mapping[str, Any],
166
174
  ) -> BackendConfig:
167
- cli_overrides = json.loads(cli_overrides_key)
168
- env = json.loads(env_key)
169
- tasks_settings = json.loads(tasks_settings_key)
170
175
  ensure_dj_queue_backend_alias(tasks_settings, backend_alias)
171
176
  backend_block = _backend_block(tasks_settings, backend_alias)
172
177
  resolved_options = _resolved_options(backend_alias, backend_block, cli_overrides, env)
@@ -185,7 +190,11 @@ def _load_backend_config_cached(
185
190
  resolved_options["preserve_finished_jobs"], "preserve_finished_jobs"
186
191
  )
187
192
  on_thread_error = _validated_callback_path(resolved_options.get("on_thread_error"))
188
- recurring = _build_recurring_config(resolved_options.get("recurring", {}))
193
+ recurring = _build_recurring_config(
194
+ resolved_options.get("recurring", {}),
195
+ allowed_queues=_as_string_tuple(backend_block.get("QUEUES", [])),
196
+ backend_alias=backend_alias,
197
+ )
189
198
  scheduler = _build_scheduler_config(resolved_options.get("scheduler", DEFAULT_SCHEDULER))
190
199
  workers = _build_worker_configs(resolved_options.get("workers", []), mode)
191
200
  dispatchers = _build_dispatcher_configs(resolved_options.get("dispatchers", []))
@@ -228,10 +237,15 @@ def _load_backend_config_cached(
228
237
  shutdown_timeout=_nonnegative_float(resolved_options["shutdown_timeout"], "shutdown_timeout"),
229
238
  supervisor_pidfile=resolved_options["supervisor_pidfile"],
230
239
  preserve_finished_jobs=preserve_finished_jobs,
231
- clear_finished_jobs_after=_optional_int(resolved_options["clear_finished_jobs_after"]),
232
- clear_failed_jobs_after=_optional_int(resolved_options["clear_failed_jobs_after"]),
233
- clear_recurring_executions_after=_optional_int(
234
- resolved_options["clear_recurring_executions_after"]
240
+ clear_finished_jobs_after=_optional_nonnegative_int(
241
+ resolved_options["clear_finished_jobs_after"], "clear_finished_jobs_after"
242
+ ),
243
+ clear_failed_jobs_after=_optional_nonnegative_int(
244
+ resolved_options["clear_failed_jobs_after"], "clear_failed_jobs_after"
245
+ ),
246
+ clear_recurring_executions_after=_optional_nonnegative_int(
247
+ resolved_options["clear_recurring_executions_after"],
248
+ "clear_recurring_executions_after",
235
249
  ),
236
250
  default_concurrency_duration=_positive_int(
237
251
  resolved_options["default_concurrency_duration"],
@@ -506,7 +520,12 @@ def _build_scheduler_config(raw_scheduler: Any) -> SchedulerConfig:
506
520
  )
507
521
 
508
522
 
509
- def _build_recurring_config(raw_recurring: Any) -> dict[str, RecurringTaskConfig]:
523
+ def _build_recurring_config(
524
+ raw_recurring: Any,
525
+ *,
526
+ allowed_queues: tuple[str, ...],
527
+ backend_alias: str,
528
+ ) -> dict[str, RecurringTaskConfig]:
510
529
  if raw_recurring is None:
511
530
  return {}
512
531
  if not isinstance(raw_recurring, Mapping):
@@ -524,14 +543,29 @@ def _build_recurring_config(raw_recurring: Any) -> dict[str, RecurringTaskConfig
524
543
  if not is_valid_cron(str(schedule)):
525
544
  raise ImproperlyConfigured(f"recurring task {key!r} has an invalid cron schedule")
526
545
 
546
+ queue_name = str(raw_entry.get("queue_name", "default"))
547
+ priority = _priority_int(raw_entry.get("priority", 0), f"recurring task {key!r} priority")
548
+ if allowed_queues and queue_name not in allowed_queues:
549
+ raise ImproperlyConfigured(
550
+ f"recurring task {key!r} is invalid: queue {queue_name!r} is not allowed for backend {backend_alias!r}"
551
+ )
552
+ try:
553
+ task = import_string(str(task_path))
554
+ except ImportError as exc:
555
+ raise ImproperlyConfigured(f"recurring task {key!r} is invalid: {exc}") from exc
556
+ if not hasattr(task, "using"):
557
+ raise ImproperlyConfigured(
558
+ f"recurring task {key!r} is invalid: task_path must reference a Django task"
559
+ )
560
+
527
561
  recurring[str(key)] = RecurringTaskConfig(
528
562
  key=str(key),
529
563
  task_path=str(task_path),
530
564
  schedule=str(schedule),
531
565
  args=tuple(raw_entry.get("args", [])),
532
566
  kwargs=dict(raw_entry.get("kwargs", {})),
533
- queue_name=str(raw_entry.get("queue_name", "default")),
534
- priority=int(raw_entry.get("priority", 0)),
567
+ queue_name=queue_name,
568
+ priority=priority,
535
569
  description=str(raw_entry.get("description", "")),
536
570
  )
537
571
  return recurring
@@ -570,13 +604,20 @@ def _as_string_tuple(value: Any) -> tuple[str, ...]:
570
604
  return tuple(str(item) for item in value)
571
605
 
572
606
 
573
- def _optional_int(value: Any) -> int | None:
607
+ def _optional_nonnegative_int(value: Any, setting_name: str) -> int | None:
574
608
  if value is None:
575
609
  return None
576
- return int(value)
610
+ number = _integer(value, setting_name, "a non-negative integer")
611
+ if number < 0:
612
+ raise ImproperlyConfigured(
613
+ f"dj_queue {setting_name} must be a non-negative integer, got {value!r}"
614
+ )
615
+ return number
577
616
 
578
617
 
579
618
  def _positive_float(value: Any, setting_name: str) -> float:
619
+ if isinstance(value, bool):
620
+ raise ImproperlyConfigured(f"dj_queue {setting_name} must be a positive number, got {value!r}")
580
621
  try:
581
622
  number = float(value)
582
623
  except (TypeError, ValueError) as exc:
@@ -590,6 +631,10 @@ def _positive_float(value: Any, setting_name: str) -> float:
590
631
 
591
632
 
592
633
  def _nonnegative_float(value: Any, setting_name: str) -> float:
634
+ if isinstance(value, bool):
635
+ raise ImproperlyConfigured(
636
+ f"dj_queue {setting_name} must be a non-negative number, got {value!r}"
637
+ )
593
638
  try:
594
639
  number = float(value)
595
640
  except (TypeError, ValueError) as exc:
@@ -605,12 +650,7 @@ def _nonnegative_float(value: Any, setting_name: str) -> float:
605
650
 
606
651
 
607
652
  def _positive_int(value: Any, setting_name: str) -> int:
608
- try:
609
- number = int(value)
610
- except (TypeError, ValueError, OverflowError) as exc:
611
- raise ImproperlyConfigured(
612
- f"dj_queue {setting_name} must be a positive integer, got {value!r}"
613
- ) from exc
653
+ number = _integer(value, setting_name, "a positive integer")
614
654
 
615
655
  if number <= 0:
616
656
  raise ImproperlyConfigured(
@@ -619,5 +659,31 @@ def _positive_int(value: Any, setting_name: str) -> int:
619
659
  return number
620
660
 
621
661
 
662
+ def _priority_int(value: Any, setting_name: str) -> int:
663
+ number = _integer(value, setting_name, "an integer from -100 to 100")
664
+
665
+ if number < -100 or number > 100:
666
+ raise ImproperlyConfigured(
667
+ f"dj_queue {setting_name} must be an integer from -100 to 100, got {value!r}"
668
+ )
669
+ return number
670
+
671
+
672
+ def _integer(value: Any, setting_name: str, expectation: str) -> int:
673
+ if isinstance(value, bool):
674
+ raise ImproperlyConfigured(f"dj_queue {setting_name} must be {expectation}, got {value!r}")
675
+ if isinstance(value, int):
676
+ return value
677
+ if isinstance(value, str):
678
+ normalized = value.strip()
679
+ unsigned = normalized[1:] if normalized[:1] in {"+", "-"} else normalized
680
+ if unsigned.isdecimal():
681
+ return int(normalized)
682
+ raise ImproperlyConfigured(f"dj_queue {setting_name} must be {expectation}, got {value!r}")
683
+
684
+
622
685
  def _cache_key(value: Any) -> str:
623
- return json.dumps(value, sort_keys=True, separators=(",", ":"), default=str)
686
+ try:
687
+ return json.dumps(value, sort_keys=True, separators=(",", ":"))
688
+ except (TypeError, ValueError) as exc:
689
+ raise ImproperlyConfigured("dj_queue config values must be JSON-serializable") from exc
@@ -0,0 +1,148 @@
1
+ import asyncio
2
+ import inspect
3
+ from contextlib import suppress
4
+
5
+ from dj_queue.runtime.errors import handle_thread_error
6
+ from dj_queue.runtime.supervisor import AsyncSupervisor
7
+
8
+
9
+ def build_supervisor(backend_alias="default"):
10
+ return AsyncSupervisor.from_backend_config(backend_alias=backend_alias, standalone=False)
11
+
12
+
13
+ class DjQueueLifespan:
14
+ def __init__(self, app, *, backend_alias="default", forward_wrapped_lifespan=True):
15
+ self.app = app
16
+ self.backend_alias = backend_alias
17
+ self.forward_wrapped_lifespan = forward_wrapped_lifespan
18
+ self.supervisor = None
19
+ self._poll_task = None
20
+ self._poll_stop = None
21
+
22
+ async def _poll_supervisor(self):
23
+ while (
24
+ self.supervisor is not None and self._poll_stop is not None and not self._poll_stop.is_set()
25
+ ):
26
+ try:
27
+ await asyncio.to_thread(self.supervisor.poll_once)
28
+ except Exception as error:
29
+ handle_thread_error(
30
+ error,
31
+ context="supervisor.run",
32
+ backend_alias=self.supervisor.backend_alias,
33
+ )
34
+
35
+ if self.supervisor is None or self._poll_stop is None or self._poll_stop.is_set():
36
+ return
37
+
38
+ try:
39
+ await asyncio.wait_for(self._poll_stop.wait(), timeout=self.supervisor.polling_interval)
40
+ except asyncio.TimeoutError:
41
+ continue
42
+
43
+ async def _start_wrapped_lifespan_app(self, scope, receive, send):
44
+ result = self.app(scope, receive, send)
45
+ if inspect.isawaitable(result):
46
+ return asyncio.create_task(result)
47
+
48
+ loop = asyncio.get_running_loop()
49
+ task = loop.create_future()
50
+ task.set_result(result)
51
+ return task
52
+
53
+ async def _forward_lifespan_message(self, app_task, receive_queue, send_queue, message):
54
+ if app_task.done():
55
+ await app_task
56
+ return None
57
+
58
+ await receive_queue.put(message)
59
+ response_task = asyncio.create_task(send_queue.get())
60
+ try:
61
+ done, _ = await asyncio.wait({app_task, response_task}, return_when=asyncio.FIRST_COMPLETED)
62
+ if response_task in done:
63
+ return response_task.result()
64
+ await app_task
65
+ return None
66
+ finally:
67
+ if not response_task.done():
68
+ response_task.cancel()
69
+ with suppress(asyncio.CancelledError):
70
+ await response_task
71
+
72
+ async def _start_supervisor(self):
73
+ self.supervisor = build_supervisor(self.backend_alias)
74
+ await asyncio.to_thread(self.supervisor.start)
75
+ self._poll_stop = asyncio.Event()
76
+ self._poll_task = asyncio.create_task(self._poll_supervisor())
77
+
78
+ @staticmethod
79
+ def _is_unsupported_lifespan_error(error):
80
+ message = str(error).lower()
81
+ return isinstance(error, ValueError) and (
82
+ "django can only handle asgi/http connections" in message and "lifespan" in message
83
+ )
84
+
85
+ async def _stop_supervisor(self):
86
+ poll_stop = self._poll_stop
87
+ self._poll_stop = None
88
+ if poll_stop is not None:
89
+ poll_stop.set()
90
+
91
+ poll_task = self._poll_task
92
+ self._poll_task = None
93
+ if poll_task is not None:
94
+ await poll_task
95
+
96
+ if self.supervisor is not None:
97
+ await asyncio.to_thread(self.supervisor.stop)
98
+ self.supervisor = None
99
+
100
+ async def __call__(self, scope, receive, send):
101
+ if scope["type"] != "lifespan":
102
+ await self.app(scope, receive, send)
103
+ return
104
+
105
+ receive_queue = asyncio.Queue()
106
+ send_queue = asyncio.Queue()
107
+ app_task = await self._start_wrapped_lifespan_app(scope, receive_queue.get, send_queue.put)
108
+ wrapped_app_supports_lifespan = self.forward_wrapped_lifespan
109
+
110
+ try:
111
+ while True:
112
+ message = await receive()
113
+ if message["type"] == "lifespan.startup":
114
+ response = None
115
+ if wrapped_app_supports_lifespan:
116
+ try:
117
+ response = await self._forward_lifespan_message(
118
+ app_task, receive_queue, send_queue, message
119
+ )
120
+ except Exception as error:
121
+ if not self._is_unsupported_lifespan_error(error):
122
+ raise
123
+ wrapped_app_supports_lifespan = False
124
+ if response is not None and response["type"] != "lifespan.startup.complete":
125
+ await send(response)
126
+ return
127
+
128
+ await self._start_supervisor()
129
+ await send({"type": "lifespan.startup.complete"})
130
+ elif message["type"] == "lifespan.shutdown":
131
+ await self._stop_supervisor()
132
+ response = None
133
+ if wrapped_app_supports_lifespan:
134
+ response = await self._forward_lifespan_message(
135
+ app_task, receive_queue, send_queue, message
136
+ )
137
+ if response is None:
138
+ response = {"type": "lifespan.shutdown.complete"}
139
+ await send(response)
140
+ if wrapped_app_supports_lifespan:
141
+ await app_task
142
+ return
143
+ finally:
144
+ await self._stop_supervisor()
145
+ if not app_task.done():
146
+ app_task.cancel()
147
+ with suppress(asyncio.CancelledError):
148
+ await app_task
@@ -0,0 +1,154 @@
1
+ import fcntl
2
+ import tempfile
3
+ import threading
4
+ from pathlib import Path
5
+
6
+ from dj_queue.runtime.errors import handle_thread_error
7
+ from dj_queue.runtime.supervisor import AsyncSupervisor
8
+
9
+ LOCK_PATH = Path(tempfile.gettempdir()) / "dj_queue_gunicorn_supervisor.lock"
10
+ LOCK_RETRY_INTERVAL = 1.0
11
+
12
+
13
+ def build_supervisor(backend_alias="default"):
14
+ return AsyncSupervisor.from_backend_config(backend_alias=backend_alias, standalone=False)
15
+
16
+
17
+ def _set_supervisor_state(worker, **state):
18
+ for name, value in state.items():
19
+ setattr(worker, f"_dj_queue_{name}", value)
20
+
21
+
22
+ def _start_embedded_supervisor(worker, *, backend_alias="default"):
23
+ lock_file = _try_acquire_supervisor_lock()
24
+ if lock_file is None:
25
+ return None
26
+
27
+ try:
28
+ supervisor = build_supervisor(backend_alias=backend_alias)
29
+ poll_stop = threading.Event()
30
+ _set_supervisor_state(
31
+ worker,
32
+ supervisor_lock=lock_file,
33
+ supervisor=supervisor,
34
+ supervisor_poll_stop=poll_stop,
35
+ )
36
+ supervisor.start()
37
+ except Exception:
38
+ _release_supervisor_lock(lock_file)
39
+ _set_supervisor_state(
40
+ worker,
41
+ supervisor_lock=None,
42
+ supervisor=None,
43
+ supervisor_poll_stop=None,
44
+ supervisor_poll_thread=None,
45
+ )
46
+ raise
47
+
48
+ def poll_supervisor():
49
+ stop_event = worker._dj_queue_supervisor_poll_stop
50
+ while stop_event.wait(supervisor.polling_interval) is False:
51
+ try:
52
+ supervisor.poll_once()
53
+ except Exception as error:
54
+ handle_thread_error(
55
+ error,
56
+ context="supervisor.run",
57
+ backend_alias=supervisor.backend_alias,
58
+ )
59
+
60
+ poll_thread = threading.Thread(target=poll_supervisor, daemon=True)
61
+ worker._dj_queue_supervisor_poll_thread = poll_thread
62
+ poll_thread.start()
63
+ return supervisor
64
+
65
+
66
+ def _start_lock_retry_loop(worker, *, backend_alias="default"):
67
+ retry_stop = threading.Event()
68
+
69
+ def retry():
70
+ while retry_stop.wait(LOCK_RETRY_INTERVAL) is False:
71
+ if getattr(worker, "_dj_queue_supervisor", None) is not None:
72
+ return
73
+ if _start_embedded_supervisor(worker, backend_alias=backend_alias) is not None:
74
+ retry_stop.set()
75
+ return
76
+
77
+ retry_thread = threading.Thread(target=retry, daemon=True)
78
+ _set_supervisor_state(
79
+ worker, supervisor_retry_stop=retry_stop, supervisor_retry_thread=retry_thread
80
+ )
81
+ retry_thread.start()
82
+
83
+
84
+ def post_fork(_server, worker):
85
+ _set_supervisor_state(
86
+ worker,
87
+ supervisor=None,
88
+ supervisor_lock=None,
89
+ supervisor_poll_stop=None,
90
+ supervisor_poll_thread=None,
91
+ supervisor_retry_stop=None,
92
+ supervisor_retry_thread=None,
93
+ )
94
+ supervisor = _start_embedded_supervisor(worker)
95
+ if supervisor is not None:
96
+ return supervisor
97
+
98
+ _start_lock_retry_loop(worker)
99
+ return None
100
+
101
+
102
+ def worker_exit(_server, worker):
103
+ supervisor = getattr(worker, "_dj_queue_supervisor", None)
104
+ lock_file = getattr(worker, "_dj_queue_supervisor_lock", None)
105
+ stop_event = getattr(worker, "_dj_queue_supervisor_poll_stop", None)
106
+ poll_thread = getattr(worker, "_dj_queue_supervisor_poll_thread", None)
107
+ retry_stop = getattr(worker, "_dj_queue_supervisor_retry_stop", None)
108
+ retry_thread = getattr(worker, "_dj_queue_supervisor_retry_thread", None)
109
+
110
+ if retry_stop is not None:
111
+ retry_stop.set()
112
+ if retry_thread is not None:
113
+ retry_thread.join(timeout=1)
114
+ worker._dj_queue_supervisor_retry_thread = None
115
+ worker._dj_queue_supervisor_retry_stop = None
116
+
117
+ if stop_event is not None:
118
+ stop_event.set()
119
+ if poll_thread is not None:
120
+ poll_thread.join()
121
+ worker._dj_queue_supervisor_poll_thread = None
122
+ worker._dj_queue_supervisor_poll_stop = None
123
+ if supervisor is None:
124
+ if lock_file is not None:
125
+ _release_supervisor_lock(lock_file)
126
+ worker._dj_queue_supervisor_lock = None
127
+ return None
128
+
129
+ supervisor.stop()
130
+ worker._dj_queue_supervisor = None
131
+ _release_supervisor_lock(lock_file)
132
+ worker._dj_queue_supervisor_lock = None
133
+ return None
134
+
135
+
136
+ def _try_acquire_supervisor_lock():
137
+ LOCK_PATH.parent.mkdir(parents=True, exist_ok=True)
138
+ lock_file = LOCK_PATH.open("a+")
139
+ try:
140
+ fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
141
+ except BlockingIOError:
142
+ lock_file.close()
143
+ return None
144
+ return lock_file
145
+
146
+
147
+ def _release_supervisor_lock(lock_file):
148
+ if lock_file is None:
149
+ return None
150
+ try:
151
+ fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
152
+ finally:
153
+ lock_file.close()
154
+ return None