dj-queue 0.7.1__tar.gz → 0.8.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. {dj_queue-0.7.1 → dj_queue-0.8.1}/PKG-INFO +23 -23
  2. {dj_queue-0.7.1 → dj_queue-0.8.1}/README.md +22 -22
  3. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/admin.py +17 -7
  4. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/api.py +32 -40
  5. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/dashboard.py +6 -7
  6. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/exceptions.py +0 -4
  7. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/models/jobs.py +0 -81
  8. dj_queue-0.8.1/dj_queue/operations/_helpers.py +41 -0
  9. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/operations/concurrency.py +63 -34
  10. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/operations/jobs.py +161 -153
  11. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/operations/recurring.py +12 -14
  12. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/runtime/base.py +2 -1
  13. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/runtime/dispatcher.py +0 -3
  14. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/runtime/supervisor.py +12 -1
  15. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/runtime/worker.py +4 -4
  16. {dj_queue-0.7.1 → dj_queue-0.8.1}/pyproject.toml +1 -1
  17. {dj_queue-0.7.1 → dj_queue-0.8.1}/LICENSE +0 -0
  18. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/__init__.py +0 -0
  19. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/apps.py +0 -0
  20. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/backend.py +0 -0
  21. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/config.py +0 -0
  22. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/contrib/__init__.py +0 -0
  23. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/contrib/asgi.py +0 -0
  24. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/contrib/gunicorn.py +0 -0
  25. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/contrib/prometheus.py +0 -0
  26. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/db.py +0 -0
  27. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/hooks.py +0 -0
  28. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/log.py +0 -0
  29. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/management/__init__.py +0 -0
  30. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/management/commands/__init__.py +0 -0
  31. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/management/commands/dj_queue.py +0 -0
  32. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/management/commands/dj_queue_health.py +0 -0
  33. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/management/commands/dj_queue_prune.py +0 -0
  34. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/migrations/0001_initial.py +0 -0
  35. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/migrations/0002_pause_semaphore.py +0 -0
  36. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/migrations/0003_recurringtask_recurringexecution.py +0 -0
  37. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/migrations/0004_dashboard.py +0 -0
  38. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/migrations/0005_remove_recurringexecution_dj_queue_recurring_executions_task_key_run_at_unique_and_more.py +0 -0
  39. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/migrations/0006_blockedexecution_dj_queue_bl_concurr_2d8393_idx_and_more.py +0 -0
  40. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/migrations/0007_recurringtask_next_run_at.py +0 -0
  41. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/migrations/0008_remove_blockedexecution_dj_queue_bl_concurr_1ce730_idx_and_more.py +0 -0
  42. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/migrations/__init__.py +0 -0
  43. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/models/__init__.py +0 -0
  44. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/models/recurring.py +0 -0
  45. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/models/runtime.py +0 -0
  46. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/observability.py +0 -0
  47. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/operations/__init__.py +0 -0
  48. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/operations/_insert.py +0 -0
  49. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/operations/cleanup.py +0 -0
  50. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/operations/queues.py +0 -0
  51. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/routers.py +0 -0
  52. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/runtime/__init__.py +0 -0
  53. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/runtime/connection_budget.py +0 -0
  54. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/runtime/errors.py +0 -0
  55. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/runtime/interruptible.py +0 -0
  56. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/runtime/notify.py +0 -0
  57. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/runtime/pidfile.py +0 -0
  58. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/runtime/pool.py +0 -0
  59. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/runtime/procline.py +0 -0
  60. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/runtime/scheduler.py +0 -0
  61. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/templates/admin/dj_queue/_dashboard_process_rows.html +0 -0
  62. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/templates/admin/dj_queue/_dashboard_recurring_rows.html +0 -0
  63. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/templates/admin/dj_queue/_dashboard_section_table.html +0 -0
  64. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/templates/admin/dj_queue/_dashboard_semaphore_rows.html +0 -0
  65. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/templates/admin/dj_queue/_paginator.html +0 -0
  66. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/templates/admin/dj_queue/_queue_controls.html +0 -0
  67. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/templates/admin/dj_queue/_sortable_header_cells.html +0 -0
  68. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/templates/admin/dj_queue/change_form.html +0 -0
  69. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/templates/admin/dj_queue/change_list.html +0 -0
  70. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/templates/admin/dj_queue/dashboard.html +0 -0
  71. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/templates/admin/dj_queue/includes/fieldset.html +0 -0
  72. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/templates/admin/dj_queue/queue_jobs.html +0 -0
  73. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/templatetags/__init__.py +0 -0
  74. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/templatetags/dj_queue_admin.py +0 -0
  75. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/urls.py +0 -0
  76. {dj_queue-0.7.1 → dj_queue-0.8.1}/dj_queue/views.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dj-queue
3
- Version: 0.7.1
3
+ Version: 0.8.1
4
4
  Summary: Database-backed task queue backend for Django’s Tasks framework.
5
5
  License-Expression: MIT
6
6
  License-File: LICENSE
@@ -461,7 +461,7 @@ expecting one semaphore namespace per backend alias.
461
461
  tables:
462
462
 
463
463
  ```python
464
- from dj_queue.api import QueueInfo
464
+ from dj_queue.api import QueueInfo, claim_ready_jobs, execute_claimed_job
465
465
 
466
466
  orders = QueueInfo("orders")
467
467
 
@@ -472,6 +472,10 @@ print(orders.paused)
472
472
  orders.pause()
473
473
  orders.resume()
474
474
  orders.clear()
475
+
476
+ claimed_jobs = claim_ready_jobs(limit=1, queues=["orders"])
477
+ if claimed_jobs:
478
+ execute_claimed_job(claimed_jobs[0])
475
479
  ```
476
480
 
477
481
  Notes:
@@ -480,6 +484,8 @@ Notes:
480
484
  - pause rows are scoped per backend alias
481
485
  - `clear()` discards ready jobs only
482
486
  - pass `backend_alias=` when you want to target a non-default `TASKS` alias
487
+ - `claim_ready_jobs()` returns `ClaimedJob` objects, so inspect `claimed_job.job` for the persisted row
488
+ - the low-level claim/execute helpers are exposed on `dj_queue.api` for scripts and examples
483
489
 
484
490
  Operational commands:
485
491
 
@@ -506,32 +512,26 @@ queue database, including the exception class, message, and traceback.
506
512
 
507
513
  You can retry and discard failed jobs through Django admin, and any raw job
508
514
  detail page can enqueue a fresh copy of that stored job. The failed-job actions
509
- also stay available directly through the operations layer:
515
+ also stay available through the public API:
510
516
 
511
517
  ```python
512
- from dj_queue.operations.jobs import discard_failed_job, retry_failed_job
518
+ from dj_queue.api import (
519
+ discard_blocked_jobs,
520
+ discard_failed_job,
521
+ discard_failed_jobs,
522
+ discard_ready_jobs,
523
+ discard_scheduled_jobs,
524
+ retry_failed_job,
525
+ retry_failed_jobs,
526
+ )
513
527
 
514
528
  retry_failed_job(job_id)
515
529
  discard_failed_job(job_id)
516
- ```
517
-
518
- Model helpers are available too:
519
-
520
- ```python
521
- from dj_queue.exceptions import UndiscardableError
522
- from dj_queue.models import ClaimedExecution, FailedExecution
523
-
524
- failed = FailedExecution.objects.get(job_id=job_id)
525
- failed.retry()
526
- failed.discard()
527
-
528
- FailedExecution.retry_all(FailedExecution.objects.order_by("job_id"))
529
- FailedExecution.discard_all_in_batches()
530
-
531
- try:
532
- ClaimedExecution.discard_all_in_batches()
533
- except UndiscardableError:
534
- pass
530
+ retry_failed_jobs(job_ids=[job_id_a, job_id_b], batch_size=2)
531
+ discard_ready_jobs(job_ids=[ready_job_id], batch_size=1)
532
+ discard_failed_jobs(batch_size=500)
533
+ discard_scheduled_jobs(job_ids=[scheduled_job_id], batch_size=1)
534
+ discard_blocked_jobs(job_ids=[blocked_job_id], batch_size=1)
535
535
  ```
536
536
 
537
537
  Failures stay inspectable until you act on them.
@@ -433,7 +433,7 @@ expecting one semaphore namespace per backend alias.
433
433
  tables:
434
434
 
435
435
  ```python
436
- from dj_queue.api import QueueInfo
436
+ from dj_queue.api import QueueInfo, claim_ready_jobs, execute_claimed_job
437
437
 
438
438
  orders = QueueInfo("orders")
439
439
 
@@ -444,6 +444,10 @@ print(orders.paused)
444
444
  orders.pause()
445
445
  orders.resume()
446
446
  orders.clear()
447
+
448
+ claimed_jobs = claim_ready_jobs(limit=1, queues=["orders"])
449
+ if claimed_jobs:
450
+ execute_claimed_job(claimed_jobs[0])
447
451
  ```
448
452
 
449
453
  Notes:
@@ -452,6 +456,8 @@ Notes:
452
456
  - pause rows are scoped per backend alias
453
457
  - `clear()` discards ready jobs only
454
458
  - pass `backend_alias=` when you want to target a non-default `TASKS` alias
459
+ - `claim_ready_jobs()` returns `ClaimedJob` objects, so inspect `claimed_job.job` for the persisted row
460
+ - the low-level claim/execute helpers are exposed on `dj_queue.api` for scripts and examples
455
461
 
456
462
  Operational commands:
457
463
 
@@ -478,32 +484,26 @@ queue database, including the exception class, message, and traceback.
478
484
 
479
485
  You can retry and discard failed jobs through Django admin, and any raw job
480
486
  detail page can enqueue a fresh copy of that stored job. The failed-job actions
481
- also stay available directly through the operations layer:
487
+ also stay available through the public API:
482
488
 
483
489
  ```python
484
- from dj_queue.operations.jobs import discard_failed_job, retry_failed_job
490
+ from dj_queue.api import (
491
+ discard_blocked_jobs,
492
+ discard_failed_job,
493
+ discard_failed_jobs,
494
+ discard_ready_jobs,
495
+ discard_scheduled_jobs,
496
+ retry_failed_job,
497
+ retry_failed_jobs,
498
+ )
485
499
 
486
500
  retry_failed_job(job_id)
487
501
  discard_failed_job(job_id)
488
- ```
489
-
490
- Model helpers are available too:
491
-
492
- ```python
493
- from dj_queue.exceptions import UndiscardableError
494
- from dj_queue.models import ClaimedExecution, FailedExecution
495
-
496
- failed = FailedExecution.objects.get(job_id=job_id)
497
- failed.retry()
498
- failed.discard()
499
-
500
- FailedExecution.retry_all(FailedExecution.objects.order_by("job_id"))
501
- FailedExecution.discard_all_in_batches()
502
-
503
- try:
504
- ClaimedExecution.discard_all_in_batches()
505
- except UndiscardableError:
506
- pass
502
+ retry_failed_jobs(job_ids=[job_id_a, job_id_b], batch_size=2)
503
+ discard_ready_jobs(job_ids=[ready_job_id], batch_size=1)
504
+ discard_failed_jobs(batch_size=500)
505
+ discard_scheduled_jobs(job_ids=[scheduled_job_id], batch_size=1)
506
+ discard_blocked_jobs(job_ids=[blocked_job_id], batch_size=1)
507
507
  ```
508
508
 
509
509
  Failures stay inspectable until you act on them.
@@ -28,7 +28,13 @@ from dj_queue.models import (
28
28
  RecurringTask,
29
29
  Semaphore,
30
30
  )
31
- from dj_queue.operations.jobs import dispatch_scheduled_job_now, enqueue_job_again
31
+ from dj_queue.operations.jobs import (
32
+ discard_failed_job,
33
+ dispatch_scheduled_job_now,
34
+ enqueue_job_again,
35
+ retry_failed_job,
36
+ retry_failed_jobs,
37
+ )
32
38
 
33
39
 
34
40
  class DjQueueFirstAdminSite(admin.AdminSite):
@@ -648,12 +654,12 @@ class JobAdmin(HiddenSidebarAdminMixin, admin.ModelAdmin):
648
654
  return self._current_object_redirect(obj, backend_alias=obj.backend_alias)
649
655
 
650
656
  if action == "retry":
651
- obj.failed_execution.retry()
657
+ retry_failed_job(obj.failed_execution.job_id, backend_alias=obj.backend_alias)
652
658
  self.message_user(request, "Retried failed job", level=messages.SUCCESS)
653
659
  return self._current_object_redirect(obj, backend_alias=obj.backend_alias)
654
660
 
655
661
  if action == "discard":
656
- obj.failed_execution.discard()
662
+ discard_failed_job(obj.failed_execution.job_id, backend_alias=obj.backend_alias)
657
663
  self.message_user(request, "Discarded failed job", level=messages.SUCCESS)
658
664
  return HttpResponseRedirect(self._changelist_url(backend_alias=obj.backend_alias))
659
665
 
@@ -676,14 +682,18 @@ class FailedExecutionAdmin(HiddenSidebarAdminMixin, admin.ModelAdmin):
676
682
 
677
683
  @admin.action(description="Retry selected failed jobs")
678
684
  def retry_jobs(self, request, queryset):
679
- retried = FailedExecution.retry_all(queryset)
685
+ retried = retry_failed_jobs(
686
+ job_ids=list(queryset.values_list("job_id", flat=True)),
687
+ batch_size=queryset.count() or 1,
688
+ backend_alias=self._backend_alias(request),
689
+ )
680
690
  self.message_user(request, f"Retried {retried} failed jobs", level=messages.SUCCESS)
681
691
 
682
692
  @admin.action(description="Discard selected failed jobs")
683
693
  def discard_jobs(self, request, queryset):
684
694
  discarded = 0
685
695
  for execution in queryset.select_related("job"):
686
- discarded += execution.discard()
696
+ discarded += discard_failed_job(execution.job_id, backend_alias=execution.job.backend_alias)
687
697
  self.message_user(request, f"Discarded {discarded} failed jobs", level=messages.SUCCESS)
688
698
 
689
699
  @admin.display(description="created at", ordering="created_at")
@@ -707,13 +717,13 @@ class FailedExecutionAdmin(HiddenSidebarAdminMixin, admin.ModelAdmin):
707
717
 
708
718
  if action == "retry":
709
719
  job_id = obj.job_id
710
- obj.retry()
720
+ retry_failed_job(job_id, backend_alias=backend_alias)
711
721
  self.message_user(request, "Retried failed job", level=messages.SUCCESS)
712
722
  url = reverse("admin:dj_queue_job_change", args=[job_id])
713
723
  return HttpResponseRedirect(f"{url}?{urlencode({'backend': backend_alias})}")
714
724
 
715
725
  if action == "discard":
716
- obj.discard()
726
+ discard_failed_job(obj.job_id, backend_alias=backend_alias)
717
727
  self.message_user(request, "Discarded failed job", level=messages.SUCCESS)
718
728
  return HttpResponseRedirect(self._changelist_url(backend_alias=backend_alias))
719
729
 
@@ -3,10 +3,40 @@ from functools import partial
3
3
  from django.db.models.functions import Coalesce
4
4
  from django.db import transaction
5
5
  from django.utils import timezone
6
- from django.utils.module_loading import import_string
7
6
 
8
7
  from dj_queue.db import get_database_alias
9
8
  from dj_queue.models import Pause, ReadyExecution
9
+ from dj_queue.operations.jobs import (
10
+ ClaimedJob,
11
+ claim_ready_jobs,
12
+ discard_blocked_jobs,
13
+ discard_failed_job,
14
+ discard_failed_jobs,
15
+ discard_ready_jobs,
16
+ discard_scheduled_jobs,
17
+ execute_claimed_job,
18
+ retry_failed_job,
19
+ retry_failed_jobs,
20
+ )
21
+ from dj_queue.operations.queues import pause_queue, resume_queue
22
+ from dj_queue.operations.recurring import schedule_recurring_task, unschedule_recurring_task
23
+
24
+ __all__ = [
25
+ "ClaimedJob",
26
+ "QueueInfo",
27
+ "claim_ready_jobs",
28
+ "discard_blocked_jobs",
29
+ "discard_failed_job",
30
+ "discard_failed_jobs",
31
+ "discard_ready_jobs",
32
+ "discard_scheduled_jobs",
33
+ "enqueue_on_commit",
34
+ "execute_claimed_job",
35
+ "retry_failed_job",
36
+ "retry_failed_jobs",
37
+ "schedule_recurring_task",
38
+ "unschedule_recurring_task",
39
+ ]
10
40
 
11
41
 
12
42
  class QueueInfo:
@@ -44,11 +74,9 @@ class QueueInfo:
44
74
  )
45
75
 
46
76
  def pause(self):
47
- pause_queue = import_string("dj_queue.operations.queues.pause_queue")
48
77
  pause_queue(self.queue_name, backend_alias=self.backend_alias)
49
78
 
50
79
  def resume(self):
51
- resume_queue = import_string("dj_queue.operations.queues.resume_queue")
52
80
  resume_queue(self.queue_name, backend_alias=self.backend_alias)
53
81
 
54
82
  def clear(self, *, batch_size=500):
@@ -57,7 +85,7 @@ class QueueInfo:
57
85
  job_ids = list(self._ready_queryset().values_list("job_id", flat=True)[:batch_size])
58
86
  if not job_ids:
59
87
  return deleted
60
- deleted += _discard_ready_jobs(
88
+ deleted += discard_ready_jobs(
61
89
  job_ids=job_ids,
62
90
  batch_size=batch_size,
63
91
  backend_alias=self.backend_alias,
@@ -86,41 +114,5 @@ class QueueInfo:
86
114
  )
87
115
 
88
116
 
89
- def _discard_ready_jobs(*, job_ids, batch_size, backend_alias):
90
- discard_ready_jobs = import_string("dj_queue.operations.jobs.discard_ready_jobs")
91
- return discard_ready_jobs(job_ids=job_ids, batch_size=batch_size, backend_alias=backend_alias)
92
-
93
-
94
117
  def enqueue_on_commit(task, *args, using=None, **kwargs):
95
118
  transaction.on_commit(partial(task.enqueue, *args, **kwargs), using=using)
96
-
97
-
98
- def schedule_recurring_task(
99
- *,
100
- key,
101
- task_path,
102
- schedule,
103
- args=(),
104
- kwargs=None,
105
- queue_name="default",
106
- priority=0,
107
- description="",
108
- backend_alias="default",
109
- ):
110
- operation = import_string("dj_queue.operations.recurring.schedule_recurring_task")
111
- return operation(
112
- key=key,
113
- task_path=task_path,
114
- schedule=schedule,
115
- args=args,
116
- kwargs=kwargs,
117
- queue_name=queue_name,
118
- priority=priority,
119
- description=description,
120
- backend_alias=backend_alias,
121
- )
122
-
123
-
124
- def unschedule_recurring_task(key, *, backend_alias="default"):
125
- operation = import_string("dj_queue.operations.recurring.unschedule_recurring_task")
126
- return operation(key, backend_alias=backend_alias)
@@ -13,13 +13,14 @@ from dj_queue import observability
13
13
  from dj_queue.api import QueueInfo
14
14
  from dj_queue.config import load_backend_config
15
15
  from dj_queue.db import database_capabilities, get_database_alias
16
- from dj_queue.models import FailedExecution, Job
16
+ from dj_queue.models import Job
17
17
  from dj_queue.operations.jobs import (
18
18
  discard_blocked_jobs,
19
19
  discard_failed_jobs,
20
20
  discard_ready_jobs,
21
21
  discard_scheduled_jobs,
22
22
  enqueue_job_again,
23
+ retry_failed_jobs,
23
24
  )
24
25
 
25
26
  QUEUE_STATE_CONFIG = {
@@ -521,13 +522,11 @@ def apply_job_action(*, backend_alias, queue_name, state, action, job_ids):
521
522
  return f"discarded {deleted} blocked jobs from {queue_name}"
522
523
 
523
524
  if state == "failed" and action == "retry":
524
- alias = get_database_alias(backend_alias)
525
- queryset = FailedExecution.objects.using(alias).filter(
526
- job_id__in=job_ids,
527
- job__backend_alias=backend_alias,
528
- job__queue_name=queue_name,
525
+ retried = retry_failed_jobs(
526
+ job_ids=job_ids,
527
+ batch_size=max(len(job_ids), 1),
528
+ backend_alias=backend_alias,
529
529
  )
530
- retried = FailedExecution.retry_all(queryset.select_related("job"))
531
530
  return f"retried {retried} failed jobs from {queue_name}"
532
531
 
533
532
  if state == "failed" and action == "discard":
@@ -6,10 +6,6 @@ class EnqueueError(DjQueueError):
6
6
  pass
7
7
 
8
8
 
9
- class UndiscardableError(DjQueueError):
10
- pass
11
-
12
-
13
9
  class AlreadyRecorded(DjQueueError):
14
10
  pass
15
11
 
@@ -3,10 +3,6 @@ import uuid
3
3
  from django.core.exceptions import ObjectDoesNotExist
4
4
  from django.db import models
5
5
  from django.db.models import Q
6
- from django.utils.module_loading import import_string
7
-
8
- from dj_queue.db import get_database_alias
9
- from dj_queue.exceptions import UndiscardableError
10
6
 
11
7
  JOB_STATUS_RELATIONS = (
12
8
  ("ready", "ready_execution"),
@@ -140,16 +136,6 @@ class ReadyExecution(models.Model):
140
136
  ),
141
137
  ]
142
138
 
143
- @classmethod
144
- def discard_all_in_batches(cls, *, batch_size=500, backend_alias="default"):
145
- operation = import_string("dj_queue.operations.jobs.discard_ready_jobs")
146
- return _discard_jobs_for_state(
147
- cls,
148
- operation,
149
- batch_size=batch_size,
150
- backend_alias=backend_alias,
151
- )
152
-
153
139
 
154
140
  class ScheduledExecution(models.Model):
155
141
  job = models.OneToOneField(
@@ -196,10 +182,6 @@ class ClaimedExecution(models.Model):
196
182
  db_table = "dj_queue_claimed_executions"
197
183
  indexes = [models.Index(fields=["process", "job"])]
198
184
 
199
- @classmethod
200
- def discard_all_in_batches(cls, **_kwargs):
201
- raise UndiscardableError("cannot discard in-progress jobs")
202
-
203
185
 
204
186
  class BlockedExecution(models.Model):
205
187
  job = models.OneToOneField(
@@ -235,16 +217,6 @@ class BlockedExecution(models.Model):
235
217
  ),
236
218
  ]
237
219
 
238
- @classmethod
239
- def discard_all_in_batches(cls, *, batch_size=500, backend_alias="default"):
240
- operation = import_string("dj_queue.operations.jobs.discard_blocked_jobs")
241
- return _discard_jobs_for_state(
242
- cls,
243
- operation,
244
- batch_size=batch_size,
245
- backend_alias=backend_alias,
246
- )
247
-
248
220
 
249
221
  class FailedExecution(models.Model):
250
222
  job = models.OneToOneField(
@@ -260,56 +232,3 @@ class FailedExecution(models.Model):
260
232
  class Meta:
261
233
  db_table = "dj_queue_failed_executions"
262
234
  indexes = [models.Index(fields=["created_at", "job"])]
263
-
264
- def retry(self):
265
- return _retry_failed_job(self.job_id, backend_alias=self.job.backend_alias)
266
-
267
- def discard(self):
268
- return _discard_failed_job(self.job_id, backend_alias=self.job.backend_alias)
269
-
270
- @classmethod
271
- def retry_all(cls, queryset):
272
- retried = 0
273
- for execution in queryset.select_related("job"):
274
- execution.retry()
275
- retried += 1
276
- return retried
277
-
278
- @classmethod
279
- def discard_all_in_batches(cls, *, batch_size=500, backend_alias="default"):
280
- operation = import_string("dj_queue.operations.jobs.discard_failed_jobs")
281
- return _discard_jobs_for_state(
282
- cls,
283
- operation,
284
- batch_size=batch_size,
285
- backend_alias=backend_alias,
286
- )
287
-
288
-
289
- def _retry_failed_job(job_id, *, backend_alias):
290
- operation = import_string("dj_queue.operations.jobs.retry_failed_job")
291
- return operation(job_id, backend_alias=backend_alias)
292
-
293
-
294
- def _discard_failed_job(job_id, *, backend_alias):
295
- operation = import_string("dj_queue.operations.jobs.discard_failed_job")
296
- return operation(job_id, backend_alias=backend_alias)
297
-
298
-
299
- def _discard_jobs_for_state(model, operation, *, batch_size, backend_alias):
300
- alias = get_database_alias(backend_alias)
301
- deleted = 0
302
- while True:
303
- filter_kwargs = (
304
- {"backend_alias": backend_alias}
305
- if any(field.name == "backend_alias" for field in model._meta.fields)
306
- else {"job__backend_alias": backend_alias}
307
- )
308
- job_ids = list(
309
- model.objects.using(alias)
310
- .filter(**filter_kwargs)
311
- .values_list("job_id", flat=True)[:batch_size]
312
- )
313
- if not job_ids:
314
- return deleted
315
- deleted += operation(job_ids=job_ids, batch_size=batch_size, backend_alias=backend_alias)
@@ -0,0 +1,41 @@
1
+ import json
2
+
3
+ from dj_queue.db import database_capabilities
4
+ from dj_queue.exceptions import EnqueueError
5
+ from dj_queue.models import Pause
6
+
7
+
8
+ def _normalize_payload(args, kwargs):
9
+ try:
10
+ return json.loads(json.dumps({"args": list(args), "kwargs": dict(kwargs)}))
11
+ except (TypeError, ValueError) as exc:
12
+ raise EnqueueError("payload must be JSON round-trippable") from exc
13
+
14
+
15
+ def _task_option(task, name, default=None):
16
+ if hasattr(task, name):
17
+ return getattr(task, name)
18
+ return getattr(task.func, name, default)
19
+
20
+
21
+ def _lock_active_pauses(alias, backend_alias, queue_names=None):
22
+ queryset = Pause.objects.using(alias).select_for_update().filter(backend_alias=backend_alias)
23
+ if queue_names is not None:
24
+ active_queue_names = tuple(queue_name for queue_name in queue_names if queue_name)
25
+ if not active_queue_names:
26
+ return set()
27
+ queryset = queryset.filter(queue_name__in=active_queue_names)
28
+ return set(queryset.values_list("queue_name", flat=True))
29
+
30
+
31
+ def _consume_selected_rows(alias, model, rows):
32
+ if not database_capabilities(alias).uses_serialized_writes:
33
+ model.objects.using(alias).filter(pk__in=[row.pk for row in rows]).delete()
34
+ return rows
35
+
36
+ consumed_rows = []
37
+ for row in rows:
38
+ deleted, _ = model.objects.using(alias).filter(pk=row.pk).delete()
39
+ if deleted:
40
+ consumed_rows.append(row)
41
+ return consumed_rows
@@ -1,6 +1,6 @@
1
1
  from datetime import timedelta
2
2
 
3
- from django.db import transaction
3
+ from django.db import connections, transaction
4
4
  from django.db.models import F
5
5
  from django.utils import timezone
6
6
  from django.utils.module_loading import import_string
@@ -9,7 +9,12 @@ from dj_queue.config import load_backend_config
9
9
  from dj_queue.db import database_capabilities, get_database_alias, locked_queryset
10
10
  from dj_queue.exceptions import EnqueueError
11
11
  from dj_queue.log import log_event
12
- from dj_queue.models import BlockedExecution, ClaimedExecution, Pause, ReadyExecution, Semaphore
12
+ from dj_queue.models import BlockedExecution, ClaimedExecution, ReadyExecution, Semaphore
13
+ from dj_queue.operations._helpers import (
14
+ _consume_selected_rows,
15
+ _lock_active_pauses,
16
+ _task_option,
17
+ )
13
18
  from dj_queue.operations._insert import create_ignore_conflicts
14
19
  from dj_queue.runtime import notify as runtime_notify
15
20
 
@@ -24,6 +29,16 @@ def semaphore_acquire(
24
29
  alias = get_database_alias(backend_alias)
25
30
  now = timezone.now()
26
31
  expires_at = now + timedelta(seconds=duration_seconds)
32
+ backend_family = database_capabilities(alias).backend_family
33
+
34
+ if backend_family in {"mysql", "mariadb"}:
35
+ return _mysql_family_semaphore_acquire(
36
+ alias,
37
+ key,
38
+ limit=limit,
39
+ expires_at=expires_at,
40
+ now=now,
41
+ )
27
42
 
28
43
  with transaction.atomic(using=alias):
29
44
  if create_ignore_conflicts(
@@ -36,7 +51,6 @@ def semaphore_acquire(
36
51
  ):
37
52
  return True
38
53
 
39
- # mysql-family backends can deadlock if a skipped insert and row lock happen in one tx
40
54
  with transaction.atomic(using=alias):
41
55
  updated = (
42
56
  Semaphore.objects.using(alias)
@@ -50,6 +64,43 @@ def semaphore_acquire(
50
64
  return updated > 0
51
65
 
52
66
 
67
+ def _mysql_family_semaphore_acquire(alias, key, *, limit, expires_at, now):
68
+ connection = connections[alias]
69
+ table = connection.ops.quote_name(Semaphore._meta.db_table)
70
+ key_column = connection.ops.quote_name("key")
71
+ value_column = connection.ops.quote_name("value")
72
+ limit_column = connection.ops.quote_name("limit")
73
+ expires_at_column = connection.ops.quote_name("expires_at")
74
+ created_at_column = connection.ops.quote_name("created_at")
75
+ updated_at_column = connection.ops.quote_name("updated_at")
76
+
77
+ # one upsert avoids mysql-family deadlocks from mixing ignored inserts and follow-up updates
78
+ with connection.cursor() as cursor:
79
+ cursor.execute(
80
+ f"""
81
+ INSERT INTO {table} (
82
+ {key_column},
83
+ {value_column},
84
+ {limit_column},
85
+ {expires_at_column},
86
+ {created_at_column},
87
+ {updated_at_column}
88
+ )
89
+ VALUES (%s, %s, %s, %s, %s, %s)
90
+ ON DUPLICATE KEY UPDATE
91
+ {expires_at_column} = IF({value_column} > 0, %s, {expires_at_column}),
92
+ {updated_at_column} = IF({value_column} > 0, %s, {updated_at_column}),
93
+ {value_column} = IF(
94
+ {value_column} > 0,
95
+ LAST_INSERT_ID({value_column}) - 1,
96
+ LAST_INSERT_ID(0) + {value_column}
97
+ )
98
+ """,
99
+ [key, limit - 1, limit, expires_at, now, now, expires_at, now],
100
+ )
101
+ return cursor.lastrowid != 0
102
+
103
+
53
104
  def semaphore_release(key, *, duration_seconds, backend_alias="default"):
54
105
  alias = get_database_alias(backend_alias)
55
106
  expires_at = timezone.now() + timedelta(seconds=duration_seconds)
@@ -139,16 +190,23 @@ def unblock_next_blocked_job(
139
190
 
140
191
  def cleanup_expired_semaphores(*, backend_alias="default"):
141
192
  alias = get_database_alias(backend_alias)
142
- active_concurrency_keys = (
193
+ claimed_concurrency_keys = (
143
194
  ClaimedExecution.objects.using(alias)
144
195
  .exclude(job__concurrency_key__isnull=True)
145
196
  .exclude(job__concurrency_key="")
146
197
  .values_list("job__concurrency_key", flat=True)
147
198
  )
199
+ ready_concurrency_keys = (
200
+ ReadyExecution.objects.using(alias)
201
+ .exclude(job__concurrency_key__isnull=True)
202
+ .exclude(job__concurrency_key="")
203
+ .values_list("job__concurrency_key", flat=True)
204
+ )
148
205
  queryset = (
149
206
  Semaphore.objects.using(alias)
150
207
  .filter(expires_at__lte=timezone.now())
151
- .exclude(key__in=active_concurrency_keys)
208
+ .exclude(key__in=claimed_concurrency_keys)
209
+ .exclude(key__in=ready_concurrency_keys)
152
210
  )
153
211
  deleted = queryset.count()
154
212
  if not deleted:
@@ -230,29 +288,6 @@ def promote_expired_blocked_jobs(*, batch_size=500, backend_alias="default", use
230
288
  return promoted_jobs
231
289
 
232
290
 
233
- def _lock_active_pauses(alias, backend_alias, queue_names):
234
- active_queue_names = tuple(queue_name for queue_name in queue_names if queue_name)
235
- if not active_queue_names:
236
- return None
237
-
238
- list(
239
- Pause.objects.using(alias)
240
- .select_for_update()
241
- .filter(backend_alias=backend_alias, queue_name__in=active_queue_names)
242
- .values_list("queue_name", flat=True)
243
- )
244
- return None
245
-
246
-
247
- def _consume_selected_rows(alias, model, rows):
248
- consumed_rows = []
249
- for row in rows:
250
- deleted, _ = model.objects.using(alias).filter(pk=row.pk).delete()
251
- if deleted:
252
- consumed_rows.append(row)
253
- return consumed_rows
254
-
255
-
256
291
  def _positive_int_option(value, name):
257
292
  try:
258
293
  number = int(value)
@@ -262,9 +297,3 @@ def _positive_int_option(value, name):
262
297
  if number <= 0:
263
298
  raise EnqueueError(f"{name} must be a positive integer")
264
299
  return number
265
-
266
-
267
- def _task_option(task, name, default=None):
268
- if hasattr(task, name):
269
- return getattr(task, name)
270
- return getattr(task.func, name, default)