steady-queue 0.2.0__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. {steady_queue-0.2.0/steady_queue.egg-info → steady_queue-0.3.0}/PKG-INFO +15 -5
  2. {steady_queue-0.2.0 → steady_queue-0.3.0}/README.md +14 -4
  3. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/__init__.py +1 -1
  4. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/app_executor.py +4 -5
  5. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/queue.py +13 -2
  6. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/recurring_execution.py +8 -4
  7. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/recurring_task.py +5 -1
  8. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/processes/runnable.py +22 -5
  9. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/processes/supervisor.py +58 -18
  10. steady_queue-0.3.0/steady_queue/signals.py +37 -0
  11. {steady_queue-0.2.0 → steady_queue-0.3.0/steady_queue.egg-info}/PKG-INFO +15 -5
  12. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue.egg-info/SOURCES.txt +4 -0
  13. steady_queue-0.3.0/tests/test_app_executor.py +84 -0
  14. {steady_queue-0.2.0 → steady_queue-0.3.0}/tests/test_executions.py +57 -1
  15. steady_queue-0.3.0/tests/test_fork_safety.py +183 -0
  16. steady_queue-0.3.0/tests/test_observability_signals.py +203 -0
  17. {steady_queue-0.2.0 → steady_queue-0.3.0}/tests/test_process_lifecycle.py +24 -10
  18. steady_queue-0.3.0/tests/test_recurring_execution.py +36 -0
  19. steady_queue-0.2.0/tests/test_fork_safety.py +0 -98
  20. {steady_queue-0.2.0 → steady_queue-0.3.0}/LICENSE +0 -0
  21. {steady_queue-0.2.0 → steady_queue-0.3.0}/pyproject.toml +0 -0
  22. {steady_queue-0.2.0 → steady_queue-0.3.0}/setup.cfg +0 -0
  23. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/admin.py +0 -0
  24. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/apps.py +0 -0
  25. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/arguments.py +0 -0
  26. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/backend.py +0 -0
  27. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/collections.py +0 -0
  28. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/concurrency.py +0 -0
  29. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/configuration.py +0 -0
  30. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/db_router.py +0 -0
  31. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/management/__init__.py +0 -0
  32. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/management/commands/__init__.py +0 -0
  33. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/management/commands/steady_queue.py +0 -0
  34. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/migrations/0001_initial.py +0 -0
  35. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/migrations/__init__.py +0 -0
  36. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/__init__.py +0 -0
  37. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/base.py +0 -0
  38. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/blocked_execution.py +0 -0
  39. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/claimed_execution.py +0 -0
  40. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/clearable.py +0 -0
  41. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/concurrency_controls.py +0 -0
  42. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/dispatching.py +0 -0
  43. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/executable.py +0 -0
  44. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/execution.py +0 -0
  45. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/executor.py +0 -0
  46. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/failed_execution.py +0 -0
  47. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/job.py +0 -0
  48. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/pause.py +0 -0
  49. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/process.py +0 -0
  50. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/prunable.py +0 -0
  51. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/ready_execution.py +0 -0
  52. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/retryable.py +0 -0
  53. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/schedulable.py +0 -0
  54. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/scheduled_execution.py +0 -0
  55. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/semaphore.py +0 -0
  56. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/processes/__init__.py +0 -0
  57. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/processes/base.py +0 -0
  58. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/processes/concurrent.py +0 -0
  59. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/processes/dispatcher.py +0 -0
  60. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/processes/errors.py +0 -0
  61. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/processes/interruptible.py +0 -0
  62. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/processes/maintenance.py +0 -0
  63. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/processes/pidfile.py +0 -0
  64. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/processes/pidfiled.py +0 -0
  65. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/processes/poller.py +0 -0
  66. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/processes/pool.py +0 -0
  67. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/processes/registrable.py +0 -0
  68. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/processes/scheduler.py +0 -0
  69. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/processes/signals.py +0 -0
  70. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/processes/supervised.py +0 -0
  71. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/processes/timer.py +0 -0
  72. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/processes/worker.py +0 -0
  73. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/queue_selector.py +0 -0
  74. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/recurring_schedule.py +0 -0
  75. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/recurring_task.py +0 -0
  76. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/task.py +0 -0
  77. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue.egg-info/dependency_links.txt +0 -0
  78. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue.egg-info/requires.txt +0 -0
  79. {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue.egg-info/top_level.txt +0 -0
  80. {steady_queue-0.2.0 → steady_queue-0.3.0}/tests/test_arguments.py +0 -0
  81. {steady_queue-0.2.0 → steady_queue-0.3.0}/tests/test_backend.py +0 -0
  82. {steady_queue-0.2.0 → steady_queue-0.3.0}/tests/test_concurrency_controls.py +0 -0
  83. {steady_queue-0.2.0 → steady_queue-0.3.0}/tests/test_configuration.py +0 -0
  84. {steady_queue-0.2.0 → steady_queue-0.3.0}/tests/test_db_router.py +0 -0
  85. {steady_queue-0.2.0 → steady_queue-0.3.0}/tests/test_job.py +0 -0
  86. {steady_queue-0.2.0 → steady_queue-0.3.0}/tests/test_pidfile.py +0 -0
  87. {steady_queue-0.2.0 → steady_queue-0.3.0}/tests/test_queue_selector.py +0 -0
  88. {steady_queue-0.2.0 → steady_queue-0.3.0}/tests/test_recurring_task.py +0 -0
  89. {steady_queue-0.2.0 → steady_queue-0.3.0}/tests/test_semaphore.py +0 -0
  90. {steady_queue-0.2.0 → steady_queue-0.3.0}/tests/test_task.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: steady-queue
3
- Version: 0.2.0
3
+ Version: 0.3.0
4
4
  Summary: An asynchronous task library for Django that supports delayed tasks, concurrency controls, recurring tasks, pausing queues, numeric priorities per task, priorities by queue order and bulk enqueueing.
5
5
  Author-email: Elias Hernandis <elias@hernandis.me>
6
6
  License-Expression: MIT
@@ -535,8 +535,17 @@ instance that is handling the task, as well as the `task_result` (a
535
535
  `django.tasks.TaskResult` instance) with information on how the task was called
536
536
  and its status.
537
537
 
538
- Unlike Solid Queue, steady queue doesn't yet emit signals related to the
539
- lifecycle of its processes.
538
+ Steady Queue also emits operational signals from ``steady_queue.signals``:
539
+
540
+ - `process_started` and `process_stopped` when a supervisor or child process
541
+ enters or leaves its run loop.
542
+ - `process_restarted` when the supervisor replaces a terminated child.
543
+ - `queue_paused` and `queue_resumed` after queue control actions.
544
+
545
+ Process signals include process identity and metadata. Queue signals include a
546
+ `changed` flag so receivers can distinguish state transitions from idempotent
547
+ actions. See the [configuration documentation](https://steady-queue.readthedocs.io/en/latest/configuration.html#signals)
548
+ for the complete payloads.
540
549
 
541
550
  ## Logging
542
551
 
@@ -824,8 +833,9 @@ there are a few differences which we outline below.
824
833
  will ever be supported, but we've kept the database column for compatibility.
825
834
  - Steady Queue worker processes do not set the process name (or procline)
826
835
  because doing so requires introducing an external dependency.
827
- - Steady Queue does not expose rich instrumentation like Solid Queue does due to
828
- the lack of a framework-native equivalent to `ActiveSupport::Notifications`.
836
+ - Steady Queue exposes Django signals for task and operational lifecycle events,
837
+ but does not yet provide the full timed instrumentation event set emitted by
838
+ Solid Queue through `ActiveSupport::Notifications`.
829
839
  - **Priority ordering:** Steady Queue follows Django's convention where larger
830
840
  numbers indicate higher priority (e.g., a task with priority 10 runs before
831
841
  priority 0), whereas Solid Queue uses the inverse (smaller numbers = higher
@@ -506,8 +506,17 @@ instance that is handling the task, as well as the `task_result` (a
506
506
  `django.tasks.TaskResult` instance) with information on how the task was called
507
507
  and its status.
508
508
 
509
- Unlike Solid Queue, steady queue doesn't yet emit signals related to the
510
- lifecycle of its processes.
509
+ Steady Queue also emits operational signals from ``steady_queue.signals``:
510
+
511
+ - `process_started` and `process_stopped` when a supervisor or child process
512
+ enters or leaves its run loop.
513
+ - `process_restarted` when the supervisor replaces a terminated child.
514
+ - `queue_paused` and `queue_resumed` after queue control actions.
515
+
516
+ Process signals include process identity and metadata. Queue signals include a
517
+ `changed` flag so receivers can distinguish state transitions from idempotent
518
+ actions. See the [configuration documentation](https://steady-queue.readthedocs.io/en/latest/configuration.html#signals)
519
+ for the complete payloads.
511
520
 
512
521
  ## Logging
513
522
 
@@ -795,8 +804,9 @@ there are a few differences which we outline below.
795
804
  will ever be supported, but we've kept the database column for compatibility.
796
805
  - Steady Queue worker processes do not set the process name (or procline)
797
806
  because doing so requires introducing an external dependency.
798
- - Steady Queue does not expose rich instrumentation like Solid Queue does due to
799
- the lack of a framework-native equivalent to `ActiveSupport::Notifications`.
807
+ - Steady Queue exposes Django signals for task and operational lifecycle events,
808
+ but does not yet provide the full timed instrumentation event set emitted by
809
+ Solid Queue through `ActiveSupport::Notifications`.
800
810
  - **Priority ordering:** Steady Queue follows Django's convention where larger
801
811
  numbers indicate higher priority (e.g., a task with priority 10 runs before
802
812
  priority 0), whereas Solid Queue uses the inverse (smaller numbers = higher
@@ -1,7 +1,7 @@
1
1
  from datetime import timedelta
2
2
  from typing import Optional
3
3
 
4
- VERSION = (0, 2, 0)
4
+ VERSION = (0, 3, 0)
5
5
 
6
6
  __version__ = ".".join(map(str, VERSION))
7
7
 
@@ -40,9 +40,8 @@ class AppExecutor:
40
40
  connection.close()
41
41
  raise
42
42
  finally:
43
- # Django automatically manages connection lifecycle,
44
- # but we can explicitly close if needed for long-running processes
45
43
  if threading.current_thread() != threading.main_thread():
46
- # Only close connections in background threads
47
- # Main thread connections are handled by Django's request/response cycle
48
- connection.close()
44
+ # Worker threads are reused without Django's request cleanup.
45
+ # Return connections for every initialized alias, including the
46
+ # application databases used by the task, to their pools.
47
+ connections.close_all()
@@ -2,6 +2,7 @@ from django.db import models
2
2
 
3
3
  from steady_queue.models.pause import Pause
4
4
  from steady_queue.models.ready_execution import ReadyExecution
5
+ from steady_queue.signals import QueueLifecycle, queue_paused, queue_resumed
5
6
 
6
7
 
7
8
  class QueueQuerySet(models.QuerySet):
@@ -48,10 +49,20 @@ class Queue(models.Model):
48
49
  return not self.is_paused
49
50
 
50
51
  def pause(self) -> None:
51
- Pause.objects.get_or_create(queue_name=self.queue_name)
52
+ _, changed = Pause.objects.get_or_create(queue_name=self.queue_name)
53
+ queue_paused.send(
54
+ sender=QueueLifecycle,
55
+ queue_name=self.queue_name,
56
+ changed=changed,
57
+ )
52
58
 
53
59
  def resume(self) -> None:
54
- Pause.objects.filter(queue_name=self.queue_name).delete()
60
+ deleted, _ = Pause.objects.filter(queue_name=self.queue_name).delete()
61
+ queue_resumed.send(
62
+ sender=QueueLifecycle,
63
+ queue_name=self.queue_name,
64
+ changed=deleted > 0,
65
+ )
55
66
 
56
67
  def __str__(self) -> str:
57
68
  return self.queue_name
@@ -1,4 +1,4 @@
1
- from django.db import models
1
+ from django.db import IntegrityError, models
2
2
  from django.tasks import TaskResult
3
3
 
4
4
  from .execution import Execution, ExecutionQuerySet
@@ -9,9 +9,10 @@ class RecurringExecutionQuerySet(ExecutionQuerySet):
9
9
  return self.filter(job__isnull=True)
10
10
 
11
11
  def record(self, task_result: TaskResult, task, run_at):
12
- self.update_or_create(
13
- task=task, run_at=run_at, defaults={"job_id": task_result.id}
14
- )
12
+ try:
13
+ self.create(task=task, run_at=run_at, job_id=task_result.id)
14
+ except IntegrityError as e:
15
+ raise self.model.AlreadyRecorded from e
15
16
 
16
17
  def clear_in_batches(self, batch_size=500):
17
18
  while True:
@@ -21,6 +22,9 @@ class RecurringExecutionQuerySet(ExecutionQuerySet):
21
22
 
22
23
 
23
24
  class RecurringExecution(Execution):
25
+ class AlreadyRecorded(Exception):
26
+ pass
27
+
24
28
  class Meta:
25
29
  verbose_name = "recurring execution"
26
30
  verbose_name_plural = "recurring executions"
@@ -97,7 +97,10 @@ class RecurringTask(UpdatedAtMixin, BaseModel):
97
97
  return self.recurring_executions.max("run_at")
98
98
 
99
99
  def enqueue(self, run_at: datetime):
100
- return self.enqueue_and_record(run_at)
100
+ try:
101
+ return self.enqueue_and_record(run_at)
102
+ except RecurringExecution.AlreadyRecorded:
103
+ return False
101
104
 
102
105
  def enqueue_and_record(self, run_at: datetime):
103
106
  with transaction.atomic(using=self._state.db):
@@ -108,6 +111,7 @@ class RecurringTask(UpdatedAtMixin, BaseModel):
108
111
  queue_name=self.queue_name, priority=self.priority
109
112
  ).enqueue(*args, **kwargs)
110
113
  RecurringExecution.objects.record(task_result, self, run_at)
114
+ return task_result
111
115
 
112
116
  @property
113
117
  def previous_time(self) -> datetime:
@@ -1,6 +1,12 @@
1
1
  import logging
2
2
 
3
3
  from steady_queue.processes.supervised import Supervised
4
+ from steady_queue.signals import (
5
+ _process_signal_context,
6
+ _send_process_signal,
7
+ process_started,
8
+ process_stopped,
9
+ )
4
10
 
5
11
  logger = logging.getLogger("steady_queue")
6
12
 
@@ -10,11 +16,22 @@ class Runnable(Supervised):
10
16
 
11
17
  def start(self):
12
18
  self.boot()
13
-
14
- if self.is_running_async:
15
- raise NotImplementedError
16
- else:
17
- self.run()
19
+ signal_context = _process_signal_context(self)
20
+ _send_process_signal(process_started, self, context=signal_context)
21
+
22
+ error = None
23
+ try:
24
+ if self.is_running_async:
25
+ raise NotImplementedError
26
+ else:
27
+ self.run()
28
+ except BaseException as exception:
29
+ error = exception
30
+ raise
31
+ finally:
32
+ _send_process_signal(
33
+ process_stopped, self, context=signal_context, error=error
34
+ )
18
35
 
19
36
  def stop(self):
20
37
  super().stop()
@@ -15,6 +15,14 @@ from steady_queue.processes.pidfiled import Pidfiled
15
15
  from steady_queue.processes.registrable import Registrable
16
16
  from steady_queue.processes.signals import Signals
17
17
  from steady_queue.processes.timer import wait_until
18
+ from steady_queue.signals import (
19
+ ProcessLifecycle,
20
+ _process_signal_context,
21
+ _send_process_signal,
22
+ process_restarted,
23
+ process_started,
24
+ process_stopped,
25
+ )
18
26
 
19
27
  logger = logging.getLogger("steady_queue")
20
28
 
@@ -40,19 +48,35 @@ class Supervisor(Maintenance, Signals, Pidfiled, Registrable, Interruptible, Bas
40
48
  super().__init__()
41
49
 
42
50
  def start(self) -> None:
43
- logger.info("starting supervisor with PID %(pid)d", {"pid": self.pid})
51
+ originating_pid = self.pid
52
+ logger.info("starting supervisor with PID %(pid)d", {"pid": originating_pid})
53
+ started = False
54
+ error = None
55
+ signal_context = None
44
56
  try:
45
- self.boot()
46
- # Fork only after resetting DB state (connections + psycopg pools).
47
- self.reset_database_connections()
48
- self.start_processes()
49
- self.launch_maintenance_task()
50
- except SystemExit:
51
- logger.info("supervisor interrupted during boot, shutting down")
52
- self.restore_default_signal_handlers()
53
- self.shutdown()
54
- return
55
- self.supervise()
57
+ try:
58
+ self.boot()
59
+ started = True
60
+ signal_context = _process_signal_context(self)
61
+ _send_process_signal(process_started, self, context=signal_context)
62
+ # Fork only after resetting DB state (connections + psycopg pools).
63
+ self.reset_database_connections()
64
+ self.start_processes()
65
+ self.launch_maintenance_task()
66
+ except SystemExit:
67
+ logger.info("supervisor interrupted during boot, shutting down")
68
+ self.restore_default_signal_handlers()
69
+ self.shutdown()
70
+ return
71
+ self.supervise()
72
+ except BaseException as exception:
73
+ error = exception
74
+ raise
75
+ finally:
76
+ if started and self.pid == originating_pid:
77
+ _send_process_signal(
78
+ process_stopped, self, context=signal_context, error=error
79
+ )
56
80
 
57
81
  def boot(self) -> None:
58
82
  super().boot()
@@ -80,12 +104,15 @@ class Supervisor(Maintenance, Signals, Pidfiled, Registrable, Interruptible, Bas
80
104
  logger.debug("supervisor finally block")
81
105
  self.shutdown()
82
106
 
83
- def start_process(self, process: Configuration.Process) -> None:
107
+ def start_process(self, process: Configuration.Process) -> int:
84
108
  logger.info("starting process %(process)s", {"process": process})
85
109
  instance = process.instantiate()
86
110
  instance.supervisor = self.process
87
111
  instance.mode = "fork"
88
112
 
113
+ # Replacement recovery queries the database after the startup reset.
114
+ # Clear that connection and pool state before the child inherits it.
115
+ self.reset_database_connections()
89
116
  if (pid := os.fork()) == 0:
90
117
  # child
91
118
  instance.start()
@@ -95,6 +122,7 @@ class Supervisor(Maintenance, Signals, Pidfiled, Registrable, Interruptible, Bas
95
122
  self.reset_database_connections()
96
123
  self.configured_processes[pid] = process
97
124
  self.forks[pid] = instance
125
+ return pid
98
126
 
99
127
  def set_procline(self) -> None:
100
128
  pass
@@ -130,14 +158,14 @@ class Supervisor(Maintenance, Signals, Pidfiled, Registrable, Interruptible, Bas
130
158
  def reap_and_replace_terminated_forks(self) -> None:
131
159
  while True:
132
160
  try:
133
- pid, exitcode = os.waitpid(-1, os.WNOHANG)
161
+ pid, wait_status = os.waitpid(-1, os.WNOHANG)
134
162
  except ChildProcessError:
135
163
  break
136
164
  else:
137
165
  if not pid:
138
166
  break
139
167
 
140
- self.replace_fork(pid, exitcode)
168
+ self.replace_fork(pid, wait_status)
141
169
 
142
170
  def reap_terminated_forks(self) -> None:
143
171
  while True:
@@ -158,11 +186,23 @@ class Supervisor(Maintenance, Signals, Pidfiled, Registrable, Interruptible, Bas
158
186
 
159
187
  self.configured_processes.pop(pid, None)
160
188
 
161
- def replace_fork(self, pid: int, exitcode: int) -> None:
189
+ def replace_fork(self, pid: int, wait_status: int) -> None:
190
+ exitcode = os.waitstatus_to_exitcode(wait_status)
162
191
  logger.info("replacing fork %s due to exit code %s", pid, exitcode)
163
192
  if terminated_fork := self.forks.pop(pid, None):
164
- self.handle_claimed_jobs_by(terminated_fork, exitcode)
165
- self.start_process(self.configured_processes.pop(pid))
193
+ self.handle_claimed_jobs_by(terminated_fork, wait_status)
194
+ replacement_pid = self.start_process(self.configured_processes.pop(pid))
195
+ process_restarted.send(
196
+ sender=ProcessLifecycle,
197
+ process_kind=terminated_fork.kind,
198
+ process_name=terminated_fork.name,
199
+ pid=pid,
200
+ hostname=terminated_fork.hostname,
201
+ metadata=terminated_fork.metadata,
202
+ exitcode=exitcode,
203
+ replacement_pid=replacement_pid,
204
+ supervisor_pid=self.pid,
205
+ )
166
206
 
167
207
  def handle_claimed_jobs_by(self, terminated_fork: Base, exitcode: int) -> None:
168
208
  if not self.process:
@@ -0,0 +1,37 @@
1
+ from django.dispatch import Signal
2
+
3
+
4
+ class ProcessLifecycle:
5
+ """Public sender for process lifecycle signals."""
6
+
7
+
8
+ class QueueLifecycle:
9
+ """Public sender for queue lifecycle signals."""
10
+
11
+
12
+ # Process lifecycle signals carry stable data suitable for logs and metrics.
13
+ process_started = Signal()
14
+ process_stopped = Signal()
15
+ process_restarted = Signal()
16
+
17
+ # Queue control signals. Receivers get ``queue_name`` and ``changed``.
18
+ queue_paused = Signal()
19
+ queue_resumed = Signal()
20
+
21
+
22
+ def _process_signal_context(process) -> dict:
23
+ return {
24
+ "process_kind": process.kind,
25
+ "process_name": process.name,
26
+ "pid": process.pid,
27
+ "hostname": process.hostname,
28
+ "metadata": process.metadata,
29
+ }
30
+
31
+
32
+ def _send_process_signal(signal: Signal, process, *, context=None, **kwargs) -> None:
33
+ signal.send(
34
+ sender=ProcessLifecycle,
35
+ **(context or _process_signal_context(process)),
36
+ **kwargs,
37
+ )
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: steady-queue
3
- Version: 0.2.0
3
+ Version: 0.3.0
4
4
  Summary: An asynchronous task library for Django that supports delayed tasks, concurrency controls, recurring tasks, pausing queues, numeric priorities per task, priorities by queue order and bulk enqueueing.
5
5
  Author-email: Elias Hernandis <elias@hernandis.me>
6
6
  License-Expression: MIT
@@ -535,8 +535,17 @@ instance that is handling the task, as well as the `task_result` (a
535
535
  `django.tasks.TaskResult` instance) with information on how the task was called
536
536
  and its status.
537
537
 
538
- Unlike Solid Queue, steady queue doesn't yet emit signals related to the
539
- lifecycle of its processes.
538
+ Steady Queue also emits operational signals from ``steady_queue.signals``:
539
+
540
+ - `process_started` and `process_stopped` when a supervisor or child process
541
+ enters or leaves its run loop.
542
+ - `process_restarted` when the supervisor replaces a terminated child.
543
+ - `queue_paused` and `queue_resumed` after queue control actions.
544
+
545
+ Process signals include process identity and metadata. Queue signals include a
546
+ `changed` flag so receivers can distinguish state transitions from idempotent
547
+ actions. See the [configuration documentation](https://steady-queue.readthedocs.io/en/latest/configuration.html#signals)
548
+ for the complete payloads.
540
549
 
541
550
  ## Logging
542
551
 
@@ -824,8 +833,9 @@ there are a few differences which we outline below.
824
833
  will ever be supported, but we've kept the database column for compatibility.
825
834
  - Steady Queue worker processes do not set the process name (or procline)
826
835
  because doing so requires introducing an external dependency.
827
- - Steady Queue does not expose rich instrumentation like Solid Queue does due to
828
- the lack of a framework-native equivalent to `ActiveSupport::Notifications`.
836
+ - Steady Queue exposes Django signals for task and operational lifecycle events,
837
+ but does not yet provide the full timed instrumentation event set emitted by
838
+ Solid Queue through `ActiveSupport::Notifications`.
829
839
  - **Priority ordering:** Steady Queue follows Django's convention where larger
830
840
  numbers indicate higher priority (e.g., a task with priority 10 runs before
831
841
  priority 0), whereas Solid Queue uses the inverse (smaller numbers = higher
@@ -14,6 +14,7 @@ steady_queue/db_router.py
14
14
  steady_queue/queue_selector.py
15
15
  steady_queue/recurring_schedule.py
16
16
  steady_queue/recurring_task.py
17
+ steady_queue/signals.py
17
18
  steady_queue/task.py
18
19
  steady_queue.egg-info/PKG-INFO
19
20
  steady_queue.egg-info/SOURCES.txt
@@ -67,6 +68,7 @@ steady_queue/processes/supervised.py
67
68
  steady_queue/processes/supervisor.py
68
69
  steady_queue/processes/timer.py
69
70
  steady_queue/processes/worker.py
71
+ tests/test_app_executor.py
70
72
  tests/test_arguments.py
71
73
  tests/test_backend.py
72
74
  tests/test_concurrency_controls.py
@@ -75,9 +77,11 @@ tests/test_db_router.py
75
77
  tests/test_executions.py
76
78
  tests/test_fork_safety.py
77
79
  tests/test_job.py
80
+ tests/test_observability_signals.py
78
81
  tests/test_pidfile.py
79
82
  tests/test_process_lifecycle.py
80
83
  tests/test_queue_selector.py
84
+ tests/test_recurring_execution.py
81
85
  tests/test_recurring_task.py
82
86
  tests/test_semaphore.py
83
87
  tests/test_task.py
@@ -0,0 +1,84 @@
1
+ from concurrent.futures import ThreadPoolExecutor
2
+ from unittest import skipUnless
3
+
4
+ from django.db import connections, transaction
5
+ from django.test import TransactionTestCase
6
+
7
+ from steady_queue.app_executor import AppExecutor
8
+ from tests.dummy.models import Dummy
9
+
10
+
11
+ @skipUnless(
12
+ connections["default"].vendor == "postgresql",
13
+ "Requires PostgreSQL application connection pooling",
14
+ )
15
+ class AppExecutorConnectionsTest(TransactionTestCase):
16
+ databases = {"default", "queue"}
17
+
18
+ def setUp(self):
19
+ self.application = connections["default"]
20
+ self.application.close()
21
+ self.application.close_pool()
22
+ self.original_options = self.application.settings_dict["OPTIONS"]
23
+ self.application.settings_dict["OPTIONS"] = {
24
+ **self.original_options,
25
+ "pool": {"min_size": 0, "max_size": 1, "timeout": 0.25},
26
+ }
27
+ self.addCleanup(self.restore_application_pool)
28
+
29
+ def restore_application_pool(self):
30
+ self.application.close()
31
+ self.application.close_pool()
32
+ self.application.settings_dict["OPTIONS"] = self.original_options
33
+
34
+ @staticmethod
35
+ def perform_task(*, fail=False):
36
+ with AppExecutor.wrap_in_app_executor():
37
+ Dummy.objects.count()
38
+ if fail:
39
+ raise ValueError("Task failed")
40
+ return (
41
+ connections["default"].connection is None,
42
+ connections["queue"].connection is None,
43
+ )
44
+
45
+ def assert_pool_reusable_by_another_worker(self, *, fail=False):
46
+ # Keep both threads alive: ending a thread must not stand in for the
47
+ # per-task cleanup that long-lived worker threads need.
48
+ workers = [ThreadPoolExecutor(max_workers=1) for _ in range(2)]
49
+ try:
50
+ first = workers[0].submit(self.perform_task, fail=fail)
51
+ if fail:
52
+ with self.assertRaisesMessage(ValueError, "Task failed"):
53
+ first.result(timeout=5)
54
+ else:
55
+ first.result(timeout=5)
56
+
57
+ # A leaked application connection leaves this separate thread
58
+ # unable to borrow the only connection in the application pool.
59
+ state = workers[1].submit(self.perform_task).result(timeout=5)
60
+ self.assertEqual(state, (True, True))
61
+ self.assertEqual(self.application.pool.get_stats()["pool_available"], 1)
62
+ finally:
63
+ for worker in workers:
64
+ worker.submit(connections.close_all).result(timeout=5)
65
+ worker.shutdown()
66
+
67
+ def test_success_returns_application_connection_for_other_workers(self):
68
+ self.assert_pool_reusable_by_another_worker()
69
+
70
+ def test_failure_returns_application_connection_for_other_workers(self):
71
+ self.assert_pool_reusable_by_another_worker(fail=True)
72
+
73
+ def test_main_thread_preserves_callers_connections_and_transaction(self):
74
+ connections["queue"].ensure_connection()
75
+ queue_connection = connections["queue"].connection
76
+ with transaction.atomic():
77
+ Dummy.objects.count()
78
+ application_connection = self.application.connection
79
+ with AppExecutor.wrap_in_app_executor():
80
+ Dummy.objects.count()
81
+ self.assertIs(self.application.connection, application_connection)
82
+ self.assertIs(connections["queue"].connection, queue_connection)
83
+ self.assertTrue(self.application.in_atomic_block)
84
+ Dummy.objects.count()
@@ -1,6 +1,10 @@
1
+ from concurrent.futures import ThreadPoolExecutor
1
2
  from datetime import timedelta
3
+ from unittest import skipUnless
4
+ from unittest.mock import patch
2
5
 
3
- from django.test import TestCase
6
+ from django.db import connections
7
+ from django.test import TestCase, TransactionTestCase
4
8
  from django.utils import timezone
5
9
 
6
10
  import steady_queue
@@ -14,6 +18,7 @@ from steady_queue.models import (
14
18
  ScheduledExecution,
15
19
  Semaphore,
16
20
  )
21
+ from steady_queue.models.claimed_execution import ClaimedExecutionQuerySet
17
22
  from tests.dummy.tasks import dummy_task, limited_task, limited_task_with_lambda_key
18
23
 
19
24
 
@@ -106,6 +111,57 @@ class ReadyExecutionTestCase(TestHelperMixin, TestCase):
106
111
  self.assertEqual(ReadyExecution.objects.count(), 1)
107
112
 
108
113
 
114
+ class ConcurrentClaimTestCase(TestHelperMixin, TransactionTestCase):
115
+ databases = {"default", "queue"}
116
+
117
+ @skipUnless(
118
+ connections["queue"].features.has_select_for_update_skip_locked,
119
+ "The queue database must support SELECT FOR UPDATE SKIP LOCKED",
120
+ )
121
+ def test_higher_priority_arrival_does_not_change_claimed_candidates(self):
122
+ """#25: keep the selected jobs stable across insertion and retrieval."""
123
+ process = self.create_test_process()
124
+ original = self.create_job_in_queue("default", priority=0)
125
+ bulk_create = ClaimedExecutionQuerySet.bulk_create
126
+
127
+ def enqueue_higher_priority_job():
128
+ try:
129
+ return self.create_job_in_queue("default", priority=10).pk
130
+ finally:
131
+ connections.close_all()
132
+
133
+ def insert_claims_then_enqueue(queryset, *args, **kwargs):
134
+ result = bulk_create(queryset, *args, **kwargs)
135
+ # Commit an arrival on another connection after claims are inserted,
136
+ # but before claiming() retrieves them. No sleeps or timing races.
137
+ with ThreadPoolExecutor(max_workers=1) as executor:
138
+ arrival = executor.submit(enqueue_higher_priority_job)
139
+ arrivals.append(arrival.result(timeout=10))
140
+ return result
141
+
142
+ arrivals = []
143
+ with patch.object(
144
+ ClaimedExecutionQuerySet, "bulk_create", insert_claims_then_enqueue
145
+ ):
146
+ claimed = ReadyExecution.objects.claim(["*"], 1, process.pk)
147
+
148
+ self.assertEqual([execution.job_id for execution in claimed], [original.pk])
149
+ self.assertFalse(
150
+ ReadyExecution.objects.filter(
151
+ job_id__in=ClaimedExecution.objects.values("job_id")
152
+ ).exists()
153
+ )
154
+ self.assertEqual(
155
+ list(ReadyExecution.objects.values_list("job_id", flat=True)), arrivals
156
+ )
157
+
158
+ # Draining the remaining queue must not try to claim the original again.
159
+ next_claimed = ReadyExecution.objects.claim(["*"], 1, process.pk)
160
+ self.assertEqual([execution.job_id for execution in next_claimed], arrivals)
161
+ self.assertEqual(ReadyExecution.objects.claim(["*"], 1, process.pk), [])
162
+ self.assertEqual(ClaimedExecution.objects.count(), 2)
163
+
164
+
109
165
  class ClaimedExecutionTestCase(TestHelperMixin, TestCase):
110
166
  """Tests for ClaimedExecution behavior."""
111
167