steady-queue 0.2.1__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. {steady_queue-0.2.1/steady_queue.egg-info → steady_queue-0.3.0}/PKG-INFO +15 -5
  2. {steady_queue-0.2.1 → steady_queue-0.3.0}/README.md +14 -4
  3. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/__init__.py +1 -1
  4. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/app_executor.py +4 -5
  5. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/queue.py +13 -2
  6. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/processes/runnable.py +22 -5
  7. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/processes/supervisor.py +55 -18
  8. steady_queue-0.3.0/steady_queue/signals.py +37 -0
  9. {steady_queue-0.2.1 → steady_queue-0.3.0/steady_queue.egg-info}/PKG-INFO +15 -5
  10. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue.egg-info/SOURCES.txt +3 -0
  11. steady_queue-0.3.0/tests/test_app_executor.py +84 -0
  12. steady_queue-0.3.0/tests/test_observability_signals.py +203 -0
  13. {steady_queue-0.2.1 → steady_queue-0.3.0}/LICENSE +0 -0
  14. {steady_queue-0.2.1 → steady_queue-0.3.0}/pyproject.toml +0 -0
  15. {steady_queue-0.2.1 → steady_queue-0.3.0}/setup.cfg +0 -0
  16. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/admin.py +0 -0
  17. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/apps.py +0 -0
  18. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/arguments.py +0 -0
  19. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/backend.py +0 -0
  20. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/collections.py +0 -0
  21. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/concurrency.py +0 -0
  22. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/configuration.py +0 -0
  23. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/db_router.py +0 -0
  24. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/management/__init__.py +0 -0
  25. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/management/commands/__init__.py +0 -0
  26. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/management/commands/steady_queue.py +0 -0
  27. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/migrations/0001_initial.py +0 -0
  28. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/migrations/__init__.py +0 -0
  29. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/__init__.py +0 -0
  30. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/base.py +0 -0
  31. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/blocked_execution.py +0 -0
  32. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/claimed_execution.py +0 -0
  33. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/clearable.py +0 -0
  34. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/concurrency_controls.py +0 -0
  35. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/dispatching.py +0 -0
  36. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/executable.py +0 -0
  37. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/execution.py +0 -0
  38. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/executor.py +0 -0
  39. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/failed_execution.py +0 -0
  40. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/job.py +0 -0
  41. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/pause.py +0 -0
  42. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/process.py +0 -0
  43. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/prunable.py +0 -0
  44. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/ready_execution.py +0 -0
  45. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/recurring_execution.py +0 -0
  46. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/recurring_task.py +0 -0
  47. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/retryable.py +0 -0
  48. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/schedulable.py +0 -0
  49. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/scheduled_execution.py +0 -0
  50. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/semaphore.py +0 -0
  51. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/processes/__init__.py +0 -0
  52. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/processes/base.py +0 -0
  53. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/processes/concurrent.py +0 -0
  54. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/processes/dispatcher.py +0 -0
  55. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/processes/errors.py +0 -0
  56. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/processes/interruptible.py +0 -0
  57. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/processes/maintenance.py +0 -0
  58. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/processes/pidfile.py +0 -0
  59. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/processes/pidfiled.py +0 -0
  60. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/processes/poller.py +0 -0
  61. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/processes/pool.py +0 -0
  62. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/processes/registrable.py +0 -0
  63. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/processes/scheduler.py +0 -0
  64. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/processes/signals.py +0 -0
  65. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/processes/supervised.py +0 -0
  66. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/processes/timer.py +0 -0
  67. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/processes/worker.py +0 -0
  68. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/queue_selector.py +0 -0
  69. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/recurring_schedule.py +0 -0
  70. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/recurring_task.py +0 -0
  71. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/task.py +0 -0
  72. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue.egg-info/dependency_links.txt +0 -0
  73. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue.egg-info/requires.txt +0 -0
  74. {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue.egg-info/top_level.txt +0 -0
  75. {steady_queue-0.2.1 → steady_queue-0.3.0}/tests/test_arguments.py +0 -0
  76. {steady_queue-0.2.1 → steady_queue-0.3.0}/tests/test_backend.py +0 -0
  77. {steady_queue-0.2.1 → steady_queue-0.3.0}/tests/test_concurrency_controls.py +0 -0
  78. {steady_queue-0.2.1 → steady_queue-0.3.0}/tests/test_configuration.py +0 -0
  79. {steady_queue-0.2.1 → steady_queue-0.3.0}/tests/test_db_router.py +0 -0
  80. {steady_queue-0.2.1 → steady_queue-0.3.0}/tests/test_executions.py +0 -0
  81. {steady_queue-0.2.1 → steady_queue-0.3.0}/tests/test_fork_safety.py +0 -0
  82. {steady_queue-0.2.1 → steady_queue-0.3.0}/tests/test_job.py +0 -0
  83. {steady_queue-0.2.1 → steady_queue-0.3.0}/tests/test_pidfile.py +0 -0
  84. {steady_queue-0.2.1 → steady_queue-0.3.0}/tests/test_process_lifecycle.py +0 -0
  85. {steady_queue-0.2.1 → steady_queue-0.3.0}/tests/test_queue_selector.py +0 -0
  86. {steady_queue-0.2.1 → steady_queue-0.3.0}/tests/test_recurring_execution.py +0 -0
  87. {steady_queue-0.2.1 → steady_queue-0.3.0}/tests/test_recurring_task.py +0 -0
  88. {steady_queue-0.2.1 → steady_queue-0.3.0}/tests/test_semaphore.py +0 -0
  89. {steady_queue-0.2.1 → steady_queue-0.3.0}/tests/test_task.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: steady-queue
3
- Version: 0.2.1
3
+ Version: 0.3.0
4
4
  Summary: An asynchronous task library for Django that supports delayed tasks, concurrency controls, recurring tasks, pausing queues, numeric priorities per task, priorities by queue order and bulk enqueueing.
5
5
  Author-email: Elias Hernandis <elias@hernandis.me>
6
6
  License-Expression: MIT
@@ -535,8 +535,17 @@ instance that is handling the task, as well as the `task_result` (a
535
535
  `django.tasks.TaskResult` instance) with information on how the task was called
536
536
  and its status.
537
537
 
538
- Unlike Solid Queue, steady queue doesn't yet emit signals related to the
539
- lifecycle of its processes.
538
+ Steady Queue also emits operational signals from ``steady_queue.signals``:
539
+
540
+ - `process_started` and `process_stopped` when a supervisor or child process
541
+ enters or leaves its run loop.
542
+ - `process_restarted` when the supervisor replaces a terminated child.
543
+ - `queue_paused` and `queue_resumed` after queue control actions.
544
+
545
+ Process signals include process identity and metadata. Queue signals include a
546
+ `changed` flag so receivers can distinguish state transitions from idempotent
547
+ actions. See the [configuration documentation](https://steady-queue.readthedocs.io/en/latest/configuration.html#signals)
548
+ for the complete payloads.
540
549
 
541
550
  ## Logging
542
551
 
@@ -824,8 +833,9 @@ there are a few differences which we outline below.
824
833
  will ever be supported, but we've kept the database column for compatibility.
825
834
  - Steady Queue worker processes do not set the process name (or procline)
826
835
  because doing so requires introducing an external dependency.
827
- - Steady Queue does not expose rich instrumentation like Solid Queue does due to
828
- the lack of a framework-native equivalent to `ActiveSupport::Notifications`.
836
+ - Steady Queue exposes Django signals for task and operational lifecycle events,
837
+ but does not yet provide the full timed instrumentation event set emitted by
838
+ Solid Queue through `ActiveSupport::Notifications`.
829
839
  - **Priority ordering:** Steady Queue follows Django's convention where larger
830
840
  numbers indicate higher priority (e.g., a task with priority 10 runs before
831
841
  priority 0), whereas Solid Queue uses the inverse (smaller numbers = higher
@@ -506,8 +506,17 @@ instance that is handling the task, as well as the `task_result` (a
506
506
  `django.tasks.TaskResult` instance) with information on how the task was called
507
507
  and its status.
508
508
 
509
- Unlike Solid Queue, steady queue doesn't yet emit signals related to the
510
- lifecycle of its processes.
509
+ Steady Queue also emits operational signals from ``steady_queue.signals``:
510
+
511
+ - `process_started` and `process_stopped` when a supervisor or child process
512
+ enters or leaves its run loop.
513
+ - `process_restarted` when the supervisor replaces a terminated child.
514
+ - `queue_paused` and `queue_resumed` after queue control actions.
515
+
516
+ Process signals include process identity and metadata. Queue signals include a
517
+ `changed` flag so receivers can distinguish state transitions from idempotent
518
+ actions. See the [configuration documentation](https://steady-queue.readthedocs.io/en/latest/configuration.html#signals)
519
+ for the complete payloads.
511
520
 
512
521
  ## Logging
513
522
 
@@ -795,8 +804,9 @@ there are a few differences which we outline below.
795
804
  will ever be supported, but we've kept the database column for compatibility.
796
805
  - Steady Queue worker processes do not set the process name (or procline)
797
806
  because doing so requires introducing an external dependency.
798
- - Steady Queue does not expose rich instrumentation like Solid Queue does due to
799
- the lack of a framework-native equivalent to `ActiveSupport::Notifications`.
807
+ - Steady Queue exposes Django signals for task and operational lifecycle events,
808
+ but does not yet provide the full timed instrumentation event set emitted by
809
+ Solid Queue through `ActiveSupport::Notifications`.
800
810
  - **Priority ordering:** Steady Queue follows Django's convention where larger
801
811
  numbers indicate higher priority (e.g., a task with priority 10 runs before
802
812
  priority 0), whereas Solid Queue uses the inverse (smaller numbers = higher
@@ -1,7 +1,7 @@
1
1
  from datetime import timedelta
2
2
  from typing import Optional
3
3
 
4
- VERSION = (0, 2, 1)
4
+ VERSION = (0, 3, 0)
5
5
 
6
6
  __version__ = ".".join(map(str, VERSION))
7
7
 
@@ -40,9 +40,8 @@ class AppExecutor:
40
40
  connection.close()
41
41
  raise
42
42
  finally:
43
- # Django automatically manages connection lifecycle,
44
- # but we can explicitly close if needed for long-running processes
45
43
  if threading.current_thread() != threading.main_thread():
46
- # Only close connections in background threads
47
- # Main thread connections are handled by Django's request/response cycle
48
- connection.close()
44
+ # Worker threads are reused without Django's request cleanup.
45
+ # Return connections for every initialized alias, including the
46
+ # application databases used by the task, to their pools.
47
+ connections.close_all()
@@ -2,6 +2,7 @@ from django.db import models
2
2
 
3
3
  from steady_queue.models.pause import Pause
4
4
  from steady_queue.models.ready_execution import ReadyExecution
5
+ from steady_queue.signals import QueueLifecycle, queue_paused, queue_resumed
5
6
 
6
7
 
7
8
  class QueueQuerySet(models.QuerySet):
@@ -48,10 +49,20 @@ class Queue(models.Model):
48
49
  return not self.is_paused
49
50
 
50
51
  def pause(self) -> None:
51
- Pause.objects.get_or_create(queue_name=self.queue_name)
52
+ _, changed = Pause.objects.get_or_create(queue_name=self.queue_name)
53
+ queue_paused.send(
54
+ sender=QueueLifecycle,
55
+ queue_name=self.queue_name,
56
+ changed=changed,
57
+ )
52
58
 
53
59
  def resume(self) -> None:
54
- Pause.objects.filter(queue_name=self.queue_name).delete()
60
+ deleted, _ = Pause.objects.filter(queue_name=self.queue_name).delete()
61
+ queue_resumed.send(
62
+ sender=QueueLifecycle,
63
+ queue_name=self.queue_name,
64
+ changed=deleted > 0,
65
+ )
55
66
 
56
67
  def __str__(self) -> str:
57
68
  return self.queue_name
@@ -1,6 +1,12 @@
1
1
  import logging
2
2
 
3
3
  from steady_queue.processes.supervised import Supervised
4
+ from steady_queue.signals import (
5
+ _process_signal_context,
6
+ _send_process_signal,
7
+ process_started,
8
+ process_stopped,
9
+ )
4
10
 
5
11
  logger = logging.getLogger("steady_queue")
6
12
 
@@ -10,11 +16,22 @@ class Runnable(Supervised):
10
16
 
11
17
  def start(self):
12
18
  self.boot()
13
-
14
- if self.is_running_async:
15
- raise NotImplementedError
16
- else:
17
- self.run()
19
+ signal_context = _process_signal_context(self)
20
+ _send_process_signal(process_started, self, context=signal_context)
21
+
22
+ error = None
23
+ try:
24
+ if self.is_running_async:
25
+ raise NotImplementedError
26
+ else:
27
+ self.run()
28
+ except BaseException as exception:
29
+ error = exception
30
+ raise
31
+ finally:
32
+ _send_process_signal(
33
+ process_stopped, self, context=signal_context, error=error
34
+ )
18
35
 
19
36
  def stop(self):
20
37
  super().stop()
@@ -15,6 +15,14 @@ from steady_queue.processes.pidfiled import Pidfiled
15
15
  from steady_queue.processes.registrable import Registrable
16
16
  from steady_queue.processes.signals import Signals
17
17
  from steady_queue.processes.timer import wait_until
18
+ from steady_queue.signals import (
19
+ ProcessLifecycle,
20
+ _process_signal_context,
21
+ _send_process_signal,
22
+ process_restarted,
23
+ process_started,
24
+ process_stopped,
25
+ )
18
26
 
19
27
  logger = logging.getLogger("steady_queue")
20
28
 
@@ -40,19 +48,35 @@ class Supervisor(Maintenance, Signals, Pidfiled, Registrable, Interruptible, Bas
40
48
  super().__init__()
41
49
 
42
50
  def start(self) -> None:
43
- logger.info("starting supervisor with PID %(pid)d", {"pid": self.pid})
51
+ originating_pid = self.pid
52
+ logger.info("starting supervisor with PID %(pid)d", {"pid": originating_pid})
53
+ started = False
54
+ error = None
55
+ signal_context = None
44
56
  try:
45
- self.boot()
46
- # Fork only after resetting DB state (connections + psycopg pools).
47
- self.reset_database_connections()
48
- self.start_processes()
49
- self.launch_maintenance_task()
50
- except SystemExit:
51
- logger.info("supervisor interrupted during boot, shutting down")
52
- self.restore_default_signal_handlers()
53
- self.shutdown()
54
- return
55
- self.supervise()
57
+ try:
58
+ self.boot()
59
+ started = True
60
+ signal_context = _process_signal_context(self)
61
+ _send_process_signal(process_started, self, context=signal_context)
62
+ # Fork only after resetting DB state (connections + psycopg pools).
63
+ self.reset_database_connections()
64
+ self.start_processes()
65
+ self.launch_maintenance_task()
66
+ except SystemExit:
67
+ logger.info("supervisor interrupted during boot, shutting down")
68
+ self.restore_default_signal_handlers()
69
+ self.shutdown()
70
+ return
71
+ self.supervise()
72
+ except BaseException as exception:
73
+ error = exception
74
+ raise
75
+ finally:
76
+ if started and self.pid == originating_pid:
77
+ _send_process_signal(
78
+ process_stopped, self, context=signal_context, error=error
79
+ )
56
80
 
57
81
  def boot(self) -> None:
58
82
  super().boot()
@@ -80,7 +104,7 @@ class Supervisor(Maintenance, Signals, Pidfiled, Registrable, Interruptible, Bas
80
104
  logger.debug("supervisor finally block")
81
105
  self.shutdown()
82
106
 
83
- def start_process(self, process: Configuration.Process) -> None:
107
+ def start_process(self, process: Configuration.Process) -> int:
84
108
  logger.info("starting process %(process)s", {"process": process})
85
109
  instance = process.instantiate()
86
110
  instance.supervisor = self.process
@@ -98,6 +122,7 @@ class Supervisor(Maintenance, Signals, Pidfiled, Registrable, Interruptible, Bas
98
122
  self.reset_database_connections()
99
123
  self.configured_processes[pid] = process
100
124
  self.forks[pid] = instance
125
+ return pid
101
126
 
102
127
  def set_procline(self) -> None:
103
128
  pass
@@ -133,14 +158,14 @@ class Supervisor(Maintenance, Signals, Pidfiled, Registrable, Interruptible, Bas
133
158
  def reap_and_replace_terminated_forks(self) -> None:
134
159
  while True:
135
160
  try:
136
- pid, exitcode = os.waitpid(-1, os.WNOHANG)
161
+ pid, wait_status = os.waitpid(-1, os.WNOHANG)
137
162
  except ChildProcessError:
138
163
  break
139
164
  else:
140
165
  if not pid:
141
166
  break
142
167
 
143
- self.replace_fork(pid, exitcode)
168
+ self.replace_fork(pid, wait_status)
144
169
 
145
170
  def reap_terminated_forks(self) -> None:
146
171
  while True:
@@ -161,11 +186,23 @@ class Supervisor(Maintenance, Signals, Pidfiled, Registrable, Interruptible, Bas
161
186
 
162
187
  self.configured_processes.pop(pid, None)
163
188
 
164
- def replace_fork(self, pid: int, exitcode: int) -> None:
189
+ def replace_fork(self, pid: int, wait_status: int) -> None:
190
+ exitcode = os.waitstatus_to_exitcode(wait_status)
165
191
  logger.info("replacing fork %s due to exit code %s", pid, exitcode)
166
192
  if terminated_fork := self.forks.pop(pid, None):
167
- self.handle_claimed_jobs_by(terminated_fork, exitcode)
168
- self.start_process(self.configured_processes.pop(pid))
193
+ self.handle_claimed_jobs_by(terminated_fork, wait_status)
194
+ replacement_pid = self.start_process(self.configured_processes.pop(pid))
195
+ process_restarted.send(
196
+ sender=ProcessLifecycle,
197
+ process_kind=terminated_fork.kind,
198
+ process_name=terminated_fork.name,
199
+ pid=pid,
200
+ hostname=terminated_fork.hostname,
201
+ metadata=terminated_fork.metadata,
202
+ exitcode=exitcode,
203
+ replacement_pid=replacement_pid,
204
+ supervisor_pid=self.pid,
205
+ )
169
206
 
170
207
  def handle_claimed_jobs_by(self, terminated_fork: Base, exitcode: int) -> None:
171
208
  if not self.process:
@@ -0,0 +1,37 @@
1
+ from django.dispatch import Signal
2
+
3
+
4
+ class ProcessLifecycle:
5
+ """Public sender for process lifecycle signals."""
6
+
7
+
8
+ class QueueLifecycle:
9
+ """Public sender for queue lifecycle signals."""
10
+
11
+
12
+ # Process lifecycle signals carry stable data suitable for logs and metrics.
13
+ process_started = Signal()
14
+ process_stopped = Signal()
15
+ process_restarted = Signal()
16
+
17
+ # Queue control signals. Receivers get ``queue_name`` and ``changed``.
18
+ queue_paused = Signal()
19
+ queue_resumed = Signal()
20
+
21
+
22
+ def _process_signal_context(process) -> dict:
23
+ return {
24
+ "process_kind": process.kind,
25
+ "process_name": process.name,
26
+ "pid": process.pid,
27
+ "hostname": process.hostname,
28
+ "metadata": process.metadata,
29
+ }
30
+
31
+
32
+ def _send_process_signal(signal: Signal, process, *, context=None, **kwargs) -> None:
33
+ signal.send(
34
+ sender=ProcessLifecycle,
35
+ **(context or _process_signal_context(process)),
36
+ **kwargs,
37
+ )
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: steady-queue
3
- Version: 0.2.1
3
+ Version: 0.3.0
4
4
  Summary: An asynchronous task library for Django that supports delayed tasks, concurrency controls, recurring tasks, pausing queues, numeric priorities per task, priorities by queue order and bulk enqueueing.
5
5
  Author-email: Elias Hernandis <elias@hernandis.me>
6
6
  License-Expression: MIT
@@ -535,8 +535,17 @@ instance that is handling the task, as well as the `task_result` (a
535
535
  `django.tasks.TaskResult` instance) with information on how the task was called
536
536
  and its status.
537
537
 
538
- Unlike Solid Queue, steady queue doesn't yet emit signals related to the
539
- lifecycle of its processes.
538
+ Steady Queue also emits operational signals from ``steady_queue.signals``:
539
+
540
+ - `process_started` and `process_stopped` when a supervisor or child process
541
+ enters or leaves its run loop.
542
+ - `process_restarted` when the supervisor replaces a terminated child.
543
+ - `queue_paused` and `queue_resumed` after queue control actions.
544
+
545
+ Process signals include process identity and metadata. Queue signals include a
546
+ `changed` flag so receivers can distinguish state transitions from idempotent
547
+ actions. See the [configuration documentation](https://steady-queue.readthedocs.io/en/latest/configuration.html#signals)
548
+ for the complete payloads.
540
549
 
541
550
  ## Logging
542
551
 
@@ -824,8 +833,9 @@ there are a few differences which we outline below.
824
833
  will ever be supported, but we've kept the database column for compatibility.
825
834
  - Steady Queue worker processes do not set the process name (or procline)
826
835
  because doing so requires introducing an external dependency.
827
- - Steady Queue does not expose rich instrumentation like Solid Queue does due to
828
- the lack of a framework-native equivalent to `ActiveSupport::Notifications`.
836
+ - Steady Queue exposes Django signals for task and operational lifecycle events,
837
+ but does not yet provide the full timed instrumentation event set emitted by
838
+ Solid Queue through `ActiveSupport::Notifications`.
829
839
  - **Priority ordering:** Steady Queue follows Django's convention where larger
830
840
  numbers indicate higher priority (e.g., a task with priority 10 runs before
831
841
  priority 0), whereas Solid Queue uses the inverse (smaller numbers = higher
@@ -14,6 +14,7 @@ steady_queue/db_router.py
14
14
  steady_queue/queue_selector.py
15
15
  steady_queue/recurring_schedule.py
16
16
  steady_queue/recurring_task.py
17
+ steady_queue/signals.py
17
18
  steady_queue/task.py
18
19
  steady_queue.egg-info/PKG-INFO
19
20
  steady_queue.egg-info/SOURCES.txt
@@ -67,6 +68,7 @@ steady_queue/processes/supervised.py
67
68
  steady_queue/processes/supervisor.py
68
69
  steady_queue/processes/timer.py
69
70
  steady_queue/processes/worker.py
71
+ tests/test_app_executor.py
70
72
  tests/test_arguments.py
71
73
  tests/test_backend.py
72
74
  tests/test_concurrency_controls.py
@@ -75,6 +77,7 @@ tests/test_db_router.py
75
77
  tests/test_executions.py
76
78
  tests/test_fork_safety.py
77
79
  tests/test_job.py
80
+ tests/test_observability_signals.py
78
81
  tests/test_pidfile.py
79
82
  tests/test_process_lifecycle.py
80
83
  tests/test_queue_selector.py
@@ -0,0 +1,84 @@
1
+ from concurrent.futures import ThreadPoolExecutor
2
+ from unittest import skipUnless
3
+
4
+ from django.db import connections, transaction
5
+ from django.test import TransactionTestCase
6
+
7
+ from steady_queue.app_executor import AppExecutor
8
+ from tests.dummy.models import Dummy
9
+
10
+
11
+ @skipUnless(
12
+ connections["default"].vendor == "postgresql",
13
+ "Requires PostgreSQL application connection pooling",
14
+ )
15
+ class AppExecutorConnectionsTest(TransactionTestCase):
16
+ databases = {"default", "queue"}
17
+
18
+ def setUp(self):
19
+ self.application = connections["default"]
20
+ self.application.close()
21
+ self.application.close_pool()
22
+ self.original_options = self.application.settings_dict["OPTIONS"]
23
+ self.application.settings_dict["OPTIONS"] = {
24
+ **self.original_options,
25
+ "pool": {"min_size": 0, "max_size": 1, "timeout": 0.25},
26
+ }
27
+ self.addCleanup(self.restore_application_pool)
28
+
29
+ def restore_application_pool(self):
30
+ self.application.close()
31
+ self.application.close_pool()
32
+ self.application.settings_dict["OPTIONS"] = self.original_options
33
+
34
+ @staticmethod
35
+ def perform_task(*, fail=False):
36
+ with AppExecutor.wrap_in_app_executor():
37
+ Dummy.objects.count()
38
+ if fail:
39
+ raise ValueError("Task failed")
40
+ return (
41
+ connections["default"].connection is None,
42
+ connections["queue"].connection is None,
43
+ )
44
+
45
+ def assert_pool_reusable_by_another_worker(self, *, fail=False):
46
+ # Keep both threads alive: ending a thread must not stand in for the
47
+ # per-task cleanup that long-lived worker threads need.
48
+ workers = [ThreadPoolExecutor(max_workers=1) for _ in range(2)]
49
+ try:
50
+ first = workers[0].submit(self.perform_task, fail=fail)
51
+ if fail:
52
+ with self.assertRaisesMessage(ValueError, "Task failed"):
53
+ first.result(timeout=5)
54
+ else:
55
+ first.result(timeout=5)
56
+
57
+ # A leaked application connection leaves this separate thread
58
+ # unable to borrow the only connection in the application pool.
59
+ state = workers[1].submit(self.perform_task).result(timeout=5)
60
+ self.assertEqual(state, (True, True))
61
+ self.assertEqual(self.application.pool.get_stats()["pool_available"], 1)
62
+ finally:
63
+ for worker in workers:
64
+ worker.submit(connections.close_all).result(timeout=5)
65
+ worker.shutdown()
66
+
67
+ def test_success_returns_application_connection_for_other_workers(self):
68
+ self.assert_pool_reusable_by_another_worker()
69
+
70
+ def test_failure_returns_application_connection_for_other_workers(self):
71
+ self.assert_pool_reusable_by_another_worker(fail=True)
72
+
73
+ def test_main_thread_preserves_callers_connections_and_transaction(self):
74
+ connections["queue"].ensure_connection()
75
+ queue_connection = connections["queue"].connection
76
+ with transaction.atomic():
77
+ Dummy.objects.count()
78
+ application_connection = self.application.connection
79
+ with AppExecutor.wrap_in_app_executor():
80
+ Dummy.objects.count()
81
+ self.assertIs(self.application.connection, application_connection)
82
+ self.assertIs(connections["queue"].connection, queue_connection)
83
+ self.assertTrue(self.application.in_atomic_block)
84
+ Dummy.objects.count()
@@ -0,0 +1,203 @@
1
+ import os
2
+ from unittest.mock import MagicMock, patch
3
+
4
+ from django.test import SimpleTestCase, TestCase
5
+
6
+ from steady_queue.configuration import Configuration
7
+ from steady_queue.models import Job, Queue
8
+ from steady_queue.processes.base import Base
9
+ from steady_queue.processes.runnable import Runnable
10
+ from steady_queue.processes.supervisor import Supervisor
11
+ from steady_queue.signals import (
12
+ ProcessLifecycle,
13
+ QueueLifecycle,
14
+ process_restarted,
15
+ process_started,
16
+ process_stopped,
17
+ queue_paused,
18
+ queue_resumed,
19
+ )
20
+ from tests.dummy.tasks import dummy_task
21
+
22
+
23
+ class FakeRunnable(Runnable, Base):
24
+ mode = "inline"
25
+
26
+ def run(self):
27
+ pass
28
+
29
+
30
+ class ProcessLifecycleSignalTests(SimpleTestCase):
31
+ def test_runnable_emits_started_and_stopped_with_operational_context(self):
32
+ process = FakeRunnable()
33
+ started = MagicMock()
34
+ stopped = MagicMock()
35
+
36
+ process_started.connect(started)
37
+ process_stopped.connect(stopped)
38
+ self.addCleanup(process_started.disconnect, started)
39
+ self.addCleanup(process_stopped.disconnect, stopped)
40
+
41
+ process.start()
42
+
43
+ started.assert_called_once_with(
44
+ signal=process_started,
45
+ sender=ProcessLifecycle,
46
+ process_kind="fakerunnable",
47
+ process_name=process.name,
48
+ pid=process.pid,
49
+ hostname=process.hostname,
50
+ metadata={},
51
+ )
52
+ stopped.assert_called_once_with(
53
+ signal=process_stopped,
54
+ sender=ProcessLifecycle,
55
+ process_kind="fakerunnable",
56
+ process_name=process.name,
57
+ pid=process.pid,
58
+ hostname=process.hostname,
59
+ metadata={},
60
+ error=None,
61
+ )
62
+
63
+ def test_stopped_signal_includes_run_error(self):
64
+ process = FakeRunnable()
65
+ error = RuntimeError("broken")
66
+ stopped = MagicMock()
67
+ process.run = MagicMock(side_effect=error)
68
+
69
+ process_stopped.connect(stopped)
70
+ self.addCleanup(process_stopped.disconnect, stopped)
71
+
72
+ with self.assertRaises(RuntimeError):
73
+ process.start()
74
+
75
+ self.assertIs(stopped.call_args.kwargs["error"], error)
76
+
77
+ def test_forked_child_does_not_emit_supervisor_stopped(self):
78
+ supervisor = Supervisor(Configuration(Configuration.Options()))
79
+ originating_pid = os.getpid()
80
+ read_fd, write_fd = os.pipe()
81
+ child_pids = []
82
+
83
+ def record_stop(**kwargs):
84
+ message = f"{os.getpid()}:{kwargs['pid']}\n".encode()
85
+ os.write(write_fd, message)
86
+
87
+ def fork_child():
88
+ pid = os.fork()
89
+ if pid == 0:
90
+ os.close(read_fd)
91
+ raise SystemExit
92
+ child_pids.append(pid)
93
+
94
+ process_stopped.connect(record_stop)
95
+ try:
96
+ with (
97
+ patch.object(supervisor, "boot"),
98
+ patch.object(supervisor, "reset_database_connections"),
99
+ patch.object(supervisor, "start_processes", side_effect=fork_child),
100
+ patch.object(supervisor, "launch_maintenance_task"),
101
+ patch.object(supervisor, "restore_default_signal_handlers"),
102
+ patch.object(supervisor, "shutdown"),
103
+ patch.object(supervisor, "supervise"),
104
+ ):
105
+ supervisor.start()
106
+
107
+ if os.getpid() != originating_pid:
108
+ os.close(write_fd)
109
+ os._exit(0)
110
+
111
+ os.waitpid(child_pids[0], 0)
112
+ os.close(write_fd)
113
+ write_fd = None
114
+ records = os.read(read_fd, 4096).decode().splitlines()
115
+ finally:
116
+ process_stopped.disconnect(record_stop)
117
+ os.close(read_fd)
118
+ if write_fd is not None:
119
+ os.close(write_fd)
120
+
121
+ self.assertEqual(records, [f"{originating_pid}:{originating_pid}"])
122
+
123
+ def test_supervisor_emits_restart_after_replacing_child(self):
124
+ supervisor = Supervisor(Configuration(Configuration.Options()))
125
+ terminated = FakeRunnable()
126
+ configured = MagicMock()
127
+ supervisor.forks[123] = terminated
128
+ supervisor.configured_processes[123] = configured
129
+ restarted = MagicMock()
130
+
131
+ process_restarted.connect(restarted)
132
+ self.addCleanup(process_restarted.disconnect, restarted)
133
+
134
+ with (
135
+ patch.object(supervisor, "handle_claimed_jobs_by") as recover_jobs,
136
+ patch.object(supervisor, "start_process", return_value=456),
137
+ ):
138
+ supervisor.replace_fork(123, 7 << 8)
139
+
140
+ recover_jobs.assert_called_once_with(terminated, 7 << 8)
141
+ restarted.assert_called_once_with(
142
+ signal=process_restarted,
143
+ sender=ProcessLifecycle,
144
+ process_kind="fakerunnable",
145
+ process_name=terminated.name,
146
+ pid=123,
147
+ hostname=terminated.hostname,
148
+ metadata={},
149
+ exitcode=7,
150
+ replacement_pid=456,
151
+ supervisor_pid=supervisor.pid,
152
+ )
153
+
154
+
155
+ class QueueLifecycleSignalTests(TestCase):
156
+ databases = {"default", "queue"}
157
+
158
+ def setUp(self):
159
+ Job.objects.enqueue(dummy_task, [], {})
160
+ self.queue = Queue.objects.get(queue_name="default")
161
+
162
+ def test_pause_and_resume_report_real_and_idempotent_actions(self):
163
+ paused = MagicMock()
164
+ resumed = MagicMock()
165
+ queue_paused.connect(paused)
166
+ queue_resumed.connect(resumed)
167
+ self.addCleanup(queue_paused.disconnect, paused)
168
+ self.addCleanup(queue_resumed.disconnect, resumed)
169
+
170
+ self.queue.pause()
171
+ self.queue.pause()
172
+ self.queue.resume()
173
+ self.queue.resume()
174
+
175
+ self.assertEqual(
176
+ [call.kwargs["changed"] for call in paused.call_args_list], [True, False]
177
+ )
178
+ self.assertEqual(
179
+ [call.kwargs["changed"] for call in resumed.call_args_list], [True, False]
180
+ )
181
+ self.assertTrue(
182
+ all(
183
+ call.kwargs["queue_name"] == "default" for call in paused.call_args_list
184
+ )
185
+ )
186
+ self.assertTrue(
187
+ all("queue" not in call.kwargs for call in paused.call_args_list)
188
+ )
189
+ self.assertTrue(
190
+ all("queue" not in call.kwargs for call in resumed.call_args_list)
191
+ )
192
+ self.assertTrue(
193
+ all(
194
+ call.kwargs["sender"] is QueueLifecycle
195
+ for call in paused.call_args_list
196
+ )
197
+ )
198
+ self.assertTrue(
199
+ all(
200
+ call.kwargs["sender"] is QueueLifecycle
201
+ for call in resumed.call_args_list
202
+ )
203
+ )
File without changes
File without changes