steady-queue 0.2.1__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {steady_queue-0.2.1/steady_queue.egg-info → steady_queue-0.3.0}/PKG-INFO +15 -5
- {steady_queue-0.2.1 → steady_queue-0.3.0}/README.md +14 -4
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/__init__.py +1 -1
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/app_executor.py +4 -5
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/queue.py +13 -2
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/processes/runnable.py +22 -5
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/processes/supervisor.py +55 -18
- steady_queue-0.3.0/steady_queue/signals.py +37 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0/steady_queue.egg-info}/PKG-INFO +15 -5
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue.egg-info/SOURCES.txt +3 -0
- steady_queue-0.3.0/tests/test_app_executor.py +84 -0
- steady_queue-0.3.0/tests/test_observability_signals.py +203 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/LICENSE +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/pyproject.toml +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/setup.cfg +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/admin.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/apps.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/arguments.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/backend.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/collections.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/concurrency.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/configuration.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/db_router.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/management/__init__.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/management/commands/__init__.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/management/commands/steady_queue.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/migrations/0001_initial.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/migrations/__init__.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/__init__.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/base.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/blocked_execution.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/claimed_execution.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/clearable.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/concurrency_controls.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/dispatching.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/executable.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/execution.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/executor.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/failed_execution.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/job.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/pause.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/process.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/prunable.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/ready_execution.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/recurring_execution.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/recurring_task.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/retryable.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/schedulable.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/scheduled_execution.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/models/semaphore.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/processes/__init__.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/processes/base.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/processes/concurrent.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/processes/dispatcher.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/processes/errors.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/processes/interruptible.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/processes/maintenance.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/processes/pidfile.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/processes/pidfiled.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/processes/poller.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/processes/pool.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/processes/registrable.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/processes/scheduler.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/processes/signals.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/processes/supervised.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/processes/timer.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/processes/worker.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/queue_selector.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/recurring_schedule.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/recurring_task.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue/task.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue.egg-info/dependency_links.txt +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue.egg-info/requires.txt +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/steady_queue.egg-info/top_level.txt +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/tests/test_arguments.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/tests/test_backend.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/tests/test_concurrency_controls.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/tests/test_configuration.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/tests/test_db_router.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/tests/test_executions.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/tests/test_fork_safety.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/tests/test_job.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/tests/test_pidfile.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/tests/test_process_lifecycle.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/tests/test_queue_selector.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/tests/test_recurring_execution.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/tests/test_recurring_task.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/tests/test_semaphore.py +0 -0
- {steady_queue-0.2.1 → steady_queue-0.3.0}/tests/test_task.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: steady-queue
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: An asynchronous task library for Django that supports delayed tasks, concurrency controls, recurring tasks, pausing queues, numeric priorities per task, priorities by queue order and bulk enqueueing.
|
|
5
5
|
Author-email: Elias Hernandis <elias@hernandis.me>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -535,8 +535,17 @@ instance that is handling the task, as well as the `task_result` (a
|
|
|
535
535
|
`django.tasks.TaskResult` instance) with information on how the task was called
|
|
536
536
|
and its status.
|
|
537
537
|
|
|
538
|
-
|
|
539
|
-
|
|
538
|
+
Steady Queue also emits operational signals from ``steady_queue.signals``:
|
|
539
|
+
|
|
540
|
+
- `process_started` and `process_stopped` when a supervisor or child process
|
|
541
|
+
enters or leaves its run loop.
|
|
542
|
+
- `process_restarted` when the supervisor replaces a terminated child.
|
|
543
|
+
- `queue_paused` and `queue_resumed` after queue control actions.
|
|
544
|
+
|
|
545
|
+
Process signals include process identity and metadata. Queue signals include a
|
|
546
|
+
`changed` flag so receivers can distinguish state transitions from idempotent
|
|
547
|
+
actions. See the [configuration documentation](https://steady-queue.readthedocs.io/en/latest/configuration.html#signals)
|
|
548
|
+
for the complete payloads.
|
|
540
549
|
|
|
541
550
|
## Logging
|
|
542
551
|
|
|
@@ -824,8 +833,9 @@ there are a few differences which we outline below.
|
|
|
824
833
|
will ever be supported, but we've kept the database column for compatibility.
|
|
825
834
|
- Steady Queue worker processes do not set the process name (or procline)
|
|
826
835
|
because doing so requires introducing an external dependency.
|
|
827
|
-
- Steady Queue
|
|
828
|
-
the
|
|
836
|
+
- Steady Queue exposes Django signals for task and operational lifecycle events,
|
|
837
|
+
but does not yet provide the full timed instrumentation event set emitted by
|
|
838
|
+
Solid Queue through `ActiveSupport::Notifications`.
|
|
829
839
|
- **Priority ordering:** Steady Queue follows Django's convention where larger
|
|
830
840
|
numbers indicate higher priority (e.g., a task with priority 10 runs before
|
|
831
841
|
priority 0), whereas Solid Queue uses the inverse (smaller numbers = higher
|
|
@@ -506,8 +506,17 @@ instance that is handling the task, as well as the `task_result` (a
|
|
|
506
506
|
`django.tasks.TaskResult` instance) with information on how the task was called
|
|
507
507
|
and its status.
|
|
508
508
|
|
|
509
|
-
|
|
510
|
-
|
|
509
|
+
Steady Queue also emits operational signals from ``steady_queue.signals``:
|
|
510
|
+
|
|
511
|
+
- `process_started` and `process_stopped` when a supervisor or child process
|
|
512
|
+
enters or leaves its run loop.
|
|
513
|
+
- `process_restarted` when the supervisor replaces a terminated child.
|
|
514
|
+
- `queue_paused` and `queue_resumed` after queue control actions.
|
|
515
|
+
|
|
516
|
+
Process signals include process identity and metadata. Queue signals include a
|
|
517
|
+
`changed` flag so receivers can distinguish state transitions from idempotent
|
|
518
|
+
actions. See the [configuration documentation](https://steady-queue.readthedocs.io/en/latest/configuration.html#signals)
|
|
519
|
+
for the complete payloads.
|
|
511
520
|
|
|
512
521
|
## Logging
|
|
513
522
|
|
|
@@ -795,8 +804,9 @@ there are a few differences which we outline below.
|
|
|
795
804
|
will ever be supported, but we've kept the database column for compatibility.
|
|
796
805
|
- Steady Queue worker processes do not set the process name (or procline)
|
|
797
806
|
because doing so requires introducing an external dependency.
|
|
798
|
-
- Steady Queue
|
|
799
|
-
the
|
|
807
|
+
- Steady Queue exposes Django signals for task and operational lifecycle events,
|
|
808
|
+
but does not yet provide the full timed instrumentation event set emitted by
|
|
809
|
+
Solid Queue through `ActiveSupport::Notifications`.
|
|
800
810
|
- **Priority ordering:** Steady Queue follows Django's convention where larger
|
|
801
811
|
numbers indicate higher priority (e.g., a task with priority 10 runs before
|
|
802
812
|
priority 0), whereas Solid Queue uses the inverse (smaller numbers = higher
|
|
@@ -40,9 +40,8 @@ class AppExecutor:
|
|
|
40
40
|
connection.close()
|
|
41
41
|
raise
|
|
42
42
|
finally:
|
|
43
|
-
# Django automatically manages connection lifecycle,
|
|
44
|
-
# but we can explicitly close if needed for long-running processes
|
|
45
43
|
if threading.current_thread() != threading.main_thread():
|
|
46
|
-
#
|
|
47
|
-
#
|
|
48
|
-
|
|
44
|
+
# Worker threads are reused without Django's request cleanup.
|
|
45
|
+
# Return connections for every initialized alias, including the
|
|
46
|
+
# application databases used by the task, to their pools.
|
|
47
|
+
connections.close_all()
|
|
@@ -2,6 +2,7 @@ from django.db import models
|
|
|
2
2
|
|
|
3
3
|
from steady_queue.models.pause import Pause
|
|
4
4
|
from steady_queue.models.ready_execution import ReadyExecution
|
|
5
|
+
from steady_queue.signals import QueueLifecycle, queue_paused, queue_resumed
|
|
5
6
|
|
|
6
7
|
|
|
7
8
|
class QueueQuerySet(models.QuerySet):
|
|
@@ -48,10 +49,20 @@ class Queue(models.Model):
|
|
|
48
49
|
return not self.is_paused
|
|
49
50
|
|
|
50
51
|
def pause(self) -> None:
|
|
51
|
-
Pause.objects.get_or_create(queue_name=self.queue_name)
|
|
52
|
+
_, changed = Pause.objects.get_or_create(queue_name=self.queue_name)
|
|
53
|
+
queue_paused.send(
|
|
54
|
+
sender=QueueLifecycle,
|
|
55
|
+
queue_name=self.queue_name,
|
|
56
|
+
changed=changed,
|
|
57
|
+
)
|
|
52
58
|
|
|
53
59
|
def resume(self) -> None:
|
|
54
|
-
Pause.objects.filter(queue_name=self.queue_name).delete()
|
|
60
|
+
deleted, _ = Pause.objects.filter(queue_name=self.queue_name).delete()
|
|
61
|
+
queue_resumed.send(
|
|
62
|
+
sender=QueueLifecycle,
|
|
63
|
+
queue_name=self.queue_name,
|
|
64
|
+
changed=deleted > 0,
|
|
65
|
+
)
|
|
55
66
|
|
|
56
67
|
def __str__(self) -> str:
|
|
57
68
|
return self.queue_name
|
|
@@ -1,6 +1,12 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
|
|
3
3
|
from steady_queue.processes.supervised import Supervised
|
|
4
|
+
from steady_queue.signals import (
|
|
5
|
+
_process_signal_context,
|
|
6
|
+
_send_process_signal,
|
|
7
|
+
process_started,
|
|
8
|
+
process_stopped,
|
|
9
|
+
)
|
|
4
10
|
|
|
5
11
|
logger = logging.getLogger("steady_queue")
|
|
6
12
|
|
|
@@ -10,11 +16,22 @@ class Runnable(Supervised):
|
|
|
10
16
|
|
|
11
17
|
def start(self):
|
|
12
18
|
self.boot()
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
19
|
+
signal_context = _process_signal_context(self)
|
|
20
|
+
_send_process_signal(process_started, self, context=signal_context)
|
|
21
|
+
|
|
22
|
+
error = None
|
|
23
|
+
try:
|
|
24
|
+
if self.is_running_async:
|
|
25
|
+
raise NotImplementedError
|
|
26
|
+
else:
|
|
27
|
+
self.run()
|
|
28
|
+
except BaseException as exception:
|
|
29
|
+
error = exception
|
|
30
|
+
raise
|
|
31
|
+
finally:
|
|
32
|
+
_send_process_signal(
|
|
33
|
+
process_stopped, self, context=signal_context, error=error
|
|
34
|
+
)
|
|
18
35
|
|
|
19
36
|
def stop(self):
|
|
20
37
|
super().stop()
|
|
@@ -15,6 +15,14 @@ from steady_queue.processes.pidfiled import Pidfiled
|
|
|
15
15
|
from steady_queue.processes.registrable import Registrable
|
|
16
16
|
from steady_queue.processes.signals import Signals
|
|
17
17
|
from steady_queue.processes.timer import wait_until
|
|
18
|
+
from steady_queue.signals import (
|
|
19
|
+
ProcessLifecycle,
|
|
20
|
+
_process_signal_context,
|
|
21
|
+
_send_process_signal,
|
|
22
|
+
process_restarted,
|
|
23
|
+
process_started,
|
|
24
|
+
process_stopped,
|
|
25
|
+
)
|
|
18
26
|
|
|
19
27
|
logger = logging.getLogger("steady_queue")
|
|
20
28
|
|
|
@@ -40,19 +48,35 @@ class Supervisor(Maintenance, Signals, Pidfiled, Registrable, Interruptible, Bas
|
|
|
40
48
|
super().__init__()
|
|
41
49
|
|
|
42
50
|
def start(self) -> None:
|
|
43
|
-
|
|
51
|
+
originating_pid = self.pid
|
|
52
|
+
logger.info("starting supervisor with PID %(pid)d", {"pid": originating_pid})
|
|
53
|
+
started = False
|
|
54
|
+
error = None
|
|
55
|
+
signal_context = None
|
|
44
56
|
try:
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
57
|
+
try:
|
|
58
|
+
self.boot()
|
|
59
|
+
started = True
|
|
60
|
+
signal_context = _process_signal_context(self)
|
|
61
|
+
_send_process_signal(process_started, self, context=signal_context)
|
|
62
|
+
# Fork only after resetting DB state (connections + psycopg pools).
|
|
63
|
+
self.reset_database_connections()
|
|
64
|
+
self.start_processes()
|
|
65
|
+
self.launch_maintenance_task()
|
|
66
|
+
except SystemExit:
|
|
67
|
+
logger.info("supervisor interrupted during boot, shutting down")
|
|
68
|
+
self.restore_default_signal_handlers()
|
|
69
|
+
self.shutdown()
|
|
70
|
+
return
|
|
71
|
+
self.supervise()
|
|
72
|
+
except BaseException as exception:
|
|
73
|
+
error = exception
|
|
74
|
+
raise
|
|
75
|
+
finally:
|
|
76
|
+
if started and self.pid == originating_pid:
|
|
77
|
+
_send_process_signal(
|
|
78
|
+
process_stopped, self, context=signal_context, error=error
|
|
79
|
+
)
|
|
56
80
|
|
|
57
81
|
def boot(self) -> None:
|
|
58
82
|
super().boot()
|
|
@@ -80,7 +104,7 @@ class Supervisor(Maintenance, Signals, Pidfiled, Registrable, Interruptible, Bas
|
|
|
80
104
|
logger.debug("supervisor finally block")
|
|
81
105
|
self.shutdown()
|
|
82
106
|
|
|
83
|
-
def start_process(self, process: Configuration.Process) ->
|
|
107
|
+
def start_process(self, process: Configuration.Process) -> int:
|
|
84
108
|
logger.info("starting process %(process)s", {"process": process})
|
|
85
109
|
instance = process.instantiate()
|
|
86
110
|
instance.supervisor = self.process
|
|
@@ -98,6 +122,7 @@ class Supervisor(Maintenance, Signals, Pidfiled, Registrable, Interruptible, Bas
|
|
|
98
122
|
self.reset_database_connections()
|
|
99
123
|
self.configured_processes[pid] = process
|
|
100
124
|
self.forks[pid] = instance
|
|
125
|
+
return pid
|
|
101
126
|
|
|
102
127
|
def set_procline(self) -> None:
|
|
103
128
|
pass
|
|
@@ -133,14 +158,14 @@ class Supervisor(Maintenance, Signals, Pidfiled, Registrable, Interruptible, Bas
|
|
|
133
158
|
def reap_and_replace_terminated_forks(self) -> None:
|
|
134
159
|
while True:
|
|
135
160
|
try:
|
|
136
|
-
pid,
|
|
161
|
+
pid, wait_status = os.waitpid(-1, os.WNOHANG)
|
|
137
162
|
except ChildProcessError:
|
|
138
163
|
break
|
|
139
164
|
else:
|
|
140
165
|
if not pid:
|
|
141
166
|
break
|
|
142
167
|
|
|
143
|
-
self.replace_fork(pid,
|
|
168
|
+
self.replace_fork(pid, wait_status)
|
|
144
169
|
|
|
145
170
|
def reap_terminated_forks(self) -> None:
|
|
146
171
|
while True:
|
|
@@ -161,11 +186,23 @@ class Supervisor(Maintenance, Signals, Pidfiled, Registrable, Interruptible, Bas
|
|
|
161
186
|
|
|
162
187
|
self.configured_processes.pop(pid, None)
|
|
163
188
|
|
|
164
|
-
def replace_fork(self, pid: int,
|
|
189
|
+
def replace_fork(self, pid: int, wait_status: int) -> None:
|
|
190
|
+
exitcode = os.waitstatus_to_exitcode(wait_status)
|
|
165
191
|
logger.info("replacing fork %s due to exit code %s", pid, exitcode)
|
|
166
192
|
if terminated_fork := self.forks.pop(pid, None):
|
|
167
|
-
self.handle_claimed_jobs_by(terminated_fork,
|
|
168
|
-
self.start_process(self.configured_processes.pop(pid))
|
|
193
|
+
self.handle_claimed_jobs_by(terminated_fork, wait_status)
|
|
194
|
+
replacement_pid = self.start_process(self.configured_processes.pop(pid))
|
|
195
|
+
process_restarted.send(
|
|
196
|
+
sender=ProcessLifecycle,
|
|
197
|
+
process_kind=terminated_fork.kind,
|
|
198
|
+
process_name=terminated_fork.name,
|
|
199
|
+
pid=pid,
|
|
200
|
+
hostname=terminated_fork.hostname,
|
|
201
|
+
metadata=terminated_fork.metadata,
|
|
202
|
+
exitcode=exitcode,
|
|
203
|
+
replacement_pid=replacement_pid,
|
|
204
|
+
supervisor_pid=self.pid,
|
|
205
|
+
)
|
|
169
206
|
|
|
170
207
|
def handle_claimed_jobs_by(self, terminated_fork: Base, exitcode: int) -> None:
|
|
171
208
|
if not self.process:
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
from django.dispatch import Signal
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class ProcessLifecycle:
|
|
5
|
+
"""Public sender for process lifecycle signals."""
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class QueueLifecycle:
|
|
9
|
+
"""Public sender for queue lifecycle signals."""
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
# Process lifecycle signals carry stable data suitable for logs and metrics.
|
|
13
|
+
process_started = Signal()
|
|
14
|
+
process_stopped = Signal()
|
|
15
|
+
process_restarted = Signal()
|
|
16
|
+
|
|
17
|
+
# Queue control signals. Receivers get ``queue_name`` and ``changed``.
|
|
18
|
+
queue_paused = Signal()
|
|
19
|
+
queue_resumed = Signal()
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _process_signal_context(process) -> dict:
|
|
23
|
+
return {
|
|
24
|
+
"process_kind": process.kind,
|
|
25
|
+
"process_name": process.name,
|
|
26
|
+
"pid": process.pid,
|
|
27
|
+
"hostname": process.hostname,
|
|
28
|
+
"metadata": process.metadata,
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _send_process_signal(signal: Signal, process, *, context=None, **kwargs) -> None:
|
|
33
|
+
signal.send(
|
|
34
|
+
sender=ProcessLifecycle,
|
|
35
|
+
**(context or _process_signal_context(process)),
|
|
36
|
+
**kwargs,
|
|
37
|
+
)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: steady-queue
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: An asynchronous task library for Django that supports delayed tasks, concurrency controls, recurring tasks, pausing queues, numeric priorities per task, priorities by queue order and bulk enqueueing.
|
|
5
5
|
Author-email: Elias Hernandis <elias@hernandis.me>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -535,8 +535,17 @@ instance that is handling the task, as well as the `task_result` (a
|
|
|
535
535
|
`django.tasks.TaskResult` instance) with information on how the task was called
|
|
536
536
|
and its status.
|
|
537
537
|
|
|
538
|
-
|
|
539
|
-
|
|
538
|
+
Steady Queue also emits operational signals from ``steady_queue.signals``:
|
|
539
|
+
|
|
540
|
+
- `process_started` and `process_stopped` when a supervisor or child process
|
|
541
|
+
enters or leaves its run loop.
|
|
542
|
+
- `process_restarted` when the supervisor replaces a terminated child.
|
|
543
|
+
- `queue_paused` and `queue_resumed` after queue control actions.
|
|
544
|
+
|
|
545
|
+
Process signals include process identity and metadata. Queue signals include a
|
|
546
|
+
`changed` flag so receivers can distinguish state transitions from idempotent
|
|
547
|
+
actions. See the [configuration documentation](https://steady-queue.readthedocs.io/en/latest/configuration.html#signals)
|
|
548
|
+
for the complete payloads.
|
|
540
549
|
|
|
541
550
|
## Logging
|
|
542
551
|
|
|
@@ -824,8 +833,9 @@ there are a few differences which we outline below.
|
|
|
824
833
|
will ever be supported, but we've kept the database column for compatibility.
|
|
825
834
|
- Steady Queue worker processes do not set the process name (or procline)
|
|
826
835
|
because doing so requires introducing an external dependency.
|
|
827
|
-
- Steady Queue
|
|
828
|
-
the
|
|
836
|
+
- Steady Queue exposes Django signals for task and operational lifecycle events,
|
|
837
|
+
but does not yet provide the full timed instrumentation event set emitted by
|
|
838
|
+
Solid Queue through `ActiveSupport::Notifications`.
|
|
829
839
|
- **Priority ordering:** Steady Queue follows Django's convention where larger
|
|
830
840
|
numbers indicate higher priority (e.g., a task with priority 10 runs before
|
|
831
841
|
priority 0), whereas Solid Queue uses the inverse (smaller numbers = higher
|
|
@@ -14,6 +14,7 @@ steady_queue/db_router.py
|
|
|
14
14
|
steady_queue/queue_selector.py
|
|
15
15
|
steady_queue/recurring_schedule.py
|
|
16
16
|
steady_queue/recurring_task.py
|
|
17
|
+
steady_queue/signals.py
|
|
17
18
|
steady_queue/task.py
|
|
18
19
|
steady_queue.egg-info/PKG-INFO
|
|
19
20
|
steady_queue.egg-info/SOURCES.txt
|
|
@@ -67,6 +68,7 @@ steady_queue/processes/supervised.py
|
|
|
67
68
|
steady_queue/processes/supervisor.py
|
|
68
69
|
steady_queue/processes/timer.py
|
|
69
70
|
steady_queue/processes/worker.py
|
|
71
|
+
tests/test_app_executor.py
|
|
70
72
|
tests/test_arguments.py
|
|
71
73
|
tests/test_backend.py
|
|
72
74
|
tests/test_concurrency_controls.py
|
|
@@ -75,6 +77,7 @@ tests/test_db_router.py
|
|
|
75
77
|
tests/test_executions.py
|
|
76
78
|
tests/test_fork_safety.py
|
|
77
79
|
tests/test_job.py
|
|
80
|
+
tests/test_observability_signals.py
|
|
78
81
|
tests/test_pidfile.py
|
|
79
82
|
tests/test_process_lifecycle.py
|
|
80
83
|
tests/test_queue_selector.py
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
2
|
+
from unittest import skipUnless
|
|
3
|
+
|
|
4
|
+
from django.db import connections, transaction
|
|
5
|
+
from django.test import TransactionTestCase
|
|
6
|
+
|
|
7
|
+
from steady_queue.app_executor import AppExecutor
|
|
8
|
+
from tests.dummy.models import Dummy
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@skipUnless(
|
|
12
|
+
connections["default"].vendor == "postgresql",
|
|
13
|
+
"Requires PostgreSQL application connection pooling",
|
|
14
|
+
)
|
|
15
|
+
class AppExecutorConnectionsTest(TransactionTestCase):
|
|
16
|
+
databases = {"default", "queue"}
|
|
17
|
+
|
|
18
|
+
def setUp(self):
|
|
19
|
+
self.application = connections["default"]
|
|
20
|
+
self.application.close()
|
|
21
|
+
self.application.close_pool()
|
|
22
|
+
self.original_options = self.application.settings_dict["OPTIONS"]
|
|
23
|
+
self.application.settings_dict["OPTIONS"] = {
|
|
24
|
+
**self.original_options,
|
|
25
|
+
"pool": {"min_size": 0, "max_size": 1, "timeout": 0.25},
|
|
26
|
+
}
|
|
27
|
+
self.addCleanup(self.restore_application_pool)
|
|
28
|
+
|
|
29
|
+
def restore_application_pool(self):
|
|
30
|
+
self.application.close()
|
|
31
|
+
self.application.close_pool()
|
|
32
|
+
self.application.settings_dict["OPTIONS"] = self.original_options
|
|
33
|
+
|
|
34
|
+
@staticmethod
|
|
35
|
+
def perform_task(*, fail=False):
|
|
36
|
+
with AppExecutor.wrap_in_app_executor():
|
|
37
|
+
Dummy.objects.count()
|
|
38
|
+
if fail:
|
|
39
|
+
raise ValueError("Task failed")
|
|
40
|
+
return (
|
|
41
|
+
connections["default"].connection is None,
|
|
42
|
+
connections["queue"].connection is None,
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
def assert_pool_reusable_by_another_worker(self, *, fail=False):
|
|
46
|
+
# Keep both threads alive: ending a thread must not stand in for the
|
|
47
|
+
# per-task cleanup that long-lived worker threads need.
|
|
48
|
+
workers = [ThreadPoolExecutor(max_workers=1) for _ in range(2)]
|
|
49
|
+
try:
|
|
50
|
+
first = workers[0].submit(self.perform_task, fail=fail)
|
|
51
|
+
if fail:
|
|
52
|
+
with self.assertRaisesMessage(ValueError, "Task failed"):
|
|
53
|
+
first.result(timeout=5)
|
|
54
|
+
else:
|
|
55
|
+
first.result(timeout=5)
|
|
56
|
+
|
|
57
|
+
# A leaked application connection leaves this separate thread
|
|
58
|
+
# unable to borrow the only connection in the application pool.
|
|
59
|
+
state = workers[1].submit(self.perform_task).result(timeout=5)
|
|
60
|
+
self.assertEqual(state, (True, True))
|
|
61
|
+
self.assertEqual(self.application.pool.get_stats()["pool_available"], 1)
|
|
62
|
+
finally:
|
|
63
|
+
for worker in workers:
|
|
64
|
+
worker.submit(connections.close_all).result(timeout=5)
|
|
65
|
+
worker.shutdown()
|
|
66
|
+
|
|
67
|
+
def test_success_returns_application_connection_for_other_workers(self):
|
|
68
|
+
self.assert_pool_reusable_by_another_worker()
|
|
69
|
+
|
|
70
|
+
def test_failure_returns_application_connection_for_other_workers(self):
|
|
71
|
+
self.assert_pool_reusable_by_another_worker(fail=True)
|
|
72
|
+
|
|
73
|
+
def test_main_thread_preserves_callers_connections_and_transaction(self):
|
|
74
|
+
connections["queue"].ensure_connection()
|
|
75
|
+
queue_connection = connections["queue"].connection
|
|
76
|
+
with transaction.atomic():
|
|
77
|
+
Dummy.objects.count()
|
|
78
|
+
application_connection = self.application.connection
|
|
79
|
+
with AppExecutor.wrap_in_app_executor():
|
|
80
|
+
Dummy.objects.count()
|
|
81
|
+
self.assertIs(self.application.connection, application_connection)
|
|
82
|
+
self.assertIs(connections["queue"].connection, queue_connection)
|
|
83
|
+
self.assertTrue(self.application.in_atomic_block)
|
|
84
|
+
Dummy.objects.count()
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from unittest.mock import MagicMock, patch
|
|
3
|
+
|
|
4
|
+
from django.test import SimpleTestCase, TestCase
|
|
5
|
+
|
|
6
|
+
from steady_queue.configuration import Configuration
|
|
7
|
+
from steady_queue.models import Job, Queue
|
|
8
|
+
from steady_queue.processes.base import Base
|
|
9
|
+
from steady_queue.processes.runnable import Runnable
|
|
10
|
+
from steady_queue.processes.supervisor import Supervisor
|
|
11
|
+
from steady_queue.signals import (
|
|
12
|
+
ProcessLifecycle,
|
|
13
|
+
QueueLifecycle,
|
|
14
|
+
process_restarted,
|
|
15
|
+
process_started,
|
|
16
|
+
process_stopped,
|
|
17
|
+
queue_paused,
|
|
18
|
+
queue_resumed,
|
|
19
|
+
)
|
|
20
|
+
from tests.dummy.tasks import dummy_task
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class FakeRunnable(Runnable, Base):
|
|
24
|
+
mode = "inline"
|
|
25
|
+
|
|
26
|
+
def run(self):
|
|
27
|
+
pass
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class ProcessLifecycleSignalTests(SimpleTestCase):
|
|
31
|
+
def test_runnable_emits_started_and_stopped_with_operational_context(self):
|
|
32
|
+
process = FakeRunnable()
|
|
33
|
+
started = MagicMock()
|
|
34
|
+
stopped = MagicMock()
|
|
35
|
+
|
|
36
|
+
process_started.connect(started)
|
|
37
|
+
process_stopped.connect(stopped)
|
|
38
|
+
self.addCleanup(process_started.disconnect, started)
|
|
39
|
+
self.addCleanup(process_stopped.disconnect, stopped)
|
|
40
|
+
|
|
41
|
+
process.start()
|
|
42
|
+
|
|
43
|
+
started.assert_called_once_with(
|
|
44
|
+
signal=process_started,
|
|
45
|
+
sender=ProcessLifecycle,
|
|
46
|
+
process_kind="fakerunnable",
|
|
47
|
+
process_name=process.name,
|
|
48
|
+
pid=process.pid,
|
|
49
|
+
hostname=process.hostname,
|
|
50
|
+
metadata={},
|
|
51
|
+
)
|
|
52
|
+
stopped.assert_called_once_with(
|
|
53
|
+
signal=process_stopped,
|
|
54
|
+
sender=ProcessLifecycle,
|
|
55
|
+
process_kind="fakerunnable",
|
|
56
|
+
process_name=process.name,
|
|
57
|
+
pid=process.pid,
|
|
58
|
+
hostname=process.hostname,
|
|
59
|
+
metadata={},
|
|
60
|
+
error=None,
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
def test_stopped_signal_includes_run_error(self):
|
|
64
|
+
process = FakeRunnable()
|
|
65
|
+
error = RuntimeError("broken")
|
|
66
|
+
stopped = MagicMock()
|
|
67
|
+
process.run = MagicMock(side_effect=error)
|
|
68
|
+
|
|
69
|
+
process_stopped.connect(stopped)
|
|
70
|
+
self.addCleanup(process_stopped.disconnect, stopped)
|
|
71
|
+
|
|
72
|
+
with self.assertRaises(RuntimeError):
|
|
73
|
+
process.start()
|
|
74
|
+
|
|
75
|
+
self.assertIs(stopped.call_args.kwargs["error"], error)
|
|
76
|
+
|
|
77
|
+
def test_forked_child_does_not_emit_supervisor_stopped(self):
|
|
78
|
+
supervisor = Supervisor(Configuration(Configuration.Options()))
|
|
79
|
+
originating_pid = os.getpid()
|
|
80
|
+
read_fd, write_fd = os.pipe()
|
|
81
|
+
child_pids = []
|
|
82
|
+
|
|
83
|
+
def record_stop(**kwargs):
|
|
84
|
+
message = f"{os.getpid()}:{kwargs['pid']}\n".encode()
|
|
85
|
+
os.write(write_fd, message)
|
|
86
|
+
|
|
87
|
+
def fork_child():
|
|
88
|
+
pid = os.fork()
|
|
89
|
+
if pid == 0:
|
|
90
|
+
os.close(read_fd)
|
|
91
|
+
raise SystemExit
|
|
92
|
+
child_pids.append(pid)
|
|
93
|
+
|
|
94
|
+
process_stopped.connect(record_stop)
|
|
95
|
+
try:
|
|
96
|
+
with (
|
|
97
|
+
patch.object(supervisor, "boot"),
|
|
98
|
+
patch.object(supervisor, "reset_database_connections"),
|
|
99
|
+
patch.object(supervisor, "start_processes", side_effect=fork_child),
|
|
100
|
+
patch.object(supervisor, "launch_maintenance_task"),
|
|
101
|
+
patch.object(supervisor, "restore_default_signal_handlers"),
|
|
102
|
+
patch.object(supervisor, "shutdown"),
|
|
103
|
+
patch.object(supervisor, "supervise"),
|
|
104
|
+
):
|
|
105
|
+
supervisor.start()
|
|
106
|
+
|
|
107
|
+
if os.getpid() != originating_pid:
|
|
108
|
+
os.close(write_fd)
|
|
109
|
+
os._exit(0)
|
|
110
|
+
|
|
111
|
+
os.waitpid(child_pids[0], 0)
|
|
112
|
+
os.close(write_fd)
|
|
113
|
+
write_fd = None
|
|
114
|
+
records = os.read(read_fd, 4096).decode().splitlines()
|
|
115
|
+
finally:
|
|
116
|
+
process_stopped.disconnect(record_stop)
|
|
117
|
+
os.close(read_fd)
|
|
118
|
+
if write_fd is not None:
|
|
119
|
+
os.close(write_fd)
|
|
120
|
+
|
|
121
|
+
self.assertEqual(records, [f"{originating_pid}:{originating_pid}"])
|
|
122
|
+
|
|
123
|
+
def test_supervisor_emits_restart_after_replacing_child(self):
|
|
124
|
+
supervisor = Supervisor(Configuration(Configuration.Options()))
|
|
125
|
+
terminated = FakeRunnable()
|
|
126
|
+
configured = MagicMock()
|
|
127
|
+
supervisor.forks[123] = terminated
|
|
128
|
+
supervisor.configured_processes[123] = configured
|
|
129
|
+
restarted = MagicMock()
|
|
130
|
+
|
|
131
|
+
process_restarted.connect(restarted)
|
|
132
|
+
self.addCleanup(process_restarted.disconnect, restarted)
|
|
133
|
+
|
|
134
|
+
with (
|
|
135
|
+
patch.object(supervisor, "handle_claimed_jobs_by") as recover_jobs,
|
|
136
|
+
patch.object(supervisor, "start_process", return_value=456),
|
|
137
|
+
):
|
|
138
|
+
supervisor.replace_fork(123, 7 << 8)
|
|
139
|
+
|
|
140
|
+
recover_jobs.assert_called_once_with(terminated, 7 << 8)
|
|
141
|
+
restarted.assert_called_once_with(
|
|
142
|
+
signal=process_restarted,
|
|
143
|
+
sender=ProcessLifecycle,
|
|
144
|
+
process_kind="fakerunnable",
|
|
145
|
+
process_name=terminated.name,
|
|
146
|
+
pid=123,
|
|
147
|
+
hostname=terminated.hostname,
|
|
148
|
+
metadata={},
|
|
149
|
+
exitcode=7,
|
|
150
|
+
replacement_pid=456,
|
|
151
|
+
supervisor_pid=supervisor.pid,
|
|
152
|
+
)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
class QueueLifecycleSignalTests(TestCase):
|
|
156
|
+
databases = {"default", "queue"}
|
|
157
|
+
|
|
158
|
+
def setUp(self):
|
|
159
|
+
Job.objects.enqueue(dummy_task, [], {})
|
|
160
|
+
self.queue = Queue.objects.get(queue_name="default")
|
|
161
|
+
|
|
162
|
+
def test_pause_and_resume_report_real_and_idempotent_actions(self):
|
|
163
|
+
paused = MagicMock()
|
|
164
|
+
resumed = MagicMock()
|
|
165
|
+
queue_paused.connect(paused)
|
|
166
|
+
queue_resumed.connect(resumed)
|
|
167
|
+
self.addCleanup(queue_paused.disconnect, paused)
|
|
168
|
+
self.addCleanup(queue_resumed.disconnect, resumed)
|
|
169
|
+
|
|
170
|
+
self.queue.pause()
|
|
171
|
+
self.queue.pause()
|
|
172
|
+
self.queue.resume()
|
|
173
|
+
self.queue.resume()
|
|
174
|
+
|
|
175
|
+
self.assertEqual(
|
|
176
|
+
[call.kwargs["changed"] for call in paused.call_args_list], [True, False]
|
|
177
|
+
)
|
|
178
|
+
self.assertEqual(
|
|
179
|
+
[call.kwargs["changed"] for call in resumed.call_args_list], [True, False]
|
|
180
|
+
)
|
|
181
|
+
self.assertTrue(
|
|
182
|
+
all(
|
|
183
|
+
call.kwargs["queue_name"] == "default" for call in paused.call_args_list
|
|
184
|
+
)
|
|
185
|
+
)
|
|
186
|
+
self.assertTrue(
|
|
187
|
+
all("queue" not in call.kwargs for call in paused.call_args_list)
|
|
188
|
+
)
|
|
189
|
+
self.assertTrue(
|
|
190
|
+
all("queue" not in call.kwargs for call in resumed.call_args_list)
|
|
191
|
+
)
|
|
192
|
+
self.assertTrue(
|
|
193
|
+
all(
|
|
194
|
+
call.kwargs["sender"] is QueueLifecycle
|
|
195
|
+
for call in paused.call_args_list
|
|
196
|
+
)
|
|
197
|
+
)
|
|
198
|
+
self.assertTrue(
|
|
199
|
+
all(
|
|
200
|
+
call.kwargs["sender"] is QueueLifecycle
|
|
201
|
+
for call in resumed.call_args_list
|
|
202
|
+
)
|
|
203
|
+
)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|