steady-queue 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {steady_queue-0.2.0/steady_queue.egg-info → steady_queue-0.3.0}/PKG-INFO +15 -5
- {steady_queue-0.2.0 → steady_queue-0.3.0}/README.md +14 -4
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/__init__.py +1 -1
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/app_executor.py +4 -5
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/queue.py +13 -2
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/recurring_execution.py +8 -4
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/recurring_task.py +5 -1
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/processes/runnable.py +22 -5
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/processes/supervisor.py +58 -18
- steady_queue-0.3.0/steady_queue/signals.py +37 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0/steady_queue.egg-info}/PKG-INFO +15 -5
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue.egg-info/SOURCES.txt +4 -0
- steady_queue-0.3.0/tests/test_app_executor.py +84 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/tests/test_executions.py +57 -1
- steady_queue-0.3.0/tests/test_fork_safety.py +183 -0
- steady_queue-0.3.0/tests/test_observability_signals.py +203 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/tests/test_process_lifecycle.py +24 -10
- steady_queue-0.3.0/tests/test_recurring_execution.py +36 -0
- steady_queue-0.2.0/tests/test_fork_safety.py +0 -98
- {steady_queue-0.2.0 → steady_queue-0.3.0}/LICENSE +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/pyproject.toml +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/setup.cfg +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/admin.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/apps.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/arguments.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/backend.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/collections.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/concurrency.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/configuration.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/db_router.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/management/__init__.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/management/commands/__init__.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/management/commands/steady_queue.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/migrations/0001_initial.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/migrations/__init__.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/__init__.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/base.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/blocked_execution.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/claimed_execution.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/clearable.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/concurrency_controls.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/dispatching.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/executable.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/execution.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/executor.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/failed_execution.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/job.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/pause.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/process.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/prunable.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/ready_execution.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/retryable.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/schedulable.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/scheduled_execution.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/models/semaphore.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/processes/__init__.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/processes/base.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/processes/concurrent.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/processes/dispatcher.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/processes/errors.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/processes/interruptible.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/processes/maintenance.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/processes/pidfile.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/processes/pidfiled.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/processes/poller.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/processes/pool.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/processes/registrable.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/processes/scheduler.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/processes/signals.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/processes/supervised.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/processes/timer.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/processes/worker.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/queue_selector.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/recurring_schedule.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/recurring_task.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue/task.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue.egg-info/dependency_links.txt +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue.egg-info/requires.txt +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/steady_queue.egg-info/top_level.txt +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/tests/test_arguments.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/tests/test_backend.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/tests/test_concurrency_controls.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/tests/test_configuration.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/tests/test_db_router.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/tests/test_job.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/tests/test_pidfile.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/tests/test_queue_selector.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/tests/test_recurring_task.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/tests/test_semaphore.py +0 -0
- {steady_queue-0.2.0 → steady_queue-0.3.0}/tests/test_task.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: steady-queue
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: An asynchronous task library for Django that supports delayed tasks, concurrency controls, recurring tasks, pausing queues, numeric priorities per task, priorities by queue order and bulk enqueueing.
|
|
5
5
|
Author-email: Elias Hernandis <elias@hernandis.me>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -535,8 +535,17 @@ instance that is handling the task, as well as the `task_result` (a
|
|
|
535
535
|
`django.tasks.TaskResult` instance) with information on how the task was called
|
|
536
536
|
and its status.
|
|
537
537
|
|
|
538
|
-
|
|
539
|
-
|
|
538
|
+
Steady Queue also emits operational signals from ``steady_queue.signals``:
|
|
539
|
+
|
|
540
|
+
- `process_started` and `process_stopped` when a supervisor or child process
|
|
541
|
+
enters or leaves its run loop.
|
|
542
|
+
- `process_restarted` when the supervisor replaces a terminated child.
|
|
543
|
+
- `queue_paused` and `queue_resumed` after queue control actions.
|
|
544
|
+
|
|
545
|
+
Process signals include process identity and metadata. Queue signals include a
|
|
546
|
+
`changed` flag so receivers can distinguish state transitions from idempotent
|
|
547
|
+
actions. See the [configuration documentation](https://steady-queue.readthedocs.io/en/latest/configuration.html#signals)
|
|
548
|
+
for the complete payloads.
|
|
540
549
|
|
|
541
550
|
## Logging
|
|
542
551
|
|
|
@@ -824,8 +833,9 @@ there are a few differences which we outline below.
|
|
|
824
833
|
will ever be supported, but we've kept the database column for compatibility.
|
|
825
834
|
- Steady Queue worker processes do not set the process name (or procline)
|
|
826
835
|
because doing so requires introducing an external dependency.
|
|
827
|
-
- Steady Queue
|
|
828
|
-
the
|
|
836
|
+
- Steady Queue exposes Django signals for task and operational lifecycle events,
|
|
837
|
+
but does not yet provide the full timed instrumentation event set emitted by
|
|
838
|
+
Solid Queue through `ActiveSupport::Notifications`.
|
|
829
839
|
- **Priority ordering:** Steady Queue follows Django's convention where larger
|
|
830
840
|
numbers indicate higher priority (e.g., a task with priority 10 runs before
|
|
831
841
|
priority 0), whereas Solid Queue uses the inverse (smaller numbers = higher
|
|
@@ -506,8 +506,17 @@ instance that is handling the task, as well as the `task_result` (a
|
|
|
506
506
|
`django.tasks.TaskResult` instance) with information on how the task was called
|
|
507
507
|
and its status.
|
|
508
508
|
|
|
509
|
-
|
|
510
|
-
|
|
509
|
+
Steady Queue also emits operational signals from ``steady_queue.signals``:
|
|
510
|
+
|
|
511
|
+
- `process_started` and `process_stopped` when a supervisor or child process
|
|
512
|
+
enters or leaves its run loop.
|
|
513
|
+
- `process_restarted` when the supervisor replaces a terminated child.
|
|
514
|
+
- `queue_paused` and `queue_resumed` after queue control actions.
|
|
515
|
+
|
|
516
|
+
Process signals include process identity and metadata. Queue signals include a
|
|
517
|
+
`changed` flag so receivers can distinguish state transitions from idempotent
|
|
518
|
+
actions. See the [configuration documentation](https://steady-queue.readthedocs.io/en/latest/configuration.html#signals)
|
|
519
|
+
for the complete payloads.
|
|
511
520
|
|
|
512
521
|
## Logging
|
|
513
522
|
|
|
@@ -795,8 +804,9 @@ there are a few differences which we outline below.
|
|
|
795
804
|
will ever be supported, but we've kept the database column for compatibility.
|
|
796
805
|
- Steady Queue worker processes do not set the process name (or procline)
|
|
797
806
|
because doing so requires introducing an external dependency.
|
|
798
|
-
- Steady Queue
|
|
799
|
-
the
|
|
807
|
+
- Steady Queue exposes Django signals for task and operational lifecycle events,
|
|
808
|
+
but does not yet provide the full timed instrumentation event set emitted by
|
|
809
|
+
Solid Queue through `ActiveSupport::Notifications`.
|
|
800
810
|
- **Priority ordering:** Steady Queue follows Django's convention where larger
|
|
801
811
|
numbers indicate higher priority (e.g., a task with priority 10 runs before
|
|
802
812
|
priority 0), whereas Solid Queue uses the inverse (smaller numbers = higher
|
|
@@ -40,9 +40,8 @@ class AppExecutor:
|
|
|
40
40
|
connection.close()
|
|
41
41
|
raise
|
|
42
42
|
finally:
|
|
43
|
-
# Django automatically manages connection lifecycle,
|
|
44
|
-
# but we can explicitly close if needed for long-running processes
|
|
45
43
|
if threading.current_thread() != threading.main_thread():
|
|
46
|
-
#
|
|
47
|
-
#
|
|
48
|
-
|
|
44
|
+
# Worker threads are reused without Django's request cleanup.
|
|
45
|
+
# Return connections for every initialized alias, including the
|
|
46
|
+
# application databases used by the task, to their pools.
|
|
47
|
+
connections.close_all()
|
|
@@ -2,6 +2,7 @@ from django.db import models
|
|
|
2
2
|
|
|
3
3
|
from steady_queue.models.pause import Pause
|
|
4
4
|
from steady_queue.models.ready_execution import ReadyExecution
|
|
5
|
+
from steady_queue.signals import QueueLifecycle, queue_paused, queue_resumed
|
|
5
6
|
|
|
6
7
|
|
|
7
8
|
class QueueQuerySet(models.QuerySet):
|
|
@@ -48,10 +49,20 @@ class Queue(models.Model):
|
|
|
48
49
|
return not self.is_paused
|
|
49
50
|
|
|
50
51
|
def pause(self) -> None:
|
|
51
|
-
Pause.objects.get_or_create(queue_name=self.queue_name)
|
|
52
|
+
_, changed = Pause.objects.get_or_create(queue_name=self.queue_name)
|
|
53
|
+
queue_paused.send(
|
|
54
|
+
sender=QueueLifecycle,
|
|
55
|
+
queue_name=self.queue_name,
|
|
56
|
+
changed=changed,
|
|
57
|
+
)
|
|
52
58
|
|
|
53
59
|
def resume(self) -> None:
|
|
54
|
-
Pause.objects.filter(queue_name=self.queue_name).delete()
|
|
60
|
+
deleted, _ = Pause.objects.filter(queue_name=self.queue_name).delete()
|
|
61
|
+
queue_resumed.send(
|
|
62
|
+
sender=QueueLifecycle,
|
|
63
|
+
queue_name=self.queue_name,
|
|
64
|
+
changed=deleted > 0,
|
|
65
|
+
)
|
|
55
66
|
|
|
56
67
|
def __str__(self) -> str:
|
|
57
68
|
return self.queue_name
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from django.db import models
|
|
1
|
+
from django.db import IntegrityError, models
|
|
2
2
|
from django.tasks import TaskResult
|
|
3
3
|
|
|
4
4
|
from .execution import Execution, ExecutionQuerySet
|
|
@@ -9,9 +9,10 @@ class RecurringExecutionQuerySet(ExecutionQuerySet):
|
|
|
9
9
|
return self.filter(job__isnull=True)
|
|
10
10
|
|
|
11
11
|
def record(self, task_result: TaskResult, task, run_at):
|
|
12
|
-
|
|
13
|
-
task=task, run_at=run_at,
|
|
14
|
-
|
|
12
|
+
try:
|
|
13
|
+
self.create(task=task, run_at=run_at, job_id=task_result.id)
|
|
14
|
+
except IntegrityError as e:
|
|
15
|
+
raise self.model.AlreadyRecorded from e
|
|
15
16
|
|
|
16
17
|
def clear_in_batches(self, batch_size=500):
|
|
17
18
|
while True:
|
|
@@ -21,6 +22,9 @@ class RecurringExecutionQuerySet(ExecutionQuerySet):
|
|
|
21
22
|
|
|
22
23
|
|
|
23
24
|
class RecurringExecution(Execution):
|
|
25
|
+
class AlreadyRecorded(Exception):
|
|
26
|
+
pass
|
|
27
|
+
|
|
24
28
|
class Meta:
|
|
25
29
|
verbose_name = "recurring execution"
|
|
26
30
|
verbose_name_plural = "recurring executions"
|
|
@@ -97,7 +97,10 @@ class RecurringTask(UpdatedAtMixin, BaseModel):
|
|
|
97
97
|
return self.recurring_executions.max("run_at")
|
|
98
98
|
|
|
99
99
|
def enqueue(self, run_at: datetime):
|
|
100
|
-
|
|
100
|
+
try:
|
|
101
|
+
return self.enqueue_and_record(run_at)
|
|
102
|
+
except RecurringExecution.AlreadyRecorded:
|
|
103
|
+
return False
|
|
101
104
|
|
|
102
105
|
def enqueue_and_record(self, run_at: datetime):
|
|
103
106
|
with transaction.atomic(using=self._state.db):
|
|
@@ -108,6 +111,7 @@ class RecurringTask(UpdatedAtMixin, BaseModel):
|
|
|
108
111
|
queue_name=self.queue_name, priority=self.priority
|
|
109
112
|
).enqueue(*args, **kwargs)
|
|
110
113
|
RecurringExecution.objects.record(task_result, self, run_at)
|
|
114
|
+
return task_result
|
|
111
115
|
|
|
112
116
|
@property
|
|
113
117
|
def previous_time(self) -> datetime:
|
|
@@ -1,6 +1,12 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
|
|
3
3
|
from steady_queue.processes.supervised import Supervised
|
|
4
|
+
from steady_queue.signals import (
|
|
5
|
+
_process_signal_context,
|
|
6
|
+
_send_process_signal,
|
|
7
|
+
process_started,
|
|
8
|
+
process_stopped,
|
|
9
|
+
)
|
|
4
10
|
|
|
5
11
|
logger = logging.getLogger("steady_queue")
|
|
6
12
|
|
|
@@ -10,11 +16,22 @@ class Runnable(Supervised):
|
|
|
10
16
|
|
|
11
17
|
def start(self):
|
|
12
18
|
self.boot()
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
19
|
+
signal_context = _process_signal_context(self)
|
|
20
|
+
_send_process_signal(process_started, self, context=signal_context)
|
|
21
|
+
|
|
22
|
+
error = None
|
|
23
|
+
try:
|
|
24
|
+
if self.is_running_async:
|
|
25
|
+
raise NotImplementedError
|
|
26
|
+
else:
|
|
27
|
+
self.run()
|
|
28
|
+
except BaseException as exception:
|
|
29
|
+
error = exception
|
|
30
|
+
raise
|
|
31
|
+
finally:
|
|
32
|
+
_send_process_signal(
|
|
33
|
+
process_stopped, self, context=signal_context, error=error
|
|
34
|
+
)
|
|
18
35
|
|
|
19
36
|
def stop(self):
|
|
20
37
|
super().stop()
|
|
@@ -15,6 +15,14 @@ from steady_queue.processes.pidfiled import Pidfiled
|
|
|
15
15
|
from steady_queue.processes.registrable import Registrable
|
|
16
16
|
from steady_queue.processes.signals import Signals
|
|
17
17
|
from steady_queue.processes.timer import wait_until
|
|
18
|
+
from steady_queue.signals import (
|
|
19
|
+
ProcessLifecycle,
|
|
20
|
+
_process_signal_context,
|
|
21
|
+
_send_process_signal,
|
|
22
|
+
process_restarted,
|
|
23
|
+
process_started,
|
|
24
|
+
process_stopped,
|
|
25
|
+
)
|
|
18
26
|
|
|
19
27
|
logger = logging.getLogger("steady_queue")
|
|
20
28
|
|
|
@@ -40,19 +48,35 @@ class Supervisor(Maintenance, Signals, Pidfiled, Registrable, Interruptible, Bas
|
|
|
40
48
|
super().__init__()
|
|
41
49
|
|
|
42
50
|
def start(self) -> None:
|
|
43
|
-
|
|
51
|
+
originating_pid = self.pid
|
|
52
|
+
logger.info("starting supervisor with PID %(pid)d", {"pid": originating_pid})
|
|
53
|
+
started = False
|
|
54
|
+
error = None
|
|
55
|
+
signal_context = None
|
|
44
56
|
try:
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
57
|
+
try:
|
|
58
|
+
self.boot()
|
|
59
|
+
started = True
|
|
60
|
+
signal_context = _process_signal_context(self)
|
|
61
|
+
_send_process_signal(process_started, self, context=signal_context)
|
|
62
|
+
# Fork only after resetting DB state (connections + psycopg pools).
|
|
63
|
+
self.reset_database_connections()
|
|
64
|
+
self.start_processes()
|
|
65
|
+
self.launch_maintenance_task()
|
|
66
|
+
except SystemExit:
|
|
67
|
+
logger.info("supervisor interrupted during boot, shutting down")
|
|
68
|
+
self.restore_default_signal_handlers()
|
|
69
|
+
self.shutdown()
|
|
70
|
+
return
|
|
71
|
+
self.supervise()
|
|
72
|
+
except BaseException as exception:
|
|
73
|
+
error = exception
|
|
74
|
+
raise
|
|
75
|
+
finally:
|
|
76
|
+
if started and self.pid == originating_pid:
|
|
77
|
+
_send_process_signal(
|
|
78
|
+
process_stopped, self, context=signal_context, error=error
|
|
79
|
+
)
|
|
56
80
|
|
|
57
81
|
def boot(self) -> None:
|
|
58
82
|
super().boot()
|
|
@@ -80,12 +104,15 @@ class Supervisor(Maintenance, Signals, Pidfiled, Registrable, Interruptible, Bas
|
|
|
80
104
|
logger.debug("supervisor finally block")
|
|
81
105
|
self.shutdown()
|
|
82
106
|
|
|
83
|
-
def start_process(self, process: Configuration.Process) ->
|
|
107
|
+
def start_process(self, process: Configuration.Process) -> int:
|
|
84
108
|
logger.info("starting process %(process)s", {"process": process})
|
|
85
109
|
instance = process.instantiate()
|
|
86
110
|
instance.supervisor = self.process
|
|
87
111
|
instance.mode = "fork"
|
|
88
112
|
|
|
113
|
+
# Replacement recovery queries the database after the startup reset.
|
|
114
|
+
# Clear that connection and pool state before the child inherits it.
|
|
115
|
+
self.reset_database_connections()
|
|
89
116
|
if (pid := os.fork()) == 0:
|
|
90
117
|
# child
|
|
91
118
|
instance.start()
|
|
@@ -95,6 +122,7 @@ class Supervisor(Maintenance, Signals, Pidfiled, Registrable, Interruptible, Bas
|
|
|
95
122
|
self.reset_database_connections()
|
|
96
123
|
self.configured_processes[pid] = process
|
|
97
124
|
self.forks[pid] = instance
|
|
125
|
+
return pid
|
|
98
126
|
|
|
99
127
|
def set_procline(self) -> None:
|
|
100
128
|
pass
|
|
@@ -130,14 +158,14 @@ class Supervisor(Maintenance, Signals, Pidfiled, Registrable, Interruptible, Bas
|
|
|
130
158
|
def reap_and_replace_terminated_forks(self) -> None:
|
|
131
159
|
while True:
|
|
132
160
|
try:
|
|
133
|
-
pid,
|
|
161
|
+
pid, wait_status = os.waitpid(-1, os.WNOHANG)
|
|
134
162
|
except ChildProcessError:
|
|
135
163
|
break
|
|
136
164
|
else:
|
|
137
165
|
if not pid:
|
|
138
166
|
break
|
|
139
167
|
|
|
140
|
-
self.replace_fork(pid,
|
|
168
|
+
self.replace_fork(pid, wait_status)
|
|
141
169
|
|
|
142
170
|
def reap_terminated_forks(self) -> None:
|
|
143
171
|
while True:
|
|
@@ -158,11 +186,23 @@ class Supervisor(Maintenance, Signals, Pidfiled, Registrable, Interruptible, Bas
|
|
|
158
186
|
|
|
159
187
|
self.configured_processes.pop(pid, None)
|
|
160
188
|
|
|
161
|
-
def replace_fork(self, pid: int,
|
|
189
|
+
def replace_fork(self, pid: int, wait_status: int) -> None:
|
|
190
|
+
exitcode = os.waitstatus_to_exitcode(wait_status)
|
|
162
191
|
logger.info("replacing fork %s due to exit code %s", pid, exitcode)
|
|
163
192
|
if terminated_fork := self.forks.pop(pid, None):
|
|
164
|
-
self.handle_claimed_jobs_by(terminated_fork,
|
|
165
|
-
self.start_process(self.configured_processes.pop(pid))
|
|
193
|
+
self.handle_claimed_jobs_by(terminated_fork, wait_status)
|
|
194
|
+
replacement_pid = self.start_process(self.configured_processes.pop(pid))
|
|
195
|
+
process_restarted.send(
|
|
196
|
+
sender=ProcessLifecycle,
|
|
197
|
+
process_kind=terminated_fork.kind,
|
|
198
|
+
process_name=terminated_fork.name,
|
|
199
|
+
pid=pid,
|
|
200
|
+
hostname=terminated_fork.hostname,
|
|
201
|
+
metadata=terminated_fork.metadata,
|
|
202
|
+
exitcode=exitcode,
|
|
203
|
+
replacement_pid=replacement_pid,
|
|
204
|
+
supervisor_pid=self.pid,
|
|
205
|
+
)
|
|
166
206
|
|
|
167
207
|
def handle_claimed_jobs_by(self, terminated_fork: Base, exitcode: int) -> None:
|
|
168
208
|
if not self.process:
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
from django.dispatch import Signal
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class ProcessLifecycle:
|
|
5
|
+
"""Public sender for process lifecycle signals."""
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class QueueLifecycle:
|
|
9
|
+
"""Public sender for queue lifecycle signals."""
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
# Process lifecycle signals carry stable data suitable for logs and metrics.
|
|
13
|
+
process_started = Signal()
|
|
14
|
+
process_stopped = Signal()
|
|
15
|
+
process_restarted = Signal()
|
|
16
|
+
|
|
17
|
+
# Queue control signals. Receivers get ``queue_name`` and ``changed``.
|
|
18
|
+
queue_paused = Signal()
|
|
19
|
+
queue_resumed = Signal()
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _process_signal_context(process) -> dict:
|
|
23
|
+
return {
|
|
24
|
+
"process_kind": process.kind,
|
|
25
|
+
"process_name": process.name,
|
|
26
|
+
"pid": process.pid,
|
|
27
|
+
"hostname": process.hostname,
|
|
28
|
+
"metadata": process.metadata,
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _send_process_signal(signal: Signal, process, *, context=None, **kwargs) -> None:
|
|
33
|
+
signal.send(
|
|
34
|
+
sender=ProcessLifecycle,
|
|
35
|
+
**(context or _process_signal_context(process)),
|
|
36
|
+
**kwargs,
|
|
37
|
+
)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: steady-queue
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: An asynchronous task library for Django that supports delayed tasks, concurrency controls, recurring tasks, pausing queues, numeric priorities per task, priorities by queue order and bulk enqueueing.
|
|
5
5
|
Author-email: Elias Hernandis <elias@hernandis.me>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -535,8 +535,17 @@ instance that is handling the task, as well as the `task_result` (a
|
|
|
535
535
|
`django.tasks.TaskResult` instance) with information on how the task was called
|
|
536
536
|
and its status.
|
|
537
537
|
|
|
538
|
-
|
|
539
|
-
|
|
538
|
+
Steady Queue also emits operational signals from ``steady_queue.signals``:
|
|
539
|
+
|
|
540
|
+
- `process_started` and `process_stopped` when a supervisor or child process
|
|
541
|
+
enters or leaves its run loop.
|
|
542
|
+
- `process_restarted` when the supervisor replaces a terminated child.
|
|
543
|
+
- `queue_paused` and `queue_resumed` after queue control actions.
|
|
544
|
+
|
|
545
|
+
Process signals include process identity and metadata. Queue signals include a
|
|
546
|
+
`changed` flag so receivers can distinguish state transitions from idempotent
|
|
547
|
+
actions. See the [configuration documentation](https://steady-queue.readthedocs.io/en/latest/configuration.html#signals)
|
|
548
|
+
for the complete payloads.
|
|
540
549
|
|
|
541
550
|
## Logging
|
|
542
551
|
|
|
@@ -824,8 +833,9 @@ there are a few differences which we outline below.
|
|
|
824
833
|
will ever be supported, but we've kept the database column for compatibility.
|
|
825
834
|
- Steady Queue worker processes do not set the process name (or procline)
|
|
826
835
|
because doing so requires introducing an external dependency.
|
|
827
|
-
- Steady Queue
|
|
828
|
-
the
|
|
836
|
+
- Steady Queue exposes Django signals for task and operational lifecycle events,
|
|
837
|
+
but does not yet provide the full timed instrumentation event set emitted by
|
|
838
|
+
Solid Queue through `ActiveSupport::Notifications`.
|
|
829
839
|
- **Priority ordering:** Steady Queue follows Django's convention where larger
|
|
830
840
|
numbers indicate higher priority (e.g., a task with priority 10 runs before
|
|
831
841
|
priority 0), whereas Solid Queue uses the inverse (smaller numbers = higher
|
|
@@ -14,6 +14,7 @@ steady_queue/db_router.py
|
|
|
14
14
|
steady_queue/queue_selector.py
|
|
15
15
|
steady_queue/recurring_schedule.py
|
|
16
16
|
steady_queue/recurring_task.py
|
|
17
|
+
steady_queue/signals.py
|
|
17
18
|
steady_queue/task.py
|
|
18
19
|
steady_queue.egg-info/PKG-INFO
|
|
19
20
|
steady_queue.egg-info/SOURCES.txt
|
|
@@ -67,6 +68,7 @@ steady_queue/processes/supervised.py
|
|
|
67
68
|
steady_queue/processes/supervisor.py
|
|
68
69
|
steady_queue/processes/timer.py
|
|
69
70
|
steady_queue/processes/worker.py
|
|
71
|
+
tests/test_app_executor.py
|
|
70
72
|
tests/test_arguments.py
|
|
71
73
|
tests/test_backend.py
|
|
72
74
|
tests/test_concurrency_controls.py
|
|
@@ -75,9 +77,11 @@ tests/test_db_router.py
|
|
|
75
77
|
tests/test_executions.py
|
|
76
78
|
tests/test_fork_safety.py
|
|
77
79
|
tests/test_job.py
|
|
80
|
+
tests/test_observability_signals.py
|
|
78
81
|
tests/test_pidfile.py
|
|
79
82
|
tests/test_process_lifecycle.py
|
|
80
83
|
tests/test_queue_selector.py
|
|
84
|
+
tests/test_recurring_execution.py
|
|
81
85
|
tests/test_recurring_task.py
|
|
82
86
|
tests/test_semaphore.py
|
|
83
87
|
tests/test_task.py
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
2
|
+
from unittest import skipUnless
|
|
3
|
+
|
|
4
|
+
from django.db import connections, transaction
|
|
5
|
+
from django.test import TransactionTestCase
|
|
6
|
+
|
|
7
|
+
from steady_queue.app_executor import AppExecutor
|
|
8
|
+
from tests.dummy.models import Dummy
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@skipUnless(
|
|
12
|
+
connections["default"].vendor == "postgresql",
|
|
13
|
+
"Requires PostgreSQL application connection pooling",
|
|
14
|
+
)
|
|
15
|
+
class AppExecutorConnectionsTest(TransactionTestCase):
|
|
16
|
+
databases = {"default", "queue"}
|
|
17
|
+
|
|
18
|
+
def setUp(self):
|
|
19
|
+
self.application = connections["default"]
|
|
20
|
+
self.application.close()
|
|
21
|
+
self.application.close_pool()
|
|
22
|
+
self.original_options = self.application.settings_dict["OPTIONS"]
|
|
23
|
+
self.application.settings_dict["OPTIONS"] = {
|
|
24
|
+
**self.original_options,
|
|
25
|
+
"pool": {"min_size": 0, "max_size": 1, "timeout": 0.25},
|
|
26
|
+
}
|
|
27
|
+
self.addCleanup(self.restore_application_pool)
|
|
28
|
+
|
|
29
|
+
def restore_application_pool(self):
|
|
30
|
+
self.application.close()
|
|
31
|
+
self.application.close_pool()
|
|
32
|
+
self.application.settings_dict["OPTIONS"] = self.original_options
|
|
33
|
+
|
|
34
|
+
@staticmethod
|
|
35
|
+
def perform_task(*, fail=False):
|
|
36
|
+
with AppExecutor.wrap_in_app_executor():
|
|
37
|
+
Dummy.objects.count()
|
|
38
|
+
if fail:
|
|
39
|
+
raise ValueError("Task failed")
|
|
40
|
+
return (
|
|
41
|
+
connections["default"].connection is None,
|
|
42
|
+
connections["queue"].connection is None,
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
def assert_pool_reusable_by_another_worker(self, *, fail=False):
|
|
46
|
+
# Keep both threads alive: ending a thread must not stand in for the
|
|
47
|
+
# per-task cleanup that long-lived worker threads need.
|
|
48
|
+
workers = [ThreadPoolExecutor(max_workers=1) for _ in range(2)]
|
|
49
|
+
try:
|
|
50
|
+
first = workers[0].submit(self.perform_task, fail=fail)
|
|
51
|
+
if fail:
|
|
52
|
+
with self.assertRaisesMessage(ValueError, "Task failed"):
|
|
53
|
+
first.result(timeout=5)
|
|
54
|
+
else:
|
|
55
|
+
first.result(timeout=5)
|
|
56
|
+
|
|
57
|
+
# A leaked application connection leaves this separate thread
|
|
58
|
+
# unable to borrow the only connection in the application pool.
|
|
59
|
+
state = workers[1].submit(self.perform_task).result(timeout=5)
|
|
60
|
+
self.assertEqual(state, (True, True))
|
|
61
|
+
self.assertEqual(self.application.pool.get_stats()["pool_available"], 1)
|
|
62
|
+
finally:
|
|
63
|
+
for worker in workers:
|
|
64
|
+
worker.submit(connections.close_all).result(timeout=5)
|
|
65
|
+
worker.shutdown()
|
|
66
|
+
|
|
67
|
+
def test_success_returns_application_connection_for_other_workers(self):
|
|
68
|
+
self.assert_pool_reusable_by_another_worker()
|
|
69
|
+
|
|
70
|
+
def test_failure_returns_application_connection_for_other_workers(self):
|
|
71
|
+
self.assert_pool_reusable_by_another_worker(fail=True)
|
|
72
|
+
|
|
73
|
+
def test_main_thread_preserves_callers_connections_and_transaction(self):
|
|
74
|
+
connections["queue"].ensure_connection()
|
|
75
|
+
queue_connection = connections["queue"].connection
|
|
76
|
+
with transaction.atomic():
|
|
77
|
+
Dummy.objects.count()
|
|
78
|
+
application_connection = self.application.connection
|
|
79
|
+
with AppExecutor.wrap_in_app_executor():
|
|
80
|
+
Dummy.objects.count()
|
|
81
|
+
self.assertIs(self.application.connection, application_connection)
|
|
82
|
+
self.assertIs(connections["queue"].connection, queue_connection)
|
|
83
|
+
self.assertTrue(self.application.in_atomic_block)
|
|
84
|
+
Dummy.objects.count()
|
|
@@ -1,6 +1,10 @@
|
|
|
1
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
1
2
|
from datetime import timedelta
|
|
3
|
+
from unittest import skipUnless
|
|
4
|
+
from unittest.mock import patch
|
|
2
5
|
|
|
3
|
-
from django.
|
|
6
|
+
from django.db import connections
|
|
7
|
+
from django.test import TestCase, TransactionTestCase
|
|
4
8
|
from django.utils import timezone
|
|
5
9
|
|
|
6
10
|
import steady_queue
|
|
@@ -14,6 +18,7 @@ from steady_queue.models import (
|
|
|
14
18
|
ScheduledExecution,
|
|
15
19
|
Semaphore,
|
|
16
20
|
)
|
|
21
|
+
from steady_queue.models.claimed_execution import ClaimedExecutionQuerySet
|
|
17
22
|
from tests.dummy.tasks import dummy_task, limited_task, limited_task_with_lambda_key
|
|
18
23
|
|
|
19
24
|
|
|
@@ -106,6 +111,57 @@ class ReadyExecutionTestCase(TestHelperMixin, TestCase):
|
|
|
106
111
|
self.assertEqual(ReadyExecution.objects.count(), 1)
|
|
107
112
|
|
|
108
113
|
|
|
114
|
+
class ConcurrentClaimTestCase(TestHelperMixin, TransactionTestCase):
|
|
115
|
+
databases = {"default", "queue"}
|
|
116
|
+
|
|
117
|
+
@skipUnless(
|
|
118
|
+
connections["queue"].features.has_select_for_update_skip_locked,
|
|
119
|
+
"The queue database must support SELECT FOR UPDATE SKIP LOCKED",
|
|
120
|
+
)
|
|
121
|
+
def test_higher_priority_arrival_does_not_change_claimed_candidates(self):
|
|
122
|
+
"""#25: keep the selected jobs stable across insertion and retrieval."""
|
|
123
|
+
process = self.create_test_process()
|
|
124
|
+
original = self.create_job_in_queue("default", priority=0)
|
|
125
|
+
bulk_create = ClaimedExecutionQuerySet.bulk_create
|
|
126
|
+
|
|
127
|
+
def enqueue_higher_priority_job():
|
|
128
|
+
try:
|
|
129
|
+
return self.create_job_in_queue("default", priority=10).pk
|
|
130
|
+
finally:
|
|
131
|
+
connections.close_all()
|
|
132
|
+
|
|
133
|
+
def insert_claims_then_enqueue(queryset, *args, **kwargs):
|
|
134
|
+
result = bulk_create(queryset, *args, **kwargs)
|
|
135
|
+
# Commit an arrival on another connection after claims are inserted,
|
|
136
|
+
# but before claiming() retrieves them. No sleeps or timing races.
|
|
137
|
+
with ThreadPoolExecutor(max_workers=1) as executor:
|
|
138
|
+
arrival = executor.submit(enqueue_higher_priority_job)
|
|
139
|
+
arrivals.append(arrival.result(timeout=10))
|
|
140
|
+
return result
|
|
141
|
+
|
|
142
|
+
arrivals = []
|
|
143
|
+
with patch.object(
|
|
144
|
+
ClaimedExecutionQuerySet, "bulk_create", insert_claims_then_enqueue
|
|
145
|
+
):
|
|
146
|
+
claimed = ReadyExecution.objects.claim(["*"], 1, process.pk)
|
|
147
|
+
|
|
148
|
+
self.assertEqual([execution.job_id for execution in claimed], [original.pk])
|
|
149
|
+
self.assertFalse(
|
|
150
|
+
ReadyExecution.objects.filter(
|
|
151
|
+
job_id__in=ClaimedExecution.objects.values("job_id")
|
|
152
|
+
).exists()
|
|
153
|
+
)
|
|
154
|
+
self.assertEqual(
|
|
155
|
+
list(ReadyExecution.objects.values_list("job_id", flat=True)), arrivals
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
# Draining the remaining queue must not try to claim the original again.
|
|
159
|
+
next_claimed = ReadyExecution.objects.claim(["*"], 1, process.pk)
|
|
160
|
+
self.assertEqual([execution.job_id for execution in next_claimed], arrivals)
|
|
161
|
+
self.assertEqual(ReadyExecution.objects.claim(["*"], 1, process.pk), [])
|
|
162
|
+
self.assertEqual(ClaimedExecution.objects.count(), 2)
|
|
163
|
+
|
|
164
|
+
|
|
109
165
|
class ClaimedExecutionTestCase(TestHelperMixin, TestCase):
|
|
110
166
|
"""Tests for ClaimedExecution behavior."""
|
|
111
167
|
|