steady-queue 0.2.0__tar.gz → 0.2.1__tar.gz

{steady_queue-0.2.0/steady_queue.egg-info → steady_queue-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: steady-queue
-Version: 0.2.0
+Version: 0.2.1
 Summary: An asynchronous task library for Django that supports delayed tasks, concurrency controls, recurring tasks, pausing queues, numeric priorities per task, priorities by queue order and bulk enqueueing.
 Author-email: Elias Hernandis <elias@hernandis.me>
 License-Expression: MIT

{steady_queue-0.2.0 → steady_queue-0.2.1}/steady_queue/__init__.py

@@ -1,7 +1,7 @@
 from datetime import timedelta
 from typing import Optional
 
-VERSION = (0, 2, 0)
+VERSION = (0, 2, 1)
 
 __version__ = ".".join(map(str, VERSION))
 

{steady_queue-0.2.0 → steady_queue-0.2.1}/steady_queue/models/recurring_execution.py

@@ -1,4 +1,4 @@
-from django.db import models
+from django.db import IntegrityError, models
 from django.tasks import TaskResult
 
 from .execution import Execution, ExecutionQuerySet
@@ -9,9 +9,10 @@ class RecurringExecutionQuerySet(ExecutionQuerySet):
         return self.filter(job__isnull=True)
 
     def record(self, task_result: TaskResult, task, run_at):
-        self.update_or_create(
-            task=task, run_at=run_at, defaults={"job_id": task_result.id}
-        )
+        try:
+            self.create(task=task, run_at=run_at, job_id=task_result.id)
+        except IntegrityError as e:
+            raise self.model.AlreadyRecorded from e
 
     def clear_in_batches(self, batch_size=500):
         while True:
@@ -21,6 +22,9 @@ class RecurringExecutionQuerySet(ExecutionQuerySet):
 
 
 class RecurringExecution(Execution):
+    class AlreadyRecorded(Exception):
+        pass
+
     class Meta:
         verbose_name = "recurring execution"
         verbose_name_plural = "recurring executions"

{steady_queue-0.2.0 → steady_queue-0.2.1}/steady_queue/models/recurring_task.py

@@ -97,7 +97,10 @@ class RecurringTask(UpdatedAtMixin, BaseModel):
         return self.recurring_executions.max("run_at")
 
     def enqueue(self, run_at: datetime):
-        return self.enqueue_and_record(run_at)
+        try:
+            return self.enqueue_and_record(run_at)
+        except RecurringExecution.AlreadyRecorded:
+            return False
 
     def enqueue_and_record(self, run_at: datetime):
         with transaction.atomic(using=self._state.db):
@@ -108,6 +111,7 @@ class RecurringTask(UpdatedAtMixin, BaseModel):
                 queue_name=self.queue_name, priority=self.priority
             ).enqueue(*args, **kwargs)
             RecurringExecution.objects.record(task_result, self, run_at)
+            return task_result
 
     @property
     def previous_time(self) -> datetime:

{steady_queue-0.2.0 → steady_queue-0.2.1}/steady_queue/processes/supervisor.py

@@ -86,6 +86,9 @@ class Supervisor(Maintenance, Signals, Pidfiled, Registrable, Interruptible, Bas
         instance.supervisor = self.process
         instance.mode = "fork"
 
+        # Replacement recovery queries the database after the startup reset.
+        # Clear that connection and pool state before the child inherits it.
+        self.reset_database_connections()
         if (pid := os.fork()) == 0:
             # child
             instance.start()

{steady_queue-0.2.0 → steady_queue-0.2.1/steady_queue.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: steady-queue
-Version: 0.2.0
+Version: 0.2.1
 Summary: An asynchronous task library for Django that supports delayed tasks, concurrency controls, recurring tasks, pausing queues, numeric priorities per task, priorities by queue order and bulk enqueueing.
 Author-email: Elias Hernandis <elias@hernandis.me>
 License-Expression: MIT

{steady_queue-0.2.0 → steady_queue-0.2.1}/steady_queue.egg-info/SOURCES.txt

@@ -78,6 +78,7 @@ tests/test_job.py
 tests/test_pidfile.py
 tests/test_process_lifecycle.py
 tests/test_queue_selector.py
+tests/test_recurring_execution.py
 tests/test_recurring_task.py
 tests/test_semaphore.py
 tests/test_task.py

{steady_queue-0.2.0 → steady_queue-0.2.1}/tests/test_executions.py

@@ -1,6 +1,10 @@
+from concurrent.futures import ThreadPoolExecutor
 from datetime import timedelta
+from unittest import skipUnless
+from unittest.mock import patch
 
-from django.test import TestCase
+from django.db import connections
+from django.test import TestCase, TransactionTestCase
 from django.utils import timezone
 
 import steady_queue
@@ -14,6 +18,7 @@ from steady_queue.models import (
     ScheduledExecution,
     Semaphore,
 )
+from steady_queue.models.claimed_execution import ClaimedExecutionQuerySet
 from tests.dummy.tasks import dummy_task, limited_task, limited_task_with_lambda_key
 
 
@@ -106,6 +111,57 @@ class ReadyExecutionTestCase(TestHelperMixin, TestCase):
         self.assertEqual(ReadyExecution.objects.count(), 1)
 
 
+class ConcurrentClaimTestCase(TestHelperMixin, TransactionTestCase):
+    databases = {"default", "queue"}
+
+    @skipUnless(
+        connections["queue"].features.has_select_for_update_skip_locked,
+        "The queue database must support SELECT FOR UPDATE SKIP LOCKED",
+    )
+    def test_higher_priority_arrival_does_not_change_claimed_candidates(self):
+        """#25: keep the selected jobs stable across insertion and retrieval."""
+        process = self.create_test_process()
+        original = self.create_job_in_queue("default", priority=0)
+        bulk_create = ClaimedExecutionQuerySet.bulk_create
+
+        def enqueue_higher_priority_job():
+            try:
+                return self.create_job_in_queue("default", priority=10).pk
+            finally:
+                connections.close_all()
+
+        def insert_claims_then_enqueue(queryset, *args, **kwargs):
+            result = bulk_create(queryset, *args, **kwargs)
+            # Commit an arrival on another connection after claims are inserted,
+            # but before claiming() retrieves them. No sleeps or timing races.
+            with ThreadPoolExecutor(max_workers=1) as executor:
+                arrival = executor.submit(enqueue_higher_priority_job)
+                arrivals.append(arrival.result(timeout=10))
+            return result
+
+        arrivals = []
+        with patch.object(
+            ClaimedExecutionQuerySet, "bulk_create", insert_claims_then_enqueue
+        ):
+            claimed = ReadyExecution.objects.claim(["*"], 1, process.pk)
+
+        self.assertEqual([execution.job_id for execution in claimed], [original.pk])
+        self.assertFalse(
+            ReadyExecution.objects.filter(
+                job_id__in=ClaimedExecution.objects.values("job_id")
+            ).exists()
+        )
+        self.assertEqual(
+            list(ReadyExecution.objects.values_list("job_id", flat=True)), arrivals
+        )
+
+        # Draining the remaining queue must not try to claim the original again.
+        next_claimed = ReadyExecution.objects.claim(["*"], 1, process.pk)
+        self.assertEqual([execution.job_id for execution in next_claimed], arrivals)
+        self.assertEqual(ReadyExecution.objects.claim(["*"], 1, process.pk), [])
+        self.assertEqual(ClaimedExecution.objects.count(), 2)
+
+
 class ClaimedExecutionTestCase(TestHelperMixin, TestCase):
     """Tests for ClaimedExecution behavior."""
 

steady_queue-0.2.1/tests/test_fork_safety.py

@@ -0,0 +1,183 @@
+from unittest import skipUnless
+from unittest.mock import patch
+
+from django.db import connections
+from django.test import SimpleTestCase, TransactionTestCase
+
+from steady_queue.configuration import Configuration
+from steady_queue.models import (
+    ClaimedExecution,
+    FailedExecution,
+    Job,
+    Process,
+    ReadyExecution,
+)
+from steady_queue.processes.base import Base
+from steady_queue.processes.supervisor import Supervisor
+from tests.dummy.tasks import dummy_task
+
+
+class SupervisorForkSafetyTest(SimpleTestCase):
+    def build_supervisor(self) -> Supervisor:
+        options = Configuration.Options(
+            workers=[],
+            dispatchers=[],
+            recurring_tasks=[],
+            skip_recurring=True,
+        )
+        return Supervisor(Configuration(options))
+
+    def test_supervisor_start_resets_connections_before_forking(self):
+        supervisor = self.build_supervisor()
+        calls = []
+
+        supervisor.boot = lambda: calls.append("boot")
+        supervisor.reset_database_connections = lambda: calls.append("reset")
+        supervisor.start_processes = lambda: calls.append("start_processes")
+        supervisor.launch_maintenance_task = lambda: calls.append("launch_maintenance")
+        supervisor.supervise = lambda: calls.append("supervise")
+
+        supervisor.start()
+
+        self.assertEqual(
+            calls,
+            [
+                "boot",
+                "reset",
+                "start_processes",
+                "launch_maintenance",
+                "supervise",
+            ],
+        )
+
+
+@skipUnless(
+    connections["queue"].vendor == "postgresql",
+    "Requires PostgreSQL connection pooling on the queue database",
+)
+class SupervisorPostgreSQLForkTest(TransactionTestCase):
+    databases = {"default", "queue"}
+
+    def setUp(self):
+        options = Configuration.Options(
+            workers=[], dispatchers=[], recurring_tasks=[], skip_recurring=True
+        )
+        self.supervisor = Supervisor(Configuration(options))
+        self.supervisor.register()
+        self.worker_config = Configuration.Process(
+            kind="worker", attributes=Configuration.Worker()
+        )
+        self.connection = connections["queue"]
+        self.pool_options = self.connection.settings_dict["OPTIONS"]["pool"].copy()
+        self.addCleanup(self.supervisor.reset_database_connections)
+
+    def assert_reset_at_fork(self):
+        # Intercept only the OS fork; exercise real Django connections, psycopg
+        # pools, and replacement recovery up to the boundary children inherit.
+        self.assertIsNone(self.connection.connection)
+        self.assertEqual(self.connection.__class__._connection_pools, {})
+        self.assertEqual(
+            self.connection.settings_dict["OPTIONS"]["pool"], self.pool_options
+        )
+        return 12346
+
+    def test_start_process_resets_open_connection_and_pool_before_fork(self):
+        self.assertIsNotNone(self.connection.connection)
+        self.assertIn("queue", self.connection.__class__._connection_pools)
+
+        with patch(
+            "steady_queue.processes.supervisor.os.fork",
+            side_effect=self.assert_reset_at_fork,
+        ) as fork:
+            self.supervisor.start_process(self.worker_config)
+
+        fork.assert_called_once_with()
+        self.assertIn(12346, self.supervisor.forks)
+        # Pooling remains usable by the parent after the reset.
+        self.assertTrue(Process.objects.filter(pk=self.supervisor.process.pk).exists())
+        self.assertIn("queue", self.connection.__class__._connection_pools)
+
+    def test_replacement_resets_connection_reopened_by_job_recovery(self):
+        old_worker = self.worker_config.instantiate()
+        self.addCleanup(old_worker.pool.shutdown)
+        registered_worker = Process.register(
+            kind="worker",
+            name=old_worker.name,
+            pid=12345,
+            hostname="test-host",
+            supervisor=self.supervisor.process,
+        )
+        job = Job.objects.enqueue(dummy_task, [], {})
+        ReadyExecution.objects.claim(["*"], 1, registered_worker.pk)
+        self.supervisor.forks[12345] = old_worker
+        self.supervisor.configured_processes[12345] = self.worker_config
+        self.supervisor.reset_database_connections()
+
+        with patch(
+            "steady_queue.processes.supervisor.os.fork",
+            side_effect=self.assert_reset_at_fork,
+        ) as fork:
+            self.supervisor.replace_fork(12345, 11)
+
+        fork.assert_called_once_with()
+        self.assertNotIn(12345, self.supervisor.forks)
+        self.assertIn(12346, self.supervisor.forks)
+        self.assertFalse(ClaimedExecution.objects.filter(job=job).exists())
+        self.assertTrue(FailedExecution.objects.filter(job=job).exists())
+
+
+class ResetDatabaseConnectionsTest(SimpleTestCase):
+    def test_reset_connections_clears_psycopg_pool_cache_without_disabling_pooling(
+        self,
+    ):
+        class FakePool:
+            def __init__(self):
+                self.closed = False
+
+            def close(self):
+                self.closed = True
+
+        class FakeConnection:
+            _connection_pools = {}
+
+            def __init__(self, engine: str):
+                self.settings_dict = {
+                    "ENGINE": engine,
+                    "OPTIONS": {"pool": {"min_size": 1, "max_size": 4}},
+                }
+
+        class FakeConnections(dict):
+            close_all_called = False
+
+            def __iter__(self):
+                return iter(self.keys())
+
+            def close_all(self):
+                self.close_all_called = True
+
+        pool_default = FakePool()
+        pool_queue = FakePool()
+        FakeConnection._connection_pools = {
+            "default": pool_default,
+            "queue": pool_queue,
+        }
+
+        fake_connections = FakeConnections(
+            {
+                "default": FakeConnection("django.db.backends.postgresql"),
+                "queue": FakeConnection("django.db.backends.postgresql"),
+                "sqlite": FakeConnection("django.db.backends.sqlite3"),
+            }
+        )
+
+        with patch("steady_queue.processes.base.connections", fake_connections):
+            Base().reset_database_connections()
+
+        self.assertIn("pool", fake_connections["default"].settings_dict["OPTIONS"])
+        self.assertIn("pool", fake_connections["queue"].settings_dict["OPTIONS"])
+        self.assertIn("pool", fake_connections["sqlite"].settings_dict["OPTIONS"])
+
+        self.assertTrue(fake_connections.close_all_called)
+        self.assertTrue(pool_default.closed)
+        self.assertTrue(pool_queue.closed)
+        self.assertEqual(FakeConnection._connection_pools, {})

{steady_queue-0.2.0 → steady_queue-0.2.1}/tests/test_process_lifecycle.py

@@ -1,5 +1,5 @@
 from datetime import timedelta
-from unittest.mock import MagicMock
+from unittest.mock import MagicMock, patch
 
 from django.test import TestCase
 from django.utils import timezone
@@ -272,11 +272,11 @@ class PoolPostNilProcessTest(TestCase):
 
     def test_pool_post_does_not_access_execution_process(self):
         """Pool.post should not crash when execution.process is None."""
-        import time
-
         from steady_queue.processes.pool import Pool
 
-        pool = Pool(size=1, on_idle=lambda: None)
+        on_idle = MagicMock()
+        pool = Pool(size=1, on_idle=on_idle, worker_name="test-worker")
+        self.addCleanup(pool.executor.shutdown, wait=True)
 
         execution = MagicMock()
         execution.process = None
@@ -284,11 +284,25 @@ class PoolPostNilProcessTest(TestCase):
         execution.job.class_name = "test_task"
         execution.pk = 1
 
-        # Should not raise AttributeError
-        pool.post(execution)
-        # Give the thread pool time to execute
-        time.sleep(0.2)
-        pool.shutdown()
+        submit = pool.executor.submit
+        futures = []
+
+        def capture_future(*args, **kwargs):
+            future = submit(*args, **kwargs)
+            futures.append(future)
+            return future
+
+        with self.assertLogs("steady_queue", level="INFO") as logs:
+            with patch.object(pool.executor, "submit", capture_future):
+                pool.post(execution)
+            # Exceptions after perform() live in the Future. Checking only that
+            # perform() ran allowed the original #26 logging crash to pass.
+            futures[0].result(timeout=5)
 
-        # Verify perform was called successfully
         execution.perform.assert_called_once()
+        self.assertEqual(
+            logs.output,
+            ["INFO:steady_queue:test-worker completed job 1 test_task"],
+        )
+        self.assertEqual(pool.idle_threads, 1)
+        on_idle.assert_called_once_with()

steady_queue-0.2.1/tests/test_recurring_execution.py

@@ -0,0 +1,36 @@
+from django.test import TestCase
+from django.utils import timezone
+
+from steady_queue.configuration import Configuration
+from steady_queue.models import Job, RecurringExecution, RecurringTask
+from tests.dummy.tasks import dummy_task
+
+
+class RecurringExecutionTestCase(TestCase):
+    def test_enqueue_same_run_at_only_enqueues_once(self):
+        recurring_task = RecurringTask.from_configuration(
+            Configuration.RecurringTask(
+                key="test-recurring-task",
+                class_name="tests.dummy.tasks.dummy_task",
+                schedule="* * * * *",
+                arguments=dummy_task.serialize([], {}),
+                queue_name="default",
+                priority=0,
+            )
+        )
+        recurring_task.save()
+
+        run_at = timezone.now().replace(second=0, microsecond=0)
+
+        first_result = recurring_task.enqueue(run_at=run_at)
+        second_result = recurring_task.enqueue(run_at=run_at)
+
+        self.assertNotEqual(first_result, False)
+        self.assertFalse(second_result)
+
+        self.assertEqual(Job.objects.count(), 1)
+        self.assertEqual(RecurringExecution.objects.count(), 1)
+
+        recurring_execution = RecurringExecution.objects.get()
+        self.assertEqual(recurring_execution.task_id, recurring_task.key)
+        self.assertEqual(recurring_execution.run_at, run_at)

steady_queue-0.2.0/tests/test_fork_safety.py

@@ -1,98 +0,0 @@
-from unittest.mock import patch
-
-from django.test import SimpleTestCase
-
-from steady_queue.configuration import Configuration
-from steady_queue.processes.base import Base
-from steady_queue.processes.supervisor import Supervisor
-
-
-class SupervisorForkSafetyTest(SimpleTestCase):
-    def build_supervisor(self) -> Supervisor:
-        options = Configuration.Options(
-            workers=[],
-            dispatchers=[],
-            recurring_tasks=[],
-            skip_recurring=True,
-        )
-        return Supervisor(Configuration(options))
-
-    def test_supervisor_start_resets_connections_before_forking(self):
-        supervisor = self.build_supervisor()
-        calls = []
-
-        supervisor.boot = lambda: calls.append("boot")
-        supervisor.reset_database_connections = lambda: calls.append("reset")
-        supervisor.start_processes = lambda: calls.append("start_processes")
-        supervisor.launch_maintenance_task = lambda: calls.append("launch_maintenance")
-        supervisor.supervise = lambda: calls.append("supervise")
-
-        supervisor.start()
-
-        self.assertEqual(
-            calls,
-            [
-                "boot",
-                "reset",
-                "start_processes",
-                "launch_maintenance",
-                "supervise",
-            ],
-        )
-
-
-class ResetDatabaseConnectionsTest(SimpleTestCase):
-    def test_reset_connections_clears_psycopg_pool_cache_without_disabling_pooling(
-        self,
-    ):
-        class FakePool:
-            def __init__(self):
-                self.closed = False
-
-            def close(self):
-                self.closed = True
-
-        class FakeConnection:
-            _connection_pools = {}
-
-            def __init__(self, engine: str):
-                self.settings_dict = {
-                    "ENGINE": engine,
-                    "OPTIONS": {"pool": {"min_size": 1, "max_size": 4}},
-                }
-
-        class FakeConnections(dict):
-            close_all_called = False
-
-            def __iter__(self):
-                return iter(self.keys())
-
-            def close_all(self):
-                self.close_all_called = True
-
-        pool_default = FakePool()
-        pool_queue = FakePool()
-        FakeConnection._connection_pools = {
-            "default": pool_default,
-            "queue": pool_queue,
-        }
-
-        fake_connections = FakeConnections(
-            {
-                "default": FakeConnection("django.db.backends.postgresql"),
-                "queue": FakeConnection("django.db.backends.postgresql"),
-                "sqlite": FakeConnection("django.db.backends.sqlite3"),
-            }
-        )
-
-        with patch("steady_queue.processes.base.connections", fake_connections):
-            Base().reset_database_connections()
-
-        self.assertIn("pool", fake_connections["default"].settings_dict["OPTIONS"])
-        self.assertIn("pool", fake_connections["queue"].settings_dict["OPTIONS"])
-        self.assertIn("pool", fake_connections["sqlite"].settings_dict["OPTIONS"])
-
-        self.assertTrue(fake_connections.close_all_called)
-        self.assertTrue(pool_default.closed)
-        self.assertTrue(pool_queue.closed)
-        self.assertEqual(FakeConnection._connection_pools, {})