steady-queue 0.1.8__tar.gz → 0.2.0__tar.gz

{steady_queue-0.1.8/steady_queue.egg-info → steady_queue-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: steady-queue
-Version: 0.1.8
+Version: 0.2.0
 Summary: An asynchronous task library for Django that supports delayed tasks, concurrency controls, recurring tasks, pausing queues, numeric priorities per task, priorities by queue order and bulk enqueueing.
 Author-email: Elias Hernandis <elias@hernandis.me>
 License-Expression: MIT
@@ -10,12 +10,15 @@ Project-URL: Documentation, https://steady-queue.readthedocs.io/en/latest/
 Project-URL: Tracker, https://github.com/knifecake/steady-queue/issues
 Project-URL: Releases, https://github.com/knifecake/steady-queue/releases
 Keywords: django,tasks,async,jobs,queues
+Classifier: Development Status :: 4 - Beta
 Classifier: Framework :: Django
 Classifier: Framework :: Django :: 6.0
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
-Classifier: Operating System :: OS Independent
+Classifier: Operating System :: MacOS
+Classifier: Operating System :: POSIX
+Classifier: Operating System :: Unix
 Classifier: Natural Language :: English
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
@@ -63,6 +66,7 @@ The database backend exposed by Steady Queue doesn't support the following optio
   result = task.return_value` will result in an error (even if the task has
   completed processing). Instead, we recommend saving the results directly on
   your database or file storage system if they need to be kept around.
+- **Requires a POSIX system** due to the concurrency model based on `os.fork`. Steady Queue doesn't run on windows.
 
 The Steady Queue backend advertises these limitations by [setting the
 corresponding flags][django-tasks-flags] on its backend.

{steady_queue-0.1.8 → steady_queue-0.2.0}/README.md

@@ -37,6 +37,7 @@ The database backend exposed by Steady Queue doesn't support the following optio
   result = task.return_value` will result in an error (even if the task has
   completed processing). Instead, we recommend saving the results directly on
   your database or file storage system if they need to be kept around.
+- **Requires a POSIX system** due to the concurrency model based on `os.fork`. Steady Queue doesn't run on windows.
 
 The Steady Queue backend advertises these limitations by [setting the
 corresponding flags][django-tasks-flags] on its backend.

{steady_queue-0.1.8 → steady_queue-0.2.0}/pyproject.toml

@@ -10,13 +10,17 @@ authors = [
 requires-python = ">=3.12"
 keywords = ["django", "tasks", "async", "jobs", "queues"]
 classifiers = [
+    "Development Status :: 4 - Beta",
     "Framework :: Django",
     "Framework :: Django :: 6.0",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
     "Programming Language :: Python :: 3.14",
-    "Operating System :: OS Independent",
+    "Operating System :: MacOS",
+    "Operating System :: POSIX",
+    "Operating System :: Unix",
     "Natural Language :: English",
+
 ]
 dependencies = [
     "crontab>=1.0.5",

{steady_queue-0.1.8 → steady_queue-0.2.0}/steady_queue/__init__.py

@@ -1,7 +1,7 @@
 from datetime import timedelta
 from typing import Optional
 
-VERSION = (0, 1, 8)
+VERSION = (0, 2, 0)
 
 __version__ = ".".join(map(str, VERSION))
 

{steady_queue-0.1.8 → steady_queue-0.2.0}/steady_queue/configuration.py

@@ -3,9 +3,11 @@ from datetime import timedelta
 from typing import Optional
 
 from crontab import CronTab
+from django.conf import settings
 from django.core.exceptions import ValidationError
 from django.utils.module_loading import import_string
 
+from steady_queue.db_router import steady_queue_database_alias
 from steady_queue.processes.base import Base
 
 
@@ -167,6 +169,7 @@ class Configuration:
     def is_valid(self):
         self.errors = []
         self.errors.extend(self.validate_configured_processes())
+        self.errors.extend(self.validate_database_pool_size())
         self.errors.extend(self.validate_recurring_tasks())
 
         return len(self.errors) == 0
@@ -177,6 +180,39 @@ class Configuration:
 
         return []
 
+    def validate_database_pool_size(self) -> list[ValidationError]:
+        # Match Solid Queue behavior by validating worker thread count against
+        # the queue DB connection pool size when a max_size is explicitly set.
+        if len(self.options.workers) == 0:
+            return []
+
+        db_alias = steady_queue_database_alias()
+        db_config = settings.DATABASES.get(db_alias, {})
+
+        if db_config.get("ENGINE") != "django.db.backends.postgresql":
+            return []
+
+        pool_options = db_config.get("OPTIONS", {}).get("pool")
+        if not isinstance(pool_options, dict):
+            return []
+
+        pool_max_size = pool_options.get("max_size")
+        if not isinstance(pool_max_size, int):
+            return []
+
+        if pool_max_size < self.estimated_number_of_threads:
+            return [
+                ValidationError(
+                    "Steady Queue is configured to use "
+                    f"{self.estimated_number_of_threads} threads but the "
+                    f"database connection pool max_size for '{db_alias}' is "
+                    f"{pool_max_size}. Increase "
+                    f"DATABASES['{db_alias}']['OPTIONS']['pool']['max_size']."
+                )
+            ]
+
+        return []
+
     def validate_recurring_tasks(self) -> list[ValidationError]:
         if self.skip_recurring:
             return []
@@ -195,6 +231,13 @@ class Configuration:
 
         return errors
 
+    @property
+    def estimated_number_of_threads(self) -> int:
+        # At most `threads` in each worker + 2 additional threads (worker loop
+        # and heartbeat), mirroring Solid Queue's sizing heuristic.
+        max_worker_threads = max((w.threads for w in self.options.workers), default=1)
+        return max_worker_threads + 2
+
     @property
     def skip_recurring(self) -> bool:
         return self.options.skip_recurring or self.options.only_work

steady_queue-0.2.0/steady_queue/processes/base.py

@@ -0,0 +1,98 @@
+import logging
+import os
+import secrets
+import socket
+from typing import Any
+
+from django.db import connections
+
+logger = logging.getLogger("steady_queue")
+
+
+class Base:
+    name: str
+    stopped: bool = False
+
+    def __init__(self):
+        self.name = self.generate_name()
+        self.stopped = False
+
+    def boot(self):
+        pass
+
+    def shutdown(self):
+        pass
+
+    def stop(self):
+        self.stopped = True
+
+    @property
+    def kind(self) -> str:
+        return self.__class__.__name__.lower()
+
+    @property
+    def pid(self) -> int:
+        return os.getpid()
+
+    @property
+    def hostname(self) -> str:
+        return socket.gethostname()
+
+    @property
+    def metadata(self) -> dict[str, Any]:
+        return {}
+
+    @property
+    def is_stopped(self) -> bool:
+        return self.stopped
+
+    def generate_name(self) -> str:
+        return "-".join((self.kind, secrets.token_hex(10)))
+
+    def close_postgresql_connection_pools(self):
+        """
+        Close and clear Django's class-level psycopg pool cache.
+
+        Django stores psycopg pools in DatabaseWrapper._connection_pools, so we
+        must clear those references to avoid inheriting stale pools across fork.
+        """
+        closed_pool_maps: set[int] = set()
+
+        for alias in connections:
+            connection = connections[alias]
+            if (
+                connection.settings_dict.get("ENGINE")
+                != "django.db.backends.postgresql"
+            ):
+                continue
+
+            pool_map = getattr(connection.__class__, "_connection_pools", None)
+            if not isinstance(pool_map, dict) or id(pool_map) in closed_pool_maps:
+                continue
+
+            for pool_alias, pool in list(pool_map.items()):
+                try:
+                    pool.close()
+                    logger.debug(
+                        "%(name)s closed psycopg pool for '%(alias)s'",
+                        {"name": self.name, "alias": pool_alias},
+                    )
+                except Exception as e:
+                    logger.debug(
+                        "%(name)s failed to close pool for '%(alias)s': %(e)s",
+                        {"name": self.name, "alias": pool_alias, "e": e},
+                    )
+                finally:
+                    pool_map.pop(pool_alias, None)
+
+            closed_pool_maps.add(id(pool_map))
+
+    def reset_database_connections(self):
+        """
+        Reset database connections for forked processes.
+
+        This closes all current connections and clears Django's class-level
+        psycopg pool cache so child processes don't inherit parent pool state.
+        """
+        connections.close_all()
+        self.close_postgresql_connection_pools()

{steady_queue-0.1.8 → steady_queue-0.2.0}/steady_queue/processes/supervisor.py

@@ -43,6 +43,8 @@ class Supervisor(Maintenance, Signals, Pidfiled, Registrable, Interruptible, Bas
         logger.info("starting supervisor with PID %(pid)d", {"pid": self.pid})
         try:
             self.boot()
+            # Fork only after resetting DB state (connections + psycopg pools).
+            self.reset_database_connections()
             self.start_processes()
             self.launch_maintenance_task()
         except SystemExit:

{steady_queue-0.1.8 → steady_queue-0.2.0/steady_queue.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: steady-queue
-Version: 0.1.8
+Version: 0.2.0
 Summary: An asynchronous task library for Django that supports delayed tasks, concurrency controls, recurring tasks, pausing queues, numeric priorities per task, priorities by queue order and bulk enqueueing.
 Author-email: Elias Hernandis <elias@hernandis.me>
 License-Expression: MIT
@@ -10,12 +10,15 @@ Project-URL: Documentation, https://steady-queue.readthedocs.io/en/latest/
 Project-URL: Tracker, https://github.com/knifecake/steady-queue/issues
 Project-URL: Releases, https://github.com/knifecake/steady-queue/releases
 Keywords: django,tasks,async,jobs,queues
+Classifier: Development Status :: 4 - Beta
 Classifier: Framework :: Django
 Classifier: Framework :: Django :: 6.0
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
-Classifier: Operating System :: OS Independent
+Classifier: Operating System :: MacOS
+Classifier: Operating System :: POSIX
+Classifier: Operating System :: Unix
 Classifier: Natural Language :: English
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
@@ -63,6 +66,7 @@ The database backend exposed by Steady Queue doesn't support the following optio
   result = task.return_value` will result in an error (even if the task has
   completed processing). Instead, we recommend saving the results directly on
   your database or file storage system if they need to be kept around.
+- **Requires a POSIX system** due to the concurrency model based on `os.fork`. Steady Queue doesn't run on windows.
 
 The Steady Queue backend advertises these limitations by [setting the
 corresponding flags][django-tasks-flags] on its backend.

{steady_queue-0.1.8 → steady_queue-0.2.0}/steady_queue.egg-info/SOURCES.txt

@@ -73,6 +73,7 @@ tests/test_concurrency_controls.py
 tests/test_configuration.py
 tests/test_db_router.py
 tests/test_executions.py
+tests/test_fork_safety.py
 tests/test_job.py
 tests/test_pidfile.py
 tests/test_process_lifecycle.py

{steady_queue-0.1.8 → steady_queue-0.2.0}/tests/test_configuration.py

@@ -1,3 +1,4 @@
+import warnings
 from datetime import timedelta
 
 from django.core.exceptions import ValidationError
@@ -153,6 +154,54 @@ class ConfigurationValidationTestCase(SimpleTestCase):
         self.assertGreater(len(config.errors), 0)
         self.assertIn("No processes configured", str(config.errors[0]))
 
+    def test_small_postgres_pool_fails_validation(self):
+        """Configured worker threads must fit in postgres pool max_size."""
+        options = Configuration.Options(workers=[Configuration.Worker(threads=3)])
+        config = Configuration(options)
+
+        with warnings.catch_warnings():
+            warnings.filterwarnings(
+                "ignore",
+                message="Overriding setting DATABASES can lead to unexpected behavior.",
+                category=UserWarning,
+            )
+
+            with self.settings(
+                DATABASES={
+                    "queue": {
+                        "ENGINE": "django.db.backends.postgresql",
+                        "OPTIONS": {"pool": {"min_size": 1, "max_size": 4}},
+                    }
+                }
+            ):
+                self.assertFalse(config.is_valid)
+
+        self.assertTrue(
+            any("pool max_size" in error.message for error in config.errors)
+        )
+
+    def test_sufficient_postgres_pool_passes_validation(self):
+        """Validation passes when postgres pool max_size is large enough."""
+        options = Configuration.Options(workers=[Configuration.Worker(threads=3)])
+        config = Configuration(options)
+
+        with warnings.catch_warnings():
+            warnings.filterwarnings(
+                "ignore",
+                message="Overriding setting DATABASES can lead to unexpected behavior.",
+                category=UserWarning,
+            )
+
+            with self.settings(
+                DATABASES={
+                    "queue": {
+                        "ENGINE": "django.db.backends.postgresql",
+                        "OPTIONS": {"pool": {"min_size": 1, "max_size": 5}},
+                    }
+                }
+            ):
+                self.assertTrue(config.is_valid)
+
     def test_invalid_recurring_task_schedule_fails_validation(self):
         """Invalid cron schedule in recurring task fails validation."""
         invalid_task = Configuration.RecurringTask(

steady_queue-0.2.0/tests/test_fork_safety.py

@@ -0,0 +1,98 @@
+from unittest.mock import patch
+
+from django.test import SimpleTestCase
+
+from steady_queue.configuration import Configuration
+from steady_queue.processes.base import Base
+from steady_queue.processes.supervisor import Supervisor
+
+
+class SupervisorForkSafetyTest(SimpleTestCase):
+    def build_supervisor(self) -> Supervisor:
+        options = Configuration.Options(
+            workers=[],
+            dispatchers=[],
+            recurring_tasks=[],
+            skip_recurring=True,
+        )
+        return Supervisor(Configuration(options))
+
+    def test_supervisor_start_resets_connections_before_forking(self):
+        supervisor = self.build_supervisor()
+        calls = []
+
+        supervisor.boot = lambda: calls.append("boot")
+        supervisor.reset_database_connections = lambda: calls.append("reset")
+        supervisor.start_processes = lambda: calls.append("start_processes")
+        supervisor.launch_maintenance_task = lambda: calls.append("launch_maintenance")
+        supervisor.supervise = lambda: calls.append("supervise")
+
+        supervisor.start()
+
+        self.assertEqual(
+            calls,
+            [
+                "boot",
+                "reset",
+                "start_processes",
+                "launch_maintenance",
+                "supervise",
+            ],
+        )
+
+
+class ResetDatabaseConnectionsTest(SimpleTestCase):
+    def test_reset_connections_clears_psycopg_pool_cache_without_disabling_pooling(
+        self,
+    ):
+        class FakePool:
+            def __init__(self):
+                self.closed = False
+
+            def close(self):
+                self.closed = True
+
+        class FakeConnection:
+            _connection_pools = {}
+
+            def __init__(self, engine: str):
+                self.settings_dict = {
+                    "ENGINE": engine,
+                    "OPTIONS": {"pool": {"min_size": 1, "max_size": 4}},
+                }
+
+        class FakeConnections(dict):
+            close_all_called = False
+
+            def __iter__(self):
+                return iter(self.keys())
+
+            def close_all(self):
+                self.close_all_called = True
+
+        pool_default = FakePool()
+        pool_queue = FakePool()
+        FakeConnection._connection_pools = {
+            "default": pool_default,
+            "queue": pool_queue,
+        }
+
+        fake_connections = FakeConnections(
+            {
+                "default": FakeConnection("django.db.backends.postgresql"),
+                "queue": FakeConnection("django.db.backends.postgresql"),
+                "sqlite": FakeConnection("django.db.backends.sqlite3"),
+            }
+        )
+
+        with patch("steady_queue.processes.base.connections", fake_connections):
+            Base().reset_database_connections()
+
+        self.assertIn("pool", fake_connections["default"].settings_dict["OPTIONS"])
+        self.assertIn("pool", fake_connections["queue"].settings_dict["OPTIONS"])
+        self.assertIn("pool", fake_connections["sqlite"].settings_dict["OPTIONS"])
+
+        self.assertTrue(fake_connections.close_all_called)
+        self.assertTrue(pool_default.closed)
+        self.assertTrue(pool_queue.closed)
+        self.assertEqual(FakeConnection._connection_pools, {})

steady_queue-0.1.8/steady_queue/processes/base.py

@@ -1,102 +0,0 @@
-import os
-import secrets
-import socket
-from typing import Any
-
-from django.db import connections
-
-
-class Base:
-    name: str
-    stopped: bool = False
-
-    def __init__(self):
-        self.name = self.generate_name()
-        self.stopped = False
-
-    def boot(self):
-        pass
-
-    def shutdown(self):
-        pass
-
-    def stop(self):
-        self.stopped = True
-
-    @property
-    def kind(self) -> str:
-        return self.__class__.__name__.lower()
-
-    @property
-    def pid(self) -> int:
-        return os.getpid()
-
-    @property
-    def hostname(self) -> str:
-        return socket.gethostname()
-
-    @property
-    def metadata(self) -> dict[str, Any]:
-        return {}
-
-    @property
-    def is_stopped(self) -> bool:
-        return self.stopped
-
-    def generate_name(self) -> str:
-        return "-".join((self.kind, secrets.token_hex(10)))
-
-    def disable_connection_pooling(self):
-        """
-        Disable connection pooling for steady_queue processes.
-
-        Connection pooling with psycopg doesn't work with forked processes.
-        This method removes pool configuration from database settings to prevent
-        pool-related errors in steady_queue workers.
-        """
-        import logging
-
-        from django.conf import settings
-
-        logger = logging.getLogger("steady_queue")
-
-        # Disable pooling in database configuration
-        if hasattr(settings, "DATABASES"):
-            for alias, db_config in settings.DATABASES.items():
-                if db_config.get("ENGINE") == "django.db.backends.postgresql":
-                    # Remove pool configuration if it exists
-                    options = db_config.setdefault("OPTIONS", {})
-                    if "pool" in options:
-                        logger.info(
-                            "%(name)s disabling connection pooling for database '%(alias)s'",
-                            {"name": self.name, "alias": alias},
-                        )
-                        del options["pool"]
-
-        # Also disable on any existing connections
-        for alias in connections:
-            connection = connections[alias]
-            if hasattr(connection, "pool") and connection.pool is not None:
-                try:
-                    connection.pool.close()
-                    connection.pool = None
-                    logger.debug(
-                        "%(name)s removed existing pool for '%(alias)s'",
-                        {"name": self.name, "alias": alias},
-                    )
-                except Exception as e:
-                    logger.debug(
-                        "%(name)s failed to close existing pool for '%(alias)s': %(e)s",
-                        {"name": self.name, "alias": alias, "e": e},
-                    )
-
-    def reset_database_connections(self):
-        """
-        Reset database connections for forked processes.
-
-        This disables connection pooling and resets connection state to prevent
-        issues with shared connections between parent and child processes.
-        """
-        self.disable_connection_pooling()
-
-        connections.close_all()