p1-taskqueue 0.1.31__tar.gz → 0.2.0__tar.gz

{p1_taskqueue-0.1.31 → p1_taskqueue-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: p1-taskqueue
-Version: 0.1.31
+Version: 0.2.0
 Summary: A Task Queue Wrapper for Dekoruma Backend
 Author-email: Chalvin <engineering@dekoruma.com>
 Project-URL: Homepage, https://github.com/Dekoruma/p1-taskqueue

{p1_taskqueue-0.1.31 → p1_taskqueue-0.2.0}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 [project]
 name = "p1-taskqueue"
 # DO NOT CHANGE THIS VERSION - it gets automatically replaced by CI/CD with the git tag version
-version = "0.1.31"
+version = "0.2.0"
 description = "A Task Queue Wrapper for Dekoruma Backend"
 authors = [
     {name = "Chalvin", email = "engineering@dekoruma.com"}

{p1_taskqueue-0.1.31 → p1_taskqueue-0.2.0}/src/p1_taskqueue.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: p1-taskqueue
-Version: 0.1.31
+Version: 0.2.0
 Summary: A Task Queue Wrapper for Dekoruma Backend
 Author-email: Chalvin <engineering@dekoruma.com>
 Project-URL: Homepage, https://github.com/Dekoruma/p1-taskqueue

{p1_taskqueue-0.1.31 → p1_taskqueue-0.2.0}/src/p1_taskqueue.egg-info/SOURCES.txt

@@ -9,10 +9,15 @@ src/taskqueue/__init__.py
 src/taskqueue/apps.py
 src/taskqueue/celery_app.py
 src/taskqueue/cmanager.py
+src/taskqueue/dlq.py
 src/taskqueue/models.py
 src/taskqueue/slack_notifier.py
 src/taskqueue/libs/__init__.py
 src/taskqueue/libs/helper_test.py
+src/taskqueue/management/__init__.py
+src/taskqueue/management/commands/__init__.py
+src/taskqueue/management/commands/retry_dlq_messages.py
+src/taskqueue/management/commands/setup_taskqueue.py
 src/taskqueue/migrations/0001_initial.py
 src/taskqueue/migrations/0002_add_channel_to_taskreconstruction.py
 src/taskqueue/migrations/0003_alter_taskreconstruction_id.py
@@ -20,5 +25,7 @@ src/taskqueue/migrations/0004_add_callable_name_meta_to_taskreconstruction.py
 src/taskqueue/migrations/__init__.py
 tests/test_celery_app.py
 tests/test_cmanager.py
+tests/test_dlq.py
 tests/test_helper_test_functions.py
+tests/test_management_command.py
 tests/test_test_utils.py

{p1_taskqueue-0.1.31 → p1_taskqueue-0.2.0}/src/taskqueue/celery_app.py

@@ -36,8 +36,10 @@ def create_celery_app():
         'result_serializer': getattr(settings, 'CELERY_RESULT_SERIALIZER', 'pickle'),
         'accept_content': getattr(settings, 'CELERY_ACCEPT_CONTENT', ['pickle']),
         'timezone': getattr(settings, 'CELERY_TIMEZONE', 'UTC+7'),
-        'task_time_limit': getattr(settings, 'CELERY_TASK_TIME_LIMIT', 30 * 60),
-        'task_soft_time_limit': getattr(settings, 'CELERY_TASK_SOFT_TIME_LIMIT', 25 * 60),
+        'task_time_limit': getattr(settings, 'CELERY_TASK_TIME_LIMIT', 10 * 60),
+        'task_soft_time_limit': getattr(
+            settings, 'CELERY_TASK_SOFT_TIME_LIMIT', 8 * 60
+        ),
         # 14 days
         'result_expires': getattr(settings, 'CELERY_RESULT_EXPIRES', 14 * 24 * 60 * 60),
         'task_track_started': True,
@@ -54,22 +56,21 @@ def create_celery_app():
         'broker_pool_limit': 2,
     }
 
+    configure_queues(settings, celery_config)
     app.conf.update(celery_config)
     app.autodiscover_tasks(['taskqueue'])
 
     return app
 
 
-def setup_queues(app, settings):
+def _build_queue_setup(settings):
     app_name = getattr(settings, 'TASKQUEUE_APP_NAME', 'taskqueue')
     queue_names = getattr(settings, 'TASKQUEUE_QUEUES',
                           ['default', 'high', 'low'])
-    if queue_names is None:
-        queue_names = ['default', 'high', 'low']
     dlq_name_prefix = getattr(settings, 'TASKQUEUE_DLQ_NAME_PREFIX', 'dlq')
 
     logger.info(
-        f"[TaskQueue] Configuring app: {app_name}, queues: {queue_names}")
+        f'[TaskQueue] Configuring app: {app_name}, queues: {queue_names}')
 
     main_exchange = Exchange(app_name, type='direct')
     dlx_exchange = Exchange(f'{app_name}.dlx', type='direct')
@@ -78,68 +79,83 @@ def setup_queues(app, settings):
 
     for queue_name in queue_names:
         dlq_name = f'{dlq_name_prefix}.{queue_name}'
-        dlx_name = f'{app_name}.dlx'
-
-        queue_args = {
-            'x-dead-letter-exchange': dlx_name,
-            'x-dead-letter-routing-key': dlq_name
-        }
 
         queue = Queue(
             queue_name,
             main_exchange,
             routing_key=queue_name,
-            queue_arguments=queue_args
+            queue_arguments={
+                'x-dead-letter-exchange': dlx_exchange.name,
+                'x-dead-letter-routing-key': dlq_name,
+            },
         )
         queues.append(queue)
         logger.info(
-            f"[TaskQueue] Queue '{queue_name}' configured with DLX: {dlx_name}, DLQ routing key: {dlq_name}")
+            f"[TaskQueue] Queue '{queue_name}' configured with DLX: {dlx_exchange.name}, DLQ routing key: {dlq_name}"
+        )
 
-    for queue_name in queue_names:
-        dlq_name = f'{dlq_name_prefix}.{queue_name}'
         dlq = Queue(dlq_name, dlx_exchange, routing_key=dlq_name)
         queues.append(dlq)
         logger.info(f"[TaskQueue] DLQ '{dlq_name}' configured")
 
-    app.conf.update({
-        'task_default_queue': 'default',
-        'task_default_exchange': app_name,
-        'task_default_exchange_type': 'direct',
-        'task_queues': tuple(queues),
-    })
+    return app_name, queues, main_exchange, dlx_exchange
 
-    try:
-        with app.connection_or_acquire() as conn:
-            channel = conn.default_channel
 
-            try:
-                main_exchange.declare(channel=channel)
-                logger.info(f"[TaskQueue] Exchange declared: {app_name}")
-            except PreconditionFailed:
-                logger.info(f"[TaskQueue] Exchange already exists: {app_name}")
+def configure_queues(settings, celery_config):
+    app_name, queues, _, _ = _build_queue_setup(settings)
+    celery_config.update(
+        {
+            'task_default_queue': 'default',
+            'task_default_exchange': app_name,
+            'task_default_exchange_type': 'direct',
+            'task_queues': queues,
+        }
+    )
+
+
+def setup_queues(app=None, settings=None):
+    if settings is None:
+        settings = get_django_settings()
+    if app is None:
+        app = celery_app
+
+    app_name, queues, main_exchange, dlx_exchange = _build_queue_setup(
+        settings)
+    app.conf.update(
+        {
+            'task_default_queue': 'default',
+            'task_default_exchange': app_name,
+            'task_default_exchange_type': 'direct',
+            'task_queues': queues,
+        }
+    )
+
+    with app.connection_or_acquire() as conn:
+        channel = conn.default_channel
+
+        try:
+            main_exchange.declare(channel=channel)
+            logger.info(f"[TaskQueue] Exchange declared: {app_name}")
+        except PreconditionFailed:
+            logger.info(f"[TaskQueue] Exchange already exists: {app_name}")
 
+        try:
+            dlx_exchange.declare(channel=channel)
+            logger.info(f"[TaskQueue] DLX Exchange declared: {app_name}.dlx")
+        except PreconditionFailed:
+            logger.info(
+                f"[TaskQueue] DLX Exchange already exists: {app_name}.dlx")
+
+        for queue in queues:
             try:
-                dlx_exchange.declare(channel=channel)
-                logger.info(
-                    f"[TaskQueue] DLX Exchange declared: {app_name}.dlx")
+                queue.declare(channel=channel)
+                logger.info(f"[TaskQueue] Queue declared: {queue.name}")
             except PreconditionFailed:
                 logger.info(
-                    f"[TaskQueue] DLX Exchange already exists: {app_name}.dlx")
-
-            for queue in queues:
-                try:
-                    queue.declare(channel=channel)
-                    logger.info(f"[TaskQueue] Queue declared: {queue.name}")
-                except PreconditionFailed:
-                    logger.info(
-                        f"[TaskQueue] Queue already exists with different config: {queue.name}. Using existing queue.")
-                except Exception as e:
-                    logger.warning(
-                        f"[TaskQueue] Failed to declare queue {queue.name}: {e}")
-
-    except Exception as e:
-        logger.warning(
-            f"[TaskQueue] Failed to setup queues: {str(e.__class__.__name__)} {e}")
+                    f"[TaskQueue] Queue already exists with different config: {queue.name}. Using existing queue."
+                )
+
+    return queues
 
 
 celery_app = create_celery_app()

p1_taskqueue-0.2.0/src/taskqueue/dlq.py

@@ -0,0 +1,192 @@
+import logging
+from datetime import datetime
+from datetime import timezone
+from typing import Any
+from typing import Dict
+from typing import Optional
+from typing import Tuple
+
+from kombu import Producer
+from taskqueue.cmanager import K_TASK_STATUS_REPUBLISHED
+
+logger = logging.getLogger(__name__)
+
+K_DEFAULT_MAX_ETA_DAYS = 3
+K_DEFAULT_MAX_RETENTION_DAYS = 14
+K_SUPPORTED_PUBLISH_PROPERTIES = (
+    "correlation_id",
+    "reply_to",
+    "expiration",
+    "message_id",
+    "timestamp",
+    "type",
+    "user_id",
+    "app_id",
+    "cluster_id",
+    "delivery_mode",
+    "priority",
+)
+
+
+def _get_celery_app() -> Any:
+    from taskqueue.celery_app import celery_app
+
+    return celery_app
+
+
+def _get_task_result_model():
+    from django_celery_results.models import TaskResult
+
+    return TaskResult
+
+
+def _get_retry_limits() -> Tuple[int, int]:
+    from django.conf import settings
+
+    max_eta_days = getattr(
+        settings, "TASKQUEUE_DLQ_RETRY_MAX_ETA_DAYS", K_DEFAULT_MAX_ETA_DAYS
+    )
+    max_retention_days = getattr(
+        settings, "TASKQUEUE_DLQ_MAX_RETENTION_DAYS", K_DEFAULT_MAX_RETENTION_DAYS
+    )
+    return max_eta_days, max_retention_days
+
+
+def _extract_publish_properties(message: Any) -> Dict[str, Any]:
+    properties = getattr(message, "properties", {}) or {}
+    return {
+        key: properties[key]
+        for key in K_SUPPORTED_PUBLISH_PROPERTIES
+        if key in properties
+    }
+
+
+def retry_dlq_messages(
+    dlq_name: str, main_queue_name: str, app: Any = None
+) -> Dict[str, int]:
+    app = app or _get_celery_app()
+
+    results = {
+        "moved": 0,
+        "skipped": 0,
+        "discarded": 0,
+    }
+
+    with app.connection_or_acquire() as conn:
+        channel = conn.default_channel
+
+        dlq_factory = app.amqp.queues.get(dlq_name)
+        if dlq_factory is None:
+            raise ValueError(f"DLQ '{dlq_name}' is not configured")
+
+        main_queue_factory = app.amqp.queues.get(main_queue_name)
+        if main_queue_factory is None:
+            raise ValueError(f"Queue '{main_queue_name}' is not configured")
+
+        dlq_queue = dlq_factory(channel)
+        main_queue = main_queue_factory(channel)
+        producer = Producer(channel)
+
+        while True:
+            message = dlq_queue.get(no_ack=False)
+            if not message:
+                break
+
+            headers = message.headers or {}
+
+            discard_info = get_discard_info(headers)
+            if discard_info:
+                task_name, task_id = discard_info
+                logger.warning(
+                    f"[DLQRetry] Discarding task: {task_name} (ID: {task_id})"
+                )
+                message.ack()
+                results["discarded"] += 1
+                continue
+
+            if not should_retry_message(headers):
+                logger.warning(
+                    f"[DLQRetry] Message from {dlq_name} is not eligible for retry"
+                )
+                message.ack()
+                results["skipped"] += 1
+                continue
+
+            producer.publish(
+                message.body,
+                exchange=main_queue.exchange,
+                routing_key=main_queue.routing_key,
+                headers=headers,
+                content_type=message.content_type,
+                content_encoding=message.content_encoding,
+                declare=[main_queue],
+                retry=True,
+                **_extract_publish_properties(message),
+            )
+            message.ack()
+            results["moved"] += 1
+
+    logger.info(
+        f"[DLQRetry] Moved {results['moved']} messages from {dlq_name} to {main_queue_name}"
+    )
+    return results
+
+
+def should_retry_message(headers: Dict[str, Any]) -> bool:
+    try:
+        task_id = headers.get("id")
+
+        if task_id:
+            try:
+                TaskResult = _get_task_result_model()
+                task_result = TaskResult.objects.filter(
+                    task_id=task_id).first()
+                if task_result and task_result.status == K_TASK_STATUS_REPUBLISHED:
+                    logger.info(
+                        f"[DLQRetry] Task {task_id} has status '{K_TASK_STATUS_REPUBLISHED}', skipping retry"
+                    )
+                    return False
+            except Exception as e:
+                logger.warning(
+                    f"[DLQRetry] Error checking TaskResult for task {task_id}: {e.__class__.__name__}: {e}"
+                )
+
+        eta = headers.get("eta")
+        if not eta:
+            logger.warning(
+                f"[DLQRetry] Message {task_id} has no ETA, skipping retry")
+            return False
+
+        eta_datetime = datetime.fromisoformat(eta.replace("Z", "+00:00"))
+        current_datetime = datetime.now(timezone.utc)
+        max_eta_days, _ = _get_retry_limits()
+        days_difference = (current_datetime - eta_datetime).days
+
+        return days_difference <= max_eta_days
+    except Exception as e:
+        logger.error(
+            f"[DLQRetry] Error checking ETA: {e.__class__.__name__}: {e}")
+        return False
+
+
+def get_discard_info(headers: Dict[str, Any]) -> Optional[Tuple[str, str]]:
+    try:
+        eta = headers.get("eta")
+        if not eta:
+            return None
+
+        eta_datetime = datetime.fromisoformat(eta.replace("Z", "+00:00"))
+        current_datetime = datetime.now(timezone.utc)
+        _, max_retention_days = _get_retry_limits()
+        days_difference = (current_datetime - eta_datetime).days
+
+        if days_difference > max_retention_days:
+            task_name = headers.get("task", "unknown")
+            task_id = headers.get("id", "unknown")
+            return task_name, task_id
+
+        return None
+    except Exception as e:
+        logger.error(
+            f"[DLQRetry] Error checking ETA: {e.__class__.__name__}: {e}")
+        return None

p1_taskqueue-0.2.0/src/taskqueue/management/commands/retry_dlq_messages.py

@@ -0,0 +1,33 @@
+import logging
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+from taskqueue.dlq import retry_dlq_messages
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    help = "Retry messages from TaskQueue dead letter queues back to the main queues."
+
+    def handle(self, *args, **options):
+        logger.info("[DLQRetry] Starting DLQ retry process")
+
+        queue_names = getattr(settings, "TASKQUEUE_QUEUES", [
+                              "default", "high", "low"])
+        dlq_prefix = getattr(settings, "TASKQUEUE_DLQ_NAME_PREFIX", "dlq")
+
+        total_moved = 0
+        total_skipped = 0
+        total_discarded = 0
+
+        for queue_name in queue_names:
+            dlq_name = f"{dlq_prefix}.{queue_name}"
+            result = retry_dlq_messages(dlq_name, queue_name)
+            total_moved += result["moved"]
+            total_skipped += result["skipped"]
+            total_discarded += result["discarded"]
+
+        logger.info("[DLQRetry] Finished DLQ retry process")
+        logger.info(
+            f'Retried {total_moved} messages, skipped {total_skipped}, discarded {total_discarded}.')

p1_taskqueue-0.2.0/src/taskqueue/management/commands/setup_taskqueue.py

@@ -0,0 +1,14 @@
+import logging
+
+from django.core.management.base import BaseCommand
+from taskqueue.celery_app import setup_queues
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    help = "Declare TaskQueue exchanges and queues on the configured broker."
+
+    def handle(self, *args, **options):
+        queues = setup_queues()
+        logger.info(f"Declared {len(queues)} TaskQueue queues successfully.")

p1_taskqueue-0.2.0/tests/test_celery_app.py

@@ -0,0 +1,225 @@
+"""Tests for the celery_app module."""
+from unittest.mock import MagicMock
+from unittest.mock import patch
+
+import pytest
+from taskqueue.celery_app import configure_queues
+from taskqueue.celery_app import create_celery_app
+from taskqueue.celery_app import get_django_settings
+from taskqueue.celery_app import setup_queues
+
+
+class TestGetDjangoSettings:
+    def test_get_django_settings_given_django_available_expect_return_settings(self):
+        with patch("django.conf.settings") as mock_settings:
+            result = get_django_settings()
+            assert result == mock_settings
+
+    def test_get_django_settings_given_django_not_available_expect_raise_import_error(
+        self,
+    ):
+        with patch(
+            "builtins.__import__", side_effect=ImportError("No module named 'django'")
+        ):
+            with pytest.raises(
+                ImportError, match="\\[TaskQueue\\] Django settings not found\\."
+            ):
+                get_django_settings()
+
+
+class TestCreateCeleryApp:
+    @patch("taskqueue.celery_app.configure_queues")
+    @patch("taskqueue.celery_app.setup_queues")
+    @patch("taskqueue.celery_app.get_django_settings")
+    @patch("taskqueue.celery_app.Celery")
+    def test_create_celery_app_given_valid_settings_expect_celery_app_created(
+        self,
+        mock_celery_class,
+        mock_get_settings,
+        mock_setup_queues,
+        mock_configure_queues,
+    ):
+        mock_settings = MagicMock()
+        mock_settings.TASKQUEUE_APP_NAME = "testapp"
+        mock_settings.CELERY_BROKER_URL = "redis://localhost:6379/0"
+        mock_settings.CELERY_RESULT_BACKEND = "redis://localhost:6379/0"
+        mock_settings.CELERY_TASK_SERIALIZER = "pickle"
+        mock_settings.CELERY_RESULT_SERIALIZER = "pickle"
+        mock_settings.CELERY_ACCEPT_CONTENT = ["pickle"]
+        mock_settings.CELERY_TIMEZONE = "UTC"
+        mock_settings.CELERY_TASK_TRACK_STARTED = True
+        mock_settings.CELERY_TASK_TIME_LIMIT = 1800
+        mock_settings.CELERY_TASK_SOFT_TIME_LIMIT = 1500
+        mock_settings.CELERY_TASK_ALWAYS_EAGER = False
+        mock_settings.CELERY_TASK_EAGER_PROPAGATES = True
+
+        mock_get_settings.return_value = mock_settings
+        mock_app = MagicMock()
+        mock_celery_class.return_value = mock_app
+
+        result = create_celery_app()
+
+        mock_celery_class.assert_called_once_with("testapp")
+        mock_configure_queues.assert_called_once()
+        mock_setup_queues.assert_not_called()
+        mock_app.conf.update.assert_called_once()
+        mock_app.autodiscover_tasks.assert_called_once_with(["taskqueue"])
+        assert result == mock_app
+
+    @patch("taskqueue.celery_app.configure_queues")
+    @patch("taskqueue.celery_app.setup_queues")
+    @patch("taskqueue.celery_app.get_django_settings")
+    @patch("taskqueue.celery_app.Celery")
+    def test_create_celery_app_given_missing_settings_expect_defaults_used(
+        self,
+        mock_celery_class,
+        mock_get_settings,
+        mock_setup_queues,
+        mock_configure_queues,
+    ):
+        mock_settings = MagicMock()
+        del mock_settings.TASKQUEUE_APP_NAME
+        del mock_settings.CELERY_BROKER_URL
+        del mock_settings.CELERY_RESULT_BACKEND
+        del mock_settings.CELERY_TASK_SERIALIZER
+        del mock_settings.CELERY_RESULT_SERIALIZER
+        del mock_settings.CELERY_ACCEPT_CONTENT
+        del mock_settings.CELERY_TIMEZONE
+        del mock_settings.CELERY_TASK_TRACK_STARTED
+        del mock_settings.CELERY_TASK_TIME_LIMIT
+        del mock_settings.CELERY_TASK_SOFT_TIME_LIMIT
+        del mock_settings.CELERY_TASK_ALWAYS_EAGER
+        del mock_settings.CELERY_TASK_EAGER_PROPAGATES
+
+        mock_get_settings.return_value = mock_settings
+        mock_app = MagicMock()
+        mock_celery_class.return_value = mock_app
+
+        result = create_celery_app()
+
+        mock_celery_class.assert_called_once_with("taskqueue")
+        mock_configure_queues.assert_called_once()
+        mock_setup_queues.assert_not_called()
+        assert result == mock_app
+
+
+class TestConfigureQueues:
+    """Test configure_queues function."""
+
+    def test_configure_queues_given_valid_settings_expect_queues_configured(self):
+        mock_settings = MagicMock()
+        mock_settings.TASKQUEUE_APP_NAME = "testapp"
+        mock_settings.TASKQUEUE_QUEUES = ["default", "high", "low"]
+        mock_settings.TASKQUEUE_DLQ_NAME_PREFIX = "dlq"
+
+        celery_config = {}
+
+        configure_queues(mock_settings, celery_config)
+
+        assert celery_config["task_default_queue"] == "default"
+        assert celery_config["task_default_exchange"] == "testapp"
+        assert celery_config["task_default_exchange_type"] == "direct"
+        assert len(celery_config["task_queues"]) == 6
+
+        main_queues = [
+            q for q in celery_config["task_queues"] if not q.name.startswith("dlq.")
+        ]
+        assert len(main_queues) == 3
+        assert any(q.name == "default" for q in main_queues)
+        assert any(q.name == "high" for q in main_queues)
+        assert any(q.name == "low" for q in main_queues)
+
+        dlq_queues = [
+            q for q in celery_config["task_queues"] if q.name.startswith("dlq.")
+        ]
+        assert len(dlq_queues) == 3
+        assert any(q.name == "dlq.default" for q in dlq_queues)
+        assert any(q.name == "dlq.high" for q in dlq_queues)
+        assert any(q.name == "dlq.low" for q in dlq_queues)
+
+    def test_configure_queues_given_missing_settings_expect_defaults_used(self):
+        mock_settings = MagicMock()
+        del mock_settings.TASKQUEUE_APP_NAME
+        del mock_settings.TASKQUEUE_QUEUES
+        del mock_settings.TASKQUEUE_DLQ_NAME_PREFIX
+
+        celery_config = {}
+
+        configure_queues(mock_settings, celery_config)
+
+        assert celery_config["task_default_queue"] == "default"
+        assert celery_config["task_default_exchange"] == "taskqueue"
+        assert celery_config["task_default_exchange_type"] == "direct"
+        assert len(celery_config["task_queues"]) == 6
+
+    def test_configure_queues_given_single_queue_expect_correct_configuration(self):
+        mock_settings = MagicMock()
+        mock_settings.TASKQUEUE_APP_NAME = "singleapp"
+        mock_settings.TASKQUEUE_QUEUES = ["single"]
+        mock_settings.TASKQUEUE_DLQ_NAME_PREFIX = "dead"
+
+        celery_config = {}
+
+        configure_queues(mock_settings, celery_config)
+
+        assert len(celery_config["task_queues"]) == 2
+
+        main_queue = next(
+            q for q in celery_config["task_queues"] if q.name == "single")
+        assert main_queue.queue_arguments["x-dead-letter-exchange"] == "singleapp.dlx"
+        assert main_queue.queue_arguments["x-dead-letter-routing-key"] == "dead.single"
+
+        dlq = next(
+            q for q in celery_config["task_queues"] if q.name == "dead.single")
+        assert dlq.exchange.name == "singleapp.dlx"
+
+
+class TestSetupQueues:
+    @patch("taskqueue.celery_app.Queue.declare")
+    @patch("taskqueue.celery_app.Exchange.declare")
+    def test_setup_queues_given_valid_settings_expect_broker_declaration_runs(
+        self, mock_exchange_declare, mock_queue_declare
+    ):
+        mock_app = MagicMock()
+        mock_settings = MagicMock()
+        mock_settings.TASKQUEUE_APP_NAME = "testapp"
+        mock_settings.TASKQUEUE_QUEUES = ["default", "high", "low"]
+        mock_settings.TASKQUEUE_DLQ_NAME_PREFIX = "dlq"
+
+        queues = setup_queues(mock_app, mock_settings)
+
+        mock_app.connection_or_acquire.assert_called_once()
+        mock_app.conf.update.assert_called_once()
+        assert len(queues) == 6
+        assert mock_exchange_declare.call_count == 2
+        assert mock_queue_declare.call_count == 6
+
+
+class TestCeleryAppIntegration:
+    """Integration tests for the celery_app module."""
+
+    @patch("taskqueue.celery_app.get_django_settings")
+    def test_celery_app_import_given_django_configured_expect_app_created(
+        self, mock_get_settings
+    ):
+        mock_settings = MagicMock()
+        mock_settings.TASKQUEUE_APP_NAME = "testapp"
+        mock_settings.CELERY_BROKER_URL = "redis://localhost:6379/0"
+        mock_settings.CELERY_RESULT_BACKEND = "redis://localhost:6379/0"
+        mock_settings.CELERY_TASK_SERIALIZER = "pickle"
+        mock_settings.CELERY_RESULT_SERIALIZER = "pickle"
+        mock_settings.CELERY_ACCEPT_CONTENT = ["pickle"]
+        mock_settings.CELERY_TIMEZONE = "UTC"
+        mock_settings.CELERY_TASK_TRACK_STARTED = True
+        mock_settings.CELERY_TASK_TIME_LIMIT = 1800
+        mock_settings.CELERY_TASK_SOFT_TIME_LIMIT = 1500
+        mock_settings.CELERY_TASK_ALWAYS_EAGER = False
+        mock_settings.CELERY_TASK_EAGER_PROPAGATES = True
+        mock_settings.TASKQUEUE_QUEUES = ["default"]
+        mock_settings.TASKQUEUE_DLQ_NAME_PREFIX = "dlq"
+
+        mock_get_settings.return_value = mock_settings
+
+        from taskqueue.celery_app import celery_app
+
+        assert celery_app is not None

p1_taskqueue-0.2.0/tests/test_dlq.py

@@ -0,0 +1,240 @@
+from datetime import datetime
+from datetime import timedelta
+from datetime import timezone
+from unittest.mock import MagicMock
+from unittest.mock import patch
+
+from taskqueue.cmanager import K_TASK_STATUS_REPUBLISHED
+from taskqueue.dlq import get_discard_info
+from taskqueue.dlq import retry_dlq_messages
+from taskqueue.dlq import should_retry_message
+from taskqueue.management.commands.retry_dlq_messages import Command
+
+
+class TestShouldRetryMessage:
+    @patch("taskqueue.dlq._get_task_result_model")
+    def test_should_retry_message_given_republished_task_expect_false(
+        self, mock_get_task_result_model
+    ):
+        mock_task_result_model = MagicMock()
+        mock_task_result_model.objects.filter.return_value.first.return_value = (
+            MagicMock(status=K_TASK_STATUS_REPUBLISHED)
+        )
+        mock_get_task_result_model.return_value = mock_task_result_model
+        eta = (datetime.now(timezone.utc) - timedelta(days=1)).isoformat()
+
+        result = should_retry_message({"id": "task-1", "eta": eta})
+
+        assert result is False
+
+    @patch("taskqueue.dlq._get_task_result_model")
+    def test_should_retry_message_given_eta_within_limit_expect_true(
+        self, mock_get_task_result_model
+    ):
+        mock_task_result_model = MagicMock()
+        mock_task_result_model.objects.filter.return_value.first.return_value = None
+        mock_get_task_result_model.return_value = mock_task_result_model
+        eta = (datetime.now(timezone.utc) - timedelta(days=2)).isoformat()
+
+        result = should_retry_message({"id": "task-1", "eta": eta})
+
+        assert result is True
+
+    @patch("taskqueue.dlq._get_task_result_model")
+    def test_should_retry_message_given_eta_older_than_limit_expect_false(
+        self, mock_get_task_result_model
+    ):
+        mock_task_result_model = MagicMock()
+        mock_task_result_model.objects.filter.return_value.first.return_value = None
+        mock_get_task_result_model.return_value = mock_task_result_model
+        eta = (datetime.now(timezone.utc) - timedelta(days=8)).isoformat()
+
+        result = should_retry_message({"id": "task-1", "eta": eta})
+
+        assert result is False
+
+    @patch("taskqueue.dlq._get_task_result_model")
+    def test_should_retry_message_given_missing_eta_expect_false(
+        self, mock_get_task_result_model
+    ):
+        mock_task_result_model = MagicMock()
+        mock_task_result_model.objects.filter.return_value.first.return_value = None
+        mock_get_task_result_model.return_value = mock_task_result_model
+
+        result = should_retry_message({"id": "task-1"})
+
+        assert result is False
+
+
+class TestGetDiscardInfo:
+    def test_get_discard_info_given_eta_beyond_retention_expect_discard_info(self):
+        eta = (datetime.now(timezone.utc) - timedelta(days=20)).isoformat()
+
+        result = get_discard_info(
+            {
+                "id": "task-1",
+                "task": "my.task",
+                "eta": eta,
+            }
+        )
+
+        assert result == ("my.task", "task-1")
+
+    def test_get_discard_info_given_eta_within_retention_expect_none(self):
+        eta = (datetime.now(timezone.utc) - timedelta(days=5)).isoformat()
+
+        result = get_discard_info(
+            {
+                "id": "task-1",
+                "task": "my.task",
+                "eta": eta,
+            }
+        )
+
+        assert result is None
+
+
+class TestRetryDlqMessages:
+    @patch("taskqueue.dlq.Producer")
+    @patch("taskqueue.dlq.get_discard_info", return_value=None)
+    @patch("taskqueue.dlq.should_retry_message", return_value=True)
+    def test_retry_dlq_messages_given_retryable_message_expect_republished(
+        self,
+        mock_should_retry_message,
+        mock_get_discard_info,
+        mock_producer_class,
+    ):
+        eta = (datetime.now(timezone.utc) - timedelta(days=1)).isoformat()
+        mock_message = MagicMock()
+        mock_message.headers = {"id": "task-1", "eta": eta}
+        mock_message.body = b"raw-payload"
+        mock_message.content_type = "application/json"
+        mock_message.content_encoding = "utf-8"
+        mock_message.properties = {"priority": 5}
+
+        mock_dlq_queue = MagicMock()
+        mock_dlq_queue.get.side_effect = [mock_message, None]
+
+        mock_main_queue = MagicMock()
+        mock_main_queue.exchange = "test_taskqueue"
+        mock_main_queue.routing_key = "default"
+
+        mock_app = MagicMock()
+        mock_app.amqp.queues = {
+            "dlq.default": MagicMock(return_value=mock_dlq_queue),
+            "default": MagicMock(return_value=mock_main_queue),
+        }
+        mock_app.connection_or_acquire.return_value.__enter__.return_value.default_channel = "channel"
+
+        result = retry_dlq_messages("dlq.default", "default", app=mock_app)
+
+        mock_should_retry_message.assert_called_once_with(mock_message.headers)
+        mock_get_discard_info.assert_called_once_with(mock_message.headers)
+        mock_producer_class.return_value.publish.assert_called_once_with(
+            b"raw-payload",
+            exchange="test_taskqueue",
+            routing_key="default",
+            headers=mock_message.headers,
+            content_type="application/json",
+            content_encoding="utf-8",
+            declare=[mock_main_queue],
+            retry=True,
+            priority=5,
+        )
+        mock_message.ack.assert_called_once_with()
+        assert result == {"moved": 1, "skipped": 0, "discarded": 0}
+
+    @patch("taskqueue.dlq.Producer")
+    @patch("taskqueue.dlq.get_discard_info", return_value=("my.task", "task-1"))
+    def test_retry_dlq_messages_given_discarded_message_expect_ack_without_publish(
+        self,
+        mock_get_discard_info,
+        mock_producer_class,
+    ):
+        eta = (datetime.now(timezone.utc) - timedelta(days=20)).isoformat()
+        mock_message = MagicMock()
+        mock_message.headers = {
+            "id": "task-1",
+            "task": "my.task",
+            "eta": eta,
+        }
+
+        mock_dlq_queue = MagicMock()
+        mock_dlq_queue.get.side_effect = [mock_message, None]
+
+        mock_main_queue = MagicMock()
+        mock_app = MagicMock()
+        mock_app.amqp.queues = {
+            "dlq.default": MagicMock(return_value=mock_dlq_queue),
+            "default": MagicMock(return_value=mock_main_queue),
+        }
+        mock_app.connection_or_acquire.return_value.__enter__.return_value.default_channel = "channel"
+
+        result = retry_dlq_messages("dlq.default", "default", app=mock_app)
+
+        mock_get_discard_info.assert_called_once_with(mock_message.headers)
+        mock_producer_class.return_value.publish.assert_not_called()
+        mock_message.ack.assert_called_once_with()
+        assert result == {"moved": 0, "skipped": 0, "discarded": 1}
+
+    @patch("taskqueue.dlq.Producer")
+    @patch("taskqueue.dlq.get_discard_info", return_value=None)
+    @patch("taskqueue.dlq.should_retry_message", return_value=False)
+    def test_retry_dlq_messages_given_skipped_message_expect_ack_without_publish(
+        self,
+        mock_should_retry_message,
+        mock_get_discard_info,
+        mock_producer_class,
+    ):
+        eta = (datetime.now(timezone.utc) - timedelta(days=8)).isoformat()
+        mock_message = MagicMock()
+        mock_message.headers = {"id": "task-1", "eta": eta}
+
+        mock_dlq_queue = MagicMock()
+        mock_dlq_queue.get.side_effect = [mock_message, None]
+
+        mock_main_queue = MagicMock()
+        mock_app = MagicMock()
+        mock_app.amqp.queues = {
+            "dlq.default": MagicMock(return_value=mock_dlq_queue),
+            "default": MagicMock(return_value=mock_main_queue),
+        }
+        mock_app.connection_or_acquire.return_value.__enter__.return_value.default_channel = "channel"
+
+        result = retry_dlq_messages("dlq.default", "default", app=mock_app)
+
+        mock_get_discard_info.assert_called_once_with(mock_message.headers)
+        mock_should_retry_message.assert_called_once_with(mock_message.headers)
+        mock_producer_class.return_value.publish.assert_not_called()
+        mock_message.ack.assert_called_once_with()
+        assert result == {"moved": 0, "skipped": 1, "discarded": 0}
+
+
+class TestRetryDlqMessagesCommand:
+    @patch("taskqueue.management.commands.retry_dlq_messages.retry_dlq_messages")
+    def test_command_given_multiple_queues_expect_each_dlq_processed(
+        self, mock_retry_dlq_messages, caplog
+    ):
+        mock_retry_dlq_messages.side_effect = [
+            {"moved": 1, "skipped": 0, "discarded": 0},
+            {"moved": 2, "skipped": 1, "discarded": 1},
+        ]
+
+        command = Command()
+
+        with patch(
+            "taskqueue.management.commands.retry_dlq_messages.settings"
+        ) as mock_settings:
+            mock_settings.TASKQUEUE_QUEUES = ["default", "high"]
+            mock_settings.TASKQUEUE_DLQ_NAME_PREFIX = "dlq"
+
+            with caplog.at_level("INFO"):
+                command.handle()
+
+        assert mock_retry_dlq_messages.call_args_list[0].args == (
+            "dlq.default",
+            "default",
+        )
+        assert mock_retry_dlq_messages.call_args_list[1].args == (
+            "dlq.high", "high")
+        assert "Retried 3 messages, skipped 1, discarded 1." in caplog.text

p1_taskqueue-0.2.0/tests/test_management_command.py

@@ -0,0 +1,19 @@
+from unittest.mock import patch
+
+from taskqueue.management.commands.setup_taskqueue import Command
+
+
+class TestSetupTaskqueueCommand:
+    @patch("taskqueue.management.commands.setup_taskqueue.setup_queues")
+    def test_setup_taskqueue_command_given_success_expect_queues_declared(
+        self, mock_setup_queues, caplog
+    ):
+        mock_setup_queues.return_value = ["default", "high"]
+
+        command = Command()
+
+        with caplog.at_level("INFO"):
+            command.handle()
+
+        mock_setup_queues.assert_called_once_with()
+        assert "Declared 2 TaskQueue queues successfully." in caplog.text

p1_taskqueue-0.1.31/tests/test_celery_app.py

@@ -1,183 +0,0 @@
-"""Tests for the celery_app module."""
-from unittest.mock import MagicMock
-from unittest.mock import patch
-
-import pytest
-from taskqueue.celery_app import create_celery_app
-from taskqueue.celery_app import get_django_settings
-from taskqueue.celery_app import setup_queues
-
-
-class TestGetDjangoSettings:
-
-    def test_get_django_settings_given_django_available_expect_return_settings(self):
-        with patch('django.conf.settings') as mock_settings:
-            result = get_django_settings()
-            assert result == mock_settings
-
-    def test_get_django_settings_given_django_not_available_expect_raise_import_error(self):
-        with patch('builtins.__import__', side_effect=ImportError("No module named 'django'")):
-            with pytest.raises(ImportError, match="\\[TaskQueue\\] Django settings not found\\."):
-                get_django_settings()
-
-
-class TestCreateCeleryApp:
-
-    @patch('taskqueue.celery_app.setup_queues')
-    @patch('taskqueue.celery_app.get_django_settings')
-    @patch('taskqueue.celery_app.Celery')
-    def test_create_celery_app_given_valid_settings_expect_celery_app_created(
-        self, mock_celery_class, mock_get_settings, mock_setup_queues
-    ):
-        mock_settings = MagicMock()
-        mock_settings.TASKQUEUE_APP_NAME = 'testapp'
-        mock_settings.CELERY_BROKER_URL = 'redis://localhost:6379/0'
-        mock_settings.CELERY_RESULT_BACKEND = 'redis://localhost:6379/0'
-        mock_settings.CELERY_TASK_SERIALIZER = 'pickle'
-        mock_settings.CELERY_RESULT_SERIALIZER = 'pickle'
-        mock_settings.CELERY_ACCEPT_CONTENT = ['pickle']
-        mock_settings.CELERY_TIMEZONE = 'UTC'
-        mock_settings.CELERY_TASK_TRACK_STARTED = True
-        mock_settings.CELERY_TASK_TIME_LIMIT = 1800
-        mock_settings.CELERY_TASK_SOFT_TIME_LIMIT = 1500
-        mock_settings.CELERY_TASK_ALWAYS_EAGER = False
-        mock_settings.CELERY_TASK_EAGER_PROPAGATES = True
-
-        mock_get_settings.return_value = mock_settings
-        mock_app = MagicMock()
-        mock_celery_class.return_value = mock_app
-
-        result = create_celery_app()
-
-        mock_celery_class.assert_called_once_with('testapp')
-        # mock_setup_queues.assert_called_once()
-        mock_app.conf.update.assert_called_once()
-        mock_app.autodiscover_tasks.assert_called_once_with(['taskqueue'])
-        assert result == mock_app
-
-    @patch('taskqueue.celery_app.setup_queues')
-    @patch('taskqueue.celery_app.get_django_settings')
-    @patch('taskqueue.celery_app.Celery')
-    def test_create_celery_app_given_missing_settings_expect_defaults_used(
-        self, mock_celery_class, mock_get_settings, mock_setup_queues
-    ):
-        mock_settings = MagicMock()
-        del mock_settings.TASKQUEUE_APP_NAME
-        del mock_settings.CELERY_BROKER_URL
-        del mock_settings.CELERY_RESULT_BACKEND
-        del mock_settings.CELERY_TASK_SERIALIZER
-        del mock_settings.CELERY_RESULT_SERIALIZER
-        del mock_settings.CELERY_ACCEPT_CONTENT
-        del mock_settings.CELERY_TIMEZONE
-        del mock_settings.CELERY_TASK_TRACK_STARTED
-        del mock_settings.CELERY_TASK_TIME_LIMIT
-        del mock_settings.CELERY_TASK_SOFT_TIME_LIMIT
-        del mock_settings.CELERY_TASK_ALWAYS_EAGER
-        del mock_settings.CELERY_TASK_EAGER_PROPAGATES
-
-        mock_get_settings.return_value = mock_settings
-        mock_app = MagicMock()
-        mock_celery_class.return_value = mock_app
-
-        result = create_celery_app()
-
-        mock_celery_class.assert_called_once_with('taskqueue')
-        assert result == mock_app
-
-
-class TestSetupQueues:
-    """Test setup_queues function."""
-
-    def test_setup_queues_given_valid_settings_expect_queues_configured(self):
-        """Test that setup_queues configures queues and DLQs correctly."""
-        mock_app = MagicMock()
-        mock_app.conf = {}
-        mock_settings = MagicMock()
-        mock_settings.TASKQUEUE_APP_NAME = 'testapp'
-        mock_settings.TASKQUEUE_QUEUES = ['default', 'high', 'low']
-        mock_settings.TASKQUEUE_DLQ_NAME_PREFIX = 'dlq'
-
-        setup_queues(mock_app, mock_settings)
-
-        # Assertions
-        assert mock_app.conf['task_default_queue'] == 'default'
-        assert mock_app.conf['task_default_exchange'] == 'testapp'
-        assert mock_app.conf['task_default_exchange_type'] == 'direct'
-        assert len(mock_app.conf['task_queues']) == 6
-
-        main_queues = [q for q in mock_app.conf['task_queues']
-                       if not q.name.startswith('dlq.')]
-        assert len(main_queues) == 3
-        assert any(q.name == 'default' for q in main_queues)
-        assert any(q.name == 'high' for q in main_queues)
-        assert any(q.name == 'low' for q in main_queues)
-
-        dlq_queues = [q for q in mock_app.conf['task_queues']
-                      if q.name.startswith('dlq.')]
-        assert len(dlq_queues) == 3
-        assert any(q.name == 'dlq.default' for q in dlq_queues)
-        assert any(q.name == 'dlq.high' for q in dlq_queues)
-        assert any(q.name == 'dlq.low' for q in dlq_queues)
-
-    def test_setup_queues_given_missing_settings_expect_defaults_used(self):
-        mock_app = MagicMock()
-        mock_app.conf = {}
-        mock_settings = MagicMock()
-        del mock_settings.TASKQUEUE_APP_NAME
-        del mock_settings.TASKQUEUE_QUEUES
-        del mock_settings.TASKQUEUE_DLQ_NAME_PREFIX
-
-        setup_queues(mock_app, mock_settings)
-
-        assert mock_app.conf['task_default_queue'] == 'default'
-        assert mock_app.conf['task_default_exchange'] == 'taskqueue'
-        assert mock_app.conf['task_default_exchange_type'] == 'direct'
-        assert len(mock_app.conf['task_queues']) == 6
-
-    def test_setup_queues_given_single_queue_expect_correct_configuration(self):
-        mock_app = MagicMock()
-        mock_app.conf = {}
-        mock_settings = MagicMock()
-        mock_settings.TASKQUEUE_APP_NAME = 'singleapp'
-        mock_settings.TASKQUEUE_QUEUES = ['single']
-        mock_settings.TASKQUEUE_DLQ_NAME_PREFIX = 'dead'
-
-        setup_queues(mock_app, mock_settings)
-
-        assert len(mock_app.conf['task_queues']) == 2
-
-        main_queue = next(
-            q for q in mock_app.conf['task_queues'] if q.name == 'single')
-        assert main_queue.queue_arguments['x-dead-letter-exchange'] == 'singleapp.dlx'
-        assert main_queue.queue_arguments['x-dead-letter-routing-key'] == 'dead.single'
-
-        dlq = next(
-            q for q in mock_app.conf['task_queues'] if q.name == 'dead.single')
-        assert dlq.exchange.name == 'singleapp.dlx'
-
-
-class TestCeleryAppIntegration:
-    """Integration tests for the celery_app module."""
-
-    @patch('taskqueue.celery_app.get_django_settings')
-    def test_celery_app_import_given_django_configured_expect_app_created(self, mock_get_settings):
-        mock_settings = MagicMock()
-        mock_settings.TASKQUEUE_APP_NAME = 'testapp'
-        mock_settings.CELERY_BROKER_URL = 'redis://localhost:6379/0'
-        mock_settings.CELERY_RESULT_BACKEND = 'redis://localhost:6379/0'
-        mock_settings.CELERY_TASK_SERIALIZER = 'pickle'
-        mock_settings.CELERY_RESULT_SERIALIZER = 'pickle'
-        mock_settings.CELERY_ACCEPT_CONTENT = ['pickle']
-        mock_settings.CELERY_TIMEZONE = 'UTC'
-        mock_settings.CELERY_TASK_TRACK_STARTED = True
-        mock_settings.CELERY_TASK_TIME_LIMIT = 1800
-        mock_settings.CELERY_TASK_SOFT_TIME_LIMIT = 1500
-        mock_settings.CELERY_TASK_ALWAYS_EAGER = False
-        mock_settings.CELERY_TASK_EAGER_PROPAGATES = True
-        mock_settings.TASKQUEUE_QUEUES = ['default']
-        mock_settings.TASKQUEUE_DLQ_NAME_PREFIX = 'dlq'
-
-        mock_get_settings.return_value = mock_settings
-
-        from taskqueue.celery_app import celery_app
-        assert celery_app is not None