p1-taskqueue 0.1.12__py3-none-any.whl → 0.1.13__py3-none-any.whl

{p1_taskqueue-0.1.12.dist-info → p1_taskqueue-0.1.13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: p1-taskqueue
-Version: 0.1.12
+Version: 0.1.13
 Summary: A Task Queue Wrapper for Dekoruma Backend
 Author-email: Chalvin <engineering@dekoruma.com>
 Project-URL: Homepage, https://github.com/Dekoruma/p1-taskqueue
@@ -19,6 +19,7 @@ Requires-Dist: kombu>=5.5.4
 Requires-Dist: django>=4.0.0
 Requires-Dist: django-celery-results>=2.6.0
 Requires-Dist: django-celery-beat>=2.8.1
+Requires-Dist: requests>=2.32.3
 Provides-Extra: dev
 Requires-Dist: pytest>=7.0.0; extra == "dev"
 Requires-Dist: pytest-cov>=4.0.0; extra == "dev"

p1_taskqueue-0.1.13.dist-info/RECORD

@@ -0,0 +1,10 @@
+taskqueue/__init__.py,sha256=nNHXIyysNLom9f8of-d__pWJ-YF53Mbrbsb1frPzPPI,298
+taskqueue/celery_app.py,sha256=rLIYRBBgYaQiXEqz-zCntvS8xU4eFZ9YuhJBfIye9E0,3931
+taskqueue/cmanager.py,sha256=9jxcTsWOpzexV3SkRhlY-PkhrobEdMuQVm6tWVVzgTs,16260
+taskqueue/slack_notifier.py,sha256=ZvTbWa1XXHUiciYF14_SH5af0BvGoPCBQohPtxx4FgU,1419
+taskqueue/libs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+taskqueue/libs/helper_test.py,sha256=JCdh2S29PpL8RqUxqkcIqwIvr3M9puqHglBZAmfPkuw,7722
+p1_taskqueue-0.1.13.dist-info/METADATA,sha256=pr73Jir2XbTBsq2HieYSEO4h0a_FM5KV15THqfedOM0,1597
+p1_taskqueue-0.1.13.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+p1_taskqueue-0.1.13.dist-info/top_level.txt,sha256=hA3SM1ik2K8iPqtlt_-_nJ4TAePwHPN3vsOc4EiynqU,10
+p1_taskqueue-0.1.13.dist-info/RECORD,,

taskqueue/celery_app.py

@@ -2,10 +2,14 @@
 Celery application setup for TaskQueue.
 Reads configuration from Django settings and auto-configures queues with DLQ.
 """
+import logging
+
 from celery import Celery
 from kombu import Exchange
 from kombu import Queue
 
+logger = logging.getLogger(__name__)
+
 
 def get_django_settings():
     """Get Django settings, fail fast if not properly configured."""
@@ -63,7 +67,6 @@ def setup_queues(app, settings, celery_config):
         queue_names = ['default', 'high', 'low']
     dlq_name_prefix = getattr(settings, 'TASKQUEUE_DLQ_NAME_PREFIX', 'dlq')
 
-    # Create exchanges
     main_exchange = Exchange(app_name, type='direct')
     dlx_exchange = Exchange(f'{app_name}.dlx', type='direct')
 
@@ -95,5 +98,16 @@ def setup_queues(app, settings, celery_config):
         'task_queues': tuple(queues),
     })
 
+    try:
+        with app.connection_or_acquire() as conn:
+            main_exchange.declare(channel=conn.default_channel)
+            dlx_exchange.declare(channel=conn.default_channel)
+
+            for queue in queues:
+                queue.declare(channel=conn.default_channel)
+    except Exception as e:
+        logger.warning(
+            f"[TaskQueue] Failed to declare queues: {str(e.__class__.__name__)} {e}")
+
 
 celery_app = create_celery_app()

taskqueue/cmanager.py

@@ -1,13 +1,13 @@
 import importlib
 import inspect
 import logging
-from datetime import datetime
-from datetime import timedelta
 from typing import Any
 from typing import Dict
 from typing import Tuple
 
 from celery import shared_task
+from celery.exceptions import Reject
+from taskqueue.slack_notifier import SlackbotManager
 
 # Setup logger
 logger = logging.getLogger(__name__)
@@ -50,7 +50,8 @@ def _extract_init_args_from_instance(instance: Any) -> Tuple[list, dict]:
 def _split_function_and_queue_kwargs(kwargs: Dict[str, Any]) -> Tuple[Dict[str, Any], Dict[str, Any]]:
     # To prevent confusion whether a kwargs is for function or queue kwargs(i.e celery options and on_commit),
     # ignore confusing kwargs while give warning
-    supported_queue_keys = {"channel", "retry", "on_commit", "job_timeout"}
+    supported_queue_keys = {"channel", "retry",
+                            "on_commit", "job_timeout", "use_legacy_executor"}
     ignored_non_function_keys = {
         "queue", "countdown", "eta", "expires", "priority", "task_id", "routing_key",
         "serializer", "compression", "headers", "link", "link_error", "retry_policy",
@@ -74,6 +75,17 @@ def _split_function_and_queue_kwargs(kwargs: Dict[str, Any]) -> Tuple[Dict[str,
     return func_kwargs, queue_kwargs
 
 
+def _build_callable_task_call(func: Any, func_args: tuple, func_kwargs: dict) -> Tuple[str, list, dict]:
+    task_name = "taskqueue.cmanager.callable_executor"
+    task_args = []
+    task_kwargs = {
+        "callable_obj": func,
+        "args": list(func_args),
+        "kwargs": dict(func_kwargs),
+    }
+    return task_name, task_args, task_kwargs
+
+
 def _build_dynamic_task_call(func: Any, *args: Any, **func_kwargs: Any) -> Tuple[str, list, dict]:
     if _is_class_method(func):
         instance = getattr(func, "__self__")
@@ -155,47 +167,21 @@ class CManager:
             'enqueue_op_type', K_ENQUEUE_OP_TYPE_ENQUEUE)
 
         try:
-            if enqueue_op_type == K_ENQUEUE_OP_TYPE_ENQUEUE:
-                if not args:
-                    raise ValueError(
-                        "enqueue requires a callable as the first positional argument")
-                func = args[0]
-                func_args = args[1:]
-
-            elif enqueue_op_type == K_ENQUEUE_OP_TYPE_ENQUEUE_AT:
-                if len(args) < 2:
-                    raise ValueError(
-                        "enqueue_at requires (eta_datetime, func, *func_args)")
-                eta: datetime = args[0]
-                func = args[1]
-                func_args = args[2:]
-
-            elif enqueue_op_type == K_ENQUEUE_OP_TYPE_ENQUEUE_IN:
-                if len(args) < 2:
-                    raise ValueError(
-                        "enqueue_in requires (countdown_delta, func, *func_args)")
-                delta: timedelta = args[0]
-                func = args[1]
-                func_args = args[2:]
-            else:
-                raise ValueError(
-                    f"Unknown enqueue operation type: {enqueue_op_type}")
-
-            func_kwargs, queue_kwargs = _split_function_and_queue_kwargs(
-                kwargs)
+            func, func_args, func_kwargs, queue_options = self._parse_enqueue_args(
+                enqueue_op_type, args, kwargs)
 
-            if enqueue_op_type == K_ENQUEUE_OP_TYPE_ENQUEUE_AT:
-                queue_kwargs = dict(queue_kwargs)
-                queue_kwargs["eta"] = eta
-            elif enqueue_op_type == K_ENQUEUE_OP_TYPE_ENQUEUE_IN:
-                queue_kwargs = dict(queue_kwargs)
-                queue_kwargs["countdown"] = int(delta.total_seconds())
+            use_legacy_executor = queue_options.pop(
+                'use_legacy_executor', True)
 
-            task_name, task_args, task_kwargs = _build_dynamic_task_call(
-                func, *func_args, **func_kwargs)
+            if use_legacy_executor:
+                task_name, task_args, task_kwargs = _build_dynamic_task_call(
+                    func, *func_args, **func_kwargs)
+            else:
+                task_name, task_args, task_kwargs = _build_callable_task_call(
+                    func, func_args, func_kwargs)
 
             task_id = self._send_task(task_name, task_args,
-                                      task_kwargs, queue_kwargs)
+                                      task_kwargs, queue_options)
 
             logger.info('[_enqueue_op_base %s] Submit Celery Task SUCCESS, task_name: %s args: %s, kwargs: %s, task_id: %s' % (
                 enqueue_op_type, task_name, task_args, task_kwargs, task_id))
@@ -205,6 +191,46 @@ class CManager:
                 enqueue_op_type, str(e), args, kwargs))
             raise e
 
+    def _parse_enqueue_args(self, enqueue_op_type: str, args: tuple, kwargs: dict) -> Tuple[Any, tuple, dict, dict]:
+        """Parse enqueue arguments and return func, func_args, func_kwargs, and queue_options."""
+        if enqueue_op_type == K_ENQUEUE_OP_TYPE_ENQUEUE:
+            if not args:
+                raise ValueError(
+                    "enqueue requires a callable as the first positional argument")
+            func = args[0]
+            func_args = args[1:]
+            eta, delta = None, None
+
+        elif enqueue_op_type == K_ENQUEUE_OP_TYPE_ENQUEUE_AT:
+            if len(args) < 2:
+                raise ValueError(
+                    "enqueue_at requires (eta_datetime, func, *func_args)")
+            eta = args[0]
+            func = args[1]
+            func_args = args[2:]
+            delta = None
+
+        elif enqueue_op_type == K_ENQUEUE_OP_TYPE_ENQUEUE_IN:
+            if len(args) < 2:
+                raise ValueError(
+                    "enqueue_in requires (countdown_delta, func, *func_args)")
+            delta = args[0]
+            func = args[1]
+            func_args = args[2:]
+            eta = None
+        else:
+            raise ValueError(
+                f"Unknown enqueue operation type: {enqueue_op_type}")
+
+        func_kwargs, queue_options = _split_function_and_queue_kwargs(kwargs)
+
+        if eta is not None:
+            queue_options["eta"] = eta
+        elif delta is not None:
+            queue_options["countdown"] = int(delta.total_seconds())
+
+        return func, func_args, func_kwargs, queue_options
+
     def _send_task(self, task_name: str, task_args: list, task_kwargs: dict, queue_kwargs: Dict[str, Any]) -> str:
         celery_app = self._get_celery_app()
 
@@ -237,7 +263,54 @@ class CManager:
 cm = CManager()
 
 
-@shared_task(bind=True, max_retries=K_MAX_RETRY_COUNT)
+@shared_task(bind=True, max_retries=K_MAX_RETRY_COUNT, acks_late=True, reject_on_worker_lost=True)
+def callable_executor(self, callable_obj=None, args=None, kwargs=None, retry=None):
+    job_id = self.request.id
+    try:
+        args = args or []
+        kwargs = kwargs or {}
+        callable_name = getattr(callable_obj, '__name__', str(callable_obj))
+
+        logger.info(
+            f"[TaskQueue] Executing callable: {callable_name} with args: {args} and kwargs: {kwargs}, job_id: {job_id}")
+
+        callable_obj(*args, **kwargs)
+
+        logger.info(
+            f"[TaskQueue] Callable execution completed successfully, callable: {callable_name}, args: {args}, kwargs: {kwargs}, job_id: {job_id}")
+        return None
+    except Exception as e:
+        logger.exception(
+            f"[TaskQueue] Error executing callable: {callable_name}, args: {args}, kwargs: {kwargs}, error_class: {e.__class__.__name__}, error: {e}, job_id: {job_id}")
+
+        current_retries = getattr(self.request, 'retries', 0) or 0
+        max_retries = self.max_retries or K_MAX_RETRY_COUNT
+        if isinstance(retry, dict) and 'max_retries' in retry:
+            max_retries = retry['max_retries']
+
+        if current_retries >= max_retries:
+            logger.error(
+                f"[TaskQueue] Max retries ({max_retries}) reached for callable: {callable_name}, job_id: {job_id}")
+            self.update_state(state='FAILURE', meta={
+                              'exc_type': type(e).__name__, 'exc_message': str(e)})
+
+            SlackbotManager.send_message(
+                f"Job Failed Too Many Times - Moving back to dlq.\n"
+                f"function name: {callable_name}\n"
+                f"args: {args}\n"
+                f"kwargs: {kwargs}"
+            )
+
+            raise Reject(reason=str(e), requeue=False)
+
+        countdown = K_DEFAULT_RETRY_COUNTDOWN
+        if isinstance(retry, dict) and 'countdown' in retry:
+            countdown = retry['countdown']
+
+        raise self.retry(exc=e, countdown=countdown, max_retries=max_retries)
+
+
+@shared_task(bind=True, max_retries=K_MAX_RETRY_COUNT, acks_late=True, reject_on_worker_lost=True)
 def dynamic_function_executor(self, module_path=None, function_name=None, args=None, kwargs=None, retry=None):
     job_id = self.request.id
     try:
@@ -261,8 +334,18 @@ def dynamic_function_executor(self, module_path=None, function_name=None, args=N
 
         if current_retries >= max_retries:
             logger.error(
-                f"[TaskQueue] Max retries ({max_retries}) reached for function: {function_name}, marking task as FAILED, job_id: {job_id}")
-            raise
+                f"[TaskQueue] Max retries ({max_retries}) reached for function: {function_name}, job_id: {job_id}")
+            self.update_state(state='FAILURE', meta={
+                              'exc_type': type(e).__name__, 'exc_message': str(e)})
+
+            SlackbotManager.send_message(
+                f"Job Failed Too Many Times - Moving back to dlq.\n"
+                f"function name: {function_name}\n"
+                f"args: {args}\n"
+                f"kwargs: {kwargs}"
+            )
+
+            raise Reject(reason=str(e), requeue=False)
 
         countdown = K_DEFAULT_RETRY_COUNTDOWN
         if isinstance(retry, dict) and 'countdown' in retry:
@@ -271,7 +354,7 @@ def dynamic_function_executor(self, module_path=None, function_name=None, args=N
         raise self.retry(exc=e, countdown=countdown, max_retries=max_retries)
 
 
-@shared_task(bind=True, max_retries=K_MAX_RETRY_COUNT)
+@shared_task(bind=True, max_retries=K_MAX_RETRY_COUNT, acks_late=True, reject_on_worker_lost=True)
 def dynamic_class_method_executor(self, module_path=None, class_name=None, method_name=None, args=None, kwargs=None, init_args=None, init_kwargs=None, retry=None):
     job_id = self.request.id
     try:
@@ -299,8 +382,18 @@ def dynamic_class_method_executor(self, module_path=None, class_name=None, metho
 
         if current_retries >= max_retries:
             logger.error(
-                f"[TaskQueue] Max retries ({max_retries}) reached for method: {method_name}, marking task as FAILED, job_id: {job_id}")
-            raise
+                f"[TaskQueue] Max retries ({max_retries}) reached for method: {method_name}, job_id: {job_id}")
+            self.update_state(state='FAILURE', meta={
+                              'exc_type': type(e).__name__, 'exc_message': str(e)})
+
+            SlackbotManager.send_message(
+                f"Job Failed Too Many Times - Moving back to dlq.\n"
+                f"function name: {class_name}.{method_name}\n"
+                f"args: {args}\n"
+                f"kwargs: {kwargs}"
+            )
+
+            raise Reject(reason=str(e), requeue=False)
 
         countdown = K_DEFAULT_RETRY_COUNTDOWN
         if isinstance(retry, dict) and 'countdown' in retry:

taskqueue/slack_notifier.py

@@ -0,0 +1,51 @@
+"""
+Slack notification for TaskQueue.
+"""
+import json
+import logging
+
+import requests
+
+logger = logging.getLogger(__name__)
+
+
+class SlackbotManager:
+
+    @classmethod
+    def send_message(cls, message: str) -> None:
+        try:
+            from django.conf import settings
+        except ImportError:
+            return
+
+        if not getattr(settings, 'TASKQUEUE_SLACK_ENABLED', False):
+            return
+
+        hook_url = getattr(settings, 'TASKQUEUE_SLACK_HOOK_URL', None)
+        if not hook_url:
+            return
+
+        channel = getattr(
+            settings, 'TASKQUEUE_SLACK_CHANNEL_NAME', '#tech-automation')
+        username = getattr(
+            settings, 'TASKQUEUE_SLACK_USERNAME', 'TaskQueueBot')
+        icon_emoji = getattr(
+            settings, 'TASKQUEUE_SLACK_ICON_EMOJI', ':robot_face:')
+
+        is_staging = getattr(settings, 'IS_RUN_IN_STAGING_ENV', False)
+        if is_staging:
+            message = '[STAGING] ' + message
+
+        try:
+            requests.post(
+                hook_url,
+                data=json.dumps({
+                    'channel': channel,
+                    'username': username,
+                    'text': message,
+                    'icon_emoji': icon_emoji,
+                }),
+                headers={"Content-Type": "application/json"}
+            )
+        except Exception as e:
+            logger.exception('[TaskQueue Slack] Error: %s', str(e))

p1_taskqueue-0.1.12.dist-info/RECORD

@@ -1,9 +0,0 @@
-taskqueue/__init__.py,sha256=nNHXIyysNLom9f8of-d__pWJ-YF53Mbrbsb1frPzPPI,298
-taskqueue/celery_app.py,sha256=neNjLNDouBwUJsDHn6q3cgQQnmwjBFJ0xTVWHR8cWvY,3482
-taskqueue/cmanager.py,sha256=dzYubR9SkMzEYuatt3AL_umkVX_l5BHenz3urp14mLw,12411
-taskqueue/libs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-taskqueue/libs/helper_test.py,sha256=JCdh2S29PpL8RqUxqkcIqwIvr3M9puqHglBZAmfPkuw,7722
-p1_taskqueue-0.1.12.dist-info/METADATA,sha256=9NOpVhO1MhpRLe7B96FfLZmgLE8Sn8AIEpXeDsoISHU,1565
-p1_taskqueue-0.1.12.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-p1_taskqueue-0.1.12.dist-info/top_level.txt,sha256=hA3SM1ik2K8iPqtlt_-_nJ4TAePwHPN3vsOc4EiynqU,10
-p1_taskqueue-0.1.12.dist-info/RECORD,,