p1-taskqueue 0.1.21__tar.gz → 0.1.23__tar.gz

{p1_taskqueue-0.1.21 → p1_taskqueue-0.1.23}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: p1-taskqueue
-Version: 0.1.21
+Version: 0.1.23
 Summary: A Task Queue Wrapper for Dekoruma Backend
 Author-email: Chalvin <engineering@dekoruma.com>
 Project-URL: Homepage, https://github.com/Dekoruma/p1-taskqueue

{p1_taskqueue-0.1.21 → p1_taskqueue-0.1.23}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 [project]
 name = "p1-taskqueue"
 # DO NOT CHANGE THIS VERSION - it gets automatically replaced by CI/CD with the git tag version
-version = "0.1.21"
+version = "0.1.23"
 description = "A Task Queue Wrapper for Dekoruma Backend"
 authors = [
     {name = "Chalvin", email = "engineering@dekoruma.com"}

{p1_taskqueue-0.1.21 → p1_taskqueue-0.1.23}/src/p1_taskqueue.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: p1-taskqueue
-Version: 0.1.21
+Version: 0.1.23
 Summary: A Task Queue Wrapper for Dekoruma Backend
 Author-email: Chalvin <engineering@dekoruma.com>
 Project-URL: Homepage, https://github.com/Dekoruma/p1-taskqueue

{p1_taskqueue-0.1.21 → p1_taskqueue-0.1.23}/src/p1_taskqueue.egg-info/SOURCES.txt

@@ -6,11 +6,15 @@ src/p1_taskqueue.egg-info/dependency_links.txt
 src/p1_taskqueue.egg-info/requires.txt
 src/p1_taskqueue.egg-info/top_level.txt
 src/taskqueue/__init__.py
+src/taskqueue/apps.py
 src/taskqueue/celery_app.py
 src/taskqueue/cmanager.py
+src/taskqueue/models.py
 src/taskqueue/slack_notifier.py
 src/taskqueue/libs/__init__.py
 src/taskqueue/libs/helper_test.py
+src/taskqueue/migrations/0001_initial.py
+src/taskqueue/migrations/__init__.py
 tests/test_celery_app.py
 tests/test_cmanager.py
 tests/test_helper_test_functions.py

p1_taskqueue-0.1.23/src/taskqueue/apps.py

@@ -0,0 +1,8 @@
+from django.apps import AppConfig
+
+
+class TaskqueueConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'taskqueue'
+    verbose_name = 'TaskQueue'
+

{p1_taskqueue-0.1.21 → p1_taskqueue-0.1.23}/src/taskqueue/celery_app.py

@@ -31,7 +31,7 @@ def create_celery_app():
     # https://docs.celeryq.dev/en/latest/userguide/configuration.html
     celery_config = {
         'broker_url': getattr(settings, 'CELERY_BROKER_URL', 'amqp://localhost:5672//'),
-        'result_backend': getattr(settings, 'CELERY_RESULT_BACKEND', 'rpc://localhost:5672//'),
+        'result_backend': getattr(settings, 'CELERY_RESULT_BACKEND', 'django-db'),
         'task_serializer': getattr(settings, 'CELERY_TASK_SERIALIZER', 'pickle'),
         'result_serializer': getattr(settings, 'CELERY_RESULT_SERIALIZER', 'pickle'),
         'accept_content': getattr(settings, 'CELERY_ACCEPT_CONTENT', ['pickle']),

{p1_taskqueue-0.1.21 → p1_taskqueue-0.1.23}/src/taskqueue/cmanager.py

@@ -1,6 +1,9 @@
+import base64
 import importlib
 import inspect
 import logging
+import pickle
+import uuid
 from datetime import datetime
 from typing import Any
 from typing import Dict
@@ -18,8 +21,10 @@ K_ENQUEUE_OP_TYPE_ENQUEUE = 'enqueue'
 K_ENQUEUE_OP_TYPE_ENQUEUE_AT = 'enqueue_at'
 K_ENQUEUE_OP_TYPE_ENQUEUE_IN = 'enqueue_in'
 
-K_MAX_RETRY_COUNT = 3
+K_MAX_RETRY_COUNT = 2
 K_DEFAULT_RETRY_COUNTDOWN = 3600
+K_TASK_STATUS_REPUBLISHED = 'FAILURE - REPUBLISHED'
+K_TASK_STATUS_FAILURE = 'FAILURE'
 
 
 def _is_class_method(func: Any) -> bool:
@@ -138,6 +143,63 @@ class CManager:
         from .celery_app import celery_app
         return celery_app
 
+    def republish_task(self, task_id: str) -> str:
+        try:
+            from taskqueue.models import TaskReconstruction
+        except ImportError:
+            raise ImportError(
+                "taskqueue.models is required for republishing tasks")
+
+        try:
+            task_recon = TaskReconstruction.objects.get(task_id=task_id)
+        except TaskReconstruction.DoesNotExist:
+            raise ValueError(f"Task with ID {task_id} not found")
+
+        try:
+            from django_celery_results.models import TaskResult
+            from django.utils import timezone
+            task_result = TaskResult.objects.filter(task_id=task_id).first()
+            if not task_result:
+                raise ValueError(
+                    f"Task with ID {task_id} not found in TaskResult. Cannot republish task without TaskResult.")
+            if task_result.status != K_TASK_STATUS_FAILURE:
+                raise ValueError(
+                    f"Task with ID {task_id} cannot be republished. Only tasks with status '{K_TASK_STATUS_FAILURE}' can be republished. Current status: {task_result.status}")
+
+            TaskResult.objects.filter(task_id=task_id).update(
+                status=K_TASK_STATUS_REPUBLISHED,
+                date_done=timezone.now(),
+            )
+        except ImportError:
+            raise ImportError(
+                "django_celery_results is required for republishing tasks")
+        except ValueError:
+            raise
+        except Exception as e:
+            logger.warning(f"[CManager] Failed to update old task status: {e}")
+            raise
+
+        task_name = task_recon.task_name
+        task_args = self._unpickle_data(task_recon.task_args)
+        task_kwargs = self._unpickle_data(task_recon.task_kwargs)
+        queue_options = task_recon.queue_options
+
+        eta = queue_options.get("eta")
+        if eta and isinstance(eta, str):
+            from datetime import datetime as dt
+            queue_options["eta"] = dt.fromisoformat(eta.replace('Z', '+00:00'))
+            eta = queue_options["eta"]
+
+        queue_name = queue_options.get("channel")
+        job_timeout = queue_options.get("job_timeout")
+        retry_policy = queue_options.get("retry")
+        countdown = queue_options.get("countdown")
+
+        new_task_id = self._create_task_result(task_name, task_args, task_kwargs,
+                                               queue_name, job_timeout, retry_policy, countdown, eta)
+
+        return self._send_task(task_name, task_args, task_kwargs, queue_options, new_task_id)
+
     def enqueue(self, *args: Any, **kwargs: Any) -> None:
         self._enqueue_op(
             *args, enqueue_op_type=K_ENQUEUE_OP_TYPE_ENQUEUE, **kwargs)
@@ -181,8 +243,17 @@ class CManager:
                 task_name, task_args, task_kwargs = _build_callable_task_call(
                     func, func_args, func_kwargs)
 
+            queue_name = queue_options.get("channel")
+            job_timeout = queue_options.get("job_timeout")
+            retry_policy = queue_options.get("retry")
+            countdown = queue_options.get("countdown")
+            eta = queue_options.get("eta")
+
+            task_id = self._create_task_result(task_name, task_args, task_kwargs,
+                                               queue_name, job_timeout, retry_policy, countdown, eta)
+
             task_id = self._send_task(task_name, task_args,
-                                      task_kwargs, queue_options)
+                                      task_kwargs, queue_options, task_id)
 
             logger.info('[_enqueue_op_base %s] Submit Celery Task SUCCESS, task_name: %s args: %s, kwargs: %s, task_id: %s' % (
                 enqueue_op_type, task_name, task_args, task_kwargs, task_id))
@@ -232,12 +303,71 @@ class CManager:
 
         return func, func_args, func_kwargs, queue_options
 
-    def _send_task(self, task_name: str, task_args: list, task_kwargs: dict, queue_kwargs: Dict[str, Any]) -> str:
+    def _pickle_data(self, data: Any) -> str:
+        """Pickle data and return as base64-encoded string."""
+        pickled = pickle.dumps(data)
+        return base64.b64encode(pickled).decode('utf-8')
+
+    def _unpickle_data(self, data: str) -> Any:
+        """Unpickle base64-encoded pickled string."""
+        pickled = base64.b64decode(data.encode('utf-8'))
+        return pickle.loads(pickled)
+
+    def _create_task_result(self, task_name: str, task_args: list, task_kwargs: dict,
+                            queue_name: str, job_timeout: int, retry_policy: dict, countdown: int, eta: Any) -> str:
+        task_id = str(uuid.uuid4())
+
+        try:
+            from django_celery_results.models import TaskResult
+            from django.utils import timezone
+
+            TaskResult.objects.create(
+                task_id=task_id,
+                task_name=task_name,
+                status='PENDING',
+                date_created=timezone.now(),
+                date_done=timezone.now(),
+            )
+        except ImportError:
+            pass
+        except Exception as e:
+            logger.warning(f"[CManager] Failed to create TaskResult: {e}")
+
+        try:
+            from taskqueue.models import TaskReconstruction
+
+            pickled_task_args = self._pickle_data(task_args)
+            pickled_task_kwargs = self._pickle_data(task_kwargs)
+
+            TaskReconstruction.objects.create(
+                task_id=task_id,
+                task_name=task_name,
+                task_args=pickled_task_args,
+                task_kwargs=pickled_task_kwargs,
+                queue_options={
+                    "channel": queue_name,
+                    "job_timeout": job_timeout,
+                    "retry": retry_policy,
+                    "countdown": countdown,
+                    "eta": eta.isoformat() if eta else None,
+                },
+            )
+        except ImportError:
+            pass
+        except Exception as e:
+            logger.warning(
+                f"[CManager] Failed to create TaskReconstruction: {e}")
+
+        return task_id
+
+    def _send_task(self, task_name: str, task_args: list, task_kwargs: dict, queue_kwargs: Dict[str, Any], task_id: str = None) -> str:
         celery_app = self._get_celery_app()
 
         queue_name = queue_kwargs.pop("channel", None)
         job_timeout = queue_kwargs.pop("job_timeout", None)
         retry_policy = queue_kwargs.pop("retry", None)
+        countdown = queue_kwargs.pop("countdown", None)
+        eta = queue_kwargs.pop("eta", None)
 
         created_at = datetime.now().isoformat()
 
@@ -250,10 +380,10 @@ class CManager:
             send_opts["queue"] = queue_name
         if job_timeout is not None:
             send_opts["time_limit"] = job_timeout
-        if "countdown" in queue_kwargs:
-            send_opts["countdown"] = queue_kwargs["countdown"]
-        if "eta" in queue_kwargs:
-            send_opts["eta"] = queue_kwargs["eta"]
+        if countdown is not None:
+            send_opts["countdown"] = countdown
+        if eta is not None:
+            send_opts["eta"] = eta
 
         task_kwargs_with_retry = dict(task_kwargs)
         if retry_policy is None:
@@ -262,8 +392,12 @@ class CManager:
         else:
             task_kwargs_with_retry["retry"] = retry_policy
 
+        if task_id:
+            send_opts["task_id"] = task_id
+
         task = celery_app.send_task(task_name, args=task_args,
                                     kwargs=task_kwargs_with_retry, **send_opts)
+
         return str(task.id)
 
 

p1_taskqueue-0.1.23/src/taskqueue/migrations/0001_initial.py

@@ -0,0 +1,27 @@
+from django.db import migrations
+from django.db import models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = []
+
+    operations = [
+        migrations.CreateModel(
+            name='TaskReconstruction',
+            fields=[
+                ('task_id', models.CharField(
+                    max_length=255, primary_key=True, serialize=False)),
+                ('task_name', models.CharField(max_length=255)),
+                ('task_args', models.TextField()),
+                ('task_kwargs', models.TextField()),
+                ('queue_options', models.JSONField(default=dict)),
+                ('created_at', models.DateTimeField(auto_now_add=True)),
+            ],
+            options={
+                'db_table': 'taskqueue_task_reconstruction',
+            },
+        ),
+    ]

p1_taskqueue-0.1.23/src/taskqueue/models.py

@@ -0,0 +1,17 @@
+from django.db import models
+
+
+class TaskReconstruction(models.Model):
+    task_id = models.CharField(max_length=255, primary_key=True)
+    task_name = models.CharField(max_length=255)
+    task_args = models.TextField()
+    task_kwargs = models.TextField()
+    queue_options = models.JSONField(default=dict)
+    created_at = models.DateTimeField(auto_now_add=True)
+
+    class Meta:
+        db_table = 'taskqueue_task_reconstruction'
+        app_label = 'taskqueue'
+
+    def __str__(self):
+        return f"TaskReconstruction({self.task_id})"