django-celery-logs 0.2.0__py3-none-any.whl

django_celery_logs/__init__.py

@@ -0,0 +1 @@
+default_app_config = "django_celery_logs.apps.CeleryLogsConfig"

django_celery_logs/admin.py

@@ -0,0 +1,358 @@
+import json
+from datetime import timedelta
+
+from celery import current_app
+from django.conf import settings
+from django.contrib import admin, messages
+from django.db.models import Avg, Count, JSONField, Max, Min, Q
+from django.http import HttpResponseRedirect
+from django.urls import path, reverse
+from django.utils import timezone
+from django.utils.html import escape
+from django.utils.safestring import mark_safe
+from django.utils.translation import gettext_lazy as _
+
+from .models import TaskConfig, TaskLog, TaskLogStatistics
+from .utils import sort_dict_recursively
+from .widgets import PrettyJSONWidget
+
+
+@admin.register(TaskConfig)
+class TaskConfigAdmin(admin.ModelAdmin):
+    list_display = ("logs_expiration_days",)
+
+    def has_add_permission(self, request):
+        return not TaskConfig.objects.exists()
+
+    def has_delete_permission(self, request, obj=None):
+        return False
+
+
+@admin.register(TaskLog)
+class TaskLogAdmin(admin.ModelAdmin):
+    model = TaskLog
+    date_hierarchy = "timestamp"
+    list_display = ("id", "task_name", "_timestamp", "status", "result_snippet")
+    list_filter = ("status", "timestamp", "task_name", "queue_name")
+    readonly_fields = (
+        "timestamp",
+        "worker",
+        "status",
+        "task_id",
+        "task_name",
+        "periodic_task_name",
+        "queue_name",
+        "exc_type",
+        "exc_msg",
+        "result_pretty",
+        "traceback_pretty",
+    )
+    search_fields = ("task_name", "task_id", "worker")
+    fieldsets = (
+        (
+            None,
+            {
+                "fields": ("task_id", "task_name", "periodic_task_name", "queue_name", "status", "worker"),
+                "classes": ("extrapretty", "wide"),
+            },
+        ),
+        (_("Parameters"), {"fields": ("task_args", "task_kwargs"), "classes": ("extrapretty", "wide")}),
+        (
+            _("Result"),
+            {
+                "fields": ("result_pretty", "timestamp", "exc_type", "exc_msg", "traceback_pretty"),
+                "classes": ("extrapretty", "wide"),
+            },
+        ),
+    )
+    formfield_overrides = {JSONField: {"widget": PrettyJSONWidget}}
+    list_per_page = 25
+
+    def get_changelist_instance(self, request):
+        list_per_page_options = [25, 50, 75, 100]
+        selected_per_page = request.GET.get("list_per_page", "25")
+
+        try:
+            selected_per_page = int(selected_per_page)
+            self.list_per_page = selected_per_page if selected_per_page in list_per_page_options else 25
+        except (ValueError, TypeError):
+            self.list_per_page = 25
+
+        if "list_per_page" in request.GET:
+            get_params = request.GET.copy()
+            del get_params["list_per_page"]
+            request.GET = get_params
+
+        return super().get_changelist_instance(request)
+
+    def changelist_view(self, request, extra_context=None):
+        list_per_page_options = [25, 50, 75, 100]
+        selected_per_page = request.GET.get("list_per_page", "25")
+
+        try:
+            selected_per_page = int(selected_per_page)
+            if selected_per_page not in list_per_page_options:
+                selected_per_page = 25
+        except (ValueError, TypeError):
+            selected_per_page = 25
+
+        extra_context = extra_context or {}
+        extra_context.update(
+            {
+                "list_per_page_options": list_per_page_options,
+                "selected_per_page": selected_per_page,
+            }
+        )
+        return super().changelist_view(request, extra_context=extra_context)
+
+    def has_add_permission(self, request):
+        return False
+
+    def has_change_permission(self, request, obj=None):
+        return False
+
+    def get_urls(self):
+        custom_urls = [
+            path(
+                "<path:object_id>/rerun/",
+                self.admin_site.admin_view(self.rerun_task_view),
+                name="celery_log_tasklog_rerun",
+            ),
+        ]
+        return custom_urls + super().get_urls()
+
+    def rerun_task_view(self, request, object_id):
+        task_log = self.get_object(request, object_id)
+        if not task_log:
+            messages.error(request, _("Task log not found."))
+            return HttpResponseRedirect(reverse("admin:celery_log_tasklog_changelist"))
+
+        try:
+            result = current_app.send_task(
+                task_log.task_name,
+                args=task_log.task_args or [],
+                kwargs=task_log.task_kwargs or {},
+                queue=task_log.queue_name,
+            )
+            messages.success(request, _("Task '%(task)s' has been re-queued with ID: %(task_id)s") % {
+                "task": task_log.task_name,
+                "task_id": result.id,
+            })
+        except Exception as exc:
+            messages.error(request, _("Failed to re-run task '%(task)s': %(error)s") % {
+                "task": task_log.task_name,
+                "error": exc,
+            })
+
+        return HttpResponseRedirect(reverse("admin:celery_log_tasklog_change", args=[object_id]))
+
+    def exc_type(self, obj):
+        return (obj.traceback or {}).get("exc_type", "")
+
+    exc_type.short_description = _("Exception Type")
+
+    def exc_msg(self, obj):
+        return (obj.traceback or {}).get("exc_msg", "")
+
+    exc_msg.short_description = _("Exception Message")
+
+    def result_pretty(self, obj):
+        data = obj.result or {}
+        safe = escape(json.dumps(data, indent=2, sort_keys=True, default=str))
+        return mark_safe(
+            '<pre style="background:#f7f7f7;padding:8px;border:1px solid #ddd;'
+            'white-space:pre-wrap;word-break:break-all;max-height:500px;overflow:auto;">'
+            f"{safe}</pre>"
+        )
+
+    result_pretty.short_description = _("Result")
+
+    def result_snippet(self, obj):
+        data = obj.result or {}
+        if not data and obj.status == TaskLog.Status.FAILURE:
+            data = {"error": obj.error_message}
+
+        data = sort_dict_recursively(data)
+        compact = escape(json.dumps(data, separators=(",", ":"), default=str))
+        pretty = escape(json.dumps(data, indent=2, sort_keys=True, default=str))
+        return mark_safe(
+            '<details style="display:inline-block;max-width:500px;vertical-align:top;">'
+            '<summary style="white-space:nowrap;overflow:hidden;text-overflow:ellipsis;cursor:pointer;">'
+            f"{compact}</summary>"
+            '<pre style="background:#f7f7f7;padding:4px;border:1px solid #ddd;margin:4px 0;'
+            f'white-space:pre-wrap;word-wrap:break-word;max-height:500px;overflow:auto;">{pretty}</pre>'
+            "</details>"
+        )
+
+    result_snippet.short_description = _("Result")
+
+    def traceback_pretty(self, obj):
+        frames = (obj.traceback or {}).get("exc_tb", {}).get("frames", [])
+        html_chunks = []
+        total = len(frames)
+
+        for index, frame in enumerate(frames):
+            details_attr = " open" if index == total - 1 else ""
+            header = "Frame {index}: {func} @ {module}:{line}".format(
+                index=index,
+                func=frame.get("func_name"),
+                module=frame.get("module_name"),
+                line=frame.get("lineno"),
+            )
+            payload = escape(json.dumps(frame, indent=2, sort_keys=True, default=str))
+            html_chunks.append(
+                f'<details{details_attr} style="margin-bottom:8px;">'
+                f'<summary style="font-weight:bold;">{escape(header)}</summary>'
+                '<pre style="background:#f7f7f7;padding:8px;border:1px solid #ddd;'
+                f'white-space:pre-wrap;word-break:break-all;max-height:300px;overflow:auto;">{payload}</pre>'
+                "</details>"
+            )
+
+        return mark_safe("".join(html_chunks)) if html_chunks else _("No traceback available.")
+
+    traceback_pretty.short_description = _("Traceback details")
+
+    def _timestamp(self, obj):
+        return obj.timestamp.strftime("%Y-%m-%d %H:%M")
+
+    _timestamp.short_description = _("Timestamp")
+    _timestamp.admin_order_field = "timestamp"
+
+
+@admin.register(TaskLogStatistics)
+class TaskLogStatisticsAdmin(admin.ModelAdmin):
+    model = TaskLogStatistics
+
+    def has_add_permission(self, request):
+        return False
+
+    def has_change_permission(self, request, obj=None):
+        return False
+
+    def has_delete_permission(self, request, obj=None):
+        return False
+
+    def changelist_view(self, request, extra_context=None):
+        days_to_show = getattr(settings, "CELERY_TASK_LOGS_EXPIRES", 2)
+        end_date = timezone.now().date()
+        start_date = end_date - timedelta(days=days_to_show)
+        base_queryset = TaskLog.objects.filter(timestamp__date__gte=start_date, timestamp__date__lte=end_date)
+
+        messages_per_day = (
+            base_queryset.extra(select={"day": "date(timestamp)"})
+            .values("day")
+            .annotate(total=Count("id"))
+            .order_by("day")
+        )
+        failures_per_day = (
+            base_queryset.filter(status=TaskLog.Status.FAILURE)
+            .extra(select={"day": "date(timestamp)"})
+            .values("day")
+            .annotate(failures=Count("id"))
+            .order_by("day")
+        )
+        success_failure_stats = base_queryset.aggregate(
+            total_success=Count("id", filter=Q(status=TaskLog.Status.SUCCESS)),
+            total_failure=Count("id", filter=Q(status=TaskLog.Status.FAILURE)),
+        )
+        top_tasks = list(
+            base_queryset.values("task_name")
+            .annotate(
+                execution_count=Count("id"),
+                success_count=Count("id", filter=Q(status=TaskLog.Status.SUCCESS)),
+                failure_count=Count("id", filter=Q(status=TaskLog.Status.FAILURE)),
+                success_rate=Count("id", filter=Q(status=TaskLog.Status.SUCCESS)) * 100.0 / Count("id"),
+                avg_duration=Avg("duration_secs", filter=Q(duration_secs__gt=0)),
+                max_duration=Max("duration_secs", filter=Q(duration_secs__gt=0)),
+            )
+            .order_by("-execution_count")[:10]
+        )
+        top_queues = list(
+            base_queryset.values("queue_name")
+            .annotate(
+                message_count=Count("id"),
+                success_count=Count("id", filter=Q(status=TaskLog.Status.SUCCESS)),
+                failure_count=Count("id", filter=Q(status=TaskLog.Status.FAILURE)),
+            )
+            .order_by("-message_count")[:10]
+        )
+        worker_distribution = list(
+            base_queryset.values("worker")
+            .annotate(
+                task_count=Count("id"),
+                success_count=Count("id", filter=Q(status=TaskLog.Status.SUCCESS)),
+                failure_count=Count("id", filter=Q(status=TaskLog.Status.FAILURE)),
+            )
+            .order_by("-task_count")[:10]
+        )
+        periodic_task_distribution = list(
+            base_queryset.filter(periodic_task_name__isnull=False)
+            .values("periodic_task_name")
+            .annotate(
+                execution_count=Count("id"),
+                success_count=Count("id", filter=Q(status=TaskLog.Status.SUCCESS)),
+                failure_count=Count("id", filter=Q(status=TaskLog.Status.FAILURE)),
+            )
+            .order_by("-execution_count")[:10]
+        )
+        error_analysis = list(
+            base_queryset.filter(status=TaskLog.Status.FAILURE, error_message__isnull=False)
+            .exclude(error_message="")
+            .values("error_message")
+            .annotate(error_count=Count("id"))
+            .order_by("-error_count")[:10]
+        )
+        slowest_tasks = list(
+            base_queryset.filter(duration_secs__gt=0)
+            .values("task_name")
+            .annotate(message_count=Count("id"), avg_duration=Avg("duration_secs"), max_duration=Max("duration_secs"))
+            .order_by("-avg_duration", "-max_duration")[:10]
+        )
+        performance_stats = base_queryset.filter(duration_secs__gt=0).aggregate(
+            avg_duration=Avg("duration_secs"),
+            min_duration=Min("duration_secs"),
+            max_duration=Max("duration_secs"),
+            total_with_duration=Count("id"),
+        )
+
+        for task in top_tasks + slowest_tasks:
+            task["avg_duration"] = int(round(task.get("avg_duration") or 0))
+
+        chart_data = {
+            "messages_per_day": list(messages_per_day),
+            "failures_per_day": list(failures_per_day),
+            "success_failure_stats": success_failure_stats,
+            "top_queues": top_queues,
+            "top_tasks": top_tasks,
+            "worker_distribution": worker_distribution,
+            "periodic_task_distribution": periodic_task_distribution,
+            "error_analysis": error_analysis,
+            "slowest_tasks": slowest_tasks,
+            "date_range": {"start": start_date.isoformat(), "end": end_date.isoformat()},
+        }
+
+        total_messages = success_failure_stats["total_success"] + success_failure_stats["total_failure"]
+        days = max((end_date - start_date).days + 1, 1)
+        extra_context = extra_context or {}
+        extra_context.update(
+            {
+                "chart_data": json.dumps(chart_data, default=str),
+                "days_to_show": days_to_show,
+                "top_tasks": top_tasks,
+                "slowest_tasks": slowest_tasks,
+                "statistics": {
+                    "total_messages": total_messages,
+                    "total_success": success_failure_stats["total_success"],
+                    "total_failure": success_failure_stats["total_failure"],
+                    "success_rate": round((success_failure_stats["total_success"] / max(total_messages, 1)) * 100, 2),
+                    "avg_messages_per_day": int(round(total_messages / days)),
+                    "avg_messages_per_hour": int(round(total_messages / max(days * 24, 1))),
+                    "avg_messages_per_minute": int(round(total_messages / max(days * 24 * 60, 1))),
+                    "avg_duration": int(round(performance_stats["avg_duration"] or 0)),
+                    "max_duration": performance_stats["max_duration"] or 0,
+                    "min_duration": performance_stats["min_duration"] or 0,
+                    "total_with_duration": performance_stats["total_with_duration"],
+                },
+            }
+        )
+        return super().changelist_view(request, extra_context=extra_context)

django_celery_logs/apps.py

@@ -0,0 +1,12 @@
+from django.apps import AppConfig
+from django.utils.translation import gettext_lazy as _
+
+
+class CeleryLogsConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "django_celery_logs"
+    label = "celery_log"
+    verbose_name = _("Celery Task Logs")
+
+    def ready(self):
+        from . import signals  # noqa: F401

django_celery_logs/migrations/0001_initial.py

@@ -0,0 +1,65 @@
+import django_celery_logs.models
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    initial = True
+
+    dependencies = []
+
+    operations = [
+        migrations.CreateModel(
+            name="TaskConfig",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                (
+                    "logs_expiration_days",
+                    models.PositiveIntegerField(
+                        default=django_celery_logs.models.get_default_logs_expiration_days,
+                        help_text="Number of days to keep celery task logs.",
+                        verbose_name="Logs Expiration Days",
+                    ),
+                ),
+            ],
+            options={
+                "verbose_name": "Task Configuration",
+                "verbose_name_plural": "Task Configurations",
+            },
+        ),
+        migrations.CreateModel(
+            name="TaskLog",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("task_id", models.CharField(max_length=255, unique=True)),
+                ("task_name", models.CharField(max_length=255)),
+                ("periodic_task_name", models.CharField(blank=True, max_length=255, null=True)),
+                ("queue_name", models.CharField(blank=True, max_length=255, null=True)),
+                ("worker", models.CharField(max_length=255)),
+                ("status", models.CharField(choices=[("SUCCESS", "Success"), ("FAILURE", "Failure")], max_length=10)),
+                ("task_args", models.JSONField(blank=True, null=True)),
+                ("task_kwargs", models.JSONField(blank=True, null=True)),
+                ("result", models.JSONField(blank=True, null=True)),
+                ("duration_secs", models.FloatField(blank=True, null=True, verbose_name="Duration (Seconds)")),
+                ("error_message", models.TextField(blank=True)),
+                ("traceback", models.JSONField(blank=True, null=True)),
+                ("timestamp", models.DateTimeField(auto_now_add=True)),
+            ],
+            options={
+                "verbose_name": "Task Log",
+                "verbose_name_plural": "Task Logs",
+                "ordering": ["-timestamp"],
+            },
+        ),
+        migrations.CreateModel(
+            name="TaskLogStatistics",
+            fields=[],
+            options={
+                "verbose_name": "Task Log Statistics",
+                "verbose_name_plural": "Task Log Statistics",
+                "proxy": True,
+                "indexes": [],
+                "constraints": [],
+            },
+            bases=("celery_log.tasklog",),
+        ),
+    ]

django_celery_logs/migrations/__init__.py

@@ -0,0 +1 @@
+

django_celery_logs/models.py

@@ -0,0 +1,62 @@
+from django.conf import settings
+from django.db import models
+from django.utils.translation import gettext_lazy as _
+
+
+def get_default_logs_expiration_days():
+    return getattr(settings, "CELERY_TASK_LOGS_EXPIRES", 2)
+
+
+class TaskConfig(models.Model):
+    logs_expiration_days = models.PositiveIntegerField(
+        default=get_default_logs_expiration_days,
+        verbose_name=_("Logs Expiration Days"),
+        help_text=_("Number of days to keep celery task logs."),
+    )
+
+    class Meta:
+        verbose_name = _("Task Configuration")
+        verbose_name_plural = _("Task Configurations")
+
+    def __str__(self):
+        return str(_("Task Configuration"))
+
+    @classmethod
+    def get_solo(cls):
+        obj, _created = cls.objects.get_or_create(pk=1)
+        return obj
+
+
+class TaskLog(models.Model):
+    class Status(models.TextChoices):
+        SUCCESS = "SUCCESS", _("Success")
+        FAILURE = "FAILURE", _("Failure")
+
+    task_id = models.CharField(max_length=255, unique=True)
+    task_name = models.CharField(max_length=255)
+    periodic_task_name = models.CharField(max_length=255, null=True, blank=True)
+    queue_name = models.CharField(max_length=255, null=True, blank=True)
+    worker = models.CharField(max_length=255)
+    status = models.CharField(max_length=10, choices=Status.choices)
+    task_args = models.JSONField(null=True, blank=True)
+    task_kwargs = models.JSONField(null=True, blank=True)
+    result = models.JSONField(null=True, blank=True)
+    duration_secs = models.FloatField(null=True, blank=True, verbose_name=_("Duration (Seconds)"))
+    error_message = models.TextField(blank=True)
+    traceback = models.JSONField(blank=True, null=True)
+    timestamp = models.DateTimeField(auto_now_add=True)
+
+    class Meta:
+        ordering = ["-timestamp"]
+        verbose_name = _("Task Log")
+        verbose_name_plural = _("Task Logs")
+
+    def __str__(self):
+        return f"{self.task_id} ({self.task_name}) -> {self.status}"
+
+
+class TaskLogStatistics(TaskLog):
+    class Meta:
+        proxy = True
+        verbose_name = _("Task Log Statistics")
+        verbose_name_plural = _("Task Log Statistics")

django_celery_logs/signals.py

@@ -0,0 +1,13 @@
+from celery.signals import task_failure, task_success
+
+from .utils import create_task_log
+
+
+@task_success.connect
+def log_task_success(sender=None, result=None, **kwargs):
+    create_task_log(sender=sender, result=result)
+
+
+@task_failure.connect
+def log_task_failure(sender=None, exception=None, **kwargs):
+    create_task_log(sender=sender, exception=exception)

django_celery_logs/static/django_celery_logs/css/admin_filters.css

@@ -0,0 +1,7 @@
+#changelist-filter.hidden {
+    display: none !important;
+}
+
+.filtered.hidden-filters #changelist {
+    margin-right: 0 !important;
+}

django_celery_logs/static/django_celery_logs/js/admin_filters.js

@@ -0,0 +1,35 @@
+document.addEventListener("DOMContentLoaded", function () {
+    const toggleBtn = document.createElement("button");
+    const filterDiv = document.getElementById("changelist-filter");
+
+    if (!filterDiv) return;
+
+    const i18n = window.adminFiltersI18n || {
+        showFilters: "Show Filters",
+        hideFilters: "Hide Filters"
+    };
+
+    toggleBtn.className = "historylink";
+    toggleBtn.textContent = i18n.showFilters;
+    toggleBtn.type = "button";
+
+    filterDiv.classList.add("hidden");
+    document.body.classList.add("hidden-filters");
+    filterDiv.parentNode.insertBefore(toggleBtn, filterDiv);
+
+    toggleBtn.addEventListener("click", function () {
+        const currentlyHidden = filterDiv.classList.contains("hidden");
+
+        if (currentlyHidden) {
+            filterDiv.classList.remove("hidden");
+            document.body.classList.remove("hidden-filters");
+            toggleBtn.textContent = i18n.hideFilters;
+            localStorage.setItem("admin-filters-hidden", "false");
+        } else {
+            filterDiv.classList.add("hidden");
+            document.body.classList.add("hidden-filters");
+            toggleBtn.textContent = i18n.showFilters;
+            localStorage.setItem("admin-filters-hidden", "true");
+        }
+    });
+});

django_celery_logs/tasks.py

@@ -0,0 +1,45 @@
+import time
+from datetime import timedelta
+
+from celery import shared_task
+from celery.utils.log import get_task_logger
+from django.apps import apps as django_apps
+from django.db import transaction
+from django.utils import timezone
+
+logger = get_task_logger(__name__)
+
+
+@shared_task(name="clear_celery_task_logs")
+def clear_celery_task_logs():
+    start_time = time.perf_counter()
+    logger.debug("Starting clear_celery_task_logs")
+
+    TaskLog = django_apps.get_model("celery_log", "TaskLog")
+    TaskConfig = django_apps.get_model("celery_log", "TaskConfig")
+    config = TaskConfig.get_solo()
+    cutoff = timezone.now() - timedelta(days=config.logs_expiration_days)
+
+    total_deleted = 0
+    batch_size = 1000
+
+    try:
+        with transaction.atomic():
+            queryset = TaskLog.objects.filter(timestamp__lte=cutoff)
+            while True:
+                batch_ids = list(queryset.values_list("id", flat=True)[:batch_size])
+                if not batch_ids:
+                    break
+                deleted, _details = TaskLog.objects.filter(id__in=batch_ids).delete()
+                total_deleted += deleted
+    except Exception:
+        logger.exception("clear_celery_task_logs failed during batch delete")
+
+    duration = time.perf_counter() - start_time
+    logger.info("clear_celery_task_logs completed: removed %s entries in %.2fs", total_deleted, duration)
+
+    return {
+        "status": "Celery task logs cleaned",
+        "deleted": total_deleted,
+        "duration_s": round(duration, 2),
+    }

django_celery_logs/templates/admin/celery_log/tasklog/change_form.html

@@ -0,0 +1,21 @@
+{% extends "admin/change_form.html" %}
+{% load i18n admin_urls %}
+
+{% block object-tools %}
+    {% if change %}{% if not is_popup %}
+        <ul class="object-tools">
+            {% block object-tools-items %}
+                {% if original %}
+                    <li>
+                        <a href="{% url 'admin:celery_log_tasklog_rerun' original.pk %}"
+                           class="historylink"
+                           onclick="return confirm('{% trans "Are you sure you want to re-run this task?" %}');">
+                            {% trans "Re-run Task" %}
+                        </a>
+                    </li>
+                {% endif %}
+                {{ block.super }}
+            {% endblock %}
+        </ul>
+    {% endif %}{% endif %}
+{% endblock %}