django-simpletask5 0.1.0__py3-none-any.whl

django_simpletask5/core/signals.py

@@ -0,0 +1,120 @@
+import threading
+
+from django.db.models.signals import post_save, pre_save
+from django.dispatch import receiver
+
+from django_simpletask5.models import Task
+from django_simpletask5.core.publisher import create_and_publish, _model_to_dict
+
+
+class _DeleteIntercepted(Exception):
+    def __init__(self, instance):
+        self.instance = instance
+        super().__init__('Delete intercepted, TaskExecution created instead')
+
+
+_pre_save_data = threading.local()
+
+
+@receiver(pre_save)
+def handle_task_pre_save(sender, instance, **kwargs):
+    if not _is_task_subclass(sender):
+        return
+    if instance.pk and not getattr(instance, '_skip_pre_save_snapshot', False):
+        try:
+            old = sender.objects.get(pk=instance.pk)
+            _pre_save_data.data = _model_to_dict(old)
+        except sender.DoesNotExist:
+            _pre_save_data.data = None
+    else:
+        _pre_save_data.data = None
+
+
+@receiver(post_save)
+def handle_task_post_save(sender, instance, created, **kwargs):
+    if not _is_task_subclass(sender):
+        return
+    if created:
+        create_and_publish(
+            task=instance,
+            trigger_event='create',
+        )
+    else:
+        old_data = getattr(_pre_save_data, 'data', None)
+        new_data = _model_to_dict(instance)
+        if old_data:
+            changed_fields = _get_changed_fields(old_data, new_data)
+        else:
+            changed_fields = list(new_data.keys())
+        if not changed_fields:
+            return
+        if not _should_trigger_update(instance, changed_fields):
+            return
+        create_and_publish(
+            task=instance,
+            trigger_event='update',
+            extra_context={
+                'old_data': old_data or {},
+                'new_data': new_data,
+                'changed_fields': changed_fields,
+            },
+        )
+
+
+def _is_task_subclass(sender):
+    if not issubclass(sender, Task):
+        return False
+    if sender._meta.abstract:
+        return False
+    return True
+
+
+def _should_trigger_update(instance, update_fields):
+    whitelist = instance.get_trigger_update_fields()
+    blacklist = instance.get_trigger_update_blacklist()
+
+    if whitelist is not None:
+        if update_fields:
+            return any(f in whitelist for f in update_fields)
+        return True
+
+    if blacklist is not None:
+        if update_fields:
+            return not all(f in blacklist for f in update_fields)
+        return True
+
+    return True
+
+
+def _get_changed_field_names(old_data, new_data):
+    changed = []
+    for key in new_data:
+        if key in old_data:
+            old_val = old_data[key]
+            new_val = new_data[key]
+            if old_val != new_val:
+                try:
+                    from decimal import Decimal
+                    if Decimal(str(old_val)) == Decimal(str(new_val)):
+                        continue
+                except Exception:
+                    pass
+                changed.append(key)
+    return changed
+
+
+def _get_changed_fields(old, new):
+    changed = []
+    for key in new:
+        if key in old:
+            old_val = old[key]
+            new_val = new[key]
+            if old_val != new_val:
+                try:
+                    from decimal import Decimal
+                    if Decimal(str(old_val)) == Decimal(str(new_val)):
+                        continue
+                except Exception:
+                    pass
+                changed.append(key)
+    return changed

django_simpletask5/core/worker_registry.py

@@ -0,0 +1,134 @@
+import json
+import logging
+import os
+import socket
+import threading
+import time
+
+from django.conf import settings
+
+logger = logging.getLogger(__name__)
+
+WORKER_TTL = 300
+
+_REDIS_UNAVAILABLE = False
+
+
+def _get_redis_client():
+    import redis as _redis
+    lock_config = getattr(settings, 'DJANGO_SIMPLETASK_LOCK_CONFIG', {})
+    opts = lock_config.get('global_lock_engine_options', {})
+    return _redis.Redis(
+        host=opts.get('host', '127.0.0.1'),
+        port=opts.get('port', 6379),
+        db=opts.get('db', 0),
+        password=opts.get('password', None) or None,
+        decode_responses=True,
+        socket_connect_timeout=3,
+    )
+
+
+def _safe_redis_call(fn, default=None, log_level=logging.WARNING):
+    global _REDIS_UNAVAILABLE
+    try:
+        result = fn()
+        if _REDIS_UNAVAILABLE:
+            _REDIS_UNAVAILABLE = False
+            logger.info('Redis connection restored')
+        return result
+    except Exception as e:
+        if not _REDIS_UNAVAILABLE:
+            _REDIS_UNAVAILABLE = True
+            logger.log(log_level, 'Redis operation failed: %s', e)
+        return default
+
+
+def _worker_key(pid, tid):
+    return f'simpletask:worker:{pid}.{tid}'
+
+
+def _workers_pattern():
+    return 'simpletask:worker:*'
+
+
+def register_worker(queue_name, services=None, pid=None, tid=None):
+    pid = pid or os.getpid()
+    tid = tid or threading.get_ident()
+    key = _worker_key(pid, tid)
+    client = _get_redis_client()
+    info = {
+        'pid': str(pid),
+        'tid': str(tid),
+        'hostname': socket.gethostname(),
+        'queue': queue_name,
+        'services': json.dumps(services or []),
+        'status': 'idle',
+        'current_execution_id': '',
+        'started_at': str(time.time()),
+        'updated_at': str(time.time()),
+    }
+    _safe_redis_call(lambda: (client.hset(key, mapping=info), client.expire(key, WORKER_TTL)) and None)
+    return key
+
+
+def unregister_worker(pid=None, tid=None):
+    pid = pid or os.getpid()
+    tid = tid or threading.get_ident()
+    key = _worker_key(pid, tid)
+    client = _get_redis_client()
+    _safe_redis_call(lambda: client.delete(key))
+
+
+def update_worker_status(status, execution_id='', pid=None, tid=None):
+    pid = pid or os.getpid()
+    tid = tid or threading.get_ident()
+    key = _worker_key(pid, tid)
+    client = _get_redis_client()
+    _safe_redis_call(lambda: (client.hset(key, 'status', status), client.hset(key, 'current_execution_id', execution_id or ''), client.hset(key, 'updated_at', str(time.time())), client.expire(key, WORKER_TTL)) and None)
+
+
+def heartbeat_worker(pid=None, tid=None):
+    pid = pid or os.getpid()
+    tid = tid or threading.get_ident()
+    key = _worker_key(pid, tid)
+    client = _get_redis_client()
+    _safe_redis_call(lambda: (client.hset(key, 'updated_at', str(time.time())), client.expire(key, WORKER_TTL)) and None)
+
+
+def get_all_workers():
+    client = _get_redis_client()
+    keys = _safe_redis_call(lambda: client.keys(_workers_pattern()), default=[])
+    if not keys:
+        return []
+    workers = []
+    for key in keys:
+        data = _safe_redis_call(lambda k=key: client.hgetall(k), default={})
+        if data:
+            data['services'] = json.loads(data.get('services', '[]'))
+            data['redis_key'] = key
+            workers.append(data)
+    return workers
+
+
+def get_workers_by_queue(queue_name):
+    return [w for w in get_all_workers() if w.get('queue') == queue_name]
+
+
+def get_worker_stats():
+    workers = get_all_workers()
+    total = len(workers)
+    by_queue = {}
+    by_status = {}
+    for w in workers:
+        q = w.get('queue', 'unknown')
+        s = w.get('status', 'unknown')
+        by_queue.setdefault(q, 0)
+        by_queue[q] += 1
+        by_status.setdefault(s, 0)
+        by_status[s] += 1
+    return {
+        'total': total,
+        'by_queue': by_queue,
+        'by_status': by_status,
+        'workers': workers,
+    }

django_simpletask5/dashboards.py

@@ -0,0 +1,274 @@
+from datetime import date, timedelta
+
+from django.db.models import Count
+from django.utils.translation import gettext_lazy as _
+
+from django_admin_dashboards.base import Dashboard, Layout, CardComponent, ChartComponent, TableComponent
+
+from django_simpletask5.models import TaskExecution, CronJob, TaskExecutionArchive
+from django_simpletask5.core.worker_registry import get_worker_stats as _get_worker_stats
+
+
+class TaskDashboard(Dashboard):
+    title = _("Task Manager Dashboard")
+    auto_refresh = 60
+
+    class Media:
+        css = {
+            "all": (
+                "remixicon/remixicon.css",
+                "admin/css/vendor/select2/select2.min.css",
+                "django_admin_dashboards/css/dashboard.css",
+                "django_simpletask5/css/dashboard.css",
+            ),
+        }
+        js = (
+            "django_admin_dashboards/js/chart.umd.js",
+            "admin/js/vendor/jquery/jquery.min.js",
+            "admin/js/vendor/select2/select2.full.min.js",
+            "admin/js/jquery.init.js",
+            "django_admin_dashboards/js/dashboard.js",
+        )
+
+    def get_layout(self):
+        layout = Layout(columns=12)
+
+        today = date.today()
+
+        total_today = TaskExecution.objects.filter(
+            created_at__date=today
+        ).count()
+        success_today = TaskExecution.objects.filter(
+            created_at__date=today, status="success"
+        ).count()
+        pending = TaskExecution.objects.filter(status="pending").count()
+        running = TaskExecution.objects.filter(status="running").count()
+        active_cronjobs = CronJob.objects.filter(is_active=True).count()
+        total_cronjobs = CronJob.objects.count()
+        archives_count = TaskExecutionArchive.objects.count()
+        failed_today = TaskExecution.objects.filter(
+            created_at__date=today, status="failed"
+        ).count()
+
+        worker_stats = _get_worker_stats()
+        worker_total = worker_stats["total"]
+        worker_idle = worker_stats["by_status"].get("idle", 0)
+        worker_running_count = worker_stats["by_status"].get("running", 0)
+
+        success_rate = 0
+        if total_today > 0:
+            success_rate = round(success_today / total_today * 100)
+
+        layout.add_row([
+            (CardComponent(
+                title=_("Today Executions"),
+                value=total_today,
+                icon_class="ri-flashlight-line",
+                color="primary",
+                link="../django_simpletask5/taskexecution/?created_at__date__gte=" + today.isoformat(),
+            ), 3),
+            (CardComponent(
+                title=_("Success Rate"),
+                value=f"{success_rate}%",
+                change=f"{success_today}/{total_today}",
+                trend="up" if success_rate >= 80 else "down",
+                color="success" if success_rate >= 80 else "danger",
+                icon_class="ri-checkbox-circle-line",
+            ), 3),
+            (CardComponent(
+                title=_("Workers"),
+                value=worker_total,
+                values=[worker_idle, worker_running_count],
+                value_labels=[_("Idle"), _("Running")],
+                color="info",
+                icon_class="ri-server-line",
+            ), 3),
+            (CardComponent(
+                title=_("Cronjobs"),
+                value=active_cronjobs,
+                values=[active_cronjobs, total_cronjobs - active_cronjobs],
+                value_labels=[_("Active"), _("Inactive")],
+                links=[
+                    "../django_simpletask5/cronjob/?is_active__exact=1",
+                    "../django_simpletask5/cronjob/?is_active__exact=0",
+                ],
+                color="info",
+                icon_class="ri-calendar-check-line",
+            ), 3),
+        ])
+
+        layout.add_row([
+            (CardComponent(
+                title=_("Pending / Running"),
+                value=pending,
+                values=[pending, running],
+                value_labels=[_("Pending"), _("Running")],
+                links=[
+                    "../django_simpletask5/taskexecution/?status__exact=pending",
+                    "../django_simpletask5/taskexecution/?status__exact=running",
+                ],
+                color="warning",
+                icon_class="ri-timer-line",
+            ), 3),
+            (CardComponent(
+                title=_("Failed Today"),
+                value=failed_today,
+                icon_class="ri-error-warning-line",
+                color="danger",
+                link="../django_simpletask5/taskexecution/?status__exact=failed&created_at__date__gte=" + today.isoformat(),
+            ), 3),
+            (CardComponent(
+                title=_("Archives"),
+                value=archives_count,
+                icon_class="ri-archive-line",
+                color="secondary",
+                link="../django_simpletask5/taskexecutionarchive/",
+            ), 3),
+            (CardComponent(
+                title=_("Retries"),
+                value=TaskExecution.objects.filter(status="retry").count(),
+                icon_class="ri-refresh-line",
+                color="warning",
+                link="../django_simpletask5/taskexecution/?status__exact=retry",
+            ), 3),
+        ])
+
+        status_counts = TaskExecution.objects.values("status").annotate(
+            count=Count("execution_id")
+        )
+        status_labels = []
+        status_data = []
+        status_colors = []
+        color_map = {
+            "pending": "#ffc107",
+            "running": "#0d6efd",
+            "success": "#198754",
+            "failed": "#dc3545",
+            "retry": "#fd7e14",
+            "timeout": "#6c757d",
+        }
+        label_map = {
+            "pending": _("Pending"),
+            "running": _("Running"),
+            "success": _("Success"),
+            "failed": _("Failed"),
+            "retry": _("Retry"),
+            "timeout": _("Timeout"),
+        }
+        for item in status_counts:
+            status_labels.append(str(label_map.get(item["status"], item["status"])))
+            status_data.append(item["count"])
+            status_colors.append(color_map.get(item["status"], "#adb5bd"))
+
+        layout.add_row([
+            (ChartComponent(
+                title=_("Execution Status Distribution"),
+                chart_type="doughnut",
+                data={
+                    "labels": status_labels,
+                    "datasets": [{
+                        "data": status_data,
+                        "backgroundColor": status_colors,
+                    }],
+                },
+                options={"responsive": True, "maintainAspectRatio": False},
+                height=350,
+            ), 6),
+            (ChartComponent(
+                title=_("Daily Executions (Last 30 Days)"),
+                chart_type="bar",
+                data=self._daily_executions_chart(),
+                options={"responsive": True, "maintainAspectRatio": False, "scales": {"x": {"stacked": True}, "y": {"stacked": True}}},
+                height=350,
+            ), 6),
+        ])
+
+        recent_failed = TaskExecution.objects.filter(
+            status__in=["failed", "timeout"]
+        ).order_by("-created_at")[:10]
+
+        failed_table_data = []
+        for ex in recent_failed:
+            failed_table_data.append([
+                str(ex.execution_id)[:8],
+                ex.executor_class,
+                ex.trigger_event,
+                ex.status,
+                ex.created_at.strftime("%m-%d %H:%M"),
+            ])
+
+        layout.add_row([
+            (TableComponent(
+                title=_("Recent Failed / Timeout Executions"),
+                columns=[_("ID"), _("Executor"), _("Event"), _("Status"), _("Time")],
+                data=failed_table_data,
+            ), 6),
+            (TableComponent(
+                title=_("Recent Executions"),
+                columns=[_("ID"), _("Executor"), _("Event"), _("Status"), _("Time")],
+                data=[
+                    [
+                        str(ex.execution_id)[:8],
+                        ex.executor_class,
+                        ex.trigger_event,
+                        ex.status,
+                        ex.created_at.strftime("%m-%d %H:%M"),
+                    ]
+                    for ex in TaskExecution.objects.order_by("-created_at")[:10]
+                ],
+            ), 6),
+        ])
+
+        worker_rows = []
+        for w in worker_stats["workers"]:
+            worker_rows.append([
+                w.get("hostname", "?"),
+                w.get("queue", "?"),
+                w.get("status", "?"),
+                w.get("current_execution_id", "")[:8] or "-",
+                w.get("services", "-") if isinstance(w.get("services"), str) else ",".join(w.get("services", [])) or "-",
+            ])
+
+        layout.add_row([
+            (TableComponent(
+                title=_("Active Workers"),
+                columns=[_("Hostname"), _("Queue"), _("Status"), _("Current Execution"), _("Services")],
+                data=worker_rows,
+            ), 12),
+        ])
+
+        return layout
+
+    def _daily_executions_chart(self):
+        today = date.today()
+        labels = []
+        success_data = []
+        failed_data = []
+        for i in range(29, -1, -1):
+            d = today - timedelta(days=i)
+            labels.append(d.strftime("%m-%d"))
+            success_data.append(
+                TaskExecution.objects.filter(
+                    created_at__date=d, status="success"
+                ).count()
+            )
+            failed_data.append(
+                TaskExecution.objects.filter(
+                    created_at__date=d, status__in=["failed", "timeout"]
+                ).count()
+            )
+        return {
+            "labels": labels,
+            "datasets": [
+                {
+                    "label": str(_("Success")),
+                    "data": success_data,
+                    "backgroundColor": "#198754",
+                },
+                {
+                    "label": str(_("Failed/Timeout")),
+                    "data": failed_data,
+                    "backgroundColor": "#dc3545",
+                },
+            ],
+        }

django_simpletask5/executors/archive.py

@@ -0,0 +1,24 @@
+import json
+import logging
+
+from django.utils import timezone
+from datetime import timedelta
+
+from django_simpletask5.models import TaskExecution
+from django_simpletask5.executors.base import BaseExecutor
+
+logger = logging.getLogger(__name__)
+
+
+class ArchiveExecutor(BaseExecutor):
+    def execute(self, execution: TaskExecution) -> str | None:
+        from django_simpletask5.services.archive import archive_executions
+
+        yesterday = timezone.now().date() - timedelta(days=1)
+        count = archive_executions(yesterday)
+
+        return json.dumps({
+            'archived_count': count,
+            'archive_date': yesterday.isoformat(),
+            'success': True,
+        })

django_simpletask5/executors/base.py

@@ -0,0 +1,10 @@
+from django_simpletask5.models import TaskExecution
+
+
+class BaseExecutor:
+    def execute(self, execution: TaskExecution) -> str | None:
+        raise NotImplementedError('Subclasses must implement execute()')
+
+    @classmethod
+    def get_parameter_schema(cls):
+        return []

django_simpletask5/executors/bash_script.py

@@ -0,0 +1,61 @@
+import json
+import os
+import subprocess
+
+from django.conf import settings
+
+from django_simpletask5.models import TaskExecution
+from django_simpletask5.executors.base import BaseExecutor
+
+
+class BashScriptExecutor(BaseExecutor):
+    @classmethod
+    def get_parameter_schema(cls):
+        return [
+            {'name': 'script', 'type': 'text', 'required': False, 'default': '', 'description': 'Shell 脚本内容'},
+            {'name': 'script_path', 'type': 'string', 'required': False, 'default': '', 'description': 'Shell 脚本文件路径'},
+            {'name': 'args', 'type': 'array', 'required': False, 'default': [], 'description': '脚本参数列表'},
+            {'name': 'env', 'type': 'object', 'required': False, 'default': {}, 'description': '环境变量 JSON 对象'},
+        ]
+
+    def execute(self, execution: TaskExecution) -> str | None:
+        params = execution.get_context_dict()
+        script = params.get('script', '')
+        script_path = params.get('script_path', '')
+        args = params.get('args', [])
+        env = params.get('env', {})
+
+        if script_path:
+            script_whitelist = getattr(settings, 'DJANGO_SIMPLETASK_SCRIPT_WHITELIST', [])
+            if script_whitelist:
+                abs_path = os.path.abspath(script_path)
+                allowed = any(abs_path.startswith(os.path.abspath(p)) for p in script_whitelist)
+                if not allowed:
+                    raise Exception('Script path not in whitelist')
+            with open(script_path, 'r') as f:
+                script = f.read()
+
+        if not script:
+            raise Exception('No script provided')
+
+        cmd = ['bash', '-c', script] + list(args)
+        merged_env = os.environ.copy()
+        merged_env.update(env)
+
+        result = subprocess.run(
+            cmd,
+            capture_output=True,
+            text=True,
+            timeout=execution.timeout_seconds or 300,
+            env=merged_env,
+        )
+
+        if result.returncode != 0:
+            raise Exception(result.stderr[:2000])
+
+        return json.dumps({
+            'stdout': result.stdout[:2000],
+            'stderr': result.stderr[:2000],
+            'return_code': result.returncode,
+            'success': True,
+        })

django_simpletask5/executors/loader.py

@@ -0,0 +1,17 @@
+import importlib
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def load_executor(executor_class_path):
+    if not executor_class_path or '.' not in executor_class_path:
+        raise ImportError(f'Cannot load executor {executor_class_path}: invalid path')
+    try:
+        module_path, class_name = executor_class_path.rsplit('.', 1)
+        module = importlib.import_module(module_path)
+        executor_cls = getattr(module, class_name)
+        return executor_cls()
+    except (ImportError, AttributeError) as e:
+        logger.exception('Failed to load executor %s', executor_class_path)
+        raise ImportError(f'Cannot load executor {executor_class_path}: {e}')

django_simpletask5/executors/ping_pong.py

@@ -0,0 +1,12 @@
+import logging
+
+from django_simpletask5.models import TaskExecution
+from django_simpletask5.executors.base import BaseExecutor
+
+logger = logging.getLogger(__name__)
+
+
+class PingPongExecutor(BaseExecutor):
+    def execute(self, execution: TaskExecution) -> str | None:
+        logger.debug('PingPongExecutor executing execution %s', execution.pk)
+        return 'pong'