django-simpletask5 0.1.0__py3-none-any.whl

django_simpletask5/__init__.py

@@ -0,0 +1,7 @@
+default_app_config = 'django_simpletask5.apps.DjangoSimpletask5Config'
+
+app_requires = [
+    'django_admin_dashboards',
+    'django_safe_fields',
+]
+

django_simpletask5/admin.py

@@ -0,0 +1,205 @@
+import json
+
+from django import forms
+from django.contrib import admin
+from django.utils.translation import gettext_lazy as _
+
+
+from django_simpletask5.models import TaskExecution, CronJob, TaskExecutionStat, TaskExecutionArchive
+from django_simpletask5.core.executor_scanner import (
+    get_executor_choices,
+    get_executor_schema_map,
+)
+
+
+class ExecutorClassWidget(forms.Select):
+    def __init__(self, attrs=None):
+        schema_map = get_executor_schema_map()
+        choices = [("", "---------")]
+        for group_name, items in get_executor_choices():
+            group_choices = [(v, v) for v, _ in items]
+            choices.append((group_name, group_choices))
+        attrs = attrs or {}
+        attrs.setdefault("class", "select2-executor")
+        attrs.setdefault("style", "width: 600px;")
+        attrs["data-schemas"] = json.dumps(schema_map, ensure_ascii=False)
+        super().__init__(attrs=attrs, choices=choices)
+
+
+class CronJobForm(forms.ModelForm):
+    executor_class = forms.CharField(
+        label=_("Executor class"),
+        widget=ExecutorClassWidget(),
+    )
+    context = forms.CharField(
+        label=_("Context"),
+        required=False,
+        widget=forms.Textarea(
+            attrs={
+                "rows": 8,
+                "style": "width: 600px; font-family: monospace;",
+            }
+        ),
+        help_text=_("Supports JSON or YAML format. YAML is more concise and compatible with JSON standard format."),
+    )
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if self.instance and self.instance.pk and self.instance.context:
+            try:
+                parsed = json.loads(self.instance.context)
+                import yaml
+
+                self.initial["context"] = yaml.dump(
+                    parsed,
+                    default_flow_style=False,
+                    allow_unicode=True,
+                    sort_keys=False,
+                ).strip()
+            except Exception:
+                pass
+
+    def clean_context(self):
+        raw = self.cleaned_data.get("context")
+        if not raw:
+            return None
+        try:
+            parsed = json.loads(raw)
+            return json.dumps(parsed, ensure_ascii=False)
+        except json.JSONDecodeError:
+            pass
+        try:
+            import yaml
+
+            parsed = yaml.safe_load(raw)
+            if parsed is None:
+                return None
+            return json.dumps(parsed, ensure_ascii=False)
+        except Exception:
+            raise forms.ValidationError(_("Invalid parameter format. Please use JSON or YAML format."))
+
+    class Meta:
+        model = CronJob
+        fields = "__all__"
+
+    class Media:
+        css = {
+            "all": ["admin/css/vendor/select2/select2.min.css"],
+        }
+        js = [
+            "admin/js/vendor/jquery/jquery.min.js",
+            "admin/js/vendor/select2/select2.full.min.js",
+            "django_simpletask5/js/select2_executor.js",
+            "admin/js/jquery.init.js",
+        ]
+
+
+@admin.register(TaskExecution)
+class TaskExecutionAdmin(admin.ModelAdmin):
+    list_display = [
+        "execution_id",
+        "task_id",
+        "trigger_event",
+        "executor_class",
+        "status",
+        "retry_count",
+        "max_retries",
+        "created_at",
+    ]
+    list_filter = ["status", "trigger_event", "created_at"]
+    search_fields = ["execution_id", "task_id", "executor_class"]
+    readonly_fields = ["execution_id", "task_id", "created_at", "updated_at"]
+    ordering = ["-created_at"]
+
+
+@admin.register(CronJob)
+class CronJobAdmin(admin.ModelAdmin):
+    form = CronJobForm
+    list_display = [
+        "name",
+        "cron_expression",
+        "executor_class",
+        "is_active",
+        "is_modified_by_user",
+        "updated_at",
+    ]
+    list_filter = ["is_active", "is_modified_by_user"]
+    search_fields = ["name", "executor_class"]
+    actions = [
+        "reset_from_code",
+        "mark_as_modified",
+        "unmark_as_modified",
+        "activate_cronjobs",
+        "deactivate_cronjobs",
+    ]
+
+    def save_model(self, request, obj, form, change):
+        if change:
+            obj.is_modified_by_user = True
+        super().save_model(request, obj, form, change)
+
+    def save_related(self, request, form, formsets, change):
+        super().save_related(request, form, formsets, change)
+
+    def reset_from_code(self, request, queryset):
+        from django_simpletask5.core.cronjob_registry import _CRONJOB_REGISTRY
+
+        updated = 0
+        for cronjob in queryset:
+            if cronjob.name in _CRONJOB_REGISTRY:
+                registered = _CRONJOB_REGISTRY[cronjob.name]
+                cronjob.cron_expression = registered["cron_expression"]
+                cronjob.executor_class = registered["executor_class"]
+                cronjob.context = registered.get("context")
+                cronjob.description = registered.get("description", "")
+                cronjob.is_modified_by_user = False
+                cronjob.save()
+                updated += 1
+        self.message_user(request, f"{updated} cronjob(s) reset from code definitions.")
+
+    reset_from_code.short_description = _("Reset from code definitions")
+
+    def mark_as_modified(self, request, queryset):
+        updated = queryset.update(is_modified_by_user=True)
+        self.message_user(request, f"{updated} cronjob(s) marked as user-modified.")
+
+    mark_as_modified.short_description = _("Mark as user-modified")
+
+    def unmark_as_modified(self, request, queryset):
+        updated = queryset.update(is_modified_by_user=False)
+        self.message_user(request, f"{updated} cronjob(s) unmarked as user-modified.")
+
+    unmark_as_modified.short_description = _("Unmark as user-modified")
+
+    def activate_cronjobs(self, request, queryset):
+        updated = queryset.update(is_active=True)
+        self.message_user(request, f"{updated} cronjob(s) activated.")
+
+    activate_cronjobs.short_description = _("Activate selected tasks")
+
+    def deactivate_cronjobs(self, request, queryset):
+        updated = queryset.update(is_active=False)
+        self.message_user(request, f"{updated} cronjob(s) deactivated.")
+
+    deactivate_cronjobs.short_description = _("Deactivate selected tasks")
+
+
+@admin.register(TaskExecutionStat)
+class TaskExecutionStatAdmin(admin.ModelAdmin):
+    list_display = [
+        "date",
+        "trigger_event",
+        "total_count",
+        "success_count",
+        "failed_count",
+        "avg_duration_seconds",
+    ]
+    list_filter = ["date", "trigger_event"]
+    ordering = ["-date"]
+
+
+@admin.register(TaskExecutionArchive)
+class TaskExecutionArchiveAdmin(admin.ModelAdmin):
+    list_display = ["archive_date", "created_at"]
+    readonly_fields = ["archive_date", "file", "created_at"]
+    ordering = ["-archive_date"]

django_simpletask5/apps.py

@@ -0,0 +1,8 @@
+from django.apps import AppConfig
+from django.utils.translation import gettext_lazy as _
+
+
+class DjangoSimpletask5Config(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'django_simpletask5'
+    verbose_name = _('Simple Task Manager')

django_simpletask5/core/cronjob_registry.py

@@ -0,0 +1,96 @@
+import json
+import logging
+from datetime import datetime
+
+from django.conf import settings
+from django.utils import timezone
+
+logger = logging.getLogger(__name__)
+
+_CRONJOB_REGISTRY = {}
+
+
+def register_cronjob(name, cron_expression, executor_class, context=None, description=''):
+    _CRONJOB_REGISTRY[name] = {
+        'cron_expression': cron_expression,
+        'executor_class': executor_class,
+        'context': json.dumps(context) if context else None,
+        'description': description,
+    }
+
+
+def _compute_next_run_time(cron_expression, base=None):
+    try:
+        from croniter import croniter
+    except ImportError:
+        return None
+    if base is None:
+        base = timezone.now()
+    try:
+        cron = croniter(cron_expression, base)
+        return cron.get_next(datetime)
+    except (ValueError, KeyError):
+        return None
+
+
+def sync_cronjobs():
+    from django_simpletask5.models import CronJob
+
+    if not getattr(settings, 'DJANGO_SIMPLETASK_CRONJOB_AUTO_SYNC', True):
+        return
+
+    try:
+        CronJob.objects.exists()
+    except Exception:
+        return
+
+    for name, config in _CRONJOB_REGISTRY.items():
+        try:
+            cronjob = CronJob.objects.get(name=name)
+            if not cronjob.is_modified_by_user:
+                cronjob.cron_expression = config['cron_expression']
+                cronjob.executor_class = config['executor_class']
+                cronjob.context = config.get('context')
+                cronjob.description = config.get('description', '')
+                if cronjob.next_run_time is None:
+                    cronjob.next_run_time = _compute_next_run_time(config['cron_expression'])
+                cronjob.save()
+        except CronJob.DoesNotExist:
+            CronJob.objects.create(
+                name=name,
+                cron_expression=config['cron_expression'],
+                executor_class=config['executor_class'],
+                context=config.get('context'),
+                description=config.get('description', ''),
+                is_modified_by_user=False,
+                next_run_time=_compute_next_run_time(config['cron_expression']),
+            )
+        except Exception:
+            logger.exception('Failed to sync cronjob %s', name)
+
+
+def register_builtin_cronjobs():
+    register_cronjob(
+        name='simpletask_status_check',
+        cron_expression='*/5 * * * *',
+        executor_class='django_simpletask5.executors.status_check.StatusCheckExecutor',
+        description='Scan long-running pending/running tasks and mark them as timeout',
+    )
+    register_cronjob(
+        name='simpletask_retry_timeout',
+        cron_expression='*/10 * * * *',
+        executor_class='django_simpletask5.executors.retry_timeout.RetryTimeoutExecutor',
+        description='Retry timeout tasks that still have retry attempts remaining',
+    )
+    register_cronjob(
+        name='simpletask_archive',
+        cron_expression='0 2 * * *',
+        executor_class='django_simpletask5.executors.archive.ArchiveExecutor',
+        description='Archive previous day TaskExecution records',
+    )
+    register_cronjob(
+        name='simpletask_ping_pong',
+        cron_expression='* * * * *',
+        executor_class='django_simpletask5.executors.ping_pong.PingPongExecutor',
+        description='Ping pong health check',
+    )

django_simpletask5/core/defaults.py

@@ -0,0 +1,24 @@
+from django.conf import settings
+
+DJANGO_SIMPLETASK_LOCK_TIMEOUT = getattr(settings, 'DJANGO_SIMPLETASK_LOCK_TIMEOUT', 600)
+
+DJANGO_SIMPLETASK_DEFAULT_MAX_RETRIES = getattr(settings, 'DJANGO_SIMPLETASK_DEFAULT_MAX_RETRIES', 3)
+DJANGO_SIMPLETASK_DEFAULT_TIMEOUT_SECONDS = getattr(settings, 'DJANGO_SIMPLETASK_DEFAULT_TIMEOUT_SECONDS', 300)
+
+DJANGO_SIMPLETASK_UPDATE_FILTER_POLICY = getattr(settings, 'DJANGO_SIMPLETASK_UPDATE_FILTER_POLICY', 'whitelist')
+
+DJANGO_SIMPLETASK_ARCHIVE_PATH = getattr(settings, 'DJANGO_SIMPLETASK_ARCHIVE_PATH', 'django_simpletask5_archives')
+DJANGO_SIMPLETASK_ARCHIVE_SALT = getattr(settings, 'DJANGO_SIMPLETASK_ARCHIVE_SALT', 'django-simpletask5-archive')
+
+DJANGO_SIMPLETASK_CRONJOB_AUTO_SYNC = getattr(settings, 'DJANGO_SIMPLETASK_CRONJOB_AUTO_SYNC', True)
+
+DJANGO_SIMPLETASK_DEFAULT_QUEUE = getattr(settings, 'DJANGO_SIMPLETASK_DEFAULT_QUEUE', 'django_simpletask5.queue.default')
+DJANGO_SIMPLETASK_HIGH_PRIORITY_QUEUE = getattr(settings, 'DJANGO_SIMPLETASK_HIGH_PRIORITY_QUEUE', 'django_simpletask5.queue.high_priority')
+
+DJANGO_SIMPLETASK_SCRIPT_WHITELIST = getattr(settings, 'DJANGO_SIMPLETASK_SCRIPT_WHITELIST', [])
+
+DJANGO_SIMPLETASK_FIELD_CIPHER_CLASS = getattr(settings, 'DJANGO_SIMPLETASK_FIELD_CIPHER_CLASS', None)
+
+DJANGO_SIMPLETASK_RESULT_PASSWORD = getattr(settings, 'DJANGO_SIMPLETASK_RESULT_PASSWORD', None)
+DJANGO_SIMPLETASK_ERROR_MSG_PASSWORD = getattr(settings, 'DJANGO_SIMPLETASK_ERROR_MSG_PASSWORD', None)
+DJANGO_SIMPLETASK_CONTEXT_PASSWORD = getattr(settings, 'DJANGO_SIMPLETASK_CONTEXT_PASSWORD', None)

django_simpletask5/core/executor_scanner.py

@@ -0,0 +1,82 @@
+import importlib
+import inspect
+import logging
+import pkgutil
+
+from django.apps import apps
+
+from django_simpletask5.executors.base import BaseExecutor
+
+logger = logging.getLogger(__name__)
+
+
+def _scan_module(module, base_path, app_label, verbose_name, executors):
+    for name, cls in inspect.getmembers(module, inspect.isclass):
+        if cls is BaseExecutor:
+            continue
+        if not issubclass(cls, BaseExecutor):
+            continue
+        path = f'{base_path}.{name}'
+        schema = []
+        try:
+            schema = cls.get_parameter_schema()
+        except Exception:
+            pass
+        executors[path] = {
+            'name': name,
+            'path': path,
+            'app_label': app_label,
+            'app_name': verbose_name,
+            'schema': schema,
+        }
+
+
+def _scan_package(package_name, app_label, verbose_name, executors):
+    try:
+        package = importlib.import_module(package_name)
+    except ImportError:
+        return
+    _scan_module(package, package_name, app_label, verbose_name, executors)
+    try:
+        for importer, modname, ispkg in pkgutil.iter_modules(package.__path__):
+            if modname.startswith('_'):
+                continue
+            full_name = f'{package_name}.{modname}'
+            if ispkg:
+                _scan_package(full_name, app_label, verbose_name, executors)
+            else:
+                try:
+                    mod = importlib.import_module(full_name)
+                    _scan_module(mod, full_name, app_label, verbose_name, executors)
+                except ImportError:
+                    continue
+    except Exception:
+        pass
+
+
+def discover_executors():
+    executors = {}
+    for app_config in apps.get_app_configs():
+        app_label = app_config.label
+        verbose_name = app_config.verbose_name
+        _scan_package(f'{app_config.name}.executors', app_label, verbose_name, executors)
+    return executors
+
+
+def get_executor_choices():
+    executors = discover_executors()
+    groups = {}
+    for path, info in executors.items():
+        group = groups.setdefault(info['app_name'], [])
+        group.append((path, f"{info['name']}"))
+    sorted_groups = sorted(groups.items(), key=lambda x: x[0])
+    choices = []
+    for group_name, items in sorted_groups:
+        sorted_items = sorted(items, key=lambda x: x[1])
+        choices.append((group_name, sorted_items))
+    return choices
+
+
+def get_executor_schema_map():
+    executors = discover_executors()
+    return {path: info['schema'] for path, info in executors.items()}

django_simpletask5/core/lock.py

@@ -0,0 +1,24 @@
+import logging
+
+from django.conf import settings
+
+logger = logging.getLogger(__name__)
+
+
+def get_lock_manager():
+    from globallock import GlobalLockManager
+    config = getattr(settings, 'DJANGO_SIMPLETASK_LOCK_CONFIG', None)
+    if config is None:
+        raise RuntimeError(
+            'DJANGO_SIMPLETASK_LOCK_CONFIG is not configured. '
+            'Set it in Django settings, e.g.:\n'
+            'DJANGO_SIMPLETASK_LOCK_CONFIG = {\n'
+            '    "global_lock_engine_class": "globallock.redis_global_lock.RedisGlobalLock",\n'
+            '    "global_lock_engine_options": {\n'
+            '        "host": "127.0.0.1",\n'
+            '        "port": 6379,\n'
+            '        "db": 0,\n'
+            '    },\n'
+            '}'
+        )
+    return GlobalLockManager(config)

django_simpletask5/core/message_queue.py

@@ -0,0 +1,102 @@
+import logging
+import socket
+
+from django.conf import settings
+from kombu import Connection, Exchange, Queue, Producer, Consumer
+
+logger = logging.getLogger(__name__)
+
+
+def _get_broker_url():
+    url = getattr(settings, 'DJANGO_SIMPLETASK_BROKER_URL', None)
+    if url:
+        return url
+    transport = getattr(settings, 'DJANGO_SIMPLETASK_MESSAGE_QUEUE_TYPE', 'memory')
+    if transport == 'rabbitmq':
+        host = getattr(settings, 'DJANGO_SIMPLETASK_RABBITMQ_HOST', '127.0.0.1')
+        port = getattr(settings, 'DJANGO_SIMPLETASK_RABBITMQ_PORT', 5672)
+        user = getattr(settings, 'DJANGO_SIMPLETASK_RABBITMQ_USER', 'guest')
+        pw = getattr(settings, 'DJANGO_SIMPLETASK_RABBITMQ_PASSWORD', 'guest')
+        return f'amqp://{user}:{pw}@{host}:{port}//'
+    if transport == 'redis':
+        host = getattr(settings, 'DJANGO_SIMPLETASK_REDIS_MQ_HOST', '127.0.0.1')
+        port = getattr(settings, 'DJANGO_SIMPLETASK_REDIS_MQ_PORT', 6379)
+        db = getattr(settings, 'DJANGO_SIMPLETASK_REDIS_MQ_DB', 0)
+        return f'redis://{host}:{port}/{db}'
+    return 'memory://'
+
+
+class KombuQueue:
+    """Kombu-based message queue with publish/consume support."""
+
+    def __init__(self):
+        self._exchange = Exchange('django_simpletask5', type='direct', durable=True)
+        self._stopped = False
+
+    def _declare(self, conn, routing_key):
+        ch = conn.channel()
+        self._exchange.declare(channel=ch)
+        queue = Queue(routing_key, self._exchange, routing_key=routing_key, durable=True)
+        queue.declare(channel=ch)
+        ch.close()
+
+    def publish(self, routing_key, message):
+        with Connection(_get_broker_url()) as conn:
+            conn.connect()
+            self._declare(conn, routing_key)
+            with Producer(conn) as producer:
+                producer.publish(
+                    message,
+                    exchange=self._exchange,
+                    routing_key=routing_key,
+                    serializer='json',
+                    retry=True,
+                )
+
+    def consume(self, queue_name, callback, auto_ack=True):
+        retry_delay = 1
+        while not self._stopped:
+            try:
+                with Connection(_get_broker_url()) as conn:
+                    conn.connect()
+                    queue = Queue(queue_name, self._exchange, routing_key=queue_name, durable=True)
+
+                    with Consumer(
+                        conn,
+                        queues=[queue],
+                        callbacks=[lambda body, msg: self._on_message(body, msg, callback, auto_ack)],
+                        accept=['json'],
+                    ):
+                        retry_delay = 1
+                        while not self._stopped:
+                            try:
+                                conn.drain_events(timeout=0.5)
+                            except socket.timeout:
+                                continue
+            except Exception as e:
+                if not self._stopped:
+                    logger.debug('Consumer connection lost: %s, reconnecting in %ss...', e, retry_delay)
+                    import time as _time
+                    _time.sleep(retry_delay)
+                    retry_delay = min(retry_delay * 2, 10)
+
+    @staticmethod
+    def _on_message(body, message, callback, auto_ack):
+        try:
+            result = callback(body)
+            if result is False:
+                message.reject(requeue=True)
+                return
+        except Exception:
+            logger.exception('Error in message callback')
+            message.reject(requeue=True)
+            return
+        if auto_ack:
+            message.ack()
+
+    def stop(self):
+        self._stopped = True
+
+
+def get_message_queue():
+    return KombuQueue()

django_simpletask5/core/publisher.py

@@ -0,0 +1,93 @@
+import json
+import logging
+
+
+from django_simpletask5.models import TaskExecution
+from django_simpletask5.core.defaults import DJANGO_SIMPLETASK_DEFAULT_QUEUE
+
+logger = logging.getLogger(__name__)
+
+def create_and_publish(task, trigger_event, extra_context=None):
+    executor_class_path = task.get_executor_class_path(event=trigger_event)
+    queue_name = task.get_queue_name(event=trigger_event)
+
+    context = {
+        'task_id': str(task.task_id),
+        'trigger_event': trigger_event,
+        'data': _model_to_dict(task),
+    }
+    if extra_context:
+        context.update(extra_context)
+
+    serialized_context = _serialize_context(context)
+
+    execution = TaskExecution(
+        task_id=task.task_id,
+        task_model=task._meta.label,
+        trigger_event=trigger_event,
+        executor_class=executor_class_path,
+        status='pending',
+        context=json.dumps(serialized_context, default=str) if serialized_context else None,
+    )
+    execution.save()
+
+    _publish_message(execution, routing_key=queue_name)
+
+    return execution
+
+
+def _publish_message(execution, routing_key=None):
+    message = {
+        'execution_id': str(execution.execution_id),
+        'task_id': str(execution.task_id) if execution.task_id else None,
+        'trigger_event': execution.trigger_event,
+        'executor_class': execution.executor_class,
+    }
+
+    try:
+        from django_simpletask5.core.message_queue import get_message_queue
+        mq = get_message_queue()
+        mq.publish(
+            routing_key=routing_key or DJANGO_SIMPLETASK_DEFAULT_QUEUE,
+            message=message,
+        )
+        logger.debug('Published message for execution %s', execution.execution_id)
+    except Exception:
+        logger.exception('Failed to publish message for execution %s', execution.execution_id)
+
+
+def _serialize_context(context):
+    if context is None:
+        return None
+    serialized = {}
+    for key, value in context.items():
+        if isinstance(value, dict):
+            serialized[key] = {k: _serialize_value(v) for k, v in value.items()}
+        else:
+            serialized[key] = _serialize_value(value)
+    return serialized
+
+
+def _serialize_value(value):
+    if value is None or isinstance(value, (str, bool)):
+        return value
+    return str(value)
+
+
+def _model_to_dict(instance):
+    data = {}
+    for field in instance._meta.fields:
+        value = getattr(instance, field.attname)
+        if value is None:
+            data[field.attname] = None
+        elif isinstance(value, bool):
+            data[field.attname] = value
+        elif isinstance(value, (int, float)):
+            data[field.attname] = str(value)
+        elif isinstance(value, str):
+            data[field.attname] = value
+        elif isinstance(value, bytes):
+            data[field.attname] = value.decode('utf-8', errors='replace')
+        else:
+            data[field.attname] = str(value)
+    return data