plain.jobs 0.33.0__py3-none-any.whl

plain/jobs/scheduling.py

@@ -0,0 +1,251 @@
+from __future__ import annotations
+
+import datetime
+import subprocess
+from typing import Any
+
+from plain.utils import timezone
+
+from .jobs import Job
+from .registry import jobs_registry, register_job
+
+_MONTH_NAMES = {
+    "JAN": 1,
+    "FEB": 2,
+    "MAR": 3,
+    "APR": 4,
+    "MAY": 5,
+    "JUN": 6,
+    "JUL": 7,
+    "AUG": 8,
+    "SEP": 9,
+    "OCT": 10,
+    "NOV": 11,
+    "DEC": 12,
+}
+_DAY_NAMES = {
+    "MON": 0,
+    "TUE": 1,
+    "WED": 2,
+    "THU": 3,
+    "FRI": 4,
+    "SAT": 5,
+    "SUN": 6,
+}
+
+
+class _ScheduleComponent:
+    def __init__(self, values: list[int], raw: str | int = "") -> None:
+        self.values = sorted(values)
+        self._raw = raw
+
+    def __str__(self) -> str:
+        if self._raw:
+            return str(self._raw)
+        return ",".join(str(v) for v in self.values)
+
+    def __eq__(self, other: Any) -> bool:
+        return self.values == other.values
+
+    @classmethod
+    def parse(
+        cls,
+        value: int | str,
+        min_allowed: int,
+        max_allowed: int,
+        str_conversions: dict[str, int] | None = None,
+    ) -> _ScheduleComponent:
+        if str_conversions is None:
+            str_conversions = {}
+
+        if isinstance(value, int):
+            if value < min_allowed or value > max_allowed:
+                raise ValueError(
+                    f"Schedule component should be between {min_allowed} and {max_allowed}"
+                )
+            return cls([value], raw=value)
+
+        if not isinstance(value, str):
+            raise ValueError("Schedule component should be an int or str")
+
+        # First split any subcomponents and re-parse them
+        if "," in value:
+            return cls(
+                sum(
+                    (
+                        cls.parse(
+                            sub_value, min_allowed, max_allowed, str_conversions
+                        ).values
+                        for sub_value in value.split(",")
+                    ),
+                    [],
+                ),
+                raw=value,
+            )
+
+        if value == "*":
+            return cls(list(range(min_allowed, max_allowed + 1)), raw=value)
+
+        def _convert(value: str) -> int:
+            result = str_conversions.get(value.upper(), value)
+            return int(result)
+
+        if "/" in value:
+            values, step = value.split("/")
+            values = cls.parse(values, min_allowed, max_allowed, str_conversions)
+            return cls([v for v in values.values if v % int(step) == 0], raw=value)
+
+        if "-" in value:
+            start, end = value.split("-")
+            return cls(list(range(_convert(start), _convert(end) + 1)), raw=value)
+
+        return cls([_convert(value)], raw=value)
+
+
+class Schedule:
+    def __init__(
+        self,
+        *,
+        minute: int | str = "*",
+        hour: int | str = "*",
+        day_of_month: int | str = "*",
+        month: int | str = "*",
+        day_of_week: int | str = "*",
+        raw: str = "",
+    ) -> None:
+        self.minute = _ScheduleComponent.parse(minute, min_allowed=0, max_allowed=59)
+        self.hour = _ScheduleComponent.parse(hour, min_allowed=0, max_allowed=23)
+        self.day_of_month = _ScheduleComponent.parse(
+            day_of_month, min_allowed=1, max_allowed=31
+        )
+        self.month = _ScheduleComponent.parse(
+            month,
+            min_allowed=1,
+            max_allowed=12,
+            str_conversions=_MONTH_NAMES,
+        )
+        self.day_of_week = _ScheduleComponent.parse(
+            day_of_week,
+            min_allowed=0,
+            max_allowed=6,
+            str_conversions=_DAY_NAMES,
+        )
+        self._raw = raw
+
+    def __str__(self) -> str:
+        if self._raw:
+            return self._raw
+        return f"{self.minute} {self.hour} {self.day_of_month} {self.month} {self.day_of_week}"
+
+    def __repr__(self) -> str:
+        return f"<Schedule {self}>"
+
+    @classmethod
+    def from_cron(cls, cron: str) -> Schedule:
+        raw = cron
+
+        if cron == "@yearly" or cron == "@annually":
+            cron = "0 0 1 1 *"
+        elif cron == "@monthly":
+            cron = "0 0 1 * *"
+        elif cron == "@weekly":
+            cron = "0 0 * * 0"
+        elif cron == "@daily" or cron == "@midnight":
+            cron = "0 0 * * *"
+        elif cron == "@hourly":
+            cron = "0 * * * *"
+
+        minute, hour, day_of_month, month, day_of_week = cron.split()
+
+        return cls(
+            minute=minute,
+            hour=hour,
+            day_of_month=day_of_month,
+            month=month,
+            day_of_week=day_of_week,
+            raw=raw,
+        )
+
+    def next(self, now: datetime.datetime | None = None) -> datetime.datetime:
+        """
+        Find the next datetime that matches the schedule after the given datetime.
+        """
+        dt = now or timezone.localtime()  # Use the defined plain timezone by default
+
+        # We only care about minutes, so immediately jump to the next minute
+        dt += datetime.timedelta(minutes=1)
+        dt = dt.replace(second=0, microsecond=0)
+
+        def _go_to_next_day(v: datetime.datetime) -> datetime.datetime:
+            v = v + datetime.timedelta(days=1)
+            return v.replace(
+                hour=self.hour.values[0],
+                minute=self.minute.values[0],
+            )
+
+        # If we don't find a value in the next 500 days,
+        # then the schedule is probably never going to match (i.e. Feb 31)
+        max_future = dt + datetime.timedelta(days=500)
+
+        while True:
+            is_valid_day = (
+                dt.month in self.month.values
+                and dt.day in self.day_of_month.values
+                and dt.weekday() in self.day_of_week.values
+            )
+            if is_valid_day:
+                # We're on a valid day, now find the next valid hour and minute
+                for hour in self.hour.values:
+                    if hour < dt.hour:
+                        continue
+                    for minute in self.minute.values:
+                        if hour == dt.hour and minute < dt.minute:
+                            continue
+                        candidate_datetime = dt.replace(hour=hour, minute=minute)
+                        if candidate_datetime >= dt:
+                            return candidate_datetime
+                # If no valid time is found today, reset to the first valid minute and hour of the next day
+                dt = _go_to_next_day(dt)
+            else:
+                # Increment the day until a valid month/day/weekday combination is found
+                dt = _go_to_next_day(dt)
+
+            if dt > max_future:
+                raise ValueError("No valid schedule match found in the next 500 days")
+
+
+@register_job
+class ScheduledCommand(Job):
+    def __init__(self, command: str) -> None:
+        self.command = command
+
+    def __repr__(self) -> str:
+        return f"<ScheduledCommand: {self.command}>"
+
+    def run(self) -> None:
+        subprocess.run(self.command, shell=True, check=True)
+
+    def get_unique_key(self) -> str:
+        # The ScheduledCommand can be used for different commands,
+        # so we need the unique_key to separate them in the scheduling uniqueness logic
+        return self.command
+
+
+def load_schedule(
+    schedules: list[tuple[str | Job, str | Schedule]],
+) -> list[tuple[Job, Schedule]]:
+    jobs_schedule: list[tuple[Job, Schedule]] = []
+
+    for job, schedule in schedules:
+        if isinstance(job, str):
+            if job.startswith("cmd:"):
+                job = ScheduledCommand(job[4:])
+            else:
+                job = jobs_registry.load_job(job, {"args": [], "kwargs": {}})
+
+        if isinstance(schedule, str):
+            schedule = Schedule.from_cron(schedule)
+
+        jobs_schedule.append((job, schedule))
+
+    return jobs_schedule

plain/jobs/templates/admin/plainqueue/jobresult_detail.html

@@ -0,0 +1,8 @@
+{% extends "admin/detail.html" %}
+
+{% block actions %}
+<form method="post">
+    <input type="hidden" name="action" value="retry">
+    <button type="submit">Retry job</button>
+</form>
+{% endblock %}

plain/jobs/workers.py

@@ -0,0 +1,322 @@
+from __future__ import annotations
+
+import gc
+import logging
+import multiprocessing
+import os
+import time
+from concurrent.futures import Future, ProcessPoolExecutor
+from functools import partial
+from typing import Any
+
+from plain import models
+from plain.models import transaction
+from plain.runtime import settings
+from plain.signals import request_finished, request_started
+from plain.utils import timezone
+from plain.utils.module_loading import import_string
+
+from .models import JobProcess, JobRequest, JobResult, JobResultStatuses
+from .registry import jobs_registry
+
+logger = logging.getLogger("plain.jobs")
+
+
+class Worker:
+    def __init__(
+        self,
+        queues: list[str],
+        jobs_schedule: list[Any] | None = None,
+        max_processes: int | None = None,
+        max_jobs_per_process: int | None = None,
+        max_pending_per_process: int = 10,
+        stats_every: int | None = None,
+    ) -> None:
+        if jobs_schedule is None:
+            jobs_schedule = []
+
+        self.executor = ProcessPoolExecutor(
+            max_workers=max_processes,
+            max_tasks_per_child=max_jobs_per_process,
+            mp_context=multiprocessing.get_context("spawn"),
+        )
+
+        self.queues = queues
+
+        # Filter the jobs schedule to those that are in the same queue as this worker
+        self.jobs_schedule = [x for x in jobs_schedule if x[0].get_queue() in queues]
+
+        # How often to log the stats (in seconds)
+        self.stats_every = stats_every
+
+        self.max_processes = self.executor._max_workers
+        self.max_jobs_per_process = max_jobs_per_process
+        self.max_pending_per_process = max_pending_per_process
+
+        self._is_shutting_down = False
+
+    def run(self) -> None:
+        logger.info(
+            "⬣ Starting Plain worker\n    Registered jobs: %s\n    Queues: %s\n    Jobs schedule: %s\n    Stats every: %s seconds\n    Max processes: %s\n    Max jobs per process: %s\n    Max pending per process: %s\n    PID: %s",
+            "\n                     ".join(
+                f"{name}: {cls}" for name, cls in jobs_registry.jobs.items()
+            ),
+            ", ".join(self.queues),
+            "\n                   ".join(str(x) for x in self.jobs_schedule),
+            self.stats_every,
+            self.max_processes,
+            self.max_jobs_per_process,
+            self.max_pending_per_process,
+            os.getpid(),
+        )
+
+        while not self._is_shutting_down:
+            try:
+                self.maybe_log_stats()
+                self.maybe_check_job_results()
+                self.maybe_schedule_jobs()
+            except Exception as e:
+                # Log the issue, but don't stop the worker
+                # (these tasks are kind of ancilarry to the main job processing)
+                logger.exception(e)
+
+            if len(self.executor._pending_work_items) >= (
+                self.max_processes * self.max_pending_per_process
+            ):
+                # We don't want to convert too many JobRequests to Jobs,
+                # because anything not started yet will be cancelled on deploy etc.
+                # It's easier to leave them in the JobRequest db queue as long as possible.
+                time.sleep(0.1)
+                continue
+
+            with transaction.atomic():
+                job_request = (
+                    JobRequest.query.select_for_update(skip_locked=True)
+                    .filter(
+                        queue__in=self.queues,
+                    )
+                    .filter(
+                        models.Q(start_at__isnull=True)
+                        | models.Q(start_at__lte=timezone.now())
+                    )
+                    .order_by("priority", "-start_at", "-created_at")
+                    .first()
+                )
+                if not job_request:
+                    # Potentially no jobs to process (who knows for how long)
+                    # but sleep for a second to give the CPU and DB a break
+                    time.sleep(1)
+                    continue
+
+                logger.info(
+                    'Preparing to execute job job_class=%s job_request_uuid=%s job_priority=%s job_source="%s" job_queues="%s"',
+                    job_request.job_class,
+                    job_request.uuid,
+                    job_request.priority,
+                    job_request.source,
+                    job_request.queue,
+                )
+
+                job = job_request.convert_to_job_process()
+
+            job_process_uuid = str(job.uuid)  # Make a str copy
+
+            # Release these now
+            del job_request
+            del job
+
+            future = self.executor.submit(process_job, job_process_uuid)
+            future.add_done_callback(
+                partial(future_finished_callback, job_process_uuid)
+            )
+
+            # Do a quick sleep regardless to see if it
+            # gives processes a chance to start up
+            time.sleep(0.1)
+
+    def shutdown(self) -> None:
+        if self._is_shutting_down:
+            # Already shutting down somewhere else
+            return
+
+        logger.info("Job worker shutdown started")
+        self._is_shutting_down = True
+        self.executor.shutdown(wait=True, cancel_futures=True)
+        logger.info("Job worker shutdown complete")
+
+    def maybe_log_stats(self) -> None:
+        if not self.stats_every:
+            return
+
+        now = time.time()
+
+        if not hasattr(self, "_stats_logged_at"):
+            self._stats_logged_at = now
+
+        if now - self._stats_logged_at > self.stats_every:
+            self._stats_logged_at = now
+            self.log_stats()
+
+    def maybe_check_job_results(self) -> None:
+        now = time.time()
+
+        if not hasattr(self, "_job_results_checked_at"):
+            self._job_results_checked_at = now
+
+        check_every = 60  # Only need to check once a minute
+
+        if now - self._job_results_checked_at > check_every:
+            self._job_results_checked_at = now
+            self.rescue_job_results()
+
+    def maybe_schedule_jobs(self) -> None:
+        if not self.jobs_schedule:
+            return
+
+        now = time.time()
+
+        if not hasattr(self, "_jobs_schedule_checked_at"):
+            self._jobs_schedule_checked_at = now
+
+        check_every = 60  # Only need to check once every 60 seconds
+
+        if now - self._jobs_schedule_checked_at > check_every:
+            for job, schedule in self.jobs_schedule:
+                next_start_at = schedule.next()
+
+                # Leverage the unique_key to prevent duplicate scheduled
+                # jobs with the same start time (also works if unique_key == "")
+                schedule_unique_key = (
+                    f"{job.get_unique_key()}:scheduled:{int(next_start_at.timestamp())}"
+                )
+
+                # Drawback here is if scheduled job is running, and detected by unique_key
+                # so it doesn't schedule the next one? Maybe an ok downside... prevents
+                # overlapping executions...?
+                result = job.run_in_worker(
+                    delay=next_start_at,
+                    unique_key=schedule_unique_key,
+                )
+                # Results are a list if it found scheduled/running jobs...
+                if not isinstance(result, list):
+                    logger.info(
+                        'Scheduling job job_class=%s job_queue="%s" job_start_at="%s" job_schedule="%s" job_unique_key="%s"',
+                        result.job_class,
+                        result.queue,
+                        result.start_at,
+                        schedule,
+                        result.unique_key,
+                    )
+
+            self._jobs_schedule_checked_at = now
+
+    def log_stats(self) -> None:
+        try:
+            num_proccesses = len(self.executor._processes)
+        except (AttributeError, TypeError):
+            # Depending on shutdown timing and internal behavior, this might not work
+            num_proccesses = 0
+
+        jobs_requested = JobRequest.query.filter(queue__in=self.queues).count()
+        jobs_processing = JobProcess.query.filter(queue__in=self.queues).count()
+
+        logger.info(
+            'Job worker stats worker_processes=%s worker_queues="%s" jobs_requested=%s jobs_processing=%s worker_max_processes=%s worker_max_jobs_per_process=%s',
+            num_proccesses,
+            ",".join(self.queues),
+            jobs_requested,
+            jobs_processing,
+            self.max_processes,
+            self.max_jobs_per_process,
+        )
+
+    def rescue_job_results(self) -> None:
+        """Find any lost or failed jobs on this worker's queues and handle them."""
+        # TODO return results and log them if there are any?
+        JobProcess.query.filter(queue__in=self.queues).mark_lost_jobs()
+        JobResult.query.filter(queue__in=self.queues).retry_failed_jobs()
+
+
+def future_finished_callback(job_process_uuid: str, future: Future) -> None:
+    if future.cancelled():
+        logger.warning("Job cancelled job_process_uuid=%s", job_process_uuid)
+        try:
+            job = JobProcess.query.get(uuid=job_process_uuid)
+            job.convert_to_result(status=JobResultStatuses.CANCELLED)
+        except JobProcess.DoesNotExist:
+            # Job may have already been cleaned up
+            pass
+    elif exception := future.exception():
+        # Process pool may have been killed...
+        logger.warning(
+            "Job failed job_process_uuid=%s",
+            job_process_uuid,
+            exc_info=exception,
+        )
+        try:
+            job = JobProcess.query.get(uuid=job_process_uuid)
+            job.convert_to_result(status=JobResultStatuses.CANCELLED)
+        except JobProcess.DoesNotExist:
+            # Job may have already been cleaned up
+            pass
+    else:
+        logger.debug("Job finished job_process_uuid=%s", job_process_uuid)
+
+
+def process_job(job_process_uuid: str) -> None:
+    try:
+        worker_pid = os.getpid()
+
+        request_started.send(sender=None)
+
+        job_process = JobProcess.query.get(uuid=job_process_uuid)
+
+        logger.info(
+            'Executing job worker_pid=%s job_class=%s job_request_uuid=%s job_priority=%s job_source="%s" job_queue="%s"',
+            worker_pid,
+            job_process.job_class,
+            job_process.job_request_uuid,
+            job_process.priority,
+            job_process.source,
+            job_process.queue,
+        )
+
+        def middleware_chain(job: JobProcess) -> JobResult:
+            return job.run()
+
+        for middleware_path in reversed(settings.JOBS_MIDDLEWARE):
+            middleware_class = import_string(middleware_path)
+            middleware_instance = middleware_class(middleware_chain)
+            middleware_chain = middleware_instance
+
+        job_result = middleware_chain(job_process)
+
+        # Release it now
+        del job_process
+
+        duration = job_result.ended_at - job_result.started_at  # type: ignore[unsupported-operator]
+        duration = duration.total_seconds()
+
+        logger.info(
+            'Completed job worker_pid=%s job_class=%s job_process_uuid=%s job_request_uuid=%s job_result_uuid=%s job_priority=%s job_source="%s" job_queue="%s" job_duration=%s',
+            worker_pid,
+            job_result.job_class,
+            job_result.job_process_uuid,
+            job_result.job_request_uuid,
+            job_result.uuid,
+            job_result.priority,
+            job_result.source,
+            job_result.queue,
+            duration,
+        )
+
+        del job_result
+    except Exception as e:
+        # Raising exceptions inside the worker process doesn't
+        # seem to be caught/shown anywhere as configured.
+        # So we at least log it out here.
+        # (A job should catch it's own user-code errors, so this is for library errors)
+        logger.exception(e)
+    finally:
+        request_finished.send(sender=None)
+        gc.collect()