threadmill 0.6.2__tar.gz → 0.7.0__tar.gz

{threadmill-0.6.2 → threadmill-0.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: threadmill
-Version: 0.6.2
+Version: 0.7.0
 Summary: The most reliable backend for Django's task framework.
 Keywords: Django,tasks,worker
 Author-email: Johannes Maron <johannes@maron.family>

{threadmill-0.6.2 → threadmill-0.7.0}/threadmill/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.6.2'
-__version_tuple__ = version_tuple = (0, 6, 2)
+__version__ = version = '0.7.0'
+__version_tuple__ = version_tuple = (0, 7, 0)
 
-__commit_id__ = commit_id = 'g9ded4014b'
+__commit_id__ = commit_id = 'g0fc4d7ade'

{threadmill-0.6.2 → threadmill-0.7.0}/threadmill/executor.py

@@ -3,6 +3,7 @@
 import asyncio
 import dataclasses
 import datetime
+import json
 import logging
 import multiprocessing
 import random
@@ -16,6 +17,7 @@ from queue import Empty
 from traceback import format_exception
 
 import django
+from django.core.serializers.json import DjangoJSONEncoder
 from django.tasks import TaskResult, task_backends
 from django.tasks.base import TaskContext, TaskError, TaskResultStatus
 from django.tasks.signals import task_finished, task_started
@@ -25,12 +27,62 @@ from django.utils.json import normalize_json
 if typing.TYPE_CHECKING:
     from .backends.base import Broker, ThreadmillTaskBackend
 
+
+class JsonFormatter(logging.Formatter):
+    """Format log records as single-line JSON objects."""
+
+    standard_attributes = frozenset(
+        {
+            "args",
+            "asctime",
+            "created",
+            "exc_info",
+            "exc_text",
+            "filename",
+            "funcName",
+            "levelname",
+            "levelno",
+            "lineno",
+            "message",
+            "module",
+            "msecs",
+            "msg",
+            "name",
+            "pathname",
+            "process",
+            "processName",
+            "relativeCreated",
+            "stack_info",
+            "taskName",
+            "thread",
+            "threadName",
+        }
+    )
+
+    def format(self, record: logging.LogRecord) -> str:
+        payload = {
+            "created_at": datetime.datetime.fromtimestamp(
+                record.created, tz=timezone.get_current_timezone()
+            ),
+            "level": record.levelname,
+            "logger": record.name,
+            "message": record.getMessage(),
+            "process": record.process,
+            "process_name": record.processName,
+            "thread": record.threadName,
+        } | {
+            key: value
+            for key, value in record.__dict__.items()
+            if key not in self.standard_attributes
+        }
+        if record.exc_info:
+            payload["exception"] = "".join(format_exception(*record.exc_info))
+        return json.dumps(payload, cls=DjangoJSONEncoder)
+
+
 logger = multiprocessing.get_logger()
-formatter = logging.Formatter(
-    "%(levelname)s: %(asctime)s - pid=%(process)s - %(message)s"
-)
 handler = logging.StreamHandler()
-handler.setFormatter(formatter)
+handler.setFormatter(JsonFormatter())
 logger.addHandler(handler)
 logger.setLevel(logging.INFO)
 
@@ -53,6 +105,7 @@ class TaskExecutor:
     queues: tuple[str]
     broker: Broker | None = dataclasses.field(default=None, init=False)
     exit_empty: bool = False
+    log_formatter: logging.Formatter = dataclasses.field(default_factory=JsonFormatter)
 
     def __post_init__(self) -> None:
         """Initialize derived orchestration fields and queues."""
@@ -69,17 +122,19 @@ class TaskExecutor:
     def create_worker_process(self) -> WorkerProcess:
         """Create and start a new worker process."""
         worker = WorkerProcess(
-            self.thread_count,
-            self.get_maximum_tasks_per_child(),
-            self.backend.alias,
-            self.queues,
-            self.exit_empty,
+            thread_count=self.thread_count,
+            max_tasks=self.get_maximum_tasks_per_child(),
+            backend_alias=self.backend.alias,
+            queues=self.queues,
+            exit_empty=self.exit_empty,
+            log_formatter=self.log_formatter,
         )
         worker.start()
         return worker
 
     def run(self) -> None:
         """Start consuming tasks until shutdown is requested."""
+        handler.setFormatter(self.log_formatter)
         self.worker_processes = [
             self.create_worker_process() for _ in range(self.process_count)
         ]
@@ -127,11 +182,13 @@ class WorkerProcess(multiprocessing.Process):
 
     def __init__(
         self,
+        *,
         thread_count: int,
         max_tasks: int | None = None,
         backend_alias: str = "",
         queues: tuple[str, ...] = (),
         exit_empty: bool = False,
+        log_formatter: logging.Formatter,
     ) -> None:
         """Create process with dedicated thread pool for task execution."""
         self.shutdown_requested = multiprocessing.Event()
@@ -141,12 +198,14 @@ class WorkerProcess(multiprocessing.Process):
         self.backend_alias = backend_alias
         self.queues = queues
         self.exit_empty = exit_empty
+        self.log_formatter = log_formatter
         self.task_count = 0
         self.lock: threading.Lock | None = None
         self.expired: threading.Event | None = None
 
     def run(self) -> None:
         """Start consumer execution inside this process."""
+        handler.setFormatter(self.log_formatter)
         django.setup()
         logger.info("Starting worker process %s", self.name)
         self.lock = threading.Lock()
@@ -229,11 +288,19 @@ class WorkerThread(threading.Thread):
             try:
                 return task_result.task.retry(TaskContext(task_result=task_result))
             except Exception:
-                logger.exception("Retry callback failed for task %r", task_result.id)
+                logger.exception(
+                    "Retry callback failed for task '%s@%s'",
+                    task_result.id,
+                    task_result.task.module_path,
+                )
 
     def execute_task_result(self, task_result: TaskResult) -> TaskResult:
         """Execute task from task result and update result lifecycle state."""
-        logger.info("Executing task %r", task_result.id)
+        logger.info(
+            "Executing task '%s@%s'",
+            task_result.id,
+            task_result.task.module_path,
+        )
         started_at = timezone.now()
         task_result = dataclasses.replace(
             task_result,
@@ -252,7 +319,11 @@ class WorkerThread(threading.Thread):
                 errors=[*task_result.errors, WorkerThread.create_task_error(exception)],
                 finished_at=timezone.now(),
             )
-            logger.exception("Task failed %r", task_result.id)
+            logger.exception(
+                "Task '%s@%s' failed",
+                task_result.id,
+                task_result.task.module_path,
+            )
         else:
             task_result = dataclasses.replace(
                 task_result,
@@ -262,7 +333,11 @@ class WorkerThread(threading.Thread):
             object.__setattr__(
                 task_result, "_return_value", normalize_json(return_value)
             )
-            logger.info("Task successful %r", task_result.id)
+            logger.info(
+                "Task '%s@%s' succeeded",
+                task_result.id,
+                task_result.task.module_path,
+            )
         finally:
             task_finished.send(TaskExecutor, task_result=task_result)
 

{threadmill-0.6.2 → threadmill-0.7.0}/threadmill/management/commands/threadmill.py

@@ -1,3 +1,4 @@
+import logging
 import signal
 import sys
 
@@ -10,7 +11,7 @@ from django.tasks import (
     task_backends,
 )
 
-from ...executor import TaskExecutor
+from ...executor import JsonFormatter, TaskExecutor
 
 
 def kill_softly(signum, frame):
@@ -71,6 +72,13 @@ class WorkerCommand(DjangoBaseCommand):
             action="store_true",
             help="Drain the task queue and exit with 0.",
         )
+        parser.add_argument(
+            "--log-format",
+            help=(
+                "Logging format string for worker log records, e.g."
+                " '%%(levelname)s %%(message)s'. Defaults to JSON."
+            ),
+        )
 
     def handle(
         self,
@@ -83,6 +91,7 @@ class WorkerCommand(DjangoBaseCommand):
         max_tasks,
         max_tasks_jitter,
         exit_empty,
+        log_format,
         **options,
     ):
         match sys.platform:
@@ -101,6 +110,17 @@ class WorkerCommand(DjangoBaseCommand):
             raise CommandError(
                 f"Backend does not support all specified queues: {_non_queues!r}"
             )
+        try:
+            log_formatter = (
+                JsonFormatter() if log_format is None else logging.Formatter(log_format)
+            )
+            log_formatter.format(
+                logging.LogRecord(
+                    "threadmill", logging.INFO, __file__, 1, "Ready", (), None
+                )
+            )
+        except (TypeError, ValueError) as e:
+            raise CommandError(f"Invalid log format: {log_format!r}") from e
         exe = TaskExecutor(
             backend=backend,
             workers=workers,
@@ -109,6 +129,7 @@ class WorkerCommand(DjangoBaseCommand):
             max_tasks_jitter=max_tasks_jitter,
             exit_empty=exit_empty,
             queues=queues,
+            log_formatter=log_formatter,
         )
         try:
             exe.run()