threadmill 0.8.2__tar.gz → 0.8.4__tar.gz

{threadmill-0.8.2 → threadmill-0.8.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: threadmill
-Version: 0.8.2
+Version: 0.8.4
 Summary: The most reliable backend for Django's task framework.
 Keywords: Django,tasks,worker
 Author-email: Johannes Maron <johannes@maron.family>

{threadmill-0.8.2 → threadmill-0.8.4}/threadmill/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.8.2'
-__version_tuple__ = version_tuple = (0, 8, 2)
+__version__ = version = '0.8.4'
+__version_tuple__ = version_tuple = (0, 8, 4)
 
-__commit_id__ = commit_id = 'gdff213884'
+__commit_id__ = commit_id = 'g071a4ce76'

{threadmill-0.8.2 → threadmill-0.8.4}/threadmill/executor.py

@@ -12,7 +12,6 @@ import socket
 import sys
 import sysconfig
 import threading
-import time
 import typing
 from concurrent.futures import Future, ThreadPoolExecutor
 from inspect import iscoroutinefunction
@@ -129,7 +128,12 @@ class TaskExecutor:
     prefetch_count: int | None = None
     poll_interval: datetime.timedelta = datetime.timedelta(seconds=0.01)
     poll_max_interval: datetime.timedelta = datetime.timedelta(seconds=1)
-    is_publishing: bool = dataclasses.field(default=True, init=False)
+    shutdown_requested: threading.Event = dataclasses.field(
+        default_factory=threading.Event, init=False
+    )
+    pool_maintainer: threading.Thread | None = dataclasses.field(
+        default=None, init=False
+    )
     worker_processes: list[WorkerProcess] = dataclasses.field(
         default_factory=list, init=False
     )
@@ -176,9 +180,10 @@ class TaskExecutor:
         self.worker_processes = [
             self.create_worker_process() for _ in range(self.process_count)
         ]
-        threads = [
-            threading.Thread(target=self.maintain_worker_pool, daemon=True),
-        ]
+        self.pool_maintainer = threading.Thread(
+            target=self.maintain_worker_pool, daemon=True
+        )
+        threads = [self.pool_maintainer]
         if self.backend.broker_class:
             self.broker = self.backend.broker_class(self.backend)
             threads.append(self.broker)
@@ -189,17 +194,24 @@ class TaskExecutor:
             thread.join()
 
     def shutdown(self) -> None:
-        """Stop queue consumption and terminate all worker processes."""
+        """Stop pool maintenance first, then queue consumption and all workers."""
         logger.info("Shutting down task executor")
+        self.shutdown_requested.set()
+        if (
+            self.pool_maintainer is not None
+            and self.pool_maintainer.is_alive()
+            # The maintainer calls shutdown() itself on exit_empty.
+            and self.pool_maintainer is not threading.current_thread()
+        ):
+            self.pool_maintainer.join()
         if self.broker is not None:
             self.broker.shutdown()
         with ThreadPoolExecutor(max_workers=self.process_count) as executor:
             executor.map(lambda worker: worker.shutdown(), self.worker_processes)
-        self.is_publishing = False
 
     def maintain_worker_pool(self) -> None:
-        """Restart worker processes that have exited."""
-        while self.is_publishing:
+        """Restart worker processes that have exited, until shutdown is requested."""
+        while not self.shutdown_requested.is_set():
             all_dead = True
             for index, worker in enumerate(self.worker_processes):
                 if worker.is_alive():
@@ -218,7 +230,7 @@ class TaskExecutor:
             if all_dead and self.exit_empty:
                 self.shutdown()
                 return
-            time.sleep(1)
+            self.shutdown_requested.wait(1)
 
 
 class WorkerProcess(multiprocessing.Process):
@@ -414,11 +426,6 @@ class WorkerThread(threading.Thread):
 
     def execute_task_result(self, task_result: TaskResult) -> TaskResult:
         """Execute task from task result and update result lifecycle state."""
-        logger.info(
-            "Executing task '%s@%s'",
-            task_result.id,
-            task_result.task.module_path,
-        )
         started_at = timezone.now()
         task_result = dataclasses.replace(
             task_result,
@@ -430,7 +437,7 @@ class WorkerThread(threading.Thread):
 
         try:
             return_value = WorkerThread.call_task(task_result)
-        except Exception as exception:
+        except Exception as exception:  # noqa: BLE001
             task_result = dataclasses.replace(
                 task_result,
                 status=TaskResultStatus.FAILED,
@@ -440,11 +447,7 @@ class WorkerThread(threading.Thread):
                 ],
                 finished_at=timezone.now(),
             )
-            logger.exception(
-                "Task '%s@%s' failed",
-                task_result.id,
-                task_result.task.module_path,
-            )
+            task_finished.send(TaskExecutor, task_result=task_result)
         else:
             task_result = dataclasses.replace(
                 task_result,
@@ -454,12 +457,6 @@ class WorkerThread(threading.Thread):
             object.__setattr__(
                 task_result, "_return_value", normalize_json(return_value)
             )
-            logger.info(
-                "Task '%s@%s' succeeded",
-                task_result.id,
-                task_result.task.module_path,
-            )
-        finally:
             task_finished.send(TaskExecutor, task_result=task_result)
 
         return task_result