threadmill 0.4.0__tar.gz → 0.5.0__tar.gz

{threadmill-0.4.0 → threadmill-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: threadmill
-Version: 0.4.0
+Version: 0.5.0
 Summary: The most reliable backend for Django's task framework.
 Keywords: Django,tasks,worker
 Author-email: Johannes Maron <johannes@maron.family>

{threadmill-0.4.0 → threadmill-0.5.0}/threadmill/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.4.0'
-__version_tuple__ = version_tuple = (0, 4, 0)
+__version__ = version = '0.5.0'
+__version_tuple__ = version_tuple = (0, 5, 0)
 
-__commit_id__ = commit_id = 'gf9749f870'
+__commit_id__ = commit_id = 'g190acbe27'

{threadmill-0.4.0 → threadmill-0.5.0}/threadmill/backends/base.py

@@ -217,7 +217,12 @@ class ThreadmillTaskBackend(BaseTaskBackend, ABC):
         raise NotImplementedError
 
     def requeue(self, task_result: TaskResult, run_after: datetime.datetime) -> None:
-        """Re-queue a failed task result for a retry attempt after `run_after`."""
+        """Re-queue a failed task result for a retry attempt after `run_after`.
+
+        Cleans up any persisted failed result so the method works both for
+        in-flight retries (task still running) and inspector-driven requeues
+        of already-failed tasks.
+        """
         raise NotImplementedError
 
     def peek(
@@ -246,3 +251,15 @@ class ThreadmillTaskBackend(BaseTaskBackend, ABC):
             interval: The time window for rolling rates.
         """
         raise NotImplementedError
+
+    def dequeue(self, task_result: TaskResult) -> None:
+        """Delete a single task from its current status segment."""
+        raise NotImplementedError
+
+    def purge(self, queue_name: str) -> None:
+        """Delete every task across all segments of a queue.
+
+        Args:
+            queue_name: The queue to purge.
+        """
+        raise NotImplementedError

{threadmill-0.4.0 → threadmill-0.5.0}/threadmill/backends/lua/acquire.lua

@@ -7,7 +7,7 @@
 --                KEYS[4] = queue, etc.
 -- ARGV[1]     -- current time in milliseconds
 -- ARGV[2]     -- current time as ISO-8601 string
--- ARGV[3]     -- task key prefix (e.g. "threadmill:default:task:")
+-- ARGV[3]     -- task key prefix (e.g. "threadmill:task:")
 -- ARGV[4]     -- number of queue pairs (N/2)
 -- ARGV[5]     -- worker name
 -- ARGV[6]     -- lease TTL in milliseconds

{threadmill-0.4.0 → threadmill-0.5.0}/threadmill/backends/redis.py

@@ -55,14 +55,12 @@ class RedisBroker(Broker):
         deferred_key = self.backend.DEFERRED_KEY.format(
             prefix=self.backend.key_prefix, queue_name=queue_name
         )
-        queue_key = self.backend.QUEUE_KEY.format(
-            prefix=self.backend.key_prefix, queue_name=queue_name
-        )
+        queue_key = self.backend._segment_key(TaskResultStatus.READY, queue_name)
         self._mover_script(
             keys=[deferred_key, queue_key],
             args=[
                 str(time.time() * 1000),
-                self.backend.key_prefix + ":task:",
+                f"{self.backend.key_prefix}:task:",
                 str(self.backend.batch_size),
             ],
         )
@@ -72,18 +70,16 @@ class RedisBroker(Broker):
         now = timezone.now()
         now_ms = now.timestamp() * 1000
         finished_at_iso = now.isoformat()
-        running_key = self.backend.RUNNING_KEY.format(
-            prefix=self.backend.key_prefix, queue_name=queue_name
-        )
-        failed_results_key = self.backend.FAILED_RESULTS_KEY.format(
-            prefix=self.backend.key_prefix, queue_name=queue_name
+        running_key = self.backend._segment_key(TaskResultStatus.RUNNING, queue_name)
+        failed_results_key = self.backend._segment_key(
+            TaskResultStatus.FAILED, queue_name
         )
         self._reaper_script(
             keys=[running_key, failed_results_key],
             args=[
                 str(now_ms),
-                self.backend.key_prefix + ":task:",
-                self.backend.key_prefix + ":result:",
+                f"{self.backend.key_prefix}:task:",
+                f"{self.backend.key_prefix}:result:",
                 str(self.backend.batch_size),
                 str(int(self.backend.result_ttl.total_seconds())),
                 finished_at_iso,
@@ -125,20 +121,25 @@ class RedisTaskBackend(ThreadmillTaskBackend):
 
     broker_class = RedisBroker
 
-    QUEUE_KEY = "{prefix}:queue:{queue_name}"
-    RUNNING_KEY = "{prefix}:running:{queue_name}"
-    DEFERRED_KEY = "{prefix}:deferred:{queue_name}"
     TASK_KEY = "{prefix}:task:{task_id}"
     RESULT_KEY = "{prefix}:result:{result_id}"
-    SUCCESSFUL_RESULTS_KEY = "{prefix}:results:{queue_name}:successful"
-    FAILED_RESULTS_KEY = "{prefix}:results:{queue_name}:failed"
-    INGRESS_KEY = "{prefix}:ingress:{queue_name}:events"
+    SEGMENT_KEY = "{prefix}:{queue_name}:{status}"
+    DEFERRED_KEY = "{prefix}:{queue_name}:deferred"
+
+    INGRESS_KEY = "{prefix}:{queue_name}:ingress:events"
 
     ACQUIRE_SCRIPT = _load_lua("acquire")
     """Pop the next task from a priority queue and move it directly to the running set."""
     ACKNOWLEDGE_SCRIPT = _load_lua("acknowledge")
     """Remove from running, persist the result, and clean up."""
 
+    def _segment_key(self, status: TaskResultStatus, queue_name: str) -> str:
+        return self.SEGMENT_KEY.format(
+            prefix=self.key_prefix,
+            queue_name=queue_name,
+            status=status.value.lower(),
+        )
+
     def __init__(self, alias: str, params: dict) -> None:
         super().__init__(alias=alias, params=params)
 
@@ -224,9 +225,7 @@ class RedisTaskBackend(ThreadmillTaskBackend):
             run_after_ms = task.run_after.timestamp() * 1000
             pipe.zadd(deferred_key, {task_result.id: run_after_ms})
         else:
-            queue_key = self.QUEUE_KEY.format(
-                prefix=self.key_prefix, queue_name=task.queue_name
-            )
+            queue_key = self._segment_key(TaskResultStatus.READY, task.queue_name)
             pipe.zadd(queue_key, {task_result.id: score})
 
         pipe.execute()
@@ -246,8 +245,8 @@ class RedisTaskBackend(ThreadmillTaskBackend):
             key
             for queue_name in queue_names
             for key in (
-                self.RUNNING_KEY.format(prefix=self.key_prefix, queue_name=queue_name),
-                self.QUEUE_KEY.format(prefix=self.key_prefix, queue_name=queue_name),
+                self._segment_key(TaskResultStatus.RUNNING, queue_name),
+                self._segment_key(TaskResultStatus.READY, queue_name),
             )
         ]
 
@@ -261,7 +260,7 @@ class RedisTaskBackend(ThreadmillTaskBackend):
                 args=[
                     str(now_ms),
                     now_iso,
-                    self.key_prefix + ":task:",
+                    f"{self.key_prefix}:task:",
                     str(len(queue_names)),
                     worker,
                     str(int(self.lease_ttl.total_seconds() * 1000)),
@@ -281,18 +280,18 @@ class RedisTaskBackend(ThreadmillTaskBackend):
 
     def acknowledge(self, task_result: TaskResult) -> None:
         serialized = self.serialize_task_result(task_result)
-        running_key = self.RUNNING_KEY.format(
-            prefix=self.key_prefix, queue_name=task_result.task.queue_name
+        running_key = self._segment_key(
+            TaskResultStatus.RUNNING, task_result.task.queue_name
         )
         result_key = self.RESULT_KEY.format(
             prefix=self.key_prefix, result_id=task_result.id
         )
         task_key = self.TASK_KEY.format(prefix=self.key_prefix, task_id=task_result.id)
-        successful_results_key = self.SUCCESSFUL_RESULTS_KEY.format(
-            prefix=self.key_prefix, queue_name=task_result.task.queue_name
+        successful_results_key = self._segment_key(
+            TaskResultStatus.SUCCESSFUL, task_result.task.queue_name
         )
-        failed_results_key = self.FAILED_RESULTS_KEY.format(
-            prefix=self.key_prefix, queue_name=task_result.task.queue_name
+        failed_results_key = self._segment_key(
+            TaskResultStatus.FAILED, task_result.task.queue_name
         )
         finished_at = task_result.finished_at or timezone.now()
         finish_score = finished_at.timestamp() * 1000
@@ -322,13 +321,19 @@ class RedisTaskBackend(ThreadmillTaskBackend):
             finished_at=None,
         )
         serialized = self.serialize_task_result(task_result)
-        running_key = self.RUNNING_KEY.format(
-            prefix=self.key_prefix, queue_name=task_result.task.queue_name
+        running_key = self._segment_key(
+            TaskResultStatus.RUNNING, task_result.task.queue_name
         )
         deferred_key = self.DEFERRED_KEY.format(
             prefix=self.key_prefix, queue_name=task_result.task.queue_name
         )
+        failed_key = self._segment_key(
+            TaskResultStatus.FAILED, task_result.task.queue_name
+        )
         task_key = self.TASK_KEY.format(prefix=self.key_prefix, task_id=task_result.id)
+        result_key = self.RESULT_KEY.format(
+            prefix=self.key_prefix, result_id=task_result.id
+        )
         score = self._compute_score(task_result.task.priority, task_result.enqueued_at)
         run_after_ms = run_after.timestamp() * 1000
         task_data_ttl = int(
@@ -337,11 +342,26 @@ class RedisTaskBackend(ThreadmillTaskBackend):
 
         pipe = self.client.pipeline()
         pipe.zrem(running_key, task_result.id)
+        pipe.zrem(failed_key, task_result.id)
+        pipe.delete(result_key)
         pipe.hset(task_key, mapping={"data": serialized, "score": str(score)})
         pipe.expire(task_key, task_data_ttl)
         pipe.zadd(deferred_key, {task_result.id: run_after_ms})
         pipe.execute()
 
+    def dequeue(self, task_result: TaskResult) -> None:
+        self.client.zrem(
+            self._segment_key(task_result.status, task_result.task.queue_name),
+            task_result.id,
+        )
+
+    def purge(self, queue_name: str) -> None:
+        pattern = f"{self.key_prefix}:{queue_name}:*"
+        pipe = self.client.pipeline()
+        for key in self.client.scan_iter(match=pattern):
+            pipe.delete(key)
+        pipe.execute()
+
     def peek(
         self,
         queue_name: str = DEFAULT_TASK_QUEUE_NAME,
@@ -350,34 +370,27 @@ class RedisTaskBackend(ThreadmillTaskBackend):
         count: int = 1,
     ) -> Generator[TaskResult]:
         match status:
-            case TaskResultStatus.READY:
+            case TaskResultStatus.READY | TaskResultStatus.RUNNING:
                 yield from self._peek(
-                    self.QUEUE_KEY, self.TASK_KEY, queue_name, count, "data"
+                    self._segment_key(status, queue_name),
+                    self.TASK_KEY,
+                    count,
+                    "data",
                 )
-            case TaskResultStatus.RUNNING:
+            case TaskResultStatus.SUCCESSFUL | TaskResultStatus.FAILED:
                 yield from self._peek(
-                    self.RUNNING_KEY, self.TASK_KEY, queue_name, count, "data"
-                )
-            case TaskResultStatus.SUCCESSFUL:
-                yield from self._peek(
-                    self.SUCCESSFUL_RESULTS_KEY, self.RESULT_KEY, queue_name, count
-                )
-            case TaskResultStatus.FAILED:
-                yield from self._peek(
-                    self.FAILED_RESULTS_KEY, self.RESULT_KEY, queue_name, count
+                    self._segment_key(status, queue_name),
+                    self.RESULT_KEY,
+                    count,
                 )
 
     def _peek(
         self,
-        zset_key_template: str,
+        zset_key: str,
         data_key_template: str,
-        queue_name: str,
         count: int,
         field: str | None = None,
     ) -> Generator[TaskResult]:
-        zset_key = zset_key_template.format(
-            prefix=self.key_prefix, queue_name=queue_name
-        )
         pipe = self.client.pipeline()
         for member in self.client.zrange(zset_key, 0, count - 1):
             member_id = member.decode() if isinstance(member, bytes) else member
@@ -412,16 +425,12 @@ class RedisTaskBackend(ThreadmillTaskBackend):
             cutoff,
         )
         pipe.zremrangebyscore(
-            self.SUCCESSFUL_RESULTS_KEY.format(
-                prefix=self.key_prefix, queue_name=queue_name
-            ),
+            self._segment_key(TaskResultStatus.SUCCESSFUL, queue_name),
             0,
             cutoff,
         )
         pipe.zremrangebyscore(
-            self.FAILED_RESULTS_KEY.format(
-                prefix=self.key_prefix, queue_name=queue_name
-            ),
+            self._segment_key(TaskResultStatus.FAILED, queue_name),
             0,
             cutoff,
         )
@@ -436,21 +445,15 @@ class RedisTaskBackend(ThreadmillTaskBackend):
         retention_start_ms = now_ms - self.result_ttl.total_seconds() * 1000
         pipe = self.client.pipeline()
         for queue_name in self.queues:
-            pipe.zcard(
-                self.QUEUE_KEY.format(prefix=self.key_prefix, queue_name=queue_name)
-            )
-            pipe.zcard(
-                self.RUNNING_KEY.format(prefix=self.key_prefix, queue_name=queue_name)
-            )
+            pipe.zcard(self._segment_key(TaskResultStatus.READY, queue_name))
+            pipe.zcard(self._segment_key(TaskResultStatus.RUNNING, queue_name))
             pipe.zcard(
                 self.DEFERRED_KEY.format(prefix=self.key_prefix, queue_name=queue_name)
             )
-            successful_results_key = self.SUCCESSFUL_RESULTS_KEY.format(
-                prefix=self.key_prefix, queue_name=queue_name
-            )
-            failed_results_key = self.FAILED_RESULTS_KEY.format(
-                prefix=self.key_prefix, queue_name=queue_name
+            successful_results_key = self._segment_key(
+                TaskResultStatus.SUCCESSFUL, queue_name
             )
+            failed_results_key = self._segment_key(TaskResultStatus.FAILED, queue_name)
             pipe.zcard(successful_results_key)
             pipe.zcard(failed_results_key)
             ingress_key = self.INGRESS_KEY.format(

{threadmill-0.4.0 → threadmill-0.5.0}/threadmill/inspector/app.py

@@ -30,6 +30,7 @@ from textual.widgets import (
 )
 
 from ..backends.base import BackendTelemetry, ThreadmillTaskBackend
+from .screens import ConfirmScreen, PurgeScreen
 
 logger = logging.getLogger(__name__)
 
@@ -163,6 +164,21 @@ class TaskList(Vertical):
     counts: reactive[dict[str, int]] = reactive({})
     selected_task: reactive[TaskResult | None] = reactive(None)
 
+    BINDINGS = [
+        Binding("f5", "refresh", "Refresh"),
+        Binding("r", "requeue", "Requeue"),
+        Binding("d", "dequeue", "Drop"),
+    ]
+
+    def check_action(self, action: str, parameters: tuple[object, ...]) -> bool | None:
+        """Show requeue only on the Failed tab, dequeue on non-Running tabs."""
+        status = self.active_status()
+        if action == "requeue":
+            return status == TaskResultStatus.FAILED
+        if action == "dequeue":
+            return status is not None and status != TaskResultStatus.RUNNING
+        return True
+
     def compose(self) -> ComposeResult:
         with TabbedContent(initial="tab-ready") as tabs:
             for label, _ in TAB_STATUSES:
@@ -212,6 +228,11 @@ class TaskList(Vertical):
         """Activate the tab with the given id."""
         self._tabs.active = tab_id
 
+    def action_refresh(self) -> None:
+        """Refresh telemetry and re-fetch the task list."""
+        self.app._refresh_telemetry()
+        self.refresh_tasks()
+
     def on_data_table_row_selected(self, event: DataTable.RowSelected) -> None:
         """Notify the app when a task row is selected."""
         if isinstance(event.row_key.value, str):
@@ -227,6 +248,7 @@ class TaskList(Vertical):
     ) -> None:
         """Refresh the visible table when the user switches tabs."""
         self._refresh_data()
+        self.refresh_bindings()
 
     def _select_task_by_id(self, task_id: str) -> None:
         """Find the task result matching the row key in the current results."""
@@ -235,6 +257,57 @@ class TaskList(Vertical):
             None,
         )
 
+    def active_status(self) -> TaskResultStatus | None:
+        """Return the status segment of the currently active task tab."""
+        tab_id = self._tabs.active.removeprefix("tab-")
+        return next(
+            (status for label, status in TAB_STATUSES if label.lower() == tab_id),
+            None,
+        )
+
+    def action_requeue(self) -> None:
+        """Requeue the selected failed task after confirmation."""
+        task = self.selected_task
+        if task and self.active_status() == TaskResultStatus.FAILED:
+            self.app.push_screen(
+                ConfirmScreen(f"Requeue task {task.id[:8]}?"),
+                lambda confirmed: self._do_requeue(confirmed, task),
+            )
+
+    def _do_requeue(self, confirmed: bool, task: TaskResult) -> None:
+        if not confirmed:
+            return
+        try:
+            self.backend.requeue(task, datetime.datetime.now(datetime.UTC))
+        except Exception:  # noqa: BLE001
+            logger.exception("Requeue failed for task %s", task.id)
+            self.app.notify("Requeue failed.", severity="error")
+            return
+        self.app._after_action("Task requeued.")
+
+    def action_dequeue(self) -> None:
+        """Remove the selected task from its current segment after confirmation."""
+        task = self.selected_task
+        if task and self.active_status() is not None:
+            self.app.push_screen(
+                ConfirmScreen(
+                    f"Delete task {task.task.func.__name__} ({task.id[:8]})?",
+                    danger=True,
+                ),
+                lambda confirmed: self._do_dequeue(confirmed, task),
+            )
+
+    def _do_dequeue(self, confirmed: bool, task: TaskResult) -> None:
+        if not confirmed:
+            return
+        try:
+            self.backend.dequeue(task)
+        except Exception:  # noqa: BLE001
+            logger.exception("Delete failed for task %s", task.id)
+            self.app.notify("Delete failed.", severity="error")
+            return
+        self.app._after_action("Task deleted.")
+
     def _refresh_data(self) -> None:
         """Fetch and display tasks for the current queue and active tab."""
         backend = self.backend
@@ -291,6 +364,10 @@ class QueueList(ListView):
 
     telemetry: reactive[BackendTelemetry | None] = reactive(None)
 
+    BINDINGS = [
+        Binding("p", "purge", "Purge"),
+    ]
+
     def __init__(self, **kwargs: Any) -> None:
         super().__init__(**kwargs)
         self._items: dict[str, QueueItem] = {}
@@ -300,12 +377,10 @@ class QueueList(ListView):
         self.border_title = "Queues"
 
     def watch_telemetry(self, telemetry: BackendTelemetry | None) -> None:
-        """Refresh queue labels when a new telemetry snapshot arrives."""
         if telemetry is not None:
             self.update_telemetry(telemetry)
 
     def update_telemetry(self, telemetry: BackendTelemetry) -> None:
-        """Refresh queue labels from a new telemetry snapshot."""
         queues = telemetry.queues
         for queue_name, stats in sorted(queues.items()):
             theme = BUILTIN_THEMES[self.app.theme]
@@ -334,21 +409,40 @@ class QueueList(ListView):
             self._notify_selection(target)
 
     def _notify_selection(self, queue_name: str) -> None:
-        """Tell the app which queue to display."""
         self.app._task_list.queue_name = queue_name
 
+    def action_purge(self) -> None:
+        item = self.highlighted_child
+        if not isinstance(item, QueueItem):
+            return
+        queue_name = item.queue_name
+        self.app.push_screen(
+            PurgeScreen(queue_name),
+            lambda confirmed: self._do_purge(confirmed, queue_name),
+        )
+
+    def _do_purge(self, confirmed: bool, queue_name: str) -> None:
+        if not confirmed:
+            return
+        try:
+            self.app.backend.purge(queue_name)
+        except Exception:  # noqa: BLE001
+            logger.exception("Purge failed for %r", queue_name)
+            self.app.notify("Purge failed.", severity="error")
+            return
+        self.app._after_action(f"Purged queue {queue_name}.")
+
 
 class InspectorApp(App):
     """Threadmill TUI inspector with backend/queue/task panes."""
 
     CSS_PATH = "inspector.scss"
     BINDINGS = [
-        Binding("q", "quit", "Quit"),
-        Binding("f5", "refresh", "Refresh"),
         *(
             Binding(key, f"switch_tab('tab-{tab_id}')", tab_id.capitalize())
             for tab_id, key in TAB_KEYS.items()
         ),
+        Binding("q", "quit", "Quit"),
     ]
 
     backend: reactive[ThreadmillTaskBackend] = reactive(None)
@@ -415,8 +509,9 @@ class InspectorApp(App):
         """Exit the TUI."""
         self.exit()
 
-    def action_refresh(self) -> None:
-        """Refresh queue stats and re-fetch the task list on demand."""
+    def _after_action(self, message: str) -> None:
+        """Refresh telemetry and tasks after a mutating queue action."""
+        self.notify(message)
         self._refresh_telemetry()
         self._task_list.refresh_tasks()
 

{threadmill-0.4.0 → threadmill-0.5.0}/threadmill/inspector/inspector.scss

@@ -2,6 +2,11 @@ Screen {
     background: $background;
 }
 
+ConfirmScreen,
+PurgeScreen {
+    align: center middle;
+}
+
 * {
     scrollbar-color: $primary 10%;
     scrollbar-color-hover: $primary 80%;
@@ -104,3 +109,29 @@ TaskDetail {
     overflow: scroll;
     padding: 0 1;
 }
+
+#dialog {
+    width: 60;
+    height: auto;
+    background: $surface;
+    padding: 1 2;
+}
+
+#dialog.warning {
+    border: thick $warning;
+}
+
+#dialog.danger {
+    border: thick $error;
+}
+
+#prompt {
+    text-align: center;
+    margin-bottom: 1;
+}
+
+#instruction {
+    text-align: center;
+    color: $text-muted;
+    margin-top: 1;
+}

threadmill-0.5.0/threadmill/inspector/screens.py

@@ -0,0 +1,52 @@
+"""Modal confirmation screens for inspector queue actions."""
+
+from textual.app import ComposeResult
+from textual.containers import Vertical
+from textual.events import Key
+from textual.screen import ModalScreen
+from textual.widgets import Input, Label
+
+
+class ConfirmScreen(ModalScreen[bool]):
+    """Yes/no confirmation: Enter confirms, Esc cancels."""
+
+    def __init__(self, prompt: str, *, danger: bool = False) -> None:
+        super().__init__()
+        self._prompt = prompt
+        self._danger = danger
+
+    def compose(self) -> ComposeResult:
+        with Vertical(id="dialog", classes="danger" if self._danger else "warning"):
+            yield Label(self._prompt, id="prompt")
+            yield Label("Press Enter to confirm, Esc to cancel.", id="instruction")
+
+    def on_key(self, event: Key) -> None:
+        match event.key:
+            case "enter":
+                self.dismiss(True)
+            case "escape":
+                self.dismiss(False)
+
+
+class PurgeScreen(ModalScreen[bool]):
+    """Type the queue name to confirm a purge."""
+
+    def __init__(self, queue_name: str) -> None:
+        super().__init__()
+        self._queue_name = queue_name
+
+    def compose(self) -> ComposeResult:
+        with Vertical(id="dialog", classes="danger"):
+            yield Label(f"Purge queue [bold]{self._queue_name}[/bold]?", id="prompt")
+            yield Label(
+                "Type the queue name to confirm, Esc to cancel.", id="instruction"
+            )
+            yield Input(placeholder=self._queue_name, id="confirm-input")
+
+    def on_input_submitted(self, event: Input.Submitted) -> None:
+        if event.value == self._queue_name:
+            self.dismiss(True)
+
+    def on_key(self, event: Key) -> None:
+        if event.key == "escape":
+            self.dismiss(False)