threadmill 0.4.0__tar.gz → 0.5.1__tar.gz

{threadmill-0.4.0 → threadmill-0.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: threadmill
-Version: 0.4.0
+Version: 0.5.1
 Summary: The most reliable backend for Django's task framework.
 Keywords: Django,tasks,worker
 Author-email: Johannes Maron <johannes@maron.family>

{threadmill-0.4.0 → threadmill-0.5.1}/threadmill/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.4.0'
-__version_tuple__ = version_tuple = (0, 4, 0)
+__version__ = version = '0.5.1'
+__version_tuple__ = version_tuple = (0, 5, 1)
 
-__commit_id__ = commit_id = 'gf9749f870'
+__commit_id__ = commit_id = 'ga1884237b'

{threadmill-0.4.0 → threadmill-0.5.1}/threadmill/backends/base.py

@@ -1,6 +1,7 @@
 import collections.abc
 import dataclasses
 import datetime
+import enum
 import json
 import threading
 import typing
@@ -95,6 +96,21 @@ class BackendTelemetry:
     queues: dict[str, QueueStats]
 
 
+class TelemetryDirection(enum.Enum):
+    """Direction of a telemetry event: a task entering or leaving a queue."""
+
+    INGRESS = "ingress"
+    EGRESS = "egress"
+
+
+@dataclasses.dataclass(frozen=True, slots=True)
+class TelemetryEvent:
+    """A single ingress/egress event published by a worker via pub/sub."""
+
+    direction: TelemetryDirection
+    queue_name: str
+
+
 class Broker(threading.Thread):
     """Backend maintenance thread launched by the task executor."""
 
@@ -217,7 +233,12 @@ class ThreadmillTaskBackend(BaseTaskBackend, ABC):
         raise NotImplementedError
 
     def requeue(self, task_result: TaskResult, run_after: datetime.datetime) -> None:
-        """Re-queue a failed task result for a retry attempt after `run_after`."""
+        """Re-queue a failed task result for a retry attempt after `run_after`.
+
+        Cleans up any persisted failed result so the method works both for
+        in-flight retries (task still running) and inspector-driven requeues
+        of already-failed tasks.
+        """
         raise NotImplementedError
 
     def peek(
@@ -237,12 +258,26 @@ class ThreadmillTaskBackend(BaseTaskBackend, ABC):
         """
         raise NotImplementedError
 
-    def telemetry(
+    async def queue_stats(
         self, *, interval: datetime.timedelta = datetime.timedelta(seconds=60)
     ) -> BackendTelemetry:
-        """Return a snapshot of stats for all configured queues.
+        """Return per-queue task counts for all configured queues."""
+        raise NotImplementedError
+
+    async def worker_telemetry(
+        self,
+    ) -> collections.abc.AsyncIterator[TelemetryEvent]:
+        """Yield ingress/egress telemetry events from the backend's pub/sub stream."""
+        raise NotImplementedError
+
+    def dequeue(self, task_result: TaskResult) -> None:
+        """Delete a single task from its current status segment."""
+        raise NotImplementedError
+
+    def purge(self, queue_name: str) -> None:
+        """Delete every task across all segments of a queue.
 
         Args:
-            interval: The time window for rolling rates.
+            queue_name: The queue to purge.
         """
         raise NotImplementedError

{threadmill-0.4.0 → threadmill-0.5.1}/threadmill/backends/lua/acknowledge.lua

@@ -3,7 +3,8 @@
 -- per-status results history set. Evicts results whose finish score falls
 -- outside the retention window (result_ttl). The per-status history sets are
 -- the time series the inspector counts over for telemetry, so no separate
--- egress window or status counters are needed here.
+-- egress window or status counters are needed here. A pub/sub event is
+-- published so the inspector can track live egress throughput.
 --
 -- KEYS[1]  -- running set (ZSET)
 -- KEYS[2]  -- result key (STRING, stores serialized TaskResult)
@@ -15,6 +16,8 @@
 -- ARGV[3]  -- result TTL in seconds
 -- ARGV[4]  -- finish timestamp in milliseconds (score for the history set)
 -- ARGV[5]  -- status (SUCCESSFUL or FAILED)
+-- ARGV[6]  -- telemetry pub/sub channel
+-- ARGV[7]  -- queue name
 -- Returns: 1 on success, 0 if task was not in the running set
 
 local removed = redis.call('ZREM', KEYS[1], ARGV[1])
@@ -31,4 +34,5 @@ if ARGV[5] ~= 'SUCCESSFUL' then
 end
 redis.call('ZADD', results_key, finish, ARGV[1])
 redis.call('ZREMRANGEBYSCORE', results_key, 0, cutoff)
+redis.call('PUBLISH', ARGV[6], 'egress:' .. ARGV[7])
 return 1

{threadmill-0.4.0 → threadmill-0.5.1}/threadmill/backends/lua/acquire.lua

@@ -7,7 +7,7 @@
 --                KEYS[4] = queue, etc.
 -- ARGV[1]     -- current time in milliseconds
 -- ARGV[2]     -- current time as ISO-8601 string
--- ARGV[3]     -- task key prefix (e.g. "threadmill:default:task:")
+-- ARGV[3]     -- task key prefix (e.g. "threadmill:task:")
 -- ARGV[4]     -- number of queue pairs (N/2)
 -- ARGV[5]     -- worker name
 -- ARGV[6]     -- lease TTL in milliseconds

{threadmill-0.4.0 → threadmill-0.5.1}/threadmill/backends/redis.py

@@ -1,5 +1,6 @@
 """Redis-backed durable priority queue backend for Django's task framework."""
 
+import collections.abc
 import dataclasses
 import datetime
 import logging
@@ -10,6 +11,7 @@ from collections.abc import Generator, Sequence
 from pathlib import Path
 
 import redis
+import redis.asyncio
 from django.tasks import DEFAULT_TASK_QUEUE_NAME, TaskResult, TaskResultStatus
 from django.tasks.exceptions import TaskResultDoesNotExist
 from django.tasks.signals import task_enqueued
@@ -21,6 +23,8 @@ from threadmill.backends.base import (
     QueueCounts,
     QueueRates,
     QueueStats,
+    TelemetryDirection,
+    TelemetryEvent,
     ThreadmillTaskBackend,
 )
 
@@ -55,14 +59,12 @@ class RedisBroker(Broker):
         deferred_key = self.backend.DEFERRED_KEY.format(
             prefix=self.backend.key_prefix, queue_name=queue_name
         )
-        queue_key = self.backend.QUEUE_KEY.format(
-            prefix=self.backend.key_prefix, queue_name=queue_name
-        )
+        queue_key = self.backend._segment_key(TaskResultStatus.READY, queue_name)
         self._mover_script(
             keys=[deferred_key, queue_key],
             args=[
                 str(time.time() * 1000),
-                self.backend.key_prefix + ":task:",
+                f"{self.backend.key_prefix}:task:",
                 str(self.backend.batch_size),
             ],
         )
@@ -72,18 +74,16 @@ class RedisBroker(Broker):
         now = timezone.now()
         now_ms = now.timestamp() * 1000
         finished_at_iso = now.isoformat()
-        running_key = self.backend.RUNNING_KEY.format(
-            prefix=self.backend.key_prefix, queue_name=queue_name
-        )
-        failed_results_key = self.backend.FAILED_RESULTS_KEY.format(
-            prefix=self.backend.key_prefix, queue_name=queue_name
+        running_key = self.backend._segment_key(TaskResultStatus.RUNNING, queue_name)
+        failed_results_key = self.backend._segment_key(
+            TaskResultStatus.FAILED, queue_name
         )
         self._reaper_script(
             keys=[running_key, failed_results_key],
             args=[
                 str(now_ms),
-                self.backend.key_prefix + ":task:",
-                self.backend.key_prefix + ":result:",
+                f"{self.backend.key_prefix}:task:",
+                f"{self.backend.key_prefix}:result:",
                 str(self.backend.batch_size),
                 str(int(self.backend.result_ttl.total_seconds())),
                 finished_at_iso,
@@ -92,7 +92,6 @@ class RedisBroker(Broker):
 
     def main(self) -> None:
         """Run mover and running reaper passes for all queues."""
-        now_ms = time.time() * 1000
         for queue_name in self.backend.queues:
             try:
                 self._move_queue(queue_name)
@@ -104,11 +103,6 @@ class RedisBroker(Broker):
             except Exception:  # noqa: BLE001
                 logger.exception("Running reaper error for queue %r", queue_name)
 
-            try:
-                self.backend._trim_telemetry(queue_name, now_ms=now_ms)
-            except Exception:  # noqa: BLE001
-                logger.exception("Telemetry trim error for queue %r", queue_name)
-
 
 class RedisTaskBackend(ThreadmillTaskBackend):
     """Redis-backed durable priority queue backend.
@@ -125,20 +119,25 @@ class RedisTaskBackend(ThreadmillTaskBackend):
 
     broker_class = RedisBroker
 
-    QUEUE_KEY = "{prefix}:queue:{queue_name}"
-    RUNNING_KEY = "{prefix}:running:{queue_name}"
-    DEFERRED_KEY = "{prefix}:deferred:{queue_name}"
     TASK_KEY = "{prefix}:task:{task_id}"
     RESULT_KEY = "{prefix}:result:{result_id}"
-    SUCCESSFUL_RESULTS_KEY = "{prefix}:results:{queue_name}:successful"
-    FAILED_RESULTS_KEY = "{prefix}:results:{queue_name}:failed"
-    INGRESS_KEY = "{prefix}:ingress:{queue_name}:events"
+    SEGMENT_KEY = "{prefix}:{queue_name}:{status}"
+    DEFERRED_KEY = "{prefix}:{queue_name}:deferred"
+
+    TELEMETRY_CHANNEL = "{prefix}:telemetry"
 
     ACQUIRE_SCRIPT = _load_lua("acquire")
     """Pop the next task from a priority queue and move it directly to the running set."""
     ACKNOWLEDGE_SCRIPT = _load_lua("acknowledge")
     """Remove from running, persist the result, and clean up."""
 
+    def _segment_key(self, status: TaskResultStatus, queue_name: str) -> str:
+        return self.SEGMENT_KEY.format(
+            prefix=self.key_prefix,
+            queue_name=queue_name,
+            status=status.value.lower(),
+        )
+
     def __init__(self, alias: str, params: dict) -> None:
         super().__init__(alias=alias, params=params)
 
@@ -149,13 +148,23 @@ class RedisTaskBackend(ThreadmillTaskBackend):
                 f"REDIS_URL must be specified in your settings for the {type(self).__name__}."
             ) from e
         self.client = redis.from_url(redis_url)
+        self._async_client: redis.asyncio.Redis | None = None
+        self.redis_url = redis_url
         self.key_prefix = f"threadmill:{{{alias}}}"
+        self.telemetry_channel = self.TELEMETRY_CHANNEL.format(prefix=self.key_prefix)
         self.lease_ttl = self.options.get("lease_ttl", datetime.timedelta(hours=1))
         self.result_ttl = self.options.get("result_ttl", datetime.timedelta(days=1))
         self.batch_size = self.options.get("batch_size", 100)
         self._acquire_script = self.client.register_script(self.ACQUIRE_SCRIPT)
         self._acknowledge_script = self.client.register_script(self.ACKNOWLEDGE_SCRIPT)
 
+    @property
+    def async_client(self) -> redis.asyncio.Redis:
+        """Lazily-created async Redis client, reused across calls."""
+        if self._async_client is None:
+            self._async_client = redis.asyncio.Redis.from_url(self.redis_url)
+        return self._async_client
+
     def _compute_score(self, priority: int, enqueued_at: datetime.datetime) -> float:
         """Compute a ZSET score for priority-ordered FIFO queueing.
 
@@ -195,15 +204,11 @@ class RedisTaskBackend(ThreadmillTaskBackend):
         )
 
         score = self._compute_score(task.priority, enqueued_at)
-        enqueued_at_ms = enqueued_at.timestamp() * 1000
         serialized = self.serialize_task_result(task_result)
         task_key = self.TASK_KEY.format(prefix=self.key_prefix, task_id=task_result.id)
         task_data_ttl = int(
             self.lease_ttl.total_seconds() * 3 + self.result_ttl.total_seconds()
         )
-        ingress_key = self.INGRESS_KEY.format(
-            prefix=self.key_prefix, queue_name=task.queue_name
-        )
 
         pipe = self.client.pipeline()
         pipe.hset(
@@ -215,7 +220,7 @@ class RedisTaskBackend(ThreadmillTaskBackend):
             },
         )
         pipe.expire(task_key, task_data_ttl)
-        pipe.zadd(ingress_key, {task_result.id: enqueued_at_ms})
+        pipe.publish(self.telemetry_channel, f"ingress:{task.queue_name}")
 
         if task.run_after is not None:
             deferred_key = self.DEFERRED_KEY.format(
@@ -224,9 +229,7 @@ class RedisTaskBackend(ThreadmillTaskBackend):
             run_after_ms = task.run_after.timestamp() * 1000
             pipe.zadd(deferred_key, {task_result.id: run_after_ms})
         else:
-            queue_key = self.QUEUE_KEY.format(
-                prefix=self.key_prefix, queue_name=task.queue_name
-            )
+            queue_key = self._segment_key(TaskResultStatus.READY, task.queue_name)
             pipe.zadd(queue_key, {task_result.id: score})
 
         pipe.execute()
@@ -246,8 +249,8 @@ class RedisTaskBackend(ThreadmillTaskBackend):
             key
             for queue_name in queue_names
             for key in (
-                self.RUNNING_KEY.format(prefix=self.key_prefix, queue_name=queue_name),
-                self.QUEUE_KEY.format(prefix=self.key_prefix, queue_name=queue_name),
+                self._segment_key(TaskResultStatus.RUNNING, queue_name),
+                self._segment_key(TaskResultStatus.READY, queue_name),
             )
         ]
 
@@ -261,7 +264,7 @@ class RedisTaskBackend(ThreadmillTaskBackend):
                 args=[
                     str(now_ms),
                     now_iso,
-                    self.key_prefix + ":task:",
+                    f"{self.key_prefix}:task:",
                     str(len(queue_names)),
                     worker,
                     str(int(self.lease_ttl.total_seconds() * 1000)),
@@ -281,18 +284,18 @@ class RedisTaskBackend(ThreadmillTaskBackend):
 
     def acknowledge(self, task_result: TaskResult) -> None:
         serialized = self.serialize_task_result(task_result)
-        running_key = self.RUNNING_KEY.format(
-            prefix=self.key_prefix, queue_name=task_result.task.queue_name
+        running_key = self._segment_key(
+            TaskResultStatus.RUNNING, task_result.task.queue_name
         )
         result_key = self.RESULT_KEY.format(
             prefix=self.key_prefix, result_id=task_result.id
         )
         task_key = self.TASK_KEY.format(prefix=self.key_prefix, task_id=task_result.id)
-        successful_results_key = self.SUCCESSFUL_RESULTS_KEY.format(
-            prefix=self.key_prefix, queue_name=task_result.task.queue_name
+        successful_results_key = self._segment_key(
+            TaskResultStatus.SUCCESSFUL, task_result.task.queue_name
         )
-        failed_results_key = self.FAILED_RESULTS_KEY.format(
-            prefix=self.key_prefix, queue_name=task_result.task.queue_name
+        failed_results_key = self._segment_key(
+            TaskResultStatus.FAILED, task_result.task.queue_name
         )
         finished_at = task_result.finished_at or timezone.now()
         finish_score = finished_at.timestamp() * 1000
@@ -311,6 +314,8 @@ class RedisTaskBackend(ThreadmillTaskBackend):
                 str(int(self.result_ttl.total_seconds())),
                 str(finish_score),
                 task_result.status.name,
+                self.telemetry_channel,
+                task_result.task.queue_name,
             ],
         )
 
@@ -322,13 +327,19 @@ class RedisTaskBackend(ThreadmillTaskBackend):
             finished_at=None,
         )
         serialized = self.serialize_task_result(task_result)
-        running_key = self.RUNNING_KEY.format(
-            prefix=self.key_prefix, queue_name=task_result.task.queue_name
+        running_key = self._segment_key(
+            TaskResultStatus.RUNNING, task_result.task.queue_name
         )
         deferred_key = self.DEFERRED_KEY.format(
             prefix=self.key_prefix, queue_name=task_result.task.queue_name
         )
+        failed_key = self._segment_key(
+            TaskResultStatus.FAILED, task_result.task.queue_name
+        )
         task_key = self.TASK_KEY.format(prefix=self.key_prefix, task_id=task_result.id)
+        result_key = self.RESULT_KEY.format(
+            prefix=self.key_prefix, result_id=task_result.id
+        )
         score = self._compute_score(task_result.task.priority, task_result.enqueued_at)
         run_after_ms = run_after.timestamp() * 1000
         task_data_ttl = int(
@@ -337,9 +348,25 @@ class RedisTaskBackend(ThreadmillTaskBackend):
 
         pipe = self.client.pipeline()
         pipe.zrem(running_key, task_result.id)
+        pipe.zrem(failed_key, task_result.id)
+        pipe.delete(result_key)
         pipe.hset(task_key, mapping={"data": serialized, "score": str(score)})
         pipe.expire(task_key, task_data_ttl)
         pipe.zadd(deferred_key, {task_result.id: run_after_ms})
+        pipe.publish(self.telemetry_channel, f"ingress:{task_result.task.queue_name}")
+        pipe.execute()
+
+    def dequeue(self, task_result: TaskResult) -> None:
+        self.client.zrem(
+            self._segment_key(task_result.status, task_result.task.queue_name),
+            task_result.id,
+        )
+
+    def purge(self, queue_name: str) -> None:
+        pattern = f"{self.key_prefix}:{queue_name}:*"
+        pipe = self.client.pipeline()
+        for key in self.client.scan_iter(match=pattern):
+            pipe.delete(key)
         pipe.execute()
 
     def peek(
@@ -350,34 +377,27 @@ class RedisTaskBackend(ThreadmillTaskBackend):
         count: int = 1,
     ) -> Generator[TaskResult]:
         match status:
-            case TaskResultStatus.READY:
-                yield from self._peek(
-                    self.QUEUE_KEY, self.TASK_KEY, queue_name, count, "data"
-                )
-            case TaskResultStatus.RUNNING:
+            case TaskResultStatus.READY | TaskResultStatus.RUNNING:
                 yield from self._peek(
-                    self.RUNNING_KEY, self.TASK_KEY, queue_name, count, "data"
+                    self._segment_key(status, queue_name),
+                    self.TASK_KEY,
+                    count,
+                    "data",
                 )
-            case TaskResultStatus.SUCCESSFUL:
+            case TaskResultStatus.SUCCESSFUL | TaskResultStatus.FAILED:
                 yield from self._peek(
-                    self.SUCCESSFUL_RESULTS_KEY, self.RESULT_KEY, queue_name, count
-                )
-            case TaskResultStatus.FAILED:
-                yield from self._peek(
-                    self.FAILED_RESULTS_KEY, self.RESULT_KEY, queue_name, count
+                    self._segment_key(status, queue_name),
+                    self.RESULT_KEY,
+                    count,
                 )
 
     def _peek(
         self,
-        zset_key_template: str,
+        zset_key: str,
         data_key_template: str,
-        queue_name: str,
         count: int,
         field: str | None = None,
     ) -> Generator[TaskResult]:
-        zset_key = zset_key_template.format(
-            prefix=self.key_prefix, queue_name=queue_name
-        )
         pipe = self.client.pipeline()
         for member in self.client.zrange(zset_key, 0, count - 1):
             member_id = member.decode() if isinstance(member, bytes) else member
@@ -403,94 +423,61 @@ class RedisTaskBackend(ThreadmillTaskBackend):
             return self.deserialize_task_result(data)
         raise TaskResultDoesNotExist(f"Task result {result_id!r} does not exist.")
 
-    def _trim_telemetry(self, queue_name: str, *, now_ms: float) -> None:
-        cutoff = now_ms - self.result_ttl.total_seconds() * 1000
-        pipe = self.client.pipeline()
-        pipe.zremrangebyscore(
-            self.INGRESS_KEY.format(prefix=self.key_prefix, queue_name=queue_name),
-            0,
-            cutoff,
-        )
-        pipe.zremrangebyscore(
-            self.SUCCESSFUL_RESULTS_KEY.format(
-                prefix=self.key_prefix, queue_name=queue_name
-            ),
-            0,
-            cutoff,
-        )
-        pipe.zremrangebyscore(
-            self.FAILED_RESULTS_KEY.format(
-                prefix=self.key_prefix, queue_name=queue_name
-            ),
-            0,
-            cutoff,
-        )
-        pipe.execute()
-
-    def telemetry(
+    async def queue_stats(
         self, *, interval: datetime.timedelta = datetime.timedelta(seconds=60)
     ) -> BackendTelemetry:
-        now_ms = time.time() * 1000
-        window_start_ms = now_ms - interval.total_seconds() * 1000
-        exclusive_start = f"({window_start_ms}"
-        retention_start_ms = now_ms - self.result_ttl.total_seconds() * 1000
-        pipe = self.client.pipeline()
+        client = self.async_client
+        pipe = client.pipeline()
         for queue_name in self.queues:
-            pipe.zcard(
-                self.QUEUE_KEY.format(prefix=self.key_prefix, queue_name=queue_name)
-            )
-            pipe.zcard(
-                self.RUNNING_KEY.format(prefix=self.key_prefix, queue_name=queue_name)
-            )
+            pipe.zcard(self._segment_key(TaskResultStatus.READY, queue_name))
+            pipe.zcard(self._segment_key(TaskResultStatus.RUNNING, queue_name))
             pipe.zcard(
                 self.DEFERRED_KEY.format(prefix=self.key_prefix, queue_name=queue_name)
             )
-            successful_results_key = self.SUCCESSFUL_RESULTS_KEY.format(
-                prefix=self.key_prefix, queue_name=queue_name
-            )
-            failed_results_key = self.FAILED_RESULTS_KEY.format(
-                prefix=self.key_prefix, queue_name=queue_name
-            )
-            pipe.zcard(successful_results_key)
-            pipe.zcard(failed_results_key)
-            ingress_key = self.INGRESS_KEY.format(
-                prefix=self.key_prefix, queue_name=queue_name
-            )
-            pipe.zcount(ingress_key, exclusive_start, now_ms)
-            pipe.zcount(successful_results_key, exclusive_start, now_ms)
-            pipe.zcount(failed_results_key, exclusive_start, now_ms)
-            pipe.zremrangebyscore(ingress_key, 0, retention_start_ms)
-        results = pipe.execute()
+            pipe.zcard(self._segment_key(TaskResultStatus.SUCCESSFUL, queue_name))
+            pipe.zcard(self._segment_key(TaskResultStatus.FAILED, queue_name))
+        results = await pipe.execute()
+        zero_rates = QueueRates(interval=interval, ingress=0, egress=0)
         queues: dict[str, QueueStats] = {}
         for index, queue_name in enumerate(self.queues):
-            base = index * 9
-            (
-                ready,
-                running,
-                deferred,
-                successful,
-                failed,
-                ingress,
-                successful_rate,
-                failed_rate,
-                _,
-            ) = (int(c or 0) for c in results[base : base + 9])
+            base = index * 5
             queues[queue_name] = QueueStats(
                 counts=QueueCounts(
-                    ready=ready,
-                    running=running,
-                    deferred=deferred,
-                    successful=successful,
-                    failed=failed,
-                ),
-                rates=QueueRates(
-                    interval=interval,
-                    ingress=ingress,
-                    egress=successful_rate + failed_rate,
+                    ready=int(results[base] or 0),
+                    running=int(results[base + 1] or 0),
+                    deferred=int(results[base + 2] or 0),
+                    successful=int(results[base + 3] or 0),
+                    failed=int(results[base + 4] or 0),
                 ),
+                rates=zero_rates,
             )
         return BackendTelemetry(queues=queues)
 
+    async def worker_telemetry(
+        self,
+    ) -> collections.abc.AsyncGenerator[TelemetryEvent]:
+        client = self.async_client
+        pubsub = client.pubsub()
+        await pubsub.subscribe(self.telemetry_channel)
+        pubsub.ignore_subscribe_messages = True
+        try:
+            async for message in pubsub.listen():
+                if (data := message.get("data")) is not None:
+                    payload = data.decode() if isinstance(data, bytes) else data
+                    direction, _, queue_name = payload.partition(":")
+                    try:
+                        event = TelemetryEvent(
+                            direction=TelemetryDirection(direction),
+                            queue_name=queue_name,
+                        )
+                    except ValueError:
+                        continue
+                    yield event
+        finally:
+            await pubsub.unsubscribe(self.telemetry_channel)
+            await pubsub.aclose()
+
     def close(self) -> None:
-        """Close the Redis connection."""
+        """Close the Redis connections."""
         self.client.close()
+        self._async_client = None

{threadmill-0.4.0 → threadmill-0.5.1}/threadmill/inspector/app.py

@@ -1,8 +1,11 @@
 """Textual app for the inspector TUI."""
 
+import asyncio
+import dataclasses
 import datetime
 import logging
 import math
+import time
 import typing
 from typing import Any
 
@@ -13,6 +16,7 @@ from django.tasks import (
     TaskResultStatus,
     task_backends,
 )
+from textual import work
 from textual.app import App, ComposeResult
 from textual.binding import Binding
 from textual.containers import Horizontal, Vertical
@@ -24,17 +28,27 @@ from textual.widgets import (
     ListItem,
     ListView,
     Select,
+    Sparkline,
     Static,
     TabbedContent,
     TabPane,
 )
 
-from ..backends.base import BackendTelemetry, ThreadmillTaskBackend
+from ..backends.base import (
+    BackendTelemetry,
+    TelemetryDirection,
+    ThreadmillTaskBackend,
+)
+from .screens import ConfirmScreen, PurgeScreen
+from .telemetry import TelemetryBuffer
 
 logger = logging.getLogger(__name__)
 
-TELEMETRY_INTERVAL_SECONDS = 2.0
-"""Seconds between automatic queue-stat refreshes; the task list stays manual."""
+TELEMETRY_INTERVAL = datetime.timedelta(seconds=2)
+"""Between automatic queue-stat refreshes; the task list stays manual."""
+
+SPARKLINE_INTERVAL = datetime.timedelta(seconds=1)
+"""Between sparkline refreshes from the rolling telemetry buffer."""
 
 TAB_STATUSES: list[tuple[str, TaskResultStatus | None]] = [
     ("Running", TaskResultStatus.RUNNING),
@@ -163,6 +177,21 @@ class TaskList(Vertical):
     counts: reactive[dict[str, int]] = reactive({})
     selected_task: reactive[TaskResult | None] = reactive(None)
 
+    BINDINGS = [
+        Binding("f5", "refresh", "Refresh"),
+        Binding("r", "requeue", "Requeue"),
+        Binding("d", "dequeue", "Drop"),
+    ]
+
+    def check_action(self, action: str, parameters: tuple[object, ...]) -> bool | None:
+        """Show requeue only on the Failed tab, dequeue on non-Running tabs."""
+        status = self.active_status()
+        if action == "requeue":
+            return status == TaskResultStatus.FAILED
+        if action == "dequeue":
+            return status is not None and status != TaskResultStatus.RUNNING
+        return True
+
     def compose(self) -> ComposeResult:
         with TabbedContent(initial="tab-ready") as tabs:
             for label, _ in TAB_STATUSES:
@@ -212,6 +241,11 @@ class TaskList(Vertical):
         """Activate the tab with the given id."""
         self._tabs.active = tab_id
 
+    def action_refresh(self) -> None:
+        """Refresh telemetry and re-fetch the task list."""
+        self.app._poll_queue_stats()
+        self.refresh_tasks()
+
     def on_data_table_row_selected(self, event: DataTable.RowSelected) -> None:
         """Notify the app when a task row is selected."""
         if isinstance(event.row_key.value, str):
@@ -227,6 +261,7 @@ class TaskList(Vertical):
     ) -> None:
         """Refresh the visible table when the user switches tabs."""
         self._refresh_data()
+        self.refresh_bindings()
 
     def _select_task_by_id(self, task_id: str) -> None:
         """Find the task result matching the row key in the current results."""
@@ -235,6 +270,55 @@ class TaskList(Vertical):
             None,
         )
 
+    def active_status(self) -> TaskResultStatus | None:
+        """Return the status segment of the currently active task tab."""
+        tab_id = self._tabs.active.removeprefix("tab-")
+        return next(
+            (status for label, status in TAB_STATUSES if label.lower() == tab_id),
+            None,
+        )
+
+    def action_requeue(self) -> None:
+        """Requeue the selected failed task after confirmation."""
+        task = self.selected_task
+        if task and self.active_status() == TaskResultStatus.FAILED:
+            self.app.push_screen(
+                ConfirmScreen(f"Requeue task {task.id[:8]}?"),
+                lambda confirmed: self._do_requeue(confirmed, task),
+            )
+
+    def _do_requeue(self, confirmed: bool, task: TaskResult) -> None:
+        if confirmed:
+            try:
+                self.backend.requeue(task, datetime.datetime.now(datetime.UTC))
+            except Exception:  # noqa: BLE001
+                logger.exception("Requeue failed for task %s", task.id)
+                self.app.notify("Requeue failed.", severity="error")
+            else:
+                self.app._after_action("Task requeued.")
+
+    def action_dequeue(self) -> None:
+        """Remove the selected task from its current segment after confirmation."""
+        task = self.selected_task
+        if task and self.active_status() is not None:
+            self.app.push_screen(
+                ConfirmScreen(
+                    f"Delete task {task.task.func.__name__} ({task.id[:8]})?",
+                    danger=True,
+                ),
+                lambda confirmed: self._do_dequeue(confirmed, task),
+            )
+
+    def _do_dequeue(self, confirmed: bool, task: TaskResult) -> None:
+        if confirmed:
+            try:
+                self.backend.dequeue(task)
+            except Exception:  # noqa: BLE001
+                logger.exception("Delete failed for task %s", task.id)
+                self.app.notify("Delete failed.", severity="error")
+            else:
+                self.app._after_action("Task deleted.")
+
     def _refresh_data(self) -> None:
         """Fetch and display tasks for the current queue and active tab."""
         backend = self.backend
@@ -291,6 +375,10 @@ class QueueList(ListView):
 
     telemetry: reactive[BackendTelemetry | None] = reactive(None)
 
+    BINDINGS = [
+        Binding("p", "purge", "Purge"),
+    ]
+
     def __init__(self, **kwargs: Any) -> None:
         super().__init__(**kwargs)
         self._items: dict[str, QueueItem] = {}
@@ -300,12 +388,10 @@ class QueueList(ListView):
         self.border_title = "Queues"
 
     def watch_telemetry(self, telemetry: BackendTelemetry | None) -> None:
-        """Refresh queue labels when a new telemetry snapshot arrives."""
         if telemetry is not None:
             self.update_telemetry(telemetry)
 
     def update_telemetry(self, telemetry: BackendTelemetry) -> None:
-        """Refresh queue labels from a new telemetry snapshot."""
         queues = telemetry.queues
         for queue_name, stats in sorted(queues.items()):
             theme = BUILTIN_THEMES[self.app.theme]
@@ -334,21 +420,38 @@ class QueueList(ListView):
             self._notify_selection(target)
 
     def _notify_selection(self, queue_name: str) -> None:
-        """Tell the app which queue to display."""
         self.app._task_list.queue_name = queue_name
 
+    def action_purge(self) -> None:
+        item = self.highlighted_child
+        if isinstance(item, QueueItem):
+            queue_name = item.queue_name
+            self.app.push_screen(
+                PurgeScreen(queue_name),
+                lambda confirmed: self._do_purge(confirmed, queue_name),
+            )
+
+    def _do_purge(self, confirmed: bool, queue_name: str) -> None:
+        if confirmed:
+            try:
+                self.app.backend.purge(queue_name)
+            except Exception:  # noqa: BLE001
+                logger.exception("Purge failed for %r", queue_name)
+                self.app.notify("Purge failed.", severity="error")
+            else:
+                self.app._after_action(f"Purged queue {queue_name}.")
+
 
 class InspectorApp(App):
     """Threadmill TUI inspector with backend/queue/task panes."""
 
     CSS_PATH = "inspector.scss"
     BINDINGS = [
-        Binding("q", "quit", "Quit"),
-        Binding("f5", "refresh", "Refresh"),
         *(
             Binding(key, f"switch_tab('tab-{tab_id}')", tab_id.capitalize())
             for tab_id, key in TAB_KEYS.items()
         ),
+        Binding("q", "quit", "Quit"),
     ]
 
     backend: reactive[ThreadmillTaskBackend] = reactive(None)
@@ -365,8 +468,10 @@ class InspectorApp(App):
         self._task_list: TaskList | None = None
         self._task_detail: TaskDetail | None = None
         self._options_static: Static | None = None
+        self._sparkline_timer = None
         self._telemetry_timer = None
         self._auto_refresh = auto_refresh
+        self.telemetry_buffer = TelemetryBuffer()
         self.set_reactive(InspectorApp.backend, backend)
 
     def compose(self) -> ComposeResult:
@@ -387,6 +492,9 @@ class InspectorApp(App):
                         telemetry=InspectorApp.telemetry
                     )
                 with Vertical(id="right-pane"):
+                    with Horizontal(id="sparkline-row"):
+                        yield Sparkline(id="ingress-sparkline")
+                        yield Sparkline(id="egress-sparkline")
                     yield TaskList(id="task-list").data_bind(
                         backend=InspectorApp.backend,
                         telemetry=InspectorApp.telemetry,
@@ -402,22 +510,31 @@ class InspectorApp(App):
         self._task_detail = self.query_one("#task-detail", TaskDetail)
         self._options_static = self.query_one("#backend-options", Static)
         self._refresh_options()
-        self._refresh_telemetry()
+        self.query_one("#ingress-sparkline", Sparkline).border_title = "Ingress"
+        self.query_one("#egress-sparkline", Sparkline).border_title = "Egress"
+        self._poll_queue_stats()
+        self._listen_telemetry()
         if self._auto_refresh:
             self._telemetry_timer = self.set_interval(
-                TELEMETRY_INTERVAL_SECONDS,
-                self._refresh_telemetry,
+                TELEMETRY_INTERVAL.total_seconds(),
+                self._poll_queue_stats,
                 name="telemetry-refresh",
             )
+            self._sparkline_timer = self.set_interval(
+                SPARKLINE_INTERVAL.total_seconds(),
+                self._refresh_sparklines,
+                name="sparkline-refresh",
+            )
         self._queue_list.focus()
 
     def action_quit(self) -> None:
         """Exit the TUI."""
         self.exit()
 
-    def action_refresh(self) -> None:
-        """Refresh queue stats and re-fetch the task list on demand."""
-        self._refresh_telemetry()
+    def _after_action(self, message: str) -> None:
+        """Refresh telemetry and tasks after a mutating queue action."""
+        self.notify(message)
+        self._poll_queue_stats()
         self._task_list.refresh_tasks()
 
     def action_switch_tab(self, tab_id: str) -> None:
@@ -426,7 +543,9 @@ class InspectorApp(App):
 
     def watch_backend(self) -> None:
         self._refresh_options()
-        self._refresh_telemetry()
+        self.telemetry_buffer = TelemetryBuffer()
+        self._poll_queue_stats()
+        self._listen_telemetry()
 
     def on_select_changed(self, event: Select.Changed) -> None:
         """Switch backend when the user selects a different alias."""
@@ -453,9 +572,53 @@ class InspectorApp(App):
         ]
         self._options_static.update(" ".join(parts) or "No options")
 
-    def _refresh_telemetry(self) -> None:
-        """Poll the backend for fresh telemetry."""
+    @work(exclusive=True, group="queue-stats")
+    async def _poll_queue_stats(self) -> None:
+        """Fetch queue counts and merge live pub/sub rates into the telemetry."""
         try:
-            self.telemetry = self.backend.telemetry()
+            telemetry = await self.backend.queue_stats()
         except Exception:  # noqa: BLE001
-            logger.exception("Failed to refresh telemetry")
+            logger.exception("Failed to refresh queue stats")
+        else:
+            for queue_name, stats in telemetry.queues.items():
+                live_rates = self.telemetry_buffer.rates_for(queue_name)
+                telemetry.queues[queue_name] = dataclasses.replace(
+                    stats, rates=live_rates
+                )
+            self.telemetry = telemetry
+
+    def _refresh_sparklines(self) -> None:
+        """Push the latest 60-second rolling series into the ingress/egress sparklines."""
+        queue_name = self._task_list.queue_name
+        ingress = self.query_one("#ingress-sparkline", Sparkline)
+        egress = self.query_one("#egress-sparkline", Sparkline)
+        now = time.monotonic()
+        ingress.data = (
+            self.telemetry_buffer.series(
+                queue_name, TelemetryDirection.INGRESS, now=now
+            )
+            if queue_name
+            else [0.0] * 60
+        )
+        egress.data = (
+            self.telemetry_buffer.series(queue_name, TelemetryDirection.EGRESS, now=now)
+            if queue_name
+            else [0.0] * 60
+        )
+
+    @work(exclusive=True, group="telemetry")
+    async def _listen_telemetry(self) -> None:
+        """Feed the telemetry buffer from the backend's pub/sub stream."""
+        try:
+            stream = self.backend.worker_telemetry()
+            if asyncio.iscoroutine(stream):
+                stream = await stream
+        except NotImplementedError:
+            pass
+        else:
+            if stream is not None:
+                try:
+                    async for event in stream:
+                        self.telemetry_buffer.record(event, now=time.monotonic())
+                except asyncio.CancelledError:
+                    raise

{threadmill-0.4.0 → threadmill-0.5.1}/threadmill/inspector/inspector.scss

@@ -2,6 +2,11 @@ Screen {
     background: $background;
 }
 
+ConfirmScreen,
+PurgeScreen {
+    align: center middle;
+}
+
 * {
     scrollbar-color: $primary 10%;
     scrollbar-color-hover: $primary 80%;
@@ -37,6 +42,29 @@ Select {
     width: 42;
 }
 
+#sparkline-row {
+    height: 5;
+    margin-bottom: 0;
+}
+
+#ingress-sparkline,
+#egress-sparkline {
+    width: 1fr;
+    height: 1fr;
+    padding: 0;
+    margin: 0;
+    border: round $accent 40%;
+    border-title-color: $primary;
+}
+
+#ingress-sparkline > .sparkline--max-color {
+    color: $success;
+}
+
+#egress-sparkline > .sparkline--max-color {
+    color: $error;
+}
+
 #backend-row {
     height: 3;
 }
@@ -71,7 +99,7 @@ QueueList ListItem {
 }
 
 TaskList {
-    height: 60%;
+    height: 1fr;
 
     TabbedContent,
     ContentSwitcher,
@@ -99,8 +127,34 @@ DataTable {
 }
 
 TaskDetail {
-    height: 40%;
+    height: 1fr;
     width: 100%;
     overflow: scroll;
     padding: 0 1;
 }
+
+#dialog {
+    width: 60;
+    height: auto;
+    background: $surface;
+    padding: 1 2;
+}
+
+#dialog.warning {
+    border: thick $warning;
+}
+
+#dialog.danger {
+    border: thick $error;
+}
+
+#prompt {
+    text-align: center;
+    margin-bottom: 1;
+}
+
+#instruction {
+    text-align: center;
+    color: $text-muted;
+    margin-top: 1;
+}

threadmill-0.5.1/threadmill/inspector/screens.py

@@ -0,0 +1,52 @@
+"""Modal confirmation screens for inspector queue actions."""
+
+from textual.app import ComposeResult
+from textual.containers import Vertical
+from textual.events import Key
+from textual.screen import ModalScreen
+from textual.widgets import Input, Label
+
+
+class ConfirmScreen(ModalScreen[bool]):
+    """Yes/no confirmation: Enter confirms, Esc cancels."""
+
+    def __init__(self, prompt: str, *, danger: bool = False) -> None:
+        super().__init__()
+        self._prompt = prompt
+        self._danger = danger
+
+    def compose(self) -> ComposeResult:
+        with Vertical(id="dialog", classes="danger" if self._danger else "warning"):
+            yield Label(self._prompt, id="prompt")
+            yield Label("Press Enter to confirm, Esc to cancel.", id="instruction")
+
+    def on_key(self, event: Key) -> None:
+        match event.key:
+            case "enter":
+                self.dismiss(True)
+            case "escape":
+                self.dismiss(False)
+
+
+class PurgeScreen(ModalScreen[bool]):
+    """Type the queue name to confirm a purge."""
+
+    def __init__(self, queue_name: str) -> None:
+        super().__init__()
+        self._queue_name = queue_name
+
+    def compose(self) -> ComposeResult:
+        with Vertical(id="dialog", classes="danger"):
+            yield Label(f"Purge queue [bold]{self._queue_name}[/bold]?", id="prompt")
+            yield Label(
+                "Type the queue name to confirm, Esc to cancel.", id="instruction"
+            )
+            yield Input(placeholder=self._queue_name, id="confirm-input")
+
+    def on_input_submitted(self, event: Input.Submitted) -> None:
+        if event.value == self._queue_name:
+            self.dismiss(True)
+
+    def on_key(self, event: Key) -> None:
+        if event.key == "escape":
+            self.dismiss(False)

threadmill-0.5.1/threadmill/inspector/telemetry.py

@@ -0,0 +1,81 @@
+"""In-memory rolling telemetry buffer fed by backend pub/sub events."""
+
+import collections
+import datetime
+import time
+
+from threadmill.backends.base import QueueRates, TelemetryDirection, TelemetryEvent
+
+WINDOW_SECONDS = 60
+"""Rolling window the inspector displays for ingress/egress sparklines."""
+
+
+class TelemetryBuffer:
+    """Per-(queue, direction) rolling 60-second count buffer."""
+
+    def __init__(self, *, window: int = WINDOW_SECONDS) -> None:
+        self._window = window
+        self._counts: dict[
+            tuple[str, TelemetryDirection], collections.OrderedDict[int, int]
+        ] = collections.defaultdict(collections.OrderedDict)
+
+    def record(self, event: TelemetryEvent, *, now: float | None = None) -> None:
+        """Increment the current-second bucket for the event's queue+direction."""
+        second = int(now if now is not None else time.monotonic())
+        buckets = self._counts[event.queue_name, event.direction]
+        buckets[second] = buckets.get(second, 0) + 1
+        self._evict(buckets, second)
+
+    def series(
+        self,
+        queue_name: str,
+        direction: TelemetryDirection,
+        *,
+        now: float | None = None,
+    ) -> list[float]:
+        """Return ``window`` per-second counts, oldest to newest, for the pair."""
+        second = int(now if now is not None else time.monotonic())
+        buckets = self._counts.get((queue_name, direction))
+        if buckets is not None:
+            self._evict(buckets, second)
+        return [
+            float(buckets.get(second - offset, 0)) if buckets else 0.0
+            for offset in reversed(range(self._window))
+        ]
+
+    def rates_for(
+        self,
+        queue_name: str,
+        *,
+        interval: datetime.timedelta = datetime.timedelta(seconds=WINDOW_SECONDS),
+        now: float | None = None,
+    ) -> QueueRates:
+        """Sum the last ``interval`` of events into per-second ingress/egress rates."""
+        second = int(now if now is not None else time.monotonic())
+        span = min(int(interval.total_seconds()), self._window)
+        ingress = self._sum_direction(
+            queue_name, TelemetryDirection.INGRESS, second, span
+        )
+        egress = self._sum_direction(
+            queue_name, TelemetryDirection.EGRESS, second, span
+        )
+        return QueueRates(
+            interval=datetime.timedelta(seconds=span), ingress=ingress, egress=egress
+        )
+
+    def _sum_direction(
+        self, queue_name: str, direction: TelemetryDirection, second: int, span: int
+    ) -> int:
+        buckets = self._counts.get((queue_name, direction))
+        if buckets is None:
+            return 0
+        self._evict(buckets, second)
+        return sum(buckets.get(second - offset, 0) for offset in range(span))
+
+    def _evict(
+        self, buckets: collections.OrderedDict[int, int], now_second: int
+    ) -> None:
+        """Drop buckets older than the window (LRU pop from front)."""
+        cutoff = now_second - self._window
+        while buckets and next(iter(buckets)) <= cutoff:
+            buckets.popitem(last=False)