threadmill 0.2.0__tar.gz → 0.3.1__tar.gz

{threadmill-0.2.0 → threadmill-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: threadmill
-Version: 0.2.0
+Version: 0.3.1
 Summary: The most reliable backend for Django's task framework.
 Keywords: Django,tasks,worker
 Author-email: Johannes Maron <johannes@maron.family>
@@ -24,9 +24,11 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Classifier: Framework :: Django
+Classifier: Framework :: Django :: 6.0
 Classifier: Framework :: Django :: 6.1
 License-File: LICENSE
-Requires-Dist: django>=6.1a1
+Requires-Dist: django>=6.0
+Requires-Dist: textual>=8.2.7 ; extra == "inspector"
 Requires-Dist: redis>=5.0 ; extra == "redis"
 Project-URL: Changelog, https://github.com/codingjoe/threadmill/releases
 Project-URL: Documentation, https://github.com/codingjoe/threadmill/
@@ -35,15 +37,14 @@ Project-URL: Homepage, https://github.com/codingjoe/threadmill
 Project-URL: Issues, https://github.com/codingjoe/threadmill/issues
 Project-URL: Releasenotes, https://github.com/codingjoe/threadmill/releases/latest
 Project-URL: Source, https://github.com/codingjoe/threadmill
+Provides-Extra: inspector
 Provides-Extra: redis
 
-# Threadmill
-
 <p align="center">
   <picture>
-    <source media="(prefers-color-scheme: dark)" srcset="https://github.com/codingjoe/threadmill/raw/main/images/logo-dark.svg">
-    <source media="(prefers-color-scheme: light)" srcset="https://github.com/codingjoe/threadmill/raw/main/images/logo-light.svg">
-    <img alt="Threadmill: A queue agnostic worker for Django's task framework." src="https://github.com/codingjoe/threadmill/raw/main/images/logo-light.svg">
+    <source media="(prefers-color-scheme: dark)" srcset="https://github.com/codingjoe/threadmill/raw/main/docs/images/logo-dark.svg">
+    <source media="(prefers-color-scheme: light)" srcset="https://github.com/codingjoe/threadmill/raw/main/docs/images/logo-light.svg">
+    <img alt="Threadmill: Durable high-performance backend for Django's task framework." src="https://github.com/codingjoe/threadmill/raw/main/docs/images/logo-light.svg">
   </picture>
 <br>
   <a href="https://github.com/codingjoe/threadmill/">Documentation</a> |
@@ -52,7 +53,9 @@ Provides-Extra: redis
   <a href="https://github.com/sponsors/codingjoe">Funding</a> 💚
 </p>
 
-**A queue agnostic worker for Django's task framework.**
+# Threadmill [![PyPi Version](https://img.shields.io/pypi/v/threadmill.svg)](https://pypi.python.org/pypi/threadmill/) [![Test Coverage](https://codecov.io/gh/codingjoe/threadmill/branch/main/graph/badge.svg)](https://codecov.io/gh/codingjoe/threadmill) [![GitHub License](https://img.shields.io/github/license/codingjoe/threadmill)](https://raw.githubusercontent.com/codingjoe/threadmill/master/LICENSE)
+
+**Durable high-performance backend for Django's task framework.**
 
 ## Design Principles
 
@@ -60,13 +63,6 @@ Provides-Extra: redis
 - **Consistency** – We never lose data, even if someone unplugs the power or network.
 - **Utilization** – We keep the CPU saturated with tasks, not with idle time or waiting for locks.
 
-> [!WARNING]
-> Threadmill requires a development version of Django and is in a preview stage.
-
-[![PyPi Version](https://img.shields.io/pypi/v/threadmill.svg)](https://pypi.python.org/pypi/threadmill/)
-[![Test Coverage](https://codecov.io/gh/codingjoe/threadmill/branch/main/graph/badge.svg)](https://codecov.io/gh/codingjoe/threadmill)
-[![GitHub License](https://img.shields.io/github/license/codingjoe/threadmill)](https://raw.githubusercontent.com/codingjoe/threadmill/master/LICENSE)
-
 ## Sponsors
 
 [![Sponsors](https://django.the-box.sh/sponsors/codingjoe/threadmill.svg)](https://github.com/sponsors/codingjoe)
@@ -100,10 +96,16 @@ TASKS = {
 }
 ```
 
-Finally, you launch the worker pool:
+Optionally, install the inspector dependency if you want the TUI:
+
+```console
+uv add threadmill[inspector]
+```
+
+Then launch the worker pool:
 
 ```console
-uv run manage.py threadmill
+uv run manage.py threadmill worker
 ```
 
 ## Usage
@@ -118,7 +120,7 @@ Depending on your workload, you can tweak the number of processes and threads.
 Processes allow for parallel compute (no GIL) while threads are great for low-memory concurrent IO.
 
 ```console
-uv run manage.py threadmill --processes 4 --threads 2
+uv run manage.py threadmill worker --processes 4 --threads 2
 ```
 
 #### Health
@@ -126,7 +128,7 @@ uv run manage.py threadmill --processes 4 --threads 2
 If your tasks leak memory, you can recycle (restart) the workers after a certain number of tasks have been processed:
 
 ```console
-uv run manage.py threadmill --max-tasks 1000 --max-tasks-jitter 100
+uv run manage.py threadmill worker --max-tasks 1000 --max-tasks-jitter 100
 ```
 
 This will restart the workers after 1000 tasks have been processed, with a random jitter of up to 100 tasks to avoid all workers restarting at the same time.
@@ -141,6 +143,18 @@ All workers will finish the tasks they acquired and acknowledge them.
 You can use `--exit-empty` to exit immediately after all tasks have been processed,
 which might be useful for draining a one-off queue.
 
+### Inspector
+
+![Inspector TUI screenshot](https://github.com/codingjoe/threadmill/raw/main/docs/images/TUI-screenshot.svg)
+
+The optional TUI inspector lets you watch queues, tasks, and task details in real-time.
+Install it with the `inspector` extra and launch it from a separate terminal:
+
+```console
+uv add threadmill[inspector]
+uv run manage.py threadmill inspector
+```
+
 ### Redis Backend Options
 
 The `RedisTaskBackend` accepts the following options under `OPTIONS` in your

{threadmill-0.2.0 → threadmill-0.3.1}/README.md

@@ -1,10 +1,8 @@
-# Threadmill
-
 <p align="center">
   <picture>
-    <source media="(prefers-color-scheme: dark)" srcset="https://github.com/codingjoe/threadmill/raw/main/images/logo-dark.svg">
-    <source media="(prefers-color-scheme: light)" srcset="https://github.com/codingjoe/threadmill/raw/main/images/logo-light.svg">
-    <img alt="Threadmill: A queue agnostic worker for Django's task framework." src="https://github.com/codingjoe/threadmill/raw/main/images/logo-light.svg">
+    <source media="(prefers-color-scheme: dark)" srcset="https://github.com/codingjoe/threadmill/raw/main/docs/images/logo-dark.svg">
+    <source media="(prefers-color-scheme: light)" srcset="https://github.com/codingjoe/threadmill/raw/main/docs/images/logo-light.svg">
+    <img alt="Threadmill: Durable high-performance backend for Django's task framework." src="https://github.com/codingjoe/threadmill/raw/main/docs/images/logo-light.svg">
   </picture>
 <br>
   <a href="https://github.com/codingjoe/threadmill/">Documentation</a> |
@@ -13,7 +11,9 @@
   <a href="https://github.com/sponsors/codingjoe">Funding</a> 💚
 </p>
 
-**A queue agnostic worker for Django's task framework.**
+# Threadmill [![PyPi Version](https://img.shields.io/pypi/v/threadmill.svg)](https://pypi.python.org/pypi/threadmill/) [![Test Coverage](https://codecov.io/gh/codingjoe/threadmill/branch/main/graph/badge.svg)](https://codecov.io/gh/codingjoe/threadmill) [![GitHub License](https://img.shields.io/github/license/codingjoe/threadmill)](https://raw.githubusercontent.com/codingjoe/threadmill/master/LICENSE)
+
+**Durable high-performance backend for Django's task framework.**
 
 ## Design Principles
 
@@ -21,13 +21,6 @@
 - **Consistency** – We never lose data, even if someone unplugs the power or network.
 - **Utilization** – We keep the CPU saturated with tasks, not with idle time or waiting for locks.
 
-> [!WARNING]
-> Threadmill requires a development version of Django and is in a preview stage.
-
-[![PyPi Version](https://img.shields.io/pypi/v/threadmill.svg)](https://pypi.python.org/pypi/threadmill/)
-[![Test Coverage](https://codecov.io/gh/codingjoe/threadmill/branch/main/graph/badge.svg)](https://codecov.io/gh/codingjoe/threadmill)
-[![GitHub License](https://img.shields.io/github/license/codingjoe/threadmill)](https://raw.githubusercontent.com/codingjoe/threadmill/master/LICENSE)
-
 ## Sponsors
 
 [![Sponsors](https://django.the-box.sh/sponsors/codingjoe/threadmill.svg)](https://github.com/sponsors/codingjoe)
@@ -61,10 +54,16 @@ TASKS = {
 }
 ```
 
-Finally, you launch the worker pool:
+Optionally, install the inspector dependency if you want the TUI:
+
+```console
+uv add threadmill[inspector]
+```
+
+Then launch the worker pool:
 
 ```console
-uv run manage.py threadmill
+uv run manage.py threadmill worker
 ```
 
 ## Usage
@@ -79,7 +78,7 @@ Depending on your workload, you can tweak the number of processes and threads.
 Processes allow for parallel compute (no GIL) while threads are great for low-memory concurrent IO.
 
 ```console
-uv run manage.py threadmill --processes 4 --threads 2
+uv run manage.py threadmill worker --processes 4 --threads 2
 ```
 
 #### Health
@@ -87,7 +86,7 @@ uv run manage.py threadmill --processes 4 --threads 2
 If your tasks leak memory, you can recycle (restart) the workers after a certain number of tasks have been processed:
 
 ```console
-uv run manage.py threadmill --max-tasks 1000 --max-tasks-jitter 100
+uv run manage.py threadmill worker --max-tasks 1000 --max-tasks-jitter 100
 ```
 
 This will restart the workers after 1000 tasks have been processed, with a random jitter of up to 100 tasks to avoid all workers restarting at the same time.
@@ -102,6 +101,18 @@ All workers will finish the tasks they acquired and acknowledge them.
 You can use `--exit-empty` to exit immediately after all tasks have been processed,
 which might be useful for draining a one-off queue.
 
+### Inspector
+
+![Inspector TUI screenshot](https://github.com/codingjoe/threadmill/raw/main/docs/images/TUI-screenshot.svg)
+
+The optional TUI inspector lets you watch queues, tasks, and task details in real-time.
+Install it with the `inspector` extra and launch it from a separate terminal:
+
+```console
+uv add threadmill[inspector]
+uv run manage.py threadmill inspector
+```
+
 ### Redis Backend Options
 
 The `RedisTaskBackend` accepts the following options under `OPTIONS` in your

{threadmill-0.2.0 → threadmill-0.3.1}/pyproject.toml

@@ -30,13 +30,17 @@ classifiers = [
   "Programming Language :: Python :: 3.13",
   "Programming Language :: Python :: 3.14",
   "Framework :: Django",
+  "Framework :: Django :: 6.0",
   "Framework :: Django :: 6.1",
 ]
 requires-python = ">=3.12"
-dependencies = ["django>=6.1a1"]
+dependencies = ["django>=6.0"]
 
 [project.optional-dependencies]
 redis = ["redis>=5.0"]
+inspector = [
+    "textual>=8.2.7",
+]
 
 [project.urls]
 # https://packaging.python.org/en/latest/specifications/well-known-project-urls/#well-known-labels

{threadmill-0.2.0 → threadmill-0.3.1}/threadmill/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0'
-__version_tuple__ = version_tuple = (0, 2, 0)
+__version__ = version = '0.3.1'
+__version_tuple__ = version_tuple = (0, 3, 1)
 
-__commit_id__ = commit_id = 'gb135558ff'
+__commit_id__ = commit_id = 'g32b680b8b'

{threadmill-0.2.0 → threadmill-0.3.1}/threadmill/backends/base.py

@@ -7,12 +7,69 @@ import json
 import threading
 from abc import ABC
 
+import django
 from django.core.serializers.json import DjangoJSONEncoder
 from django.tasks import DEFAULT_TASK_QUEUE_NAME, Task, TaskResult, TaskResultStatus
 from django.tasks.backends.base import BaseTaskBackend
 from django.tasks.base import TaskError
 from django.utils.module_loading import import_string
 
+if django.VERSION == (6, 0):
+    # https://github.com/django/django/commit/8c8b833d32c02d3ae6f43b04bb1e45968796b402
+    @dataclasses.dataclass(frozen=True, slots=True, kw_only=True)
+    class Task(Task):
+        @classmethod
+        def _reconstruct(cls, kwargs):
+            func_path = kwargs["func"]
+            try:
+                func = import_string(func_path)
+                kwargs["func"] = func.func
+            except (ImportError, AttributeError) as e:
+                msg = f"Expected {func_path!r} to point to a Task instance."
+                raise ValueError(msg) from e
+            return cls(**kwargs)
+
+        def __reduce__(self):
+            kwargs = {f.name: getattr(self, f.name) for f in dataclasses.fields(self)}
+            kwargs["func"] = self.module_path
+
+            return (self.__class__._reconstruct, (kwargs,))
+
+
+@dataclasses.dataclass(kw_only=True, slots=True)
+class QueueCounts:
+    """Point-in-time cardinality of each queue segment."""
+
+    ready: int
+    running: int
+    deferred: int
+    successful: int
+    failed: int
+
+
+@dataclasses.dataclass(kw_only=True, slots=True)
+class QueueRates:
+    """Rolling ingress/egress throughput over a time window (time-series data)."""
+
+    interval: datetime.timedelta
+    ingress: int
+    egress: int
+
+
+@dataclasses.dataclass(kw_only=True, slots=True)
+class QueueStats:
+    """Telemetry for a single queue: point-in-time counts plus rolling rates."""
+
+    counts: QueueCounts
+    rates: QueueRates
+
+
+@dataclasses.dataclass(kw_only=True, slots=True)
+class BackendTelemetry:
+    """Snapshot of counts and rates across a backend's queues."""
+
+    queues: dict[str, QueueStats]
+
 
 class Broker(threading.Thread):
     """Backend maintenance thread launched by the task executor."""
@@ -71,6 +128,7 @@ class TaskResultEncoder(DjangoJSONEncoder):
 class ThreadmillTaskBackend(BaseTaskBackend, ABC):
     """Interface for task queues to be processed by the executor."""
 
+    task_class = Task  # can be removed in the future when Django 6.0 support is dropped
     supports_async_task = True
     supports_get_result = True
     broker_class: type[Broker] | None = None
@@ -138,8 +196,25 @@ class ThreadmillTaskBackend(BaseTaskBackend, ABC):
         self,
         queue_name: str = DEFAULT_TASK_QUEUE_NAME,
         *,
-        status: TaskResultStatus | None = None,
+        status: TaskResultStatus,
         count: int = 1,
     ) -> collections.abc.Generator[TaskResult, None, None]:
-        """Yield acknowledged results from a queue, optionally filtered by status."""
+        """
+        Yield up to ``count`` tasks from a queue in the given status segment.
+
+        Args:
+            queue_name: The name of the queue to peek into.
+            status: The status of the tasks to yield.
+            count: The maximum number of tasks to yield. If 0, yield all available tasks.
+        """
+        raise NotImplementedError
+
+    def telemetry(
+        self, *, interval: datetime.timedelta = datetime.timedelta(seconds=60)
+    ) -> BackendTelemetry:
+        """Return a snapshot of stats for all configured queues.
+
+        Args:
+            interval: The time window for rolling rates.
+        """
         raise NotImplementedError

threadmill-0.3.1/threadmill/backends/lua/acknowledge.lua

@@ -0,0 +1,34 @@
+-- Finalize a completed task: remove it from the running set, persist the
+-- result with a TTL, delete the task data hash, and add the result to the
+-- per-status results history set. Evicts results whose finish score falls
+-- outside the retention window (result_ttl). The per-status history sets are
+-- the time series the inspector counts over for telemetry, so no separate
+-- egress window or status counters are needed here.
+--
+-- KEYS[1]  -- running set (ZSET)
+-- KEYS[2]  -- result key (STRING, stores serialized TaskResult)
+-- KEYS[3]  -- task data key (HASH, deleted after acknowledge)
+-- KEYS[4]  -- successful results history (ZSET, scored by finish time)
+-- KEYS[5]  -- failed results history (ZSET, scored by finish time)
+-- ARGV[1]  -- task ID
+-- ARGV[2]  -- serialized TaskResult JSON
+-- ARGV[3]  -- result TTL in seconds
+-- ARGV[4]  -- finish timestamp in milliseconds (score for the history set)
+-- ARGV[5]  -- status (SUCCESSFUL or FAILED)
+-- Returns: 1 on success, 0 if task was not in the running set
+
+local removed = redis.call('ZREM', KEYS[1], ARGV[1])
+if removed == 0 then
+  return 0  -- Task already reaped, skip
+end
+redis.call('SET', KEYS[2], ARGV[2], 'EX', ARGV[3])
+redis.call('DEL', KEYS[3])
+local finish = tonumber(ARGV[4])
+local cutoff = finish - tonumber(ARGV[3]) * 1000
+local results_key = KEYS[4]
+if ARGV[5] ~= 'SUCCESSFUL' then
+  results_key = KEYS[5]
+end
+redis.call('ZADD', results_key, finish, ARGV[1])
+redis.call('ZREMRANGEBYSCORE', results_key, 0, cutoff)
+return 1

{threadmill-0.2.0 → threadmill-0.3.1}/threadmill/backends/lua/reaper.lua

@@ -1,7 +1,9 @@
--- Fail tasks whose processing lease has expired.
+-- Fail tasks whose processing lease has expired. Reaped tasks are recorded
+-- in the failed results history (the time series the inspector counts over),
+-- then evicted once older than result_ttl.
 --
 -- KEYS[1]  -- running set (ZSET)
--- KEYS[2]  -- results history set (ZSET)
+-- KEYS[2]  -- failed results history (ZSET, scored by finish time)
 -- ARGV[1]  -- current time in milliseconds (for score comparison)
 -- ARGV[2]  -- task key prefix (e.g. "threadmill:default:task:")
 -- ARGV[3]  -- result key prefix (e.g. "threadmill:default:result:")
@@ -33,4 +35,6 @@ for _, task_id in ipairs(stale) do
     end
   end
 end
+-- Evict failed results older than result_ttl to bound the history.
+redis.call('ZREMRANGEBYSCORE', KEYS[2], 0, tonumber(ARGV[1]) - tonumber(ARGV[5]) * 1000)
 return #stale