toro-queue 0.5.0__tar.gz → 0.5.1__tar.gz

{toro_queue-0.5.0 → toro_queue-0.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: toro-queue
-Version: 0.5.0
+Version: 0.5.1
 Summary: An async-first, Redis-backed job queue for Python.
 Project-URL: Homepage, https://github.com/ilovepixelart/toro
 Project-URL: Repository, https://github.com/ilovepixelart/toro

{toro_queue-0.5.0 → toro_queue-0.5.1}/docs/processing.md

@@ -55,7 +55,11 @@ event loop - see [workers vs. slots](concepts.md). Two practical consequences:
   treated as stalled ([Reliability](reliability.md)).
 - **Connections scale with concurrency.** Each idle slot parks a (blocking-pop)
   connection, so the worker sizes its own pool to `concurrency + headroom`. If
-  you pass your own `connection`, size its pool accordingly.
+  you pass your own `connection`, size its pool accordingly, and give it a read
+  timeout (`socket_timeout`) above `block_timeout`. redis-py 8 defaults it to 5 s.
+  The worker keeps its blocking pop under whatever it finds, so a tighter read
+  timeout means idle slots re-poll sooner than `block_timeout` asks, with a
+  warning logged.
 
 A busy slot doesn't return to the blocking wait between jobs: the finish call
 also claims the next job in the same round trip (fetch-next - see

{toro_queue-0.5.0 → toro_queue-0.5.1}/pyproject.toml

@@ -2,7 +2,7 @@
 # PyPI distribution name - plain `toro` is taken (abandoned). The import package
 # stays `toro` (e.g. `pip install toro-queue` then `import toro`).
 name = "toro-queue"
-version = "0.5.0"
+version = "0.5.1"
 description = "An async-first, Redis-backed job queue for Python."
 readme = "README.md"
 requires-python = ">=3.10"

toro_queue-0.5.1/tests/integration/test_shutdown.py

@@ -0,0 +1,39 @@
+"""Integration: shutdown must not abandon a job the worker has already claimed."""
+
+import asyncio
+
+from toro import Worker
+
+PREFIX = "torotest"
+
+
+async def test_stop_during_a_finish_round_trip_keeps_the_claimed_job(q, run_until):
+    """The fetch flag is read BEFORE the finish script runs. If stop() lands while
+    that round trip is in flight, the script has already claimed the next job. It
+    is in flight like any other and must be processed, not dropped: dropped, it
+    sits locked in `active` until its lock expires and the sweep comes round."""
+    await q.add("a", {})
+    await q.add("b", {})
+    done: list[str] = []
+
+    async def proc(job):
+        done.append(job.name)
+
+    w = Worker(q.name, proc, prefix=PREFIX, stalled_interval=0)
+    real_finish = w._finish_completed
+
+    async def finish_then_stop(job, result):
+        nxt = await real_finish(job, result)  # fetch was "1": the script claimed `b`
+        w._running = False  # stop() lands while that round trip was in flight
+        return nxt
+
+    w._finish_completed = finish_then_stop
+    task = asyncio.create_task(w.run())
+    try:
+        assert await run_until(lambda: "a" in done)
+        await asyncio.sleep(0.3)  # room for the loop to either process `b` or drop it
+        assert done == ["a", "b"]
+        assert await q.redis.llen(q.keys.active) == 0
+    finally:
+        await w.stop(grace_period=1)
+        task.cancel()

{toro_queue-0.5.0 → toro_queue-0.5.1}/tests/integration/test_worker_resilience.py

@@ -6,6 +6,8 @@ Redis error between the blocking pop and the finish.
 import asyncio
 import time
 
+import redis.asyncio as aioredis
+
 
 async def test_raising_event_callback_does_not_kill_the_slot(q, run_worker, run_until):
     done = []
@@ -80,3 +82,20 @@ async def test_corrupt_job_hash_does_not_kill_the_slot(q, run_worker, run_until)
         await q.add("good", {})
         assert await run_until(lambda: len(done) >= 1, timeout=10.0), "slot died on corrupt data"
         assert done == ["good"]
+
+
+async def test_idle_repoll_survives_a_read_timeout_shorter_than_the_pop(q, run_worker, run_until):
+    """The blocking pop must come back before the connection's read timeout, or it
+    raises instead of timing out quietly and the loop never reaches the claim: a
+    job whose wake was missed is then stranded for good. A caller-provided
+    connection can carry any read timeout, so the worker has to stay under it."""
+    await q.add("stranded", {})
+    await q.redis.delete(q.keys.marker)  # the missed wake: work waiting, no marker
+    done = []
+
+    async def proc(job):
+        done.append(job.id)
+
+    conn = aioredis.from_url("redis://localhost:6379", socket_timeout=0.4, decode_responses=True)
+    async with run_worker(q, proc, connection=conn, block_timeout=1.0, stalled_interval=0):
+        assert await run_until(lambda: done, timeout=3.0), "the idle re-poll never claimed it"

toro_queue-0.5.1/tests/unit/test_worker_connection.py

@@ -0,0 +1,58 @@
+"""Unit: the connection a Worker builds for itself must outlast its blocking pop."""
+
+import pytest
+import redis.asyncio as aioredis
+from redis.asyncio.connection import Connection
+
+from toro import Worker
+from toro.connection import read_timeout
+from toro.worker import pop_timeout
+
+
+async def _noop(job):
+    return None
+
+
+def test_read_timeout_exceeds_the_blocking_pop_at_defaults():
+    """redis-py 8 defaults socket_timeout to 5s, the same as block_timeout. Equal is
+    not enough: the client gives up on the read as the server answers it. The value
+    is spelled out because conftest shrinks the default block_timeout in tests."""
+    w = Worker("q", _noop, block_timeout=5.0)
+    timeout = read_timeout(w.redis)
+    assert timeout is None or timeout > w.block_timeout
+
+
+def test_read_timeout_follows_a_longer_block_timeout():
+    w = Worker("q", _noop, block_timeout=30.0)
+    timeout = read_timeout(w.redis)
+    assert timeout is None or timeout > 30.0
+
+
+@pytest.mark.parametrize(
+    ("read_timeout_s", "block_timeout", "expected"),
+    [
+        (None, 5.0, 5.0),  # no read timeout: nothing to stay under
+        (10.0, 5.0, 5.0),  # fits with room
+        (35.0, 30.0, 30.0),  # a worker-built connection: block_timeout + margin
+        (5.0, 5.0, 4.0),  # redis-py 8 defaults on a caller-provided connection
+        (0.4, 1.0, 0.2),  # tiny read timeout: half of it, never zero or negative
+    ],
+)
+def test_pop_timeout_stays_under_the_read_timeout(read_timeout_s, block_timeout, expected):
+    assert pop_timeout(read_timeout_s, block_timeout) == expected
+
+
+def test_a_caller_provided_connection_on_library_defaults_is_clamped():
+    """The common real case: a client built with no socket_timeout at all. The
+    library default applies but never shows up in the pool's kwargs. The oracle is
+    redis-py's own Connection, not toro's helper, so a helper that misses the
+    default cannot simply agree with itself."""
+    library_default = Connection().socket_timeout
+    conn = aioredis.from_url("redis://localhost:6379", decode_responses=True)
+    assert read_timeout(conn) == library_default
+
+    w = Worker("q", _noop, connection=conn, block_timeout=30.0)
+    if library_default is None:
+        assert w._pop_timeout == 30.0
+    else:
+        assert w._pop_timeout < library_default

{toro_queue-0.5.0 → toro_queue-0.5.1}/toro/__init__.py

@@ -26,4 +26,4 @@ __all__ = [
     "ToroError",
     "Worker",
 ]
-__version__ = "0.5.0"
+__version__ = "0.5.1"

{toro_queue-0.5.0 → toro_queue-0.5.1}/toro/connection.py

@@ -2,14 +2,21 @@
 
 from __future__ import annotations
 
+from typing import Any
+
 import redis.asyncio as aioredis
 from redis.asyncio.retry import Retry
 from redis.backoff import ExponentialBackoff
 from redis.exceptions import ConnectionError as RedisConnectionError
 from redis.exceptions import TimeoutError as RedisTimeoutError
 
+# How far a connection's read timeout sits above the longest blocking pop it serves.
+READ_MARGIN = 5.0
+
 
-def connect(url: str, *, max_connections: int = 50) -> aioredis.Redis:
+def connect(
+    url: str, *, max_connections: int = 50, blocking_timeout: float | None = None
+) -> aioredis.Redis:
     """Open a ``decode_responses`` client tuned for toro's long-lived connections.
 
     Connections that sit idle for a while (a worker's blocking pop, the result()
@@ -27,7 +34,15 @@ def connect(url: str, *, max_connections: int = 50) -> aioredis.Redis:
     ``max_connections`` must exceed the count of connections held LONG-term: a
     worker parks one per process loop inside BZPOPMIN, so it sizes the pool from
     its concurrency (measured: concurrency=100 on a 50-pool starves and errors).
+
+    ``blocking_timeout`` is the longest server-side block this client will issue.
+    The read timeout is set above it: a blocking pop has to come back before the
+    client gives up on the read, or it raises instead of timing out quietly.
+    redis-py 8 defaults the read timeout to 5s, which a 5s pop does not beat.
     """
+    extra: dict[str, Any] = {}
+    if blocking_timeout is not None:
+        extra["socket_timeout"] = blocking_timeout + READ_MARGIN
     pool = aioredis.BlockingConnectionPool.from_url(
         url,
         max_connections=max_connections,
@@ -36,5 +51,17 @@ def connect(url: str, *, max_connections: int = 50) -> aioredis.Redis:
         socket_keepalive=True,
         retry=Retry(ExponentialBackoff(), retries=3),
         retry_on_error=[RedisConnectionError, RedisTimeoutError],
+        **extra,
     )
     return aioredis.Redis(connection_pool=pool)
+
+
+def read_timeout(client: aioredis.Redis) -> float | None:
+    """Return the read timeout a connection from this client's pool will really use.
+
+    Read off a connection from the pool's own factory, so the library default
+    counts: it is absent from ``connection_kwargs`` unless the caller passed it.
+    The factory is what a pool subclass overrides, and building the object opens
+    nothing.
+    """
+    return client.connection_pool.make_connection().socket_timeout

{toro_queue-0.5.0 → toro_queue-0.5.1}/toro/worker.py

@@ -33,7 +33,7 @@ from typing import Any, TypedDict, cast
 from redis.asyncio import Redis
 
 from . import scripts
-from .connection import connect
+from .connection import connect, read_timeout
 from .job import Backoff, Job, JobContext, JobOptions
 from .keys import Keys
 from .scheduler import next_run
@@ -64,6 +64,19 @@ def _pairs(flat: list[str] | None) -> dict[str, str]:
     return dict(zip(it, it, strict=False))
 
 
+def pop_timeout(read_timeout_s: float | None, block_timeout: float) -> float:
+    """How long the blocking pop may block: `block_timeout`, held under the
+    connection's read timeout. A pop that outlasts the read raises instead of
+    timing out quietly, the loop never reaches the claim, and a job whose wake was
+    missed is stranded. A connection the worker built has room by construction; a
+    caller-provided one can carry any read timeout.
+    """
+    if read_timeout_s is None:
+        return block_timeout
+    ceiling = max(read_timeout_s - 1.0, read_timeout_s / 2)
+    return min(block_timeout, ceiling)
+
+
 def compute_backoff(backoff: Backoff, attempts_made: int) -> int:
     """Delay (ms) before the next attempt. `backoff` is None/0, an int (fixed ms),
     or {"type": "fixed"|"exponential", "delay": ms}. Exponential doubles per attempt.
@@ -107,7 +120,9 @@ class Worker:
         # Each process loop PARKS a connection inside BZPOPMIN, so the pool must
         # exceed the concurrency or loops starve waiting for connections. A
         # caller-provided connection must be sized accordingly by the caller.
-        self.redis = connection or connect(url, max_connections=max(50, concurrency + 10))
+        self.redis = connection or connect(
+            url, max_connections=max(50, concurrency + 10), blocking_timeout=block_timeout
+        )
         self.concurrency = concurrency
         # Queue-wide rate limit, shared by all workers via one token bucket in Redis.
         # `{"max": N, "duration": ms}` = at most N jobs per duration. All workers on a
@@ -119,6 +134,14 @@ class Worker:
         self.rl_max = int(rate_limit["max"]) if rate_limit else 0
         self.rl_duration = int(rate_limit["duration"]) if rate_limit else 0
         self.block_timeout = block_timeout
+        self._pop_timeout = pop_timeout(read_timeout(self.redis), block_timeout)
+        if self._pop_timeout < block_timeout:
+            logger.warning(
+                "block_timeout %.2fs does not fit under the connection's read timeout; "
+                "idle workers re-poll every %.2fs instead",
+                block_timeout,
+                self._pop_timeout,
+            )
 
         # Reliability knobs.
         self.token = uuid.uuid4().hex
@@ -282,12 +305,16 @@ class Worker:
                 # The marker only wakes us; the real claim is the atomic
                 # MOVE_TO_ACTIVE below. A timeout (None) is fine - we still try
                 # to acquire, so a missed marker can never strand a job.
-                await self.redis.bzpopmin(self.keys.marker, self.block_timeout)
+                await self.redis.bzpopmin(self.keys.marker, self._pop_timeout)
                 if not self._running:
                     break  # shutting down - don't claim a new job
                 loaded = await self._acquire()
-                # Keep processing as long as each finish hands us the next job.
-                while loaded is not None and self._running:
+                # Keep processing as long as each finish hands us the next job. No
+                # `_running` check here: a job in hand is already claimed, and stop()
+                # can land during the very round trip that claimed it. Dropped, it
+                # would sit locked in `active` until the sweep. Shutdown ends the
+                # chain by itself: a stopping worker finishes with fetch=0.
+                while loaded is not None:
                     loaded = await self._handle(loaded)
             except asyncio.CancelledError:
                 raise

{toro_queue-0.5.0 → toro_queue-0.5.1}/uv.lock

@@ -424,7 +424,7 @@ wheels = [
 
 [[package]]
 name = "toro-queue"
-version = "0.5.0"
+version = "0.5.1"
 source = { editable = "." }
 dependencies = [
     { name = "croniter" },