pydocket 0.1.2__tar.gz → 0.1.4__tar.gz

{pydocket-0.1.2 → pydocket-0.1.4}/.github/workflows/ci.yml

@@ -7,15 +7,16 @@ on:
     branches: [main]
   workflow_call:
 
-
 jobs:
   test:
-    name: Test Python ${{ matrix.python-version }}
+    name: Test Python ${{ matrix.python-version }}, Redis ${{ matrix.redis-version }}, redis-py ${{ matrix.redis-py-version }}
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         python-version: ["3.12", "3.13"]
-
+        redis-version: ["6.2", "7.4"]
+        redis-py-version: [">=4.6,<5", ">=5"]
 
     steps:
       - uses: actions/checkout@v4
@@ -28,9 +29,11 @@ jobs:
           cache-dependency-glob: "pyproject.toml"
 
       - name: Install dependencies
-        run: uv sync --dev
+        run: uv sync --dev --upgrade-package 'redis${{ matrix.redis-py-version }}'
 
       - name: Run tests
+        env:
+          REDIS_VERSION: ${{ matrix.redis-version }}
         run: uv run pytest --cov-branch --cov-report=xml --cov-report=term-missing:skip-covered
 
       - name: Upload coverage reports to Codecov

{pydocket-0.1.2 → pydocket-0.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pydocket
-Version: 0.1.2
+Version: 0.1.4
 Summary: A distributed background task system for Python functions
 Project-URL: Homepage, https://github.com/chrisguidry/docket
 Project-URL: Bug Tracker, https://github.com/chrisguidry/docket/issues

{pydocket-0.1.2 → pydocket-0.1.4}/pyproject.toml

@@ -70,7 +70,6 @@ asyncio_mode = "auto"
 asyncio_default_fixture_loop_scope = "session"
 filterwarnings = ["error"]
 
-
 [tool.pyright]
 include = ["src", "tests"]
 typeCheckingMode = "strict"

{pydocket-0.1.2 → pydocket-0.1.4}/src/docket/cli.py

@@ -228,6 +228,14 @@ def worker(
             envvar="DOCKET_WORKER_RECONNECTION_DELAY",
         ),
     ] = timedelta(seconds=5),
+    minimum_check_interval: Annotated[
+        timedelta,
+        typer.Option(
+            parser=duration,
+            help="The minimum interval to check for tasks",
+            envvar="DOCKET_WORKER_MINIMUM_CHECK_INTERVAL",
+        ),
+    ] = timedelta(milliseconds=100),
     until_finished: Annotated[
         bool,
         typer.Option(
@@ -244,6 +252,7 @@ def worker(
             concurrency=concurrency,
             redelivery_timeout=redelivery_timeout,
             reconnection_delay=reconnection_delay,
+            minimum_check_interval=minimum_check_interval,
             until_finished=until_finished,
             tasks=tasks,
         )

{pydocket-0.1.2 → pydocket-0.1.4}/src/docket/docket.py

@@ -147,17 +147,17 @@ class Docket:
         self._monitor_strikes_task = asyncio.create_task(self._monitor_strikes())
 
         # Ensure that the stream and worker group exist
-        async with self.redis() as r:
-            try:
+        try:
+            async with self.redis() as r:
                 await r.xgroup_create(
                     groupname=self.worker_group_name,
                     name=self.stream_key,
                     id="0-0",
                     mkstream=True,
                 )
-            except redis.exceptions.RedisError as e:
-                if "BUSYGROUP" not in repr(e):
-                    raise
+        except redis.exceptions.RedisError as e:
+            if "BUSYGROUP" not in repr(e):
+                raise
 
         return self
 
@@ -178,8 +178,21 @@ class Docket:
 
     @asynccontextmanager
     async def redis(self) -> AsyncGenerator[Redis, None]:
-        async with Redis.from_url(self.url) as redis:  # type: ignore
-            yield redis
+        redis: Redis | None = None
+        try:
+            redis = await Redis.from_url(
+                self.url,
+                single_connection_client=True,
+            )
+            await redis.__aenter__()
+            try:
+                yield redis
+            finally:
+                await asyncio.shield(redis.__aexit__(None, None, None))
+        finally:
+            # redis 4.6.0 doesn't automatically disconnect and leaves connections open
+            if redis:
+                await asyncio.shield(redis.connection_pool.disconnect())
 
     def register(self, function: Callable[..., Awaitable[Any]]) -> None:
         from .dependencies import validate_dependencies
@@ -513,7 +526,7 @@ class Docket:
                 )
                 running.append(RunningExecution(execution, worker_name, started))
             else:
-                future.append(execution)
+                future.append(execution)  # pragma: no cover
 
         for message in queued_messages:
             function = self.tasks[message[b"function"].decode()]

{pydocket-0.1.2 → pydocket-0.1.4}/src/docket/worker.py

@@ -67,7 +67,7 @@ class Worker:
         concurrency: int = 10,
         redelivery_timeout: timedelta = timedelta(minutes=5),
         reconnection_delay: timedelta = timedelta(seconds=5),
-        minimum_check_interval: timedelta = timedelta(milliseconds=10),
+        minimum_check_interval: timedelta = timedelta(milliseconds=100),
     ) -> None:
         self.docket = docket
         self.name = name or f"worker:{uuid4()}"
@@ -110,6 +110,7 @@ class Worker:
         concurrency: int = 10,
         redelivery_timeout: timedelta = timedelta(minutes=5),
         reconnection_delay: timedelta = timedelta(seconds=5),
+        minimum_check_interval: timedelta = timedelta(milliseconds=100),
         until_finished: bool = False,
         tasks: list[str] = ["docket.tasks:standard_tasks"],
     ) -> None:
@@ -123,6 +124,7 @@ class Worker:
                 concurrency=concurrency,
                 redelivery_timeout=redelivery_timeout,
                 reconnection_delay=reconnection_delay,
+                minimum_check_interval=minimum_check_interval,
             ) as worker:
                 if until_finished:
                     await worker.run_until_finished()
@@ -170,28 +172,31 @@ class Worker:
                     """
                 local total_work = redis.call('ZCARD', KEYS[1])
                 local due_work = 0
-                local tasks = redis.call('ZRANGEBYSCORE', KEYS[1], 0, ARGV[1])
 
-                for i, key in ipairs(tasks) do
-                    local hash_key = ARGV[2] .. ":" .. key
-                    local task_data = redis.call('HGETALL', hash_key)
-
-                    if #task_data > 0 then
-                        local task = {}
-                        for j = 1, #task_data, 2 do
-                            task[task_data[j]] = task_data[j+1]
+                if total_work > 0 then
+                    local tasks = redis.call('ZRANGEBYSCORE', KEYS[1], 0, ARGV[1])
+
+                    for i, key in ipairs(tasks) do
+                        local hash_key = ARGV[2] .. ":" .. key
+                        local task_data = redis.call('HGETALL', hash_key)
+
+                        if #task_data > 0 then
+                            local task = {}
+                            for j = 1, #task_data, 2 do
+                                task[task_data[j]] = task_data[j+1]
+                            end
+
+                            redis.call('XADD', KEYS[2], '*',
+                                'key', task['key'],
+                                'when', task['when'],
+                                'function', task['function'],
+                                'args', task['args'],
+                                'kwargs', task['kwargs'],
+                                'attempt', task['attempt']
+                            )
+                            redis.call('DEL', hash_key)
+                            due_work = due_work + 1
                         end
-
-                        redis.call('XADD', KEYS[2], '*',
-                            'key', task['key'],
-                            'when', task['when'],
-                            'function', task['function'],
-                            'args', task['args'],
-                            'kwargs', task['kwargs'],
-                            'attempt', task['attempt']
-                        )
-                        redis.call('DEL', hash_key)
-                        due_work = due_work + 1
                     end
                 end
 
@@ -275,8 +280,6 @@ class Worker:
 
                     for message_id, message in redeliveries:
                         start_task(message_id, message)
-                        if available_slots <= 0:
-                            break
 
                     if available_slots <= 0:
                         continue
@@ -295,8 +298,7 @@ class Worker:
                     for _, messages in new_deliveries:
                         for message_id, message in messages:
                             start_task(message_id, message)
-                            if available_slots <= 0:
-                                break
+
             except asyncio.CancelledError:
                 if active_tasks:  # pragma: no cover
                     logger.info(

{pydocket-0.1.2 → pydocket-0.1.4}/tests/conftest.py

@@ -1,4 +1,6 @@
+import os
 import time
+from contextlib import contextmanager
 from datetime import datetime, timezone
 from functools import partial
 from typing import Any, AsyncGenerator, Callable, Generator, Iterable, cast
@@ -9,10 +11,12 @@ import pytest
 import redis.exceptions
 from docker import DockerClient
 from docker.models.containers import Container
-from redis import Redis
+from redis import ConnectionPool, Redis
 
 from docket import Docket, Worker
 
+REDIS_VERSION = os.environ.get("REDIS_VERSION", "7.4")
+
 
 @pytest.fixture
 def now() -> Callable[[], datetime]:
@@ -24,6 +28,19 @@ def redis_port(unused_tcp_port_factory: Callable[[], int]) -> int:
     return unused_tcp_port_factory()
 
 
+@contextmanager
+def _sync_redis(url: str) -> Generator[Redis, None, None]:
+    pool: ConnectionPool | None = None
+    redis = Redis.from_url(url, single_connection_client=True)  # type: ignore
+    try:
+        with redis:
+            pool = redis.connection_pool  # type: ignore
+            yield redis
+    finally:
+        if pool:  # pragma: no branch
+            pool.disconnect()
+
+
 @pytest.fixture(scope="session")
 def redis_server(redis_port: int) -> Generator[Container, None, None]:
     client = DockerClient.from_env()
@@ -39,7 +56,7 @@ def redis_server(redis_port: int) -> Generator[Container, None, None]:
         container.remove(force=True)
 
     container = client.containers.run(
-        "redis:7.4.2",
+        f"redis:{REDIS_VERSION}",
         detach=True,
         ports={"6379/tcp": redis_port},
         labels={"source": "docket-unit-tests"},
@@ -50,8 +67,9 @@ def redis_server(redis_port: int) -> Generator[Container, None, None]:
 
     while True:
         try:
-            with Redis.from_url(url) as r:  # type: ignore
-                if r.ping():  # type: ignore
+            with _sync_redis(url) as r:
+                success = r.ping()  # type: ignore
+                if success:  # pragma: no branch
                     break
         except redis.exceptions.ConnectionError:  # pragma: no cover
             pass
@@ -61,7 +79,7 @@ def redis_server(redis_port: int) -> Generator[Container, None, None]:
     try:
         yield container
     finally:
-        with Redis.from_url(url) as r:  # type: ignore
+        with _sync_redis(url) as r:
             info: dict[str, Any] = r.info()  # type: ignore
 
         container.stop()
@@ -75,10 +93,10 @@ def redis_server(redis_port: int) -> Generator[Container, None, None]:
 
 @pytest.fixture
 def redis_url(redis_server: Container, redis_port: int) -> str:
-    with Redis.from_url(f"redis://localhost:{redis_port}/0") as r:  # type: ignore
+    url = f"redis://localhost:{redis_port}/0"
+    with _sync_redis(url) as r:
         r.flushdb()  # type: ignore
-
-    return f"redis://localhost:{redis_port}/0"
+    return url
 
 
 @pytest.fixture