pydocket 0.1.2__py3-none-any.whl → 0.1.4__py3-none-any.whl

docket/cli.py

@@ -228,6 +228,14 @@ def worker(
             envvar="DOCKET_WORKER_RECONNECTION_DELAY",
         ),
     ] = timedelta(seconds=5),
+    minimum_check_interval: Annotated[
+        timedelta,
+        typer.Option(
+            parser=duration,
+            help="The minimum interval to check for tasks",
+            envvar="DOCKET_WORKER_MINIMUM_CHECK_INTERVAL",
+        ),
+    ] = timedelta(milliseconds=100),
     until_finished: Annotated[
         bool,
         typer.Option(
@@ -244,6 +252,7 @@ def worker(
             concurrency=concurrency,
             redelivery_timeout=redelivery_timeout,
             reconnection_delay=reconnection_delay,
+            minimum_check_interval=minimum_check_interval,
             until_finished=until_finished,
             tasks=tasks,
         )

docket/docket.py

@@ -147,17 +147,17 @@ class Docket:
         self._monitor_strikes_task = asyncio.create_task(self._monitor_strikes())
 
         # Ensure that the stream and worker group exist
-        async with self.redis() as r:
-            try:
+        try:
+            async with self.redis() as r:
                 await r.xgroup_create(
                     groupname=self.worker_group_name,
                     name=self.stream_key,
                     id="0-0",
                     mkstream=True,
                 )
-            except redis.exceptions.RedisError as e:
-                if "BUSYGROUP" not in repr(e):
-                    raise
+        except redis.exceptions.RedisError as e:
+            if "BUSYGROUP" not in repr(e):
+                raise
 
         return self
 
@@ -178,8 +178,21 @@ class Docket:
 
     @asynccontextmanager
     async def redis(self) -> AsyncGenerator[Redis, None]:
-        async with Redis.from_url(self.url) as redis:  # type: ignore
-            yield redis
+        redis: Redis | None = None
+        try:
+            redis = await Redis.from_url(
+                self.url,
+                single_connection_client=True,
+            )
+            await redis.__aenter__()
+            try:
+                yield redis
+            finally:
+                await asyncio.shield(redis.__aexit__(None, None, None))
+        finally:
+            # redis 4.6.0 doesn't automatically disconnect and leaves connections open
+            if redis:
+                await asyncio.shield(redis.connection_pool.disconnect())
 
     def register(self, function: Callable[..., Awaitable[Any]]) -> None:
         from .dependencies import validate_dependencies
@@ -513,7 +526,7 @@ class Docket:
                 )
                 running.append(RunningExecution(execution, worker_name, started))
             else:
-                future.append(execution)
+                future.append(execution)  # pragma: no cover
 
         for message in queued_messages:
             function = self.tasks[message[b"function"].decode()]

docket/worker.py

@@ -67,7 +67,7 @@ class Worker:
         concurrency: int = 10,
         redelivery_timeout: timedelta = timedelta(minutes=5),
         reconnection_delay: timedelta = timedelta(seconds=5),
-        minimum_check_interval: timedelta = timedelta(milliseconds=10),
+        minimum_check_interval: timedelta = timedelta(milliseconds=100),
     ) -> None:
         self.docket = docket
         self.name = name or f"worker:{uuid4()}"
@@ -110,6 +110,7 @@ class Worker:
         concurrency: int = 10,
         redelivery_timeout: timedelta = timedelta(minutes=5),
         reconnection_delay: timedelta = timedelta(seconds=5),
+        minimum_check_interval: timedelta = timedelta(milliseconds=100),
         until_finished: bool = False,
         tasks: list[str] = ["docket.tasks:standard_tasks"],
     ) -> None:
@@ -123,6 +124,7 @@ class Worker:
                 concurrency=concurrency,
                 redelivery_timeout=redelivery_timeout,
                 reconnection_delay=reconnection_delay,
+                minimum_check_interval=minimum_check_interval,
             ) as worker:
                 if until_finished:
                     await worker.run_until_finished()
@@ -170,28 +172,31 @@ class Worker:
                     """
                 local total_work = redis.call('ZCARD', KEYS[1])
                 local due_work = 0
-                local tasks = redis.call('ZRANGEBYSCORE', KEYS[1], 0, ARGV[1])
 
-                for i, key in ipairs(tasks) do
-                    local hash_key = ARGV[2] .. ":" .. key
-                    local task_data = redis.call('HGETALL', hash_key)
-
-                    if #task_data > 0 then
-                        local task = {}
-                        for j = 1, #task_data, 2 do
-                            task[task_data[j]] = task_data[j+1]
+                if total_work > 0 then
+                    local tasks = redis.call('ZRANGEBYSCORE', KEYS[1], 0, ARGV[1])
+
+                    for i, key in ipairs(tasks) do
+                        local hash_key = ARGV[2] .. ":" .. key
+                        local task_data = redis.call('HGETALL', hash_key)
+
+                        if #task_data > 0 then
+                            local task = {}
+                            for j = 1, #task_data, 2 do
+                                task[task_data[j]] = task_data[j+1]
+                            end
+
+                            redis.call('XADD', KEYS[2], '*',
+                                'key', task['key'],
+                                'when', task['when'],
+                                'function', task['function'],
+                                'args', task['args'],
+                                'kwargs', task['kwargs'],
+                                'attempt', task['attempt']
+                            )
+                            redis.call('DEL', hash_key)
+                            due_work = due_work + 1
                         end
-
-                        redis.call('XADD', KEYS[2], '*',
-                            'key', task['key'],
-                            'when', task['when'],
-                            'function', task['function'],
-                            'args', task['args'],
-                            'kwargs', task['kwargs'],
-                            'attempt', task['attempt']
-                        )
-                        redis.call('DEL', hash_key)
-                        due_work = due_work + 1
                     end
                 end
 
@@ -275,8 +280,6 @@ class Worker:
 
                     for message_id, message in redeliveries:
                         start_task(message_id, message)
-                        if available_slots <= 0:
-                            break
 
                     if available_slots <= 0:
                         continue
@@ -295,8 +298,7 @@ class Worker:
                     for _, messages in new_deliveries:
                         for message_id, message in messages:
                             start_task(message_id, message)
-                            if available_slots <= 0:
-                                break
+
             except asyncio.CancelledError:
                 if active_tasks:  # pragma: no cover
                     logger.info(

{pydocket-0.1.2.dist-info → pydocket-0.1.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pydocket
-Version: 0.1.2
+Version: 0.1.4
 Summary: A distributed background task system for Python functions
 Project-URL: Homepage, https://github.com/chrisguidry/docket
 Project-URL: Bug Tracker, https://github.com/chrisguidry/docket/issues

{pydocket-0.1.2.dist-info → pydocket-0.1.4.dist-info}/RECORD

@@ -1,16 +1,16 @@
 docket/__init__.py,sha256=GoJYpyuO6QFeBB8GNaxGGvMMuai55Eaw_8u-o1PM3hk,743
 docket/__main__.py,sha256=Vkuh7aJ-Bl7QVpVbbkUksAd_hn05FiLmWbc-8kbhZQ4,34
 docket/annotations.py,sha256=GZwOPtPXyeIhnsLh3TQMBnXrjtTtSmF4Ratv4vjPx8U,950
-docket/cli.py,sha256=6dSD94wIa0aJNigeAgPR0hXE4a6szLYAs8h5F8V0CGs,18853
+docket/cli.py,sha256=N0vp1zO5Wau4nBDMJOU34hYn11HR3PaYY3Ybk1gS8XY,19188
 docket/dependencies.py,sha256=Vht3qKbik-HQ7jsAU5k-eig4_yuru56-ZewjBVVu4yM,4325
-docket/docket.py,sha256=4bXMwDfOXbubBY1NiVyO7k598KvTDbbPzXjnN1dVGJU,19659
+docket/docket.py,sha256=TWeZ63NfN6Eq4lFzKoQTJz88ECZsH3-gqYszhQl-bXs,20124
 docket/execution.py,sha256=rHsQ60BbNREzcpUC_RvbGUctdLaprYp1x46sT6jTrdc,11416
 docket/instrumentation.py,sha256=USo8ptCFcwQj_YaUpJvsUHPb0QfQr50i9dF4tYgYde4,2992
 docket/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 docket/tasks.py,sha256=K1f_W1z4m9RVz1GJ1ymWY5ZaRmqHO1SebNBVENlkelU,1471
-docket/worker.py,sha256=nlp4sJCI2zvAhxivl5igFbKDe3z1aOOiAEz30XelAGY,18371
-pydocket-0.1.2.dist-info/METADATA,sha256=_Mpg9K1AsgbagCDZWY0DF29awyi-Ox9OMAN17bf6wKE,13092
-pydocket-0.1.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-pydocket-0.1.2.dist-info/entry_points.txt,sha256=4WOk1nUlBsUT5O3RyMci2ImuC5XFswuopElYcLHtD5k,47
-pydocket-0.1.2.dist-info/licenses/LICENSE,sha256=YuVWU_ZXO0K_k2FG8xWKe5RGxV24AhJKTvQmKfqXuyk,1087
-pydocket-0.1.2.dist-info/RECORD,,
+docket/worker.py,sha256=8wnWxHj7ctHPxEGSRxPTsHksZ6OWRoG5dKpSkvIZP88,18479
+pydocket-0.1.4.dist-info/METADATA,sha256=y8PTR9Xwh8MeMr7ZhPJzUQGtQUjQXN3QRpYTvxtKfv0,13092
+pydocket-0.1.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+pydocket-0.1.4.dist-info/entry_points.txt,sha256=4WOk1nUlBsUT5O3RyMci2ImuC5XFswuopElYcLHtD5k,47
+pydocket-0.1.4.dist-info/licenses/LICENSE,sha256=YuVWU_ZXO0K_k2FG8xWKe5RGxV24AhJKTvQmKfqXuyk,1087
+pydocket-0.1.4.dist-info/RECORD,,