pydocket 0.6.4__tar.gz → 0.7.0__tar.gz

{pydocket-0.6.4 → pydocket-0.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pydocket
-Version: 0.6.4
+Version: 0.7.0
 Summary: A distributed background task system for Python functions
 Project-URL: Homepage, https://github.com/chrisguidry/docket
 Project-URL: Bug Tracker, https://github.com/chrisguidry/docket/issues

{pydocket-0.6.4 → pydocket-0.7.0}/chaos/driver.py

@@ -153,13 +153,20 @@ async def main(
                     sent_tasks = await r.zcard("hello:sent")
                     received_tasks = await r.zcard("hello:received")
 
+                    stream_length = await r.xlen(docket.stream_key)
+                    pending = await r.xpending(
+                        docket.stream_key, docket.worker_group_name
+                    )
+
                     logger.info(
-                        "sent: %d, received: %d, clients: %d",
+                        "sent: %d, received: %d, stream: %d, pending: %d, clients: %d",
                         sent_tasks,
                         received_tasks,
+                        stream_length,
+                        pending["pending"],
                         connected_clients,
                     )
-                    if sent_tasks >= tasks:
+                    if sent_tasks >= tasks and received_tasks >= sent_tasks:
                         break
             except redis.exceptions.ConnectionError as e:
                 logger.error(
@@ -177,16 +184,13 @@ async def main(
 
                 elif chaos_chance < 0.10:
                     worker_index = random.randrange(len(worker_processes))
-                    worker_to_kill = worker_processes.pop(worker_index)
+                    worker_to_kill = worker_processes[worker_index]
 
                     logger.warning("CHAOS: Killing worker %d...", worker_index)
                     try:
-                        worker_to_kill.terminate()
+                        worker_to_kill.kill()
                     except ProcessLookupError:
                         logger.warning("  What is dead may never die!")
-
-                    logger.warning("CHAOS: Replacing worker %d...", worker_index)
-                    worker_processes.append(await spawn_worker())
                 elif chaos_chance < 0.15:
                     logger.warning("CHAOS: Queuing a toxic task...")
                     try:
@@ -194,6 +198,17 @@ async def main(
                     except redis.exceptions.ConnectionError:
                         pass
 
+            # Check if any worker processes have died and replace them
+            for i in range(len(worker_processes)):
+                process = worker_processes[i]
+                if process.returncode is not None:
+                    logger.warning(
+                        "Worker %d has died with code %d, replacing it...",
+                        i,
+                        process.returncode,
+                    )
+                    worker_processes[i] = await spawn_worker()
+
             await asyncio.sleep(0.25)
 
         async with docket.redis() as r:
@@ -225,5 +240,6 @@ async def main(
 
 if __name__ == "__main__":
     mode = sys.argv[1] if len(sys.argv) > 1 else "chaos"
+    tasks = int(sys.argv[2]) if len(sys.argv) > 2 else 20000
     assert mode in ("performance", "chaos")
-    asyncio.run(main(mode=mode))
+    asyncio.run(main(mode=mode, tasks=tasks))

{pydocket-0.6.4 → pydocket-0.7.0}/chaos/tasks.py

@@ -1,4 +1,6 @@
+import asyncio
 import logging
+import random
 import sys
 import time
 
@@ -29,7 +31,12 @@ async def hello(
 
 
 async def toxic():
-    sys.exit(42)
+    if random.random() < 0.25:
+        sys.exit(42)
+    elif random.random() < 0.5:
+        raise Exception("Boom")
+    else:
+        await asyncio.sleep(random.uniform(0.01, 0.05))
 
 
 chaos_tasks = [hello, toxic]

{pydocket-0.6.4 → pydocket-0.7.0}/src/docket/dependencies.py

@@ -3,7 +3,7 @@ import logging
 import time
 from contextlib import AsyncExitStack, asynccontextmanager
 from contextvars import ContextVar
-from datetime import timedelta
+from datetime import datetime, timedelta, timezone
 from types import TracebackType
 from typing import (
     TYPE_CHECKING,
@@ -14,6 +14,7 @@ from typing import (
     Callable,
     Counter,
     Generic,
+    NoReturn,
     TypeVar,
     cast,
 )
@@ -188,6 +189,10 @@ def TaskLogger() -> logging.LoggerAdapter[logging.Logger]:
     return cast(logging.LoggerAdapter[logging.Logger], _TaskLogger())
 
 
+class ForcedRetry(Exception):
+    """Raised when a task requests a retry via `in_` or `at`"""
+
+
 class Retry(Dependency):
     """Configures linear retries for a task.  You can specify the total number of
     attempts (or `None` to retry indefinitely), and the delay between attempts.
@@ -222,6 +227,17 @@ class Retry(Dependency):
         retry.attempt = execution.attempt
         return retry
 
+    def at(self, when: datetime) -> NoReturn:
+        now = datetime.now(timezone.utc)
+        diff = when - now
+        diff = diff if diff.total_seconds() >= 0 else timedelta(0)
+
+        self.in_(diff)
+
+    def in_(self, when: timedelta) -> NoReturn:
+        self.delay: timedelta = when
+        raise ForcedRetry()
+
 
 class ExponentialRetry(Retry):
     """Configures exponential retries for a task.  You can specify the total number
@@ -251,7 +267,6 @@ class ExponentialRetry(Retry):
             maximum_delay: The maximum delay between attempts.
         """
         super().__init__(attempts=attempts, delay=minimum_delay)
-        self.minimum_delay = minimum_delay
         self.maximum_delay = maximum_delay
 
     async def __aenter__(self) -> "ExponentialRetry":
@@ -259,14 +274,14 @@ class ExponentialRetry(Retry):
 
         retry = ExponentialRetry(
             attempts=self.attempts,
-            minimum_delay=self.minimum_delay,
+            minimum_delay=self.delay,
             maximum_delay=self.maximum_delay,
         )
         retry.attempt = execution.attempt
 
         if execution.attempt > 1:
             backoff_factor = 2 ** (execution.attempt - 1)
-            calculated_delay = self.minimum_delay * backoff_factor
+            calculated_delay = self.delay * backoff_factor
 
             if calculated_delay > self.maximum_delay:
                 retry.delay = self.maximum_delay

{pydocket-0.6.4 → pydocket-0.7.0}/tests/test_dependencies.py

@@ -1,9 +1,10 @@
 import logging
+from datetime import datetime, timedelta, timezone
 
 import pytest
 
 from docket import CurrentDocket, CurrentWorker, Docket, Worker
-from docket.dependencies import Depends, Retry, TaskArgument
+from docket.dependencies import Depends, ExponentialRetry, Retry, TaskArgument
 
 
 async def test_dependencies_may_be_duplicated(docket: Docket, worker: Worker):
@@ -95,6 +96,127 @@ async def test_user_provide_retries_are_used(docket: Docket, worker: Worker):
     assert calls == 2
 
 
+@pytest.mark.parametrize("retry_cls", [Retry, ExponentialRetry])
+async def test_user_can_request_a_retry_in_timedelta_time(
+    retry_cls: Retry, docket: Docket, worker: Worker
+):
+    calls = 0
+    first_call_time = None
+    second_call_time = None
+
+    async def the_task(
+        a: str,
+        b: str,
+        retry: Retry = retry_cls(attempts=2),  # type: ignore[reportCallIssue]
+    ):
+        assert a == "a"
+        assert b == "b"
+
+        nonlocal calls
+        calls += 1
+
+        nonlocal first_call_time
+        if not first_call_time:
+            first_call_time = datetime.now(timezone.utc)
+            retry.in_(timedelta(seconds=0.5))
+        else:
+            nonlocal second_call_time
+            second_call_time = datetime.now(timezone.utc)
+
+    await docket.add(the_task)("a", "b")
+
+    await worker.run_until_finished()
+
+    assert calls == 2
+
+    assert isinstance(first_call_time, datetime)
+    assert isinstance(second_call_time, datetime)
+
+    delay = second_call_time - first_call_time
+    assert delay.total_seconds() > 0 < 1
+
+
+@pytest.mark.parametrize("retry_cls", [Retry, ExponentialRetry])
+async def test_user_can_request_a_retry_at_a_specific_time(
+    retry_cls: Retry, docket: Docket, worker: Worker
+):
+    calls = 0
+    first_call_time = None
+    second_call_time = None
+
+    async def the_task(
+        a: str,
+        b: str,
+        retry: Retry = retry_cls(attempts=2),  # type: ignore[reportCallIssue]
+    ):
+        assert a == "a"
+        assert b == "b"
+
+        nonlocal calls
+        calls += 1
+
+        nonlocal first_call_time
+        if not first_call_time:
+            when = datetime.now(timezone.utc) + timedelta(seconds=0.5)
+            first_call_time = datetime.now(timezone.utc)
+            retry.at(when)
+        else:
+            nonlocal second_call_time
+            second_call_time = datetime.now(timezone.utc)
+
+    await docket.add(the_task)("a", "b")
+
+    await worker.run_until_finished()
+
+    assert calls == 2
+
+    assert isinstance(first_call_time, datetime)
+    assert isinstance(second_call_time, datetime)
+
+    delay = second_call_time - first_call_time
+    assert delay.total_seconds() > 0 < 1
+
+
+async def test_user_can_request_a_retry_at_a_specific_time_in_the_past(
+    docket: Docket, worker: Worker
+):
+    calls = 0
+    first_call_time = None
+    second_call_time = None
+
+    async def the_task(
+        a: str,
+        b: str,
+        retry: Retry = Retry(attempts=2),
+    ):
+        assert a == "a"
+        assert b == "b"
+
+        nonlocal calls
+        calls += 1
+
+        nonlocal first_call_time
+        if not first_call_time:
+            when = datetime.now(timezone.utc) - timedelta(days=1)
+            first_call_time = datetime.now(timezone.utc)
+            retry.at(when)
+        else:
+            nonlocal second_call_time
+            second_call_time = datetime.now(timezone.utc)
+
+    await docket.add(the_task)("a", "b")
+
+    await worker.run_until_finished()
+
+    assert calls == 2
+
+    assert isinstance(first_call_time, datetime)
+    assert isinstance(second_call_time, datetime)
+
+    delay = second_call_time - first_call_time
+    assert delay.total_seconds() > 0 < 1
+
+
 async def test_dependencies_error_for_missing_task_argument(
     docket: Docket, worker: Worker, caplog: pytest.LogCaptureFixture
 ):