pydocket 0.2.0__tar.gz → 0.2.1__tar.gz

{pydocket-0.2.0 → pydocket-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pydocket
-Version: 0.2.0
+Version: 0.2.1
 Summary: A distributed background task system for Python functions
 Project-URL: Homepage, https://github.com/chrisguidry/docket
 Project-URL: Bug Tracker, https://github.com/chrisguidry/docket/issues

{pydocket-0.2.0 → pydocket-0.2.1}/pyproject.toml

@@ -65,7 +65,14 @@ source = "vcs"
 packages = ["src/docket"]
 
 [tool.pytest.ini_options]
-addopts = "--cov=src/docket --cov=tests --cov-report=term-missing --cov-branch"
+addopts = [
+    "--numprocesses=logical",
+    "--maxprocesses=4",
+    "--cov=src/docket",
+    "--cov=tests",
+    "--cov-report=term-missing",
+    "--cov-branch",
+]
 filterwarnings = ["error"]
 
 [tool.pyright]

{pydocket-0.2.0 → pydocket-0.2.1}/src/docket/worker.py

@@ -7,6 +7,7 @@ from types import TracebackType
 from typing import (
     TYPE_CHECKING,
     Any,
+    Callable,
     Mapping,
     Protocol,
     Self,
@@ -63,6 +64,11 @@ class _stream_due_tasks(Protocol):
 class Worker:
     docket: Docket
     name: str
+    concurrency: int
+    redelivery_timeout: timedelta
+    reconnection_delay: timedelta
+    minimum_check_interval: timedelta
+    _strike_conditions: list[Callable[[Execution], bool]] = []
 
     def __init__(
         self,
@@ -80,6 +86,10 @@ class Worker:
         self.reconnection_delay = reconnection_delay
         self.minimum_check_interval = minimum_check_interval
 
+        self._strike_conditions = [
+            docket.strike_list.is_stricken,
+        ]
+
     async def __aenter__(self) -> Self:
         self._heartbeat_task = asyncio.create_task(self._heartbeat())
 
@@ -151,6 +161,35 @@ class Worker:
         """Run the worker indefinitely."""
         return await self._run(forever=True)  # pragma: no cover
 
+    async def run_at_most(self, iterations_by_key: Mapping[str, int]) -> None:
+        """
+        Run the worker until there are no more tasks to process, but limit specified
+        task keys to a maximum number of iterations.
+
+        This is particularly useful for testing self-perpetuating tasks that would
+        otherwise run indefinitely.
+
+        Args:
+            iterations_by_key: Maps task keys to their maximum allowed executions
+        """
+        execution_counts: dict[str, int] = {key: 0 for key in iterations_by_key}
+
+        def has_reached_max_iterations(execution: Execution) -> bool:
+            if execution.key not in iterations_by_key:
+                return False
+
+            if execution_counts[execution.key] >= iterations_by_key[execution.key]:
+                return True
+
+            execution_counts[execution.key] += 1
+            return False
+
+        self._strike_conditions.insert(0, has_reached_max_iterations)
+        try:
+            await self.run_until_finished()
+        finally:
+            self._strike_conditions.remove(has_reached_max_iterations)
+
     async def _run(self, forever: bool = False) -> None:
         logger.info("Starting worker %r with the following tasks:", self.name)
         for task_name, task in self.docket.tasks.items():
@@ -322,7 +361,7 @@ class Worker:
                     await process_completed_tasks()
 
     async def _execute(self, message: RedisMessage) -> None:
-        log_context: dict[str, str | float] = self._log_context()
+        log_context: Mapping[str, str | float] = self._log_context()
 
         function_name = message[b"function"].decode()
         function = self.docket.tasks.get(function_name)
@@ -334,13 +373,13 @@ class Worker:
 
         execution = Execution.from_message(function, message)
 
-        log_context |= execution.specific_labels()
+        log_context = {**log_context, **execution.specific_labels()}
         counter_labels = {**self.labels(), **execution.general_labels()}
 
         arrow = "↬" if execution.attempt > 1 else "↪"
         call = execution.call_repr()
 
-        if self.docket.strike_list.is_stricken(execution):
+        if any(condition(execution) for condition in self._strike_conditions):
             arrow = "🗙"
             logger.warning("%s %s", arrow, call, extra=log_context)
             TASKS_STRICKEN.add(1, counter_labels | {"docket.where": "worker"})
@@ -354,7 +393,7 @@ class Worker:
 
         start = datetime.now(timezone.utc)
         punctuality = start - execution.when
-        log_context["punctuality"] = punctuality.total_seconds()
+        log_context = {**log_context, "punctuality": punctuality.total_seconds()}
         duration = timedelta(0)
 
         TASKS_STARTED.add(1, counter_labels)

pydocket-0.2.1/tests/conftest.py

@@ -0,0 +1,167 @@
+import fcntl
+import os
+import socket
+import time
+from contextlib import contextmanager
+from datetime import datetime, timedelta, timezone
+from functools import partial
+from typing import AsyncGenerator, Callable, Generator, Iterable, cast
+from unittest.mock import AsyncMock
+from uuid import uuid4
+
+import pytest
+import redis.exceptions
+from docker import DockerClient
+from docker.models.containers import Container
+from redis import ConnectionPool, Redis
+
+from docket import Docket, Worker
+
+REDIS_VERSION = os.environ.get("REDIS_VERSION", "7.4")
+
+
+@pytest.fixture
+def now() -> Callable[[], datetime]:
+    return partial(datetime.now, timezone.utc)
+
+
+@contextmanager
+def _sync_redis(url: str) -> Generator[Redis, None, None]:
+    pool: ConnectionPool | None = None
+    redis = Redis.from_url(url)  # type: ignore
+    try:
+        with redis:
+            pool = redis.connection_pool  # type: ignore
+            yield redis
+    finally:
+        if pool:  # pragma: no branch
+            pool.disconnect()
+
+
+@contextmanager
+def _adminitrative_redis(port: int) -> Generator[Redis, None, None]:
+    with _sync_redis(f"redis://localhost:{port}/15") as r:
+        yield r
+
+
+def _wait_for_redis(port: int) -> None:
+    while True:
+        try:
+            with _adminitrative_redis(port) as r:
+                success = r.ping()  # type: ignore
+                if success:  # pragma: no branch
+                    return
+        except redis.exceptions.ConnectionError:  # pragma: no cover
+            time.sleep(0.1)
+
+
+@pytest.fixture(scope="session")
+def redis_server(testrun_uid: str, worker_id: str) -> Generator[Container, None, None]:
+    client = DockerClient.from_env()
+
+    container: Container | None = None
+    lock_file_name = f"/tmp/docket-unit-tests-{testrun_uid}-startup"
+
+    with open(lock_file_name, "w+") as lock_file:
+        fcntl.flock(lock_file, fcntl.LOCK_EX)
+
+        containers: Iterable[Container] = cast(
+            Iterable[Container],
+            client.containers.list(  # type: ignore
+                all=True,
+                filters={"label": "source=docket-unit-tests"},
+            ),
+        )
+        for c in containers:
+            if c.labels.get("testrun_uid") == testrun_uid:  # type: ignore
+                container = c
+            else:
+                c.remove(force=True)  # pragma: no cover
+
+        if not container:
+            with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+                s.bind(("", 0))
+                redis_port = s.getsockname()[1]
+
+            container = client.containers.run(
+                f"redis:{REDIS_VERSION}",
+                detach=True,
+                ports={"6379/tcp": redis_port},
+                labels={
+                    "source": "docket-unit-tests",
+                    "testrun_uid": testrun_uid,
+                },
+                auto_remove=True,
+            )
+
+            _wait_for_redis(redis_port)
+        else:
+            port_bindings = container.attrs["HostConfig"]["PortBindings"]["6379/tcp"]
+            redis_port = int(port_bindings[0]["HostPort"])
+
+        with _adminitrative_redis(redis_port) as r:
+            r.sadd(f"docket-unit-tests:{testrun_uid}", worker_id)
+
+    try:
+        yield container
+    finally:
+        with _adminitrative_redis(redis_port) as r:
+            with r.pipeline() as pipe:  # type: ignore
+                pipe.srem(f"docket-unit-tests:{testrun_uid}", worker_id)
+                pipe.scard(f"docket-unit-tests:{testrun_uid}")
+                count: int
+                _, count = pipe.execute()  # type: ignore
+
+        if count == 0:
+            container.stop()
+            os.remove(lock_file_name)
+
+
+@pytest.fixture
+def redis_port(redis_server: Container) -> int:
+    port_bindings = redis_server.attrs["HostConfig"]["PortBindings"]["6379/tcp"]
+    return int(port_bindings[0]["HostPort"])
+
+
+@pytest.fixture(scope="session")
+def redis_db(worker_id: str) -> int:
+    if not worker_id or "gw" not in worker_id:
+        return 0  # pragma: no cover
+    else:
+        return 0 + int(worker_id.replace("gw", ""))  # pragma: no cover
+
+
+@pytest.fixture
+def redis_url(redis_port: int, redis_db: int) -> str:
+    url = f"redis://localhost:{redis_port}/{redis_db}"
+    with _sync_redis(url) as r:
+        r.flushdb()  # type: ignore
+    return url
+
+
+@pytest.fixture
+async def docket(redis_url: str, aiolib: str) -> AsyncGenerator[Docket, None]:
+    async with Docket(name=f"test-docket-{uuid4()}", url=redis_url) as docket:
+        yield docket
+
+
+@pytest.fixture
+async def worker(docket: Docket) -> AsyncGenerator[Worker, None]:
+    async with Worker(
+        docket, minimum_check_interval=timedelta(milliseconds=10)
+    ) as worker:
+        yield worker
+
+
+@pytest.fixture
+def the_task() -> AsyncMock:
+    task = AsyncMock()
+    task.__name__ = "the_task"
+    return task
+
+
+@pytest.fixture
+def another_task() -> AsyncMock:
+    task = AsyncMock()
+    task.__name__ = "another_task"
+    return task

{pydocket-0.2.0 → pydocket-0.2.1}/tests/test_fundamentals.py

@@ -609,6 +609,42 @@ async def test_self_perpetuating_scheduled_tasks(
     assert calls["second"] == [21, 22, 23]
 
 
+async def test_infinitely_self_perpetuating_tasks(
+    docket: Docket, worker: Worker, now: Callable[[], datetime]
+):
+    """docket should support testing use cases for infinitely self-perpetuating tasks"""
+
+    calls: dict[str, list[int]] = {
+        "first": [],
+        "second": [],
+        "unaffected": [],
+    }
+
+    async def the_task(start: int, iteration: int, key: str = TaskKey()):
+        calls[key].append(start + iteration)
+        soon = now() + timedelta(milliseconds=100)
+        await docket.add(the_task, key=key, when=soon)(start, iteration + 1)
+
+    async def unaffected_task(start: int, iteration: int, key: str = TaskKey()):
+        calls[key].append(start + iteration)
+        if iteration < 3:
+            await docket.add(unaffected_task, key=key)(start, iteration + 1)
+
+    await docket.add(the_task, key="first")(10, 1)
+    await docket.add(the_task, key="second")(20, 1)
+    await docket.add(unaffected_task, key="unaffected")(30, 1)
+
+    # Using worker.run_until_finished() would hang here because the task is always
+    # queueing up a future run of itself.  With worker.run_at_most(),
+    # we can specify tasks keys that will only be allowed to run a limited number of
+    # times, thus allowing the worker to exist cleanly.
+    await worker.run_at_most({"first": 4, "second": 2})
+
+    assert calls["first"] == [11, 12, 13, 14]
+    assert calls["second"] == [21, 22]
+    assert calls["unaffected"] == [31, 32, 33]
+
+
 async def test_striking_entire_tasks(
     docket: Docket, worker: Worker, the_task: AsyncMock, another_task: AsyncMock
 ):

pydocket-0.2.0/tests/conftest.py

@@ -1,117 +0,0 @@
-import os
-import socket
-import time
-from contextlib import contextmanager
-from datetime import datetime, timezone
-from functools import partial
-from typing import AsyncGenerator, Callable, Generator, Iterable, cast
-from unittest.mock import AsyncMock
-from uuid import uuid4
-
-import pytest
-import redis.exceptions
-from docker import DockerClient
-from docker.models.containers import Container
-from redis import ConnectionPool, Redis
-
-from docket import Docket, Worker
-
-REDIS_VERSION = os.environ.get("REDIS_VERSION", "7.4")
-
-
-@pytest.fixture
-def now() -> Callable[[], datetime]:
-    return partial(datetime.now, timezone.utc)
-
-
-@pytest.fixture(scope="session")
-def redis_port() -> int:
-    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
-        s.bind(("", 0))
-        return s.getsockname()[1]
-
-
-@contextmanager
-def _sync_redis(url: str) -> Generator[Redis, None, None]:
-    pool: ConnectionPool | None = None
-    redis = Redis.from_url(url)  # type: ignore
-    try:
-        with redis:
-            pool = redis.connection_pool  # type: ignore
-            yield redis
-    finally:
-        if pool:  # pragma: no branch
-            pool.disconnect()
-
-
-@pytest.fixture(scope="session")
-def redis_server(redis_port: int) -> Generator[Container, None, None]:
-    client = DockerClient.from_env()
-
-    container: Container
-
-    # Find and remove any containers from previous test runs
-    containers: Iterable[Container] = cast(
-        Iterable[Container],
-        client.containers.list(all=True, filters={"label": "source=docket-unit-tests"}),  # type: ignore
-    )
-    for container in containers:  # pragma: no cover
-        container.remove(force=True)
-
-    container = client.containers.run(
-        f"redis:{REDIS_VERSION}",
-        detach=True,
-        ports={"6379/tcp": redis_port},
-        labels={"source": "docket-unit-tests"},
-        auto_remove=True,
-    )
-
-    url = f"redis://localhost:{redis_port}/0"
-
-    while True:
-        try:
-            with _sync_redis(url) as r:
-                success = r.ping()  # type: ignore
-                if success:  # pragma: no branch
-                    break
-        except redis.exceptions.ConnectionError:  # pragma: no cover
-            time.sleep(0.1)
-
-    try:
-        yield container
-    finally:
-        container.stop()
-
-
-@pytest.fixture
-def redis_url(redis_server: Container, redis_port: int) -> str:
-    url = f"redis://localhost:{redis_port}/0"
-    with _sync_redis(url) as r:
-        r.flushdb()  # type: ignore
-    return url
-
-
-@pytest.fixture
-async def docket(redis_url: str, aiolib: str) -> AsyncGenerator[Docket, None]:
-    async with Docket(name=f"test-docket-{uuid4()}", url=redis_url) as docket:
-        yield docket
-
-
-@pytest.fixture
-async def worker(docket: Docket) -> AsyncGenerator[Worker, None]:
-    async with Worker(docket) as worker:
-        yield worker
-
-
-@pytest.fixture
-def the_task() -> AsyncMock:
-    task = AsyncMock()
-    task.__name__ = "the_task"
-    return task
-
-
-@pytest.fixture
-def another_task() -> AsyncMock:
-    task = AsyncMock()
-    task.__name__ = "another_task"
-    return task