pydocket 0.0.1__tar.gz → 0.0.2__tar.gz

pydocket-0.0.2/.github/workflows/chaos.yml

@@ -0,0 +1,30 @@
+name: Docket Chaos Tests
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  workflow_call:
+
+jobs:
+  test:
+    name: Chaos tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 2
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv and set Python version
+        uses: astral-sh/setup-uv@v5
+        with:
+          python-version: 3.12
+          enable-cache: true
+          cache-dependency-glob: "pyproject.toml"
+
+      - name: Install dependencies
+        run: uv sync --dev
+
+      - name: Run chaos tests
+        run: python -m chaos.driver

{pydocket-0.0.1 → pydocket-0.0.2}/.github/workflows/ci.yml

@@ -15,16 +15,6 @@ jobs:
       matrix:
         python-version: ["3.12", "3.13"]
 
-    services:
-      redis:
-        image: redis
-        ports:
-          - 6379:6379
-        options: >-
-          --health-cmd "redis-cli ping"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
 
     steps:
       - uses: actions/checkout@v4

{pydocket-0.0.1 → pydocket-0.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pydocket
-Version: 0.0.1
+Version: 0.0.2
 Summary: A distributed background task system for Python functions
 Project-URL: Homepage, https://github.com/chrisguidry/docket
 Project-URL: Bug Tracker, https://github.com/chrisguidry/docket/issues
@@ -23,7 +23,12 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Python: >=3.12
 Requires-Dist: cloudpickle>=3.1.1
+Requires-Dist: opentelemetry-api>=1.30.0
+Requires-Dist: opentelemetry-exporter-prometheus>=0.51b0
+Requires-Dist: prometheus-client>=0.21.1
+Requires-Dist: python-json-logger>=3.2.1
 Requires-Dist: redis>=5.2.1
+Requires-Dist: rich>=13.9.4
 Requires-Dist: typer>=0.15.1
 Description-Content-Type: text/markdown
 

pydocket-0.0.2/chaos/README.md

@@ -0,0 +1,6 @@
+This directory includes a chaos and load testing suite that can verify `docket` under
+more real-world conditions.
+
+This test suite should have a mode that can run in under a minute or two for use in CI,
+but longer tests for performance are totally fine. Everything should be self-contained,
+reproducible, and verifiable.

pydocket-0.0.2/chaos/driver.py

@@ -0,0 +1,208 @@
+import asyncio
+import logging
+import os
+import random
+import sys
+from asyncio import subprocess
+from asyncio.subprocess import Process
+from datetime import timedelta
+from typing import Any, Literal, Sequence
+from uuid import uuid4
+
+import redis.exceptions
+from opentelemetry import trace
+from testcontainers.redis import RedisContainer
+
+from docket import Docket
+
+from .tasks import toxic
+
+logging.getLogger().setLevel(logging.INFO)
+
+# Quiets down the testcontainers logger
+testcontainers_logger = logging.getLogger("testcontainers.core.container")
+testcontainers_logger.setLevel(logging.ERROR)
+testcontainers_logger = logging.getLogger("testcontainers.core.waiting_utils")
+testcontainers_logger.setLevel(logging.ERROR)
+
+console = logging.StreamHandler(stream=sys.stdout)
+console.setFormatter(
+    logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+)
+logging.getLogger().addHandler(console)
+
+
+logger = logging.getLogger("chaos.driver")
+tracer = trace.get_tracer("chaos.driver")
+
+
+def python_entrypoint() -> list[str]:
+    if os.environ.get("OTEL_DISTRO"):
+        return ["opentelemetry-instrument", sys.executable]
+    return [sys.executable]
+
+
+async def main(
+    mode: Literal["performance", "chaos", "hard"] = "chaos",
+    tasks: int = 2000,
+    producers: int = 2,
+    workers: int = 4,
+):
+    with RedisContainer("redis:7.4.2") as redis_server:
+        redis_url = f"redis://{redis_server.get_container_host_ip()}:{redis_server.get_exposed_port(6379)}"
+        docket = Docket(
+            name=f"test-docket-{uuid4()}",
+            url=redis_url,
+        )
+        environment = {
+            **os.environ,
+            "DOCKET_NAME": docket.name,
+            "DOCKET_URL": redis_url,
+        }
+
+        if tasks % producers != 0:
+            raise ValueError("total_tasks must be divisible by total_producers")
+
+        tasks_per_producer = tasks // producers
+
+        logger.info(
+            "Spawning %d producers with %d tasks each...", producers, tasks_per_producer
+        )
+
+        async def spawn_producer() -> Process:
+            return await asyncio.create_subprocess_exec(
+                *python_entrypoint(),
+                "-m",
+                "chaos.producer",
+                str(tasks_per_producer),
+                env=environment | {"OTEL_SERVICE_NAME": "chaos-producer"},
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+            )
+
+        producer_processes: list[Process] = []
+        for _ in range(producers):
+            producer_processes.append(await spawn_producer())
+
+        logger.info("Spawning %d workers...", workers)
+
+        async def spawn_worker() -> Process:
+            return await asyncio.create_subprocess_exec(
+                *python_entrypoint(),
+                "-m",
+                "docket",
+                "worker",
+                "--docket",
+                docket.name,
+                "--url",
+                redis_url,
+                "--tasks",
+                "chaos.tasks:chaos_tasks",
+                env=environment
+                | {
+                    "OTEL_SERVICE_NAME": "chaos-worker",
+                    "DOCKET_WORKER_REDELIVERY_TIMEOUT": "5s",
+                },
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+            )
+
+        worker_processes: list[Process] = []
+        for _ in range(workers):
+            worker_processes.append(await spawn_worker())
+
+        while True:
+            try:
+                async with docket.redis() as r:
+                    info: dict[str, Any] = await r.info()
+                    connected_clients = int(info.get("connected_clients", 0))
+
+                    sent_tasks = await r.zcard("hello:sent")
+                    received_tasks = await r.zcard("hello:received")
+
+                    logger.info(
+                        "sent: %d, received: %d, clients: %d",
+                        sent_tasks,
+                        received_tasks,
+                        connected_clients,
+                    )
+                    if sent_tasks >= tasks:
+                        break
+            except redis.exceptions.ConnectionError as e:
+                logger.error(
+                    "driver: Redis connection error (%s), retrying in 5s...", e
+                )
+                await asyncio.sleep(5)
+
+            # Now apply some chaos to the system:
+
+            if mode in ("chaos", "hard"):
+                chaos_chance = random.random()
+                if chaos_chance < 0.01 and mode == "hard":
+                    logger.warning("CHAOS: Killing redis server...")
+                    redis_server.stop()
+
+                    await asyncio.sleep(5)
+
+                    logger.warning("CHAOS: Starting redis server...")
+                    while True:
+                        try:
+                            redis_server.start()
+                            break
+                        except Exception:
+                            logger.warning("  Redis server failed, retrying in 5s...")
+                            await asyncio.sleep(5)
+
+                elif chaos_chance < 0.10:
+                    worker_index = random.randrange(len(worker_processes))
+                    worker_to_kill = worker_processes.pop(worker_index)
+
+                    logger.warning("CHAOS: Killing worker %d...", worker_index)
+                    try:
+                        worker_to_kill.terminate()
+                    except ProcessLookupError:
+                        logger.warning("  What is dead may never die!")
+
+                    logger.warning("CHAOS: Replacing worker %d...", worker_index)
+                    worker_processes.append(await spawn_worker())
+                elif chaos_chance < 0.15:
+                    logger.warning("CHAOS: Queuing a toxic task...")
+                    try:
+                        async with docket:
+                            await docket.add(toxic)()
+                    except redis.exceptions.ConnectionError:
+                        pass
+
+            await asyncio.sleep(0.25)
+
+        async with docket.redis() as r:
+            first_entries: Sequence[tuple[bytes, float]] = await r.zrange(
+                "hello:received", 0, 0, withscores=True
+            )
+            last_entries: Sequence[tuple[bytes, float]] = await r.zrange(
+                "hello:received", -1, -1, withscores=True
+            )
+
+            _, min_score = first_entries[0]
+            _, max_score = last_entries[0]
+            total_time = timedelta(seconds=max_score - min_score)
+
+            logger.info(
+                "Processed %d tasks in %s, averaging %.2f/s",
+                tasks,
+                total_time,
+                tasks / total_time.total_seconds(),
+            )
+
+        for process in producer_processes + worker_processes:
+            try:
+                process.kill()
+            except ProcessLookupError:
+                continue
+            await process.wait()
+
+
+if __name__ == "__main__":
+    mode = sys.argv[1] if len(sys.argv) > 1 else "chaos"
+    assert mode in ("performance", "chaos", "hard")
+    asyncio.run(main(mode=mode))

pydocket-0.0.2/chaos/producer.py

@@ -0,0 +1,42 @@
+import asyncio
+import logging
+import os
+import sys
+import time
+
+import redis.exceptions
+
+from docket import Docket
+
+from .tasks import hello
+
+logging.getLogger().setLevel(logging.INFO)
+logger = logging.getLogger("chaos.producer")
+
+
+async def main(tasks_to_produce: int):
+    docket = Docket(
+        name=os.environ["DOCKET_NAME"],
+        url=os.environ["DOCKET_URL"],
+    )
+    tasks_sent = 0
+    while tasks_sent < tasks_to_produce:
+        try:
+            async with docket:
+                async with docket.redis() as r:
+                    for _ in range(tasks_sent, tasks_to_produce):
+                        execution = await docket.add(hello)()
+                        await r.zadd("hello:sent", {execution.key: time.time()})
+                        logger.info("Added task %s", execution.key)
+                        tasks_sent += 1
+        except redis.exceptions.ConnectionError:
+            logger.warning(
+                "producer: Redis connection error, retrying in 5s... "
+                f"({tasks_sent}/{tasks_to_produce} tasks sent)"
+            )
+            await asyncio.sleep(5)
+
+
+if __name__ == "__main__":
+    tasks = int(sys.argv[1])
+    asyncio.run(main(tasks))

pydocket-0.0.2/chaos/run

@@ -0,0 +1,8 @@
+#!/bin/bash
+export OTEL_SERVICE_NAME=chaos-driver
+export OTEL_DISTRO=otlp
+export OTEL_EXPORTER_OTLP_ENDPOINT=0.0.0.0:4317
+export OTEL_EXPORTER_OTLP_INSECURE=true
+export OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED=true
+
+opentelemetry-instrument python -m chaos.driver "$@"

pydocket-0.0.2/chaos/tasks.py

@@ -0,0 +1,25 @@
+import logging
+import sys
+import time
+
+from docket import CurrentDocket, Docket, Retry, TaskKey
+
+logger = logging.getLogger(__name__)
+
+
+async def hello(
+    key: str = TaskKey(),
+    docket: Docket = CurrentDocket(),
+    retry: Retry = Retry(attempts=sys.maxsize),
+):
+    logger.info("Starting task %s", key)
+    async with docket.redis() as redis:
+        await redis.zadd("hello:received", {key: time.time()})
+    logger.info("Finished task %s", key)
+
+
+async def toxic():
+    sys.exit(42)
+
+
+chaos_tasks = [hello, toxic]

{pydocket-0.0.1 → pydocket-0.0.2}/pyproject.toml

@@ -18,12 +18,27 @@ classifiers = [
     "License :: OSI Approved :: MIT License",
     "Operating System :: OS Independent",
 ]
-dependencies = ["cloudpickle>=3.1.1", "redis>=5.2.1", "typer>=0.15.1"]
+dependencies = [
+    "cloudpickle>=3.1.1",
+    "opentelemetry-api>=1.30.0",
+    "opentelemetry-exporter-prometheus>=0.51b0",
+    "prometheus-client>=0.21.1",
+    "python-json-logger>=3.2.1",
+    "redis>=5.2.1",
+    "rich>=13.9.4",
+    "typer>=0.15.1",
+]
 
 [dependency-groups]
 dev = [
     "codespell>=2.4.1",
     "mypy>=1.14.1",
+    "opentelemetry-distro>=0.51b0",
+    "opentelemetry-exporter-otlp>=1.30.0",
+    "opentelemetry-instrumentation>=0.51b0",
+    "opentelemetry-instrumentation-logging>=0.51b0",
+    "opentelemetry-instrumentation-redis>=0.51b0",
+    "opentelemetry-sdk>=1.30.0",
     "pre-commit>=4.1.0",
     "pyright>=1.1.394",
     "pytest>=8.3.4",

{pydocket-0.0.1 → pydocket-0.0.2}/src/docket/__init__.py

@@ -8,7 +8,16 @@ from importlib.metadata import version
 
 __version__ = version("pydocket")
 
-from .dependencies import CurrentDocket, CurrentWorker, Retry
+from .annotations import Logged
+from .dependencies import (
+    CurrentDocket,
+    CurrentExecution,
+    CurrentWorker,
+    ExponentialRetry,
+    Retry,
+    TaskKey,
+    TaskLogger,
+)
 from .docket import Docket
 from .execution import Execution
 from .worker import Worker
@@ -19,6 +28,11 @@ __all__ = [
     "Execution",
     "CurrentDocket",
     "CurrentWorker",
+    "CurrentExecution",
+    "TaskKey",
+    "TaskLogger",
     "Retry",
+    "ExponentialRetry",
+    "Logged",
     "__version__",
 ]

pydocket-0.0.2/src/docket/annotations.py

@@ -0,0 +1,30 @@
+import abc
+import inspect
+from typing import Any, Iterable, Mapping, Self
+
+
+class Annotation(abc.ABC):
+    @classmethod
+    def annotated_parameters(cls, signature: inspect.Signature) -> Mapping[str, Self]:
+        annotated: dict[str, Self] = {}
+
+        for param_name, param in signature.parameters.items():
+            if param.annotation == inspect.Parameter.empty:
+                continue
+
+            try:
+                metadata: Iterable[Any] = param.annotation.__metadata__
+            except AttributeError:
+                continue
+
+            for arg_type in metadata:
+                if isinstance(arg_type, cls):
+                    annotated[param_name] = arg_type
+                elif isinstance(arg_type, type) and issubclass(arg_type, cls):
+                    annotated[param_name] = arg_type()
+
+        return annotated
+
+
+class Logged(Annotation):
+    """Instructs docket to include arguments to this parameter in the log."""