pydocket 0.0.1__py3-none-any.whl → 0.0.2__py3-none-any.whl

docket/__init__.py

@@ -8,7 +8,16 @@ from importlib.metadata import version
 
 __version__ = version("pydocket")
 
-from .dependencies import CurrentDocket, CurrentWorker, Retry
+from .annotations import Logged
+from .dependencies import (
+    CurrentDocket,
+    CurrentExecution,
+    CurrentWorker,
+    ExponentialRetry,
+    Retry,
+    TaskKey,
+    TaskLogger,
+)
 from .docket import Docket
 from .execution import Execution
 from .worker import Worker
@@ -19,6 +28,11 @@ __all__ = [
     "Execution",
     "CurrentDocket",
     "CurrentWorker",
+    "CurrentExecution",
+    "TaskKey",
+    "TaskLogger",
     "Retry",
+    "ExponentialRetry",
+    "Logged",
     "__version__",
 ]

docket/annotations.py

@@ -0,0 +1,30 @@
+import abc
+import inspect
+from typing import Any, Iterable, Mapping, Self
+
+
+class Annotation(abc.ABC):
+    @classmethod
+    def annotated_parameters(cls, signature: inspect.Signature) -> Mapping[str, Self]:
+        annotated: dict[str, Self] = {}
+
+        for param_name, param in signature.parameters.items():
+            if param.annotation == inspect.Parameter.empty:
+                continue
+
+            try:
+                metadata: Iterable[Any] = param.annotation.__metadata__
+            except AttributeError:
+                continue
+
+            for arg_type in metadata:
+                if isinstance(arg_type, cls):
+                    annotated[param_name] = arg_type
+                elif isinstance(arg_type, type) and issubclass(arg_type, cls):
+                    annotated[param_name] = arg_type()
+
+        return annotated
+
+
+class Logged(Annotation):
+    """Instructs docket to include arguments to this parameter in the log."""

docket/cli.py

@@ -1,6 +1,16 @@
+import asyncio
+import enum
+import logging
+import socket
+import sys
+from datetime import timedelta
+from typing import Annotated
+
 import typer
 
-from docket import __version__
+from . import __version__, tasks
+from .docket import Docket
+from .worker import Worker
 
 app: typer.Typer = typer.Typer(
     help="Docket - A distributed background task system for Python functions",
@@ -9,11 +19,232 @@ app: typer.Typer = typer.Typer(
 )
 
 
+class LogLevel(enum.StrEnum):
+    DEBUG = "DEBUG"
+    INFO = "INFO"
+    WARNING = "WARNING"
+    ERROR = "ERROR"
+    CRITICAL = "CRITICAL"
+
+
+class LogFormat(enum.StrEnum):
+    RICH = "rich"
+    PLAIN = "plain"
+    JSON = "json"
+
+
+def duration(duration_str: str | timedelta) -> timedelta:
+    """
+    Parse a duration string into a timedelta.
+
+    Supported formats:
+    - 123 = 123 seconds
+    - 123s = 123 seconds
+    - 123m = 123 minutes
+    - 123h = 123 hours
+    - 00:00 = mm:ss
+    - 00:00:00 = hh:mm:ss
+    """
+    if isinstance(duration_str, timedelta):
+        return duration_str
+
+    if ":" in duration_str:
+        parts = duration_str.split(":")
+        if len(parts) == 2:  # mm:ss
+            minutes, seconds = map(int, parts)
+            return timedelta(minutes=minutes, seconds=seconds)
+        elif len(parts) == 3:  # hh:mm:ss
+            hours, minutes, seconds = map(int, parts)
+            return timedelta(hours=hours, minutes=minutes, seconds=seconds)
+        else:
+            raise ValueError(f"Invalid duration string: {duration_str}")
+    elif duration_str.endswith("s"):
+        return timedelta(seconds=int(duration_str[:-1]))
+    elif duration_str.endswith("m"):
+        return timedelta(minutes=int(duration_str[:-1]))
+    elif duration_str.endswith("h"):
+        return timedelta(hours=int(duration_str[:-1]))
+    else:
+        return timedelta(seconds=int(duration_str))
+
+
+def set_logging_format(format: LogFormat) -> None:
+    root_logger = logging.getLogger()
+    if format == LogFormat.JSON:
+        from pythonjsonlogger.json import JsonFormatter
+
+        formatter = JsonFormatter(
+            "{name}{asctime}{levelname}{message}{exc_info}", style="{"
+        )
+        handler = logging.StreamHandler(stream=sys.stdout)
+        handler.setFormatter(formatter)
+        root_logger.addHandler(handler)
+    elif format == LogFormat.PLAIN:
+        handler = logging.StreamHandler(stream=sys.stdout)
+        formatter = logging.Formatter(
+            "[%(asctime)s] %(levelname)s - %(name)s - %(message)s",
+            datefmt="%Y-%m-%d %H:%M:%S",
+        )
+        handler.setFormatter(formatter)
+        root_logger.addHandler(handler)
+    else:
+        from rich.logging import RichHandler
+
+        handler = RichHandler()
+        formatter = logging.Formatter("%(message)s", datefmt="[%X]")
+        handler.setFormatter(formatter)
+        root_logger.addHandler(handler)
+
+
+def set_logging_level(level: LogLevel) -> None:
+    logging.getLogger().setLevel(level)
+
+
 @app.command(
     help="Start a worker to process tasks",
 )
-def worker() -> None:
-    print("TODO: start the worker")
+def worker(
+    tasks: Annotated[
+        list[str],
+        typer.Option(
+            "--tasks",
+            help=(
+                "The dotted path of a task collection to register with the docket. "
+                "This can be specified multiple times.  A task collection is any "
+                "iterable of async functions."
+            ),
+        ),
+    ] = ["docket.tasks:standard_tasks"],
+    docket_: Annotated[
+        str,
+        typer.Option(
+            "--docket",
+            help="The name of the docket",
+            envvar="DOCKET_NAME",
+        ),
+    ] = "docket",
+    url: Annotated[
+        str,
+        typer.Option(
+            help="The URL of the Redis server",
+            envvar="DOCKET_URL",
+        ),
+    ] = "redis://localhost:6379/0",
+    name: Annotated[
+        str | None,
+        typer.Option(
+            help="The name of the worker",
+            envvar="DOCKET_WORKER_NAME",
+        ),
+    ] = socket.gethostname(),
+    logging_level: Annotated[
+        LogLevel,
+        typer.Option(
+            help="The logging level",
+            envvar="DOCKET_LOGGING_LEVEL",
+            callback=set_logging_level,
+        ),
+    ] = LogLevel.INFO,
+    logging_format: Annotated[
+        LogFormat,
+        typer.Option(
+            help="The logging format",
+            envvar="DOCKET_LOGGING_FORMAT",
+            callback=set_logging_format,
+        ),
+    ] = LogFormat.RICH if sys.stdout.isatty() else LogFormat.PLAIN,
+    prefetch_count: Annotated[
+        int,
+        typer.Option(
+            help="The number of tasks to request from the docket at a time",
+            envvar="DOCKET_WORKER_PREFETCH_COUNT",
+        ),
+    ] = 10,
+    redelivery_timeout: Annotated[
+        timedelta,
+        typer.Option(
+            parser=duration,
+            help="How long to wait before redelivering a task to another worker",
+            envvar="DOCKET_WORKER_REDELIVERY_TIMEOUT",
+        ),
+    ] = timedelta(minutes=5),
+    reconnection_delay: Annotated[
+        timedelta,
+        typer.Option(
+            parser=duration,
+            help=(
+                "How long to wait before reconnecting to the Redis server after "
+                "a connection error"
+            ),
+            envvar="DOCKET_WORKER_RECONNECTION_DELAY",
+        ),
+    ] = timedelta(seconds=5),
+    until_finished: Annotated[
+        bool,
+        typer.Option(
+            "--until-finished",
+            help="Exit after the current docket is finished",
+        ),
+    ] = False,
+) -> None:
+    asyncio.run(
+        Worker.run(
+            docket_name=docket_,
+            url=url,
+            name=name,
+            prefetch_count=prefetch_count,
+            redelivery_timeout=redelivery_timeout,
+            reconnection_delay=reconnection_delay,
+            until_finished=until_finished,
+            tasks=tasks,
+        )
+    )
+
+
+@app.command(help="Adds a trace task to the Docket")
+def trace(
+    docket_: Annotated[
+        str,
+        typer.Option(
+            "--docket",
+            help="The name of the docket",
+            envvar="DOCKET_NAME",
+        ),
+    ] = "docket",
+    url: Annotated[
+        str,
+        typer.Option(
+            help="The URL of the Redis server",
+            envvar="DOCKET_URL",
+        ),
+    ] = "redis://localhost:6379/0",
+    message: Annotated[
+        str,
+        typer.Argument(
+            help="The message to print",
+        ),
+    ] = "Howdy!",
+    error: Annotated[
+        bool,
+        typer.Option(
+            "--error",
+            help="Intentionally raise an error",
+        ),
+    ] = False,
+) -> None:
+    async def run() -> None:
+        async with Docket(name=docket_, url=url) as docket:
+            if error:
+                execution = await docket.add(tasks.fail)(message)
+            else:
+                execution = await docket.add(tasks.trace)(message)
+
+            print(
+                f"Added {execution.function.__name__} task {execution.key!r} to "
+                f"the docket {docket.name!r}"
+            )
+
+    asyncio.run(run())
 
 
 @app.command(

docket/dependencies.py

@@ -1,5 +1,6 @@
 import abc
 import inspect
+import logging
 from datetime import timedelta
 from typing import Any, Awaitable, Callable, Counter, cast
 
@@ -35,10 +36,52 @@ def CurrentDocket() -> Docket:
     return cast(Docket, _CurrentDocket())
 
 
+class _CurrentExecution(Dependency):
+    def __call__(
+        self, docket: Docket, worker: Worker, execution: Execution
+    ) -> Execution:
+        return execution
+
+
+def CurrentExecution() -> Execution:
+    return cast(Execution, _CurrentExecution())
+
+
+class _TaskKey(Dependency):
+    def __call__(self, docket: Docket, worker: Worker, execution: Execution) -> str:
+        return execution.key
+
+
+def TaskKey() -> str:
+    return cast(str, _TaskKey())
+
+
+class _TaskLogger(Dependency):
+    def __call__(
+        self, docket: Docket, worker: Worker, execution: Execution
+    ) -> logging.LoggerAdapter:
+        logger = logging.getLogger(f"docket.task.{execution.function.__name__}")
+
+        extra = {
+            "execution.key": execution.key,
+            "execution.attempt": execution.attempt,
+            "worker.name": worker.name,
+            "docket.name": docket.name,
+        }
+
+        return logging.LoggerAdapter(logger, extra)
+
+
+def TaskLogger() -> logging.LoggerAdapter[logging.Logger]:
+    return cast(logging.LoggerAdapter[logging.Logger], _TaskLogger())
+
+
 class Retry(Dependency):
     single: bool = True
 
-    def __init__(self, attempts: int = 1, delay: timedelta = timedelta(0)) -> None:
+    def __init__(
+        self, attempts: int | None = 1, delay: timedelta = timedelta(0)
+    ) -> None:
         self.attempts = attempts
         self.delay = delay
         self.attempt = 1
@@ -49,6 +92,41 @@ class Retry(Dependency):
         return retry
 
 
+class ExponentialRetry(Retry):
+    attempts: int
+
+    def __init__(
+        self,
+        attempts: int = 1,
+        minimum_delay: timedelta = timedelta(seconds=1),
+        maximum_delay: timedelta = timedelta(seconds=64),
+    ) -> None:
+        super().__init__(attempts=attempts, delay=minimum_delay)
+        self.minimum_delay = minimum_delay
+        self.maximum_delay = maximum_delay
+
+    def __call__(
+        self, docket: Docket, worker: Worker, execution: Execution
+    ) -> "ExponentialRetry":
+        retry = ExponentialRetry(
+            attempts=self.attempts,
+            minimum_delay=self.minimum_delay,
+            maximum_delay=self.maximum_delay,
+        )
+        retry.attempt = execution.attempt
+
+        if execution.attempt > 1:
+            backoff_factor = 2 ** (execution.attempt - 1)
+            calculated_delay = self.minimum_delay * backoff_factor
+
+            if calculated_delay > self.maximum_delay:
+                retry.delay = self.maximum_delay
+            else:
+                retry.delay = calculated_delay
+
+        return retry
+
+
 def get_dependency_parameters(
     function: Callable[..., Awaitable[Any]],
 ) -> dict[str, Dependency]:

docket/docket.py

@@ -1,3 +1,4 @@
+import importlib
 from contextlib import asynccontextmanager
 from datetime import datetime, timezone
 from types import TracebackType
@@ -6,6 +7,7 @@ from typing import (
     AsyncGenerator,
     Awaitable,
     Callable,
+    Iterable,
     ParamSpec,
     Self,
     TypeVar,
@@ -13,13 +15,26 @@ from typing import (
 )
 from uuid import uuid4
 
+from opentelemetry import propagate, trace
 from redis.asyncio import Redis
 
 from .execution import Execution
+from .instrumentation import (
+    TASKS_ADDED,
+    TASKS_CANCELLED,
+    TASKS_REPLACED,
+    TASKS_SCHEDULED,
+    message_setter,
+)
+
+tracer: trace.Tracer = trace.get_tracer(__name__)
+
 
 P = ParamSpec("P")
 R = TypeVar("R")
 
+TaskCollection = Iterable[Callable[..., Awaitable[Any]]]
+
 
 class Docket:
     tasks: dict[str, Callable[..., Awaitable[Any]]]
@@ -27,19 +42,26 @@ class Docket:
     def __init__(
         self,
         name: str = "docket",
-        host: str = "localhost",
-        port: int = 6379,
-        db: int = 0,
-        password: str | None = None,
+        url: str = "redis://localhost:6379/0",
     ) -> None:
+        """
+        Args:
+            name: The name of the docket.
+            url: The URL of the Redis server.  For example:
+                - "redis://localhost:6379/0"
+                - "redis://user:password@localhost:6379/0"
+                - "redis://user:password@localhost:6379/0?ssl=true"
+                - "rediss://localhost:6379/0"
+                - "unix:///path/to/redis.sock"
+        """
         self.name = name
-        self.host = host
-        self.port = port
-        self.db = db
-        self.password = password
+        self.url = url
 
     async def __aenter__(self) -> Self:
-        self.tasks = {}
+        from .tasks import standard_tasks
+
+        self.tasks = {fn.__name__: fn for fn in standard_tasks}
+
         return self
 
     async def __aexit__(
@@ -52,13 +74,7 @@ class Docket:
 
     @asynccontextmanager
     async def redis(self) -> AsyncGenerator[Redis, None]:
-        async with Redis(
-            host=self.host,
-            port=self.port,
-            db=self.db,
-            password=self.password,
-            single_connection_client=True,
-        ) as redis:
+        async with Redis.from_url(self.url) as redis:
             yield redis
 
     def register(self, function: Callable[..., Awaitable[Any]]) -> None:
@@ -68,6 +84,19 @@ class Docket:
 
         self.tasks[function.__name__] = function
 
+    def register_collection(self, collection_path: str) -> None:
+        """
+        Register a collection of tasks.
+
+        Args:
+            collection_path: A path in the format "module:collection".
+        """
+        module_name, _, member_name = collection_path.rpartition(":")
+        module = importlib.import_module(module_name)
+        collection = getattr(module, member_name)
+        for function in collection:
+            self.register(function)
+
     @overload
     def add(
         self,
@@ -104,6 +133,9 @@ class Docket:
         async def scheduler(*args: P.args, **kwargs: P.kwargs) -> Execution:
             execution = Execution(function, args, kwargs, when, key, attempt=1)
             await self.schedule(execution)
+
+            TASKS_ADDED.add(1, {"docket": self.name, "task": function.__name__})
+
             return execution
 
         return scheduler
@@ -137,6 +169,9 @@ class Docket:
             execution = Execution(function, args, kwargs, when, key, attempt=1)
             await self.cancel(key)
             await self.schedule(execution)
+
+            TASKS_REPLACED.add(1, {"docket": self.name, "task": function.__name__})
+
             return execution
 
         return scheduler
@@ -154,25 +189,50 @@ class Docket:
 
     async def schedule(self, execution: Execution) -> None:
         message: dict[bytes, bytes] = execution.as_message()
-        key = execution.key
-        when = execution.when
+        propagate.inject(message, setter=message_setter)
+
+        with tracer.start_as_current_span(
+            "docket.schedule",
+            attributes={
+                "docket.name": self.name,
+                "docket.execution.when": execution.when.isoformat(),
+                "docket.execution.key": execution.key,
+                "docket.execution.attempt": execution.attempt,
+                "code.function.name": execution.function.__name__,
+            },
+        ):
+            key = execution.key
+            when = execution.when
+
+            async with self.redis() as redis:
+                # if the task is already in the queue, retain it
+                if await redis.zscore(self.queue_key, key) is not None:
+                    return
+
+                if when <= datetime.now(timezone.utc):
+                    await redis.xadd(self.stream_key, message)
+                else:
+                    async with redis.pipeline() as pipe:
+                        pipe.hset(self.parked_task_key(key), mapping=message)
+                        pipe.zadd(self.queue_key, {key: when.timestamp()})
+                        await pipe.execute()
+
+        TASKS_SCHEDULED.add(
+            1, {"docket": self.name, "task": execution.function.__name__}
+        )
 
-        async with self.redis() as redis:
-            # if the task is already in the queue, retain it
-            if await redis.zscore(self.queue_key, key) is not None:
-                return
-
-            if when <= datetime.now(timezone.utc):
-                await redis.xadd(self.stream_key, message)
-            else:
+    async def cancel(self, key: str) -> None:
+        with tracer.start_as_current_span(
+            "docket.cancel",
+            attributes={
+                "docket.name": self.name,
+                "docket.execution.key": key,
+            },
+        ):
+            async with self.redis() as redis:
                 async with redis.pipeline() as pipe:
-                    pipe.hset(self.parked_task_key(key), mapping=message)
-                    pipe.zadd(self.queue_key, {key: when.timestamp()})
+                    pipe.delete(self.parked_task_key(key))
+                    pipe.zrem(self.queue_key, key)
                     await pipe.execute()
 
-    async def cancel(self, key: str) -> None:
-        async with self.redis() as redis:
-            async with redis.pipeline() as pipe:
-                pipe.delete(self.parked_task_key(key))
-                pipe.zrem(self.queue_key, key)
-                await pipe.execute()
+        TASKS_CANCELLED.add(1, {"docket": self.name})

docket/execution.py

@@ -1,8 +1,11 @@
+import inspect
 from datetime import datetime
 from typing import Any, Awaitable, Callable, Self
 
 import cloudpickle
 
+from docket.annotations import Logged
+
 Message = dict[bytes, bytes]
 
 
@@ -45,3 +48,27 @@ class Execution:
             key=message[b"key"].decode(),
             attempt=int(message[b"attempt"].decode()),
         )
+
+    def call_repr(self) -> str:
+        arguments: list[str] = []
+        signature = inspect.signature(self.function)
+        function_name = self.function.__name__
+
+        logged_parameters = Logged.annotated_parameters(signature)
+
+        parameter_names = list(signature.parameters.keys())
+
+        for i, argument in enumerate(self.args[: len(parameter_names)]):
+            parameter_name = parameter_names[i]
+            if parameter_name in logged_parameters:
+                arguments.append(repr(argument))
+            else:
+                arguments.append("...")
+
+        for parameter_name, argument in self.kwargs.items():
+            if parameter_name in logged_parameters:
+                arguments.append(f"{parameter_name}={repr(argument)}")
+            else:
+                arguments.append(f"{parameter_name}=...")
+
+        return f"{function_name}({', '.join(arguments)}){{{self.key}}}"

docket/instrumentation.py

@@ -0,0 +1,103 @@
+from opentelemetry import metrics
+from opentelemetry.propagators.textmap import Getter, Setter
+
+meter: metrics.Meter = metrics.get_meter("docket")
+
+TASKS_ADDED = meter.create_counter(
+    "docket_tasks_added",
+    description="How many tasks added to the docket",
+    unit="1",
+)
+
+TASKS_REPLACED = meter.create_counter(
+    "docket_tasks_replaced",
+    description="How many tasks replaced on the docket",
+    unit="1",
+)
+
+TASKS_SCHEDULED = meter.create_counter(
+    "docket_tasks_scheduled",
+    description="How many tasks added or replaced on the docket",
+    unit="1",
+)
+
+TASKS_CANCELLED = meter.create_counter(
+    "docket_tasks_cancelled",
+    description="How many tasks cancelled from the docket",
+    unit="1",
+)
+
+TASKS_STARTED = meter.create_counter(
+    "docket_tasks_started",
+    description="How many tasks started",
+    unit="1",
+)
+
+TASKS_COMPLETED = meter.create_counter(
+    "docket_tasks_completed",
+    description="How many tasks that have completed in any state",
+    unit="1",
+)
+
+TASKS_FAILED = meter.create_counter(
+    "docket_tasks_failed",
+    description="How many tasks that have failed",
+    unit="1",
+)
+
+TASKS_SUCCEEDED = meter.create_counter(
+    "docket_tasks_succeeded",
+    description="How many tasks that have succeeded",
+    unit="1",
+)
+
+TASKS_RETRIED = meter.create_counter(
+    "docket_tasks_retried",
+    description="How many tasks that have been retried",
+    unit="1",
+)
+
+TASK_DURATION = meter.create_histogram(
+    "docket_task_duration",
+    description="How long tasks take to complete",
+    unit="s",
+)
+
+TASK_PUNCTUALITY = meter.create_histogram(
+    "docket_task_punctuality",
+    description="How close a task was to its scheduled time",
+    unit="s",
+)
+
+TASKS_RUNNING = meter.create_up_down_counter(
+    "docket_tasks_running",
+    description="How many tasks that are currently running",
+    unit="1",
+)
+
+Message = dict[bytes, bytes]
+
+
+class MessageGetter(Getter[Message]):
+    def get(self, carrier: Message, key: str) -> list[str] | None:
+        val = carrier.get(key.encode(), None)
+        if val is None:
+            return None
+        return [val.decode()]
+
+    def keys(self, carrier: Message) -> list[str]:
+        return [key.decode() for key in carrier.keys()]
+
+
+class MessageSetter(Setter[Message]):
+    def set(
+        self,
+        carrier: Message,
+        key: str,
+        value: str,
+    ) -> None:
+        carrier[key.encode()] = value.encode()
+
+
+message_getter: MessageGetter = MessageGetter()
+message_setter: MessageSetter = MessageSetter()

docket/tasks.py

@@ -0,0 +1,50 @@
+import logging
+from datetime import datetime, timezone
+
+from .dependencies import CurrentDocket, CurrentExecution, CurrentWorker, Retry
+from .docket import Docket, TaskCollection
+from .execution import Execution
+from .worker import Worker
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+
+async def trace(
+    message: str,
+    docket: Docket = CurrentDocket(),
+    worker: Worker = CurrentWorker(),
+    execution: Execution = CurrentExecution(),
+) -> None:
+    logger.info(
+        "%s: %r added to docket %r %s ago now running on worker %r",
+        message,
+        execution.key,
+        docket.name,
+        (datetime.now(timezone.utc) - execution.when),
+        worker.name,
+        extra={
+            "docket.name": docket.name,
+            "worker.name": worker.name,
+            "execution.key": execution.key,
+        },
+    )
+
+
+async def fail(
+    message: str,
+    docket: Docket = CurrentDocket(),
+    worker: Worker = CurrentWorker(),
+    execution: Execution = CurrentExecution(),
+    retry: Retry = Retry(attempts=2),
+) -> None:
+    raise Exception(
+        f"{message}: {execution.key} added to docket "
+        f"{docket.name} {datetime.now(timezone.utc) - execution.when} "
+        f"ago now running on worker {worker.name}"
+    )
+
+
+standard_tasks: TaskCollection = [
+    trace,
+    fail,
+]

docket/worker.py

@@ -1,15 +1,41 @@
+import asyncio
+import inspect
 import logging
 import sys
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 from types import TracebackType
-from typing import TYPE_CHECKING, Any, Protocol, Self, Sequence, TypeVar, cast
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Protocol,
+    Self,
+    Sequence,
+    TypeVar,
+    cast,
+)
 from uuid import uuid4
 
+import redis.exceptions
+from opentelemetry import propagate, trace
+from opentelemetry.trace import Tracer
 from redis import RedisError
 
 from .docket import Docket, Execution
+from .instrumentation import (
+    TASK_DURATION,
+    TASK_PUNCTUALITY,
+    TASKS_COMPLETED,
+    TASKS_FAILED,
+    TASKS_RETRIED,
+    TASKS_RUNNING,
+    TASKS_STARTED,
+    TASKS_SUCCEEDED,
+    message_getter,
+)
 
 logger: logging.Logger = logging.getLogger(__name__)
+tracer: Tracer = trace.get_tracer(__name__)
+
 
 RedisStreamID = bytes
 RedisMessageID = bytes
@@ -30,14 +56,22 @@ class _stream_due_tasks(Protocol):
 
 
 class Worker:
-    name: str
     docket: Docket
+    name: str
 
-    prefetch_count: int = 10
-
-    def __init__(self, docket: Docket) -> None:
-        self.name = f"worker:{uuid4()}"
+    def __init__(
+        self,
+        docket: Docket,
+        name: str | None = None,
+        prefetch_count: int = 10,
+        redelivery_timeout: timedelta = timedelta(minutes=5),
+        reconnection_delay: timedelta = timedelta(seconds=5),
+    ) -> None:
         self.docket = docket
+        self.name = name or f"worker:{uuid4()}"
+        self.prefetch_count = prefetch_count
+        self.redelivery_timeout = redelivery_timeout
+        self.reconnection_delay = reconnection_delay
 
     async def __aenter__(self) -> Self:
         async with self.docket.redis() as redis:
@@ -49,7 +83,8 @@ class Worker:
                     mkstream=True,
                 )
             except RedisError as e:
-                assert "BUSYGROUP" in repr(e)
+                if "BUSYGROUP" not in repr(e):
+                    raise
 
         return self
 
@@ -72,7 +107,60 @@ class Worker:
             "stream_key": self.docket.stream_key,
         }
 
-    async def run_until_current(self) -> None:
+    @classmethod
+    async def run(
+        cls,
+        docket_name: str = "docket",
+        url: str = "redis://localhost:6379/0",
+        name: str | None = None,
+        prefetch_count: int = 10,
+        redelivery_timeout: timedelta = timedelta(minutes=5),
+        reconnection_delay: timedelta = timedelta(seconds=5),
+        until_finished: bool = False,
+        tasks: list[str] = ["docket.tasks:standard_tasks"],
+    ) -> None:
+        async with Docket(name=docket_name, url=url) as docket:
+            for task_path in tasks:
+                docket.register_collection(task_path)
+
+            async with Worker(
+                docket=docket,
+                name=name,
+                prefetch_count=prefetch_count,
+                redelivery_timeout=redelivery_timeout,
+                reconnection_delay=reconnection_delay,
+            ) as worker:
+                if until_finished:
+                    await worker.run_until_finished()
+                else:
+                    await worker.run_forever()  # pragma: no cover
+
+    async def run_until_finished(self) -> None:
+        """Run the worker until there are no more tasks to process."""
+        return await self._run(forever=False)
+
+    async def run_forever(self) -> None:
+        """Run the worker indefinitely."""
+        return await self._run(forever=True)  # pragma: no cover
+
+    async def _run(self, forever: bool = False) -> None:
+        logger.info("Starting worker %r with the following tasks:", self.name)
+        for task_name, task in self.docket.tasks.items():
+            signature = inspect.signature(task)
+            logger.info("* %s%s", task_name, signature)
+
+        while True:
+            try:
+                return await self._worker_loop(forever=forever)
+            except redis.exceptions.ConnectionError:
+                logger.warning(
+                    "Error connecting to redis, retrying in %s...",
+                    self.reconnection_delay,
+                    exc_info=True,
+                )
+                await asyncio.sleep(self.reconnection_delay.total_seconds())
+
+    async def _worker_loop(self, forever: bool = False):
         async with self.docket.redis() as redis:
             stream_due_tasks: _stream_due_tasks = cast(
                 _stream_due_tasks,
@@ -120,83 +208,136 @@ class Worker:
             )
 
             total_work, due_work = sys.maxsize, 0
-            while total_work:
+            while forever or total_work:
                 now = datetime.now(timezone.utc)
                 total_work, due_work = await stream_due_tasks(
                     keys=[self.docket.queue_key, self.docket.stream_key],
                     args=[now.timestamp(), self.docket.name],
                 )
-                logger.info(
-                    "Moved %d/%d due tasks from %s to %s",
-                    due_work,
-                    total_work,
-                    self.docket.queue_key,
-                    self.docket.stream_key,
-                    extra=self._log_context,
+                if due_work > 0:
+                    logger.debug(
+                        "Moved %d/%d due tasks from %s to %s",
+                        due_work,
+                        total_work,
+                        self.docket.queue_key,
+                        self.docket.stream_key,
+                        extra=self._log_context,
+                    )
+
+                _, redeliveries, _ = await redis.xautoclaim(
+                    name=self.docket.stream_key,
+                    groupname=self.consumer_group_name,
+                    consumername=self.name,
+                    min_idle_time=int(self.redelivery_timeout.total_seconds() * 1000),
+                    start_id="0-0",
+                    count=self.prefetch_count,
                 )
 
-                response: RedisReadGroupResponse = await redis.xreadgroup(
+                new_deliveries: RedisReadGroupResponse = await redis.xreadgroup(
                     groupname=self.consumer_group_name,
                     consumername=self.name,
                     streams={self.docket.stream_key: ">"},
                     count=self.prefetch_count,
                     block=10,
                 )
-                for _, messages in response:
-                    for message_id, message in messages:
-                        await self._execute(message)
-
-                        # When executing a task, there's always a chance that it was
-                        # either retried or it scheduled another task, so let's give
-                        # ourselves one more iteration of the loop to handle that.
-                        total_work += 1
-
-                        async with redis.pipeline() as pipe:
-                            pipe.xack(
-                                self.docket.stream_key,
-                                self.consumer_group_name,
-                                message_id,
-                            )
-                            pipe.xdel(
-                                self.docket.stream_key,
-                                message_id,
-                            )
-                            await pipe.execute()
+
+                for source in [[(b"redeliveries", redeliveries)], new_deliveries]:
+                    for _, messages in source:
+                        for message_id, message in messages:
+                            await self._execute(message)
+
+                            async with redis.pipeline() as pipeline:
+                                pipeline.xack(
+                                    self.docket.stream_key,
+                                    self.consumer_group_name,
+                                    message_id,
+                                )
+                                pipeline.xdel(
+                                    self.docket.stream_key,
+                                    message_id,
+                                )
+                                await pipeline.execute()
+
+                            # When executing a task, there's always a chance that it was
+                            # either retried or it scheduled another task, so let's give
+                            # ourselves one more iteration of the loop to handle that.
+                            total_work += 1
 
     async def _execute(self, message: RedisMessage) -> None:
         execution = Execution.from_message(
             self.docket.tasks[message[b"function"].decode()],
             message,
         )
+        name = execution.function.__name__
+        key = execution.key
 
-        logger.info(
-            "Executing task %s with args %s and kwargs %s",
-            execution.key,
-            execution.args,
-            execution.kwargs,
-            extra={
-                **self._log_context,
-                "function": execution.function.__name__,
-            },
-        )
+        log_context: dict[str, str | float] = {
+            **self._log_context,
+            "task": name,
+            "key": key,
+        }
+        counter_labels = {
+            "docket": self.docket.name,
+            "worker": self.name,
+            "task": name,
+        }
 
         dependencies = self._get_dependencies(execution)
 
+        context = propagate.extract(message, getter=message_getter)
+        initiating_context = trace.get_current_span(context).get_span_context()
+        links = [trace.Link(initiating_context)] if initiating_context.is_valid else []
+
+        start = datetime.now(timezone.utc)
+        punctuality = start - execution.when
+        log_context["punctuality"] = punctuality.total_seconds()
+        duration = timedelta(0)
+
+        TASKS_STARTED.add(1, counter_labels)
+        TASKS_RUNNING.add(1, counter_labels)
+        TASK_PUNCTUALITY.record(punctuality.total_seconds(), counter_labels)
+
+        arrow = "↬" if execution.attempt > 1 else "↪"
+        call = execution.call_repr()
+        logger.info("%s [%s] %s", arrow, punctuality, call, extra=log_context)
+
         try:
-            await execution.function(
-                *execution.args,
-                **{
-                    **execution.kwargs,
-                    **dependencies,
+            with tracer.start_as_current_span(
+                execution.function.__name__,
+                kind=trace.SpanKind.CONSUMER,
+                attributes={
+                    "docket.name": self.docket.name,
+                    "docket.execution.when": execution.when.isoformat(),
+                    "docket.execution.key": execution.key,
+                    "docket.execution.attempt": execution.attempt,
+                    "docket.execution.punctuality": punctuality.total_seconds(),
+                    "code.function.name": execution.function.__name__,
                 },
-            )
+                links=links,
+            ):
+                await execution.function(
+                    *execution.args,
+                    **{
+                        **execution.kwargs,
+                        **dependencies,
+                    },
+                )
+
+            TASKS_SUCCEEDED.add(1, counter_labels)
+            duration = datetime.now(timezone.utc) - start
+            log_context["duration"] = duration.total_seconds()
+            logger.info("%s [%s] %s", "↩", duration, call, extra=log_context)
         except Exception:
-            logger.exception(
-                "Error executing task %s",
-                execution.key,
-                extra=self._log_context,
-            )
-            await self._retry_if_requested(execution, dependencies)
+            TASKS_FAILED.add(1, counter_labels)
+            duration = datetime.now(timezone.utc) - start
+            log_context["duration"] = duration.total_seconds()
+            retried = await self._retry_if_requested(execution, dependencies)
+            arrow = "↫" if retried else "↩"
+            logger.exception("%s [%s] %s", arrow, duration, call, extra=log_context)
+        finally:
+            TASKS_RUNNING.add(-1, counter_labels)
+            TASKS_COMPLETED.add(1, counter_labels)
+            TASK_DURATION.record(duration.total_seconds(), counter_labels)
 
     def _get_dependencies(
         self,
@@ -208,14 +349,14 @@ class Worker:
 
         dependencies: dict[str, Any] = {}
 
-        for param_name, dependency in parameters.items():
+        for parameter_name, dependency in parameters.items():
             # If the argument is already provided, skip it, which allows users to call
             # the function directly with the arguments they want.
-            if param_name in execution.kwargs:
-                dependencies[param_name] = execution.kwargs[param_name]
+            if parameter_name in execution.kwargs:
+                dependencies[parameter_name] = execution.kwargs[parameter_name]
                 continue
 
-            dependencies[param_name] = dependency(self.docket, self, execution)
+            dependencies[parameter_name] = dependency(self.docket, self, execution)
 
         return dependencies
 
@@ -223,22 +364,26 @@ class Worker:
         self,
         execution: Execution,
         dependencies: dict[str, Any],
-    ) -> None:
+    ) -> bool:
         from .dependencies import Retry
 
         retries = [retry for retry in dependencies.values() if isinstance(retry, Retry)]
         if not retries:
-            return
+            return False
 
         retry = retries[0]
 
-        if execution.attempt < retry.attempts:
+        if retry.attempts is None or execution.attempt < retry.attempts:
             execution.when = datetime.now(timezone.utc) + retry.delay
             execution.attempt += 1
             await self.docket.schedule(execution)
-        else:
-            logger.error(
-                "Task %s failed after %d attempts",
-                execution.key,
-                retry.attempts,
-            )
+
+            counter_labels = {
+                "docket": self.docket.name,
+                "worker": self.name,
+                "task": execution.function.__name__,
+            }
+            TASKS_RETRIED.add(1, counter_labels)
+            return True
+
+        return False

{pydocket-0.0.1.dist-info → pydocket-0.0.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pydocket
-Version: 0.0.1
+Version: 0.0.2
 Summary: A distributed background task system for Python functions
 Project-URL: Homepage, https://github.com/chrisguidry/docket
 Project-URL: Bug Tracker, https://github.com/chrisguidry/docket/issues
@@ -23,7 +23,12 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Python: >=3.12
 Requires-Dist: cloudpickle>=3.1.1
+Requires-Dist: opentelemetry-api>=1.30.0
+Requires-Dist: opentelemetry-exporter-prometheus>=0.51b0
+Requires-Dist: prometheus-client>=0.21.1
+Requires-Dist: python-json-logger>=3.2.1
 Requires-Dist: redis>=5.2.1
+Requires-Dist: rich>=13.9.4
 Requires-Dist: typer>=0.15.1
 Description-Content-Type: text/markdown
 

pydocket-0.0.2.dist-info/RECORD

@@ -0,0 +1,16 @@
+docket/__init__.py,sha256=GoJYpyuO6QFeBB8GNaxGGvMMuai55Eaw_8u-o1PM3hk,743
+docket/__main__.py,sha256=Vkuh7aJ-Bl7QVpVbbkUksAd_hn05FiLmWbc-8kbhZQ4,34
+docket/annotations.py,sha256=GZwOPtPXyeIhnsLh3TQMBnXrjtTtSmF4Ratv4vjPx8U,950
+docket/cli.py,sha256=Qj2wzc3WjPDFsRiYJDVaqXKdTCUrFHDESwT8eebhvUk,7101
+docket/dependencies.py,sha256=iJ9RdrdgB7jVclxFUd2mR9OXh7GjdpyR6k8fIHn4Sz4,4309
+docket/docket.py,sha256=Z9gnS-vVFMqIThnUNEDvukN3InFz4T80SmPYxhbF9HQ,7006
+docket/execution.py,sha256=AEfNwmpBFJEf9ZFnbA37dqjWexGzmMcmvdxGnDsUQdY,2340
+docket/instrumentation.py,sha256=ZgXqjjPPspeQnfGakcBUS29FcRa-SKeqC0INTBzvIow,2515
+docket/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+docket/tasks.py,sha256=rFAO9z0ESpH1F-P02DJ3yj5E54TGMtNc2pUkpaiTLVY,1317
+docket/worker.py,sha256=DGvaSyiFKkV0k2YG9jhmRhStFm1gDhWZaxDXDDky2yo,13987
+pydocket-0.0.2.dist-info/METADATA,sha256=WWNNg9YXTL8zyPUzHFUvkS0ARvFw_Wl2w_nBf-KqFNE,2292
+pydocket-0.0.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+pydocket-0.0.2.dist-info/entry_points.txt,sha256=4WOk1nUlBsUT5O3RyMci2ImuC5XFswuopElYcLHtD5k,47
+pydocket-0.0.2.dist-info/licenses/LICENSE,sha256=YuVWU_ZXO0K_k2FG8xWKe5RGxV24AhJKTvQmKfqXuyk,1087
+pydocket-0.0.2.dist-info/RECORD,,

pydocket-0.0.1.dist-info/RECORD

@@ -1,13 +0,0 @@
-docket/__init__.py,sha256=0gzwWxJcDqU7nEZ4rHrWYk7WqEgW4Mz77E7rbjpyCHw,526
-docket/__main__.py,sha256=Vkuh7aJ-Bl7QVpVbbkUksAd_hn05FiLmWbc-8kbhZQ4,34
-docket/cli.py,sha256=ty8CirvLDvvOuOs7MHW0SYMjmbQq9rqUgPCq-HqW_6g,434
-docket/dependencies.py,sha256=yd_sIv3y69Czo_DyHh3aNtLJGFOMjg8jjJ701QN04-Q,2124
-docket/docket.py,sha256=sIgSvPUX4HG4EN_OxPSN7xCGu7DZkk5oZsuBVDp5XcU,4942
-docket/execution.py,sha256=3HT1GOeg76RMILiq06bpDZITHrqjVQ_j9FU6fuY4jqw,1375
-docket/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-docket/worker.py,sha256=N-nftCveJELKnICp1lgmzG5-r1QCYkGRxi8FcTjrLSc,8204
-pydocket-0.0.1.dist-info/METADATA,sha256=IYrB8lUbawu5I9Buh6uVJ68kJh47GUfph3F1wxxN78w,2084
-pydocket-0.0.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-pydocket-0.0.1.dist-info/entry_points.txt,sha256=4WOk1nUlBsUT5O3RyMci2ImuC5XFswuopElYcLHtD5k,47
-pydocket-0.0.1.dist-info/licenses/LICENSE,sha256=YuVWU_ZXO0K_k2FG8xWKe5RGxV24AhJKTvQmKfqXuyk,1087
-pydocket-0.0.1.dist-info/RECORD,,