pydocket 0.6.4__tar.gz → 0.7.1__tar.gz

{pydocket-0.6.4 → pydocket-0.7.1}/.github/workflows/ci.yml

@@ -15,7 +15,7 @@ jobs:
       fail-fast: false
       matrix:
         python-version: ["3.12", "3.13"]
-        redis-version: ["6.2", "7.4"]
+        redis-version: ["6.2", "7.4", "valkey-8.0"]
         redis-py-version: [">=4.6,<5", ">=5"]
 
     steps:
@@ -34,7 +34,7 @@ jobs:
       - name: Run tests
         env:
           REDIS_VERSION: ${{ matrix.redis-version }}
-        run: uv run pytest --cov-branch --cov-report=xml --cov-report=term-missing:skip-covered
+        run: uv run pytest --cov-branch --cov-fail-under=100 --cov-report=xml --cov-report=term-missing:skip-covered
 
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v5

{pydocket-0.6.4 → pydocket-0.7.1}/.github/workflows/publish.yml

@@ -13,8 +13,11 @@ jobs:
     name: Build and publish to PyPI
     runs-on: ubuntu-latest
     needs: ci
+    environment:
+      name: pypi
+      url: https://pypi.org/p/pydocket
     permissions:
-      id-token: write
+      id-token: write  # Required for trusted publishing and PEP 740 attestations
       contents: read
 
     steps:
@@ -31,10 +34,12 @@ jobs:
           cache-dependency-glob: "pyproject.toml"
 
       - name: Install build dependencies
-        run: uv pip install build hatchling hatch-vcs
+        run: uv pip install hatchling hatch-vcs
 
       - name: Build package
         run: uv build
 
-      - name: Publish to PyPI
-        run: uv publish
+      - name: Publish to PyPI with PEP 740 attestations
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: dist/

{pydocket-0.6.4 → pydocket-0.7.1}/.gitignore

@@ -1,13 +1,11 @@
-# Python-generated files
-__pycache__/
-*.py[oc]
-build/
-dist/
-wheels/
 *.egg-info
-
-# Virtual environments
-.venv
+*.py[oc]
 .coverage
 .envrc
 .python-version
+.venv
+.worktrees/
+__pycache__/
+build/
+dist/
+wheels/

pydocket-0.7.1/CLAUDE.md

@@ -0,0 +1,127 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Project Overview
+
+**Docket** (`pydocket` on PyPI) is a distributed background task system for Python functions with Redis-backed persistence. It enables scheduling both immediate and future work with comprehensive dependency injection, retry mechanisms, and fault tolerance.
+
+**Key Requirements**: Python 3.12+, Redis 6.2+ or Valkey 8.0+
+
+## Development Commands
+
+### Testing
+
+```bash
+# Run full test suite with coverage and parallel execution
+pytest
+
+# Run specific test
+pytest tests/test_docket.py::test_specific_function
+
+```
+
+The project REQUIRES 100% test coverage
+
+### Code Quality
+
+```bash
+# Lint and format code
+ruff check
+ruff format
+
+# Type checking
+pyright
+pyright tests
+
+# Run all pre-commit hooks
+pre-commit run --all-files
+```
+
+### Development Setup
+
+```bash
+# Install development dependencies
+uv sync --group dev
+
+# Install pre-commit hooks
+pre-commit install
+```
+
+### Git Workflow
+
+- This project uses Github for issue tracking
+- This project can use git worktrees under .worktrees/
+
+## Core Architecture
+
+### Key Classes
+
+- **`Docket`** (`src/docket/docket.py`): Central task registry and scheduler
+
+  - `add()`: Schedule tasks for execution
+  - `replace()`: Replace existing scheduled tasks
+  - `cancel()`: Cancel pending tasks
+  - `strike()`/`restore()`: Conditionally block/unblock tasks
+  - `snapshot()`: Get current state for observability
+
+- **`Worker`** (`src/docket/worker.py`): Task execution engine
+
+  - `run_forever()`/`run_until_finished()`: Main execution loops
+  - Handles concurrency, retries, and dependency injection
+  - Maintains heartbeat for liveness tracking
+
+- **`Execution`** (`src/docket/execution.py`): Task execution context with metadata
+
+### Dependencies System (`src/docket/dependencies.py`)
+
+Rich dependency injection supporting:
+
+- Context access: `CurrentDocket`, `CurrentWorker`, `CurrentExecution`
+- Retry strategies: `Retry`, `ExponentialRetry`
+- Special behaviors: `Perpetual` (self-rescheduling), `Timeout`
+- Custom injection: `Depends()`
+- Contextual logging: `TaskLogger`
+
+### Redis Data Model
+
+- **Streams**: `{docket}:stream` (ready tasks), `{docket}:strikes` (commands)
+- **Sorted Sets**: `{docket}:queue` (scheduled tasks), `{docket}:workers` (heartbeats)
+- **Hashes**: `{docket}:{key}` (parked task data)
+- **Sets**: `{docket}:worker-tasks:{worker}` (worker capabilities)
+
+### Task Lifecycle
+
+1. Registration with `Docket.register()` or `@docket.task`
+2. Scheduling: immediate → Redis stream, future → Redis sorted set
+3. Worker processing: scheduler moves due tasks, workers consume via consumer groups
+4. Execution: dependency injection, retry logic, acknowledgment
+
+## Project Structure
+
+### Source Code
+
+- `src/docket/` - Main package
+  - `__init__.py` - Public API exports
+  - `docket.py` - Core Docket class
+  - `worker.py` - Worker implementation
+  - `execution.py` - Task execution context
+  - `dependencies.py` - Dependency injection system
+  - `tasks.py` - Built-in utility tasks
+  - `cli.py` - Command-line interface
+
+### Testing and Examples
+
+- `tests/` - Comprehensive test suite
+- `examples/` - Usage examples
+- `chaos/` - Chaos testing framework
+
+## CLI Usage
+
+```bash
+# Run a worker
+docket worker --url redis://localhost:6379/0 --tasks your.module --concurrency 4
+
+# See all commands
+docket --help
+```

{pydocket-0.6.4 → pydocket-0.7.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pydocket
-Version: 0.6.4
+Version: 0.7.1
 Summary: A distributed background task system for Python functions
 Project-URL: Homepage, https://github.com/chrisguidry/docket
 Project-URL: Bug Tracker, https://github.com/chrisguidry/docket/issues

{pydocket-0.6.4 → pydocket-0.7.1}/chaos/driver.py

@@ -153,13 +153,20 @@ async def main(
                     sent_tasks = await r.zcard("hello:sent")
                     received_tasks = await r.zcard("hello:received")
 
+                    stream_length = await r.xlen(docket.stream_key)
+                    pending = await r.xpending(
+                        docket.stream_key, docket.worker_group_name
+                    )
+
                     logger.info(
-                        "sent: %d, received: %d, clients: %d",
+                        "sent: %d, received: %d, stream: %d, pending: %d, clients: %d",
                         sent_tasks,
                         received_tasks,
+                        stream_length,
+                        pending["pending"],
                         connected_clients,
                     )
-                    if sent_tasks >= tasks:
+                    if sent_tasks >= tasks and received_tasks >= sent_tasks:
                         break
             except redis.exceptions.ConnectionError as e:
                 logger.error(
@@ -177,16 +184,13 @@ async def main(
 
                 elif chaos_chance < 0.10:
                     worker_index = random.randrange(len(worker_processes))
-                    worker_to_kill = worker_processes.pop(worker_index)
+                    worker_to_kill = worker_processes[worker_index]
 
                     logger.warning("CHAOS: Killing worker %d...", worker_index)
                     try:
-                        worker_to_kill.terminate()
+                        worker_to_kill.kill()
                     except ProcessLookupError:
                         logger.warning("  What is dead may never die!")
-
-                    logger.warning("CHAOS: Replacing worker %d...", worker_index)
-                    worker_processes.append(await spawn_worker())
                 elif chaos_chance < 0.15:
                     logger.warning("CHAOS: Queuing a toxic task...")
                     try:
@@ -194,6 +198,17 @@ async def main(
                     except redis.exceptions.ConnectionError:
                         pass
 
+            # Check if any worker processes have died and replace them
+            for i in range(len(worker_processes)):
+                process = worker_processes[i]
+                if process.returncode is not None:
+                    logger.warning(
+                        "Worker %d has died with code %d, replacing it...",
+                        i,
+                        process.returncode,
+                    )
+                    worker_processes[i] = await spawn_worker()
+
             await asyncio.sleep(0.25)
 
         async with docket.redis() as r:
@@ -225,5 +240,6 @@ async def main(
 
 if __name__ == "__main__":
     mode = sys.argv[1] if len(sys.argv) > 1 else "chaos"
+    tasks = int(sys.argv[2]) if len(sys.argv) > 2 else 20000
     assert mode in ("performance", "chaos")
-    asyncio.run(main(mode=mode))
+    asyncio.run(main(mode=mode, tasks=tasks))

{pydocket-0.6.4 → pydocket-0.7.1}/chaos/tasks.py

@@ -1,4 +1,6 @@
+import asyncio
 import logging
+import random
 import sys
 import time
 
@@ -29,7 +31,12 @@ async def hello(
 
 
 async def toxic():
-    sys.exit(42)
+    if random.random() < 0.25:
+        sys.exit(42)
+    elif random.random() < 0.5:
+        raise Exception("Boom")
+    else:
+        await asyncio.sleep(random.uniform(0.01, 0.05))
 
 
 chaos_tasks = [hello, toxic]

{pydocket-0.6.4 → pydocket-0.7.1}/src/docket/annotations.py

@@ -2,6 +2,8 @@ import abc
 import inspect
 from typing import Any, Iterable, Mapping, Self
 
+from .instrumentation import CACHE_SIZE
+
 
 class Annotation(abc.ABC):
     _cache: dict[tuple[type[Self], inspect.Signature], Mapping[str, Self]] = {}
@@ -10,6 +12,7 @@ class Annotation(abc.ABC):
     def annotated_parameters(cls, signature: inspect.Signature) -> Mapping[str, Self]:
         key = (cls, signature)
         if key in cls._cache:
+            CACHE_SIZE.set(len(cls._cache), {"cache": "annotation"})
             return cls._cache[key]
 
         annotated: dict[str, Self] = {}
@@ -30,6 +33,7 @@ class Annotation(abc.ABC):
                     annotated[param_name] = arg_type()
 
         cls._cache[key] = annotated
+        CACHE_SIZE.set(len(cls._cache), {"cache": "annotation"})
         return annotated
 
 

{pydocket-0.6.4 → pydocket-0.7.1}/src/docket/cli.py

@@ -358,6 +358,32 @@ def strike(
     asyncio.run(run())
 
 
+@app.command(help="Clear all pending and scheduled tasks from the docket")
+def clear(
+    docket_: Annotated[
+        str,
+        typer.Option(
+            "--docket",
+            help="The name of the docket",
+            envvar="DOCKET_NAME",
+        ),
+    ] = "docket",
+    url: Annotated[
+        str,
+        typer.Option(
+            help="The URL of the Redis server",
+            envvar="DOCKET_URL",
+        ),
+    ] = "redis://localhost:6379/0",
+) -> None:
+    async def run() -> None:
+        async with Docket(name=docket_, url=url) as docket:
+            cleared_count = await docket.clear()
+            print(f"Cleared {cleared_count} tasks from docket '{docket_}'")
+
+    asyncio.run(run())
+
+
 @app.command(help="Restores a task or parameters to the Docket")
 def restore(
     function: Annotated[

{pydocket-0.6.4 → pydocket-0.7.1}/src/docket/dependencies.py

@@ -3,7 +3,7 @@ import logging
 import time
 from contextlib import AsyncExitStack, asynccontextmanager
 from contextvars import ContextVar
-from datetime import timedelta
+from datetime import datetime, timedelta, timezone
 from types import TracebackType
 from typing import (
     TYPE_CHECKING,
@@ -14,12 +14,14 @@ from typing import (
     Callable,
     Counter,
     Generic,
+    NoReturn,
     TypeVar,
     cast,
 )
 
 from .docket import Docket
 from .execution import Execution, TaskFunction, get_signature
+from .instrumentation import CACHE_SIZE
 
 if TYPE_CHECKING:  # pragma: no cover
     from .worker import Worker
@@ -188,6 +190,10 @@ def TaskLogger() -> logging.LoggerAdapter[logging.Logger]:
     return cast(logging.LoggerAdapter[logging.Logger], _TaskLogger())
 
 
+class ForcedRetry(Exception):
+    """Raised when a task requests a retry via `in_` or `at`"""
+
+
 class Retry(Dependency):
     """Configures linear retries for a task.  You can specify the total number of
     attempts (or `None` to retry indefinitely), and the delay between attempts.
@@ -222,6 +228,17 @@ class Retry(Dependency):
         retry.attempt = execution.attempt
         return retry
 
+    def at(self, when: datetime) -> NoReturn:
+        now = datetime.now(timezone.utc)
+        diff = when - now
+        diff = diff if diff.total_seconds() >= 0 else timedelta(0)
+
+        self.in_(diff)
+
+    def in_(self, when: timedelta) -> NoReturn:
+        self.delay: timedelta = when
+        raise ForcedRetry()
+
 
 class ExponentialRetry(Retry):
     """Configures exponential retries for a task.  You can specify the total number
@@ -251,7 +268,6 @@ class ExponentialRetry(Retry):
             maximum_delay: The maximum delay between attempts.
         """
         super().__init__(attempts=attempts, delay=minimum_delay)
-        self.minimum_delay = minimum_delay
         self.maximum_delay = maximum_delay
 
     async def __aenter__(self) -> "ExponentialRetry":
@@ -259,14 +275,14 @@ class ExponentialRetry(Retry):
 
         retry = ExponentialRetry(
             attempts=self.attempts,
-            minimum_delay=self.minimum_delay,
+            minimum_delay=self.delay,
             maximum_delay=self.maximum_delay,
         )
         retry.attempt = execution.attempt
 
         if execution.attempt > 1:
             backoff_factor = 2 ** (execution.attempt - 1)
-            calculated_delay = self.minimum_delay * backoff_factor
+            calculated_delay = self.delay * backoff_factor
 
             if calculated_delay > self.maximum_delay:
                 retry.delay = self.maximum_delay
@@ -400,6 +416,7 @@ def get_dependency_parameters(
     function: TaskFunction | DependencyFunction[Any],
 ) -> dict[str, Dependency]:
     if function in _parameter_cache:
+        CACHE_SIZE.set(len(_parameter_cache), {"cache": "parameter"})
         return _parameter_cache[function]
 
     dependencies: dict[str, Dependency] = {}
@@ -413,6 +430,7 @@ def get_dependency_parameters(
         dependencies[parameter] = param.default
 
     _parameter_cache[function] = dependencies
+    CACHE_SIZE.set(len(_parameter_cache), {"cache": "parameter"})
     return dependencies
 
 

{pydocket-0.6.4 → pydocket-0.7.1}/src/docket/docket.py

@@ -743,3 +743,46 @@ class Docket:
                 workers.append(WorkerInfo(worker_name, last_seen, task_names))
 
         return workers
+
+    async def clear(self) -> int:
+        """Clear all pending and scheduled tasks from the docket.
+
+        This removes all tasks from the stream (immediate tasks) and queue
+        (scheduled tasks), along with their associated parked data. Running
+        tasks are not affected.
+
+        Returns:
+            The total number of tasks that were cleared.
+        """
+        with tracer.start_as_current_span(
+            "docket.clear",
+            attributes=self.labels(),
+        ):
+            async with self.redis() as redis:
+                async with redis.pipeline() as pipeline:
+                    # Get counts before clearing
+                    pipeline.xlen(self.stream_key)
+                    pipeline.zcard(self.queue_key)
+                    pipeline.zrange(self.queue_key, 0, -1)
+
+                    stream_count: int
+                    queue_count: int
+                    scheduled_keys: list[bytes]
+                    stream_count, queue_count, scheduled_keys = await pipeline.execute()
+
+                    # Clear all data
+                    # Trim stream to 0 messages instead of deleting it to preserve consumer group
+                    if stream_count > 0:
+                        pipeline.xtrim(self.stream_key, maxlen=0, approximate=False)
+                    pipeline.delete(self.queue_key)
+
+                    # Clear parked task data and known task keys
+                    for key_bytes in scheduled_keys:
+                        key = key_bytes.decode()
+                        pipeline.delete(self.parked_task_key(key))
+                        pipeline.delete(self.known_task_key(key))
+
+                    await pipeline.execute()
+
+                    total_cleared = stream_count + queue_count
+                    return total_cleared

{pydocket-0.6.4 → pydocket-0.7.1}/src/docket/execution.py

@@ -19,7 +19,7 @@ import opentelemetry.context
 from opentelemetry import propagate, trace
 
 from .annotations import Logged
-from .instrumentation import message_getter
+from .instrumentation import CACHE_SIZE, message_getter
 
 logger: logging.Logger = logging.getLogger(__name__)
 
@@ -32,10 +32,12 @@ _signature_cache: dict[Callable[..., Any], inspect.Signature] = {}
 
 def get_signature(function: Callable[..., Any]) -> inspect.Signature:
     if function in _signature_cache:
+        CACHE_SIZE.set(len(_signature_cache), {"cache": "signature"})
         return _signature_cache[function]
 
     signature = inspect.signature(function)
     _signature_cache[function] = signature
+    CACHE_SIZE.set(len(_signature_cache), {"cache": "signature"})
     return signature
 
 

{pydocket-0.6.4 → pydocket-0.7.1}/src/docket/instrumentation.py

@@ -117,6 +117,12 @@ SCHEDULE_DEPTH = meter.create_gauge(
     unit="1",
 )
 
+CACHE_SIZE = meter.create_gauge(
+    "docket_cache_size",
+    description="Size of internal docket caches",
+    unit="1",
+)
+
 Message = dict[bytes, bytes]
 
 

{pydocket-0.6.4 → pydocket-0.7.1}/src/docket/worker.py

@@ -15,7 +15,7 @@ from typing import (
 )
 
 from opentelemetry import trace
-from opentelemetry.trace import Tracer
+from opentelemetry.trace import Status, StatusCode, Tracer
 from redis.asyncio import Redis
 from redis.exceptions import ConnectionError, LockError
 
@@ -531,7 +531,7 @@ class Worker:
                 "code.function.name": execution.function.__name__,
             },
             links=execution.incoming_span_links(),
-        ):
+        ) as span:
             try:
                 async with resolved_dependencies(self, execution) as dependencies:
                     # Preemptively reschedule the perpetual task for the future, or clear
@@ -576,6 +576,8 @@ class Worker:
                     duration = log_context["duration"] = time.time() - start
                     TASKS_SUCCEEDED.add(1, counter_labels)
 
+                    span.set_status(Status(StatusCode.OK))
+
                     rescheduled = await self._perpetuate_if_requested(
                         execution, dependencies, timedelta(seconds=duration)
                     )
@@ -584,10 +586,13 @@ class Worker:
                     logger.info(
                         "%s [%s] %s", arrow, ms(duration), call, extra=log_context
                     )
-            except Exception:
+            except Exception as e:
                 duration = log_context["duration"] = time.time() - start
                 TASKS_FAILED.add(1, counter_labels)
 
+                span.record_exception(e)
+                span.set_status(Status(StatusCode.ERROR, str(e)))
+
                 retried = await self._retry_if_requested(execution, dependencies)
                 if not retried:
                     retried = await self._perpetuate_if_requested(