pydocket 0.0.1__tar.gz

pydocket-0.0.1/.cursorrules

@@ -0,0 +1,41 @@
+docket is a distributed background task system for Python functions with a focus
+on the scheduling of future work as seamlessly and efficiency as immediate work.
+
+docket is built in Python and uses Redis as the message broker and storage system.
+
+docket integrates two modes of task execution:
+
+1. Immediate tasks are pushed onto a Redis stream and are available to be
+   picked up by any worker.
+2. Scheduled tasks are pushed onto a Redis sorted set with a schedule time.
+   A loop within each worker moves scheduled tasks onto the stream when their
+   schedule time has arrived. This move is performed as a Lua script to ensure
+   atomicity. Once a scheduled task is moved onto the stream, it is now an
+   immediate task and can't be rescheduled.
+
+docket inherently understands self-perpetuating chains of tasks, where a task
+will repeatedly reschedule itself until it is no longer needed. This is supported
+directly in the developer API so that devs don't need to worry about the mechanics.
+
+Tasks have unique identifiers that may be set by the caller in order to guarantee
+idempotency of an execution.
+
+A docket worker should be as easily usable in code as it is from the command line,
+and should be a breeze to use with test suites.
+
+# Code style
+
+When generating production code, always use full parameter and return type hints
+for every function. Never generate useless inline comments that just reiterate
+what the code is doing. It's okay to include comments in the rare case there is
+something tricky going on.
+
+When generating tests, always use parameter type hints, but never include the
+`-> None` return type hint for a test function. For `pytest` fixtures, always
+generate both the parameter and return type hints.
+
+When generating tests, favor smaller, focused tests that use fixtures for reuse.
+Don't include extraneous comments in the test code unless something needs more
+clarity. Always generate a docstring using "should" language to describe the
+aspect of the system the test is checking. Use simple direct language and avoid
+sounding stuffy, but make these complete sentences.

pydocket-0.0.1/.github/workflows/ci.yml

@@ -0,0 +1,70 @@
+name: Docket CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  workflow_call:
+
+jobs:
+  test:
+    name: Test Python ${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.12", "3.13"]
+
+    services:
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv and set Python version
+        uses: astral-sh/setup-uv@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          enable-cache: true
+          cache-dependency-glob: "pyproject.toml"
+
+      - name: Install dependencies
+        run: uv sync --dev
+
+      - name: Run tests
+        run: uv run pytest --cov-branch --cov-report=xml
+
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+
+  pre-commit:
+    name: Pre-commit checks
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv and set Python version
+        uses: astral-sh/setup-uv@v5
+        with:
+          python-version: "3.12"
+          enable-cache: true
+          cache-dependency-glob: "pyproject.toml"
+
+      - name: Install dependencies
+        run: |
+          uv sync --dev
+          uv pip install pip
+
+      - uses: pre-commit/action@v3.0.1
+        with:
+          extra_args: --all-files

pydocket-0.0.1/.github/workflows/publish.yml

@@ -0,0 +1,40 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  ci:
+    name: Run CI
+    uses: ./.github/workflows/ci.yml
+
+  publish:
+    name: Build and publish to PyPI
+    runs-on: ubuntu-latest
+    needs: ci
+    permissions:
+      id-token: write
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # Need full history for proper versioning with hatch-vcs
+          fetch-depth: 0
+
+      - name: Install uv and set Python version
+        uses: astral-sh/setup-uv@v5
+        with:
+          python-version: "3.12"
+          enable-cache: true
+          cache-dependency-glob: "pyproject.toml"
+
+      - name: Install build dependencies
+        run: uv pip install build hatchling hatch-vcs
+
+      - name: Build package
+        run: uv build
+
+      - name: Publish to PyPI
+        run: uv publish

pydocket-0.0.1/.gitignore

@@ -0,0 +1,13 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# Virtual environments
+.venv
+.coverage
+.envrc
+.python-version

pydocket-0.0.1/.pre-commit-config.yaml

@@ -0,0 +1,30 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-toml
+      - id: check-added-large-files
+
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.2.6
+    hooks:
+      - id: codespell
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.9.7
+    hooks:
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix, --show-fixes]
+      - id: ruff-format
+
+  - repo: local
+    hooks:
+      - id: pyright
+        name: pyright
+        entry: uv run pyright --verifytypes docket --ignoreexternal
+        language: system
+        types: [python]
+        pass_filenames: false

pydocket-0.0.1/LICENSE

@@ -0,0 +1,9 @@
+# Released under MIT License
+
+Copyright (c) 2025 Chris Guidry.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

pydocket-0.0.1/PKG-INFO

@@ -0,0 +1,31 @@
+Metadata-Version: 2.4
+Name: pydocket
+Version: 0.0.1
+Summary: A distributed background task system for Python functions
+Project-URL: Homepage, https://github.com/chrisguidry/docket
+Project-URL: Bug Tracker, https://github.com/chrisguidry/docket/issues
+Author-email: Chris Guidry <guid@omg.lol>
+License: # Released under MIT License
+        
+        Copyright (c) 2025 Chris Guidry.
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+License-File: LICENSE
+Classifier: Development Status :: 4 - Beta
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Python: >=3.12
+Requires-Dist: cloudpickle>=3.1.1
+Requires-Dist: redis>=5.2.1
+Requires-Dist: typer>=0.15.1
+Description-Content-Type: text/markdown
+
+docket is a distributed background task system for Python functions with a focus
+on the scheduling of future work as seamlessly and efficiency as immediate work.

pydocket-0.0.1/README.md

@@ -0,0 +1,2 @@
+docket is a distributed background task system for Python functions with a focus
+on the scheduling of future work as seamlessly and efficiency as immediate work.

pydocket-0.0.1/pyproject.toml

@@ -0,0 +1,59 @@
+[build-system]
+requires = ["hatchling", "hatch-vcs"]
+build-backend = "hatchling.build"
+
+[project]
+name = "pydocket"
+dynamic = ["version"]
+description = "A distributed background task system for Python functions"
+readme = "README.md"
+requires-python = ">=3.12"
+license = { file = "LICENSE" }
+authors = [{ name = "Chris Guidry", email = "guid@omg.lol" }]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+]
+dependencies = ["cloudpickle>=3.1.1", "redis>=5.2.1", "typer>=0.15.1"]
+
+[dependency-groups]
+dev = [
+    "codespell>=2.4.1",
+    "mypy>=1.14.1",
+    "pre-commit>=4.1.0",
+    "pyright>=1.1.394",
+    "pytest>=8.3.4",
+    "pytest-asyncio>=0.25.3",
+    "pytest-cov>=6.0.0",
+    "pytest-xdist>=3.6.1",
+    "ruff>=0.9.7",
+    "testcontainers>=4.9.1",
+]
+
+[project.scripts]
+docket = "docket.__main__:app"
+
+[project.urls]
+"Homepage" = "https://github.com/chrisguidry/docket"
+"Bug Tracker" = "https://github.com/chrisguidry/docket/issues"
+
+[tool.hatch.version]
+source = "vcs"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/docket"]
+
+[tool.pytest.ini_options]
+addopts = "--cov=src/docket --cov=tests --cov-report=term-missing --cov-branch"
+asyncio_mode = "auto"
+asyncio_default_fixture_loop_scope = "session"
+
+[tool.pyright]
+include = ["src", "tests"]
+typeCheckingMode = "strict"
+venvPath = "."
+venv = ".venv"

pydocket-0.0.1/src/docket/__init__.py

@@ -0,0 +1,24 @@
+"""
+docket - A distributed background task system for Python functions.
+
+docket focuses on scheduling future work as seamlessly and efficiently as immediate work.
+"""
+
+from importlib.metadata import version
+
+__version__ = version("pydocket")
+
+from .dependencies import CurrentDocket, CurrentWorker, Retry
+from .docket import Docket
+from .execution import Execution
+from .worker import Worker
+
+__all__ = [
+    "Docket",
+    "Worker",
+    "Execution",
+    "CurrentDocket",
+    "CurrentWorker",
+    "Retry",
+    "__version__",
+]

pydocket-0.0.1/src/docket/__main__.py

@@ -0,0 +1,3 @@
+from docket.cli import app
+
+app()

pydocket-0.0.1/src/docket/cli.py

@@ -0,0 +1,23 @@
+import typer
+
+from docket import __version__
+
+app: typer.Typer = typer.Typer(
+    help="Docket - A distributed background task system for Python functions",
+    add_completion=True,
+    no_args_is_help=True,
+)
+
+
+@app.command(
+    help="Start a worker to process tasks",
+)
+def worker() -> None:
+    print("TODO: start the worker")
+
+
+@app.command(
+    help="Print the version of Docket",
+)
+def version() -> None:
+    print(__version__)

pydocket-0.0.1/src/docket/dependencies.py

@@ -0,0 +1,77 @@
+import abc
+import inspect
+from datetime import timedelta
+from typing import Any, Awaitable, Callable, Counter, cast
+
+from .docket import Docket
+from .execution import Execution
+from .worker import Worker
+
+
+class Dependency(abc.ABC):
+    single: bool = False
+
+    @abc.abstractmethod
+    def __call__(
+        self, docket: Docket, worker: Worker, execution: Execution
+    ) -> Any: ...  # pragma: no cover
+
+
+class _CurrentWorker(Dependency):
+    def __call__(self, docket: Docket, worker: Worker, execution: Execution) -> Worker:
+        return worker
+
+
+def CurrentWorker() -> Worker:
+    return cast(Worker, _CurrentWorker())
+
+
+class _CurrentDocket(Dependency):
+    def __call__(self, docket: Docket, worker: Worker, execution: Execution) -> Docket:
+        return docket
+
+
+def CurrentDocket() -> Docket:
+    return cast(Docket, _CurrentDocket())
+
+
+class Retry(Dependency):
+    single: bool = True
+
+    def __init__(self, attempts: int = 1, delay: timedelta = timedelta(0)) -> None:
+        self.attempts = attempts
+        self.delay = delay
+        self.attempt = 1
+
+    def __call__(self, docket: Docket, worker: Worker, execution: Execution) -> "Retry":
+        retry = Retry(attempts=self.attempts, delay=self.delay)
+        retry.attempt = execution.attempt
+        return retry
+
+
+def get_dependency_parameters(
+    function: Callable[..., Awaitable[Any]],
+) -> dict[str, Dependency]:
+    dependencies: dict[str, Any] = {}
+
+    signature = inspect.signature(function)
+
+    for param_name, param in signature.parameters.items():
+        if not isinstance(param.default, Dependency):
+            continue
+
+        dependencies[param_name] = param.default
+
+    return dependencies
+
+
+def validate_dependencies(function: Callable[..., Awaitable[Any]]) -> None:
+    parameters = get_dependency_parameters(function)
+
+    counts = Counter(type(dependency) for dependency in parameters.values())
+
+    for dependency_type, count in counts.items():
+        if dependency_type.single and count > 1:
+            raise ValueError(
+                f"Only one {dependency_type.__name__} dependency is allowed per task"
+            )

pydocket-0.0.1/src/docket/docket.py

@@ -0,0 +1,178 @@
+from contextlib import asynccontextmanager
+from datetime import datetime, timezone
+from types import TracebackType
+from typing import (
+    Any,
+    AsyncGenerator,
+    Awaitable,
+    Callable,
+    ParamSpec,
+    Self,
+    TypeVar,
+    overload,
+)
+from uuid import uuid4
+
+from redis.asyncio import Redis
+
+from .execution import Execution
+
+P = ParamSpec("P")
+R = TypeVar("R")
+
+
+class Docket:
+    tasks: dict[str, Callable[..., Awaitable[Any]]]
+
+    def __init__(
+        self,
+        name: str = "docket",
+        host: str = "localhost",
+        port: int = 6379,
+        db: int = 0,
+        password: str | None = None,
+    ) -> None:
+        self.name = name
+        self.host = host
+        self.port = port
+        self.db = db
+        self.password = password
+
+    async def __aenter__(self) -> Self:
+        self.tasks = {}
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_value: BaseException | None,
+        traceback: TracebackType | None,
+    ) -> None:
+        pass
+
+    @asynccontextmanager
+    async def redis(self) -> AsyncGenerator[Redis, None]:
+        async with Redis(
+            host=self.host,
+            port=self.port,
+            db=self.db,
+            password=self.password,
+            single_connection_client=True,
+        ) as redis:
+            yield redis
+
+    def register(self, function: Callable[..., Awaitable[Any]]) -> None:
+        from .dependencies import validate_dependencies
+
+        validate_dependencies(function)
+
+        self.tasks[function.__name__] = function
+
+    @overload
+    def add(
+        self,
+        function: Callable[P, Awaitable[R]],
+        when: datetime | None = None,
+        key: str | None = None,
+    ) -> Callable[P, Awaitable[Execution]]: ...  # pragma: no cover
+
+    @overload
+    def add(
+        self,
+        function: str,
+        when: datetime | None = None,
+        key: str | None = None,
+    ) -> Callable[..., Awaitable[Execution]]: ...  # pragma: no cover
+
+    def add(
+        self,
+        function: Callable[P, Awaitable[R]] | str,
+        when: datetime | None = None,
+        key: str | None = None,
+    ) -> Callable[..., Awaitable[Execution]]:
+        if isinstance(function, str):
+            function = self.tasks[function]
+        else:
+            self.register(function)
+
+        if when is None:
+            when = datetime.now(timezone.utc)
+
+        if key is None:
+            key = f"{function.__name__}:{uuid4()}"
+
+        async def scheduler(*args: P.args, **kwargs: P.kwargs) -> Execution:
+            execution = Execution(function, args, kwargs, when, key, attempt=1)
+            await self.schedule(execution)
+            return execution
+
+        return scheduler
+
+    @overload
+    def replace(
+        self,
+        function: Callable[P, Awaitable[R]],
+        when: datetime,
+        key: str,
+    ) -> Callable[P, Awaitable[Execution]]: ...  # pragma: no cover
+
+    @overload
+    def replace(
+        self,
+        function: str,
+        when: datetime,
+        key: str,
+    ) -> Callable[..., Awaitable[Execution]]: ...  # pragma: no cover
+
+    def replace(
+        self,
+        function: Callable[P, Awaitable[R]] | str,
+        when: datetime,
+        key: str,
+    ) -> Callable[..., Awaitable[Execution]]:
+        if isinstance(function, str):
+            function = self.tasks[function]
+
+        async def scheduler(*args: P.args, **kwargs: P.kwargs) -> Execution:
+            execution = Execution(function, args, kwargs, when, key, attempt=1)
+            await self.cancel(key)
+            await self.schedule(execution)
+            return execution
+
+        return scheduler
+
+    @property
+    def queue_key(self) -> str:
+        return f"{self.name}:queue"
+
+    @property
+    def stream_key(self) -> str:
+        return f"{self.name}:stream"
+
+    def parked_task_key(self, key: str) -> str:
+        return f"{self.name}:{key}"
+
+    async def schedule(self, execution: Execution) -> None:
+        message: dict[bytes, bytes] = execution.as_message()
+        key = execution.key
+        when = execution.when
+
+        async with self.redis() as redis:
+            # if the task is already in the queue, retain it
+            if await redis.zscore(self.queue_key, key) is not None:
+                return
+
+            if when <= datetime.now(timezone.utc):
+                await redis.xadd(self.stream_key, message)
+            else:
+                async with redis.pipeline() as pipe:
+                    pipe.hset(self.parked_task_key(key), mapping=message)
+                    pipe.zadd(self.queue_key, {key: when.timestamp()})
+                    await pipe.execute()
+
+    async def cancel(self, key: str) -> None:
+        async with self.redis() as redis:
+            async with redis.pipeline() as pipe:
+                pipe.delete(self.parked_task_key(key))
+                pipe.zrem(self.queue_key, key)
+                await pipe.execute()

pydocket-0.0.1/src/docket/execution.py

@@ -0,0 +1,47 @@
+from datetime import datetime
+from typing import Any, Awaitable, Callable, Self
+
+import cloudpickle
+
+Message = dict[bytes, bytes]
+
+
+class Execution:
+    def __init__(
+        self,
+        function: Callable[..., Awaitable[Any]],
+        args: tuple[Any, ...],
+        kwargs: dict[str, Any],
+        when: datetime,
+        key: str,
+        attempt: int,
+    ) -> None:
+        self.function = function
+        self.args = args
+        self.kwargs = kwargs
+        self.when = when
+        self.key = key
+        self.attempt = attempt
+
+    def as_message(self) -> Message:
+        return {
+            b"key": self.key.encode(),
+            b"when": self.when.isoformat().encode(),
+            b"function": self.function.__name__.encode(),
+            b"args": cloudpickle.dumps(self.args),
+            b"kwargs": cloudpickle.dumps(self.kwargs),
+            b"attempt": str(self.attempt).encode(),
+        }
+
+    @classmethod
+    def from_message(
+        cls, function: Callable[..., Awaitable[Any]], message: Message
+    ) -> Self:
+        return cls(
+            function=function,
+            args=cloudpickle.loads(message[b"args"]),
+            kwargs=cloudpickle.loads(message[b"kwargs"]),
+            when=datetime.fromisoformat(message[b"when"].decode()),
+            key=message[b"key"].decode(),
+            attempt=int(message[b"attempt"].decode()),
+        )