asynk 0.1.0__py3-none-any.whl

asynk/__init__.py

@@ -0,0 +1,78 @@
+"""asynk: an asyncio task framework."""
+
+from importlib.metadata import PackageNotFoundError, version
+
+from asynk.app import App
+from asynk.codec import Codec
+from asynk.codec.json import JsonCodec
+from asynk.envelope import Envelope
+from asynk.errors import (
+    AsynkError,
+    CronError,
+    DecodeError,
+    DeliveryFailedError,
+    EncodeError,
+    InvalidEnvelopeError,
+    NotConfiguredError,
+    TaskError,
+    TaskNotFoundError,
+)
+from asynk.hooks import Next, TaskCall, TaskResult, TaskStatus
+from asynk.options import TaskOptions
+from asynk.results import TaskHandle, TaskState
+from asynk.retry import Backoff, Exponential, RetryPolicy
+from asynk.runner import Runner
+from asynk.scheduled_task import ScheduledTask
+from asynk.scheduler import Scheduler
+from asynk.store import Store
+from asynk.store.memory import MemoryStore
+from asynk.store.null import NullStore
+from asynk.task import Task
+from asynk.transport import Consumer, Delivery, Producer, PublishOptions
+from asynk.worker import Worker
+
+try:
+    __version__ = version("asynk")
+except PackageNotFoundError:
+    # Imported from a source tree without an installed distribution.
+    __version__ = "0+unknown"
+"""The installed asynk version."""
+
+__all__ = [
+    "__version__",
+    "App",
+    "AsynkError",
+    "Backoff",
+    "Codec",
+    "Consumer",
+    "CronError",
+    "DecodeError",
+    "Delivery",
+    "DeliveryFailedError",
+    "Envelope",
+    "InvalidEnvelopeError",
+    "JsonCodec",
+    "EncodeError",
+    "Exponential",
+    "NotConfiguredError",
+    "Next",
+    "Producer",
+    "PublishOptions",
+    "RetryPolicy",
+    "Runner",
+    "Task",
+    "TaskCall",
+    "TaskResult",
+    "TaskStatus",
+    "ScheduledTask",
+    "Scheduler",
+    "Worker",
+    "Store",
+    "MemoryStore",
+    "NullStore",
+    "TaskHandle",
+    "TaskOptions",
+    "TaskState",
+    "TaskError",
+    "TaskNotFoundError",
+]

asynk/_extras.py

@@ -0,0 +1,28 @@
+"""Import-time guard shared by the packages that need an optional extra."""
+
+from __future__ import annotations
+
+import importlib
+
+
+def require_extra(package: str, extra: str) -> None:
+    """Imports `package`, or raises an ImportError that names the extra to install.
+
+    Call it at the top of the package that needs the extra.
+
+    Args:
+        package: Top-level name of the third-party package the extra installs.
+        extra: Name of the extra, as in `pip install 'asynk[<extra>]'`.
+
+    Raises:
+        ImportError: If `package` is not installed. An error raised by one of
+            its own dependencies is passed on unchanged.
+    """
+    try:
+        importlib.import_module(package)
+    except ImportError as exc:
+        if exc.name != package:
+            raise
+        raise ImportError(
+            f"asynk[{extra}] needs the {package} package: pip install 'asynk[{extra}]'"
+        ) from exc

asynk/_redis.py

@@ -0,0 +1,39 @@
+"""The Redis client a Redis store or transport uses, and who closes it."""
+
+from __future__ import annotations
+
+from asynk._extras import require_extra
+
+require_extra("redis", "redis")
+
+from redis.asyncio import Redis  # noqa: E402
+
+
+class RedisConnection:
+    """A Redis client, either given by the caller or created from a URL."""
+
+    def __init__(self, url: str | None, client: Redis | None, *, owner: str) -> None:
+        """Initializes the connection.
+
+        Args:
+            url: Redis URL; the connection creates its own client and closes it.
+            client: Redis client the caller owns and closes.
+            owner: Name of the class using the connection, for error messages.
+
+        Raises:
+            ValueError: If not exactly one of `url` and `client` is given.
+        """
+        if (url is None) == (client is None):
+            raise ValueError(f"{owner} needs exactly one of url and client")
+        if client is None:
+            assert url is not None
+            client = Redis.from_url(url, socket_timeout=5.0, socket_connect_timeout=5.0)
+            self._owns_client = True
+        else:
+            self._owns_client = False
+        self.client = client
+
+    async def close(self) -> None:
+        """Closes the client if the connection created it."""
+        if self._owns_client:
+            await self.client.aclose()

asynk/app.py

@@ -0,0 +1,297 @@
+"""The `App`: tasks, transport, store, codec and hooks."""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Mapping, Sequence
+from dataclasses import replace
+from types import MappingProxyType
+from typing import Any, ParamSpec, TypeVar
+
+from asynk.codec import Codec
+from asynk.codec.json import JsonCodec
+from asynk.hooks import (
+    Hooks,
+    LifecycleHook,
+    Middleware,
+)
+from asynk.options import TaskOptions
+from asynk.results import TaskHandle
+from asynk.retry import RetryPolicy
+from asynk.scheduled_task import Schedule, ScheduledTask, resolve_timezone
+from asynk.store import Store
+from asynk.store.null import NullStore
+from asynk.task import Task
+from asynk.transport import Consumer, Producer
+
+P = ParamSpec("P")
+R = TypeVar("R")
+
+
+class App:
+    """Registry of tasks plus the producer and consumer that move their messages.
+
+    Attributes:
+        name: Name of the app.
+        producer: Used by `Task.delay`; None if the app only consumes.
+        consumer: Used by `Worker`; None if the app only produces.
+        store: Key-value store for task states, task results and scheduler claim keys.
+        codec: Codec that encodes messages, task states and results.
+        tz: Default timezone for scheduled tasks.
+        options: Default options of the app's tasks.
+        hooks: The hooks and middleware registered through the `before_*`, `after_*` and
+            `middleware` decorators.
+    """
+
+    def __init__(
+        self,
+        name: str,
+        *,
+        producer: Producer | None = None,
+        consumer: Consumer | None = None,
+        store: Store | None = None,
+        codec: Codec | None = None,
+        tz: str = "UTC",
+        options: TaskOptions | None = None,
+    ) -> None:
+        """Initializes the app.
+
+        Args:
+            name: Name of the app.
+            producer: Used by `Task.delay`; None if the app only consumes.
+            consumer: Used by `Worker`; None if the app only produces.
+            store: Key-value store for task states, task results and scheduler
+                claim keys; defaults to a `NullStore`.
+            codec: Codec that encodes messages, task states and results;
+                defaults to a `JsonCodec`.
+            tz: Default timezone for scheduled tasks.
+            options: Default options for the app's tasks.
+
+        Raises:
+            TypeError: If `options` is not a `TaskOptions`.
+            ValueError: If the timezone is unknown.
+        """
+        self.name = name
+        self.tz = resolve_timezone(tz, f"app {name!r}")
+        if options is not None and not isinstance(options, TaskOptions):
+            raise TypeError(f"options must be a TaskOptions or None, got {options!r}")
+        self.options = options if options is not None else TaskOptions()
+        self.producer = producer
+        self.consumer = consumer
+        self.store: Store = store if store is not None else NullStore()
+        self.codec: Codec = codec if codec is not None else JsonCodec()
+        self._tasks: dict[str, Task[..., Any]] = {}
+        self._scheduled_tasks: list[ScheduledTask[..., Any]] = []
+        self.hooks = Hooks()
+
+    @property
+    def tasks(self) -> Mapping[str, Task[..., Any]]:
+        """The registered tasks, by name, as a read-only mapping."""
+        return MappingProxyType(self._tasks)
+
+    @property
+    def scheduled_tasks(self) -> Sequence[ScheduledTask[..., Any]]:
+        """The registered scheduled tasks, in registration order."""
+        return tuple(self._scheduled_tasks)
+
+    def task(
+        self,
+        *,
+        name: str | None = None,
+        options: TaskOptions | None = None,
+    ) -> Callable[[Callable[P, R]], Task[P, R]]:
+        """Decorator that registers a function as a task.
+
+        Args:
+            name: Task name; defaults to the app name and the function's qualified name.
+            options: Options of this task; they override the app's defaults.
+
+        Returns:
+            A decorator producing a `Task`.
+
+        Raises:
+            TypeError: If `options` is not a `TaskOptions`.
+            ValueError: If the name is already registered.
+        """
+        policy = self._task_policy(options)
+
+        def decorator(func: Callable[P, R]) -> Task[P, R]:
+            task_name = self._claim_name(name, func)
+            task = Task(self, func, task_name, policy)
+            self._tasks[task_name] = task
+            return task
+
+        return decorator
+
+    def scheduled_task(
+        self,
+        *,
+        cron: str,
+        tz: str | None = None,
+        args: Sequence[object] = (),
+        kwargs: Mapping[str, object] | None = None,
+        name: str | None = None,
+        options: TaskOptions | None = None,
+    ) -> Callable[[Callable[P, R]], ScheduledTask[P, R]]:
+        """Decorator that registers a function as a task published on a cron schedule.
+
+        Args:
+            cron: Five-field cron expression.
+            tz: IANA timezone for the expression; defaults to the app's.
+            args: Positional arguments for each published call.
+            kwargs: Keyword arguments for each published call.
+            name: Task name; defaults to the app name and the function's qualified name.
+            options: Options of this task; they override the app's defaults.
+
+        Returns:
+            A decorator producing a `ScheduledTask`.
+
+        Raises:
+            CronError: If the cron expression is invalid.
+            TypeError: If `options` is not a `TaskOptions`.
+            ValueError: If the timezone is unknown, or the name is already registered.
+        """
+        # Built now so a bad expression fails at the call.
+        schedule = Schedule.build(cron, tz=self.tz, args=args, kwargs=kwargs)
+        policy = self._task_policy(options)
+        if tz is not None and name is not None:
+            resolve_timezone(tz, f"scheduled task {name!r}")
+
+        def decorator(func: Callable[P, R]) -> ScheduledTask[P, R]:
+            task_name = self._claim_name(name, func)
+            zone = self.tz if tz is None else resolve_timezone(tz, f"scheduled task {task_name!r}")
+            task = ScheduledTask(self, func, task_name, policy, replace(schedule, tz=zone))
+            self._tasks[task_name] = task
+            self._scheduled_tasks.append(task)
+            return task
+
+        return decorator
+
+    def _task_policy(self, options: TaskOptions | None) -> RetryPolicy | None:
+        if options is not None and not isinstance(options, TaskOptions):
+            raise TypeError(f"options must be a TaskOptions or None, got {options!r}")
+        return (options or TaskOptions()).with_defaults(self.options).policy
+
+    def _claim_name(self, name: str | None, func: Callable[..., Any]) -> str:
+        task_name = name or f"{self.name}.{func.__qualname__}"
+        if task_name in self._tasks:
+            raise ValueError(f"task {task_name!r} is already registered")
+        return task_name
+
+    def registered_task(self, name: str) -> Task[..., Any]:
+        """Looks up a registered task.
+
+        Args:
+            name: The task name.
+
+        Returns:
+            The task.
+
+        Raises:
+            KeyError: If no task has that name.
+        """
+        return self._tasks[name]
+
+    def get_task(self, task_id: str) -> TaskHandle:
+        """Returns a handle for reading the state and result of a published task.
+
+        Args:
+            task_id: The task's id (`Envelope.id`).
+
+        Returns:
+            A `TaskHandle` reading this app's store with this app's codec.
+        """
+        return TaskHandle(task_id, self.store, self.codec)
+
+    def before_startup(self, hook: LifecycleHook) -> LifecycleHook:
+        """Registers a hook run after the store starts and before serving begins.
+
+        It runs before a worker's consumer or a scheduler's loop starts. Lifecycle
+        hooks may be sync or async and receive the `Runner` they run in, a `Worker`
+        or a `Scheduler`; `runner.role` tells them apart. Registering the same hook
+        twice has no effect.
+
+        Example:
+            Initialize telemetry in every process, but open a database pool only
+            in workers:
+
+                @app.before_startup
+                async def startup(runner: Runner) -> None:
+                    init_sentry()
+                    if runner.role == "worker":
+                        await database.connect()
+
+        Args:
+            hook: The hook to register.
+
+        Returns:
+            The hook, unchanged.
+        """
+        return self.hooks.before_startup.add(hook)
+
+    def after_startup(self, hook: LifecycleHook) -> LifecycleHook:
+        """Registers a hook run once a worker or scheduler has started.
+
+        Same rules as `before_startup`.
+
+        Args:
+            hook: The hook to register.
+
+        Returns:
+            The hook, unchanged.
+        """
+        return self.hooks.after_startup.add(hook)
+
+    def before_shutdown(self, hook: LifecycleHook) -> LifecycleHook:
+        """Registers a hook run when a worker or scheduler begins to shut down.
+
+        A worker runs it when it stops fetching, before draining. Same rules as
+        `before_startup`.
+
+        Args:
+            hook: The hook to register.
+
+        Returns:
+            The hook, unchanged.
+        """
+        return self.hooks.before_shutdown.add(hook)
+
+    def after_shutdown(self, hook: LifecycleHook) -> LifecycleHook:
+        """Registers a hook run after a worker or scheduler has stopped, before the store stops.
+
+        Same rules as `before_startup`.
+
+        Args:
+            hook: The hook to register.
+
+        Returns:
+            The hook, unchanged.
+        """
+        return self.hooks.after_shutdown.add(hook)
+
+    def middleware(self, layer: Middleware) -> Middleware:
+        """Registers a middleware layer around each task.
+
+        A layer is an async function `(call: TaskCall, next: Next) -> TaskResult`.
+        It receives the call and `next`, which runs the rest of the chain and the
+        task, and returns a `TaskResult`: the one from `next` or another one, such
+        as `call.skip()`, the `SKIPPED` result that vetoes the task. The last
+        registered layer is the outermost. Registering the same layer twice has no
+        effect.
+
+        Args:
+            layer: The middleware layer to register.
+
+        Returns:
+            The layer, unchanged.
+
+        Example:
+            Log how long each task takes:
+
+                @app.middleware
+                async def timed(call: TaskCall, next: Next) -> TaskResult:
+                    started = time.monotonic()
+                    result = await next(call)
+                    logger.info("%s took %.3fs", call.envelope.task, time.monotonic() - started)
+                    return result
+        """
+        return self.hooks.middleware.add(layer)

asynk/cli.py

@@ -0,0 +1,162 @@
+"""The `asynk` command line: `worker`, `scheduler` and `health`."""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import importlib
+import logging
+import sys
+from collections.abc import Sequence
+from pathlib import Path
+from typing import Any
+
+from asynk.app import App
+from asynk.errors import AsynkError
+from asynk.logging import configure_logging
+from asynk.probes import check_probe, default_probe_dir
+from asynk.scheduler import Scheduler
+from asynk.worker import Worker
+
+
+class CliError(AsynkError):
+    """Raised for user-facing command-line errors."""
+
+
+def load_app(path: str) -> App:
+    """Imports an app from a `module:attr` path.
+
+    Args:
+        path: Import path of the form `module:attr`.
+
+    Returns:
+        The app.
+
+    Raises:
+        CliError: If the path is malformed, cannot be imported, or is not an `App`.
+    """
+    module_name, sep, attr = path.partition(":")
+    if not sep or not module_name or not attr:
+        raise CliError(f"invalid app path {path!r}: expected 'module:attr'")
+    try:
+        module = importlib.import_module(module_name)
+    except Exception as exc:
+        raise CliError(f"cannot import {module_name!r}: {exc}") from exc
+    try:
+        obj = getattr(module, attr)
+    except AttributeError:
+        raise CliError(f"module {module_name!r} has no attribute {attr!r}") from None
+    if not isinstance(obj, App):
+        raise CliError(f"{path!r} is a {type(obj).__name__}, not an asynk App")
+    return obj
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="asynk")
+    commands = parser.add_subparsers(dest="command", required=True)
+
+    worker = commands.add_parser("worker", help="run a worker for an app")
+    worker.add_argument("app", metavar="MODULE:APP")
+    worker.add_argument("--concurrency", type=int)
+    worker.add_argument("--shutdown-timeout", type=float)
+    worker.add_argument("--heartbeat-interval", type=float)
+    worker.add_argument("--probe-dir", type=Path, default=default_probe_dir())
+    worker.add_argument(
+        "--log-level",
+        default="INFO",
+        type=str.upper,
+        choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
+    )
+
+    scheduler = commands.add_parser("scheduler", help="publish an app's cron scheduled tasks")
+    scheduler.add_argument("app", metavar="MODULE:APP")
+    scheduler.add_argument("--heartbeat-interval", type=float)
+    scheduler.add_argument("--probe-dir", type=Path, default=default_probe_dir())
+    scheduler.add_argument(
+        "--log-level",
+        default="INFO",
+        type=str.upper,
+        choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
+    )
+
+    health = commands.add_parser("health", help="check a worker or scheduler probe file")
+    health.add_argument("probe", choices=["liveness", "readiness"])
+    health.add_argument("--max-age", type=float, required=True)
+    health.add_argument("--probe-dir", type=Path, default=default_probe_dir())
+    return parser
+
+
+def _run_worker(args: argparse.Namespace) -> int:
+    configure_logging(args.log_level)
+    app = load_app(args.app)
+    options: dict[str, Any] = {
+        key: value
+        for key in ("concurrency", "shutdown_timeout", "heartbeat_interval")
+        if (value := getattr(args, key)) is not None
+    }
+    try:
+        worker = Worker(app, probe_dir=args.probe_dir, **options)
+    except (AsynkError, ValueError) as exc:
+        raise CliError(str(exc)) from exc
+    logging.getLogger("asynk").info(
+        "starting worker for app %r: concurrency=%d shutdown_timeout=%s probe_dir=%s",
+        app.name,
+        worker.concurrency,
+        worker.shutdown_timeout,
+        worker.probe_dir,
+    )
+    asyncio.run(worker.run_with_signals())
+    return 0
+
+
+def _run_scheduler(args: argparse.Namespace) -> int:
+    configure_logging(args.log_level)
+    app = load_app(args.app)
+    options: dict[str, Any] = {}
+    if args.heartbeat_interval is not None:
+        options["heartbeat_interval"] = args.heartbeat_interval
+    try:
+        scheduler = Scheduler(app, probe_dir=args.probe_dir, **options)
+    except (AsynkError, ValueError) as exc:
+        raise CliError(str(exc)) from exc
+    logging.getLogger("asynk").info(
+        "starting scheduler for app %r: scheduled_tasks=[%s] probe_dir=%s",
+        app.name,
+        ", ".join(
+            f"{t.name} ({t.schedule.expression}, {t.schedule.tz_name})" for t in app.scheduled_tasks
+        ),
+        scheduler.probe_dir,
+    )
+    asyncio.run(scheduler.run_with_signals())
+    return 0
+
+
+def _run_health(args: argparse.Namespace) -> int:
+    healthy, message = check_probe(args.probe_dir / args.probe, args.max_age)
+    print(message)
+    return 0 if healthy else 1
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    """Runs the `asynk` command line.
+
+    Args:
+        argv: Arguments to parse; defaults to `sys.argv[1:]`.
+
+    Returns:
+        The process exit code.
+    """
+    args = _build_parser().parse_args(argv)
+    try:
+        if args.command == "worker":
+            return _run_worker(args)
+        if args.command == "scheduler":
+            return _run_scheduler(args)
+        return _run_health(args)
+    except CliError as exc:
+        print(f"asynk: {exc}", file=sys.stderr)
+        return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

asynk/codec/__init__.py

@@ -0,0 +1,48 @@
+"""The `Codec` protocol."""
+
+from __future__ import annotations
+
+from typing import Protocol
+
+
+class Codec(Protocol):
+    """Converts values to and from bytes."""
+
+    def validate(self, value: object) -> None:
+        """Checks that `encode` would accept a value, without encoding it.
+
+        Args:
+            value: The value to check.
+
+        Raises:
+            EncodeError: If this codec cannot encode `value`.
+        """
+        ...
+
+    def encode(self, value: object) -> bytes:
+        """Serializes a value.
+
+        Args:
+            value: The value to serialize.
+
+        Returns:
+            The encoded value.
+
+        Raises:
+            EncodeError: If this codec cannot encode `value`.
+        """
+        ...
+
+    def decode(self, data: bytes) -> object:
+        """Deserializes a value.
+
+        Args:
+            data: Bytes previously produced by `encode`.
+
+        Returns:
+            The decoded value.
+
+        Raises:
+            DecodeError: If `data` is not a valid encoded value.
+        """
+        ...