taskscheduler-client 0.6.0__py3-none-any.whl

taskscheduler/__init__.py

@@ -0,0 +1,141 @@
+"""TaskScheduler — async Python SDK for durable, distributed background jobs.
+
+A Python service talks to the same PostgreSQL database and RabbitMQ broker as the Kotlin
+side, so its jobs show up on the same dashboard, obey the same retry and cancellation
+semantics, and survive restarts the same way. Postgres holds the state; RabbitMQ only
+delivers 16-byte job ids.
+
+Producing work::
+
+    from dataclasses import dataclass
+    from taskscheduler import Scheduler, SchedulerConfig, job_type
+
+    @job_type
+    @dataclass
+    class SendInvoice:
+        order_id: int
+
+    async with Scheduler(SchedulerConfig(dsn="postgresql://...")) as scheduler:
+        await scheduler.enqueue(SendInvoice(order_id=42))
+
+Consuming it::
+
+    from taskscheduler import HandlerRegistry, JobContext, WorkerPool, WorkerConfig
+
+    registry = HandlerRegistry()
+
+    @registry.handler(SendInvoice)
+    async def send_invoice(ctx: JobContext, job: SendInvoice) -> None:
+        await billing.send(job.order_id)
+
+    worker = WorkerPool(
+        scheduler_config=SchedulerConfig(dsn="postgresql://..."),
+        worker_config=WorkerConfig(node_id="billing-1").queue("billing", concurrency=8),
+        rabbit_config=RabbitConfig(url="amqp://...", queues=["billing"]),
+        registry=registry,
+    )
+    await worker.start()
+
+The delivery guarantee is at-least-once: a job can run twice after a lease expiry or a
+broker redelivery. ``ctx.job_id`` is stable across every attempt — use it as the
+idempotency key for anything with side effects.
+
+Requires a running Kotlin ``scheduler-infra`` process: it owns the schema migrations and
+the background loops (outbox publisher, recurring cron, orphan recovery, retention).
+"""
+
+from __future__ import annotations
+
+from .config import (
+    QueueConfig,
+    RabbitConfig,
+    SchedulerConfig,
+    WorkerConfig,
+    default_node_id,
+    dsn_from_jdbc,
+)
+from .context import JobContext, ProgressBar
+from .cron import next_trigger_at, validate_cron
+from .errors import (
+    ConfigurationError,
+    HandlerNotRegisteredError,
+    JobCancellationError,
+    NonRetriableError,
+    PayloadDecodeError,
+    SchedulerError,
+    SchemaMismatchError,
+)
+from .job import HandlerRegistry, JobHandler, job_type
+from .models import (
+    ConcurrencyPolicy,
+    JobRow,
+    JobState,
+    MisfirePolicy,
+    OnFailure,
+    RecurringOverlap,
+)
+from .retry import (
+    FULL_JITTER,
+    NO_JITTER,
+    ExponentialBackoff,
+    FixedDelay,
+    Jitter,
+    NoRetry,
+    RetryPolicy,
+    equal_jitter,
+)
+from .scheduler import Scheduler
+from .storage import Storage
+from .transport import RabbitTransport
+from .worker import WorkerPool
+
+__version__ = "0.6.0"
+
+__all__ = [
+    "__version__",
+    # entry points
+    "Scheduler",
+    "WorkerPool",
+    "HandlerRegistry",
+    "JobHandler",
+    "JobContext",
+    "ProgressBar",
+    "job_type",
+    # configuration
+    "SchedulerConfig",
+    "WorkerConfig",
+    "QueueConfig",
+    "RabbitConfig",
+    "default_node_id",
+    "dsn_from_jdbc",
+    # retry
+    "RetryPolicy",
+    "NoRetry",
+    "FixedDelay",
+    "ExponentialBackoff",
+    "Jitter",
+    "NO_JITTER",
+    "FULL_JITTER",
+    "equal_jitter",
+    # enums and rows
+    "JobState",
+    "JobRow",
+    "OnFailure",
+    "ConcurrencyPolicy",
+    "MisfirePolicy",
+    "RecurringOverlap",
+    # cron
+    "next_trigger_at",
+    "validate_cron",
+    # errors
+    "SchedulerError",
+    "ConfigurationError",
+    "SchemaMismatchError",
+    "PayloadDecodeError",
+    "HandlerNotRegisteredError",
+    "JobCancellationError",
+    "NonRetriableError",
+    # lower level
+    "Storage",
+    "RabbitTransport",
+]

taskscheduler/config.py

@@ -0,0 +1,190 @@
+"""Configuration objects.
+
+Three groups, matching the Kotlin Koin modules: core scheduler defaults
+(``schedulerCoreModule``), the RabbitMQ connection (``schedulerRabbitModule``), and the
+worker pool (``schedulerWorkerModule``).
+
+This client never runs migrations. The Kotlin ``scheduler-infra`` process owns the schema
+and the background loops (outbox publisher, recurring cron, retention, orphan recovery);
+the Python service is a peer of a user-app worker, not a replacement for infra.
+"""
+
+from __future__ import annotations
+
+import os
+import socket
+from dataclasses import dataclass, field
+
+from .errors import ConfigurationError
+from .retry import ExponentialBackoff, RetryPolicy
+
+__all__ = [
+    "SchedulerConfig",
+    "RabbitConfig",
+    "WorkerConfig",
+    "QueueConfig",
+    "dsn_from_jdbc",
+    "default_node_id",
+]
+
+#: The Flyway version this client's SQL was written against. Startup fails if the database
+#: is older, because the missing columns would surface as confusing runtime errors instead.
+REQUIRED_SCHEMA_VERSION = 8
+
+
+def default_node_id() -> str:
+    """``HOSTNAME`` / ``COMPUTERNAME`` / the host name, matching Kotlin's ``defaultNodeId``."""
+    for var in ("HOSTNAME", "COMPUTERNAME"):
+        value = os.environ.get(var)
+        if value:
+            return value
+    try:
+        return socket.gethostname()
+    except OSError:
+        return "worker"
+
+
+def dsn_from_jdbc(jdbc_url: str, user: str, password: str) -> str:
+    """Turn the ``jdbc:postgresql://host:port/db`` URL used by the Kotlin side into a libpq DSN.
+
+    Handy when both services read the same ``POSTGRES_URL`` environment variable.
+    """
+    prefix = "jdbc:postgresql://"
+    if not jdbc_url.startswith(prefix):
+        raise ConfigurationError(f"expected a URL starting with {prefix!r}, got {jdbc_url!r}")
+    return f"postgresql://{user}:{password}@{jdbc_url[len(prefix):]}"
+
+
+@dataclass(slots=True)
+class SchedulerConfig:
+    """Connection to Postgres plus the defaults applied to every enqueue."""
+
+    dsn: str
+    node_id: str = field(default_factory=default_node_id)
+    default_queue: str = "default"
+    default_max_attempts: int = 3
+    default_timeout_seconds: int = 300
+    default_retry_policy: RetryPolicy | None = None
+    #: Jobs further out than this are stored as SCHEDULED without an outbox row; the infra
+    #: fast-forward loop promotes them when they come within the window. Must match the
+    #: Kotlin ``fastForwardWindow`` (24h) or far-future jobs fire at the wrong time.
+    fast_forward_window_seconds: int = 24 * 60 * 60
+    min_pool_size: int = 1
+    max_pool_size: int = 10
+    #: Emit ``scheduler_events`` NOTIFY payloads so the dashboard updates live.
+    emit_events: bool = True
+    #: Verify the Flyway schema version at startup.
+    check_schema: bool = True
+
+    def __post_init__(self) -> None:
+        if not self.dsn:
+            raise ConfigurationError("SchedulerConfig.dsn is required")
+        if self.default_max_attempts < 1:
+            raise ConfigurationError("default_max_attempts must be >= 1")
+        if self.default_timeout_seconds < 1:
+            raise ConfigurationError("default_timeout_seconds must be >= 1")
+        if self.max_pool_size < self.min_pool_size:
+            raise ConfigurationError("max_pool_size must be >= min_pool_size")
+        if self.default_retry_policy is None:
+            self.default_retry_policy = ExponentialBackoff(max_attempts=self.default_max_attempts)
+
+
+@dataclass(slots=True)
+class RabbitConfig:
+    """AMQP connection settings. Only the worker needs these — producers write to Postgres."""
+
+    url: str = "amqp://guest:guest@localhost:5672/"
+    #: Logical queue names to declare. Must be a superset of the names the worker consumes,
+    #: and must match what the Kotlin side declares, or the bindings will not line up.
+    queues: list[str] = field(default_factory=lambda: ["default"])
+    prefetch: int = 10
+    reconnect_delay_seconds: float = 5.0
+    #: Declare the exchanges and queues on connect. Leave on unless a stricter broker policy
+    #: forbids clients from declaring topology.
+    declare_topology: bool = True
+
+    def __post_init__(self) -> None:
+        if not self.url:
+            raise ConfigurationError("RabbitConfig.url is required")
+        if self.prefetch < 1:
+            raise ConfigurationError("prefetch must be >= 1")
+
+
+@dataclass(slots=True)
+class QueueConfig:
+    """One consumed queue and how much of it runs at once."""
+
+    name: str
+    concurrency: int = 10
+    prefetch: int | None = None
+    default_priority: int = 0
+
+    def __post_init__(self) -> None:
+        if self.concurrency < 1:
+            raise ConfigurationError(f"queue {self.name!r}: concurrency must be >= 1")
+        if self.prefetch is None:
+            self.prefetch = self.concurrency
+        if self.prefetch < 1:
+            raise ConfigurationError(f"queue {self.name!r}: prefetch must be >= 1")
+        if not 0 <= self.default_priority <= 10:
+            raise ConfigurationError(f"queue {self.name!r}: default_priority must be within 0..10")
+
+
+@dataclass(slots=True)
+class WorkerConfig:
+    """Worker identity, lease timings and the queues it serves."""
+
+    node_id: str = field(default_factory=default_node_id)
+    node_tags: list[str] = field(default_factory=list)
+    queues: list[QueueConfig] = field(default_factory=list)
+    #: How often the lease on in-flight jobs is extended, and how long each extension lasts.
+    #: Keep ``heartbeat <= lock_duration / 3`` so two missed ticks still leave the lock valid
+    #: (DESIGN.md 13.4) — otherwise infra's orphan recovery re-runs a job that is still going.
+    heartbeat_interval_seconds: float = 30.0
+    lock_duration_seconds: float = 90.0
+    #: Grace period for in-flight jobs to finish after ``stop()`` is called.
+    shutdown_timeout_seconds: float = 30.0
+    #: How long a cancelled job is given to stop cooperatively before its task is cancelled
+    #: outright. Handlers that never check ``ctx.is_cancellation_requested()`` need this;
+    #: ones that do stop sooner.
+    cancel_grace_seconds: float = 30.0
+    #: How long a job whose type is paused waits before redelivery. Matches Kotlin's
+    #: ``PAUSE_REDELIVER_DELAY``.
+    pause_redeliver_delay_seconds: float = 60.0
+
+    def queue(
+        self,
+        name: str,
+        *,
+        concurrency: int = 10,
+        prefetch: int | None = None,
+        default_priority: int = 0,
+    ) -> WorkerConfig:
+        """Add a consumed queue. Chainable."""
+        self.queues.append(
+            QueueConfig(
+                name=name,
+                concurrency=concurrency,
+                prefetch=prefetch,
+                default_priority=default_priority,
+            )
+        )
+        return self
+
+    def validate(self) -> None:
+        if not self.queues:
+            raise ConfigurationError(
+                "WorkerConfig has no queues — call worker_config.queue('default') at least once"
+            )
+        names = [q.name for q in self.queues]
+        duplicates = {n for n in names if names.count(n) > 1}
+        if duplicates:
+            raise ConfigurationError(f"duplicate queue names: {', '.join(sorted(duplicates))}")
+        if self.heartbeat_interval_seconds <= 0 or self.lock_duration_seconds <= 0:
+            raise ConfigurationError("heartbeat_interval and lock_duration must be positive")
+        if self.heartbeat_interval_seconds > self.lock_duration_seconds / 3:
+            raise ConfigurationError(
+                f"heartbeat_interval ({self.heartbeat_interval_seconds}s) must be at most a third "
+                f"of lock_duration ({self.lock_duration_seconds}s) — otherwise a single missed "
+                f"tick lets another node steal a job that is still running"
+            )

taskscheduler/context.py

@@ -0,0 +1,185 @@
+"""Per-execution context handed to a handler.
+
+``job_id`` is stable across every attempt of a job — retries, orphan recovery and broker
+redelivery all reuse it. That makes it the natural idempotency key for external calls
+(``Idempotency-Key: <job_id>`` on an HTTP request, a unique column in your own tables),
+which matters because the delivery guarantee is at-least-once: a job can run twice, and
+only the handler can make that harmless.
+"""
+
+from __future__ import annotations
+
+import time
+import uuid
+from datetime import datetime
+from typing import TYPE_CHECKING
+
+from .models import JobRow
+
+if TYPE_CHECKING:
+    from .worker import WorkerPool
+
+__all__ = ["JobContext", "ProgressBar"]
+
+#: Progress writes are collapsed to at most one per second per job. Calling more often is
+#: safe — only the latest value in each window reaches the database.
+_PROGRESS_THROTTLE_SECONDS = 1.0
+
+
+class ProgressBar:
+    """Counting progress over a known number of work items.
+
+        bar = ctx.progress_bar(len(orders))
+        for order in orders:
+            try:
+                await charge(order)
+                await bar.succeeded()
+            except PaymentDeclined:
+                await bar.failed()
+
+    The fraction is derived from the counters and persisted under the same throttle as
+    :meth:`JobContext.update_progress`. Creating a bar costs nothing — the first write
+    happens on the first increment.
+    """
+
+    __slots__ = ("_ctx", "_total", "_succeeded", "_failed")
+
+    def __init__(self, ctx: JobContext, total: int) -> None:
+        self._ctx = ctx
+        self._total = max(0, total)
+        self._succeeded = 0
+        self._failed = 0
+
+    @property
+    def total(self) -> int:
+        return self._total
+
+    @property
+    def succeeded_count(self) -> int:
+        return self._succeeded
+
+    @property
+    def failed_count(self) -> int:
+        return self._failed
+
+    @property
+    def processed(self) -> int:
+        return self._succeeded + self._failed
+
+    @property
+    def fraction(self) -> float:
+        if self._total <= 0:
+            return 0.0
+        return min(1.0, self.processed / self._total)
+
+    async def succeeded(self, count: int = 1, message: str | None = None) -> None:
+        self._succeeded += count
+        await self._flush(message)
+
+    async def failed(self, count: int = 1, message: str | None = None) -> None:
+        self._failed += count
+        await self._flush(message)
+
+    async def _flush(self, message: str | None) -> None:
+        await self._ctx._write_progress(
+            progress=self.fraction,
+            message=message,
+            succeeded=self._succeeded,
+            failed=self._failed,
+            total=self._total,
+        )
+
+
+class JobContext:
+    """What a handler knows about the execution it is in."""
+
+    __slots__ = ("_row", "_pool", "_last_progress_at", "_cancel_flag")
+
+    def __init__(self, row: JobRow, pool: WorkerPool) -> None:
+        self._row = row
+        self._pool = pool
+        self._last_progress_at = 0.0
+        self._cancel_flag = False
+
+    @property
+    def job_id(self) -> uuid.UUID:
+        return self._row.id
+
+    @property
+    def attempt(self) -> int:
+        """1-based: the first execution is attempt 1."""
+        return self._row.attempts
+
+    @property
+    def max_attempts(self) -> int:
+        return self._row.max_attempts
+
+    @property
+    def queue(self) -> str:
+        return self._row.queue
+
+    @property
+    def payload_type(self) -> str:
+        return self._row.payload_type
+
+    @property
+    def enqueued_at(self) -> datetime | None:
+        return self._row.created_at
+
+    @property
+    def is_last_attempt(self) -> bool:
+        return self._row.attempts >= self._row.max_attempts
+
+    def progress_bar(self, total: int) -> ProgressBar:
+        return ProgressBar(self, total)
+
+    async def update_progress(self, progress: float, message: str | None = None) -> None:
+        """Report a 0.0..1.0 fraction. Throttled to one write per second."""
+        await self._write_progress(progress=max(0.0, min(1.0, progress)), message=message)
+
+    async def is_cancellation_requested(self) -> bool:
+        """Whether someone asked this job to stop.
+
+        Poll it inside long loops and raise :class:`JobCancellationError` to end as CANCELLED::
+
+            if await ctx.is_cancellation_requested():
+                raise JobCancellationError()
+
+        Returning normally after a cancel request is also fine — the job lands in SUCCEEDED
+        because the work actually finished.
+        """
+        if self._cancel_flag:
+            return True
+        self._cancel_flag = await self._pool._check_cancelled(self._row.id)
+        return self._cancel_flag
+
+    def _mark_cancelled(self) -> None:
+        """Set by the ``job_cancel`` listener so the next poll answers without a query."""
+        self._cancel_flag = True
+
+    async def _write_progress(
+        self,
+        *,
+        progress: float,
+        message: str | None,
+        succeeded: int | None = None,
+        failed: int | None = None,
+        total: int | None = None,
+    ) -> None:
+        now = time.monotonic()
+        complete = total is not None and succeeded is not None and failed is not None and (
+            succeeded + failed >= total
+        )
+        # Always let the final tick through, so a bar that finishes inside the throttle
+        # window doesn't leave the dashboard stuck at 90%.
+        if not complete and now - self._last_progress_at < _PROGRESS_THROTTLE_SECONDS:
+            return
+        self._last_progress_at = now
+        await self._pool._report_progress(
+            job_id=self._row.id,
+            progress=progress,
+            message=message,
+            succeeded=succeeded,
+            failed=failed,
+            total=total,
+        )

taskscheduler/cron.py

@@ -0,0 +1,76 @@
+"""Cron parsing for recurring definitions.
+
+Only used to compute the *first* ``next_trigger_at`` when a definition is registered. Every
+subsequent occurrence is computed by the Kotlin infra leader when it fires the job, so this
+never drives execution — but it must agree with the Kotlin dialect or the first run lands at
+the wrong time.
+
+Kotlin (``core/backend/.../cron/CronExpr.kt``) picks its parser by field count:
+
+* **5 fields** — classic UNIX ``m h dom month dow``.
+* **6 fields** — the same with a *leading* seconds field (Spring 5.3 dialect, not Quartz:
+  day-of-week keeps UNIX numbering and there is no ``?`` placeholder).
+
+``croniter`` defaults to seconds *last* for 6-field expressions, so ``second_at_beginning``
+is required here.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
+
+from croniter import CroniterBadCronError, croniter
+
+from .errors import ConfigurationError
+
+__all__ = ["next_trigger_at", "validate_cron"]
+
+
+def _resolve_zone(timezone_name: str | None) -> timezone | ZoneInfo:
+    if not timezone_name:
+        return timezone.utc
+    try:
+        return ZoneInfo(timezone_name)
+    except (ZoneInfoNotFoundError, ValueError) as exc:
+        raise ConfigurationError(
+            f"unknown IANA timezone {timezone_name!r} — use names like 'Europe/Berlin'"
+        ) from exc
+
+
+def _build(cron: str, base: datetime) -> croniter:
+    fields = cron.split()
+    if len(fields) not in (5, 6):
+        raise ConfigurationError(
+            f"cron {cron!r} has {len(fields)} fields — expected 5 (m h dom mon dow) "
+            f"or 6 (with leading seconds)"
+        )
+    try:
+        return croniter(cron, base, second_at_beginning=True)
+    except (CroniterBadCronError, ValueError) as exc:
+        raise ConfigurationError(f"invalid cron expression {cron!r}: {exc}") from exc
+
+
+def validate_cron(cron: str, timezone_name: str | None = None) -> None:
+    """Raise :class:`ConfigurationError` if the expression will not parse."""
+    next_trigger_at(cron, timezone_name=timezone_name)
+
+
+def next_trigger_at(
+    cron: str, after: datetime | None = None, timezone_name: str | None = None
+) -> datetime:
+    """First occurrence strictly after ``after``, returned as an aware UTC datetime.
+
+    ``timezone_name`` is an IANA name; ``None`` means UTC, matching a NULL
+    ``recurring_job.timezone``. Evaluating in the target zone is what makes "every day at
+    09:00 Europe/Berlin" hold across a DST change.
+    """
+    zone = _resolve_zone(timezone_name)
+    base = after or datetime.now(timezone.utc)
+    if base.tzinfo is None:
+        base = base.replace(tzinfo=timezone.utc)
+    local_base = base.astimezone(zone)
+    nxt: datetime = _build(cron, local_base).get_next(datetime)
+    if nxt.tzinfo is None:
+        nxt = nxt.replace(tzinfo=zone)
+    return nxt.astimezone(timezone.utc)

taskscheduler/errors.py

@@ -0,0 +1,56 @@
+"""Exceptions raised by the TaskScheduler Python SDK."""
+
+from __future__ import annotations
+
+__all__ = [
+    "SchedulerError",
+    "ConfigurationError",
+    "SchemaMismatchError",
+    "PayloadDecodeError",
+    "HandlerNotRegisteredError",
+    "JobCancellationError",
+    "NonRetriableError",
+]
+
+
+class SchedulerError(Exception):
+    """Base class for every error this SDK raises."""
+
+
+class ConfigurationError(SchedulerError):
+    """Invalid or incomplete configuration — raised eagerly at construction time."""
+
+
+class SchemaMismatchError(SchedulerError):
+    """The database schema is older than this client expects.
+
+    The Kotlin ``scheduler-infra`` process owns the Flyway schema; this client only
+    verifies it. Deploy a matching infra version before starting the Python service.
+    """
+
+
+class PayloadDecodeError(SchedulerError):
+    """``payload_json`` could not be turned back into a payload object.
+
+    Treated as terminal: the bytes will not change, so retrying is pointless. Mirrors the
+    Kotlin worker's ``SerializationException`` handling (DESIGN.md 22.9).
+    """
+
+
+class HandlerNotRegisteredError(SchedulerError):
+    """No handler is registered for a job's ``payload_type`` on this node.
+
+    Also terminal — the job is marked FAILED so the dashboard surfaces the
+    misconfiguration instead of the message bouncing between nodes.
+    """
+
+
+class JobCancellationError(SchedulerError):
+    """Raise from a handler to end the job as CANCELLED rather than FAILED.
+
+    No retry is scheduled and ``on_final_failure`` is not invoked.
+    """
+
+
+class NonRetriableError(SchedulerError):
+    """Raise from a handler to fail terminally, skipping the remaining attempt budget."""