smallage 0.2.0__py3-none-any.whl

smallage/__init__.py

@@ -0,0 +1,90 @@
+"""Distributed task queue on Redis Streams with first-class Litestar integration.
+
+Delivery is at-least-once: handlers must be idempotent.
+"""
+
+from importlib.metadata import version as _version
+
+from smallage.core import (
+    BrokerHandler,
+    Codec,
+    CollectingEnqueuer,
+    ConfigurationError,
+    CronJob,
+    DeferredEnqueuer,
+    EagerEnqueuer,
+    Enqueuer,
+    Envelope,
+    FilePayloadStore,
+    JsonCodec,
+    MalformedEnvelope,
+    PayloadMissing,
+    PayloadStore,
+    PayloadTooLarge,
+    Pending,
+    Record,
+    RedisResultStore,
+    RedisScheduler,
+    RedisStreamsTransport,
+    ResultStore,
+    RetryPolicy,
+    Scheduler,
+    SmallageError,
+    StreamTransport,
+    TaskHandler,
+    TaskResult,
+    UnknownTask,
+    WorkerConfig,
+    WorkerStats,
+    current_enqueuer,
+    dlq_key,
+    from_fields,
+    run,
+    run_with_signals,
+    to_fields,
+    worker_running,
+)
+
+__version__ = _version("smallage")
+"""Read from the installed metadata: pyproject is the only place it is written."""
+
+__all__ = [
+    "BrokerHandler",
+    "Codec",
+    "CollectingEnqueuer",
+    "ConfigurationError",
+    "CronJob",
+    "DeferredEnqueuer",
+    "EagerEnqueuer",
+    "Enqueuer",
+    "Envelope",
+    "FilePayloadStore",
+    "JsonCodec",
+    "MalformedEnvelope",
+    "PayloadMissing",
+    "PayloadStore",
+    "PayloadTooLarge",
+    "Pending",
+    "Record",
+    "RedisResultStore",
+    "RedisScheduler",
+    "RedisStreamsTransport",
+    "ResultStore",
+    "RetryPolicy",
+    "Scheduler",
+    "SmallageError",
+    "StreamTransport",
+    "TaskHandler",
+    "TaskResult",
+    "UnknownTask",
+    "WorkerConfig",
+    "WorkerStats",
+    "__version__",
+    "current_enqueuer",
+    "dlq_key",
+    "from_fields",
+    "run",
+    "run_with_signals",
+    "to_fields",
+    "worker_running",
+]

smallage/core/__init__.py

@@ -0,0 +1,87 @@
+"""Transport, worker and scheduling core. Importing Litestar from here is forbidden."""
+
+from smallage.core.cron import CronJob
+from smallage.core.deferred import DeferredEnqueuer, current_enqueuer
+from smallage.core.envelope import (
+    ENVELOPE_VERSION,
+    Envelope,
+    JsonCodec,
+    Pending,
+    Record,
+    TaskResult,
+    from_fields,
+    to_fields,
+)
+from smallage.core.errors import (
+    ConfigurationError,
+    MalformedEnvelope,
+    PayloadTooLarge,
+    SmallageError,
+)
+from smallage.core.keys import dlq_key
+from smallage.core.payloads import FilePayloadStore, PayloadMissing
+from smallage.core.protocols import (
+    BrokerHandler,
+    Codec,
+    Enqueuer,
+    PayloadStore,
+    ResultStore,
+    Scheduler,
+    Sleeper,
+    StreamTransport,
+    TaskHandler,
+)
+from smallage.core.results import RedisResultStore
+from smallage.core.retry import RetryPolicy
+from smallage.core.scheduler import RedisScheduler
+from smallage.core.stats import WorkerStats
+from smallage.core.testing import (
+    CollectingEnqueuer,
+    EagerEnqueuer,
+    UnknownTask,
+    worker_running,
+)
+from smallage.core.transport import RedisStreamsTransport
+from smallage.core.worker import WorkerConfig, run, run_with_signals
+
+__all__ = [
+    "ENVELOPE_VERSION",
+    "BrokerHandler",
+    "Codec",
+    "CollectingEnqueuer",
+    "ConfigurationError",
+    "CronJob",
+    "DeferredEnqueuer",
+    "EagerEnqueuer",
+    "Enqueuer",
+    "Envelope",
+    "FilePayloadStore",
+    "JsonCodec",
+    "MalformedEnvelope",
+    "PayloadMissing",
+    "PayloadStore",
+    "PayloadTooLarge",
+    "Pending",
+    "Record",
+    "RedisResultStore",
+    "RedisScheduler",
+    "RedisStreamsTransport",
+    "ResultStore",
+    "RetryPolicy",
+    "Scheduler",
+    "Sleeper",
+    "SmallageError",
+    "StreamTransport",
+    "TaskHandler",
+    "TaskResult",
+    "UnknownTask",
+    "WorkerConfig",
+    "WorkerStats",
+    "current_enqueuer",
+    "dlq_key",
+    "from_fields",
+    "run",
+    "run_with_signals",
+    "to_fields",
+    "worker_running",
+]

smallage/core/clients.py

@@ -0,0 +1,18 @@
+"""Reading settings off a redis-py client, whatever shape it is.
+
+A standalone client keeps them behind a connection pool and a cluster client
+keeps them on itself. Both are accepted: the hash-tagged key schema exists for
+cluster mode and would be pointless if a cluster client could not be handed in.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+def connection_kwarg(client: Any, name: str) -> Any:
+    pool = getattr(client, "connection_pool", None)
+    if pool is not None:
+        kwargs: dict[str, Any] = pool.connection_kwargs
+        return kwargs.get(name)
+    return getattr(client, "connection_kwargs", {}).get(name)

smallage/core/cron.py

@@ -0,0 +1,87 @@
+"""Cron expressions, resolved in a real time zone.
+
+Occurrences are always strictly after the moment asked about, so a job cannot
+fire twice for the same minute no matter how often the leader looks.
+"""
+
+from dataclasses import dataclass
+from datetime import datetime
+from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
+
+from cronsim import CronSim, CronSimError
+
+from smallage.core.envelope import Envelope
+from smallage.core.errors import ConfigurationError
+
+
+@dataclass(frozen=True, slots=True)
+class CronJob:
+    name: str
+    expression: str
+    task: str
+    payload: bytes = b"{}"
+    """Encoded arguments. An empty object, not null: it decodes into the task's
+    argument struct, and a task taking nothing still has one."""
+    queue: str = "default"
+    timezone: str = "UTC"
+
+    def __post_init__(self) -> None:
+        if not self.name or ":" in self.name:
+            raise ConfigurationError(
+                f"cron job name must be non-empty and free of ':', got {self.name!r}"
+            )
+        try:
+            zone = ZoneInfo(self.timezone)
+        except (ZoneInfoNotFoundError, ValueError) as exc:
+            raise ConfigurationError(
+                f"cron job {self.name!r} has an unknown timezone {self.timezone!r}"
+            ) from exc
+        try:
+            CronSim(self.expression, datetime.now(zone))
+        except CronSimError as exc:
+            raise ConfigurationError(
+                f"cron job {self.name!r} has an invalid expression "
+                f"{self.expression!r}: {exc}"
+            ) from exc
+
+
+def next_fire_ms(job: CronJob, after_ms: int) -> int | None:
+    """Next occurrence strictly after ``after_ms``, or None if there is none.
+
+    Resolving in the job's own zone is what makes daylight saving behave: a time
+    that does not exist on the spring-forward day moves to the following instant
+    rather than being skipped, and one that happens twice in autumn fires once.
+    """
+    zone = ZoneInfo(job.timezone)
+    after = datetime.fromtimestamp(after_ms / 1000, zone)
+    try:
+        occurrence = next(CronSim(job.expression, after))
+    except StopIteration:
+        # Expressions that can never match are refused when the job is built, so
+        # this is unreachable in practice. It stays because an escaping
+        # StopIteration inside a coroutine surfaces as a bare RuntimeError.
+        return None
+    return int(occurrence.timestamp() * 1000)
+
+
+def occurrence_id(job: CronJob, fire_ms: int) -> str:
+    """Identify an occurrence by job and instant.
+
+    Two leaders computing the same occurrence produce the same id, so scheduling
+    it twice is a no-op rather than a duplicate job.
+    """
+    return f"cron:{job.name}:{fire_ms}"
+
+
+def occurrence_envelope(job: CronJob, fire_ms: int) -> Envelope:
+    """Build the entry for one occurrence.
+
+    ``enqueued_at`` is the instant the job was due, not the instant it reached
+    the stream, so a run delayed by an outage can be recognised as late.
+    """
+    return Envelope(
+        id=occurrence_id(job, fire_ms),
+        task=job.task,
+        payload=job.payload,
+        enqueued_at=fire_ms,
+    )

smallage/core/deferred.py

@@ -0,0 +1,81 @@
+"""Publishing jobs only once the transaction that justified them has committed.
+
+A handler that writes a row and enqueues a job before ``COMMIT`` has a race it
+cannot win: the worker is fast enough to read the row before it exists, and if
+the transaction rolls back the job has already run against data that was never
+written.
+
+Buffering here and flushing from an ``after_commit`` hook covers that. It is not
+a transactional outbox -- a crash between commit and flush loses the job -- but
+it removes the ordering hazard, which is the one that bites in practice. Work
+that must survive that crash needs an outbox in the same database.
+"""
+
+from __future__ import annotations
+
+from collections.abc import AsyncGenerator
+from contextlib import asynccontextmanager
+from contextvars import ContextVar
+
+from smallage.core.envelope import Envelope
+from smallage.core.protocols import Enqueuer
+
+current_enqueuer: ContextVar[Enqueuer | None] = ContextVar(
+    "smallage_enqueuer", default=None
+)
+"""The enqueuer in force for this unit of work, if one was bound.
+
+Set it and every enqueue inside the block routes through it, which is what lets
+publication know about the transaction without every call site being told.
+"""
+
+
+class DeferredEnqueuer:
+    """Holds jobs until something says the transaction went through.
+
+    Wrap the real enqueuer per unit of work, call ``flush`` from the commit hook
+    and ``discard`` from the rollback one.
+    """
+
+    def __init__(self, target: Enqueuer) -> None:
+        self.target = target
+        self._pending: list[tuple[Envelope, str]] = []
+
+    @property
+    def pending(self) -> tuple[tuple[Envelope, str], ...]:
+        return tuple(self._pending)
+
+    async def enqueue(self, envelope: Envelope, *, queue: str) -> bytes:
+        """Record the job. Nothing reaches Redis until ``flush``."""
+        self._pending.append((envelope, queue))
+        return envelope.id.encode()
+
+    async def flush(self) -> list[bytes]:
+        """Publish everything buffered, oldest first, and forget it.
+
+        The buffer is cleared before publishing: a failure part way through must
+        not leave jobs that would be published twice by the next flush.
+        """
+        buffered, self._pending = self._pending, []
+        return [
+            await self.target.enqueue(envelope, queue=queue)
+            for envelope, queue in buffered
+        ]
+
+    def discard(self) -> None:
+        """Drop everything buffered. For the rollback path."""
+        self._pending.clear()
+
+    @asynccontextmanager
+    async def active(self) -> AsyncGenerator[DeferredEnqueuer]:
+        """Route every enqueue in this block through the buffer.
+
+        Binding only. Whether the block ends in ``flush`` or ``discard`` is the
+        caller's decision, because only the caller knows how the transaction
+        ended.
+        """
+        token = current_enqueuer.set(self)
+        try:
+            yield self
+        finally:
+            current_enqueuer.reset(token)

smallage/core/envelope.py

@@ -0,0 +1,138 @@
+"""The wire types: what a job looks like in Redis, and what it leaves behind.
+
+A stream entry is a flat hash. ``traceparent`` sits beside ``payload`` rather than
+inside it, so restoring the span context never requires decoding the payload — and
+broker mode, which reads foreign payloads, can ignore the payload entirely.
+
+Adding a field after the first release means parsing two record formats at once,
+so every field a later milestone needs is written from the start.
+"""
+
+from collections.abc import Mapping
+
+import msgspec
+
+from smallage.core.errors import MalformedEnvelope
+
+ENVELOPE_VERSION = b"1"
+
+_REQUIRED = (b"v", b"id", b"task", b"payload", b"enqueued_at", b"attempt")
+
+
+class Envelope(msgspec.Struct, frozen=True, kw_only=True):
+    id: str
+    task: str
+    payload: bytes
+    enqueued_at: int
+    attempt: int = 0
+    payload_ref: str | None = None
+    traceparent: str | None = None
+    tracestate: str | None = None
+    dedup: str | None = None
+    """Application-chosen key. At most one job per key runs within its window."""
+    result_ttl_ms: int | None = None
+    """Keep the outcome for this long. Absent means nobody is waiting for it."""
+    history: tuple[str, ...] = ()
+    """One compact line per failed attempt, oldest first.
+
+    Bounded by the retry policy, and each line is truncated, so a job that keeps
+    failing cannot grow its own entry without limit.
+    """
+
+
+class JsonCodec:
+    """Default payload codec. The plugin layer substitutes the app's msgspec pair."""
+
+    def encode(self, value: object, /) -> bytes:
+        return msgspec.json.encode(value)
+
+    def decode(self, raw: bytes, /) -> object:
+        return msgspec.json.decode(raw)
+
+
+def to_fields(envelope: Envelope) -> dict[bytes, bytes]:
+    fields = {
+        b"v": ENVELOPE_VERSION,
+        b"id": envelope.id.encode(),
+        b"task": envelope.task.encode(),
+        b"payload": envelope.payload,
+        b"enqueued_at": str(envelope.enqueued_at).encode(),
+        b"attempt": str(envelope.attempt).encode(),
+    }
+    optional = {
+        b"payload_ref": envelope.payload_ref,
+        b"traceparent": envelope.traceparent,
+        b"tracestate": envelope.tracestate,
+        b"dedup": envelope.dedup,
+    }
+    if envelope.result_ttl_ms is not None:
+        fields[b"result_ttl_ms"] = str(envelope.result_ttl_ms).encode()
+    # Redis has no null: an unset field is written as no field at all.
+    fields.update({k: v.encode() for k, v in optional.items() if v is not None})
+    if envelope.history:
+        fields[b"history"] = "\n".join(envelope.history).encode()
+    return fields
+
+
+def from_fields(fields: Mapping[bytes, bytes]) -> Envelope:
+    for name in _REQUIRED:
+        if name not in fields:
+            raise MalformedEnvelope(f"stream entry is missing field {name.decode()!r}")
+    if fields[b"v"] != ENVELOPE_VERSION:
+        raise MalformedEnvelope(
+            f"unsupported envelope version {fields[b'v']!r}, "
+            f"expected {ENVELOPE_VERSION!r}"
+        )
+    optional = {
+        name: fields[key].decode()
+        for name, key in (
+            ("payload_ref", b"payload_ref"),
+            ("traceparent", b"traceparent"),
+            ("tracestate", b"tracestate"),
+            ("dedup", b"dedup"),
+        )
+        if key in fields
+    }
+    # Unknown fields are ignored on purpose: a newer producer must not break an
+    # older worker during a rolling deploy.
+    raw_ttl = fields.get(b"result_ttl_ms")
+    raw_history = fields.get(b"history")
+    return Envelope(
+        result_ttl_ms=int(raw_ttl) if raw_ttl is not None else None,
+        history=tuple(raw_history.decode().split("\n")) if raw_history else (),
+        id=fields[b"id"].decode(),
+        task=fields[b"task"].decode(),
+        payload=fields[b"payload"],
+        enqueued_at=int(fields[b"enqueued_at"]),
+        attempt=int(fields[b"attempt"]),
+        **optional,
+    )
+
+
+class Record(msgspec.Struct, frozen=True):
+    """A stream entry as Redis returns it.
+
+    Broker mode consumes foreign payloads and never builds an ``Envelope``, so the
+    transport hands back raw records and decoding is a separate step in the worker.
+    """
+
+    stream: str
+    entry_id: bytes
+    fields: dict[bytes, bytes]
+
+
+class Pending(msgspec.Struct, frozen=True):
+    """One unacknowledged entry, as ``XPENDING`` describes it."""
+
+    stream: str
+    entry_id: bytes
+    consumer: str
+    times_delivered: int
+
+
+class TaskResult(msgspec.Struct, frozen=True):
+    """What a finished job left behind, for whoever asked to be told."""
+
+    ok: bool
+    value: bytes = b""
+    error: str = ""

smallage/core/errors.py

@@ -0,0 +1,17 @@
+"""Exception hierarchy of the public API."""
+
+
+class SmallageError(Exception):
+    """Base of every error this library raises."""
+
+
+class ConfigurationError(SmallageError):
+    """Invalid configuration. Raised at construction, never from the worker loop."""
+
+
+class MalformedEnvelope(SmallageError):
+    """A stream entry is missing a field the envelope requires."""
+
+
+class PayloadTooLarge(SmallageError):
+    """Encoded arguments exceed the configured limit for an inline payload."""

smallage/core/keys.py

@@ -0,0 +1,81 @@
+"""Redis key schema.
+
+Every key of a namespace carries the same literal ``{ns}`` hash tag, so they all
+land in one Cluster slot: multi-key ``XREADGROUP`` and the Lua scripts require it.
+The schema cannot be changed after the first release without a data migration.
+"""
+
+import zlib
+
+from smallage.core.errors import ConfigurationError
+
+_FORBIDDEN = ("{", "}", ":")
+
+
+def _validate(name: str, value: str) -> str:
+    if not value:
+        raise ConfigurationError(f"{name} must not be empty")
+    for char in _FORBIDDEN:
+        if char in value:
+            raise ConfigurationError(f"{name} must not contain {char!r}, got {value!r}")
+    return value
+
+
+def validate_namespace(namespace: str) -> str:
+    return _validate("namespace", namespace)
+
+
+def validate_queue(queue: str) -> str:
+    return _validate("queue", queue)
+
+
+def stream_key(namespace: str, queue: str, shard: int) -> str:
+    return f"{{{namespace}}}:q:{queue}:{shard}"
+
+
+def stream_keys(namespace: str, queue: str, shards: int) -> list[str]:
+    if shards < 1:
+        raise ConfigurationError(f"shards must be at least 1, got {shards}")
+    return [stream_key(namespace, queue, shard) for shard in range(shards)]
+
+
+def stream_for(namespace: str, queue: str, shards: int, routing_key: str) -> str:
+    """Pick a shard deterministically, so one routing key always lands in one stream."""
+    streams = stream_keys(namespace, queue, shards)
+    return streams[zlib.crc32(routing_key.encode()) % len(streams)]
+
+
+def alive_key(namespace: str, entry_id: bytes | str) -> str:
+    if isinstance(entry_id, bytes):
+        entry_id = entry_id.decode("ascii")
+    return f"{{{namespace}}}:alive:{entry_id}"
+
+
+def dedup_key(namespace: str, key: str) -> str:
+    return f"{{{namespace}}}:dedup:{key}"
+
+
+def sched_key(namespace: str) -> str:
+    return f"{{{namespace}}}:sched"
+
+
+def sched_job_key(namespace: str, scheduled_id: str) -> str:
+    return f"{{{namespace}}}:sched:job:{scheduled_id}"
+
+
+def leader_key(namespace: str) -> str:
+    return f"{{{namespace}}}:leader"
+
+
+def result_key(namespace: str, job_id: str) -> str:
+    return f"{{{namespace}}}:result:{job_id}"
+
+
+def result_wait_key(namespace: str, job_id: str) -> str:
+    return f"{{{namespace}}}:result:wait:{job_id}"
+
+
+def dlq_key(namespace: str) -> str:
+    """Exported: reading the dead letter queue is an ordinary thing to want, and
+    an application should not have to rebuild the key schema to do it."""
+    return f"{{{namespace}}}:dlq"

smallage/core/payloads.py

@@ -0,0 +1,66 @@
+"""Somewhere for arguments too large to keep in a stream.
+
+Redis holds the whole stream in memory, so a payload measured in megabytes is a
+direct route to an OOM kill. Above a threshold the arguments go here and the
+record carries a reference instead.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import anyio
+
+from smallage.core.errors import ConfigurationError, SmallageError
+
+SCHEME = "file://"
+
+
+class PayloadMissing(SmallageError):
+    """A record references a payload the store no longer has."""
+
+
+class FilePayloadStore:
+    """Payloads as files under one directory.
+
+    Every worker must see the same directory -- a network volume, or a single
+    host. A path local to one pod means the job runs wherever the file happens to
+    be and fails everywhere else, so the reference records the path and reading
+    it back from the wrong place says exactly that.
+    """
+
+    def __init__(self, root: Path | str) -> None:
+        self.root = Path(root)
+        if not self.root.is_absolute():
+            # A relative path resolves against each process's working directory,
+            # which is rarely the same one twice.
+            raise ConfigurationError(f"root must be absolute, got {self.root}")
+        self.root.mkdir(parents=True, exist_ok=True)
+
+    def _path(self, job_id: str) -> Path:
+        if "/" in job_id or job_id in {"", ".", ".."}:
+            raise ConfigurationError(f"job id is not a usable file name: {job_id!r}")
+        return self.root / f"{job_id}.payload"
+
+    async def put(self, job_id: str, data: bytes) -> str:
+        path = self._path(job_id)
+        # Write beside it and rename: a reader must never see half a payload,
+        # and rename within a directory is atomic.
+        staging = path.with_suffix(".partial")
+        await anyio.Path(staging).write_bytes(data)
+        await anyio.Path(staging).rename(path)
+        return f"{SCHEME}{path}"
+
+    async def get(self, reference: str) -> bytes:
+        path = Path(reference.removeprefix(SCHEME))
+        try:
+            return await anyio.Path(path).read_bytes()
+        except FileNotFoundError as exc:
+            raise PayloadMissing(
+                f"no payload at {reference}; the store is not the one that wrote "
+                "it, or it has been cleaned up"
+            ) from exc
+
+    async def discard(self, reference: str) -> None:
+        """Remove a payload once its job is finished with it."""
+        await anyio.Path(Path(reference.removeprefix(SCHEME))).unlink(missing_ok=True)