relier 0.1.0__py3-none-any.whl

relier/__init__.py

@@ -0,0 +1,111 @@
+"""
+Relier — Reliability layer for Celery.  Zero job loss.
+
+Core Guarantees
+---------------
+- **Zero job loss** — every dispatched task is tracked in Redis and
+  automatically resurrected if its worker crashes mid-execution.
+- **Atomic execution** — SHA-256 idempotency keys ensure tasks run exactly
+  once, even under concurrent retries.
+- **Schema safety** — versioned envelopes with checksums protect against
+  payload corruption during rolling deployments.
+- **Observability** — full OpenTelemetry span hierarchy and Prometheus
+  metrics with no extra wiring required.
+
+Quick Start
+-----------
+    >>> from relier import rl_task
+    >>> from relier.tasks.context import TaskContext
+    >>>
+    >>> @rl_task(
+    ...     queue="high_priority",
+    ...     idempotent=True,
+    ...     soft_timeout=25,
+    ...     hard_timeout=30,
+    ... )
+    ... async def process_order(order_id: str, ctx: TaskContext) -> dict:
+    ...     if ctx.partial_result:
+    ...         resume_from = ctx.partial_result["step"]
+    ...     result = await charge_and_fulfil(order_id)
+    ...     return result
+    >>>
+    >>> # FastAPI / async Django — dispatch without blocking:
+    >>> receipt = await process_order.apush(order_id="ord-001")
+    >>>
+    >>> # Django views / Flask routes / scripts — sync dispatch:
+    >>> receipt = process_order.push(order_id="ord-001")
+
+Configuration
+-------------
+All settings are read from environment variables prefixed ``RELIER_``.
+See :class:`~relier.config.Settings` for the full reference, or call
+:func:`~relier.config.get_settings` to inspect the active configuration.
+
+Error Handling
+--------------
+All exceptions raised by Relier inherit from :class:`~relier.core.exceptions.RelierError`.
+Catch that base class to handle any framework error uniformly:
+
+    >>> from relier import RelierError, AdmissionRejectedError
+    >>> try:
+    ...     receipt = await process_order.apush(order_id="ord-001")
+    ... except AdmissionRejectedError as exc:
+    ...     # Back off and retry after exc.retry_after seconds.
+    ...     await asyncio.sleep(exc.retry_after)
+    ... except RelierError:
+    ...     # Unexpected framework error — log and alert.
+    ...     raise
+"""
+
+__version__ = "0.1.0"
+
+# ---------------------------------------------------------------------------
+# Public API re-exports
+#
+# Everything a user needs for day-to-day Relier usage is importable from
+# the top-level ``relier`` package.  Internal subsystem modules remain
+# accessible via their full dotted paths for advanced use cases.
+# ---------------------------------------------------------------------------
+
+from relier.config import Settings, get_settings
+from relier.core.exceptions import (
+    AdmissionRejectedError,
+    ConfigurationError,
+    IdempotencyInFlightError,
+    MaxResurrectionsExceededError,
+    PayloadIntegrityError,
+    RedisConnectionError,
+    RelierError,
+    SchemaMigrationError,
+    WorkerInitializationError,
+)
+from relier.tasks.context import TaskContext, task_context
+from relier.tasks.decorator import PUBLIC_QUEUES, RelierTask, rl_task
+
+__all__ = [
+    # Version
+    "__version__",
+    # Core decorator & typed handle
+    "rl_task",
+    "RelierTask",
+    # Task context
+    "TaskContext",
+    "task_context",
+    # Configuration
+    "Settings",
+    "get_settings",
+    # Queue topology
+    "PUBLIC_QUEUES",
+    # Exceptions — base
+    "RelierError",
+    # Exceptions — configuration & infrastructure
+    "ConfigurationError",
+    "RedisConnectionError",
+    "WorkerInitializationError",
+    # Exceptions — task lifecycle
+    "AdmissionRejectedError",
+    "IdempotencyInFlightError",
+    "MaxResurrectionsExceededError",
+    "PayloadIntegrityError",
+    "SchemaMigrationError",
+]

relier/chaos/__init__.py

@@ -0,0 +1,19 @@
+"""
+Relier Chaos, Reliability validation suite.
+
+Importing this package registers every chaos scenario with the
+``ChaosEngine`` singleton via side effects. Without this, the CLI would
+resolve ``chaos_engine.run("worker-kill", ...)`` against an empty registry
+and raise ``ValueError``.
+"""
+
+from relier.chaos import (  # noqa: F401
+    load_spike,
+    network,
+    slow_task,
+    task_corrupt,
+    worker_kill,
+)
+from relier.chaos.engine import chaos_engine
+
+__all__ = ["chaos_engine"]

relier/chaos/engine.py

@@ -0,0 +1,49 @@
+"""
+Relier Chaos, The Engine of Destruction.
+
+Registry and orchestration for chaos experiments designed to validate
+Relier's reliability mechanisms (Phoenix, timeouts, circuit breakers).
+"""
+
+import logging
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+class ChaosEngine:
+    """Registry and runner for all chaos scenarios."""
+
+    _scenarios: dict[str, type] = {}
+
+    @classmethod
+    def register(cls, name: str) -> Any:
+        """Decorator for registering a chaos scenario class."""
+
+        def decorator(scenario_class: type) -> type:
+            cls._scenarios[name] = scenario_class
+            return scenario_class
+
+        return decorator
+
+    @classmethod
+    async def run(cls, name: str, **kwargs: Any) -> Any:
+        """Instantiate and execute a named chaos scenario.
+
+        Args:
+            name:   Name of the scenario (e.g. "worker-kill").
+            kwargs: Parameters passed to the scenario constructor.
+        """
+        if name not in cls._scenarios:
+            raise ValueError(f"Chaos scenario {name!r} not found.")
+
+        scenario = cls._scenarios[name](**kwargs)
+        logger.warning(
+            "Unleashing chaos scenario.",
+            extra={"scenario": name, "params": kwargs},
+        )
+        return await scenario.execute()
+
+
+# Global Registry Singleton
+chaos_engine = ChaosEngine()

relier/chaos/load_spike.py

@@ -0,0 +1,92 @@
+"""
+Relier Chaos,  Load Spike Scenario.
+
+Drives a sustained burst of task dispatches through the Relier admission
+controller. Each dispatch goes through ``apush()``, which means the
+fixed-window admission limiter (``admission_limit`` / ``admission_window``)
+is the actual choke point, not a fake HTTP layer.
+
+When the cluster is at capacity, ``apush`` raises ``AdmissionRejectedError``.
+We tally accepts vs. rejects so the operator can confirm that admission
+control engages and that workers are not flooded with unbounded backlog.
+"""
+
+import asyncio
+import logging
+
+from relier.chaos.engine import chaos_engine
+from relier.core.exceptions import AdmissionRejectedError
+
+logger = logging.getLogger(__name__)
+
+
+@chaos_engine.register("load-spike")
+class LoadSpikeScenario:
+    """Flood Relier with task dispatches to trigger admission rejections."""
+
+    def __init__(self, rps: int = 100, duration: int = 10) -> None:
+        self.rps = rps
+        self.duration = duration
+
+    async def execute(self) -> dict[str, int]:
+        """Burst ``rps`` dispatches per second for ``duration`` seconds."""
+        # Imported lazily so the chaos package does not pull in the Celery
+        # runtime at import time (matters for `rl chaos --help`).
+        from relier.chaos.tasks import chaos_noop
+
+        logger.critical(
+            "Triggering load spike.",
+            extra={"rps": self.rps, "duration_s": self.duration},
+        )
+
+        accepted = 0
+        rejected = 0
+        errored = 0
+
+        async def _dispatch_one() -> str:
+            try:
+                await chaos_noop.apush("chaos-load-spike")
+                return "ok"
+            except AdmissionRejectedError:
+                return "rejected"
+            except Exception as exc:
+                logger.debug(
+                    "Dispatch failed unexpectedly.",
+                    extra={"error_type": type(exc).__name__},
+                )
+                return "error"
+
+        loop = asyncio.get_running_loop()
+        start = loop.time()
+        # Slice into 10 sub-intervals per second to approximate the target RPS
+        # without spawning all dispatches in a single huge gather().
+        batch_size = max(1, self.rps // 10)
+
+        while (loop.time() - start) < self.duration:
+            tick_start = loop.time()
+            results = await asyncio.gather(
+                *(_dispatch_one() for _ in range(batch_size))
+            )
+            for r in results:
+                if r == "ok":
+                    accepted += 1
+                elif r == "rejected":
+                    rejected += 1
+                else:
+                    errored += 1
+
+            elapsed = loop.time() - tick_start
+            sleep_for = max(0.0, 0.1 - elapsed)
+            await asyncio.sleep(sleep_for)
+
+        logger.info(
+            "Load spike simulation complete.",
+            extra={
+                "accepted": accepted,
+                "rejected": rejected,
+                "errored": errored,
+                "duration_s": self.duration,
+                "target_rps": self.rps,
+            },
+        )
+        return {"accepted": accepted, "rejected": rejected, "errored": errored}

relier/chaos/network.py

@@ -0,0 +1,130 @@
+"""
+Relier Chaos — Network Partition Scenario.
+
+Simulates a network disconnect between the Relier cluster and its Redis
+broker, exercising worker resilience, "fail-open" admission control, and
+Phoenix's behaviour when heartbeat refreshes cannot land.
+
+The Compose project does not declare a custom network, so the Redis
+container is attached to ``<project>_default`` (typically
+``relier-cluster_default``). Rather than hard-coding that name, we discover
+every network the ``relier-redis`` container is attached to at runtime and
+detach/reattach all of them.
+"""
+
+import asyncio
+import json
+import logging
+import subprocess
+
+from relier.chaos.engine import chaos_engine
+
+logger = logging.getLogger(__name__)
+
+REDIS_CONTAINER = "relier-redis"
+
+
+def _discover_redis_networks(
+    container: str = REDIS_CONTAINER,
+) -> dict[str, list[str]]:
+    """Return a mapping of network-name -> list of DNS aliases for the container.
+
+    Aliases must be captured *before* detaching so we can restore them on
+    reconnect. ``docker network connect`` does not preserve the original
+    compose service-name alias (e.g. ``redis``) without it, other
+    containers can still reach Redis by container name (``relier-redis``)
+    but DNS lookups for the service name fail.
+    """
+    try:
+        raw = (
+            subprocess.check_output(
+                [
+                    "docker",
+                    "inspect",
+                    container,
+                    "--format",
+                    "{{json .NetworkSettings.Networks}}",
+                ]
+            )
+            .decode()
+            .strip()
+        )
+        if not raw or raw == "null":
+            return {}
+        data = json.loads(raw)
+        return {net: (cfg.get("Aliases") or []) for net, cfg in data.items()}
+    except subprocess.CalledProcessError as exc:
+        logger.error(
+            "Failed to inspect Redis container networks.",
+            extra={"container": container, "error": str(exc)},
+        )
+        return {}
+
+
+@chaos_engine.register("network-partition")
+class NetworkPartitionScenario:
+    """Isolate the Redis broker from every network it is attached to."""
+
+    def __init__(self, duration: int = 15) -> None:
+        self.duration = duration
+
+    async def execute(self) -> None:
+        """Disconnect Redis from all its networks, then reconnect after `duration`."""
+        net_aliases = _discover_redis_networks()
+        if not net_aliases:
+            logger.error(
+                "No docker networks discovered for Redis; cannot partition.",
+                extra={"container": REDIS_CONTAINER},
+            )
+            return
+
+        logger.critical(
+            "Executing network partition (isolating Redis).",
+            extra={
+                "duration_s": self.duration,
+                "networks": list(net_aliases.keys()),
+            },
+        )
+
+        detached: dict[str, list[str]] = {}
+        try:
+            for net, aliases in net_aliases.items():
+                try:
+                    subprocess.run(
+                        ["docker", "network", "disconnect", net, REDIS_CONTAINER],
+                        check=True,
+                    )
+                    detached[net] = aliases
+                except subprocess.CalledProcessError as exc:
+                    logger.error(
+                        "Failed to disconnect Redis from network.",
+                        extra={"network": net, "error": str(exc)},
+                    )
+
+            await asyncio.sleep(self.duration)
+
+        finally:
+            # Reattach every network we successfully removed, restoring the
+            # original DNS aliases (in particular the compose service name
+            # without it, other containers cannot resolve ``redis``).
+            for net, aliases in detached.items():
+                cmd = ["docker", "network", "connect"]
+                for alias in aliases:
+                    # Skip the auto-assigned container-ID-prefix alias; it is
+                    # re-added automatically on connect and rejecting it with
+                    # --alias is harmless but noisy.
+                    if not alias or alias == REDIS_CONTAINER:
+                        continue
+                    cmd.extend(["--alias", alias])
+                cmd.extend([net, REDIS_CONTAINER])
+                try:
+                    subprocess.run(cmd, check=True)
+                except Exception as exc:
+                    logger.critical(
+                        "FAILED TO HEAL NETWORK PARTITION.",
+                        extra={"network": net, "error": str(exc)},
+                    )
+            logger.info(
+                "Network partition healed.",
+                extra={"networks": list(detached.keys())},
+            )

relier/chaos/slow_task.py

@@ -0,0 +1,48 @@
+"""
+Relier Chaos — Slow Task Scenario.
+
+Dispatches a task that intentionally sleeps for longer than its configured
+``hard_timeout`` so the two-tier (soft/hard) timeout machinery in
+``relier.core.timeouts`` is forced to fire.
+
+We use the internal ``relier.chaos.tasks.chaos_slow`` task, which is
+decorated with ``hard_timeout=2``. Any ``duration`` >= 2s exercises:
+
+    1. Soft-timeout cleanup hook (if registered)
+    2. Hard-timeout cancellation
+    3. DLQ quarantine for ``TimeoutError`` in the decorator
+"""
+
+import logging
+import uuid
+
+from relier.chaos.engine import chaos_engine
+
+logger = logging.getLogger(__name__)
+
+
+@chaos_engine.register("slow-task")
+class SlowTaskScenario:
+    """Dispatch a deliberately-slow task to force a hard-timeout cancellation."""
+
+    def __init__(self, duration: int = 60) -> None:
+        self.duration = duration
+
+    async def execute(self) -> str:
+        """Dispatch a slow task whose runtime exceeds its decorator hard_timeout."""
+        # Lazy import: avoids pulling Celery into `rl chaos --help`.
+        from relier.chaos.tasks import chaos_slow
+
+        marker = f"chaos-slow-{uuid.uuid4().hex[:8]}"
+        logger.critical(
+            "Dispatching slow task to provoke timeout enforcement.",
+            extra={"duration_s": self.duration, "marker": marker},
+        )
+
+        await chaos_slow.apush(duration=self.duration, marker_key=marker)
+
+        logger.info(
+            "Slow task dispatched; expect a hard-timeout + DLQ quarantine.",
+            extra={"marker": marker},
+        )
+        return marker

relier/chaos/task_corrupt.py

@@ -0,0 +1,74 @@
+"""
+Relier Chaos — Task Corruption Scenario.
+
+Injects a 'poison pill' envelope whose payload checksum does not match
+its body. The producer side (``apush``) normally guarantees envelope
+integrity, so this scenario bypasses it by calling
+``celery_app.send_task`` directly with a hand-crafted envelope.
+
+The receiving worker will:
+    1. Unwrap the envelope through ``SchemaRegistry.unwrap_and_migrate``
+    2. Detect the checksum mismatch and raise ``PayloadIntegrityError``
+    3. Quarantine the task to the DLQ via ``DeadLetterQueue.quarantine``
+
+This validates the full integrity-failure path end-to-end.
+"""
+
+import asyncio
+import logging
+import uuid
+from datetime import UTC, datetime
+
+from relier.chaos.engine import chaos_engine
+
+logger = logging.getLogger(__name__)
+
+# Any registered task name will do, the worker rejects the envelope before
+# the user function ever runs. We target an internal chaos task that the
+# Celery app loads in every environment (see ``relier.chaos.tasks``).
+POISON_TARGET_TASK = "relier.chaos.tasks.chaos_noop"
+
+
+@chaos_engine.register("task-corrupt")
+class TaskCorruptScenario:
+    """Inject a malformed task to trigger PayloadIntegrityError."""
+
+    async def execute(self) -> None:
+        """Send a structurally-valid envelope with a deliberately-bad checksum."""
+        from relier.tasks.app import celery_app
+
+        task_id = f"chaos-poison-{uuid.uuid4().hex[:8]}"
+
+        # Structurally valid Pydantic envelope (all required fields present),
+        # but checksum is wrong on purpose so unwrap_and_migrate rejects it
+        # and the decorator routes the task to the DLQ.
+        bad_envelope = {
+            "schema_version": 1,
+            "task_id": task_id,
+            "payload": {"args": ["poison"], "kwargs": {}},
+            "checksum": "sha256:TAMPERED_INVALID_CHECKSUM_VALUE",
+            "enqueued_at": datetime.now(UTC).isoformat(),
+        }
+
+        logger.critical(
+            "Injecting corrupted poison pill task.",
+            extra={"task_id": task_id, "target": POISON_TARGET_TASK},
+        )
+
+        # Celery's publishing API is synchronous; dispatch off the event loop
+        # so the broker round-trip does not block the asyncio runtime.
+        loop = asyncio.get_running_loop()
+        await loop.run_in_executor(
+            None,
+            lambda: celery_app.send_task(
+                POISON_TARGET_TASK,
+                args=(bad_envelope,),
+                queue="default",
+                task_id=task_id,
+            ),
+        )
+
+        logger.info(
+            "Poison pill injected; expect a DLQ quarantine for PayloadIntegrityError.",
+            extra={"task_id": task_id},
+        )

relier/chaos/tasks.py

@@ -0,0 +1,98 @@
+"""
+Relier Chaos — Internal target tasks.
+
+The chaos suite needs tasks it can dispatch against in every environment
+(including production), so it cannot depend on ``relier.tasks.debug``
+which is only included by the Celery app outside production.
+
+These tasks are deliberately minimal: they exist solely to exercise
+Relier's reliability machinery (admission control, hard timeouts,
+Phoenix resurrection, payload integrity) from the chaos CLI. They do no
+useful application work and write only to a chaos-scoped Redis namespace.
+"""
+
+import asyncio
+
+from relier.storage.redis import get_relier_redis
+from relier.tasks.context import task_context
+from relier.tasks.decorator import rl_task
+
+# Namespaced so chaos markers never collide with application state.
+_MARKER_PREFIX = "rl:chaos:marker"
+
+
+@rl_task(idempotent=True)
+async def chaos_counter(key: str) -> int:
+    """Idempotent counter. Increments once per unique key regardless of how many
+    times the task is dispatched with the same arguments."""
+    redis = await get_relier_redis()
+    val = await redis.incr(f"{_MARKER_PREFIX}:counter:{key}")
+    return int(val)
+
+
+@rl_task()
+async def chaos_noop(tag: str = "chaos") -> str:
+    """No-op target for ``load-spike`` and ``task-corrupt``.
+
+    Increments a per-tag counter so operators can confirm dispatches landed
+    on a worker (useful when validating that admission rejections were the
+    actual choke point, not silent broker drops).
+    """
+    redis = await get_relier_redis()
+    await redis.incr(f"{_MARKER_PREFIX}:noop:{tag}")
+    return "ok"
+
+
+@rl_task(hard_timeout=2)
+async def chaos_slow(duration: int, marker_key: str) -> str:
+    """Sleep for ``duration`` seconds, used by ``slow-task``.
+
+    Decorated with ``hard_timeout=2`` so any ``duration >= 2`` forces the
+    two-tier timeout machinery to fire and the task to land in the DLQ.
+    """
+    redis = await get_relier_redis()
+    await redis.set(f"{_MARKER_PREFIX}:slow:{marker_key}:started", "1", ex=300)
+    await asyncio.sleep(duration)
+    await redis.set(f"{_MARKER_PREFIX}:slow:{marker_key}:finished", "1", ex=300)
+    return "done"
+
+
+@rl_task()
+async def chaos_long_running(duration: int, marker_key: str) -> str:
+    """Sleep for ``duration`` seconds without a hard timeout.
+
+    Used as the seed task for ``worker-kill``: it must still be running when
+    the SIGKILL lands so Phoenix has something orphaned to resurrect. We
+    yield to the event loop in small steps so cancellation lands cleanly if
+    the test ever needs to abort it.
+    """
+    redis = await get_relier_redis()
+    await redis.set(f"{_MARKER_PREFIX}:longrun:{marker_key}:started", "1", ex=600)
+    for _ in range(duration * 10):
+        await asyncio.sleep(0.1)
+    await redis.set(f"{_MARKER_PREFIX}:longrun:{marker_key}:finished", "1", ex=600)
+    return "done"
+
+
+@rl_task()
+async def chaos_checkpoint(steps: int, marker: str) -> str:
+    """Checkpointed task that saves progress at each step.
+
+    Used to verify Phoenix resurrection resumes from the last persisted
+    checkpoint rather than restarting from the beginning.
+    """
+    redis = await get_relier_redis()
+    start_step = 0
+    if task_context.partial_result:
+        start_step = task_context.partial_result.get("last_step", 0)
+    for i in range(start_step, steps):
+        await asyncio.sleep(0.1)
+        await task_context.set_partial({"last_step": i + 1})
+    await redis.set(f"{_MARKER_PREFIX}:checkpoint:{marker}:finished", "1", ex=300)
+    return "done"
+
+
+@rl_task()
+async def chaos_fail() -> None:
+    """Always raises an error. Used to exercise the failure and DLQ path."""
+    raise ValueError("Intentional failure")