celery-liveops 0.1.0__py3-none-any.whl

celery_liveops/__init__.py

@@ -0,0 +1,205 @@
+"""celery-liveops -- see inside a long-running Celery task while it runs.
+
+A Celery task that takes eight hours is a black box. The result backend tells you
+what happened *after* it happened; ``flower`` tells you a task is running, not
+what it is doing. This library fills that gap with five pieces that share one
+Redis connection and one rule -- **observability must never break the job it is
+observing**:
+
+- ``logs``      -- the task's log lines, readable live from another process
+- ``presence``  -- whether a run still has a living process behind it
+- ``watchdog``  -- a hard deadline per *task*, not per container
+- ``locks``     -- the Redis locks a dead process left behind, named and releasable
+- ``queues``    -- queue depth, and the queue nobody is consuming
+- ``scale``     -- resize a running worker, and keep that size across restarts
+
+Quick start::
+
+    from celery_liveops import install, capture_logs
+
+    install(watchdog={"crawl": 3600}, watchdog_enabled=True)
+
+    @app.task(bind=True)
+    def crawl(self, url):
+        with capture_logs(self.request.id):
+            log.info("fetching %s", url)   # visible live, from your API
+
+And on the reading side::
+
+    from celery_liveops import read_any_logs, is_alive
+
+    read_any_logs(task_id)   # the terminal, live or archived
+    is_alive(task_id)        # or is this row an orphan?
+"""
+from __future__ import annotations
+
+from typing import Any, Dict, Optional
+
+from .config import Settings, configure, reset, settings
+from .locks import (
+    LockSpec,
+    clear_registry,
+    list_locks,
+    lock_for,
+    lock_state,
+    locks_owned_by,
+    register_lock,
+    registered_locks,
+    release_locks,
+)
+from .logs import (
+    LiveLogHandler,
+    active_keys,
+    cap_log,
+    capture_logs,
+    clear_logs,
+    clear_orphan_logs,
+    current_key,
+    install_logging,
+    read_any_logs,
+    read_logs,
+    read_orphan_logs,
+)
+from .presence import (
+    alive_among,
+    clear_alive,
+    heartbeat,
+    install_presence,
+    is_alive,
+    mark_alive,
+    worker_id,
+    workers,
+)
+from .queues import (
+    bind_app,
+    consumers_by_queue,
+    has_consumer,
+    orphan_queues,
+    queue_depth,
+)
+from .scale import (
+    apply_target,
+    install_boot_scale,
+    max_concurrency,
+    pool_size,
+    queues_of_process,
+    resize_pool,
+    set_autoscale_ceiling,
+)
+from .snapshots import (
+    clear_snapshot,
+    read_snapshot,
+    set_gate,
+    snapshot,
+    snapshot_stats,
+)
+from .store import client as redis_client
+from .store import reset_client
+from .watchdog import (
+    deadline_for,
+    deadlines,
+    install_watchdog,
+    safe_stop_at,
+    set_deadline,
+    set_deadlines,
+)
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "__version__",
+    # config
+    "Settings",
+    "configure",
+    "settings",
+    "reset",
+    "redis_client",
+    "reset_client",
+    "install",
+    # logs
+    "LiveLogHandler",
+    "capture_logs",
+    "current_key",
+    "install_logging",
+    "read_logs",
+    "read_orphan_logs",
+    "read_any_logs",
+    "clear_logs",
+    "clear_orphan_logs",
+    "cap_log",
+    "active_keys",
+    # presence
+    "install_presence",
+    "is_alive",
+    "alive_among",
+    "workers",
+    "worker_id",
+    "mark_alive",
+    "clear_alive",
+    "heartbeat",
+    # watchdog
+    "install_watchdog",
+    "set_deadline",
+    "set_deadlines",
+    "deadlines",
+    "deadline_for",
+    "safe_stop_at",
+    # locks
+    "LockSpec",
+    "register_lock",
+    "registered_locks",
+    "clear_registry",
+    "lock_for",
+    "list_locks",
+    "release_locks",
+    "lock_state",
+    "locks_owned_by",
+    # queues
+    "bind_app",
+    "queue_depth",
+    "consumers_by_queue",
+    "has_consumer",
+    "orphan_queues",
+    # scale
+    "apply_target",
+    "resize_pool",
+    "set_autoscale_ceiling",
+    "install_boot_scale",
+    "pool_size",
+    "queues_of_process",
+    "max_concurrency",
+    # snapshots
+    "snapshot",
+    "read_snapshot",
+    "clear_snapshot",
+    "snapshot_stats",
+    "set_gate",
+]
+
+
+def install(
+    app: Any = None,
+    logger: Any = None,
+    presence: bool = True,
+    watchdog: Optional[Dict[str, int]] = None,
+    watchdog_enabled: Optional[bool] = None,
+    context_extractor: Any = None,
+    **config_overrides: Any,
+) -> None:
+    """Wire everything up in one call, from inside your worker process.
+
+    ``watchdog`` is the deadline registry (``{task_name: seconds}``). It is armed
+    only when ``watchdog_enabled`` is true, because it kills processes.
+
+    Safe to call in a web process too: presence and the watchdog attach to Celery
+    signals that simply never fire there.
+    """
+    if config_overrides:
+        configure(**config_overrides)
+    if app is not None:
+        bind_app(app)
+    install_logging(logger)
+    if presence:
+        install_presence(context_extractor=context_extractor)
+    if watchdog or watchdog_enabled is not None:
+        install_watchdog(deadlines=watchdog, enabled=watchdog_enabled)

celery_liveops/config.py

@@ -0,0 +1,166 @@
+"""Runtime configuration for celery-liveops.
+
+Every knob has a working default, so `import celery_liveops` alone is enough for a
+local Redis on ``localhost:6379``. Anything can be overridden by environment
+variable (twelve-factor deployments) or by an explicit :func:`configure` call
+(tests, embedding the library in an app that already owns its Redis client).
+
+The one rule this module enforces: **nothing here may raise at import time**. The
+library rides inside logging handlers and Celery signals, both of which run in
+places where an exception is either swallowed or fatal to a worker boot.
+"""
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass, field, replace
+from typing import Any, Callable, Optional
+
+
+def _env_int(name: str, default: int) -> int:
+    try:
+        return int(os.environ[name])
+    except (KeyError, TypeError, ValueError):
+        return default
+
+
+def _env_float(name: str, default: float) -> float:
+    try:
+        return float(os.environ[name])
+    except (KeyError, TypeError, ValueError):
+        return default
+
+
+def _env_bool(name: str, default: bool) -> bool:
+    raw = os.environ.get(name)
+    if raw is None:
+        return default
+    return raw.strip().lower() not in ("0", "false", "no", "off", "")
+
+
+@dataclass(frozen=True)
+class Settings:
+    """Immutable snapshot of the library's configuration."""
+
+    #: Redis connection used for every live channel (logs, presence, snapshots).
+    redis_url: str = field(
+        default_factory=lambda: os.environ.get("LIVEOPS_REDIS_URL", "redis://localhost:6379/0")
+    )
+    #: Prefix for every key this library writes. Change it to share one Redis
+    #: between several applications without them stepping on each other.
+    key_prefix: str = field(
+        default_factory=lambda: os.environ.get("LIVEOPS_KEY_PREFIX", "liveops")
+    )
+
+    # ── Live log buffer ───────────────────────────────────────────────────────
+    #: Hard cap on lines kept per run. Peak memory per run is roughly
+    #: ``max_lines * max_line_length`` -- deliberately conservative, because this
+    #: buffer lives in the same Redis your broker results may be using.
+    max_lines: int = field(default_factory=lambda: _env_int("LIVEOPS_MAX_LINES", 1500))
+    #: Cap per line, so one giant record cannot blow the whole budget.
+    max_line_length: int = field(default_factory=lambda: _env_int("LIVEOPS_MAX_LINE_LENGTH", 2000))
+    #: Backstop expiry for a live buffer whose task died without cleaning up.
+    log_ttl: int = field(default_factory=lambda: _env_int("LIVEOPS_LOG_TTL", 3600))
+    #: Expiry for the *previous attempt's* buffer (see `logs.capture_logs`).
+    orphan_ttl: int = field(default_factory=lambda: _env_int("LIVEOPS_ORPHAN_TTL", 900))
+    #: Cap applied by :func:`celery_liveops.cap_log` before you archive a run's
+    #: terminal into your own database.
+    max_archive_chars: int = field(
+        default_factory=lambda: _env_int("LIVEOPS_MAX_ARCHIVE_CHARS", 200_000)
+    )
+
+    # ── Presence ──────────────────────────────────────────────────────────────
+    #: How long a presence key survives without a refresh. Generous relative to
+    #: `presence_refresh` on purpose: one network hiccup must not mark a healthy
+    #: run as dead.
+    presence_ttl: int = field(default_factory=lambda: _env_int("LIVEOPS_PRESENCE_TTL", 90))
+    #: Refresh interval of the daemon thread that keeps presence warm.
+    presence_refresh: int = field(default_factory=lambda: _env_int("LIVEOPS_PRESENCE_REFRESH", 20))
+    #: TTL of the per-worker heartbeat and "what am I running" keys.
+    worker_ttl: int = field(default_factory=lambda: _env_int("LIVEOPS_WORKER_TTL", 60))
+
+    # ── Watchdog ──────────────────────────────────────────────────────────────
+    #: Master switch. Off by default: a library that kills processes must be
+    #: opted into explicitly, never enabled by the mere act of installing it.
+    watchdog_enabled: bool = field(
+        default_factory=lambda: _env_bool("LIVEOPS_WATCHDOG_ENABLED", False)
+    )
+    #: Deadline for tasks with no registered deadline of their own.
+    default_deadline: int = field(
+        default_factory=lambda: _env_int("LIVEOPS_DEFAULT_DEADLINE", 900)
+    )
+    #: Fraction of the deadline at which a cooperative task should stop itself.
+    #: The remaining 10% pays for teardown: close the browser, write a
+    #: checkpoint, mark the run finished.
+    safe_fraction: float = field(default_factory=lambda: _env_float("LIVEOPS_SAFE_FRACTION", 0.9))
+
+    # ── Snapshots ─────────────────────────────────────────────────────────────
+    #: Only the *latest* frame is kept, so this is a short "is it still on the
+    #: login page?" TTL, not a history.
+    snapshot_ttl: int = field(default_factory=lambda: _env_int("LIVEOPS_SNAPSHOT_TTL", 300))
+    #: Minimum seconds between two captures. Grabbing a screenshot is a
+    #: synchronous round-trip to the browser; unthrottled it competes with the
+    #: work you are trying to watch.
+    snapshot_min_interval: float = field(
+        default_factory=lambda: _env_float("LIVEOPS_SNAPSHOT_MIN_INTERVAL", 1.2)
+    )
+    snapshots_enabled: bool = field(
+        default_factory=lambda: _env_bool("LIVEOPS_SNAPSHOTS_ENABLED", True)
+    )
+
+    # ── Broker introspection ──────────────────────────────────────────────────
+    #: Timeout for `celery inspect`. Short on purpose: a worker that cannot
+    #: answer in two seconds is busy or dead, and either way the number you want
+    #: is what the *others* answered.
+    inspect_timeout: float = field(
+        default_factory=lambda: _env_float("LIVEOPS_INSPECT_TIMEOUT", 2.0)
+    )
+
+
+_settings = Settings()
+_redis_client: Any = None
+_redis_factory: Optional[Callable[[], Any]] = None
+
+
+def configure(
+    *,
+    redis_client: Any = None,
+    redis_factory: Optional[Callable[[], Any]] = None,
+    **overrides: Any,
+) -> Settings:
+    """Override configuration at runtime.
+
+    ``redis_client`` lets an application hand over the connection it already
+    manages (pool sizing, TLS, sentinel) instead of having this library open a
+    second one. ``redis_factory`` does the same lazily, which is what you want
+    when the client must be created after a fork.
+
+    Any remaining keyword is a :class:`Settings` field::
+
+        configure(key_prefix="billing", max_lines=500, watchdog_enabled=True)
+    """
+    global _settings, _redis_client, _redis_factory
+
+    if overrides:
+        unknown = set(overrides) - {f for f in Settings.__dataclass_fields__}
+        if unknown:
+            raise TypeError(f"unknown setting(s): {', '.join(sorted(unknown))}")
+        _settings = replace(_settings, **overrides)
+
+    if redis_client is not None or redis_factory is not None:
+        _redis_client = redis_client
+        _redis_factory = redis_factory
+
+    return _settings
+
+
+def settings() -> Settings:
+    """The configuration in force right now."""
+    return _settings
+
+
+def reset() -> None:
+    """Restore defaults (re-reading the environment). Mainly for tests."""
+    global _settings, _redis_client, _redis_factory
+    _settings = Settings()
+    _redis_client = None
+    _redis_factory = None

celery_liveops/contrib/__init__.py

@@ -0,0 +1 @@
+"""Optional integrations."""

celery_liveops/contrib/fastapi.py

@@ -0,0 +1,119 @@
+"""A ready-made FastAPI router exposing the read side of celery-liveops.
+
+Mount it and you have the panel's backend::
+
+    from fastapi import Depends, FastAPI
+    from celery_liveops.contrib.fastapi import liveops_router
+
+    app = FastAPI()
+    app.include_router(liveops_router(dependencies=[Depends(require_operator)]))
+
+**Authentication is your job and this router will not let you forget it.** It
+exposes a task's log output, a screenshot of what it is looking at, and a
+``DELETE`` on infrastructure locks -- so ``liveops_router()`` refuses to build
+without either ``dependencies=[...]`` or an explicit ``public=True``. Releasing a
+lock is still guarded by the allowlist in :mod:`celery_liveops.locks`; the raw
+key travels from the browser, and without that check an arbitrary ``DEL`` against
+production Redis would be one POST away.
+"""
+from __future__ import annotations
+
+from typing import List, Optional, Sequence
+
+try:
+    from fastapi import APIRouter, Body, HTTPException
+except ImportError as exc:  # pragma: no cover - optional dependency
+    raise ImportError(
+        "celery_liveops.contrib.fastapi requires fastapi: pip install 'celery-liveops[fastapi]'"
+    ) from exc
+
+from .. import locks as _locks
+from .. import presence as _presence
+from .. import queues as _queues
+from .. import snapshots as _snapshots
+from ..logs import read_any_logs
+
+
+def liveops_router(
+    prefix: str = "/liveops",
+    tags: Optional[Sequence[str]] = None,
+    dependencies: Optional[Sequence] = None,
+    public: bool = False,
+    allow_release: bool = True,
+) -> APIRouter:
+    """Build the router.
+
+    ``allow_release=False`` keeps everything read-only, which is the right
+    setting for a dashboard that anyone on the team can open.
+    """
+    if not dependencies and not public:
+        raise RuntimeError(
+            "liveops_router exposes task logs, screenshots and lock release. "
+            "Pass dependencies=[Depends(your_auth)], or public=True if you have "
+            "already put authentication in front of this app."
+        )
+
+    router = APIRouter(
+        prefix=prefix,
+        tags=list(tags) if tags else ["liveops"],
+        dependencies=list(dependencies) if dependencies else None,
+    )
+
+    @router.get("/runs/{task_id}/logs")
+    def get_logs(task_id: str) -> dict:
+        """The run's terminal: the live buffer, or the archived previous attempt."""
+        return {
+            "task_id": task_id,
+            "alive": _presence.is_alive(task_id),
+            "logs": read_any_logs(task_id),
+        }
+
+    @router.get("/runs/{task_id}/snapshot")
+    def get_snapshot(task_id: str) -> dict:
+        """The latest frame captured for this run, base64-encoded PNG."""
+        image = _snapshots.read_snapshot(task_id)
+        if image is None:
+            raise HTTPException(status_code=404, detail="No snapshot for this run.")
+        return {"task_id": task_id, "image_base64": image}
+
+    @router.post("/runs/alive")
+    def post_alive(task_ids: List[str] = Body(..., embed=True)) -> dict:
+        """Which of these runs still have a living process behind them.
+
+        A POST because a live table asks about a page of ids at once, and one
+        round trip beats one request per row.
+        """
+        return {"alive": sorted(_presence.alive_among(task_ids))}
+
+    @router.get("/workers")
+    def get_workers() -> dict:
+        """Worker processes that have sent a heartbeat recently."""
+        return {"workers": _presence.workers()}
+
+    @router.get("/locks")
+    def get_locks() -> dict:
+        """Registered locks currently held, with their remaining TTL."""
+        return {"locks": _locks.list_locks()}
+
+    if allow_release:
+
+        @router.post("/locks/release")
+        def post_release(keys: List[str] = Body(..., embed=True)) -> dict:
+            """Release held locks. Keys outside the catalogue are refused, not fatal."""
+            return _locks.release_locks(keys)
+
+    @router.get("/queues")
+    def get_queues() -> dict:
+        """Who consumes what, and which declared queue nobody consumes."""
+        consumers = _queues.consumers_by_queue()
+        return {
+            "consumers": consumers,
+            "orphans": _queues.orphan_queues(),
+        }
+
+    @router.get("/queues/{queue}/depth")
+    def get_depth(queue: str) -> dict:
+        """Messages waiting in a queue. ``null`` means the broker did not answer -- not zero."""
+        return {"queue": queue, "depth": _queues.queue_depth(queue)}
+
+    return router