trueppm-api 0.4.0__py3-none-any.whl

trueppm_api/core/worker_heartbeat.py

@@ -0,0 +1,134 @@
+"""Celery worker file-touch heartbeat for the Helm chart's readiness/startup
+probes (#3346).
+
+Why this exists: the chart's celery-worker readiness probe used to run
+``celery inspect ping``, a control-plane round trip handled by the worker's
+MainProcess. An **exec probe runs INSIDE the container it measures**, so under
+load the same process that is servicing the prefork pool can miss the ping's
+window and fail the probe while doing exactly the work it exists to do — this
+is #3236: a worker with 0 restarts, processing jobs throughout, that never
+answered a single readiness probe. Widening the probe's timing cannot fix
+this; readiness has no failure budget to widen (one success is all it needs,
+and a probe that never gets one cannot be repaired by any timing value).
+
+This module replaces that mechanism for readiness (and adds a startup check)
+with a file whose mtime is refreshed by Celery's own internal ``heartbeat_sent``
+signal — fired by the worker's ``Heart`` service on a fixed timer
+(``~2s`` by default) that runs on the MainProcess's own event loop, entirely
+independent of whether the prefork pool is idle or saturated. Touching a file
+costs nothing proportional to worker load, so the probe that reads it
+(``find <file> -newermt "-Ns"``, wired in the Helm chart) is a plain filesystem
+stat with no fork of Django, Celery, or a broker round trip — and therefore
+cannot be starved by the very load it is meant to tolerate.
+
+``worker_ready`` (fired once, after mingle/gossip complete — i.e. once the
+broker connection is genuinely established) provides the STARTUP signal: the
+chart's worker startup probe only checks that the file exists at all, which is
+true exactly when this fires for the first time. ``worker_shutting_down``
+removes the file so a worker that announces its own graceful shutdown drops
+out of readiness immediately rather than waiting for the staleness window to
+elapse.
+
+Liveness keeps ``celery inspect ping`` (unchanged, see ``_helpers.tpl``): a
+missed liveness check only restarts the container, a bounded cost the chart's
+termination-grace math already accounts for, whereas a false readiness failure
+had no such budget. Replacing only the mechanism with something to lose is the
+narrower, lower-risk fix.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from pathlib import Path
+
+from celery.signals import heartbeat_sent, worker_ready, worker_shutting_down
+
+logger = logging.getLogger(__name__)
+
+# Matches the Helm chart's probes.worker.heartbeatFile default (#3346) — the
+# two must agree, since the chart's exec probe reads exactly this path inside
+# the container. /tmp is the worker container's only writable mount
+# (readOnlyRootFilesystem is on; see celery-worker/deployment.yaml).
+_DEFAULT_HEARTBEAT_FILE = "/tmp/trueppm-celery-worker-heartbeat"  # nosec B108
+
+
+def heartbeat_file() -> Path:
+    """Return the configured heartbeat file path.
+
+    Read from the environment on every call (not cached at import time) so a
+    test can override ``TRUEPPM_CELERY_WORKER_HEARTBEAT_FILE`` per-case without
+    needing to reload this module.
+    """
+    return Path(os.environ.get("TRUEPPM_CELERY_WORKER_HEARTBEAT_FILE", _DEFAULT_HEARTBEAT_FILE))
+
+
+def _touch(path: Path) -> None:
+    """Create ``path`` if absent and bump its mtime to now.
+
+    Opens with ``O_NOFOLLOW`` and mode ``0600`` rather than ``Path.touch()``:
+    ``/tmp`` is world-writable (Sonar python:S5443), so a symlink planted at
+    this path by another process on the same filesystem must not be followed,
+    and the file must not be readable/writable by anyone but this process.
+
+    Errors are logged, never raised: a failure to write the heartbeat file
+    must not crash task processing — it should only ever show up as the
+    readiness probe going stale, which is the correct, visible failure mode.
+    """
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        fd = os.open(path, os.O_CREAT | os.O_WRONLY | os.O_NOFOLLOW, 0o600)
+        os.close(fd)
+        os.utime(path, None)
+    except OSError:
+        logger.warning("worker_heartbeat: could not touch %s", path, exc_info=True)
+
+
+def _remove(path: Path) -> None:
+    """Delete ``path`` if present. Errors are logged, never raised."""
+    try:
+        path.unlink(missing_ok=True)
+    except OSError:
+        logger.warning("worker_heartbeat: could not remove %s", path, exc_info=True)
+
+
+def _on_worker_ready(**_kwargs: object) -> None:
+    """First touch: fires once mingle/gossip complete — broker connection is up.
+
+    This is the STARTUP signal the chart's ``probes.worker.startup`` check
+    reads (mere existence of the file, not freshness).
+    """
+    _touch(heartbeat_file())
+
+
+def _on_heartbeat_sent(**_kwargs: object) -> None:
+    """Refresh the file on every internal Celery heartbeat tick (~2s default).
+
+    Fired from the worker's own timer/event-loop thread, not from a task
+    execution context, so its cadence does not slow down — or speed up —
+    with task load. This is the READINESS signal the chart's
+    ``probes.worker.readiness`` check reads (freshness within a staleness
+    window).
+    """
+    _touch(heartbeat_file())
+
+
+def _on_worker_shutting_down(**_kwargs: object) -> None:
+    """Drop the file immediately on a graceful shutdown request.
+
+    Without this a worker that is winding down still reads Ready until the
+    staleness window elapses. Best-effort only — a hard kill (OOM, SIGKILL)
+    skips this signal entirely, which is fine: the staleness window is exactly
+    the backstop for that case.
+    """
+    _remove(heartbeat_file())
+
+
+# Connected as plain calls rather than via the `@signal.connect` decorator
+# form: celery's `Signal.connect` has no type stubs, and mypy --strict flags
+# the decorator form as an "untyped decorator" (it cannot see through it to
+# confirm these functions still match their declared signature). A bare call
+# has no such effect — its unused return value is simply discarded.
+worker_ready.connect(_on_worker_ready)
+heartbeat_sent.connect(_on_heartbeat_sent)
+worker_shutting_down.connect(_on_worker_shutting_down)

trueppm_api/core/worker_selfheal.py

@@ -0,0 +1,205 @@
+"""Self-exit a wedged Celery worker so Docker Compose can restart it (#3936).
+
+Why this exists: the Helm chart recovers a wedged worker (heartbeat present but
+stale — see ``worker_heartbeat``) with a ``celery inspect ping``
+``livenessProbe`` (``packages/helm/templates/celery-worker/deployment.yaml``):
+a failed liveness probe makes kubelet kill and restart the pod. Plain Docker
+Compose has **no equivalent mechanism** — a container's ``HEALTHCHECK``
+(here, the same heartbeat-freshness check the chart's *readiness* probe uses)
+only changes what ``docker compose ps`` reports; nothing in Compose reads that
+status and restarts anything. ``restart: unless-stopped`` fires on a process
+*exit*, never on a healthcheck transitioning to ``unhealthy``. Verified
+generically against a permanently-failing healthcheck + ``restart:
+unless-stopped``: the container sits reporting ``unhealthy`` with
+``RestartCount=0`` forever (#3936).
+
+So a worker that wedges after connecting — #3722's failure mode, there fixed
+only for the Helm chart's cold-start race — has no path back on Compose once
+it is running: cold start is covered (``worker_broker_wait`` holds the process
+until the broker answers before the consumer ever starts), but a broker that
+goes away and comes back *after* the worker is already up is not, and nothing
+about that recovery differs between a cold-start wedge and a mid-life one once
+the symptom (heartbeat frozen) is the same.
+
+Rather than add a sidecar that watches container health and issues
+``docker restart`` — which needs the Docker socket mounted into a container in
+the hardened production stack (``docker-compose.prod.yml`` runs
+``read_only: true`` / ``no-new-privileges``), a privilege-escalation path that
+would need its own security review — this gives ``restart: unless-stopped``
+something it already knows how to act on: a process exit. A daemon thread,
+started once the worker's own heartbeat file first exists (mirroring the
+chart's startup-probe gate — see ``worker_heartbeat.heartbeat_file``), polls
+that same file's mtime and calls ``os._exit`` once it is older than
+``stale_seconds()``. It deliberately reuses the *heartbeat file* and the
+*staleness-by-mtime* mechanism the readiness healthcheck already uses rather
+than inventing a second way to decide a worker is unwell — the two now differ
+only in what they do with a stale reading: the healthcheck reports it, this
+acts on it.
+
+Opt-in, default off (``self_heal_enabled()``): a chart-deployed worker already
+has a liveness probe for this condition, independently tuned against
+Kubernetes' own grace-period accounting (``probes.worker.liveness`` in
+``values.yaml``) — turning this on unconditionally would give that worker two
+differently-tuned kill paths for the same symptom. ``docker-compose.yml`` and
+``docker-compose.prod.yml`` are the only places that set the env var.
+
+The staleness threshold is 3x the readiness healthcheck's own 30s window
+(``_DEFAULT_STALE_SECONDS``), not the same value: a false *readiness* failure
+just drops the container out of rotation for a Docker-network-level check
+nothing else currently reads, so it costs nothing to be quick about. A false
+*self-exit* kills the only process in the container and forces a cold restart
+— it must tolerate everything a transient blip (broker reconnect, a GC pause,
+CPU throttling under ``docker-compose.prod.yml``'s ``cpus: 2.0`` limit) would
+produce, with real margin, before concluding the worker is genuinely wedged
+rather than briefly busy.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import threading
+import time
+from collections.abc import Callable
+from pathlib import Path
+
+from celery.signals import worker_ready, worker_shutting_down
+
+from trueppm_api.core.worker_heartbeat import heartbeat_file
+
+logger = logging.getLogger(__name__)
+
+_ENABLE_ENV = "TRUEPPM_CELERY_WORKER_SELF_HEAL"
+_STALE_SECONDS_ENV = "TRUEPPM_CELERY_WORKER_SELF_HEAL_STALE_SECONDS"
+
+# 3x the compose readiness healthcheck's own staleness window (30s, see
+# docker-compose.yml / docker-compose.prod.yml) — see module docstring for why
+# a self-exit needs more margin than a readiness flap.
+_DEFAULT_STALE_SECONDS = 90.0
+
+# How often the watchdog re-checks the heartbeat file's mtime. Cheap (a single
+# stat), so this can safely be more frequent than the threshold it is
+# measuring against without adding meaningful overhead.
+_CHECK_INTERVAL_SECONDS = 10.0
+
+# Sysexits.h EX_TEMPFAIL: "a temporary failure ... in which the user is
+# invited to retry" — distinguishes a deliberate self-heal exit from an
+# ordinary crash (exit 1) in `docker inspect --format '{{.State.ExitCode}}'`
+# and container logs.
+_EXIT_CODE = 75
+
+_shutting_down = threading.Event()
+
+
+def self_heal_enabled() -> bool:
+    """Return whether the self-exit watchdog should run (see module docstring).
+
+    Read on every call, not cached, so tests can toggle it per-case.
+    """
+    return os.environ.get(_ENABLE_ENV, "").strip().lower() in ("1", "true", "yes")
+
+
+def stale_seconds() -> float:
+    """Return the configured staleness threshold in seconds.
+
+    Falls back to the default on a missing, non-numeric, or non-positive
+    value — the same fallback policy as ``worker_broker_wait.wait_budget_seconds``,
+    so a typo cannot silently arm a hair-trigger self-exit.
+    """
+    raw = os.environ.get(_STALE_SECONDS_ENV)
+    if raw is None or raw.strip() == "":
+        return _DEFAULT_STALE_SECONDS
+    try:
+        value = float(raw)
+    except ValueError:
+        logger.warning("worker_selfheal: ignoring non-numeric %s=%r", _STALE_SECONDS_ENV, raw)
+        return _DEFAULT_STALE_SECONDS
+    if value <= 0:
+        logger.warning("worker_selfheal: ignoring non-positive %s=%r", _STALE_SECONDS_ENV, raw)
+        return _DEFAULT_STALE_SECONDS
+    return value
+
+
+def _watchdog_loop(
+    path: Path,
+    *,
+    threshold: float,
+    check_interval: float = _CHECK_INTERVAL_SECONDS,
+    wait: Callable[[float], bool] = _shutting_down.wait,
+    now: Callable[[], float] = time.time,
+    exit_: Callable[[int], None] = lambda code: os._exit(code),
+) -> None:
+    """Poll ``path``'s mtime; self-exit once it is older than ``threshold``.
+
+    ``wait`` is ``threading.Event.wait`` by default: it blocks for up to
+    ``check_interval`` seconds and returns ``True`` immediately if the event
+    is set in the meantime (see ``_on_worker_shutting_down``), so a graceful
+    shutdown stops this loop within one tick instead of racing it.
+
+    ``os._exit`` (not ``sys.exit``/``SystemExit``) is deliberate: this runs on
+    a daemon thread, not the MainProcess's own event loop — the wedge this
+    module recovers from is exactly that loop no longer running — so raising
+    here would only unwind this thread, leaving the process (and the
+    container) up. ``os._exit`` terminates the interpreter immediately with no
+    atexit/cleanup, which is exactly what ``restart: unless-stopped`` needs: a
+    process exit to act on.
+    """
+    while not wait(check_interval):
+        try:
+            mtime = path.stat().st_mtime
+        except OSError:
+            # File briefly absent (e.g. mid-touch, or not yet created) is not
+            # staleness — skip this tick rather than treating it as an infinite age.
+            continue
+        age = now() - mtime
+        if age > threshold:
+            logger.critical(
+                "worker_selfheal: heartbeat stale for %.0fs (threshold %.0fs) — "
+                "self-exiting so `restart: unless-stopped` can recover the container (#3936)",
+                age,
+                threshold,
+            )
+            exit_(_EXIT_CODE)
+            return
+
+
+def _start_watchdog(threshold: float) -> None:
+    thread = threading.Thread(
+        target=_watchdog_loop,
+        args=(heartbeat_file(),),
+        kwargs={"threshold": threshold},
+        name="trueppm-worker-selfheal",
+        daemon=True,
+    )
+    thread.start()
+
+
+def _on_worker_ready(**_kwargs: object) -> None:
+    """Start the watchdog once the worker is genuinely ready.
+
+    Gated on the same event ``worker_heartbeat`` uses to write the file's
+    first entry, so the watchdog's clock cannot start before there is
+    anything to read — a slow cold boot is not mistaken for staleness.
+    """
+    if not self_heal_enabled():
+        return
+    _shutting_down.clear()
+    _start_watchdog(stale_seconds())
+
+
+def _on_worker_shutting_down(**_kwargs: object) -> None:
+    """Stop the watchdog on a graceful shutdown request.
+
+    Without this, a worker that is winding down on its own could have its
+    heartbeat go stale mid-shutdown and have the watchdog treat an
+    intentional stop as a wedge. Best-effort like ``worker_heartbeat``'s own
+    handler — a hard kill skips this signal entirely, which is fine because
+    there is no process left for the watchdog to exit.
+    """
+    _shutting_down.set()
+
+
+# Plain calls rather than the decorator form, for the same mypy --strict
+# reason given in worker_heartbeat.
+worker_ready.connect(_on_worker_ready)
+worker_shutting_down.connect(_on_worker_shutting_down)

trueppm_api/fields.py

@@ -0,0 +1,36 @@
+"""Custom Django model fields."""
+
+from __future__ import annotations
+
+from django.db import models
+
+
+class LtreeField(models.Field):  # type: ignore[type-arg]
+    """PostgreSQL ltree field for hierarchical path storage.
+
+    Stores dotted-label paths (e.g. "1.2.3") that can be queried with
+    PostgreSQL ltree operators (<@, @>, ~, ?) via raw SQL or ORM annotations.
+    Maps to a GiST-indexed ltree column in PostgreSQL.
+
+    Python representation: a plain string (e.g. "1.2.3" or "").
+    """
+
+    description = "PostgreSQL ltree hierarchical path"
+
+    def db_type(self, connection: object) -> str:
+        return "ltree"
+
+    def from_db_value(self, value: object, expression: object, connection: object) -> str | None:
+        if value is None:
+            return None
+        return str(value)
+
+    def to_python(self, value: object) -> str | None:
+        if value is None:
+            return None
+        return str(value)
+
+    def get_prep_value(self, value: object) -> str | None:
+        if value is None:
+            return None
+        return str(value)

trueppm_api/permissions.py

@@ -0,0 +1,30 @@
+"""Top-level permission re-exports for backward compatibility.
+
+All real permission classes and the ProjectScopedViewSet mixin live in
+trueppm_api.apps.access.permissions. This module re-exports them so that
+any code that imported from here continues to work without changes.
+"""
+
+from __future__ import annotations
+
+from trueppm_api.apps.access.permissions import (
+    IsProjectAdmin,
+    IsProjectMember,
+    IsProjectMemberWrite,
+    IsProjectOwner,
+    IsProjectScheduler,
+    ProjectScopedViewSet,
+)
+
+# Explicit re-export surface: this shim exists purely so callers that imported
+# permissions from here keep working (trueppm-enterprise registers against it).
+# Listing the names in ``__all__`` marks them as an intentional public export,
+# which is also what tells static analysis they are not dead imports.
+__all__ = [
+    "IsProjectAdmin",
+    "IsProjectMember",
+    "IsProjectMemberWrite",
+    "IsProjectOwner",
+    "IsProjectScheduler",
+    "ProjectScopedViewSet",
+]

trueppm_api/routing.py

@@ -0,0 +1,11 @@
+"""WebSocket URL routing — registered consumers are added here as features land."""
+
+from __future__ import annotations
+
+from django.urls import path
+
+from trueppm_api.apps.sync.consumers import ProjectConsumer
+
+websocket_urlpatterns = [
+    path("ws/v1/projects/<uuid:pk>/", ProjectConsumer.as_asgi()),
+]