trueppm-api 0.4.0__py3-none-any.whl

trueppm_api/apps/sync/tasks.py

@@ -0,0 +1,260 @@
+"""Celery tasks for the sync app.
+
+``purge_sync_batches`` reaps :class:`~trueppm_api.apps.sync.models.SyncBatch`
+rows past the dedup window so the idempotency table stays bounded — the
+ADR-0081 purge convention applied to the upload envelope (ADR-0082 §Durable
+Execution #6).
+
+``reap_domain_tombstones`` hard-deletes per-row soft-deleted tombstones
+(``is_deleted=True``) from live projects. VersionedModel rows are soft-deleted
+so the mobile sync endpoint can return their IDs as tombstones to offline
+clients; once the retention window has passed there is no further value in
+keeping the row and it can be hard-deleted. Only rows belonging to live
+(non-deleted, non-archived) projects are touched — tombstones in deleted or
+archived projects are skipped because those projects may still be restoring.
+"""
+
+from __future__ import annotations
+
+import logging
+from datetime import timedelta
+from typing import Any
+
+from trueppm_api.core.idempotent import idempotent_task
+
+logger = logging.getLogger(__name__)
+
+
+@idempotent_task(
+    lock_key_template="purge_sync_batches",
+    lock_ttl=300,
+    on_contention="skip",
+    name="sync.purge_sync_batches",
+)
+def purge_sync_batches(self: object) -> None:
+    """Delete SyncBatch rows older than the retention window.
+
+    Runs nightly via Celery Beat. Rows are only dedup-relevant within
+    ``TRUEPPM_SYNC_BATCH_RETENTION_HOURS`` (default 24h); past that, the same
+    ``client_batch_id`` is allowed to re-run, so the row carries no value.
+    Idempotent and contention-skipping — a second concurrent run is a no-op.
+    """
+    _do_purge()
+
+
+def _do_purge(*, dry_run: bool = False, override_value: int | None = None) -> int:
+    """Business logic for purge_sync_batches — extracted for testability.
+
+    The window (in hours) comes from ``resolve_retention`` (operator override →
+    the TRUEPPM_SYNC_BATCH_RETENTION_HOURS default, ADR-0173). This window is
+    non-nullable, so it is never disabled. Returns rows deleted, or the eligible
+    count when ``dry_run``; ``override_value`` forces a hypothetical window.
+    """
+    from django.utils import timezone
+
+    from trueppm_api.apps.observability.retention import resolve_retention
+    from trueppm_api.apps.sync.models import SyncBatch
+
+    ttl_hours = (
+        override_value
+        if override_value is not None
+        else resolve_retention("TRUEPPM_SYNC_BATCH_RETENTION_HOURS")
+    )
+    if ttl_hours is None:
+        return 0
+    cutoff = timezone.now() - timedelta(hours=ttl_hours)
+    qs = SyncBatch.objects.filter(created_at__lt=cutoff)
+    if dry_run:
+        return qs.count()
+    deleted, _ = qs.delete()
+    if deleted:
+        logger.info("purge_sync_batches: deleted %d expired batch row(s)", deleted)
+    return deleted
+
+
+@idempotent_task(
+    lock_key_template="purge_board_events",
+    lock_ttl=300,
+    on_contention="skip",
+    name="sync.purge_board_events",
+)
+def purge_board_events(self: object) -> None:
+    """Delete BoardEvent replay-buffer rows older than the retention window.
+
+    Runs nightly via Celery Beat (ADR-0236, #321). A reconnecting client whose
+    ``?since=`` predates the retained window gets a ``resync_required`` frame and
+    refetches, so aged-out rows carry no value. Idempotent and contention-skipping
+    — a concurrent run is a no-op.
+    """
+    _do_purge_board_events()
+
+
+def _do_purge_board_events(*, dry_run: bool = False, override_value: int | None = None) -> int:
+    """Business logic for purge_board_events — extracted for testability.
+
+    Reads the window directly from ``settings.TRUEPPM_BOARD_EVENT_RETENTION_HOURS``
+    (mirrors ``reap_domain_tombstones``), not the ADR-0173 operator resolver — the
+    replay buffer is internal transport plumbing, not an operator-tunable table.
+    Returns rows deleted, or the eligible count when ``dry_run``; ``override_value``
+    forces a hypothetical window (hours).
+    """
+    from django.conf import settings
+    from django.utils import timezone
+
+    from trueppm_api.apps.sync.models import BoardEvent
+
+    ttl_hours: int = (
+        override_value
+        if override_value is not None
+        else getattr(settings, "TRUEPPM_BOARD_EVENT_RETENTION_HOURS", 24)
+    )
+    cutoff = timezone.now() - timedelta(hours=ttl_hours)
+    qs = BoardEvent.objects.filter(created_at__lt=cutoff)
+    if dry_run:
+        return qs.count()
+    deleted, _ = qs.delete()
+    if deleted:
+        logger.info("purge_board_events: deleted %d expired replay-buffer row(s)", deleted)
+    return deleted
+
+
+# ---------------------------------------------------------------------------
+# reap_domain_tombstones — hard-delete stale per-row soft-delete tombstones
+# ---------------------------------------------------------------------------
+
+# Registry of (Model, project_filter_kwargs, age_field_or_None).
+#
+# ``age_field`` is the DateTimeField used to enforce the retention window. Two
+# flavors are in use: an ``updated_at`` (auto_now=True) column that reflects the
+# most recent save (Risk, Sprint), or a ``deleted_at`` column stamped only by
+# ``soft_delete()`` (mirrors Attachment.deleted_at) — used by Task and
+# Dependency so a tombstone reliably survives TRUEPPM_TOMBSTONE_RETENTION_DAYS
+# regardless of when the row last happened to be saved for an unrelated reason.
+# ``age_field=None`` remains a valid registry entry for a model that
+# deliberately opts out of a grace window, but every model below enforces one —
+# an offline mobile client must have a window to reconnect and receive a
+# tombstone before it is hard-deleted, or the deleted row becomes a permanent
+# phantom on that device.
+#
+# Extend this list when new VersionedModel subclasses are added that carry
+# ``is_deleted`` tombstones in the projects domain.
+
+
+def _build_registry() -> list[tuple[Any, dict[str, Any], str | None]]:
+    """Build the tombstone model registry with deferred model imports.
+
+    Deferred so the function can be called safely after Django app setup
+    completes, avoiding circular imports at module load time.
+    """
+    from trueppm_api.apps.projects.models import Dependency, Risk, Sprint, Task, TaskRelation
+
+    return [
+        # Task: deleted_at is stamped in Task.soft_delete() (mirrors Attachment) —
+        # enforce the retention window so a tombstone survives long enough for an
+        # offline client to reconnect and receive it.
+        (
+            Task,
+            {"project__is_deleted": False, "project__is_archived": False},
+            "deleted_at",
+        ),
+        # Risk: has updated_at — enforce the retention window.
+        (
+            Risk,
+            {"project__is_deleted": False, "project__is_archived": False},
+            "updated_at",
+        ),
+        # Sprint: has updated_at — enforce the retention window.
+        (
+            Sprint,
+            {"project__is_deleted": False, "project__is_archived": False},
+            "updated_at",
+        ),
+        # Dependency: no direct project FK — reap via predecessor. deleted_at is
+        # stamped in Dependency.soft_delete() (mirrors Task) — enforce the same
+        # retention window. Same-project and cross-project edges are both
+        # handled: once the predecessor's project is live and the edge is past
+        # the retention window, there is no further sync value in the row.
+        (
+            Dependency,
+            {
+                "predecessor__project__is_deleted": False,
+                "predecessor__project__is_archived": False,
+            },
+            "deleted_at",
+        ),
+        # TaskRelation (ADR-0455): no direct project FK — reap via the source task,
+        # mirroring Dependency. deleted_at is stamped in TaskRelation.soft_delete(),
+        # so the same retention grace window keeps a tombstone available long enough
+        # for an offline client to reconnect and receive it.
+        (
+            TaskRelation,
+            {
+                "source__project__is_deleted": False,
+                "source__project__is_archived": False,
+            },
+            "deleted_at",
+        ),
+    ]
+
+
+@idempotent_task(
+    lock_key_template="reap_domain_tombstones",
+    lock_ttl=600,
+    on_contention="skip",
+    name="sync.reap_domain_tombstones",
+)
+def reap_domain_tombstones(self: object) -> dict[str, int]:
+    """Hard-delete per-row soft-deleted tombstones from live projects.
+
+    Runs nightly via Celery Beat. For each model in ``_build_registry()``:
+    - Filters ``is_deleted=True`` rows in live (non-deleted, non-archived) projects.
+    - For models with an ``age_field`` (``updated_at`` or ``deleted_at``), further
+      restricts to rows older than the ``TRUEPPM_TOMBSTONE_RETENTION_DAYS`` cutoff
+      (default 90 days).
+    - Hard-deletes the eligible rows.
+
+    Returns a dict mapping ``{app_label.model_name: rows_deleted}`` for each
+    model — useful for monitoring and log-based alerting.
+
+    Idempotent and contention-skipping — a concurrent run is a no-op.
+    """
+    return _do_reap()
+
+
+def _do_reap(*, override_days: int | None = None) -> dict[str, int]:
+    """Business logic for reap_domain_tombstones — extracted for testability.
+
+    Args:
+        override_days: Force a specific retention window (days) instead of
+            reading ``TRUEPPM_TOMBSTONE_RETENTION_DAYS`` from settings. Useful
+            in tests to avoid manipulating global settings.
+
+    Returns:
+        Dict mapping ``{app_label.model_name: rows_deleted}`` for every
+        model in the registry, including zeros.
+    """
+    from django.conf import settings
+    from django.utils import timezone
+
+    retention_days: int = (
+        override_days
+        if override_days is not None
+        else getattr(settings, "TRUEPPM_TOMBSTONE_RETENTION_DAYS", 90)
+    )
+    cutoff = timezone.now() - timedelta(days=retention_days)
+
+    counts: dict[str, int] = {}
+    for Model, project_filters, age_field in _build_registry():
+        label: str = Model._meta.label_lower
+        qs = Model.objects.filter(is_deleted=True, **project_filters)
+        if age_field is not None:
+            qs = qs.filter(**{f"{age_field}__lt": cutoff})
+        deleted, _ = qs.delete()
+        counts[label] = deleted
+        if deleted:
+            logger.info(
+                "reap_domain_tombstones: hard-deleted %d %s tombstone(s)",
+                deleted,
+                label,
+            )
+    return counts

trueppm_api/apps/sync/throttles.py

@@ -0,0 +1,175 @@
+"""Rate limiting for the mobile sync upload endpoint (ADR-0082, #667, #1719).
+
+Raw redis-py against ``settings.REDIS_URL/2`` (the throttle-counter DB): INCR a
+per-bucket key with a 60s TTL. Two independent windows must both pass —
+per-``(project, user)`` and a per-``user`` GLOBAL bucket — so a member of N
+projects can no longer sustain ``UPLOAD_LIMIT × N`` heavyweight batch writes/min by
+spreading them across projects (#1719).
+
+Unlike a read throttle, this gates the **write** path, and each admitted request is
+a 500-row batch inside one ``transaction.atomic()`` (row locks + a CPM recalc
+enqueue). It therefore fails **CLOSED** (deny) on any Redis error: a cache blip must
+not silently remove the only DoS bound globally. Denial is safe here because the
+offline client already retries uploads with backoff — a brief 429 during an outage
+costs a retry, whereas failing open costs an unbounded authenticated write flood.
+"""
+
+from __future__ import annotations
+
+import logging
+from contextlib import contextmanager
+from typing import TYPE_CHECKING
+
+import redis
+from django.conf import settings
+from rest_framework.exceptions import Throttled
+from rest_framework.throttling import BaseThrottle
+
+from trueppm_api.core import valkey
+from trueppm_api.core.ratelimit import bypass_when_disabled
+from trueppm_api.core.redis_throttle import incr_with_ttl, incrby_with_ttl
+
+if TYPE_CHECKING:
+    from collections.abc import Iterator
+
+    from rest_framework.request import Request
+    from rest_framework.views import APIView
+
+logger = logging.getLogger(__name__)
+
+_pool: redis.ConnectionPool | None = None
+
+# req/min, per (project, user). Generous for batched offline replay on
+# reconnect, but bounded against a runaway client looping a failing batch.
+UPLOAD_LIMIT = 60
+
+# req/min, per user across ALL projects (#1719). Caps the cross-project
+# amplification: without it a member of N projects could sustain
+# ``UPLOAD_LIMIT × N`` batch writes/min. Set above UPLOAD_LIMIT so a user legitimately
+# replaying reconnect deltas for a couple of projects is unaffected, but the
+# per-user ceiling holds regardless of how many projects the writes are spread over.
+GLOBAL_UPLOAD_LIMIT = 120
+
+
+def _client() -> redis.Redis:
+    global _pool
+    if _pool is None:
+        _pool = valkey.pool(valkey.DB_CACHE, decode_responses=True)
+    return redis.Redis(connection_pool=_pool)
+
+
+@bypass_when_disabled
+class SyncUploadThrottle(BaseThrottle):
+    """Per-(project, user) rate limit for ``POST /projects/{pk}/sync/``.
+
+    Fails CLOSED on Redis error (see module docstring), but the global operator
+    kill switch (ADR-0604) still bypasses it entirely — "off means off", including
+    this write-path guard — so the admin-facing "rate limiting disabled" status is
+    literally accurate.
+    """
+
+    def allow_request(self, request: Request, view: APIView) -> bool:
+        user = getattr(request, "user", None)
+        if user is None or not getattr(user, "is_authenticated", False):
+            return True  # auth layer rejects anonymous; nothing to bucket on
+        project_pk = view.kwargs.get("pk")
+        project_key = f"rate:sync_upload:{project_pk}:{user.pk}"
+        user_key = f"rate:sync_upload:user:{user.pk}"
+
+        try:
+            client = _client()
+            # Two independent atomic INCR+first-hit-EXPIRE counters (#1757). Each
+            # EVAL is its own atomic unit, so neither bucket can outlive its 60s
+            # window without a TTL even if the process dies between the two calls.
+            project_count = incr_with_ttl(client, project_key, 60)
+            user_count = incr_with_ttl(client, user_key, 60)
+        except redis.RedisError:
+            # Fail CLOSED on the write path (#1719): denying is safe (the offline
+            # client retries with backoff); failing open would remove the only DoS
+            # bound the instant Redis hiccups. See the module docstring.
+            logger.warning("SyncUploadThrottle: Redis error, failing closed (deny)")
+            self.wait_seconds = 60
+            return False
+
+        if project_count > UPLOAD_LIMIT or user_count > GLOBAL_UPLOAD_LIMIT:
+            self.wait_seconds = 60
+            return False
+        return True
+
+    def wait(self) -> float | None:
+        return getattr(self, "wait_seconds", None)
+
+
+@contextmanager
+def sync_batch_concurrency_slot(user_pk: object) -> Iterator[None]:
+    """Bound simultaneously in-flight sync-upload batches per user (#1756).
+
+    The ``SyncUploadThrottle`` caps requests/minute, but each accepted batch is a
+    heavy ``transaction.atomic()`` (up to ``TRUEPPM_SYNC_BATCH_MAX_ROWS`` row locks
+    plus a CPM recalc enqueue). Under lock contention a burst of *concurrent*
+    batches from one user can still tie up worker/DB resources within the rate cap.
+    This is a per-user Redis semaphore around the batch apply: INCR a slot on entry,
+    DECR it in a ``finally``. When the count exceeds
+    ``TRUEPPM_SYNC_MAX_CONCURRENT_BATCHES`` the slot is immediately released and the
+    request is rejected with a 429 (``Throttled``) carrying a short ``Retry-After``.
+
+    Two safety properties:
+
+    - **Slot-leak guard.** The counter carries a TTL
+      (``TRUEPPM_SYNC_INFLIGHT_TTL_SECONDS``), refreshed on every entry via
+      :func:`incrby_with_ttl`, so a worker that dies mid-apply (never reaching the
+      ``finally`` DECR) cannot leak its slot permanently — the whole counter is
+      reclaimed once no further batch refreshes it within the window.
+    - **Fail open.** Any Redis error degrades to running the batch *without* the
+      concurrency guard rather than blocking a legitimate upload. Unlike the rate
+      throttle (which fails closed because it is the only global DoS bound), the
+      per-minute rate cap here remains fully in force, so failing open on this
+      defense-in-depth layer never removes the hard bound.
+
+    Known precision limit (tracked in #2287): a single TTL-refreshed counter can
+    drift if an apply outlives ``TRUEPPM_SYNC_INFLIGHT_TTL_SECONDS`` (over-admit) or
+    a crash-leaked slot persists under sustained load (under-admit). Both are
+    bounded to this defense-in-depth layer — the fail-closed rate cap stays the hard
+    bound — so it ships as-is; a per-holder-token semaphore would make it exact.
+    """
+    cap = getattr(settings, "TRUEPPM_SYNC_MAX_CONCURRENT_BATCHES", 4)
+    ttl = getattr(settings, "TRUEPPM_SYNC_INFLIGHT_TTL_SECONDS", 120)
+    key = f"inflight:sync_upload:user:{user_pk}"
+
+    try:
+        client = _client()
+        # INCRBY 1 + refresh TTL atomically (#1757 helper): the slot count and its
+        # leak-guard TTL are set together, so a crash can't strand a wedged counter.
+        count = incrby_with_ttl(client, key, 1, ttl)
+    except redis.RedisError:
+        # Fail open — proceed without the guard; the rate throttle still bounds abuse.
+        logger.warning("sync_batch_concurrency_slot: Redis error, proceeding without slot")
+        yield
+        return
+
+    if count > cap:
+        # Over cap: release our own increment immediately (we are rejecting, not
+        # holding a slot) and ask the client to retry once in-flight work drains.
+        try:
+            client.decr(key)
+        except redis.RedisError:
+            # The TTL will reclaim the over-count; nothing else to do.
+            logger.warning("sync_batch_concurrency_slot: Redis error releasing over-cap slot")
+        raise Throttled(
+            wait=1,
+            detail=(
+                "Too many sync batches in flight for your account; retry once the "
+                "current uploads finish."
+            ),
+        )
+
+    try:
+        yield
+    finally:
+        # Release the held slot. A failure here is non-fatal: the TTL reclaims it.
+        try:
+            client.decr(key)
+        except redis.RedisError:
+            logger.warning(
+                "sync_batch_concurrency_slot: Redis error releasing slot; TTL will reclaim"
+            )