trueppm-api 0.4.0b2__py3-none-any.whl

trueppm_api/apps/idempotency/migrations/0001_initial.py

@@ -0,0 +1,71 @@
+"""Initial migration for the idempotency app (ADR-0170)."""
+
+from __future__ import annotations
+
+import uuid
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    initial = True
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="IdempotencyKey",
+            fields=[
+                (
+                    "id",
+                    models.UUIDField(
+                        default=uuid.uuid4,
+                        editable=False,
+                        primary_key=True,
+                        serialize=False,
+                    ),
+                ),
+                ("key", models.CharField(max_length=255)),
+                ("method", models.CharField(max_length=8)),
+                ("path", models.CharField(max_length=512)),
+                ("request_hash", models.CharField(max_length=64)),
+                (
+                    "status",
+                    models.CharField(
+                        choices=[
+                            ("processing", "Processing"),
+                            ("completed", "Completed"),
+                        ],
+                        default="processing",
+                        max_length=10,
+                    ),
+                ),
+                ("response_status", models.SmallIntegerField(blank=True, null=True)),
+                ("response_body", models.JSONField(blank=True, null=True)),
+                ("response_headers", models.JSONField(blank=True, null=True)),
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                (
+                    "user",
+                    models.ForeignKey(
+                        db_index=False,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="idempotency_keys",
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+            ],
+            options={
+                "indexes": [models.Index(fields=["created_at"], name="idempotency_created_idx")],
+                "constraints": [
+                    models.UniqueConstraint(
+                        fields=["user", "key"],
+                        name="idempotency_key_unique_per_user",
+                    )
+                ],
+            },
+        ),
+    ]

trueppm_api/apps/idempotency/mixins.py

@@ -0,0 +1,226 @@
+"""DRF dispatch-layer idempotency mixin (ADR-0170).
+
+The hook lives at the DRF dispatch layer rather than in Django middleware because
+``ATOMIC_REQUESTS = True``: the per-request transaction wraps only the view call, so a
+Django ``process_response`` middleware runs *outside* it and cannot store the response
+row atomically with the mutation. ``initial()`` and ``finalize_response()`` both run
+inside that transaction and after authentication, which is exactly what we need.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+from typing import Any
+
+from django.conf import settings
+from django.db import IntegrityError, connection, transaction
+from rest_framework.exceptions import APIException, ValidationError
+from rest_framework.request import Request
+from rest_framework.response import Response
+
+from trueppm_api.apps.idempotency.models import IdempotencyKey
+
+logger = logging.getLogger(__name__)
+
+UNSAFE_METHODS = frozenset({"POST", "PUT", "PATCH", "DELETE"})
+HEADER = "Idempotency-Key"
+# Headers replayed verbatim on a cache hit. Kept minimal on purpose.
+SAFELISTED_RESPONSE_HEADERS = ("Location",)
+DEFAULT_MAX_BODY_BYTES = 1 * 1024 * 1024  # 1 MiB
+MAX_KEY_LENGTH = 255  # matches IdempotencyKey.key max_length
+
+# Sentinel: response is not replayable (5xx, streaming, non-JSON, or oversized).
+_NOT_STORABLE = object()
+
+
+class IdempotencyConflict(APIException):
+    """Raised when a key is reused with a different request hash."""
+
+    status_code = 422
+    default_detail = "Idempotency-Key was reused with a different request."
+    default_code = "idempotency_key_conflict"
+
+
+class _IdempotentReplay(Exception):
+    """Internal control-flow signal: replay a stored response instead of running the view."""
+
+    def __init__(self, row: IdempotencyKey) -> None:
+        self.row = row
+        super().__init__("idempotent replay")
+
+
+class IdempotencyMixin:
+    """Adds ``Idempotency-Key`` replay/store semantics to a DRF view.
+
+    Apply by inheriting before the DRF base (so its ``initial`` /
+    ``finalize_response`` / ``handle_exception`` overrides precede ``APIView`` in the
+    MRO). Set ``idempotency_exempt = True`` to opt a view out (auth/token endpoints,
+    already-deduped outbox entrypoints, streaming/export views).
+    """
+
+    idempotency_exempt: bool = False
+
+    def initial(self, request: Request, *args: Any, **kwargs: Any) -> None:
+        # Run authentication/throttling/negotiation first so request.user is resolved.
+        super().initial(request, *args, **kwargs)  # type: ignore[misc]
+
+        self._idempotency_row: IdempotencyKey | None = None
+
+        if self.idempotency_exempt or request.method not in UNSAFE_METHODS:
+            return
+        key = request.headers.get(HEADER)
+        if not key:
+            return  # opt-in: no header is a no-op, preserving prior behavior
+        user = getattr(request, "user", None)
+        if user is None or not user.is_authenticated:
+            return
+        if len(key) > MAX_KEY_LENGTH:
+            raise ValidationError(
+                {HEADER: f"Idempotency-Key must be {MAX_KEY_LENGTH} characters or fewer."}
+            )
+        # Requests larger than the body cap aren't idempotency-protected — skip rather than
+        # buffer+hash a very large body. They run normally (no claim stored).
+        try:
+            content_length = int(request.META.get("CONTENT_LENGTH") or 0)
+        except (TypeError, ValueError):
+            content_length = 0
+        max_bytes = getattr(settings, "IDEMPOTENCY_MAX_BODY_BYTES", DEFAULT_MAX_BODY_BYTES)
+        if content_length > max_bytes:
+            return
+
+        request_hash = self._compute_request_hash(request)
+
+        existing = IdempotencyKey.objects.filter(user=user, key=key).first()
+        if existing is not None:
+            self._resolve_existing(existing, request_hash)
+            return
+
+        # Miss → insert a claim row inside a savepoint so a concurrent-duplicate
+        # IntegrityError can be caught without poisoning the outer request transaction.
+        try:
+            with transaction.atomic():
+                self._idempotency_row = IdempotencyKey.objects.create(
+                    user=user,
+                    key=key,
+                    method=request.method,
+                    path=request.path[:512],
+                    request_hash=request_hash,
+                    status=IdempotencyKey.Status.PROCESSING,
+                )
+        except IntegrityError:
+            # A concurrent request claimed (user, key) first. Its INSERT blocked ours on
+            # the unique constraint until it committed; re-read the winner and replay.
+            winner = IdempotencyKey.objects.filter(user=user, key=key).first()
+            if winner is None:
+                # Winner rolled back (its mutation failed) after releasing the lock.
+                # Proceed without idempotency rather than failing the legitimate request.
+                return
+            self._resolve_existing(winner, request_hash)
+
+    def _resolve_existing(self, row: IdempotencyKey, request_hash: str) -> None:
+        """Replay a completed row, or raise 422 on a hash mismatch."""
+        if row.request_hash != request_hash:
+            raise IdempotencyConflict()
+        # A committed row is always COMPLETED (see model invariant); replay it.
+        raise _IdempotentReplay(row)
+
+    def handle_exception(self, exc: Exception) -> Response:
+        if isinstance(exc, _IdempotentReplay):
+            row = exc.row
+            response = Response(row.response_body, status=row.response_status)
+            # Re-apply the safelist on replay (defense in depth — never trust the stored
+            # dict to contain only safe headers).
+            for name, value in (row.response_headers or {}).items():
+                if name in SAFELISTED_RESPONSE_HEADERS:
+                    response[name] = value
+            response["Idempotent-Replay"] = "true"
+            return response
+        return super().handle_exception(exc)  # type: ignore[misc,no-any-return]
+
+    def finalize_response(
+        self, request: Request, response: Response, *args: Any, **kwargs: Any
+    ) -> Response:
+        response = super().finalize_response(request, response, *args, **kwargs)  # type: ignore[misc]
+        row = getattr(self, "_idempotency_row", None)
+        if row is None:
+            return response
+
+        # DRF's exception_handler calls set_rollback() for handled 4xx/5xx, marking the
+        # whole ATOMIC_REQUESTS transaction (including our claim row) for rollback. We must
+        # not query a broken transaction, and the claim is discarded with it — so a retry
+        # re-runs, which is correct for deterministic errors. Skip persistence entirely.
+        if connection.get_rollback():
+            return response
+
+        body = self._storable_body(response)
+        if body is _NOT_STORABLE:
+            # Non-storable (5xx, streaming, oversized): don't consume the key — delete
+            # the claim so a retry re-runs. Same transaction, so this is atomic too.
+            row.delete()
+            return response
+
+        row.status = IdempotencyKey.Status.COMPLETED
+        row.response_status = response.status_code
+        row.response_body = body
+        headers = {
+            name: response[name]
+            for name in SAFELISTED_RESPONSE_HEADERS
+            if response.has_header(name)
+        }
+        row.response_headers = headers or None
+        row.save(
+            update_fields=[
+                "status",
+                "response_status",
+                "response_body",
+                "response_headers",
+            ]
+        )
+        return response
+
+    def _storable_body(self, response: Response) -> Any:
+        """Return JSON-native body to store, or ``_NOT_STORABLE``.
+
+        Stores the DRF-rendered JSON round-tripped through ``json.loads`` rather than raw
+        ``response.data`` — the latter may hold dates/Decimals/UUIDs that the default
+        JSONField encoder cannot serialize.
+        """
+        # Never cache server errors — they propagate/roll back and must stay retryable.
+        if response.status_code >= 500:
+            return _NOT_STORABLE
+        if not hasattr(response, "data"):
+            return _NOT_STORABLE
+        from rest_framework.renderers import JSONRenderer
+
+        try:
+            rendered = JSONRenderer().render(response.data)
+        except (TypeError, ValueError):
+            return _NOT_STORABLE
+        max_bytes = getattr(settings, "IDEMPOTENCY_MAX_BODY_BYTES", DEFAULT_MAX_BODY_BYTES)
+        if len(rendered) > max_bytes:
+            return _NOT_STORABLE
+        if not rendered:
+            return None  # e.g. 204 No Content — DRF renders None as empty bytes
+        return json.loads(rendered)
+
+    @staticmethod
+    def _compute_request_hash(request: Request) -> str:
+        """sha256 of method + full path (with query) + raw body.
+
+        Read ``request.body`` here in ``initial()`` — before the handler accesses
+        ``request.data`` — so Django buffers the raw bytes and the view can still parse
+        the body downstream.
+        """
+        try:
+            body = request.body
+        except Exception:
+            body = b""
+        hasher = hashlib.sha256()
+        hasher.update((request.method or "").encode("utf-8"))
+        hasher.update(b"\n")
+        hasher.update(request.get_full_path().encode("utf-8"))
+        hasher.update(b"\n")
+        hasher.update(body)
+        return hasher.hexdigest()

trueppm_api/apps/idempotency/models.py

@@ -0,0 +1,75 @@
+"""Models for client-driven HTTP idempotency (ADR-0170).
+
+A single model, :class:`IdempotencyKey`, stores the response to an unsafe mutation
+(POST/PATCH/PUT/DELETE) that carried an ``Idempotency-Key`` header so a retry with the
+same key replays the stored response instead of re-applying the write.
+"""
+
+from __future__ import annotations
+
+import uuid
+
+from django.conf import settings
+from django.db import models
+
+
+class IdempotencyKey(models.Model):
+    """Stored response for a client-supplied ``Idempotency-Key`` on an unsafe mutation.
+
+    Does NOT inherit ``VersionedModel`` — this is a server-side request-dedup record,
+    not synced to mobile clients.
+
+    Invariant (ADR-0170): a *committed* row is always ``status=COMPLETED``. The claim
+    row is inserted (``PROCESSING``) and updated to ``COMPLETED`` inside the same
+    ``ATOMIC_REQUESTS`` transaction as the mutation, so a rolled-back mutation (5xx /
+    unhandled exception) leaves no row and a retry re-runs. The ``PROCESSING`` state is
+    therefore only ever observed transiently *within* the originating request's open
+    transaction; it serves the concurrent-duplicate blocking semantics (a second request
+    with the same key blocks on the unique constraint until the first transaction ends).
+    """
+
+    class Status(models.TextChoices):
+        PROCESSING = "processing", "Processing"
+        COMPLETED = "completed", "Completed"
+
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    user = models.ForeignKey(
+        settings.AUTH_USER_MODEL,
+        on_delete=models.CASCADE,
+        related_name="idempotency_keys",
+        # The (user, key) unique constraint already indexes user_id as its left prefix,
+        # so the implicit per-FK index would be redundant.
+        db_index=False,
+    )
+    # Client-supplied key. A UUID is recommended but any opaque string is accepted;
+    # uniqueness is scoped per-user so a leaked key cannot replay another user's response.
+    key = models.CharField(max_length=255)
+    method = models.CharField(max_length=8)
+    path = models.CharField(max_length=512)
+    # sha256 hex of method + path-with-query + raw body. A reused key with a different
+    # request hash is rejected with 422 (idempotency_key_conflict).
+    request_hash = models.CharField(max_length=64)
+    status = models.CharField(
+        max_length=10,
+        choices=Status.choices,
+        default=Status.PROCESSING,
+    )
+    response_status = models.SmallIntegerField(null=True, blank=True)
+    response_body = models.JSONField(null=True, blank=True)
+    # Safelisted response headers replayed verbatim (currently only Location).
+    response_headers = models.JSONField(null=True, blank=True)
+    created_at = models.DateTimeField(auto_now_add=True)
+
+    class Meta:
+        constraints = [
+            models.UniqueConstraint(
+                fields=["user", "key"],
+                name="idempotency_key_unique_per_user",
+            ),
+        ]
+        indexes = [
+            models.Index(fields=["created_at"], name="idempotency_created_idx"),
+        ]
+
+    def __str__(self) -> str:
+        return f"IdempotencyKey({self.method} {self.path} {self.status})"

trueppm_api/apps/idempotency/tasks.py

@@ -0,0 +1,61 @@
+"""Celery tasks for the idempotency app — retention purge (ADR-0170)."""
+
+from __future__ import annotations
+
+import logging
+from datetime import timedelta
+from typing import Any
+
+from django.conf import settings
+from django.db import OperationalError
+from django.utils import timezone
+
+from trueppm_api.core.idempotent import idempotent_task
+
+logger = logging.getLogger(__name__)
+
+
+def _do_purge(retention_hours: int) -> int:
+    """Delete IdempotencyKey rows older than the retention window. Returns deleted count."""
+    from trueppm_api.apps.idempotency.models import IdempotencyKey
+
+    cutoff = timezone.now() - timedelta(hours=retention_hours)
+    deleted, _ = IdempotencyKey.objects.filter(created_at__lt=cutoff).delete()
+    return deleted
+
+
+@idempotent_task(
+    lock_key_template="idempotency_purge",
+    lock_ttl=300,
+    on_contention="skip",
+    name="idempotency.purge_old_keys",
+    autoretry_for=(ConnectionError, OperationalError),
+    retry_backoff=60,
+    retry_backoff_max=600,
+    retry_jitter=True,
+    max_retries=3,
+    soft_time_limit=120,
+    time_limit=180,
+    acks_late=True,
+)
+def purge_old_idempotency_keys(self: object) -> dict[str, Any]:
+    """Delete IdempotencyKey rows older than IDEMPOTENCY_RETENTION_HOURS.
+
+    Registered in CELERY_BEAT_SCHEDULE to run hourly (a nightly job would let rows live
+    up to ~48h, violating the 24h contract). When IDEMPOTENCY_RETENTION_HOURS is None,
+    purging is disabled (enterprise unlimited-retention convention, matching the
+    *_RETENTION_DAYS = None pattern).
+    """
+    retention_hours: int | None = getattr(settings, "IDEMPOTENCY_RETENTION_HOURS", 24)
+    if retention_hours is None:
+        logger.info("purge_old_idempotency_keys: retention disabled, skipping")
+        return {"status": "skipped", "reason": "unlimited retention"}
+
+    deleted = _do_purge(retention_hours)
+    if deleted:
+        logger.info(
+            "purge_old_idempotency_keys: deleted %d rows older than %d hours",
+            deleted,
+            retention_hours,
+        )
+    return {"deleted": deleted}

trueppm_api/apps/integrations/__init__.py

@@ -0,0 +1,13 @@
+"""External integrations app (ADR-0049).
+
+Owns ``IntegrationCredential`` (per-user PAT storage) and the three provider
+registries — ``TASK_LINK_PROVIDERS``, ``OUTGOING_CHANNEL_PROVIDERS``,
+``NOTIFICATION_CHANNELS`` — that Enterprise registers richer connectors
+against at ``AppConfig.ready()``.
+
+``TaskLink``, the outgoing webhook ``format`` field, and the email
+notifications preference table all hang off this app's extension points and
+ship in successor issues (#637 / #638 / #639).
+"""
+
+default_app_config = "trueppm_api.apps.integrations.apps.IntegrationsConfig"

trueppm_api/apps/integrations/apps.py

@@ -0,0 +1,59 @@
+"""App config for ``apps/integrations`` (ADR-0049)."""
+
+from __future__ import annotations
+
+from django.apps import AppConfig
+
+
+class IntegrationsConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "trueppm_api.apps.integrations"
+    verbose_name = "Integrations"
+
+    def ready(self) -> None:
+        """Register OSS provider classes against the three ADR-0049 registries.
+
+        Enterprise's ``AppConfig.ready()`` registers richer providers
+        against the same registries at the same hook — no OSS code changes
+        required when Enterprise lights up Jira / ServiceNow / Slack App.
+
+        ``NOTIFICATION_CHANNELS`` is populated below with the OSS ``in_app`` /
+        ``email`` channels, but **nothing consumes the registry yet** — channel
+        validation still runs off the ``ProjectNotificationChannel`` TextChoices,
+        so a channel registered here is not thereby accepted. Wiring it is #3252.
+        Contrast ``OUTGOING_CHANNEL_PROVIDERS``, populated here by #638 with the
+        OSS ``generic`` and ``slack`` renderers, which *is* consumed — see
+        ``apps/webhooks/serializers.py`` and ``apps/webhooks/dispatch.py``.
+        """
+        from .external_sources import EXTERNAL_TASK_SOURCES, OSS_EXTERNAL_TASK_SOURCES
+        from .notification_channels import OSS_NOTIFICATION_CHANNELS
+        from .outgoing import OSS_OUTGOING_CHANNEL_PROVIDERS
+        from .providers import OSS_TASK_LINK_PROVIDERS
+        from .registry import (
+            NOTIFICATION_CHANNELS,
+            OUTGOING_CHANNEL_PROVIDERS,
+            TASK_LINK_PROVIDERS,
+        )
+
+        # EXTERNAL_TASK_SOURCES (ADR-0097): user-scoped read-only pull sources.
+        # A separate registry from TASK_LINK_PROVIDERS — OSS owns ``jira`` here
+        # for personal read-only pull; Enterprise appends governance sources.
+        for source in OSS_EXTERNAL_TASK_SOURCES:
+            if EXTERNAL_TASK_SOURCES.get(source.key) is None:
+                EXTERNAL_TASK_SOURCES.register(source.key, source)
+
+        for handler in OSS_TASK_LINK_PROVIDERS:
+            # Idempotent: skip if a re-import (e.g. tests reloading the app)
+            # already registered the handler. ``register()`` raises on the
+            # duplicate-key path; the registry-clearing helper used by tests
+            # avoids that path by re-registering deliberately.
+            if TASK_LINK_PROVIDERS.get(handler.key) is None:
+                TASK_LINK_PROVIDERS.register(handler.key, handler)
+
+        for channel in OSS_OUTGOING_CHANNEL_PROVIDERS:
+            if OUTGOING_CHANNEL_PROVIDERS.get(channel.key) is None:
+                OUTGOING_CHANNEL_PROVIDERS.register(channel.key, channel)
+
+        for notif_channel in OSS_NOTIFICATION_CHANNELS:
+            if NOTIFICATION_CHANNELS.get(notif_channel.key) is None:
+                NOTIFICATION_CHANNELS.register(notif_channel.key, notif_channel)