trueppm-api 0.4.0__py3-none-any.whl

trueppm_api/apps/agents/__init__.py

@@ -0,0 +1,6 @@
+"""Agents app — the OSS agent-as-actor extension point (ADR-0112).
+
+Phase 0 (#1805) ships the append-only, hash-chained ``AgentAction`` audit
+record and its ``record_agent_action`` service. The fuller agent-actor identity
+model and capability scopes are Phase 1 (#1063, 0.5).
+"""

trueppm_api/apps/agents/apps.py

@@ -0,0 +1,11 @@
+"""AppConfig for the agents app — OSS agent-as-actor audit substrate (ADR-0112)."""
+
+from __future__ import annotations
+
+from django.apps import AppConfig
+
+
+class AgentsConfig(AppConfig):
+    name = "trueppm_api.apps.agents"
+    verbose_name = "Agents"
+    default_auto_field = "django.db.models.BigAutoField"

trueppm_api/apps/agents/canonical.py

@@ -0,0 +1,73 @@
+"""Canonical serialization + record hashing for the agent-action chain (ADR-0112 RC1).
+
+This is the **single source of truth** for how an ``AgentAction`` is turned into the
+bytes that get hashed. ``record_agent_action`` uses it to compute a new row's
+``record_hash``; ``manage.py audit_verify`` uses the *same* function to recompute and
+compare. If the two ever diverged, every existing chain would appear tampered — so the
+canonicalization lives here, once, and both call it.
+
+The canonical form deliberately excludes ``prev_hash`` and ``record_hash`` themselves:
+``record_hash = sha256(prev_hash ‖ canonical(record))`` chains the predecessor's hash
+*around* the canonical body, so the body must not contain the link fields.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from trueppm_api.apps.agents.models import AgentAction
+
+
+def canonical_fields(action: AgentAction) -> dict[str, object]:
+    """Return the tamper-relevant fields of ``action`` in a stable, hashable dict.
+
+    Every field that defines *what happened* is included; the chain-link fields
+    (``prev_hash``/``record_hash``) are not, because the hash wraps them around this
+    body. FK ids are stringified so the JSON is deterministic across DB backends and
+    UUID/int PK types.
+    """
+
+    return {
+        "schema_version": action.schema_version,
+        "sequence": action.sequence,
+        "actor_kind": action.actor_kind,
+        "actor_token_prefix": action.actor_token_prefix,
+        "principal_id": str(action.principal_id) if action.principal_id else None,
+        "action": action.action,
+        "method": action.method,
+        "object_type": action.object_type,
+        "object_id": action.object_id,
+        "project_id": str(action.project_id) if action.project_id else None,
+        "capability_used": action.capability_used,
+        "verdict": action.verdict,
+        "refusal_reason": action.refusal_reason,
+        "payload_hash": action.payload_hash,
+        "engine_version": action.engine_version,
+        # summary + source_ip are hashed too so the human-readable narrative and the
+        # recorded origin are tamper-evident, not just the structured decision fields.
+        "summary": action.summary,
+        "source_ip": action.source_ip,
+        # ISO-8601 with tz; the exact string is fixed at write time and rehashed here.
+        "occurred_at": action.occurred_at.isoformat(),
+    }
+
+
+def canonical_json(fields: dict[str, object]) -> str:
+    """Deterministic JSON: sorted keys, no incidental whitespace."""
+
+    return json.dumps(fields, sort_keys=True, separators=(",", ":"))
+
+
+def compute_record_hash(prev_hash: str, fields: dict[str, object]) -> str:
+    """``sha256(prev_hash ‖ '\\n' ‖ canonical_json(fields))`` as lowercase hex.
+
+    The newline separator domain-separates the (fixed-length hex) ``prev_hash`` from the
+    canonical body so no body could ever be constructed to collide with a prev_hash
+    boundary.
+    """
+
+    payload = f"{prev_hash}\n{canonical_json(fields)}".encode()
+    return hashlib.sha256(payload).hexdigest()

trueppm_api/apps/agents/deferred.py

@@ -0,0 +1,108 @@
+"""Post-transaction delivery for refusal audit rows (#3017, ADR-0902).
+
+``ATOMIC_REQUESTS = True`` wraps every view in a transaction, and DRF's
+``exception_handler`` calls ``set_rollback()`` for **every** ``APIException``. So a
+refusal audit written where the refusal is detected — inside ``finalize_response`` or
+inside an authenticator — executes its INSERT and then has it discarded when the
+request transaction rolls back. The measured effect (#3017) was an agent-action log
+containing only successes: a 403 from an MCP guard and a 401 from a revoked token both
+left zero rows, while the caller-facing refusal envelope worked fine. The half an
+auditor reads was the half that was lost.
+
+The fix is to move *when* the row is written, not *who* writes it.
+:func:`queue_agent_action` stashes the fully-resolved kwargs on the request, and
+:class:`~trueppm_api.apps.agents.middleware.AgentActionAuditMiddleware` drains the
+queue on the way out. ``ATOMIC_REQUESTS`` wraps only the view callable
+(``BaseHandler._get_response``), so by the time middleware processes the response the
+request transaction has already committed or rolled back and the connection is back in
+autocommit — a write issued there cannot be taken down with it.
+
+Two properties this deliberately preserves:
+
+**The chain stays single-writer.** ``record_agent_action`` is still the only writer and
+still serializes on ``select_for_update`` over the singleton chain head. Deferring
+changes the transaction the append runs in, not the lock that orders it, so two
+requests draining concurrently interleave exactly as two inline appends would. This is
+why the fix is a *timing* change rather than a second connection or an outbox worker,
+either of which would have introduced a second writer to a hash-chained table.
+
+**Refusal audits stay best-effort.** A refusal is already the safe outcome; a failed
+audit write must never convert a 401/403 into a 500. Failures are logged and swallowed.
+That is the inverse of the *allowed*-read path, which stays inline and fail-closed —
+an allowed read must be atomic with its own audit, because serving data we could not
+record is the outcome the substrate exists to prevent. The asymmetry is forced by the
+transaction semantics, not a style choice: an allowed read has something to be atomic
+with, and a refusal has nothing.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+#: Request attribute holding the pending audit writes for this request.
+#:
+#: Underscore-prefixed to match the ``_trueppm_refusal`` / ``_mcp_scope_filtered``
+#: convention for request-local markers set by one layer and read by another.
+QUEUE_ATTR = "_trueppm_deferred_agent_actions"
+
+
+def _host_request(request: Any) -> Any:
+    """Resolve the underlying Django ``HttpRequest`` behind a DRF ``Request``.
+
+    Producers run at different layers: the permission mixin holds a DRF ``Request``,
+    the authenticator holds one too, and the middleware that drains the queue only ever
+    sees the Django ``HttpRequest``. ``rest_framework.request.Request`` proxies attribute
+    *reads* to ``_request`` but stores writes on itself, so queueing onto the DRF wrapper
+    would put the list somewhere the middleware can never find it.
+    """
+
+    return getattr(request, "_request", request)
+
+
+def queue_agent_action(request: Any, **kwargs: Any) -> None:
+    """Queue one ``record_agent_action(**kwargs)`` to run after the request transaction.
+
+    Every value must be resolved by the caller — the drain runs after the view has
+    returned, so it must not depend on anything request-scoped that is torn down (or
+    lazily re-evaluated) on the way out.
+    """
+
+    host = _host_request(request)
+    queue = getattr(host, QUEUE_ATTR, None)
+    if queue is None:
+        queue = []
+        setattr(host, QUEUE_ATTR, queue)
+    queue.append(kwargs)
+
+
+def drain_agent_actions(request: Any) -> int:
+    """Write every queued audit row and return how many were recorded.
+
+    Called by the middleware once the request transaction has closed. Each row gets its
+    own attempt so one failure cannot silently drop the rest of the queue, and every
+    failure is logged rather than raised — see the module docstring on why a refusal
+    audit is best-effort.
+    """
+
+    host = _host_request(request)
+    queue = getattr(host, QUEUE_ATTR, None)
+    if not queue:
+        return 0
+    # Clear before writing so a re-entrant drain (or a middleware that calls the
+    # response path twice) can never double-append to a hash chain.
+    setattr(host, QUEUE_ATTR, [])
+
+    from trueppm_api.apps.agents.services import record_agent_action
+
+    written = 0
+    for kwargs in queue:
+        try:
+            record_agent_action(**kwargs)
+        except Exception:
+            logger.warning("deferred agent-action audit failed", exc_info=True)
+        else:
+            written += 1
+    return written

trueppm_api/apps/agents/management/commands/audit_prune.py

@@ -0,0 +1,137 @@
+"""``manage.py audit_prune`` — chain-aware, admin-triggered pruning of the agent-action log.
+
+The ``AgentAction`` hash chain (ADR-0112) is append-only and grows unbounded. This command
+is the OSS operator lever to bound it (ADR-0361): it deletes a contiguous **prefix** of the
+oldest rows and writes an ``AgentActionCheckpoint`` so ``manage.py audit_verify`` still
+verifies the surviving tail. It is **admin-triggered, never automatic** — OSS ships no Beat
+schedule for it; automatic/enforced retention, legal hold, and the off-server archive are the
+Enterprise layer (ADR-0112 §3, #146).
+
+Exactly one window is required:
+  * ``--before <ISO-8601>`` — delete rows older than the given timestamp;
+  * ``--keep-days <N>`` — delete rows older than N days;
+  * ``--keep-last <K>`` — keep the newest K rows, delete the rest.
+
+It is a **dry-run by default** and only reports the eligible count and resulting anchor;
+``--commit`` performs the deletion (prompting for confirmation unless ``--yes`` is given).
+So it can never silently or accidentally delete audit history.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any
+
+from django.contrib.auth import get_user_model
+from django.core.management.base import BaseCommand, CommandError
+from django.utils import timezone
+from django.utils.dateparse import parse_date, parse_datetime
+
+from trueppm_api.apps.agents.services import prune_agent_actions
+
+
+class Command(BaseCommand):
+    help = "Prune the oldest agent-action rows, re-anchoring the hash chain (ADR-0361)."
+
+    def add_arguments(self, parser: Any) -> None:
+        window = parser.add_mutually_exclusive_group(required=True)
+        window.add_argument(
+            "--before",
+            type=str,
+            help="Delete rows with occurred_at before this ISO-8601 date/datetime.",
+        )
+        window.add_argument("--keep-days", type=int, help="Delete rows older than this many days.")
+        window.add_argument(
+            "--keep-last", type=int, help="Keep the newest K rows; delete the rest."
+        )
+        parser.add_argument(
+            "--commit",
+            action="store_true",
+            help="Actually delete the rows. Without this the command is a dry-run.",
+        )
+        parser.add_argument(
+            "--yes",
+            action="store_true",
+            help="Skip the interactive confirmation prompt on --commit.",
+        )
+        parser.add_argument(
+            "--actor",
+            type=str,
+            default=None,
+            help="Username to attribute the prune to (recorded on the checkpoint).",
+        )
+
+    def _parse_before(self, raw: str) -> datetime:
+        """Parse ``--before`` into an aware datetime, accepting a bare date or a datetime."""
+        parsed = parse_datetime(raw)
+        if parsed is None:
+            as_date = parse_date(raw)
+            if as_date is not None:
+                parsed = datetime(as_date.year, as_date.month, as_date.day)
+        if parsed is None:
+            raise CommandError(f"--before: could not parse '{raw}' as an ISO-8601 date/datetime.")
+        if timezone.is_naive(parsed):
+            parsed = timezone.make_aware(parsed)
+        return parsed
+
+    def handle(self, *args: Any, **options: Any) -> None:
+        before = self._parse_before(options["before"]) if options.get("before") else None
+        keep_days: int | None = options.get("keep_days")
+        keep_last: int | None = options.get("keep_last")
+        commit: bool = options["commit"]
+        actor = self._resolve_actor(options.get("actor"))
+
+        # Dry-run first so we can show the operator exactly what --commit would do.
+        preview = prune_agent_actions(
+            before=before, keep_days=keep_days, keep_last=keep_last, commit=False
+        )
+
+        if preview.eligible == 0:
+            self.stdout.write(self.style.SUCCESS("Nothing to prune — no rows match the window."))
+            return
+
+        summary = (
+            f"{preview.eligible} row(s) eligible → would prune through sequence "
+            f"{preview.cutoff_sequence}; chain re-anchors at sequence "
+            f"{preview.first_retained_sequence}."
+        )
+
+        if not commit:
+            self.stdout.write(summary)
+            self.stdout.write(
+                self.style.WARNING("Dry-run only. Re-run with --commit to delete these rows.")
+            )
+            return
+
+        if not options["yes"]:
+            confirm = input(f"{summary}\nDelete {preview.eligible} row(s)? [y/N] ").strip().lower()
+            if confirm not in {"y", "yes"}:
+                self.stdout.write(self.style.WARNING("Aborted — nothing deleted."))
+                return
+
+        # Pin the previewed cutoff so appends between preview and commit cannot enlarge a
+        # relative window and delete more than the operator confirmed.
+        try:
+            result = prune_agent_actions(
+                through_sequence=preview.cutoff_sequence, commit=True, actor=actor
+            )
+        except Exception as exc:  # a legal-hold receiver (Enterprise) may veto the prune
+            raise CommandError(f"Prune aborted — nothing deleted: {exc}") from exc
+
+        self.stdout.write(
+            self.style.SUCCESS(
+                f"Pruned {result.deleted} row(s) through sequence {result.cutoff_sequence}; "
+                f"checkpoint written. Chain now starts at sequence "
+                f"{result.first_retained_sequence}."
+            )
+        )
+
+    def _resolve_actor(self, username: str | None) -> Any | None:
+        """Resolve ``--actor`` to a user, or ``None`` when unset (the checkpoint allows NULL)."""
+        if not username:
+            return None
+        user_model = get_user_model()
+        try:
+            return user_model.objects.get(username=username)
+        except user_model.DoesNotExist as exc:
+            raise CommandError(f"--actor: no user with username '{username}'.") from exc

trueppm_api/apps/agents/management/commands/audit_verify.py

@@ -0,0 +1,101 @@
+"""``manage.py audit_verify`` — the OSS agent-action chain integrity self-check (ADR-0112 RC1).
+
+Walks the append-only ``AgentAction`` chain in ``sequence`` order and, for each row,
+recomputes ``record_hash`` from the *previous* row's ``record_hash`` and this row's
+canonical body — the same ``compute_record_hash`` the recorder used. It reports the
+first divergence and exits non-zero, so a team (or a cron/CI check) can detect whether
+its own log was altered on its own instance. It **proves** nothing to an external
+auditor — that is the Enterprise notarized/signed archive (#146); this command *detects*
+local tampering.
+
+Checks, per row:
+  * ``sequence`` is gap-free and strictly increasing (a gap/reorder is a tamper signal);
+  * ``prev_hash`` equals the predecessor's ``record_hash`` (the chain links);
+  * ``record_hash`` recomputes from ``prev_hash ‖ canonical(record)`` (the row is intact).
+
+If the oldest rows have been pruned (``manage.py audit_prune``, ADR-0361), the walk seeds
+from the **latest** ``AgentActionCheckpoint`` — the last-deleted row's ``record_hash`` and
+the sequence at which the retained chain resumes — instead of genesis. A prune leaves a
+matching checkpoint, so the surviving tail still verifies; a deletion *without* a checkpoint
+still surfaces as a break at the first surviving row. The checkpoint is an unauthenticated
+continuity anchor, not a signature: this local self-check cannot detect a prune paired with a
+*forged* checkpoint (an actor with DB write can set both) — proving to an external auditor
+that history was not selectively truncated is the Enterprise notarized/signed archive (#146).
+
+Exit status: 0 = chain intact (or empty); 1 = a break was found.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from django.core.management.base import BaseCommand, CommandError
+
+from trueppm_api.apps.agents.canonical import canonical_fields, compute_record_hash
+from trueppm_api.apps.agents.models import (
+    GENESIS_PREV_HASH,
+    AgentAction,
+    AgentActionCheckpoint,
+)
+
+
+class Command(BaseCommand):
+    help = "Verify the integrity of the append-only agent-action hash chain (ADR-0112)."
+
+    def add_arguments(self, parser: Any) -> None:
+        parser.add_argument(
+            "--quiet",
+            action="store_true",
+            help="Suppress the per-run summary; only emit output on a detected break.",
+        )
+
+    def handle(self, *args: Any, **options: Any) -> None:
+        quiet: bool = options["quiet"]
+
+        # Re-anchor from the latest prune checkpoint if the oldest rows were pruned;
+        # otherwise the chain still starts at genesis / sequence 1 (ADR-0361).
+        checkpoint = AgentActionCheckpoint.objects.order_by("-pruned_through_sequence").first()
+        if checkpoint is None:
+            expected_prev = GENESIS_PREV_HASH
+            expected_sequence = 1
+            anchor = "genesis"
+        else:
+            expected_prev = checkpoint.pruned_through_hash
+            expected_sequence = checkpoint.first_retained_sequence
+            anchor = f"checkpoint @seq {checkpoint.pruned_through_sequence}"
+        count = 0
+
+        # Stream in chain order; .iterator() keeps memory flat for a long chain.
+        for action in AgentAction.objects.order_by("sequence").iterator():
+            if action.sequence != expected_sequence:
+                raise CommandError(
+                    f"Chain break at sequence {action.sequence} ({action.id}): "
+                    f"expected sequence {expected_sequence} (gap or reorder)."
+                )
+            if action.prev_hash != expected_prev:
+                raise CommandError(
+                    f"Chain break at sequence {action.sequence} ({action.id}): "
+                    f"prev_hash does not match the predecessor's record_hash."
+                )
+            recomputed = compute_record_hash(action.prev_hash, canonical_fields(action))
+            if recomputed != action.record_hash:
+                raise CommandError(
+                    f"Chain break at sequence {action.sequence} ({action.id}): "
+                    f"record_hash does not recompute — the row was altered."
+                )
+
+            expected_prev = action.record_hash
+            expected_sequence += 1
+            count += 1
+
+        if not quiet:
+            if count == 0:
+                self.stdout.write(
+                    self.style.SUCCESS(f"Agent-action chain empty (from {anchor}) — OK.")
+                )
+            else:
+                self.stdout.write(
+                    self.style.SUCCESS(
+                        f"Agent-action chain intact — {count} records verified (from {anchor})."
+                    )
+                )

trueppm_api/apps/agents/middleware.py

@@ -0,0 +1,44 @@
+"""Drain the deferred agent-action audit queue after the request transaction (#3017).
+
+See :mod:`trueppm_api.apps.agents.deferred` for why the write has to happen here and
+not where the refusal is detected.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+
+from django.http import HttpRequest, HttpResponse
+
+from trueppm_api.apps.agents.deferred import drain_agent_actions
+
+
+class AgentActionAuditMiddleware:
+    """Write queued agent-action audit rows once the view's transaction has closed.
+
+    ``ATOMIC_REQUESTS`` wraps only the view callable, so on the response path the
+    connection is back in autocommit and ``record_agent_action``'s own
+    ``transaction.atomic()`` opens a fresh transaction that commits independently of
+    whatever the request did. That is the whole mechanism: a row written here survives
+    the ``set_rollback()`` DRF performs for every refusal.
+
+    Placed **last** in ``MIDDLEWARE`` so its response half runs first on the way out —
+    as close to the view as a middleware can get, which keeps the audit row's
+    ``occurred_at`` honest and writes it while the request's DB connection is
+    certainly still open.
+
+    The drain is in a ``finally`` so a middleware below raising cannot strand the
+    queue. It is deliberately **not** ``async_capable``: ``record_agent_action`` does
+    blocking ORM work, and leaving this sync lets Django run it in the same threadpool
+    context it already uses for :class:`~trueppm_api.core.middleware.RejectNullBytesMiddleware`
+    rather than inviting a DB call onto the event loop.
+    """
+
+    def __init__(self, get_response: Callable[[HttpRequest], HttpResponse]) -> None:
+        self.get_response = get_response
+
+    def __call__(self, request: HttpRequest) -> HttpResponse:
+        try:
+            return self.get_response(request)
+        finally:
+            drain_agent_actions(request)