trueppm-api 0.4.0b2__py3-none-any.whl

trueppm_api/apps/agents/services.py

@@ -0,0 +1,397 @@
+"""Agent-action recording service (ADR-0112 RC1, #1805).
+
+``record_agent_action`` is the single writer of the hash chain. It allocates the next
+per-instance ``sequence``, links to the predecessor's ``record_hash``, writes the
+append-only ``AgentAction`` row, and advances the chain head — all under one
+``select_for_update`` on the singleton ``AgentActionChainHead`` so two concurrent
+callers cannot interleave the chain (ADR-0112's mechanism (a): the lock also protects
+the hash linkage, which a bare DB sequence would not). The ``agent_action_recorded``
+signal is dispatched in ``on_commit`` so it fires only once the row and its link are
+durable.
+
+The write is **fail-closed**: it runs inside the request's atomic block, so if the
+append fails the whole request rolls back rather than serving an un-audited read — an
+audit substrate must never return a read it could not record.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import ipaddress
+import json
+from dataclasses import dataclass
+from datetime import timedelta
+from typing import TYPE_CHECKING, Any
+
+from django.db import transaction
+from django.utils import timezone
+
+from trueppm_api.apps.agents.canonical import canonical_fields, compute_record_hash
+from trueppm_api.apps.agents.models import (
+    SAMPLE_SUMMARY_PREFIX,
+    SAMPLE_TOKEN_PREFIX,
+    AgentAction,
+    AgentActionChainHead,
+    AgentActionCheckpoint,
+    AgentActionRefusalDetail,
+    AgentActionVerdict,
+    AgentActorKind,
+)
+from trueppm_api.apps.agents.signals import agent_action_prune_requested, agent_action_recorded
+from trueppm_api.core.extension_signals import dispatch_extension_signal
+
+if TYPE_CHECKING:
+    from datetime import datetime
+
+    from rest_framework.request import Request
+
+    from trueppm_api.apps.projects.models import ProjectApiToken
+
+
+def _safe_ip(value: str | None) -> str | None:
+    """Return ``value`` only if it parses as a valid IP address, else ``None``.
+
+    Guards the fail-closed save: a forgeable ``X-Forwarded-For`` (which may carry a
+    hostname or garbage) must never raise on ``GenericIPAddressField`` and turn a valid
+    read into a 500.
+    """
+
+    if not value:
+        return None
+    try:
+        ipaddress.ip_address(value)
+    except ValueError:
+        return None
+    return value
+
+
+def engine_version() -> str:
+    """The trueppm-scheduler version at decision time — the answer-stamp engine anchor.
+
+    Resolved from the installed package metadata (never drifts from the wheel). Falls
+    back to ``"unknown"`` so a metadata hiccup can never break the audited request.
+    """
+
+    try:
+        from trueppm_scheduler import __version__
+
+        return str(__version__)
+    except Exception:
+        return "unknown"
+
+
+def hash_request_payload(request: Request) -> str:
+    """sha256 over the canonicalized request payload — the per-request reproducibility hash.
+
+    Hashes the method, path, sorted query parameters, and the request body when present
+    (a read usually has none). Never includes headers, so no token material is hashed.
+
+    Total by construction, and that has to be enforced rather than assumed. A malformed
+    body is hashed as its raw bytes. Reading ``request.body`` *after* something has
+    consumed the stream — which every DRF view that touches ``request.data`` on a
+    POST/PATCH has done by the time an audit runs — raises ``RawPostDataException``, and
+    ``getattr(..., default)`` does not swallow it (the default only covers
+    ``AttributeError``). Such a body is recorded as unavailable rather than raised: this
+    function runs on the audit path, where raising converts a clean refusal into a 500
+    (#3017).
+    """
+
+    parts: dict[str, Any] = {
+        "method": request.method,
+        "path": request.path,
+        "query": sorted(request.GET.items()),
+    }
+    try:
+        body = getattr(request, "body", b"") or b""
+    except Exception:
+        parts["body_sha256"] = "unavailable"
+        body = b""
+    if body:
+        parts["body_sha256"] = hashlib.sha256(body).hexdigest()
+    canonical = json.dumps(parts, sort_keys=True, separators=(",", ":"), default=str)
+    return hashlib.sha256(canonical.encode()).hexdigest()
+
+
+def _attribution(
+    actor_token: ProjectApiToken | None, summary: str, *, sample: bool
+) -> tuple[str, str]:
+    """Return the hashed ``(actor_token_prefix, summary)`` pair for a new chain row.
+
+    A sample row replaces both with the demo markers, because the marker has to live
+    in hashed fields (see :func:`record_agent_action`).
+    """
+    if sample:
+        return SAMPLE_TOKEN_PREFIX, f"{SAMPLE_SUMMARY_PREFIX}{summary or ''}"
+    return (actor_token.token_prefix if actor_token is not None else ""), summary
+
+
+def _record_refusal_detail(
+    entry: AgentAction,
+    verdict: str,
+    refusal_constraint: str,
+    projected_impact: dict[str, Any] | None,
+) -> None:
+    """Create the non-hashed refusal side-car for a refusal that names a constraint."""
+    if verdict == AgentActionVerdict.REFUSED and refusal_constraint:
+        AgentActionRefusalDetail.objects.create(
+            action=entry,
+            constraint=refusal_constraint,
+            projected_impact=projected_impact or {},
+        )
+
+
+def record_agent_action(
+    *,
+    actor_kind: str = AgentActorKind.MCP_TOKEN,
+    actor_token: ProjectApiToken | None,
+    principal: Any | None,
+    action: str,
+    method: str,
+    capability_used: str,
+    verdict: str,
+    payload_hash: str,
+    refusal_reason: str = "",
+    refusal_constraint: str = "",
+    projected_impact: dict[str, Any] | None = None,
+    object_type: str = "",
+    object_id: str = "",
+    project_id: Any | None = None,
+    summary: str = "",
+    source_ip: str | None = None,
+    engine_version_str: str | None = None,
+    occurred_at: datetime | None = None,
+    sample: bool = False,
+) -> AgentAction:
+    """Append one hash-chained ``AgentAction`` row and advance the chain head.
+
+    ``sample=True`` is for the bundled-sample loader only (#3603). The row still
+    goes through the real chain — the agent-oversight panel reads nothing else —
+    but is marked as demo data in **hashed** fields: ``actor_token_prefix`` is
+    :data:`SAMPLE_TOKEN_PREFIX` and ``summary`` leads with
+    :data:`SAMPLE_SUMMARY_PREFIX`. The marker must live in the hash because the
+    row outlives its project (a sample reload SET_NULLs ``project``), and a marker
+    anyone could strip would let fabricated history pass as evidence; stripping
+    this one breaks ``audit_verify``. Sample rows also skip the
+    ``agent_action_recorded`` extension signal, so nothing downstream notarizes
+    them.
+
+    Serializes on the singleton chain head so the ``sequence`` is gap-free and each row
+    links to the true predecessor. Must be called inside a DB transaction (the request's
+    ATOMIC_REQUESTS block, or an explicit ``transaction.atomic``); the ``on_commit``
+    signal fires on the outermost commit.
+
+    When ``verdict`` is a refusal and ``refusal_constraint`` is given, a non-hashed
+    :class:`AgentActionRefusalDetail` side-car is created **in the same atomic block**
+    (ADR-0421, #1850) so the constraint that fired and the projected impact commit
+    together with the chain row — the two can never diverge. ``projected_impact`` is
+    ``{}`` for the current MCP-scope/identity producers (no schedule impact); the 0.6
+    gated-write producers populate it. The side-car is telemetry only; it is not part of
+    the hashed record, so it never affects ``audit_verify``.
+    """
+
+    if verdict != AgentActionVerdict.REFUSED:
+        # refusal_reason/constraint are meaningful only for a refusal; never persist a
+        # dangling one (a non-refusal must not carry a refusal side-car either).
+        refusal_reason = ""
+        refusal_constraint = ""
+
+    occurred = occurred_at or timezone.now()
+    token_prefix, summary = _attribution(actor_token, summary, sample=sample)
+    resolved_engine = engine_version_str or engine_version()
+    # Sanitize the client IP: a spoofed/malformed X-Forwarded-For must not raise on the
+    # GenericIPAddressField save and 500 an otherwise-valid (fail-closed) read.
+    safe_ip = _safe_ip(source_ip)
+
+    with transaction.atomic():
+        # Lock the chain head for the duration of the append so the predecessor link and
+        # the sequence allocation are atomic against any concurrent recorder.
+        head = AgentActionChainHead.objects.select_for_update().get(pk=1)
+        sequence = head.last_sequence + 1
+        prev_hash = head.last_record_hash
+
+        entry = AgentAction(
+            schema_version=AgentAction._meta.get_field("schema_version").default,
+            actor_kind=actor_kind,
+            actor_token=actor_token,
+            actor_token_prefix=token_prefix,
+            principal=principal if getattr(principal, "pk", None) else None,
+            action=action,
+            method=(method or "")[:8],
+            object_type=object_type or "",
+            object_id=str(object_id or "")[:64],
+            project_id=project_id,
+            capability_used=capability_used or "",
+            verdict=verdict,
+            refusal_reason=refusal_reason,
+            payload_hash=payload_hash,
+            engine_version=resolved_engine,
+            sequence=sequence,
+            prev_hash=prev_hash,
+            summary=summary or "",
+            source_ip=safe_ip,
+            occurred_at=occurred,
+        )
+        entry.record_hash = compute_record_hash(prev_hash, canonical_fields(entry))
+        entry.save(force_insert=True)
+
+        head.last_sequence = sequence
+        head.last_record_hash = entry.record_hash
+        head.save(update_fields=["last_sequence", "last_record_hash", "updated_at"])
+
+        # Non-hashed telemetry side-car (ADR-0421): same atomic block as the chain
+        # append so the detail and its row commit or roll back together. Only for a
+        # refusal that names a constraint — an allowed read never carries one.
+        _record_refusal_detail(entry, verdict, refusal_constraint, projected_impact)
+
+    if not sample:
+        transaction.on_commit(
+            lambda: dispatch_extension_signal(
+                agent_action_recorded, sender=AgentAction, action=entry
+            )
+        )
+    return entry
+
+
+@dataclass(frozen=True)
+class PruneResult:
+    """Outcome of a :func:`prune_agent_actions` call (dry-run or committed).
+
+    ``eligible`` is the number of rows the window selects; ``deleted`` is how many were
+    actually removed (0 on a dry-run). ``cutoff_sequence`` / ``cutoff_hash`` are the
+    prune floor — the highest sequence removed and its ``record_hash`` (the anchor a
+    ``checkpoint`` re-seeds ``audit_verify`` from). All ``None`` when nothing was eligible.
+    """
+
+    eligible: int
+    deleted: int
+    committed: bool
+    cutoff_sequence: int | None
+    cutoff_hash: str | None
+    first_retained_sequence: int | None
+    checkpoint_id: str | None
+
+
+def _resolve_cutoff(
+    head: AgentActionChainHead,
+    *,
+    before: datetime | None,
+    keep_days: int | None,
+    keep_last: int | None,
+) -> int:
+    """Resolve a window into a **prefix** cutoff sequence (delete rows ``<=`` the result).
+
+    Count-based (``keep_last``) keeps the newest K by sequence. Age-based
+    (``before`` / ``keep_days``) deletes the longest contiguous prefix whose rows are
+    **all** older than the cutoff — it stops at the first (lowest-sequence) row that is not
+    old enough, so a row younger than the cutoff is never deleted and the surviving chain
+    can never gain a mid-chain gap even if ``occurred_at`` is slightly out of sequence order.
+    """
+
+    if keep_last is not None:
+        # Newest K rows by sequence survive; everything at or below the floor is pruned.
+        return head.last_sequence - keep_last
+
+    cutoff_at = timezone.now() - timedelta(days=keep_days) if keep_days is not None else before
+    first_not_old = (
+        AgentAction.objects.filter(occurred_at__gte=cutoff_at).order_by("sequence").first()
+    )
+    if first_not_old is None:
+        # Every row is older than the cutoff — the prefix is the whole chain.
+        return head.last_sequence
+    return first_not_old.sequence - 1
+
+
+def prune_agent_actions(
+    *,
+    before: datetime | None = None,
+    keep_days: int | None = None,
+    keep_last: int | None = None,
+    through_sequence: int | None = None,
+    commit: bool = False,
+    actor: Any | None = None,
+) -> PruneResult:
+    """Chain-aware prune of the oldest ``AgentAction`` rows (ADR-0361, OSS admin-triggered).
+
+    Deletes a contiguous **prefix** (lowest sequences) selected by exactly one window and,
+    when it actually removes rows, writes an immutable :class:`AgentActionCheckpoint` so
+    ``audit_verify`` re-anchors from the last-deleted row's ``record_hash`` instead of
+    genesis. Runs under the chain-head ``select_for_update`` so it serializes against
+    ``record_agent_action`` and concurrent prunes. With ``commit=False`` (the default) it is
+    a dry-run: it reports the eligible count and resulting anchor without deleting or
+    checkpointing. A receiver of ``agent_action_prune_requested`` may raise to veto the whole
+    prune (the Enterprise legal-hold seam) — fail-closed inside the atomic block.
+
+    ``through_sequence`` pins the cutoff to an explicit sequence instead of resolving a
+    window. The command uses it to commit exactly the prefix it previewed, so rows appended
+    between the dry-run and the ``--commit`` cannot enlarge a relative (``keep_last`` /
+    ``keep_days``) cutoff and delete more than the operator confirmed.
+    """
+
+    given = [v for v in (before, keep_days, keep_last, through_sequence) if v is not None]
+    if len(given) != 1:
+        raise ValueError("Provide exactly one of before, keep_days, keep_last, through_sequence.")
+    if keep_last is not None and keep_last < 0:
+        raise ValueError("keep_last must be non-negative.")
+    if keep_days is not None and keep_days < 0:
+        raise ValueError("keep_days must be non-negative.")
+
+    with transaction.atomic():
+        # Same lock the writer takes: serializes prune vs. append and prune vs. prune, and
+        # keeps the rows at/below the cutoff stable (appends only add rows above the head).
+        head = AgentActionChainHead.objects.select_for_update().get(pk=1)
+        cutoff_sequence = (
+            through_sequence
+            if through_sequence is not None
+            else _resolve_cutoff(head, before=before, keep_days=keep_days, keep_last=keep_last)
+        )
+
+        qs = AgentAction.objects.filter(sequence__lte=cutoff_sequence)
+        eligible = qs.count()
+        if eligible == 0:
+            return PruneResult(0, 0, False, None, None, None, None)
+
+        # The last-deleted row's record_hash is the anchor the surviving tail links to.
+        cutoff_row = AgentAction.objects.get(sequence=cutoff_sequence)
+        cutoff_hash = cutoff_row.record_hash
+        first_retained = cutoff_sequence + 1
+
+        if not commit:
+            return PruneResult(
+                eligible, 0, False, cutoff_sequence, cutoff_hash, first_retained, None
+            )
+
+        # Legal-hold veto point — a receiver raising here rolls back the whole prune.
+        #
+        # FAIL-CLOSED: `.send()` is correct here and must NOT become
+        # dispatch_extension_signal / send_robust. Every other OSS→Enterprise
+        # signal is robust so a third-party bug cannot break an OSS write path
+        # (#2606); this one is the inverse — the raise IS the mechanism. Making
+        # it robust would swallow a hold that failed to register and let the
+        # prune delete records under legal hold, inverting the safety property.
+        # scripts/check-extension-signals.sh recognizes the FAIL-CLOSED marker
+        # above and allows this site; do not remove it.
+        agent_action_prune_requested.send(
+            sender=AgentAction,
+            cutoff_sequence=cutoff_sequence,
+            cutoff_at=timezone.now(),
+            actor=actor,
+        )
+
+        deleted, _ = qs.delete()
+        checkpoint = AgentActionCheckpoint.objects.create(
+            pruned_through_sequence=cutoff_sequence,
+            pruned_through_hash=cutoff_hash,
+            first_retained_sequence=first_retained,
+            pruned_count=deleted,
+            pruned_by=actor if getattr(actor, "pk", None) else None,
+            created_at=timezone.now(),
+        )
+        return PruneResult(
+            eligible,
+            deleted,
+            True,
+            cutoff_sequence,
+            cutoff_hash,
+            first_retained,
+            str(checkpoint.id),
+        )

trueppm_api/apps/agents/signals.py

@@ -0,0 +1,30 @@
+"""The OSS agent-action extension-point signal (ADR-0112 §1.3).
+
+OSS dispatches ``agent_action_recorded`` inside the recording transaction's
+``on_commit`` — so it fires exactly when (and only when) the action durably committed
+and its chain link is written. In the community edition nothing is registered and the
+signal dispatches to nobody; OSS runs unchanged. Enterprise registers a receiver at
+app-ready to append to its immutable/notarized org trail, feed the approval-workflow
+engine, and evaluate org capability policy (#146–#148). Enterprise reads the frozen
+event; it never reaches into OSS internals.
+
+The signal is kept dependency-free (a bare ``Signal``) so importing it never drags in
+models — the receiver side lives entirely in the ``trueppm-enterprise`` repo.
+"""
+
+from __future__ import annotations
+
+from django.dispatch import Signal
+
+#: Dispatched with ``sender=AgentAction`` and ``action=<AgentAction instance>`` in the
+#: committing transaction's ``on_commit``. Additive-only kwargs (ADR-0112 contract).
+agent_action_recorded = Signal()
+
+#: The OSS legal-hold seam for chain-aware pruning (ADR-0361). Dispatched with plain
+#: ``.send()`` **inside** the prune transaction, **before** the DELETE, carrying
+#: ``cutoff_sequence``, ``cutoff_at``, and ``actor``. Any receiver that raises aborts the
+#: whole atomic prune (fail-closed) — so Enterprise registers a legal-hold guard that
+#: raises to block truncation of held rows, without OSS importing enterprise or forking
+#: the purge. Community edition registers nobody and prunes unchanged. Additive-only
+#: kwargs (ADR-0112 contract).
+agent_action_prune_requested = Signal()

trueppm_api/apps/agents/urls.py

@@ -0,0 +1,15 @@
+"""URL patterns for the agents app — the team-readable agent-action log (#1805)."""
+
+from __future__ import annotations
+
+from django.urls import path
+
+from trueppm_api.apps.agents.views import AgentActionViewSet
+
+agent_action_list = AgentActionViewSet.as_view({"get": "list"})
+agent_action_detail = AgentActionViewSet.as_view({"get": "retrieve"})
+
+urlpatterns = [
+    path("agent-actions/", agent_action_list, name="agent-action-list"),
+    path("agent-actions/<uuid:pk>/", agent_action_detail, name="agent-action-detail"),
+]

trueppm_api/apps/agents/views.py

@@ -0,0 +1,108 @@
+"""Team-readable agent-action log endpoint (ADR-0112 §1.3, #1805).
+
+``GET /api/v1/agent-actions/`` lists what the team's agents did — scoped so a caller sees
+only actions in projects they are a member of, plus their own (actions where they are the
+human principal). This is the OSS "team-readable" surface; the *immutable, notarized,
+org-wide* trail is Enterprise (#146). The endpoint is read-only: the rows are append-only
+and written only by ``record_agent_action``.
+"""
+
+from __future__ import annotations
+
+import uuid
+from typing import TYPE_CHECKING
+
+from django.db.models import Q
+from django.utils.dateparse import parse_datetime
+from rest_framework import permissions, viewsets
+
+from trueppm_api.apps.access.models import ProjectMembership
+from trueppm_api.apps.agents.models import AgentAction
+from trueppm_api.apps.agents.serializers import AgentActionSerializer
+from trueppm_api.apps.projects.models import Project
+
+if TYPE_CHECKING:
+    from django.db.models import QuerySet
+
+
+class AgentActionViewSet(viewsets.ReadOnlyModelViewSet):  # type: ignore[type-arg]
+    """Read-only, membership-scoped list/retrieve of audited agent actions.
+
+    Ordering is by ``sequence`` (chain order); filtering by ``verdict``, ``project``,
+    ``program``, ``since``, and ``constraint`` is supported for the common "show me
+    refusals" / "show me this project" / "show me this program's agents (last 7d)" /
+    "show me every graph_validation refusal" (ADR-0421, #1850) triage queries. The
+    ``program`` filter (#2020) powers the per-program agent-oversight panel: a program
+    groups projects, so a program's agent log is the union of the chain across the
+    program's member projects — always intersected with the caller's own membership
+    scope below, so the filter narrows what the caller may already see and never widens it.
+    """
+
+    serializer_class = AgentActionSerializer
+    permission_classes = [permissions.IsAuthenticated]
+
+    def get_queryset(self) -> QuerySet[AgentAction]:
+        user = self.request.user
+        if not user.is_authenticated:
+            return AgentAction.objects.none()
+
+        member_project_ids = ProjectMembership.objects.filter(
+            user=user,
+            is_deleted=False,
+        ).values_list("project_id", flat=True)
+
+        # A caller sees actions in their projects, plus their own agent's actions (which
+        # may target a project they no longer belong to, or none at all). The serializer
+        # renders project/principal as their FK ids (already on the row) and only the
+        # token *prefix*; ``refusal_detail`` is the one reverse OneToOne it reads, so
+        # select_related it to avoid an N+1 across the page (ADR-0421, #1850).
+        qs = AgentAction.objects.filter(
+            Q(project_id__in=member_project_ids) | Q(principal=user)
+        ).select_related("refusal_detail")
+
+        verdict = self.request.query_params.get("verdict")
+        if verdict:
+            qs = qs.filter(verdict=verdict)
+        constraint = self.request.query_params.get("constraint")
+        if constraint:
+            # Filters through the side-car; non-refusals (no detail) drop out naturally.
+            qs = qs.filter(refusal_detail__constraint=constraint)
+        project_id = self.request.query_params.get("project")
+        if project_id:
+            # Validate the UUID here so a malformed ?project= returns an empty list rather
+            # than raising a 500 at queryset evaluation.
+            try:
+                uuid.UUID(str(project_id))
+            except ValueError:
+                return AgentAction.objects.none()
+            qs = qs.filter(project_id=project_id)
+
+        program_id = self.request.query_params.get("program")
+        if program_id:
+            try:
+                uuid.UUID(str(program_id))
+            except ValueError:
+                return AgentAction.objects.none()
+            # Resolve the program's project ids and filter with project_id__in so the query
+            # rides the existing (project, -occurred_at) index rather than joining through
+            # Project on every row (perf: one indexed IN vs. a per-row join). The result is
+            # still bounded by the membership/principal scope set above — a non-member gains
+            # nothing by naming a program they cannot see.
+            program_project_ids = Project.objects.filter(program_id=program_id).values_list(
+                "id", flat=True
+            )
+            qs = qs.filter(project_id__in=program_project_ids)
+
+        since = self.request.query_params.get("since")
+        if since:
+            # A malformed ?since= is ignored (the range simply doesn't narrow) rather than
+            # 500ing or emptying the list — a lenient time-window filter, unlike the strict
+            # UUID filters above where a bad id can only be a client bug.
+            parsed_since = parse_datetime(since)
+            if parsed_since is not None:
+                qs = qs.filter(occurred_at__gte=parsed_since)
+
+        # Order by -occurred_at (the leading key of the (project|principal, -occurred_at)
+        # indexes so the membership/principal filter and the sort share an index), with
+        # -sequence as a strict, deterministic tiebreak for same-instant rows.
+        return qs.order_by("-occurred_at", "-sequence")

trueppm_api/apps/csvimport/__init__.py

@@ -0,0 +1,12 @@
+"""CSV / Excel spreadsheet import (#743, ADR-0632).
+
+Parses a `.csv` or `.xlsx` upload into the shared ``ProjectData`` interchange
+dataclass and hands it to ``msproject.importer.import_project`` — the same
+persistence path MS Project and Jira already use (ADR-0259). This app owns a
+parser, a fuzzy column mapper, and its own transactional outbox; it writes no
+task, dependency, or resource rows itself.
+
+The spreadsheet is the single largest source of new users: most small and
+mid-size teams run their schedule in Excel and cannot evaluate TruePPM without
+a migration path that is not manual re-entry.
+"""

trueppm_api/apps/csvimport/apps.py

@@ -0,0 +1,11 @@
+"""Django app config for CSV / Excel import."""
+
+from django.apps import AppConfig
+
+
+class CsvImportConfig(AppConfig):
+    """CSV / Excel spreadsheet import app."""
+
+    name = "trueppm_api.apps.csvimport"
+    verbose_name = "CSV Import"
+    default_auto_field = "django.db.models.BigAutoField"