trueppm-api 0.4.0b2__py3-none-any.whl

trueppm_api/apps/projects/template_divergence.py

@@ -0,0 +1,168 @@
+"""How a project has diverged from the template it was created from (#2971, epic #2743).
+
+**One digest, one function, one endpoint — and no audience parameter.** That is the
+whole of the symmetry requirement, expressed structurally rather than as a promise:
+:func:`compute_template_divergence` takes a project and nothing else, so there is no
+"team view" and "PMO view" that could drift apart, and no reader whose copy could be
+narrowed later without deleting the argument that does not exist. A report about a
+team's decisions that the team cannot read is surveillance with better typography;
+the way to not build that is to make the second version unrepresentable.
+
+Governance position (epic #2743, settled in #2970/#2909 and not relitigated here):
+**divergence produces a visible signal, never a block.** Nothing in this module
+approves, rejects, queues, or scores. It counts rows and says who wrote them.
+
+Nothing here is a second definition of anything. "A machine wrote this row and no
+person has touched it since" is :meth:`TaskManager.untouched_seeded` (ADR-0786 §3) and
+is called, not re-implemented — a copy of that filter that drifts by one clause is how
+a whole class of typed work goes silently missing. "Which rows did this adoption write"
+is ``TemplateApplication.created_task_ids``, which is exact rather than heuristic
+because it is also the undo set.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from trueppm_api.apps.projects.models import Project, TemplateApplication
+
+
+def _display_name(user: Any) -> str:
+    """The name to attribute an adoption to, or '' once the account is gone.
+
+    Empty rather than "Unknown": the adoption record survives ``applied_by`` being
+    nulled by a user deletion, and a placeholder name would read as a real person.
+    """
+    if user is None:
+        return ""
+    return str(getattr(user, "display_name", "") or getattr(user, "username", "") or "")
+
+
+def _adoptions(project: Project) -> list[TemplateApplication]:
+    """Every adoption that actually seeded rows into ``project``, newest first.
+
+    Undone applications are excluded for the same reason the gallery's usage count
+    excludes them (#2909): an application whose seeding was reversed was not adopted,
+    and counting its rows would attribute work to a template that no longer wrote any.
+    """
+    from trueppm_api.apps.projects.models import (
+        TemplateApplication,
+        TemplateApplicationStatus,
+    )
+
+    return list(
+        TemplateApplication.objects.filter(
+            project=project,
+            status=TemplateApplicationStatus.SUCCESS,
+            undone_at__isnull=True,
+        )
+        .select_related("applied_by", "template")
+        .order_by("-created_at")
+    )
+
+
+def compute_template_divergence(project: Project) -> dict[str, Any]:
+    """The digest for one project — the same body for every reader.
+
+    **Two substrates, each answering the question it was built for** (see the
+    ``TemplateApplication`` docstring, which says this explicitly): the *headline*
+    — which template, which version, applied by whom and when — comes from the
+    adoption record, because that row answers "which application wrote this batch";
+    the *counts* come from the per-row provenance columns #2730 added, because
+    those answer "what wrote this row", which is the question a count is.
+
+    That split is also what keeps the query bounded. Counting through
+    ``created_task_ids`` means a ``pk__in`` over the union of every adoption's id
+    array — 2,000 ids per apply with no cap on applies, one bind parameter each,
+    and PostgreSQL's 65,535-parameter protocol limit reached at roughly 33 full
+    applications. The endpoint would then be a permanent 500 for every member of
+    that project, self-inflicted and unclearable by the team. The provenance
+    columns express the same set as a plain indexed predicate with no id list at
+    all, so the cost does not grow with adoption history.
+
+    Four counts over the rows a template wrote into this project — *whichever*
+    adoption wrote them, which is what per-row provenance means:
+
+    * ``unchanged`` — still present and no person has touched it. Delegated
+      wholesale to ``Task.objects.untouched_seeded(..., within=None)``, THE
+      definition of untouched (ADR-0786 §3). ``within=None`` because the seven-day
+      window bounds a *delete* offer; inheriting it would report every long-lived
+      project as fully adapted, which is a false accusation rather than a stale
+      number.
+    * ``adapted`` — still present and someone edited it. This is the number the
+      whole feature exists to keep neutral: it is a team doing its job, and the
+      digest reports it without a verdict attached.
+    * ``removed`` — a template row that has since been soft-deleted. A row
+      hard-deleted by the tombstone reap is genuinely gone from the database and is
+      not counted; nothing can count it, and inferring it by subtracting from a
+      stored id array is what forces the unbounded query above.
+    * ``added`` — live rows this project has that no template wrote.
+
+    A project with no adoption returns ``adopted=False`` and zeroed counts rather
+    than raising: "this project was not created from a template" is an answer the
+    settings section renders, not an error. Its ``added`` is every live row, which
+    also covers the after-an-undo case — undo keeps rows a person had edited, and a
+    project whose only adoption was reversed is not "from" that template, so the
+    survivors are the team's.
+
+    A template deleted after adoption returns ``template=None`` with
+    ``template_name``/``template_version`` still populated — those columns are
+    denormalized onto the application precisely so "this project came from Delivery
+    Skeleton v3" outlives v3.
+    """
+    from trueppm_api.apps.projects.models import Task, TaskSource
+
+    applications = _adoptions(project)
+    rows = Task.objects.filter(project=project)
+    if not applications:
+        return {
+            "project": str(project.pk),
+            "adopted": False,
+            "application": None,
+            "application_count": 0,
+            "template": None,
+            "template_name": "",
+            "template_version": 0,
+            "template_available": False,
+            "applied_at": None,
+            "applied_by_name": "",
+            "seeded_row_count": 0,
+            "unchanged": 0,
+            "adapted": 0,
+            "removed": 0,
+            "added": rows.filter(is_deleted=False).count(),
+        }
+
+    headline = applications[0]
+    template_rows = rows.filter(source_kind=TaskSource.TEMPLATE)
+    live_seeded = template_rows.filter(is_deleted=False).count()
+    # THE definition of untouched (ADR-0786 §3), narrowed to template-written rows.
+    unchanged = (
+        Task.objects.untouched_seeded(project, within=None)
+        .filter(source_kind=TaskSource.TEMPLATE)
+        .count()
+    )
+    removed = template_rows.filter(is_deleted=True).count()
+
+    return {
+        "project": str(project.pk),
+        "adopted": True,
+        "application": str(headline.pk),
+        "application_count": len(applications),
+        "template": str(headline.template_id) if headline.template_id else None,
+        "template_name": headline.template_name,
+        "template_version": headline.template_version,
+        # False means the template row is gone, not that the adoption is in doubt.
+        # The digest still names it; what it cannot do is link you to it.
+        "template_available": headline.template_id is not None,
+        "applied_at": headline.completed_at or headline.created_at,
+        "applied_by_name": _display_name(headline.applied_by),
+        # Derived, not stored: the three parts are counted from the same predicate,
+        # so the total can never disagree with them the way a snapshot would.
+        "seeded_row_count": live_seeded + removed,
+        "unchanged": unchanged,
+        "adapted": live_seeded - unchanged,
+        "removed": removed,
+        "added": rows.filter(is_deleted=False).exclude(source_kind=TaskSource.TEMPLATE).count(),
+    }

trueppm_api/apps/projects/template_services.py

@@ -0,0 +1,160 @@
+"""Dispatch and undo for template application (ADR-0789, #2729).
+
+Every caller goes through :func:`enqueue_template_apply` — nothing calls the Celery
+task directly. Same rule, same reason as ``scheduling/services.py::enqueue_recalculate``:
+a direct ``.delay()`` at the view layer commits the database change and then loses
+the task if the broker is down, which is precisely the failure the outbox exists to
+prevent.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING, Any
+
+from django.db import transaction
+from django.utils import timezone
+
+from trueppm_api.apps.projects.restructure_hooks import (
+    broadcast_tasks_restructured_and_recalc,
+)
+
+if TYPE_CHECKING:
+    from trueppm_api.apps.projects.models import (
+        Project,
+        ProjectTemplate,
+        TemplateApplication,
+    )
+
+logger = logging.getLogger(__name__)
+
+
+def enqueue_template_apply(
+    template: ProjectTemplate,
+    project: Project,
+    *,
+    applied_by: Any = None,
+) -> TemplateApplication:
+    """Record the adoption and queue the seeding job.
+
+    The ``TemplateApplication`` row is written **inside the caller's transaction**
+    and the dispatch is deferred to ``transaction.on_commit``. That ordering is the
+    whole durability story: a row that commits without its task is recoverable (the
+    drain finds it), while a task that fires before its row commits would look up an
+    application that does not exist yet.
+
+    Returns the application immediately — the Start sheet polls it and never blocks
+    on seeding (issue §2).
+    """
+    from trueppm_api.apps.access.permissions import assert_project_not_archived
+    from trueppm_api.apps.projects.models import TemplateApplication
+
+    # Archived is lifecycle state, not authority — a property of the target plan, so
+    # it holds regardless of who is calling (#3354). This module's contract is that
+    # *every* caller comes through here, which makes it the one place the check
+    # cannot be routed around; `ProjectTemplateViewSet.apply` repeats it so the
+    # refusal arrives before any row is written (ADR-0184 defense-in-depth).
+    assert_project_not_archived(project)
+
+    application = TemplateApplication.objects.create(
+        template=template,
+        template_name=template.name,
+        template_version=template.version,
+        project=project,
+        applied_by=applied_by,
+    )
+
+    def _dispatch() -> None:
+        from trueppm_api.apps.projects.template_tasks import apply_template
+
+        try:
+            result = apply_template.delay(str(application.id))
+            TemplateApplication.objects.filter(pk=application.pk).update(
+                celery_task_id=str(result.id)
+            )
+        except Exception:
+            # Deliberately swallowed: the row is committed and `pending`, so the
+            # drain re-dispatches it within the orphan window. Raising here would
+            # turn a recoverable broker blip into a user-visible 500 on a write
+            # that already succeeded.
+            logger.warning(
+                "template.apply: dispatch failed for application %s — leaving to drain",
+                application.id,
+                exc_info=True,
+            )
+
+    transaction.on_commit(_dispatch)
+    return application
+
+
+def undo_template_application(application: TemplateApplication) -> dict[str, int]:
+    """Reverse one application — a single step from the user's side.
+
+    Soft-deletes the rows this application wrote, and **only** those: the undo set
+    is ``created_task_ids``, not a heuristic over the project. When anything was
+    removed it emits its forward path's own ``tasks_restructured`` and enqueues a
+    recalculation, both on commit — see the call at the end.
+
+    It deliberately skips rows a person has since touched. An undo that discards
+    typed work in order to reverse a machine's write is the same unrecoverable
+    failure ADR-0786 §4 is organized around, and the asymmetry is identical —
+    leaving a row behind is disappointing, deleting a sentence somebody wrote is
+    not recoverable. Whoever undoes a template five minutes after a teammate
+    started filling it in must not take the teammate's work with it.
+
+    "Has a person touched this" is ``edited_at IS NOT NULL`` (ADR-0786, merged in
+    !1900). That column is stamped by ``Task.save()`` and deliberately NOT by the
+    CPM ``bulk_update`` path (ADR-0091), so a recalculation between apply and undo
+    does not falsely protect every row and turn undo into a no-op.
+    """
+    from trueppm_api.apps.access.permissions import assert_project_not_archived
+    from trueppm_api.apps.projects.models import Task, TemplateApplicationStatus
+
+    # Lifecycle floor, mirroring `enqueue_template_apply` above and the two
+    # `task_batch_services` undos: an undo soft-deletes rows, which is a write, and
+    # an archived plan takes no writes from any caller (#3354).
+    assert_project_not_archived(application.project_id)
+
+    ids = list(application.created_task_ids or [])
+    if not ids:
+        application.status = TemplateApplicationStatus.UNDONE
+        application.undone_at = timezone.now()
+        application.save(update_fields=["status", "undone_at"])
+        return {"deleted": 0, "kept": 0}
+
+    rows = list(Task.objects.filter(pk__in=ids, is_deleted=False))
+    deleted = 0
+    kept = 0
+    for row in rows:
+        if _has_been_touched(row):
+            kept += 1
+            continue
+        row.soft_delete()
+        deleted += 1
+
+    application.status = TemplateApplicationStatus.UNDONE
+    application.undone_at = timezone.now()
+    application.result_summary = {
+        **(application.result_summary or {}),
+        "undo": {"deleted": deleted, "kept": kept},
+    }
+    application.save(update_fields=["status", "undone_at", "result_summary"])
+
+    if deleted:
+        # The forward path (`template_tasks.apply_template`) emits `tasks_restructured`
+        # and enqueues a recalculation; an undo removes those same rows, so it is the
+        # same kind of mutation and owes collaborators the same event — without it the
+        # rows appear live and never disappear, and the seeded dates never move back
+        # (#3415). Guarded on `deleted` for the same reason the forward path guards on
+        # `created`: when every row was kept, no task row changed and there is nothing
+        # to announce. Deferred to commit inside the helper, and reached only here on
+        # the success path — a broadcast scheduled on a refusal path is discarded
+        # anyway, because DRF's exception handler rolls back under ATOMIC_REQUESTS.
+        broadcast_tasks_restructured_and_recalc(application.project_id)
+
+    return {"deleted": deleted, "kept": kept}
+
+
+def _has_been_touched(task: Any) -> bool:
+    """Has a person saved this row since the template wrote it? (ADR-0786)"""
+    return task.edited_at is not None

trueppm_api/apps/projects/template_tasks.py

@@ -0,0 +1,258 @@
+"""Celery tasks for template application (ADR-0789, #2729).
+
+Mirrors ``csvimport.tasks`` — same claim-and-run idempotency, same drain, same
+orphan window — because it is the same problem: a queued seeding job that must run
+exactly once even under redelivery, worker death, or a broker outage.
+"""
+
+from __future__ import annotations
+
+import logging
+from datetime import timedelta
+from typing import Any
+
+from celery import shared_task
+
+from trueppm_api.core.idempotent import idempotent_task
+
+logger = logging.getLogger(__name__)
+
+# An apply running longer than this is considered orphaned by the drain. Slightly
+# longer than the soft limit so the task gets the chance to record its own failure
+# before the drain resets the row underneath it.
+_APPLY_ORPHAN_MINUTES = 15
+
+# Rows inside an open transaction.on_commit() callback are invisible until the
+# commit lands, so the drain skips rows younger than this rather than racing an
+# in-flight request it would otherwise dispatch a second time.
+_DRAIN_MIN_AGE_MINUTES = 10
+
+
+def _claim(application_id: str) -> Any:
+    """Atomically take an application from ``pending`` to ``running``.
+
+    Returns the claimed row, or ``None`` when another delivery already has it.
+    ``select_for_update`` makes the read-modify-write one critical section, so two
+    concurrent deliveries cannot both claim — the loser sees ``running`` and skips.
+    """
+    from django.utils import timezone
+
+    from trueppm_api.apps.projects.models import (
+        TemplateApplication,
+        TemplateApplicationStatus,
+    )
+
+    row = (
+        TemplateApplication.objects.select_for_update()
+        # `project` is joined rather than lazily fetched because the caller reads
+        # `application.project.is_archived` while this row lock is held (#3354) —
+        # a lazy FK there would add a round trip inside the critical section.
+        # `select_for_update` locks only `template_application`; the join does not
+        # widen it (Postgres would need `of=` for that).
+        .select_related("project")
+        .filter(pk=application_id, status=TemplateApplicationStatus.PENDING)
+        .first()
+    )
+    if row is None:
+        return None
+    row.status = TemplateApplicationStatus.RUNNING
+    row.started_at = timezone.now()
+    row.save(update_fields=["status", "started_at"])
+    return row
+
+
+@shared_task(  # type: ignore[untyped-decorator]
+    bind=True,
+    name="templates.apply",
+    soft_time_limit=540,
+    time_limit=600,
+    acks_late=True,
+    reject_on_worker_lost=True,
+    max_retries=3,
+)
+def apply_template(self: Any, application_id: str) -> dict[str, Any]:
+    """Materialize a template into its project, then queue a CPM pass.
+
+    The claim and the seeding share one transaction, so a mid-apply failure rolls
+    the claim back too and the row returns to ``pending`` for a clean retry. A
+    duplicate delivery — a drain re-dispatch after a worker-death window, or an
+    ``acks_late`` replay of an already-completed message — finds no ``pending`` row
+    to claim and skips.
+    """
+    from django.db import transaction
+    from django.utils import timezone
+
+    from trueppm_api.apps.projects.models import (
+        TemplateApplicationStatus,
+    )
+    from trueppm_api.apps.projects.project_templates import (
+        TemplateStructureError,
+        materialize_structure,
+    )
+
+    created: list[Any] = []
+    project_id: str | None = None
+    try:
+        with transaction.atomic():
+            application = _claim(application_id)
+            if application is None:
+                logger.info(
+                    "templates.apply: application %s already claimed — skipping duplicate",
+                    application_id,
+                )
+                return {"skipped": True, "tasks_created": 0}
+            if application.template is None:
+                # The template was deleted between enqueue and dispatch. Fail
+                # loudly rather than silently succeeding with zero rows — the user
+                # asked for a skeleton and did not get one.
+                raise TemplateStructureError("Template no longer exists.")
+            if application.project.is_archived:
+                # The project was archived between enqueue and dispatch (#3354).
+                # `enqueue_template_apply` refuses an already-archived target, but
+                # the window here is wide, not theoretical: dispatch is deferred to
+                # `on_commit`, and the drain re-dispatches a still-`pending`
+                # application every 30s for as long as it stays pending — so a
+                # broker outage at enqueue time can put this write minutes or hours
+                # after the check that admitted it. Seeding now would write a whole
+                # skeleton into a plan every other endpoint refuses writes to.
+                #
+                # Raised as a `TemplateStructureError` for the same reason the
+                # deleted-template branch above is: it is the file's established
+                # non-retryable channel — the atomic rolls back (releasing the
+                # `running` claim), `_mark_failed` records the reason outside it,
+                # and the drain does not resurrect it. Retrying would be wrong
+                # regardless: unarchiving is a human act on no schedule, and three
+                # retries 30s apart cannot wait for one.
+                raise TemplateStructureError(
+                    "This project is archived and cannot be modified. Unarchive it first."
+                )
+            project_id = str(application.project_id)
+            result = materialize_structure(
+                application.template,
+                application.project,
+                applied_by=application.applied_by,
+            )
+            created = result.task_ids
+            application.created_task_ids = created
+            application.status = TemplateApplicationStatus.SUCCESS
+            application.completed_at = timezone.now()
+            # Counts feed the seed banner (#2731, ADR-0799 §2) — read off what
+            # materialize_structure already built in memory, never re-derived from
+            # a second query against the rows it just wrote.
+            application.result_summary = {
+                "tasks_created": len(created),
+                "milestones_created": result.milestones_created,
+                "dependencies_created": result.dependencies_created,
+            }
+            application.save(
+                update_fields=[
+                    "created_task_ids",
+                    "status",
+                    "completed_at",
+                    "result_summary",
+                ]
+            )
+    except TemplateStructureError as exc:
+        # A malformed or unsupported structure is not retryable — retrying would
+        # re-read the same bad document three more times and delay the user's
+        # failure message by ten minutes.
+        _mark_failed(application_id, str(exc))
+        return {"failed": True, "tasks_created": 0}
+    except Exception as exc:
+        if self.request.retries >= self.max_retries:
+            # Partially-written rows are deliberately KEPT (ADR-0789 §Durable
+            # Execution 8) — they are real, correctly-attributed tasks, and
+            # created_task_ids records exactly which, so undo stays exact.
+            _mark_failed(application_id, str(exc))
+            return {"failed": True, "tasks_created": 0}
+        raise self.retry(exc=exc, countdown=2**self.request.retries * 30) from exc
+
+    if created and project_id:
+        # Dates come from the adopter's calendar, not the publisher's — the
+        # template carries durations and shape only, so the schedule does not
+        # exist until this pass runs. Through the service, never .delay() directly.
+        from trueppm_api.apps.scheduling.services import enqueue_recalculate
+        from trueppm_api.apps.sync.broadcast import broadcast_board_event
+
+        enqueue_recalculate(project_id)
+        transaction.on_commit(
+            lambda: broadcast_board_event(str(project_id), "tasks_restructured", {})
+        )
+
+    return {"tasks_created": len(created)}
+
+
+def _mark_failed(application_id: str, detail: str) -> None:
+    """Record a terminal failure on its own transaction.
+
+    Its own transaction because the caller's has usually just rolled back — writing
+    the failure inside it would roll back with it and leave the row at ``running``
+    forever, which the drain would then keep resurrecting.
+    """
+    from django.utils import timezone
+
+    from trueppm_api.apps.projects.models import (
+        TemplateApplication,
+        TemplateApplicationStatus,
+    )
+
+    TemplateApplication.objects.filter(pk=application_id).update(
+        status=TemplateApplicationStatus.FAILED,
+        error_detail=detail[:2000],
+        completed_at=timezone.now(),
+    )
+
+
+@idempotent_task(
+    lock_key_template="drain_template_apply_queue",
+    lock_ttl=60,
+    on_contention="skip",
+    soft_time_limit=25,
+    time_limit=30,
+    acks_late=True,
+    reject_on_worker_lost=True,
+    name="templates.drain_apply_queue",
+)
+def drain_template_apply_queue(self: object) -> None:
+    """Dispatch pending applications; recover ones whose worker died.
+
+    Runs every 30 seconds via Celery Beat — the safety net that makes
+    ``enqueue_template_apply``'s best-effort ``.delay()`` durable.
+    """
+    _do_template_apply_drain()
+
+
+def _do_template_apply_drain() -> None:
+    """Business logic for the drain — extracted so it is testable without Beat."""
+    from django.utils import timezone
+
+    from trueppm_api.apps.projects.models import (
+        TemplateApplication,
+        TemplateApplicationStatus,
+    )
+
+    now = timezone.now()
+    orphan_cutoff = now - timedelta(minutes=_APPLY_ORPHAN_MINUTES)
+    pending_cutoff = now - timedelta(minutes=_DRAIN_MIN_AGE_MINUTES)
+
+    recovered = TemplateApplication.objects.filter(
+        status=TemplateApplicationStatus.RUNNING,
+        started_at__lt=orphan_cutoff,
+    ).update(status=TemplateApplicationStatus.PENDING, celery_task_id="")
+    if recovered:
+        logger.warning("templates.drain: recovered %d orphaned application(s)", recovered)
+
+    pending = list(
+        TemplateApplication.objects.filter(
+            status=TemplateApplicationStatus.PENDING,
+            created_at__lt=pending_cutoff,
+        ).values_list("id", flat=True)[:100]
+    )
+    for application_id in pending:
+        try:
+            apply_template.delay(str(application_id))
+        except Exception:
+            # Broker still down — leave it pending and try again in 30 seconds.
+            logger.warning(
+                "templates.drain: re-dispatch failed for %s", application_id, exc_info=True
+            )