trueppm-api 0.4.0b2__py3-none-any.whl

trueppm_api/apps/sync/apps.py

@@ -0,0 +1,15 @@
+"""App config for sync."""
+
+from django.apps import AppConfig
+
+
+class SyncConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "trueppm_api.apps.sync"
+
+    def ready(self) -> None:
+        # Wire the per-project sync sequence that allocates `sync_seq`
+        # (ADR-0686, #2491).
+        from trueppm_api.apps.sync.sequence import register_sequence_receivers
+
+        register_sequence_receivers()

trueppm_api/apps/sync/backfill.py

@@ -0,0 +1,208 @@
+"""Data-backfill helper for the sync delta cursor (ADR-0686, #2491).
+
+Extracted from the migration file so tests can import it without coupling to a
+migration file *name*, which a squash deletes (CLAUDE.md migration rule 3).
+
+Seeds every existing synced row's ``sync_seq`` from its project's sequence. Every
+row in a project gets the **same** value — one step of the sequence for the whole
+pre-existing dataset. That is correct, not a shortcut: the delta is ``> since``
+and the checkpoint is the maximum, so rows sharing a value are either all before
+or all after any client checkpoint. It is the same property that lets a batched
+upload share one value (ADR-0686).
+
+Because the value is the project's old watermark **plus one**, every ``since`` an
+existing client holds is strictly below it. The first pull after upgrade
+redelivers the project once under WatermelonDB upsert semantics — which also
+repairs whatever the old scheme had already dropped — and every pull after that
+is a normal delta. No client change, and no forced re-pull from zero.
+
+Soft-deleted rows are renumbered too: they are tombstones the delta must still
+deliver, selected by the same ``sync_seq__gt=since`` filter and split on
+``is_deleted``.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+# The two join shapes every indirect source uses to reach its project row, and
+# the predicates that bind the target table to the end of that join. Named
+# because the table below is thirteen rows of near-identical SQL otherwise: the
+# names say *how* a model reaches its project (directly, or through its task),
+# which is the only thing that varies between most rows.
+_VIA_PROJECT = "{project} p"
+_VIA_TASK = "{task} t JOIN {project} p ON t.project_id = p.id"
+_ON_PROJECT_ID = "x.project_id = p.id"
+_ON_TASK_ID = "x.task_id = t.id"
+
+# (label, join clause reaching the project row, predicate binding the target).
+# The target table is aliased ``x`` and the project row ``p`` in every statement.
+_INDIRECT_SOURCES: list[tuple[str, str, str]] = [
+    ("projects.Task", _VIA_PROJECT, _ON_PROJECT_ID),
+    ("access.ProjectMembership", _VIA_PROJECT, _ON_PROJECT_ID),
+    ("projects.Risk", _VIA_PROJECT, _ON_PROJECT_ID),
+    ("projects.Sprint", _VIA_PROJECT, _ON_PROJECT_ID),
+    ("projects.Label", _VIA_PROJECT, _ON_PROJECT_ID),
+    (
+        "projects.SprintRetro",
+        "{sprint} s JOIN {project} p ON s.project_id = p.id",
+        "x.sprint_id = s.id",
+    ),
+    (
+        "projects.RetroActionItem",
+        "{sprintretro} r JOIN {sprint} s ON r.sprint_id = s.id "
+        "JOIN {project} p ON s.project_id = p.id",
+        "x.retro_id = r.id",
+    ),
+    ("projects.TaskSuggestedAssignee", _VIA_TASK, _ON_TASK_ID),
+    ("projects.TaskRecurrenceRule", _VIA_TASK, _ON_TASK_ID),
+    ("integrations.TaskLink", _VIA_TASK, _ON_TASK_ID),
+    ("timetracking.TimeEntry", _VIA_TASK, _ON_TASK_ID),
+    # Graph edges reach the project through the endpoint the delta scopes and
+    # orders by. Both are delivered as their own collection filtered on their own
+    # `sync_seq`, so an unseeded edge would stay at 0 and never be delivered.
+    ("projects.Dependency", _VIA_TASK, "x.predecessor_id = t.id"),
+    ("projects.TaskRelation", _VIA_TASK, "x.source_id = t.id"),
+]
+
+
+def seed_sync_seq(apps: Any, schema_editor: Any) -> None:
+    """Give every existing synced row a cursor drawn from its project's sequence.
+
+    Every collection the delta serves is seeded, graph edges included: each is
+    filtered on its *own* ``sync_seq``, so a row left at 0 would never be
+    delivered again — not even by a cold-start ``since=0`` pull.
+    """
+    get = apps.get_model
+    tables = {
+        "project": get("projects", "Project")._meta.db_table,
+        "task": get("projects", "Task")._meta.db_table,
+        "sprint": get("projects", "Sprint")._meta.db_table,
+        "sprintretro": get("projects", "SprintRetro")._meta.db_table,
+        "calendar": get("projects", "Calendar")._meta.db_table,
+    }
+
+    with schema_editor.connection.cursor() as cursor:
+        # The project row is itself a synced source. Do it first, while
+        # last_sync_version still holds the pre-migration watermark.
+        #
+        # Every identifier interpolated into the statements below is a db_table
+        # read from model _meta; no user input reaches them, and there is no
+        # value to bind as a parameter.
+        # nosemgrep: formatted-sql-query, sqlalchemy-execute-raw-query
+        cursor.execute(
+            f"UPDATE {tables['project']} SET sync_seq = last_sync_version + 1"  # nosec B608
+        )
+
+        for label, join, predicate in _INDIRECT_SOURCES:
+            app_label, model_name = label.split(".")
+            table = get(app_label, model_name)._meta.db_table
+            cursor.execute(
+                f"UPDATE {table} x SET sync_seq = p.last_sync_version + 1 "  # nosec B608
+                f"FROM {join.format(**tables)} WHERE {predicate}"
+            )
+
+        # A calendar can be shared, and one row carries one cursor, so it takes
+        # the maximum over its owners. The final statement then raises every
+        # owning project's watermark to at least that value — without it a shared
+        # calendar would sit permanently above one owner's checkpoint and be
+        # redelivered on every pull.
+        cursor.execute(
+            f"UPDATE {tables['calendar']} c SET sync_seq = sub.v FROM ("  # nosec B608
+            f"  SELECT calendar_id, MAX(last_sync_version) + 1 AS v"
+            f"    FROM {tables['project']} WHERE calendar_id IS NOT NULL"
+            f"   GROUP BY calendar_id"
+            f") sub WHERE c.id = sub.calendar_id"
+        )
+
+        cursor.execute(
+            f"UPDATE {tables['project']} p SET last_sync_version = GREATEST("  # nosec B608
+            f"  p.last_sync_version + 1,"
+            f"  COALESCE((SELECT c.sync_seq FROM {tables['calendar']} c"
+            f"             WHERE c.id = p.calendar_id), 0))"
+        )
+
+
+def unseed_sync_seq(apps: Any, schema_editor: Any) -> None:
+    """Reverse: leave the column at its default.
+
+    ``last_sync_version`` is deliberately *not* wound back. It is monotonic by
+    contract, and a watermark that is too high only costs a client a pull that
+    returns nothing; one that moves backwards would redeliver rows forever.
+    """
+    get = apps.get_model
+    labels = ["projects.Project", "projects.Calendar", *[s[0] for s in _INDIRECT_SOURCES]]
+    with schema_editor.connection.cursor() as cursor:
+        for label in labels:
+            app_label, model_name = label.split(".")
+            table = get(app_label, model_name)._meta.db_table
+            # nosemgrep: formatted-sql-query, sqlalchemy-execute-raw-query
+            cursor.execute(f"UPDATE {table} SET sync_seq = 0")  # nosec B608
+
+
+def seed_program_sync_seq(apps: Any, schema_editor: Any) -> None:
+    """Seed the installation-wide program cursor for existing data (ADR-0747, #2498).
+
+    ``Program`` and ``ProgramMembership`` have carried a ``sync_seq`` column since
+    #2491, but always ``0`` — neither model allocated, because the program slice
+    keyed on ``server_version``. This assigns real, ordered values and starts the
+    installation counter above them.
+
+    Unlike :func:`seed_sync_seq`, rows are numbered **individually**, in
+    ``(created_at, id)`` order, rather than all sharing one value. The project
+    backfill can share a value because a project's rows are re-delivered together
+    anyway; here the whole point is that a single scalar cursor must order rows
+    from independent programs against each other, so the backfill has to produce
+    the total order the runtime allocator will continue.
+
+    Each table is ordered by its own creation timestamp — ``Program.created_at``,
+    ``ProgramMembership.joined_at``; the two models do not share a field name — with
+    ``id`` breaking ties, since two rows can share a timestamp. That makes the
+    numbering deterministic and reproducible rather than dependent on physical row
+    order.
+
+    **The offset is deliberate.** Numbering starts at
+    ``max(server_version across both tables) + 1`` rather than at 1, so that every
+    ``since`` an existing client holds — drawn from the *old* ``server_version``
+    scale — sorts strictly below every row's new ``sync_seq``. The two scales are
+    otherwise unrelated, and a client whose stored cursor happened to exceed the
+    new values would silently skip rows on its first post-upgrade pull. Starting
+    above every plausible old value makes that first pull a full re-delivery of the
+    caller's accessible set, which is safe under WatermelonDB upsert and is the
+    only transition that cannot drop a row. It also repairs whatever the old
+    scheme had already dropped.
+    """
+    Program = apps.get_model("projects", "Program")
+    ProgramMembership = apps.get_model("access", "ProgramMembership")
+    ProgramSyncSequence = apps.get_model("sync", "ProgramSyncSequence")
+
+    # (model, creation-timestamp field). The two models do not share a field name.
+    ordered_tables = ((Program, "created_at"), (ProgramMembership, "joined_at"))
+
+    # Start above the old scale so no pre-upgrade client cursor can sit above a row.
+    highest_old = 0
+    for model, _ in ordered_tables:
+        for value in model.objects.values_list("server_version", flat=True):
+            highest_old = max(highest_old, value or 0)
+
+    seq = highest_old
+    # Soft-deleted rows are renumbered too — they are tombstones the delta must
+    # still deliver, selected by the same sync_seq__gt=since filter.
+    for model, created_field in ordered_tables:
+        for pk in model.objects.order_by(created_field, "id").values_list("pk", flat=True):
+            seq += 1
+            model.objects.filter(pk=pk).update(sync_seq=seq)
+
+    # The allocator must sit at or above every row it will be compared against, or
+    # the first watermark read would publish a checkpoint below live rows.
+    ProgramSyncSequence.objects.update_or_create(pk=1, defaults={"value": seq})
+
+
+def unseed_program_sync_seq(apps: Any, schema_editor: Any) -> None:
+    """Reverse :func:`seed_program_sync_seq` — return both tables to ``sync_seq = 0``.
+
+    The counter row is dropped with the table by the schema operation, so only the
+    stamped rows need clearing.
+    """
+    for label, name in (("projects", "Program"), ("access", "ProgramMembership")):
+        apps.get_model(label, name).objects.update(sync_seq=0)

trueppm_api/apps/sync/broadcast.py

@@ -0,0 +1,320 @@
+"""Broadcast helper — sends events to a project's WebSocket group.
+
+All mutations and CPM completions call broadcast_board_event() wrapped in
+transaction.on_commit() so that clients never receive an event for a write
+that was subsequently rolled back.
+"""
+
+from __future__ import annotations
+
+import logging
+from datetime import UTC
+from typing import Any
+
+from asgiref.sync import async_to_sync
+
+logger = logging.getLogger(__name__)
+
+# Wire-protocol version for the ``board.event`` envelope (#1325). A bare integer
+# down-payment reserved *before* 0.3 ships and freezes 60+ ``event_type`` strings:
+# once a client builds against the envelope, adding a versioning handshake later
+# would be a breaking change, whereas a field that is already present can grow a
+# meaning without one. Clients that don't read it are unaffected; future clients
+# can branch on it. Bump only on a backward-incompatible envelope change.
+WS_PROTOCOL_VERSION = 1
+
+# Event types that are deliberately NOT persisted to the BoardEvent replay buffer
+# (ADR-0236, #321). Everything else is persisted so a reconnecting client can
+# replay it via ``?since=``. A *denylist* (not an allowlist) is the fail-safe
+# default: every mutation event maps to an idempotent client cache-invalidation,
+# so replaying one is always safe, and a newly-added mutation event should be
+# replayable without anyone remembering to register it. Two reasons to exclude:
+#
+#   1. **Ephemeral** high-frequency live progress / presence pings — a stale
+#      progress bar or a presence ping for a since-departed peer carries no value
+#      on replay.
+#   2. **Post-delete** — ``project_hard_deleted`` fires from an on_commit callback
+#      *after* the project row (and its cascade-deleted buffer rows) are gone, so
+#      persisting a BoardEvent for it would dangle the project FK. The DEFERRABLE
+#      FK check surfaces at COMMIT, past ``_persist_board_event``'s try/except, so
+#      it cannot be swallowed — it must not be attempted. A hard-deleted project
+#      has no replay value anyway (a reconnecting member is evicted or 404s).
+#   3. **Program-scoped** — the ``program_*`` lifecycle events (split / deleted /
+#      closed / reopened / sponsorship_transferred) fan out on a *program*'s WS
+#      group but pass the **Program** pk as ``project_id``. A Program pk is never a
+#      ``projects.Project`` pk, so persisting a BoardEvent for one always violates
+#      the Project FK at COMMIT — an unhandled IntegrityError → 500 for an
+#      otherwise-valid request (#2126). Like ``project_hard_deleted``, the FK check
+#      surfaces past ``_persist_board_event``'s try/except, so the persist must not
+#      be attempted; these events are live-only signals with no project replay row.
+DONT_PERSIST_EVENT_TYPES = frozenset(
+    {
+        "presence_join",
+        "presence_leave",
+        "task_run_started",
+        "task_run_progress",
+        "task_run_completed",
+        "task_run_failed",
+        "task_run_cancelled",
+        "project_hard_deleted",
+        "program_split",
+        "program_deleted",
+        "program_closed",
+        "program_reopened",
+        "program_sponsorship_transferred",
+    }
+)
+
+
+def _group_name(project_id: str) -> str:
+    """Return the channel layer group name for a project."""
+    return f"project_{project_id}"
+
+
+def _record_broadcast_metric() -> None:
+    """Best-effort increment of the WS broadcast counter (#1900).
+
+    Imported lazily (matching this module's lazy-import convention) and wrapped so a
+    telemetry error can never touch the latency-sensitive broadcast path — the
+    broadcast is best-effort, and so is its metric.
+    """
+    try:
+        from trueppm_api.apps.observability.otel import metrics
+
+        metrics.record_ws_broadcast()
+    except Exception:
+        logger.debug("broadcast metric increment skipped", exc_info=True)
+
+
+def _persist_board_event(project_id: str, event_type: str, payload: dict[str, Any]) -> int | None:
+    """Persist a replayable event to the BoardEvent buffer; return its sequence.
+
+    The sequence is the row's ``BigAutoField`` PK — globally monotonic, assigned
+    atomically by Postgres, so it survives concurrent commits with no lost or
+    duplicate values (ADR-0236). Returns ``None`` for non-persisted events
+    (``DONT_PERSIST_EVENT_TYPES``) and on any DB failure: best-effort exactly like the
+    broadcast itself, so a failed insert logs and is swallowed — the live event
+    still goes out (without a ``seq``) and the client recovers the gap via a
+    ``resync_required`` on its next reconnect.
+
+    Callers already run this inside a ``transaction.on_commit`` callback, so the
+    INSERT happens post-commit in autocommit — the row is the durable truth, the
+    broadcast is the best-effort echo.
+    """
+    if event_type in DONT_PERSIST_EVENT_TYPES:
+        return None
+
+    from django.db import DatabaseError
+
+    from trueppm_api.apps.sync.models import BoardEvent
+
+    try:
+        row = BoardEvent.objects.create(
+            project_id=project_id, event_type=event_type, payload=payload
+        )
+    except DatabaseError:
+        logger.exception(
+            "broadcast: failed to persist BoardEvent %s for project %s", event_type, project_id
+        )
+        return None
+    return row.pk
+
+
+def _board_message(
+    event_type: str, payload: dict[str, Any], seq: int | None = None
+) -> dict[str, Any]:
+    """Build the ``board.event`` channel-layer envelope consumers expect.
+
+    Shared by the sync and async broadcast helpers so the wire shape stays in
+    one place — ``ProjectConsumer.board_event`` reads ``event_type``/``payload``
+    off the top level. The ``protocol_version`` field is the single source of the wire
+    version for every board event (#1325).
+
+    ``seq`` is the BoardEvent sequence for a persisted (replayable) event, or
+    ``None`` for an ephemeral event that was not buffered (ADR-0236). It is
+    additive: a client that ignores it behaves exactly as before, so
+    ``protocol_version`` stays ``1``.
+    """
+    return {
+        "type": "board.event",
+        "protocol_version": WS_PROTOCOL_VERSION,
+        "event_type": event_type,
+        "payload": payload,
+        "seq": seq,
+    }
+
+
+def broadcast_task_updated(
+    project_id: str,
+    *,
+    task_id: str,
+    changed_fields: list[str],
+    version: int | None,
+    actor_id: str | None,
+) -> None:
+    """Broadcast a field-level ``task_updated`` delta (ADR-0152, #327).
+
+    Assembles the standard mutation-delta payload so every task-mutation call site
+    emits the same shape: collaborators learn *which* fields changed, at what
+    ``server_version``, and *who* changed them — without the field *values* ever
+    crossing the wire. Values are deliberately omitted because task fields are
+    role-gated (ADR-0104 nulls ``story_points`` below the velocity audience, cost
+    fields are gated, etc.); a client that needs the new values re-reads the task
+    through the serializer, which re-applies per-user gating. ``actor_id`` lets the
+    originating client suppress its own echo and avoid clobbering an in-flight
+    optimistic update.
+
+    Like :func:`broadcast_board_event`, this is best-effort and must be wrapped in
+    ``transaction.on_commit`` by the caller. The ``id`` key is retained for
+    backward compatibility with consumers that only read it.
+    """
+    broadcast_board_event(
+        project_id,
+        "task_updated",
+        {
+            "id": task_id,
+            "changed_fields": sorted(changed_fields),
+            "version": version,
+            "actor_id": actor_id,
+            "ts": _utcnow_iso(),
+        },
+    )
+
+
+def _utcnow_iso() -> str:
+    """Current UTC time as an ISO-8601 string (factored out for test stubbing)."""
+    from datetime import datetime
+
+    return datetime.now(UTC).isoformat()
+
+
+def broadcast_board_event(
+    project_id: str,
+    event_type: str,
+    payload: dict[str, Any],
+) -> None:
+    """Send a JSON event to all WebSocket clients connected to a project group.
+
+    Synchronous helper for Django views and Celery tasks. Uses asgiref's
+    async_to_sync adapter (the official Channels approach) rather than
+    asyncio.run() so the existing thread's event-loop context is reused
+    instead of a fresh one being booted per call.
+
+    Durability: this broadcast is **best-effort by design** and safe to lose.
+    Every event it carries is durably persisted in the DB before the broadcast
+    is scheduled (callers wrap this in ``transaction.on_commit``), and clients
+    reconcile by pulling the sync delta on (re)connect. If the channel layer is
+    down at commit time the event is dropped and the client recovers it on its
+    next delta fetch — nothing durable is lost. This is why ``on_commit``
+    callbacks that *only* broadcast need no outbox row, whereas callbacks that
+    dispatch Celery work must commit one first. See
+    ``docs/durability/on-commit-audit.md`` (#659).
+
+    Args:
+        project_id:  UUID string of the project whose group to broadcast to.
+        event_type:  Short identifier for the event, e.g. "task_created" or
+                     "cpm_complete".
+        payload:     JSON-serializable dict with event-specific data.
+    """
+    from channels.layers import get_channel_layer
+
+    channel_layer = get_channel_layer()
+    if channel_layer is None:
+        logger.warning("broadcast_board_event: no channel layer configured, skipping broadcast")
+        return
+
+    # Persist the replay-buffer row first so its sequence can ride the live
+    # payload (ADR-0236). Best-effort: a persist failure yields seq=None and the
+    # event still broadcasts.
+    seq = _persist_board_event(project_id, event_type, payload)
+
+    group = _group_name(project_id)
+    try:
+        async_to_sync(channel_layer.group_send)(group, _board_message(event_type, payload, seq))
+    except Exception:
+        logger.exception("broadcast_board_event: failed to send %s to group %s", event_type, group)
+    else:
+        # Count the successful fan-out (#1900). A strict no-op unless telemetry is
+        # on, self-guarding, and in the ``else`` branch so a failed send is not
+        # counted — this only observes the broadcast, never alters its semantics.
+        _record_broadcast_metric()
+
+
+async def abroadcast_board_event(
+    project_id: str,
+    event_type: str,
+    payload: dict[str, Any],
+) -> None:
+    """Async-native broadcast for callers already on an event loop.
+
+    The synchronous ``broadcast_board_event`` wraps ``group_send`` in
+    ``async_to_sync``, which raises ``RuntimeError`` when called from a thread
+    that already runs an event loop — exactly the case inside a Channels
+    consumer. Async callers (e.g. ``ProjectConsumer`` presence join/leave) must
+    ``await`` this instead so ``group_send`` is awaited directly.
+
+    Same best-effort durability contract as the sync helper: a channel-layer
+    failure is logged and swallowed, never raised — clients reconcile on
+    reconnect (#958).
+
+    Args:
+        project_id:  UUID string of the project whose group to broadcast to.
+        event_type:  Short identifier for the event, e.g. "presence_join".
+        payload:     JSON-serializable dict with event-specific data.
+    """
+    from channels.layers import get_channel_layer
+
+    channel_layer = get_channel_layer()
+    if channel_layer is None:
+        logger.warning("abroadcast_board_event: no channel layer configured, skipping broadcast")
+        return
+
+    # Persist the replay-buffer row (ADR-0236). The only async callers today are
+    # presence join/leave, which are ephemeral and short-circuit to seq=None
+    # without touching the DB; the database_sync_to_async hop only runs if a future
+    # async caller broadcasts a replayable event, keeping this path symmetric with
+    # the sync helper rather than silently dropping such an event from the buffer.
+    if event_type in DONT_PERSIST_EVENT_TYPES:
+        seq = None
+    else:
+        from channels.db import database_sync_to_async
+
+        seq = await database_sync_to_async(_persist_board_event)(project_id, event_type, payload)
+
+    group = _group_name(project_id)
+    try:
+        await channel_layer.group_send(group, _board_message(event_type, payload, seq))
+    except Exception:
+        logger.exception("abroadcast_board_event: failed to send %s to group %s", event_type, group)
+    else:
+        # Count the successful fan-out (#1900) — same contract as the sync helper.
+        _record_broadcast_metric()
+
+
+def evict_project_connection(project_id: str, user_id: str) -> None:
+    """Push a ``connection.evict`` to a project's board WS group (#813).
+
+    Project membership is checked only at ``websocket_connect``; once accepted, a
+    socket keeps receiving every ``board_event`` until it disconnects. When a
+    ``ProjectMembership`` is soft-deleted or demoted below ``Role.MEMBER``, this
+    evicts the user's *live* sockets so they stop receiving real-time project data
+    immediately (the reconnect path is already gated — this closes the active-
+    connection gap, the analog of #419). Each consumer whose authenticated user
+    matches ``user_id`` closes with code 4003; sockets for other users ignore it.
+
+    Best-effort like ``broadcast_board_event`` — callers defer it with
+    ``transaction.on_commit``.
+    """
+    from channels.layers import get_channel_layer
+
+    channel_layer = get_channel_layer()
+    if channel_layer is None:
+        logger.warning("evict_project_connection: no channel layer configured, skipping")
+        return
+
+    message = {"type": "connection.evict", "user_id": str(user_id)}
+    group = _group_name(project_id)
+    try:
+        async_to_sync(channel_layer.group_send)(group, message)
+    except Exception:
+        logger.exception("evict_project_connection: failed to evict %s from %s", user_id, group)