trueppm-api 0.4.0b2__py3-none-any.whl

trueppm_api/apps/sync/migrations/0003_boardevent.py

@@ -0,0 +1,46 @@
+# Generated by Django 5.2.15 on 2026-07-05 23:56
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("projects", "0109_projectexportjob"),
+        (
+            "sync",
+            "0001_squashed_0002_remove_syncbatch_syncbatch_project_client_batch_uniq_and_more",
+        ),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="BoardEvent",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
+                    ),
+                ),
+                ("event_type", models.CharField(max_length=64)),
+                ("payload", models.JSONField(blank=True, default=dict)),
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                (
+                    "project",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="board_events",
+                        to="projects.project",
+                    ),
+                ),
+            ],
+            options={
+                "ordering": ["id"],
+                "indexes": [
+                    models.Index(fields=["project", "id"], name="boardevent_project_seq_idx"),
+                    models.Index(fields=["created_at"], name="boardevent_created_idx"),
+                ],
+            },
+        ),
+    ]

trueppm_api/apps/sync/migrations/0004_seed_sync_seq.py

@@ -0,0 +1,24 @@
+"""Seed the per-project sync delta cursor for existing data (ADR-0686, #2491).
+
+Runs after every app that adds the ``sync_seq`` column, because the renumber
+touches all of them in one pass. The logic lives in ``apps/sync/backfill.py`` so
+tests can import it without referencing this file's name (CLAUDE.md rule 3).
+"""
+
+from django.db import migrations
+
+from trueppm_api.apps.sync.backfill import seed_sync_seq, unseed_sync_seq
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("sync", "0003_boardevent"),
+        ("projects", "0128_acceptancecriterion_sync_seq_apitoken_sync_seq_and_more"),
+        ("access", "0018_programmembership_sync_seq_and_more"),
+        ("integrations", "0011_tasklink_sync_seq_tasklink_tasklink_syncseq_idx"),
+        ("timetracking", "0004_timeentry_sync_seq_and_more"),
+    ]
+
+    operations = [
+        migrations.RunPython(seed_sync_seq, unseed_sync_seq, elidable=True),
+    ]

trueppm_api/apps/sync/migrations/0005_program_sync_sequence.py

@@ -0,0 +1,35 @@
+"""Installation-wide allocator for the program sync cursor (ADR-0747, #2498).
+
+Creates the singleton counter, then seeds it and both program tables' ``sync_seq``
+in one ordered pass. The backfill logic lives in ``apps/sync/backfill.py`` so tests
+can import it without referencing this file's name (CLAUDE.md migration rule 3).
+"""
+
+from django.db import migrations, models
+
+from trueppm_api.apps.sync.backfill import seed_program_sync_seq, unseed_program_sync_seq
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("sync", "0004_seed_sync_seq"),
+        # The backfill reads both tables' server_version and writes their sync_seq,
+        # so both columns must already exist.
+        ("projects", "0128_acceptancecriterion_sync_seq_apitoken_sync_seq_and_more"),
+        ("access", "0018_programmembership_sync_seq_and_more"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="ProgramSyncSequence",
+            fields=[
+                ("id", models.BigAutoField(primary_key=True, serialize=False)),
+                ("value", models.BigIntegerField(default=0, editable=False)),
+            ],
+            options={
+                "verbose_name": "program sync sequence",
+                "verbose_name_plural": "program sync sequence",
+            },
+        ),
+        migrations.RunPython(seed_program_sync_seq, unseed_program_sync_seq, elidable=True),
+    ]

trueppm_api/apps/sync/models.py

@@ -0,0 +1,225 @@
+"""Models for the sync app.
+
+The only model here is :class:`SyncBatch`, the idempotency + atomicity
+envelope for a mobile offline **upload** batch (ADR-0082, issue #667). It is
+server infrastructure — the mobile client never pulls it as a domain row — so
+it is a plain ``models.Model`` (not ``VersionedModel``), mirroring the
+transactional-outbox style of ``projects.SprintCloseRequest``.
+"""
+
+from __future__ import annotations
+
+import uuid
+from datetime import timedelta
+
+from django.conf import settings
+from django.db import models
+from django.utils import timezone
+
+
+class SyncBatchStatus(models.TextChoices):
+    """Lifecycle of an upload batch envelope (ADR-0082 §D)."""
+
+    PENDING = "pending", "Pending"
+    COMPLETED = "completed", "Completed"
+
+
+class SyncBatch(models.Model):
+    """Records a mobile upload batch so a retry is a no-op, not a double-apply.
+
+    A mobile client generates a stable ``client_batch_id`` (UUID) per delta and
+    re-uploads the *same* id until it sees an ACK. The first request that wins
+    the unique-constraint race applies the delta and snapshots its HTTP response
+    here, inside the same transaction as the row writes (all-or-nothing). A
+    later request carrying the same id — the lost-ACK retry — returns the stored
+    response without re-applying anything.
+
+    Concurrency is serialized by the ``client_batch_id`` unique constraint:
+    Postgres blocks a concurrent duplicate INSERT until the first transaction
+    commits or aborts, after which the duplicate either reads the committed row
+    (replay) or proceeds (the first attempt rolled back). See
+    ``sync.views.ProjectSyncView.post`` for the full algorithm.
+
+    Rows are dedup-relevant only within the freshness window
+    (``TRUEPPM_SYNC_BATCH_RETENTION_HOURS``, default 24h); the
+    ``sync.purge_sync_batches`` Beat task reaps older rows so the table stays
+    bounded (ADR-0081 purge convention).
+    """
+
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    # Client-generated dedup key. Uniqueness (and every lookup) is scoped to
+    # (project, actor_user): a retry always targets the same project endpoint as
+    # the same authenticated user, so that composite unique index is a sufficient
+    # concurrency backstop, while scoping prevents one project — or one user —
+    # replaying another's stored response (the dedup lookup must never cross the
+    # project or actor boundary — see views.post, #887/#894).
+    client_batch_id = models.UUIDField()
+    project = models.ForeignKey(
+        "projects.Project",
+        on_delete=models.CASCADE,
+        related_name="sync_batches",
+    )
+    # Actor scoping (#894): the stored response body carries task ids,
+    # server_versions, and the sync watermark for the rows *this user* pushed.
+    # Keying dedup on (project, client_batch_id) alone let any project member
+    # replay another member's batch by reusing its client_batch_id and read back
+    # that user's response — an information leak across actors. Scoping
+    # uniqueness and the replay lookup to the actor closes it: a reused id from a
+    # different user is a distinct batch, never a replay of someone else's.
+    # Nullable so the migration is backfill-safe for rows written before this
+    # field existed (those legacy rows simply never match a new actor-scoped
+    # lookup and age out of the freshness window).
+    actor_user = models.ForeignKey(
+        settings.AUTH_USER_MODEL,
+        on_delete=models.CASCADE,
+        related_name="sync_batches",
+        null=True,
+        blank=True,
+    )
+    status = models.CharField(
+        max_length=12,
+        choices=SyncBatchStatus.choices,
+        default=SyncBatchStatus.PENDING,
+        db_index=True,
+    )
+    # Verbatim success response so a retry replays byte-identically.
+    response_body = models.JSONField(default=dict, blank=True)
+    response_status = models.PositiveSmallIntegerField(default=200)
+    created_at = models.DateTimeField(auto_now_add=True)
+
+    objects: models.Manager[SyncBatch] = models.Manager()
+
+    class Meta:
+        ordering = ["created_at"]
+        constraints = [
+            # Actor-scoped uniqueness (#894): two different users may legitimately
+            # reuse the same client_batch_id; only a same-(project, actor) duplicate
+            # is a replay. Scoping the constraint to the actor prevents one user's
+            # batch from blocking — or being replayed by — another's.
+            models.UniqueConstraint(
+                fields=["project", "actor_user", "client_batch_id"],
+                name="syncbatch_project_actor_client_batch_uniq",
+            ),
+        ]
+        indexes = [
+            # Reaper scans by age; the index keeps the nightly purge cheap.
+            models.Index(fields=["created_at"], name="syncbatch_created_idx"),
+        ]
+
+    def __str__(self) -> str:
+        return f"SyncBatch({self.client_batch_id}, {self.status})"
+
+    def is_fresh(self, *, ttl_hours: int = 24) -> bool:
+        """Return True if this batch is still within the dedup window.
+
+        A duplicate of a *fresh* completed batch replays the stored response; a
+        duplicate of an *expired* batch re-runs the apply (the stale row is
+        deleted first). ``ttl_hours`` is supplied by the caller from
+        ``settings.TRUEPPM_SYNC_BATCH_RETENTION_HOURS``.
+        """
+        return self.created_at > timezone.now() - timedelta(hours=ttl_hours)
+
+
+class BoardEvent(models.Model):
+    """Bounded per-project replay buffer for WebSocket events (ADR-0236, #321).
+
+    Real-time broadcasts are best-effort: a client that disconnects loses every
+    event sent during the gap. Each replayable broadcast persists one ``BoardEvent``
+    row here so a reconnecting client can request ``?since=<seq>`` and replay only
+    the events it missed — instead of a broad refetch storm — and recover the
+    incremental ``task_dates_updated`` CPM splices a refetch-on-focus would drop.
+
+    Like :class:`SyncBatch`, this is server infrastructure, never pulled as a synced
+    domain row, so it is a plain ``models.Model`` (no ``server_version``).
+
+    **The primary key is the sequence.** The ``BigAutoField`` ``id`` is assigned
+    atomically by Postgres at INSERT, so it is globally monotonic with no lost or
+    duplicate values under concurrent commits. Clients only need a strictly
+    increasing-per-project value for a ``seq > since`` comparison and tolerate gaps
+    (other projects' rows interleave the global sequence), so the PK *is* the
+    per-project sequence — no per-project counter row or dedicated sequence is
+    needed. The wire exposes it as ``seq``.
+
+    Rows are reaped past ``TRUEPPM_BOARD_EVENT_RETENTION_HOURS`` (default 24h) by
+    the ``sync.purge_board_events`` Beat task, keeping the buffer bounded (the
+    ADR-0081 purge convention). Because ``created_at`` is monotonic with ``id`` at
+    that granularity, the purge removes a contiguous low-``id`` prefix — which is
+    what lets the consumer use "oldest surviving id" as the gap-detection
+    watermark (see ``consumers.py``).
+    """
+
+    # id: implicit BigAutoField PK (DEFAULT_AUTO_FIELD) — doubles as the sequence.
+    project = models.ForeignKey(
+        "projects.Project",
+        on_delete=models.CASCADE,
+        related_name="board_events",
+    )
+    event_type = models.CharField(max_length=64)
+    payload = models.JSONField(default=dict, blank=True)
+    created_at = models.DateTimeField(auto_now_add=True)
+
+    objects: models.Manager[BoardEvent] = models.Manager()
+
+    class Meta:
+        # Replay streams strictly by ascending sequence (the PK).
+        ordering = ["id"]
+        indexes = [
+            # The replay query: WHERE project_id=? AND id > ? ORDER BY id.
+            models.Index(fields=["project", "id"], name="boardevent_project_seq_idx"),
+            # The reaper scans by age; keeps the nightly purge cheap.
+            models.Index(fields=["created_at"], name="boardevent_created_idx"),
+        ]
+
+    def __str__(self) -> str:
+        return f"BoardEvent(seq={self.pk}, {self.event_type}, project={self.project_id})"
+
+    @property
+    def seq(self) -> int:
+        """The wire sequence — an alias for the monotonic PK (ADR-0236)."""
+        return self.pk
+
+
+class ProgramSyncSequence(models.Model):
+    """Installation-wide monotonic allocator for the program sync cursor (ADR-0747).
+
+    Exactly one row, pk=1. Every synced write to ``Program`` or
+    ``ProgramMembership`` draws the next value from it and stamps the row's
+    ``sync_seq``, so those values are totally ordered installation-wide, ``MAX`` is
+    a meaningful checkpoint, and ``sync_seq__gt=since`` is a correct delta filter.
+
+    **Why a table row and not a PostgreSQL SEQUENCE.** ``nextval()`` would avoid
+    this row's write lock, and would be wrong. Sequences are non-transactional, so
+    they grant no relationship between allocation order and commit order: one
+    transaction could allocate 100, a later one allocate 101 and commit first, and a
+    pull between the two commits would report 101 and permanently skip 100 — the
+    exact defect this replaces, reintroduced as a race. Allocating by ``UPDATE``
+    holds this row's write lock until commit, so commit order equals allocation
+    order. The lock is the correctness mechanism, not overhead.
+
+    **Why installation-wide and not per-program.** ``Program`` rows have no single
+    owning entity to sequence from the way a ``Task`` has its project (ADR-0686).
+    The accessible set is per-user — the union of the caller's programs — so a
+    scalar cursor over per-program sequences would reintroduce the same
+    hot-row-outruns-cold-row failure one level up. Ordering every program-scoped
+    write against one counter is what makes a scalar cursor sound.
+
+    The cost is that program writes serialize installation-wide. That is acceptable
+    because program writes are rare and administrative — create, rename, add or
+    remove a member — never a hot path or a bulk import. See ADR-0747 for the
+    escape hatch if that ever changes.
+    """
+
+    #: The single row's primary key. Named so call sites read as intent.
+    SINGLETON_PK = 1
+
+    id = models.BigAutoField(primary_key=True)
+    value = models.BigIntegerField(default=0, editable=False)
+
+    objects: models.Manager[ProgramSyncSequence] = models.Manager()
+
+    class Meta:
+        verbose_name = "program sync sequence"
+        verbose_name_plural = "program sync sequence"
+
+    def __str__(self) -> str:
+        return f"ProgramSyncSequence(value={self.value})"

trueppm_api/apps/sync/pagination.py

@@ -0,0 +1,231 @@
+"""Cursor pagination for the offline delta pull (#1013).
+
+The delta pull (:class:`~trueppm_api.apps.sync.views.ProjectSyncView`) must not
+materialize an entire project into one unbounded JSON response on a cold start
+(``since=0``): a large project blows past the "500-task delta < 3s" mobile
+target. This module pages the delta so each response carries at most
+``page_size`` rows across all synced collections, and the client loops until the
+cursor is exhausted.
+
+Why a compound ``(table_index, version, id)`` keyset and not a scalar ceiling
+--------------------------------------------------------------------------------
+Rows share version values. Under the project sync cursor (``sync_seq``,
+ADR-0686) every row written in one batch draws the same value, and the program
+slice (ADR-0747) allocates from one installation-wide counter, so ties are rarer
+there but still possible across a single transaction's writes. A
+scalar "return ``since < version <= ceiling``, then ``next_since = ceiling``"
+cursor cannot bound page size in either case: to avoid splitting a version (which
+would silently drop rows) it must return *all* rows at the boundary version, and
+on a cold start that is the whole project. It is either unbounded or lossy.
+
+The fix keys the cursor on the pair ``(version, id)`` within each collection.
+``id`` is a globally unique UUID, so ``(version, id)`` is a **total order** even
+when every row shares a version — a page boundary can fall between two rows of
+the same version without ambiguity. Collections are drained in a fixed order, so
+each collection is a contiguous, non-overlapping segment of the global stream.
+The result: **no row is skipped and no row is duplicated** across pages, and
+every page is bounded by ``page_size``.
+
+Concurrency during a multi-page pull is safe because the version field only ever
+increases: a row edited mid-pull moves *forward* in ``(version, id)`` order, so a
+not-yet-reached row is delivered later and an already-delivered row is
+re-delivered under WatermelonDB upsert semantics — never lost.
+
+That argument only holds for rows the drain can still *reach*. A row written into
+a collection the cursor has already passed moves forward in a stream nobody is
+reading any more, so the drain cannot deliver it — the client's next ``since``
+has to stay low enough to catch it on the following pull. The watermark is
+therefore a property of the **pull session**, not of the request: it is computed
+once on the first (cursor-less) request, carried in the cursor as ``w``, and
+echoed unchanged on every continuation page (#2568). Recomputing it per request
+published a checkpoint above rows the session never returned, and the client
+adopting the last page's value skipped them permanently.
+
+The field is caller-supplied (``version_field``) rather than hard-coded, but both
+sync endpoints now pass ``sync_seq``: the project pull draws it from its project's
+sequence (ADR-0686) and the program pull from the installation-wide program
+sequence (ADR-0747). The parameter stays because the ordering column and the
+sources' ``__gt=since`` floor must agree, and making that agreement explicit at
+each call site is what stops the two drifting apart — the program pull used to
+default to ``server_version`` and carried the ordering defect #2498 fixed.
+"""
+
+from __future__ import annotations
+
+import base64
+import binascii
+import json
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any
+
+from django.db.models import Q
+from rest_framework.exceptions import ValidationError
+
+if TYPE_CHECKING:
+    from django.db.models import QuerySet
+
+# A single sync source: (collection_name, base_queryset, serializer_class). The
+# base queryset already carries the project scope, the ``<version_field>__gt=since``
+# floor, and any RBAC/visibility filters (retro visibility, per-user time entries).
+SyncSource = tuple[str, "QuerySet[Any]", Any]
+
+# Callable that splits a materialized row list into WatermelonDB
+# created/updated/deleted buckets (``ProjectSyncView._collect``).
+CollectFn = Callable[[list[Any], Any], dict[str, Any]]
+
+
+@dataclass(frozen=True)
+class SyncCursor:
+    """Position in the global ``(table_index, version, id)`` delta stream.
+
+    ``row_id`` is ``None`` for a *fresh-table* cursor (resume at the top of
+    ``sources[index]`` with only the ``since`` floor applied) and a UUID string
+    for an *intra-table* cursor (resume strictly after ``(version, row_id)``
+    within ``sources[index]``). Keeping the two cases distinct avoids emitting a
+    ``id > ''`` predicate that Postgres would reject when casting to ``uuid``.
+
+    ``watermark`` pins the checkpoint the whole drain must report as
+    ``timestamp`` (#2568). It is the session's state, not the page's: the view
+    computes it once on the cursor-less first request and thereafter reads it
+    back off the cursor instead of re-querying, so the client can never adopt a
+    checkpoint above a row the session had already paged past.
+    """
+
+    index: int
+    version: int
+    row_id: str | None
+    watermark: int
+
+    def encode(self) -> str:
+        """Serialize to an opaque, URL-safe token for the response envelope."""
+        raw = json.dumps(
+            {"i": self.index, "v": self.version, "id": self.row_id, "w": self.watermark},
+            separators=(",", ":"),
+        )
+        return base64.urlsafe_b64encode(raw.encode()).decode()
+
+    @classmethod
+    def decode(cls, token: str) -> SyncCursor:
+        """Parse a client-supplied cursor token, rejecting anything malformed.
+
+        The token is client-controlled, so every field is validated: a tampered
+        or truncated token yields a 400 rather than an unhandled 500. ``w`` is
+        validated exactly like ``i`` and ``v`` — required, integral, and
+        non-negative — because the view returns it verbatim as the checkpoint the
+        client adopts, which makes a junk value a data-loss vector rather than a
+        cosmetic one.
+
+        ``OverflowError`` is caught alongside the rest because Python's ``json``
+        accepts the non-standard literals ``Infinity``, ``-Infinity`` and
+        overflowing floats such as ``1e400``, and ``int()`` rejects them with an
+        ``ArithmeticError`` rather than the ``ValueError`` the other bad inputs
+        raise. Without it a one-token request reached an unhandled 500 — and this
+        endpoint is deliberately unthrottled (``get_throttles`` returns ``[]``
+        for GET), so that was a free error-log amplifier for any member.
+
+        A token minted before ``w`` existed carries no such key and is rejected
+        as malformed. That costs a drain straddling the deploy one 400; the
+        client restarts it at the same unchanged ``since`` and loses nothing.
+        """
+        try:
+            raw = base64.urlsafe_b64decode(token.encode())
+            data = json.loads(raw)
+            index = int(data["i"])
+            version = int(data["v"])
+            row_id = data["id"]
+            watermark = int(data["w"])
+        except (
+            binascii.Error,
+            ValueError,
+            KeyError,
+            TypeError,
+            OverflowError,
+            UnicodeDecodeError,
+        ) as err:
+            raise ValidationError({"cursor": "Malformed pagination cursor."}) from err
+        if (
+            index < 0
+            or version < 0
+            or watermark < 0
+            or (row_id is not None and not isinstance(row_id, str))
+        ):
+            raise ValidationError({"cursor": "Malformed pagination cursor."})
+        return cls(index=index, version=version, row_id=row_id, watermark=watermark)
+
+
+def paginate_changes(
+    sources: list[SyncSource],
+    *,
+    cursor: SyncCursor | None,
+    page_size: int,
+    collect: CollectFn,
+    watermark: int,
+    version_field: str = "server_version",
+) -> tuple[dict[str, Any], SyncCursor | None, bool]:
+    """Return one page of the delta: ``(changes, next_cursor, has_more)``.
+
+    ``changes`` always contains every collection key (empty buckets for
+    collections this page does not touch) so the response shape is stable across
+    pages. ``next_cursor`` is ``None`` exactly when the delta is fully drained
+    (``has_more`` is then ``False``); the client loops until then.
+
+    Each source is fetched ``ORDER BY <version_field>, id`` with ``LIMIT
+    page_size + 1`` (the extra row detects whether the collection still has more
+    beyond this page). Collections are drained in list order; a collection is
+    fully drained before the next one is touched, so the pages partition the
+    global stream without gaps or overlap.
+
+    ``version_field`` must be the same column the caller used for its
+    ``__gt=since`` floor — mixing the two (ordering on ``sync_seq`` while
+    filtering ``server_version``) would page a stream the floor does not bound.
+
+    ``watermark`` is stamped unchanged into every emitted cursor so the caller can
+    read the session checkpoint back off a continuation cursor rather than
+    recomputing it per page (#2568). It is threaded through here, not derived
+    here, because only the caller knows which sequence its ``since`` is drawn
+    from.
+    """
+    changes: dict[str, Any] = {
+        name: {"created": [], "updated": [], "deleted": []} for name, _, _ in sources
+    }
+    remaining = page_size
+    start = cursor.index if cursor is not None else 0
+
+    for i in range(start, len(sources)):
+        name, qs, serializer_class = sources[i]
+
+        # Intra-table resume: seek strictly past (version, row_id). A fresh-table
+        # cursor (row_id is None) or the very first page applies only the
+        # ``<version_field>__gt=since`` floor already baked into ``qs``.
+        if cursor is not None and i == cursor.index and cursor.row_id is not None:
+            qs = qs.filter(
+                Q(**{f"{version_field}__gt": cursor.version})
+                | Q(**{version_field: cursor.version, "id__gt": cursor.row_id})
+            )
+
+        rows = list(qs.order_by(version_field, "id")[: remaining + 1])
+        has_more_in_source = len(rows) > remaining
+        rows = rows[:remaining]
+        if rows:
+            changes[name] = collect(rows, serializer_class)
+        remaining -= len(rows)
+
+        if has_more_in_source:
+            # This collection still has rows; stop with an intra-table cursor.
+            last = rows[-1]
+            return (
+                changes,
+                SyncCursor(i, getattr(last, version_field), str(last.pk), watermark),
+                True,
+            )
+
+        if remaining == 0:
+            # Page is exactly full and this collection is drained. Resume at the
+            # top of the next collection, or finish if this was the last one.
+            if i + 1 < len(sources):
+                return changes, SyncCursor(i + 1, 0, None, watermark), True
+            return changes, None, False
+
+    # Every remaining collection drained within the budget — delta exhausted.
+    return changes, None, False