trueppm-api 0.4.0b2__py3-none-any.whl

trueppm_api/apps/jiraimport/tasks.py

@@ -0,0 +1,277 @@
+"""Celery tasks for offline Jira import (mirrors msproject.tasks)."""
+
+from __future__ import annotations
+
+import base64
+import logging
+from collections.abc import Generator
+from contextlib import contextmanager
+from datetime import timedelta
+from typing import Any
+
+from celery import shared_task
+
+from trueppm_api.core.idempotent import idempotent_task
+
+logger = logging.getLogger(__name__)
+
+# A Jira import running longer than this is considered orphaned by the drain
+# (slightly longer than the 9-minute soft limit so the task can record its own
+# failure before the drain resets the row).
+_IMPORT_ORPHAN_MINUTES = 15
+
+
+class _NoOpTracker:
+    """Fallback when TaskRunTracker is not available."""
+
+    task_run_id = None
+
+    def update(self, pct: int, msg: str = "") -> None:
+        logger.info("import.jira progress: %d%% -- %s", pct, msg)
+
+    def set_result(self, result: dict[str, Any]) -> None:
+        logger.info("import.jira result: %s", result)
+
+
+@contextmanager
+def _get_tracker(
+    celery_task: Any,
+    project_id: str,
+    initiated_by_id: int | None,
+) -> Generator[Any, None, None]:
+    """Try to use TaskRunTracker; fall back to no-op if not available."""
+    try:
+        from trueppm_api.apps.taskruns.tracker import TaskRunTracker
+
+        with TaskRunTracker(
+            celery_task,
+            project_id=project_id,
+            task_name="import.jira",
+            initiated_by_id=initiated_by_id,
+        ) as tracker:
+            yield tracker
+    except ImportError:
+        yield _NoOpTracker()
+
+
+@shared_task(  # type: ignore[untyped-decorator]
+    bind=True,
+    name="jira.import_file",
+    soft_time_limit=540,
+    time_limit=600,
+    acks_late=True,
+    reject_on_worker_lost=True,
+)
+def import_jira(
+    self: Any,
+    project_id: str,
+    file_content_b64: str,
+    filename: str,
+    initiated_by_id: int | None = None,
+    import_request_id: str | None = None,
+) -> dict[str, Any]:
+    """Parse an uploaded Jira XML export and import it as a CPM-schedulable network.
+
+    Parses offline (no network), validates the derived dependency graph with the
+    shared guard (#1665) **before** persisting anything, then reuses the MS
+    Project importer to bulk-create the Task + FS Dependency network and triggers
+    a CPM recalculation via the scheduling outbox.
+
+    A deterministic failure — an unparseable file or a cyclic / infeasible graph
+    — marks the outbox row DEAD so the drain stops re-dispatching it forever.
+    """
+    from trueppm_api.apps.jiraimport.parser import JiraImportError, parse_jira_xml
+
+    with _get_tracker(self, project_id, initiated_by_id) as tracker:
+        file_content = base64.b64decode(file_content_b64)
+        tracker.update(5, f"Parsing {filename}...")
+
+        try:
+            project_data = parse_jira_xml(file_content)
+        except JiraImportError:
+            # Unparseable input is deterministic: retrying the same bytes always
+            # fails. Mark the row DEAD (terminal) so the orphan drain stops.
+            if import_request_id:
+                _mark_import_dead(import_request_id)
+            raise
+
+        # Validate the derived dependency graph BEFORE any write (#1665, ADR-0259).
+        # A prospect's messy export (cyclic links) must never persist an
+        # infeasible network that then crashes the CPM / what-if engine live.
+        # Validate in uid space — a relabeling of the eventual PKs — so nothing
+        # is created if the graph is bad. The parser already quarantined
+        # self-loops and dangling edges; a genuine cycle is rejected here.
+        from trueppm_api.apps.scheduling.graph_guard import (
+            InfeasibleGraphError,
+            validate_task_graph,
+        )
+
+        edges = [
+            (str(link.predecessor_uid), str(task.uid))
+            for task in project_data.tasks
+            for link in task.predecessor_links
+        ]
+        tracker.update(20, "Validating dependency graph...")
+        try:
+            validate_task_graph(edges)
+        except InfeasibleGraphError:
+            # Deterministic: the same export always produces the same cycle.
+            if import_request_id:
+                _mark_import_dead(import_request_id)
+            raise
+
+        from django.db import transaction
+
+        from trueppm_api.apps.msproject.importer import (
+            broadcast_import_project_record_change,
+            import_project,
+        )
+        from trueppm_api.apps.projects.models import TaskSource
+
+        # Make the additive Jira import idempotent under re-dispatch (#1673).
+        # import_project defaults to wipe_existing=False, so a second delivery
+        # would bulk-create the whole network again. The outbox row is the
+        # idempotency token: claim it (DISPATCHED -> DONE) and run the
+        # bulk_create in ONE transaction. A duplicate delivery — a drain
+        # re-dispatch after a worker-death window, or an acks_late /
+        # reject_on_worker_lost redelivery of an already completed message —
+        # finds no DISPATCHED row to claim and skips the import. The claim's row
+        # lock serializes any concurrent delivery; because claim and import share
+        # the transaction, a mid-import failure rolls the claim back too,
+        # returning the row to DISPATCHED for a clean retry. Mirrors the MS
+        # Project import-into-existing fix.
+        with transaction.atomic():
+            if import_request_id and not _claim_import(import_request_id):
+                logger.info(
+                    "import.jira: request %s already completed — skipping duplicate delivery",
+                    import_request_id,
+                )
+                return {"skipped": True, "tasks_created": 0}
+            # Explicit source_kind: this path shares import_project with the MS
+            # Project importer, whose default would mislabel Jira rows (ADR-0786).
+            summary = import_project(
+                project_id,
+                project_data,
+                tracker=tracker,
+                source_kind=TaskSource.JIRA_IMPORT,
+                source_id=import_request_id or None,
+            )
+
+        tracker.set_result(summary)
+
+        if summary["tasks_created"] > 0:
+            # Trigger CPM via the outbox (survives a broker outage at this point)
+            # and emit tasks_restructured so live clients render the imported tree
+            # immediately, ahead of the async recalc's cpm_complete (mirrors the
+            # MS Project importer, #1359). Both now run just after the import's
+            # atomic() has committed (no ambient transaction remains → on_commit
+            # fires immediately), so peers never see the event for a rolled-back
+            # import.
+            from trueppm_api.apps.scheduling.services import enqueue_recalculate
+            from trueppm_api.apps.sync.broadcast import broadcast_board_event
+
+            enqueue_recalculate(project_id)
+            transaction.on_commit(
+                lambda: broadcast_board_event(project_id, "tasks_restructured", {})
+            )
+
+        # See the CSV importer's note: a start shift is not a task-tree change,
+        # so tasks_restructured does not cover it.
+        broadcast_import_project_record_change(project_id, summary)
+
+    return summary
+
+
+def _claim_import(import_request_id: str) -> bool:
+    """Atomically claim a DISPATCHED JiraImportRequest, flipping it to DONE.
+
+    The conditional ``UPDATE ... WHERE status=DISPATCHED`` is the exactly-once
+    idempotency token for the import (#1673): run inside the same transaction as
+    the bulk_create, the row transitions to DONE atomically with the imported
+    network. Returns ``True`` when this delivery won the claim (import), ``False``
+    when the row was already terminal — a duplicate delivery whose import must be
+    skipped so the additive path does not bulk-create the network twice. Clears
+    the payload (only needed for pre-terminal retry).
+    """
+    from trueppm_api.apps.jiraimport.models import JiraImportRequest, JiraImportStatus
+
+    claimed = JiraImportRequest.objects.filter(
+        id=import_request_id, status=JiraImportStatus.DISPATCHED
+    ).update(status=JiraImportStatus.DONE, file_content_b64="")
+    return bool(claimed)
+
+
+def _mark_import_dead(import_request_id: str) -> None:
+    """Flip the row to DEAD after a terminal failure and clear the payload.
+
+    DEAD is excluded from orphan-drain recovery, so a deterministically bad file
+    (unparseable, or a cyclic graph) is not re-dispatched forever.
+    """
+    from trueppm_api.apps.jiraimport.models import JiraImportRequest, JiraImportStatus
+
+    JiraImportRequest.objects.filter(id=import_request_id).exclude(
+        status=JiraImportStatus.DONE
+    ).update(status=JiraImportStatus.DEAD, file_content_b64="")
+
+
+@idempotent_task(
+    lock_key_template="drain_jira_import_queue",
+    lock_ttl=60,
+    on_contention="skip",
+    soft_time_limit=25,
+    time_limit=30,
+    acks_late=True,
+    reject_on_worker_lost=True,
+    name="jira.drain_import_queue",
+)
+def drain_jira_import_queue(self: object) -> None:
+    """Dispatch pending JiraImportRequest rows; recover orphaned dispatched rows.
+
+    Runs every 30 seconds via Celery Beat. Recovers dispatched rows older than
+    ``_IMPORT_ORPHAN_MINUTES`` (worker died mid-import).
+    """
+    _do_jira_import_drain()
+
+
+def _do_jira_import_drain() -> None:
+    """Business logic for drain_jira_import_queue — extracted for testability."""
+    from django.utils import timezone
+
+    from trueppm_api.apps.jiraimport.models import JiraImportRequest, JiraImportStatus
+
+    now = timezone.now()
+    orphan_cutoff = now - timedelta(minutes=_IMPORT_ORPHAN_MINUTES)
+
+    recovered = JiraImportRequest.objects.filter(
+        status=JiraImportStatus.DISPATCHED,
+        dispatched_at__lt=orphan_cutoff,
+    ).update(status=JiraImportStatus.PENDING, celery_task_id="")
+    if recovered:
+        logger.warning("drain_jira_import_queue: recovered %d orphaned row(s)", recovered)
+
+    pending = list(JiraImportRequest.objects.filter(status=JiraImportStatus.PENDING))
+    dispatched = 0
+    for req in pending:
+        try:
+            result = import_jira.delay(
+                project_id=str(req.project_id),
+                file_content_b64=req.file_content_b64,
+                filename=req.filename,
+                initiated_by_id=req.initiated_by_id,
+                import_request_id=str(req.id),
+            )
+        except Exception:
+            logger.warning(
+                "drain_jira_import_queue: broker unavailable — JiraImportRequest %s stays pending",
+                req.id,
+            )
+            continue
+        JiraImportRequest.objects.filter(id=req.id, status=JiraImportStatus.PENDING).update(
+            status=JiraImportStatus.DISPATCHED,
+            celery_task_id=result.id,
+            dispatched_at=now,
+        )
+        dispatched += 1
+
+    if dispatched or recovered:
+        logger.info("drain_jira_import_queue: dispatched=%d recovered=%d", dispatched, recovered)

trueppm_api/apps/jiraimport/urls.py

@@ -0,0 +1,15 @@
+"""URL routing for offline Jira import."""
+
+from __future__ import annotations
+
+from django.urls import path
+
+from trueppm_api.apps.jiraimport.views import JiraImportView
+
+urlpatterns = [
+    path(
+        "projects/<project_pk>/import/jira/",
+        JiraImportView.as_view(),
+        name="project-import-jira",
+    ),
+]

trueppm_api/apps/jiraimport/views.py

@@ -0,0 +1,153 @@
+"""REST API view for offline Jira XML import."""
+
+from __future__ import annotations
+
+import base64
+import logging
+import os
+import re
+
+from django.conf import settings
+from django.db import transaction
+from drf_spectacular.types import OpenApiTypes
+from drf_spectacular.utils import OpenApiResponse, extend_schema, inline_serializer
+from rest_framework import serializers, status
+from rest_framework.exceptions import PermissionDenied
+from rest_framework.parsers import MultiPartParser
+from rest_framework.permissions import IsAuthenticated
+from rest_framework.request import Request
+from rest_framework.response import Response
+from rest_framework.views import APIView
+
+from trueppm_api.apps.access.models import ProjectMembership, Role
+from trueppm_api.apps.access.permissions import IsProjectAdmin, IsProjectNotArchived
+from trueppm_api.apps.idempotency.mixins import IdempotencyMixin
+
+logger = logging.getLogger(__name__)
+
+# Only XML is accepted: it is the sole Jira export that carries <issuelinks>
+# (CSV does not), and links are what make the network CPM-computable (ADR-0259).
+_ALLOWED_EXTENSIONS = {"xml"}
+# Allow-list (mirrors the hardened MS Project sanitizer, #816): substitute
+# anything outside a conservative safe set so an attacker-controlled
+# UploadedFile.name can never carry an HTML/Content-Disposition-injection
+# payload into a future provenance/list surface. os.path.basename below blocks
+# path traversal.
+_FILENAME_ALLOWED = re.compile(r"[^A-Za-z0-9._\- ()]")
+
+
+def _sanitize_filename(raw: str) -> str:
+    name = os.path.basename(raw or "")
+    name = _FILENAME_ALLOWED.sub("_", name)
+    name = re.sub(r"\s+", " ", name).strip()
+    return name[:255] or "upload.xml"
+
+
+def _require_project_admin(user: object, project_pk: str) -> None:
+    """Authoritative in-body Admin check (defense-in-depth behind IsProjectAdmin)."""
+    role = (
+        ProjectMembership.objects.filter(
+            project_id=project_pk,
+            user=user,  # type: ignore[misc]
+            is_deleted=False,
+        )
+        .values_list("role", flat=True)
+        .first()
+    )
+    if role is None or role < Role.ADMIN:
+        raise PermissionDenied("You need at least Project Admin role to import.")
+
+
+def _read_validated_upload(request: Request) -> tuple[str, str] | Response:
+    """Read the multipart ``file`` field; validate extension + size.
+
+    Returns ``(filename, file_content_b64)`` or a 400 ``Response``.
+    """
+    uploaded_file = request.FILES.get("file")
+    if not uploaded_file:
+        return Response(
+            {"detail": "No file provided. Send a multipart form with a 'file' field."},
+            status=status.HTTP_400_BAD_REQUEST,
+        )
+
+    filename = _sanitize_filename(uploaded_file.name or "")
+    ext = filename.rsplit(".", 1)[-1].lower() if "." in filename else ""
+    if ext not in _ALLOWED_EXTENSIONS:
+        return Response(
+            {"detail": f"Unsupported file type: .{ext}. Export your Jira issues as XML."},
+            status=status.HTTP_400_BAD_REQUEST,
+        )
+
+    max_bytes = settings.JIRA_IMPORT_MAX_UPLOAD_MB * 1024 * 1024
+    if uploaded_file.size and uploaded_file.size > max_bytes:
+        return Response(
+            {
+                "detail": (
+                    f"File too large ({uploaded_file.size} bytes). "
+                    f"Maximum: {settings.JIRA_IMPORT_MAX_UPLOAD_MB} MB."
+                )
+            },
+            status=status.HTTP_400_BAD_REQUEST,
+        )
+
+    file_content_b64 = base64.b64encode(uploaded_file.read()).decode("ascii")
+    return filename, file_content_b64
+
+
+class JiraImportView(IdempotencyMixin, APIView):
+    """Upload and import a Jira XML export into an existing project.
+
+    Requires project Admin role. The import runs asynchronously via Celery; the
+    202 response carries the ``import_request_id`` for progress tracking.
+    """
+
+    # Multipart upload, deduped at the table level via JiraImportRequest.
+    idempotency_exempt = True
+    # IsProjectAdmin gates at the DRF layer (OpenAPI-visible, enforced before the
+    # body); the in-body _require_project_admin below stays authoritative.
+    permission_classes = [IsAuthenticated, IsProjectAdmin, IsProjectNotArchived]
+    parser_classes = [MultiPartParser]
+
+    @extend_schema(
+        summary="Import a Jira XML export into an existing project",
+        request=OpenApiTypes.BINARY,
+        responses={
+            202: inline_serializer(
+                name="JiraImportQueuedResponse",
+                fields={
+                    "detail": serializers.CharField(),
+                    "import_request_id": serializers.UUIDField(),
+                },
+            ),
+            400: OpenApiResponse(description="Missing or invalid upload (wrong type/too large)."),
+            403: OpenApiResponse(description="Caller lacks the Admin role on the project."),
+        },
+    )
+    def post(self, request: Request, project_pk: str) -> Response:
+        _require_project_admin(request.user, project_pk)
+
+        validated = _read_validated_upload(request)
+        if isinstance(validated, Response):
+            return validated
+        filename, file_content_b64 = validated
+
+        from trueppm_api.apps.jiraimport.models import JiraImportRequest
+        from trueppm_api.apps.jiraimport.services import enqueue_jira_import
+
+        # Commit the outbox row before any dispatch attempt so a broker outage
+        # cannot lose the import — the row stays PENDING and the drain retries.
+        with transaction.atomic():
+            req = JiraImportRequest.objects.create(
+                project_id=project_pk,
+                filename=filename,
+                file_content_b64=file_content_b64,
+                initiated_by_id=request.user.pk,
+            )
+
+        req_id = str(req.pk)
+        transaction.on_commit(lambda: enqueue_jira_import(req_id))
+
+        return Response(
+            {"detail": "Import queued.", "import_request_id": req_id},
+            status=status.HTTP_202_ACCEPTED,
+        )

trueppm_api/apps/msproject/apps.py

@@ -0,0 +1,11 @@
+"""Django app config for MS Project import/export."""
+
+from django.apps import AppConfig
+
+
+class MsProjectConfig(AppConfig):
+    """MS Project import/export app."""
+
+    name = "trueppm_api.apps.msproject"
+    verbose_name = "MS Project Import/Export"
+    default_auto_field = "django.db.models.BigAutoField"

trueppm_api/apps/msproject/dataclasses.py

@@ -0,0 +1,153 @@
+"""Data structures for MS Project import/export interchange."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+#: MSPDI ``<ConstraintType>`` codes the importer can express, mapped onto
+#: ``Task.planned_start`` — which *is* a start-no-earlier-than floor applied by
+#: the CPM forward pass (#2891).
+#:
+#: - **4, Start No Earlier Than** — exactly ``planned_start``'s semantics. Carried
+#:   across losslessly.
+#: - **2, Must Start On** — floors the start on the same date, but TruePPM has no
+#:   way to enforce the "and no later" half. Carried across as the floor *and*
+#:   warned, because a half-enforced commitment the operator believes is whole is
+#:   worse than one they know is advisory.
+#:
+#: Code 0 (As Soon As Possible) is TruePPM's own default and therefore loses
+#: nothing when dropped. Every remaining code (1 ALAP, 3 MFO, 5 SNLT, 6 FNET,
+#: 7 FNLT) has no model home at all and is reported in the import warnings.
+#:
+#: Lives here rather than in ``parser`` because it is a claim about the
+#: *interchange contract* — what a ``TaskData.constraint_type`` value causes the
+#: importer to do — which every adapter that fills the field needs to know.
+CONSTRAINT_TYPES_APPLIED_AS_SNET: frozenset[int] = frozenset({2, 4})
+
+
+@dataclass
+class TaskData:
+    """Parsed task from an MS Project file."""
+
+    uid: int
+    name: str
+    duration_days: int = 1
+    outline_number: str = ""
+    outline_level: int = 0
+    is_milestone: bool = False
+    percent_complete: float = 0.0
+    notes: str = ""
+    start: str | None = None  # ISO date string (YYYY-MM-DD)
+    # Canonical TaskStatus *value* string (e.g. "complete"), or None when the
+    # source did not supply an explicit status. The Jira parser maps the issue's
+    # status name onto this field; the MS Project importer derives it from the
+    # clamped percent-complete when None (#1768). Keeping the interchange value a
+    # plain string keeps this dataclass free of a Django-model import.
+    status: str | None = None
+    # Task-level <CalendarUID> (#1769). TruePPM has no per-task calendars, so
+    # the importer only uses this to warn when a task references a calendar
+    # other than the project calendar. None covers both "element absent" and
+    # the MSPDI sentinel -1 ("no task calendar").
+    calendar_uid: int | None = None
+    # Three-point / PERT estimate fields (#798, ADR-0093). Working days,
+    # nullable, all-or-none: the importer sets all three to None unless the
+    # source file supplied all three for a leaf (non-summary, non-milestone)
+    # task. The fields map to Task.optimistic_duration / most_likely_duration /
+    # pessimistic_duration (IntegerField, working days) on the model.
+    optimistic_duration_days: int | None = None
+    most_likely_duration_days: int | None = None
+    pessimistic_duration_days: int | None = None
+    # MSPDI <ConstraintType> code, 0-7 (#2891). This is the mechanism a PM uses
+    # to *pin a committed date*, so dropping it silently turned a defensible
+    # commitment into a CPM-derived guess. TruePPM models exactly one constraint
+    # — the start-no-earlier-than floor `Task.planned_start` — so only the two
+    # codes that floor a start (4 SNET, 2 MSO) reach the model; the parser warns
+    # for every other code rather than pretending it was carried over. See
+    # `MSPDI_CONSTRAINT_NAMES` / `CONSTRAINT_TYPES_APPLIED_AS_SNET` in parser.py.
+    constraint_type: int | None = None
+    # <ConstraintDate> as an ISO date string, or None. Only meaningful alongside
+    # a constraint_type that carries a date (codes 2-7).
+    constraint_date: str | None = None
+    # <ActualStart> / <ActualFinish> as ISO date strings (#2891). These map
+    # one-to-one onto Task.actual_start / Task.actual_finish, so unlike the
+    # constraint family they are carried across exactly.
+    actual_start: str | None = None
+    actual_finish: str | None = None
+    predecessor_links: list[PredecessorLinkData] = field(default_factory=list)
+    resource_assignments: list[AssignmentData] = field(default_factory=list)
+    # Label *names* (not slugs or ids) this task carries, in first-seen order
+    # (ADR-0400, #2406). Deliberately names: the parser that fills this cannot
+    # see the database, so resolving a name to a catalog entry — matching an
+    # existing one case-insensitively, or creating it — belongs to
+    # ``import_project``. Populated by the CSV/Excel adapter today; the Jira
+    # adapter's components/labels, dropped since #743, are the obvious next
+    # producer.
+    labels: list[str] = field(default_factory=list)
+
+
+@dataclass
+class PredecessorLinkData:
+    """Parsed predecessor link from an MS Project file."""
+
+    predecessor_uid: int
+    dep_type: str = "FS"  # FS, SS, FF, SF
+    lag_days: int = 0
+
+
+@dataclass
+class ResourceData:
+    """Parsed resource from an MS Project file."""
+
+    uid: int
+    name: str
+    max_units: float = 1.0
+
+
+@dataclass
+class AssignmentData:
+    """Parsed resource assignment from an MS Project file."""
+
+    task_uid: int
+    resource_uid: int
+    units: float = 1.0
+
+
+@dataclass
+class CalendarExceptionData:
+    """Parsed non-working exception (holiday, shutdown) from a calendar (#1769)."""
+
+    start: str  # ISO date string (YYYY-MM-DD)
+    end: str  # ISO date string (YYYY-MM-DD), inclusive
+    name: str = ""
+
+
+@dataclass
+class CalendarData:
+    """Parsed base calendar from an MS Project file (#1769).
+
+    ``working_days`` uses the TruePPM ``Calendar.working_days`` bitmask
+    (Mon=1, Tue=2, Wed=4, Thu=8, Fri=16, Sat=32, Sun=64) — the parser converts
+    from MSPDI ``DayType`` (1=Sunday … 7=Saturday). Only base calendars are
+    parsed; resource calendars (``IsBaseCalendar=0``) have no TruePPM home.
+    """
+
+    uid: int
+    name: str
+    working_days: int = 31  # Mon–Fri
+    hours_per_day: float = 8.0
+    exceptions: list[CalendarExceptionData] = field(default_factory=list)
+
+
+@dataclass
+class ProjectData:
+    """Complete parsed project data from an MS Project file."""
+
+    name: str = ""
+    start_date: str | None = None  # ISO date string
+    # Project-level <CalendarUID> — which parsed calendar the plan is scheduled
+    # on (#1769). None when the header omits it.
+    calendar_uid: int | None = None
+    calendars: list[CalendarData] = field(default_factory=list)
+    tasks: list[TaskData] = field(default_factory=list)
+    resources: list[ResourceData] = field(default_factory=list)
+    warnings: list[str] = field(default_factory=list)