trueppm-api 0.4.0b2__py3-none-any.whl

trueppm_api/apps/scheduling/apps.py

@@ -0,0 +1,153 @@
+"""App config for scheduling."""
+
+from __future__ import annotations
+
+import logging
+import time
+from typing import Any
+
+from django.apps import AppConfig
+
+logger = logging.getLogger(__name__)
+
+# Wall-clock start time (time.monotonic(), immune to system clock adjustments) for
+# each in-flight Celery task execution, keyed by task_id. Populated in
+# _on_task_prerun and consumed (popped) in _on_task_postrun — that pair always
+# brackets one execution 1:1, including retried and failed ones, so duration is
+# available for every outcome, not just success (#1917). Celery itself does not
+# set a usable `.runtime` attribute on the request for anything but the success
+# path, which is why this is timed here instead of read off `task.request`.
+_task_start_times: dict[str, float] = {}
+
+
+class SchedulingConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "trueppm_api.apps.scheduling"
+
+    def ready(self) -> None:
+        """Bridge Celery framework signals to Django signals.
+
+        Enterprise packages connect receivers to the Django signals in
+        scheduling.signals — they never need to import from Celery directly.
+
+        Every ``.connect(...)`` below now passes ``weak=False`` (#1917 fix). Celery's
+        ``Signal.connect`` defaults to a *weak* reference to the receiver, and the
+        four ``_on_task_*`` functions are closures local to this method call — with
+        no other strong reference anywhere, they were garbage-collected the moment
+        ``ready()`` returned, silently turning the whole Celery→Django task-lifecycle
+        bridge into dead code (confirmed via ``task_prerun.receivers`` showing a
+        dead weakref immediately after startup). None of ``celery_task_started`` /
+        ``_succeeded`` / ``_failed`` / ``_retried`` were ever actually being sent in
+        a real worker, which is why #1917 found no start/success/retry log lines to
+        begin with — this was the root cause, not just a missing receiver. A stable
+        ``dispatch_uid`` per connect call is required alongside ``weak=False``: without
+        it, a second ``ready()`` call (the test runner / autoreloader can trigger one)
+        would register a *second*, now-permanent strong-ref closure rather than being
+        recognized as the same receiver, doubling every log line and metric sample
+        from then on.
+        """
+        from celery.signals import task_failure, task_postrun, task_prerun, task_retry
+
+        # Register the OSS dead-letter alerting receiver (ADR-0084) and the
+        # start/success/retry structured-log receivers (#1917). Side-effect
+        # import: the @receiver decorators in receivers.py connect on load.
+        from trueppm_api.apps.observability.otel.metrics import record_task_duration
+        from trueppm_api.apps.scheduling import receivers  # noqa: F401
+        from trueppm_api.apps.scheduling.signals import (
+            celery_task_failed,
+            celery_task_retried,
+            celery_task_started,
+            celery_task_succeeded,
+        )
+        from trueppm_api.core.extension_signals import dispatch_extension_signal
+
+        self._register_workflows()
+
+        @task_prerun.connect(weak=False, dispatch_uid="scheduling.on_task_prerun")  # type: ignore[untyped-decorator]
+        def _on_task_prerun(sender: Any, task_id: str, task: Any, **kwargs: Any) -> None:
+            _task_start_times[task_id] = time.monotonic()
+            # sender=None: these signals are about a worker run, not a model row,
+            # so there is no class for the dispatcher to filter on. Receivers
+            # branch on the task_name kwarg (#3777).
+            dispatch_extension_signal(
+                celery_task_started,
+                sender=None,
+                task_id=task_id,
+                task_name=getattr(task, "name", ""),
+                args=kwargs.get("args", ()),
+                kwargs=kwargs.get("kwargs", {}),
+            )
+
+        @task_postrun.connect(weak=False, dispatch_uid="scheduling.on_task_postrun")  # type: ignore[untyped-decorator]
+        def _on_task_postrun(
+            sender: Any, task_id: str, task: Any, retval: Any, **kwargs: Any
+        ) -> None:
+            # task_postrun fires for every terminal state (success, failure, retry,
+            # rejected, ignored) — unlike celery_task_succeeded below, the duration
+            # metric is recorded for all of them so a stuck/slow retry loop is
+            # visible in the histogram too, not just clean runs.
+            state = kwargs.get("state", "") or ""
+            task_name = getattr(task, "name", "")
+            start = _task_start_times.pop(task_id, None)
+            duration = time.monotonic() - start if start is not None else 0.0
+            record_task_duration(
+                task_name=task_name,
+                duration_seconds=duration,
+                outcome=state.lower() or "unknown",
+            )
+            if state == "SUCCESS":
+                dispatch_extension_signal(
+                    celery_task_succeeded,
+                    sender=None,
+                    task_id=task_id,
+                    task_name=task_name,
+                    runtime_seconds=duration,
+                )
+
+        @task_failure.connect(weak=False, dispatch_uid="scheduling.on_task_failure")  # type: ignore[untyped-decorator]
+        def _on_task_failure(
+            sender: Any, task_id: str, exception: BaseException, **kwargs: Any
+        ) -> None:
+            import traceback as tb_module
+
+            dispatch_extension_signal(
+                celery_task_failed,
+                sender=None,
+                task_id=task_id,
+                task_name=getattr(sender, "name", ""),
+                exception=exception,
+                traceback_str="".join(
+                    tb_module.format_exception(type(exception), exception, exception.__traceback__)
+                ),
+            )
+
+        @task_retry.connect(weak=False, dispatch_uid="scheduling.on_task_retry")  # type: ignore[untyped-decorator]
+        def _on_task_retry(sender: Any, request: Any, reason: Any, **kwargs: Any) -> None:
+            dispatch_extension_signal(
+                celery_task_retried,
+                sender=None,
+                task_id=getattr(request, "id", ""),
+                task_name=getattr(sender, "name", ""),
+                attempt=getattr(request, "retries", 0),
+                exception=reason,
+            )
+
+    @staticmethod
+    def _register_workflows() -> None:
+        """Register the scheduling app's durable workflow definitions (ADR-0080 §A).
+
+        The requeue-failed-task workflow (ADR-0210) is the first consumer of the
+        durable backend. Registered here from the owning app's ``ready()`` — the
+        scheduling app owns ``FailedTask`` and already bridges Celery signals in
+        ``ready()``, so the workflow lives with its domain rather than in the
+        backend-neutral engine app. Guarded against duplicate registration because
+        ``ready()`` can run more than once under the test runner and the registry
+        raises on a duplicate name.
+        """
+        from trueppm_api.workflows.consumers.requeue_failed_task import (
+            RequeueFailedTaskWorkflow,
+        )
+        from trueppm_api.workflows.registry import WORKFLOWS
+
+        if RequeueFailedTaskWorkflow.name not in WORKFLOWS.all():
+            WORKFLOWS.register(RequeueFailedTaskWorkflow())

trueppm_api/apps/scheduling/calendars.py

@@ -0,0 +1,119 @@
+"""Composable working-calendar resolution (#906, ADR-0251).
+
+A project's effective non-working mask for CPM is the overlay (union) of several
+applied calendars: a base ``Project.calendar`` plus zero or more
+``ProjectCalendarLayer`` overlays. A day is non-working if *any* applied calendar
+marks it so — masks AND-composed, exception ranges unioned.
+
+The fold itself lives in ``trueppm_scheduler.models.Calendar.compose`` (the Apache
+package). This module is the Django-side seam: it resolves *which* calendars apply
+to a project (through a small resolver registry) and converts them to scheduler
+dataclasses via the single ``build_sched_calendar`` primitive.
+
+The resolver registry is the one-way (enterprise → core) extension point
+(ADR-0251 §6): OSS registers exactly one resolver (base + overlay layers);
+enterprise appends more (per-resource, cross-program calendars) against the same
+signature. Because composition is AND-of-masks + union-of-exceptions, any added
+resolver is always additive and order-independent, so a new source can never be a
+breaking change to an existing project's schedule.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import TYPE_CHECKING, Any
+
+from trueppm_api.apps.scheduling.services import build_sched_calendar
+
+if TYPE_CHECKING:
+    from trueppm_api.apps.projects.models import Calendar, Project
+
+# A resolver maps a project to the ordered Django ``Calendar`` rows it applies.
+CalendarLayerResolver = Callable[["Project"], "list[Calendar]"]
+
+_LAYER_RESOLVERS: list[CalendarLayerResolver] = []
+
+
+def register_calendar_layer_resolver(resolver: CalendarLayerResolver) -> None:
+    """Register a resolver that contributes calendars to a project's composed mask.
+
+    Idempotent: registering the same resolver twice is a no-op, so an
+    ``AppConfig.ready()`` that runs more than once (or a re-imported test module)
+    cannot double-count a source.
+    """
+    if resolver not in _LAYER_RESOLVERS:
+        _LAYER_RESOLVERS.append(resolver)
+
+
+def oss_project_layers(project: Project) -> list[Calendar]:
+    """OSS resolver: the effective base calendar plus its overlay layers, in order.
+
+    The base is resolved through the inheritance chain (ADR-0441): the project's own
+    ``calendar`` if set, else its program's, else the workspace default — computed-on-read
+    in ``apps.projects.calendar_settings`` so CPM and the serializers can never disagree
+    on which calendar a project schedules against. ``None`` up the whole chain means the
+    system default, so ``compose`` falls back to Mon-Fri/8h/UTC exactly as before.
+
+    Overlays (``ProjectCalendarLayer``, ADR-0251) still stack on top of the inherited
+    base — inheritance only decides the base; it does not change overlay composition.
+
+    Callers must have prefetched ``calendar__exceptions``,
+    ``calendar_layers__calendar__exceptions`` and — so the inherited tiers don't N+1 —
+    ``program__calendar__exceptions`` when the project may inherit from its program.
+    """
+    from trueppm_api.apps.projects.calendar_settings import resolve_effective_base_calendar
+
+    calendars: list[Calendar] = []
+    base = resolve_effective_base_calendar(project)
+    if base is not None:
+        calendars.append(base)
+    # Ordered by ProjectCalendarLayer.Meta.ordering = ["sort_order"].
+    for layer in project.calendar_layers.all():
+        calendars.append(layer.calendar)
+    return calendars
+
+
+# Registered here (module import) rather than in AppConfig.ready() because the
+# OSS resolver has no app-loading dependency — importing this module is what wires
+# it up. Enterprise registers its extra resolvers from its own AppConfig.ready().
+register_calendar_layer_resolver(oss_project_layers)
+
+
+def resolve_applied_calendars(project: Project) -> list[Calendar]:
+    """The de-duplicated, ordered list of every ``Calendar`` applied to a project.
+
+    Folds the output of every registered resolver into one ordered list, dropping
+    a calendar already contributed by an earlier resolver (the base calendar, for
+    instance, must not be counted twice if a later resolver also returns it). The
+    first occurrence — the base calendar under the OSS resolver — leads, so its
+    scalar ``hours_per_day``/``timezone`` win in :func:`compose_project_calendar`.
+    """
+    seen: set[Any] = set()
+    result: list[Calendar] = []
+    for resolver in _LAYER_RESOLVERS:
+        for cal in resolver(project):
+            if cal.id not in seen:
+                seen.add(cal.id)
+                result.append(cal)
+    return result
+
+
+def compose_project_calendar(project: Project) -> Any:
+    """Fold a project's applied calendars into one scheduler ``Calendar`` (#906).
+
+    ``working_days`` = bitwise-AND of every applied mask; ``exceptions`` = union of
+    every applied calendar's ``CalendarException`` ranges; ``hours_per_day`` /
+    ``timezone`` from the base (first) calendar. Compute-on-read (ADR-0251 §3): the
+    fold is O(layers) and the expensive per-day lookup is already indexed inside
+    the scheduler ``Calendar``. A project with no applied calendars falls back to
+    the Mon-Fri/8h/UTC default, identical to ``build_sched_calendar(None)``.
+
+    Every scheduling call site (CPM pass, Monte Carlo, program schedule) routes
+    through this one helper so composition can never drift between engines. Callers
+    must prefetch ``calendar__exceptions`` and
+    ``calendar_layers__calendar__exceptions``.
+    """
+    from trueppm_scheduler.models import Calendar as SchedCalendar
+
+    applied = resolve_applied_calendars(project)
+    return SchedCalendar.compose([build_sched_calendar(c) for c in applied])

trueppm_api/apps/scheduling/deadletter.py

@@ -0,0 +1,91 @@
+"""Dead-letter helper — records permanently failed Celery tasks."""
+
+from __future__ import annotations
+
+import logging
+import traceback as tb_module
+
+logger = logging.getLogger(__name__)
+
+
+def record_failed_task(
+    task_name: str,
+    task_id: str,
+    args: list[object] | tuple[object, ...],
+    kwargs: dict[str, object],
+    exception: BaseException,
+    project_id: str | None = None,
+) -> None:
+    """Write a FailedTask row for a permanently failed Celery task.
+
+    Called from the on_failure handler when max_retries is exhausted.
+    Safe to call from within a Celery worker — uses its own DB connection.
+
+    On the *terminal* transition (a FailedTask row is newly created) this fires
+    ``celery_task_permanently_failed`` so OSS logs an alert and Enterprise can
+    register PagerDuty/Slack receivers (ADR-0084). A repeat dead-letter of the
+    same ``task_id`` only bumps ``failure_count`` and does **not** re-alert, so
+    the signal means "newly dead-lettered", not "failed again". ``project_id``
+    is threaded through from the caller when known (it is not derivable here).
+    """
+    from trueppm_api.apps.scheduling.models import FailedTask, FailedTaskStatus
+    from trueppm_api.apps.scheduling.signals import celery_task_permanently_failed
+    from trueppm_api.core.extension_signals import dispatch_extension_signal
+
+    exc_type = type(exception).__qualname__
+    exc_msg = str(exception)
+    exc_tb = "".join(
+        tb_module.format_exception(type(exception), exception, exception.__traceback__)
+    )
+
+    defaults: dict[str, object] = {
+        "task_name": task_name,
+        "args": list(args) if args else [],
+        "kwargs": dict(kwargs) if kwargs else {},
+        "exception_type": exc_type,
+        "exception_message": exc_msg,
+        "traceback": exc_tb,
+        "status": FailedTaskStatus.DEAD,
+    }
+    # Only set project_id when this call actually knows it (#1917).
+    # update_or_create's defaults overwrite unconditionally, so unconditionally
+    # including a bare `None` here would erase a project attribution recorded by
+    # an earlier failure of the same task_id the moment a later re-failure's
+    # call site didn't have project_id in scope.
+    if project_id is not None:
+        defaults["project_id"] = project_id
+
+    obj, created = FailedTask.objects.update_or_create(
+        task_id=task_id,
+        defaults=defaults,
+    )
+    if not created:
+        FailedTask.objects.filter(pk=obj.pk).update(
+            failure_count=obj.failure_count + 1,
+        )
+
+    logger.error(
+        "Dead-lettered task %s (%s): %s: %s",
+        task_name,
+        task_id,
+        exc_type,
+        exc_msg,
+    )
+
+    if created:
+        # A misbehaving receiver (incl. enterprise PagerDuty/Slack) must never
+        # break the dead-letter recording path.
+        #
+        # sender=None: this signal is about a worker run, not a model row. This
+        # site is also why the Celery family could not use a class even if one
+        # were wanted — the dead-letter recorder has the task's name and no task
+        # object at all. Receivers branch on the task_name kwarg (#3777).
+        dispatch_extension_signal(
+            celery_task_permanently_failed,
+            sender=None,
+            task_id=task_id,
+            task_name=task_name,
+            exception=exception,
+            traceback_str=exc_tb,
+            project_id=project_id,
+        )

trueppm_api/apps/scheduling/forecast_history_settings.py

@@ -0,0 +1,215 @@
+"""Workspace → Program → Project Monte Carlo forecast-history resolution (ADR-0144, #1232).
+
+The three forecast-history settings — ``mc_history_enabled`` (whether the run
+history list is exposed at all), ``mc_history_retention_cap`` (how many runs the
+nightly purge keeps), and ``mc_history_attribution_audience`` (who may see the
+run-author name) — default at the workspace and may be overridden per
+program/project. The *effective* value is resolved here, computed-on-read
+(ADR-0108): there is no stored/denormalized effective column to keep in sync.
+Clients (web, mobile, MCP) read the serializer's ``effective_mc_history_*`` /
+``inherited_mc_history_*``; they never re-implement this precedence.
+
+Precedence: project override → program override → workspace value → Django
+settings default. ``inherited_*`` answers "what would I get if this scope's
+override were cleared?" (i.e. the parent's effective value) and drives the
+settings "Inherit (…)" affordance.
+
+This module is the forecast-history sibling of ``apps.projects.sharing_settings``
+(ADR-0135) and ``apps.projects.iteration_label`` (ADR-0116); it deliberately
+mirrors their resolver + enforcement-provider shape. ``ENFORCE`` (a workspace
+admin locking the config so lower scopes cannot override it) is an Enterprise
+capability. OSS ships the inert provider below and registers nothing, so
+``ENFORCE`` degrades to ``SUGGEST`` (no lock). trueppm-enterprise registers a
+provider in its ``AppConfig.ready()`` — the integrations-registry idiom
+(ADR-0029/0049). OSS never imports enterprise.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import TYPE_CHECKING, Any
+
+from django.conf import settings
+
+from trueppm_api.apps.scheduling.models import MCAttributionAudience
+from trueppm_api.core.extension_providers import guard_single_provider
+
+if TYPE_CHECKING:
+    from trueppm_api.apps.projects.models import Program, Project
+    from trueppm_api.apps.workspace.models import Workspace
+
+#: The three inheritable forecast-history settings. The field name is identical on
+#: Workspace (non-null root), Program, and Project (nullable override), so the
+#: resolver below is field/key-agnostic.
+MC_HISTORY_FIELDS = (
+    "mc_history_enabled",
+    "mc_history_retention_cap",
+    "mc_history_attribution_audience",
+)
+
+# Enterprise registers a zero-arg predicate that returns True when forecast-history
+# enforcement is licensed/active. OSS leaves it None → enforcement inactive, so
+# ENFORCE behaves as SUGGEST (no lock).
+_ENFORCEMENT_PROVIDER: Callable[[], bool] | None = None
+
+
+def register_forecast_history_enforcement_provider(provider: Callable[[], bool] | None) -> None:
+    """Register (or clear) the forecast-history enforcement provider. Enterprise calls this."""
+    global _ENFORCEMENT_PROVIDER
+    _ENFORCEMENT_PROVIDER = guard_single_provider(
+        _ENFORCEMENT_PROVIDER, provider, "forecast history enforcement"
+    )
+
+
+def forecast_history_enforcement_active(workspace: Workspace | None = None) -> bool:
+    """True only when an enterprise provider is registered AND reports active.
+
+    OSS has no provider → always False, so a workspace ``ENFORCE`` policy never
+    locks downstream forecast-history overrides in the community edition.
+    ``workspace`` is accepted for signature parity with the sharing seam and any
+    future provider that needs the workspace context; the inert OSS path ignores it.
+    """
+    return _ENFORCEMENT_PROVIDER is not None and bool(_ENFORCEMENT_PROVIDER())
+
+
+def _settings_default(key: str) -> Any:
+    """The installation-wide fallback for ``key`` when nothing up the chain is set.
+
+    ``MC_HISTORY_CAP`` may be ``None`` (Enterprise unlimited retention); when it is
+    we fall back to the non-null Workspace default of 100 so the resolver always
+    returns a concrete int for the retention cap.
+    """
+    if key == "mc_history_enabled":
+        return True
+    if key == "mc_history_attribution_audience":
+        return MCAttributionAudience.ADMIN_OWNER
+    if key == "mc_history_retention_cap":
+        cap = settings.MC_HISTORY_CAP
+        return cap if cap is not None else 100
+    raise ValueError(f"Unknown forecast-history key: {key}")
+
+
+def _clamp(key: str, value: Any) -> Any:
+    """Clamp a resolved value to its hard bound.
+
+    ``mc_history_retention_cap`` is clamped to ``settings.MC_HISTORY_HARD_CAP`` so a
+    misconfigured (or future Enterprise-injected) value can never unbound the
+    nightly purge, which would otherwise let per-project history grow without limit.
+    """
+    if key == "mc_history_retention_cap" and value is not None:
+        return min(int(value), settings.MC_HISTORY_HARD_CAP)
+    return value
+
+
+def _enforced(workspace: Workspace) -> bool:
+    """Whether the workspace value is a hard lock (Enterprise enforcement active)."""
+    from trueppm_api.apps.workspace.models import TermOverridePolicy
+
+    return (
+        workspace.mc_history_override_policy == TermOverridePolicy.ENFORCE
+        and forecast_history_enforcement_active(workspace)
+    )
+
+
+def resolve_effective_mc_history(
+    obj: Program | Project | Workspace,
+    key: str,
+    *,
+    workspace: Workspace | None = None,
+) -> Any:
+    """Resolve the effective value of ``key`` for a project, program, or workspace.
+
+    Walks the override chain Project → Program → Workspace → Django settings default,
+    returning the first non-null value found. ``obj`` may be a ``Project`` (resolved
+    up its program → workspace), a ``Program`` (resolved up to the workspace), or the
+    ``Workspace`` itself (its own non-null column).
+
+    ``workspace`` may be passed to avoid re-loading the singleton per object when
+    resolving a list (the serializer caches it once); otherwise it is loaded here.
+
+    Args:
+        obj: The Project, Program, or Workspace to resolve for.
+        key: One of ``MC_HISTORY_FIELDS``.
+        workspace: Optional pre-loaded Workspace singleton.
+
+    Returns:
+        The effective value (clamped for the retention cap).
+    """
+    from trueppm_api.apps.workspace.models import Workspace as WorkspaceModel
+
+    if workspace is None:
+        workspace = WorkspaceModel.load()
+
+    # Resolving the workspace itself: its own non-null column (or settings default,
+    # defensive — the column is non-null so this falls through to the value).
+    if isinstance(obj, WorkspaceModel):
+        ws_value = getattr(obj, key)
+        return _clamp(key, ws_value if ws_value is not None else _settings_default(key))
+
+    if _enforced(workspace):
+        # Enterprise lock: the workspace value wins regardless of lower overrides.
+        return _clamp(key, getattr(workspace, key))
+
+    own = getattr(obj, key)  # nullable override on Program/Project
+    if own is not None:
+        return _clamp(key, own)
+    return _clamp(key, _parent_value(obj, key, workspace=workspace))
+
+
+def resolve_inherited_mc_history(
+    obj: Program | Project,
+    key: str,
+    *,
+    workspace: Workspace | None = None,
+) -> Any:
+    """Value ``obj`` would inherit if its own override were cleared.
+
+    Drives the settings "Inherit (…)" affordance — it deliberately skips ``obj``'s
+    own override and resolves from the parent up. Under an active Enterprise lock the
+    inherited value is the (locked) workspace value.
+
+    Args:
+        obj: The Program or Project to resolve the inherited value for.
+        key: One of ``MC_HISTORY_FIELDS``.
+        workspace: Optional pre-loaded Workspace singleton.
+
+    Returns:
+        The value ``obj`` would resolve to with its own override cleared.
+    """
+    from trueppm_api.apps.workspace.models import Workspace as WorkspaceModel
+
+    if workspace is None:
+        workspace = WorkspaceModel.load()
+
+    if _enforced(workspace):
+        return _clamp(key, getattr(workspace, key))
+    return _clamp(key, _parent_value(obj, key, workspace=workspace))
+
+
+def _parent_value(
+    obj: Program | Project,
+    key: str,
+    *,
+    workspace: Workspace,
+) -> Any:
+    """Resolve ``key`` from ``obj``'s parent scope up to the workspace value.
+
+    A Program's parent is the workspace. A Project's parent is its program (whose
+    own override resolves up to the workspace) or, if standalone, the workspace.
+    The workspace columns are non-null, so the chain always terminates in a
+    concrete value (no settings-default fall-through is needed below the workspace).
+    """
+    # Local import keeps this module import-safe from migrations.
+    from trueppm_api.apps.projects.models import Program
+
+    program: Program | None = None
+    if not isinstance(obj, Program) and obj.program_id:
+        program = obj.program
+
+    if program is not None:
+        program_override = getattr(program, key)
+        if program_override is not None:
+            return program_override
+
+    ws_value = getattr(workspace, key)
+    return ws_value if ws_value is not None else _settings_default(key)