flowsense-engine 0.2.1__py3-none-any.whl

flowsense/engine/propagation.py

@@ -0,0 +1,173 @@
+from __future__ import annotations
+
+from flowsense.domain import DriftResult, PropagationResult, Severity
+from flowsense.domain.enums import SEVERITY_SCORE
+
+_HOP_DECAY = 0.8
+_MAX_SEVERITY_SCORE = max(SEVERITY_SCORE.values())
+
+
+def _build_reverse_dependencies(
+    dependencies: dict[str, list[str]],
+) -> dict[str, list[str]]:
+    reverse_dependencies: dict[str, list[str]] = {
+        task_id: [] for task_id in dependencies
+    }
+
+    for upstream_task, downstream_tasks in dependencies.items():
+        for downstream_task in downstream_tasks:
+            reverse_dependencies.setdefault(downstream_task, []).append(upstream_task)
+
+    return reverse_dependencies
+
+
+def _has_anomalous_upstream(
+    task_id: str,
+    drift_results: dict[str, DriftResult],
+    reverse_dependencies: dict[str, list[str]],
+    visited: set[str] | None = None,
+) -> bool:
+    if visited is None:
+        visited = set()
+
+    if task_id in visited:
+        return False
+
+    visited = {*visited, task_id}
+
+    for upstream_task in reverse_dependencies.get(task_id, []):
+        upstream_drift = drift_results.get(upstream_task)
+
+        if upstream_drift is None or upstream_drift.severity == Severity.NORMAL:
+            continue
+
+        if upstream_drift.severity in {
+            Severity.HIGH,
+            Severity.CRITICAL,
+        }:
+            return True
+
+        if _has_anomalous_upstream(
+            task_id=upstream_task,
+            drift_results=drift_results,
+            reverse_dependencies=reverse_dependencies,
+            visited=visited,
+        ):
+            return True
+
+    return False
+
+
+def _find_propagation_paths(
+    origin_task: str,
+    current_task: str,
+    drift_results: dict[str, DriftResult],
+    dependencies: dict[str, list[str]],
+    path: list[str],
+    visited: set[str],
+) -> list[list[str]]:
+    paths: list[list[str]] = []
+
+    for downstream_task in dependencies.get(current_task, []):
+        if downstream_task in visited:
+            continue
+
+        downstream_drift = drift_results.get(downstream_task)
+
+        if downstream_drift is None:
+            continue
+
+        if downstream_drift.severity == Severity.NORMAL:
+            continue
+
+        next_path = [*path, downstream_task]
+        next_visited = {*visited, downstream_task}
+
+        child_paths = _find_propagation_paths(
+            origin_task=origin_task,
+            current_task=downstream_task,
+            drift_results=drift_results,
+            dependencies=dependencies,
+            path=next_path,
+            visited=next_visited,
+        )
+
+        if child_paths:
+            paths.extend(child_paths)
+        else:
+            paths.append(next_path)
+
+    return paths
+
+
+def _calculate_propagation_score(
+    affected_tasks: list[str],
+    drift_results: dict[str, DriftResult],
+) -> float:
+    """Return a bounded, distance-weighted score for a propagation path."""
+    weighted_severity = 0.0
+    total_weight = 0.0
+
+    for hop, task_id in enumerate(affected_tasks):
+        weight = _HOP_DECAY**hop
+        normalized_severity = (
+            SEVERITY_SCORE[drift_results[task_id].severity] / _MAX_SEVERITY_SCORE
+        )
+        weighted_severity += normalized_severity * weight
+        total_weight += weight
+
+    if total_weight == 0.0:
+        return 0.0
+
+    return weighted_severity / total_weight
+
+
+def analyze_propagation(
+    drift_results: dict[str, DriftResult],
+    dependencies: dict[str, list[str]],
+) -> list[PropagationResult]:
+    results: list[PropagationResult] = []
+
+    reverse_dependencies = _build_reverse_dependencies(dependencies)
+
+    for task_id, drift in drift_results.items():
+        if drift.severity not in {Severity.HIGH, Severity.CRITICAL}:
+            continue
+
+        if _has_anomalous_upstream(
+            task_id=task_id,
+            drift_results=drift_results,
+            reverse_dependencies=reverse_dependencies,
+        ):
+            continue
+
+        paths = _find_propagation_paths(
+            origin_task=task_id,
+            current_task=task_id,
+            drift_results=drift_results,
+            dependencies=dependencies,
+            path=[task_id],
+            visited={task_id},
+        )
+
+        for path in paths:
+            affected_tasks = path[1:]
+
+            if not affected_tasks:
+                continue
+
+            propagation_score = _calculate_propagation_score(
+                affected_tasks=affected_tasks,
+                drift_results=drift_results,
+            )
+
+            results.append(
+                PropagationResult(
+                    origin_task=task_id,
+                    affected_tasks=affected_tasks,
+                    path=path,
+                    propagation_score=propagation_score,
+                )
+            )
+
+    return results

flowsense/engine/root_cause.py

@@ -0,0 +1,148 @@
+from __future__ import annotations
+
+from flowsense.domain import (
+    DriftResult,
+    ImpactClassification,
+    PropagationResult,
+    RootCauseResult,
+    Severity,
+    TaskImpact,
+)
+from flowsense.domain.enums import SEVERITY_SCORE
+
+
+def _build_reverse_dependencies(
+    dependencies: dict[str, list[str]],
+) -> dict[str, list[str]]:
+    reverse_dependencies: dict[str, list[str]] = {}
+
+    for upstream_task, downstream_tasks in dependencies.items():
+        reverse_dependencies.setdefault(upstream_task, [])
+
+        for downstream_task in downstream_tasks:
+            reverse_dependencies.setdefault(
+                downstream_task,
+                [],
+            ).append(upstream_task)
+
+    return reverse_dependencies
+
+
+def _has_candidate_upstream(
+    task_id: str,
+    candidate_tasks: set[str],
+    drift_results: dict[str, DriftResult],
+    reverse_dependencies: dict[str, list[str]],
+    visited: set[str] | None = None,
+) -> bool:
+    if visited is None:
+        visited = set()
+
+    if task_id in visited:
+        return False
+
+    visited.add(task_id)
+
+    for upstream_task in reverse_dependencies.get(task_id, []):
+        upstream_drift = drift_results.get(upstream_task)
+
+        if upstream_drift is None or upstream_drift.severity == Severity.NORMAL:
+            continue
+
+        if upstream_task in candidate_tasks:
+            return True
+
+        if _has_candidate_upstream(
+            task_id=upstream_task,
+            candidate_tasks=candidate_tasks,
+            drift_results=drift_results,
+            reverse_dependencies=reverse_dependencies,
+            visited=visited,
+        ):
+            return True
+
+    return False
+
+
+def select_primary_origin(
+    drift_results: dict[str, DriftResult],
+    task_impacts: dict[str, TaskImpact],
+    dependencies: dict[str, list[str]],
+    propagation_results: list[PropagationResult],
+) -> RootCauseResult | None:
+    candidate_tasks = {
+        task_id
+        for task_id, impact in task_impacts.items()
+        if impact.classification
+        in {
+            ImpactClassification.OWN_DRIFT,
+            ImpactClassification.COMBINED,
+        }
+    }
+
+    if not candidate_tasks:
+        return None
+
+    reverse_dependencies = _build_reverse_dependencies(
+        dependencies,
+    )
+
+    root_candidates = {
+        task_id
+        for task_id in candidate_tasks
+        if not _has_candidate_upstream(
+            task_id=task_id,
+            candidate_tasks=candidate_tasks,
+            drift_results=drift_results,
+            reverse_dependencies=reverse_dependencies,
+        )
+    }
+
+    if not root_candidates:
+        root_candidates = candidate_tasks
+
+    propagation_scores: dict[str, float] = {}
+
+    for result in propagation_results:
+        current_score = propagation_scores.get(
+            result.origin_task,
+            0.0,
+        )
+
+        propagation_scores[result.origin_task] = max(
+            current_score,
+            result.propagation_score,
+        )
+
+    def ranking(task_id: str) -> tuple[float, int]:
+        propagation_score = propagation_scores.get(
+            task_id,
+            0.0,
+        )
+
+        drift = drift_results.get(task_id)
+
+        severity_score = SEVERITY_SCORE[drift.severity] if drift else 0
+
+        return (
+            propagation_score,
+            severity_score,
+        )
+
+    primary_task = max(
+        root_candidates,
+        key=ranking,
+    )
+
+    impact = task_impacts[primary_task]
+    drift = drift_results[primary_task]
+
+    return RootCauseResult(
+        task_id=primary_task,
+        classification=impact.classification,
+        severity=drift.severity,
+        propagation_score=propagation_scores.get(
+            primary_task,
+            0.0,
+        ),
+    )

flowsense/engine/timing.py

@@ -0,0 +1,221 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime
+
+from flowsense.domain import (
+    DEFAULT_ANALYSIS_POLICY,
+    AnalysisPolicy,
+    DriftResult,
+    InvalidTaskTimingError,
+    TaskRun,
+)
+from flowsense.engine.drift import calculate_drift
+
+
+@dataclass
+class HandoffTiming:
+    upstream_task: str
+    downstream_task: str
+    dag_run_id: str
+    handoff_delay: float
+
+
+@dataclass(frozen=True)
+class HandoffHistoryDiagnostic:
+    code: str
+    upstream_task: str
+    downstream_task: str
+    dag_run_id: str
+    message: str
+
+
+@dataclass
+class HandoffHistoryResult:
+    history: dict[tuple[str, str], list[float]]
+    diagnostics: list[HandoffHistoryDiagnostic]
+
+
+def _required_end_date(task_run: TaskRun) -> datetime:
+    if task_run.end_date is None:
+        raise InvalidTaskTimingError("Upstream task end_date is required.")
+    return task_run.end_date
+
+
+def _required_start_date(task_run: TaskRun) -> datetime:
+    if task_run.start_date is None:
+        raise InvalidTaskTimingError("Downstream task start_date is required.")
+    return task_run.start_date
+
+
+def calculate_handoff_delay(
+    upstream_run: TaskRun,
+    downstream_run: TaskRun,
+) -> HandoffTiming:
+    if upstream_run.end_date is None:
+        raise InvalidTaskTimingError("Upstream task end_date is required.")
+
+    if downstream_run.start_date is None:
+        raise InvalidTaskTimingError("Downstream task start_date is required.")
+
+    if upstream_run.dag_run_id != downstream_run.dag_run_id:
+        raise InvalidTaskTimingError("Task runs must belong to the same DAG run.")
+
+    handoff_delay = (downstream_run.start_date - upstream_run.end_date).total_seconds()
+
+    return HandoffTiming(
+        upstream_task=upstream_run.task_id,
+        downstream_task=downstream_run.task_id,
+        dag_run_id=upstream_run.dag_run_id,
+        handoff_delay=handoff_delay,
+    )
+
+
+def build_handoff_history(
+    task_runs: list[TaskRun],
+    dependencies: dict[str, list[str]],
+) -> dict[tuple[str, str], list[float]]:
+    return build_handoff_history_with_diagnostics(
+        task_runs=task_runs,
+        dependencies=dependencies,
+    ).history
+
+
+def build_handoff_history_with_diagnostics(
+    task_runs: list[TaskRun],
+    dependencies: dict[str, list[str]],
+) -> HandoffHistoryResult:
+    """Build logical-edge history across regular and dynamically mapped tasks.
+
+    A mapped upstream is complete at its latest instance end, while a mapped
+    downstream starts at its earliest instance start.
+    """
+    runs_by_id: dict[str, dict[str, list[TaskRun]]] = {}
+
+    for task_run in task_runs:
+        tasks_in_run = runs_by_id.setdefault(task_run.dag_run_id, {})
+        tasks_in_run.setdefault(task_run.task_id, []).append(task_run)
+
+    history: dict[tuple[str, str], list[float]] = {}
+    diagnostics: list[HandoffHistoryDiagnostic] = []
+
+    for dag_run_id, tasks_in_run in runs_by_id.items():
+        for upstream_task, downstream_tasks in dependencies.items():
+            upstream_runs = tasks_in_run.get(upstream_task, [])
+
+            if not upstream_runs:
+                for downstream_task in downstream_tasks:
+                    diagnostics.append(
+                        HandoffHistoryDiagnostic(
+                            code="MISSING_UPSTREAM_TASK_RUN",
+                            upstream_task=upstream_task,
+                            downstream_task=downstream_task,
+                            dag_run_id=dag_run_id,
+                            message=(
+                                f"{upstream_task} has no task run in {dag_run_id}."
+                            ),
+                        )
+                    )
+                continue
+
+            if any(task_run.end_date is None for task_run in upstream_runs):
+                for downstream_task in downstream_tasks:
+                    diagnostics.append(
+                        HandoffHistoryDiagnostic(
+                            code="MISSING_UPSTREAM_END_DATE",
+                            upstream_task=upstream_task,
+                            downstream_task=downstream_task,
+                            dag_run_id=dag_run_id,
+                            message=(
+                                f"{upstream_task} has no complete end_date in "
+                                f"{dag_run_id}."
+                            ),
+                        )
+                    )
+                continue
+
+            upstream_run = max(
+                upstream_runs,
+                key=_required_end_date,
+            )
+
+            for downstream_task in downstream_tasks:
+                downstream_runs = tasks_in_run.get(downstream_task, [])
+
+                if not downstream_runs:
+                    diagnostics.append(
+                        HandoffHistoryDiagnostic(
+                            code="MISSING_DOWNSTREAM_TASK_RUN",
+                            upstream_task=upstream_task,
+                            downstream_task=downstream_task,
+                            dag_run_id=dag_run_id,
+                            message=(
+                                f"{downstream_task} has no task run in {dag_run_id}."
+                            ),
+                        )
+                    )
+                    continue
+
+                if any(task_run.start_date is None for task_run in downstream_runs):
+                    diagnostics.append(
+                        HandoffHistoryDiagnostic(
+                            code="MISSING_DOWNSTREAM_START_DATE",
+                            upstream_task=upstream_task,
+                            downstream_task=downstream_task,
+                            dag_run_id=dag_run_id,
+                            message=(
+                                f"{downstream_task} has no complete start_date in "
+                                f"{dag_run_id}."
+                            ),
+                        )
+                    )
+                    continue
+
+                downstream_run = min(
+                    downstream_runs,
+                    key=_required_start_date,
+                )
+
+                try:
+                    timing = calculate_handoff_delay(
+                        upstream_run=upstream_run,
+                        downstream_run=downstream_run,
+                    )
+                except InvalidTaskTimingError as exc:
+                    diagnostics.append(
+                        HandoffHistoryDiagnostic(
+                            code="INVALID_HANDOFF_TIMING",
+                            upstream_task=upstream_task,
+                            downstream_task=downstream_task,
+                            dag_run_id=dag_run_id,
+                            message=str(exc),
+                        )
+                    )
+                    continue
+
+                edge = (
+                    upstream_task,
+                    downstream_task,
+                )
+
+                history.setdefault(edge, []).append(timing.handoff_delay)
+
+    return HandoffHistoryResult(
+        history=history,
+        diagnostics=diagnostics,
+    )
+
+
+def calculate_handoff_drift(
+    upstream_task: str,
+    downstream_task: str,
+    handoff_delays: list[float],
+    policy: AnalysisPolicy = DEFAULT_ANALYSIS_POLICY,
+) -> DriftResult:
+    edge_id = f"{upstream_task}->{downstream_task}"
+
+    return calculate_drift(
+        task_id=edge_id,
+        durations=handoff_delays,
+        policy=policy,
+    )

flowsense/engine/trend.py

@@ -0,0 +1,82 @@
+from __future__ import annotations
+
+import math
+
+import numpy as np
+
+from flowsense.domain import TrendDirection, TrendResult
+
+
+def _theil_sen_slope(observations: np.ndarray) -> float:
+    slopes = [
+        float((observations[end] - observations[start]) / (end - start))
+        for start in range(len(observations) - 1)
+        for end in range(start + 1, len(observations))
+    ]
+    return float(np.median(slopes))
+
+
+def detect_trend(
+    subject_id: str,
+    values: list[float],
+    *,
+    minimum_observations: int = 5,
+    score_threshold: float = 3.5,
+    minimum_directional_consistency: float = 0.6,
+) -> TrendResult | None:
+    """Detect a sustained linear trend using a robust Theil-Sen slope."""
+    if minimum_observations < 3:
+        raise ValueError("minimum_observations must be at least 3")
+
+    if score_threshold <= 0:
+        raise ValueError("score_threshold must be positive")
+
+    if not 0 < minimum_directional_consistency <= 1:
+        raise ValueError("minimum_directional_consistency must be in (0, 1]")
+
+    if len(values) < minimum_observations:
+        return None
+
+    observations = np.asarray(values, dtype=float)
+    slope = _theil_sen_slope(observations)
+
+    if math.isclose(slope, 0.0):
+        return None
+
+    consecutive_changes = np.diff(observations)
+    aligned_changes = np.count_nonzero(consecutive_changes * slope > 0)
+    directional_consistency = float(aligned_changes / len(consecutive_changes))
+
+    if directional_consistency < minimum_directional_consistency:
+        return None
+
+    indices = np.arange(len(observations), dtype=float)
+    intercept = float(np.median(observations - slope * indices))
+    residuals = np.abs(observations - (intercept + slope * indices))
+    residual_mad = float(np.median(residuals))
+    estimated_change = slope * (len(observations) - 1)
+
+    if math.isclose(residual_mad, 0.0):
+        score = 5.0
+    else:
+        score = abs(0.6745 * estimated_change / residual_mad)
+
+    if score < score_threshold:
+        return None
+
+    change_percent = (
+        estimated_change / intercept * 100 if not math.isclose(intercept, 0.0) else None
+    )
+
+    return TrendResult(
+        subject_id=subject_id,
+        direction=(
+            TrendDirection.INCREASING if slope > 0 else TrendDirection.DECREASING
+        ),
+        slope_per_observation=slope,
+        estimated_change=estimated_change,
+        change_percent=change_percent,
+        score=score,
+        directional_consistency=directional_consistency,
+        observations=len(observations),
+    )

flowsense/infrastructure/__init__.py

@@ -0,0 +1 @@
+"""Infrastructure adapters for external systems."""

flowsense/infrastructure/airflow/__init__.py

@@ -0,0 +1,13 @@
+from flowsense.infrastructure.airflow.client import AirflowClient
+from flowsense.infrastructure.airflow.exceptions import (
+    AirflowApiError,
+    AirflowDagRunNotFoundError,
+    AirflowDataError,
+)
+
+__all__ = [
+    "AirflowApiError",
+    "AirflowClient",
+    "AirflowDagRunNotFoundError",
+    "AirflowDataError",
+]