flowsense-engine 0.2.1__py3-none-any.whl

flowsense/domain/__init__.py

@@ -0,0 +1,52 @@
+from flowsense.domain.enums import (
+    ChangeDirection,
+    ImpactClassification,
+    MappedTaskAggregation,
+    Severity,
+    TrendDirection,
+    severity_meets_threshold,
+)
+from flowsense.domain.exceptions import (
+    ConfigurationError,
+    FlowSenseError,
+    InsufficientHistoryError,
+    InvalidTaskTimingError,
+)
+from flowsense.domain.models import TaskRun
+from flowsense.domain.policy import DEFAULT_ANALYSIS_POLICY, AnalysisPolicy
+from flowsense.domain.results import (
+    AnalysisDiagnostic,
+    ChangePointResult,
+    DAGAnalysis,
+    DAGAnalysisSummary,
+    DriftResult,
+    PropagationResult,
+    RootCauseResult,
+    TaskImpact,
+    TrendResult,
+)
+
+__all__ = [
+    "DEFAULT_ANALYSIS_POLICY",
+    "AnalysisDiagnostic",
+    "AnalysisPolicy",
+    "ChangeDirection",
+    "ChangePointResult",
+    "ConfigurationError",
+    "DAGAnalysis",
+    "DAGAnalysisSummary",
+    "DriftResult",
+    "FlowSenseError",
+    "ImpactClassification",
+    "InsufficientHistoryError",
+    "InvalidTaskTimingError",
+    "MappedTaskAggregation",
+    "PropagationResult",
+    "RootCauseResult",
+    "Severity",
+    "TaskImpact",
+    "TaskRun",
+    "TrendDirection",
+    "TrendResult",
+    "severity_meets_threshold",
+]

flowsense/domain/enums.py

@@ -0,0 +1,47 @@
+from enum import StrEnum
+
+
+class Severity(StrEnum):
+    NORMAL = "NORMAL"
+    MEDIUM = "MEDIUM"
+    HIGH = "HIGH"
+    CRITICAL = "CRITICAL"
+
+
+class ImpactClassification(StrEnum):
+    NORMAL = "NORMAL"
+    OWN_DRIFT = "OWN_DRIFT"
+    INHERITED_DELAY = "INHERITED_DELAY"
+    COMBINED = "COMBINED"
+
+
+class MappedTaskAggregation(StrEnum):
+    MAX = "MAX"
+    MEAN = "MEAN"
+    SUM = "SUM"
+
+
+class ChangeDirection(StrEnum):
+    INCREASE = "INCREASE"
+    DECREASE = "DECREASE"
+
+
+class TrendDirection(StrEnum):
+    INCREASING = "INCREASING"
+    DECREASING = "DECREASING"
+
+
+SEVERITY_SCORE: dict[Severity, int] = {
+    Severity.NORMAL: 0,
+    Severity.MEDIUM: 1,
+    Severity.HIGH: 2,
+    Severity.CRITICAL: 3,
+}
+
+
+def severity_meets_threshold(
+    severity: Severity,
+    threshold: Severity,
+) -> bool:
+    """Return whether a severity is at or above the requested threshold."""
+    return SEVERITY_SCORE[severity] >= SEVERITY_SCORE[threshold]

flowsense/domain/exceptions.py

@@ -0,0 +1,25 @@
+class FlowSenseError(Exception):
+    """Base exception for expected FlowSense failures."""
+
+
+class ConfigurationError(FlowSenseError, ValueError):
+    """Raised when FlowSense configuration is missing or invalid."""
+
+
+class InsufficientHistoryError(FlowSenseError, ValueError):
+    def __init__(
+        self,
+        subject_id: str,
+        required: int,
+        actual: int,
+    ) -> None:
+        self.subject_id = subject_id
+        self.required = required
+        self.actual = actual
+        super().__init__(
+            f"{subject_id} için drift hesaplamak için en az {required} run gerekli."
+        )
+
+
+class InvalidTaskTimingError(FlowSenseError, ValueError):
+    """Raised when task timestamps cannot produce a valid handoff timing."""

flowsense/domain/models.py

@@ -0,0 +1,19 @@
+from datetime import datetime
+
+from pydantic import BaseModel
+
+
+class TaskRun(BaseModel):
+    dag_id: str
+    dag_run_id: str
+    task_id: str
+
+    state: str | None = None
+
+    start_date: datetime | None = None
+    end_date: datetime | None = None
+
+    duration: float | None = None
+    try_number: int = 0
+
+    map_index: int = -1

flowsense/domain/policy.py

@@ -0,0 +1,55 @@
+from dataclasses import dataclass
+
+from flowsense.domain.enums import MappedTaskAggregation
+from flowsense.domain.exceptions import ConfigurationError
+
+
+@dataclass(frozen=True)
+class AnalysisPolicy:
+    minimum_history: int = 5
+    baseline_window: int | None = None
+    medium_threshold: float = 2.0
+    high_threshold: float = 3.5
+    critical_threshold: float = 5.0
+    mapped_task_aggregation: MappedTaskAggregation = MappedTaskAggregation.MAX
+    change_point_detection_enabled: bool = True
+    change_point_minimum_segment_size: int = 3
+    change_point_score_threshold: float = 3.5
+    trend_detection_enabled: bool = True
+    trend_minimum_observations: int = 5
+    trend_score_threshold: float = 3.5
+    trend_minimum_directional_consistency: float = 0.6
+
+    def __post_init__(self) -> None:
+        if self.minimum_history < 2:
+            raise ConfigurationError("minimum_history must be at least 2.")
+        if self.baseline_window is not None:
+            if self.baseline_window < 1:
+                raise ConfigurationError("baseline_window must be at least 1.")
+            if self.baseline_window < self.minimum_history - 1:
+                raise ConfigurationError(
+                    "baseline_window must contain enough values for minimum_history."
+                )
+        if not (
+            0 < self.medium_threshold < self.high_threshold < self.critical_threshold
+        ):
+            raise ConfigurationError(
+                "Severity thresholds must be positive and increasing."
+            )
+        if self.change_point_minimum_segment_size < 2:
+            raise ConfigurationError(
+                "change_point_minimum_segment_size must be at least 2."
+            )
+        if self.change_point_score_threshold <= 0:
+            raise ConfigurationError("change_point_score_threshold must be positive.")
+        if self.trend_minimum_observations < 3:
+            raise ConfigurationError("trend_minimum_observations must be at least 3.")
+        if self.trend_score_threshold <= 0:
+            raise ConfigurationError("trend_score_threshold must be positive.")
+        if not 0 < self.trend_minimum_directional_consistency <= 1:
+            raise ConfigurationError(
+                "trend_minimum_directional_consistency must be in (0, 1]."
+            )
+
+
+DEFAULT_ANALYSIS_POLICY = AnalysisPolicy()

flowsense/domain/results.py

@@ -0,0 +1,187 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from flowsense.domain.enums import (
+    ChangeDirection,
+    ImpactClassification,
+    Severity,
+    TrendDirection,
+)
+from flowsense.domain.policy import DEFAULT_ANALYSIS_POLICY, AnalysisPolicy
+
+
+@dataclass(frozen=True)
+class DriftResult:
+    task_id: str
+    baseline: float
+    current: float
+    mad: float
+    robust_z_score: float
+    deviation_percent: float
+    severity: Severity
+
+
+@dataclass(frozen=True)
+class ChangePointResult:
+    subject_id: str
+    change_index: int
+    before_median: float
+    after_median: float
+    change_percent: float | None
+    score: float
+    direction: ChangeDirection
+
+
+@dataclass(frozen=True)
+class TrendResult:
+    subject_id: str
+    direction: TrendDirection
+    slope_per_observation: float
+    estimated_change: float
+    change_percent: float | None
+    score: float
+    directional_consistency: float
+    observations: int
+
+
+@dataclass(frozen=True)
+class TaskImpact:
+    task_id: str
+    classification: ImpactClassification
+    task_severity: Severity
+    upstream_handoff_severity: Severity | None
+
+
+@dataclass(frozen=True)
+class PropagationResult:
+    origin_task: str
+    affected_tasks: list[str]
+    path: list[str]
+    propagation_score: float
+
+
+@dataclass(frozen=True)
+class RootCauseResult:
+    task_id: str
+    classification: ImpactClassification
+    severity: Severity
+    propagation_score: float
+
+
+@dataclass(frozen=True)
+class AnalysisDiagnostic:
+    code: str
+    subject_id: str
+    message: str
+
+
+@dataclass(frozen=True)
+class DAGAnalysisSummary:
+    total_tasks: int
+    analyzed_tasks: int
+    analysis_coverage_percent: float
+    normal_tasks: int
+    medium_tasks: int
+    high_tasks: int
+    critical_tasks: int
+    anomalous_tasks: int
+    anomalous_handoffs: int
+    affected_tasks: int
+    change_points: int
+    trends: int
+    diagnostics: int
+
+
+@dataclass
+class DAGAnalysis:
+    dag_id: str
+    runs_analyzed: int
+    overall_severity: Severity
+    primary_origin: RootCauseResult | None
+    drift_results: dict[str, DriftResult]
+    handoff_drift_results: dict[tuple[str, str], DriftResult]
+    task_impacts: dict[str, TaskImpact]
+    propagation_results: list[PropagationResult]
+    dependencies: dict[str, list[str]]
+    diagnostics: list[AnalysisDiagnostic] = field(default_factory=list)
+    policy: AnalysisPolicy = DEFAULT_ANALYSIS_POLICY
+    change_point_results: dict[str, ChangePointResult] = field(default_factory=dict)
+    handoff_change_point_results: dict[tuple[str, str], ChangePointResult] = field(
+        default_factory=dict
+    )
+    trend_results: dict[str, TrendResult] = field(default_factory=dict)
+    handoff_trend_results: dict[tuple[str, str], TrendResult] = field(
+        default_factory=dict
+    )
+    current_dag_run_id: str | None = None
+
+    @property
+    def summary(self) -> DAGAnalysisSummary:
+        task_ids = {
+            *self.dependencies,
+            *(task for tasks in self.dependencies.values() for task in tasks),
+            *self.drift_results,
+            *self.task_impacts,
+            *self.change_point_results,
+            *self.trend_results,
+            *(
+                task
+                for edge in (
+                    *self.handoff_drift_results,
+                    *self.handoff_change_point_results,
+                    *self.handoff_trend_results,
+                )
+                for task in edge
+            ),
+            *(
+                task
+                for result in self.propagation_results
+                for task in (result.origin_task, *result.affected_tasks)
+            ),
+        }
+        severity_counts = {
+            severity: sum(
+                result.severity == severity for result in self.drift_results.values()
+            )
+            for severity in Severity
+        }
+        analyzed_tasks = len(self.drift_results)
+        total_tasks = len(task_ids)
+        anomalous_severities = {
+            Severity.MEDIUM,
+            Severity.HIGH,
+            Severity.CRITICAL,
+        }
+
+        return DAGAnalysisSummary(
+            total_tasks=total_tasks,
+            analyzed_tasks=analyzed_tasks,
+            analysis_coverage_percent=(
+                analyzed_tasks / total_tasks * 100 if total_tasks else 0.0
+            ),
+            normal_tasks=severity_counts[Severity.NORMAL],
+            medium_tasks=severity_counts[Severity.MEDIUM],
+            high_tasks=severity_counts[Severity.HIGH],
+            critical_tasks=severity_counts[Severity.CRITICAL],
+            anomalous_tasks=sum(
+                result.severity in anomalous_severities
+                for result in self.drift_results.values()
+            ),
+            anomalous_handoffs=sum(
+                result.severity in anomalous_severities
+                for result in self.handoff_drift_results.values()
+            ),
+            affected_tasks=len(
+                {
+                    task
+                    for result in self.propagation_results
+                    for task in result.affected_tasks
+                }
+            ),
+            change_points=(
+                len(self.change_point_results) + len(self.handoff_change_point_results)
+            ),
+            trends=len(self.trend_results) + len(self.handoff_trend_results),
+            diagnostics=len(self.diagnostics),
+        )

flowsense/engine/analyzer.py

@@ -0,0 +1,17 @@
+"""Backward-compatible analyzer entry point.
+
+New integrations should inject a data source into
+``flowsense.application.analyze_dag``.
+"""
+
+from flowsense.application import analyze_dag as analyze_dag_with_source
+from flowsense.domain import DAGAnalysis
+from flowsense.infrastructure.airflow import AirflowClient
+
+
+def analyze_dag(dag_id: str) -> DAGAnalysis:
+    with AirflowClient() as source:
+        return analyze_dag_with_source(
+            dag_id=dag_id,
+            source=source,
+        )

flowsense/engine/change_point.py

@@ -0,0 +1,87 @@
+from __future__ import annotations
+
+import math
+
+import numpy as np
+
+from flowsense.domain import ChangeDirection, ChangePointResult
+
+
+def _candidate_score(
+    before: np.ndarray, after: np.ndarray
+) -> tuple[float, float, float]:
+    before_median = float(np.median(before))
+    after_median = float(np.median(after))
+    difference = after_median - before_median
+
+    residuals = np.concatenate(
+        (
+            np.abs(before - before_median),
+            np.abs(after - after_median),
+        )
+    )
+    noise = float(np.median(residuals))
+
+    if math.isclose(noise, 0.0):
+        score = 0.0 if math.isclose(difference, 0.0) else 5.0
+    else:
+        score = abs(0.6745 * difference / noise)
+
+    return score, before_median, after_median
+
+
+def detect_change_point(
+    subject_id: str,
+    values: list[float],
+    *,
+    minimum_segment_size: int = 3,
+    score_threshold: float = 3.5,
+) -> ChangePointResult | None:
+    """Detect the strongest persistent level shift in an ordered time series."""
+    if minimum_segment_size < 2:
+        raise ValueError("minimum_segment_size must be at least 2")
+
+    if score_threshold <= 0:
+        raise ValueError("score_threshold must be positive")
+
+    if len(values) < minimum_segment_size * 2:
+        return None
+
+    observations = np.asarray(values, dtype=float)
+    best_candidate: tuple[float, int, float, float] | None = None
+
+    for change_index in range(
+        minimum_segment_size,
+        len(observations) - minimum_segment_size + 1,
+    ):
+        score, before_median, after_median = _candidate_score(
+            observations[:change_index],
+            observations[change_index:],
+        )
+
+        if best_candidate is None or score > best_candidate[0]:
+            best_candidate = (score, change_index, before_median, after_median)
+
+    if best_candidate is None or best_candidate[0] < score_threshold:
+        return None
+
+    score, change_index, before_median, after_median = best_candidate
+    change_percent = (
+        (after_median - before_median) / before_median * 100
+        if not math.isclose(before_median, 0.0)
+        else None
+    )
+
+    return ChangePointResult(
+        subject_id=subject_id,
+        change_index=change_index,
+        before_median=before_median,
+        after_median=after_median,
+        change_percent=change_percent,
+        score=score,
+        direction=(
+            ChangeDirection.INCREASE
+            if after_median > before_median
+            else ChangeDirection.DECREASE
+        ),
+    )

flowsense/engine/drift.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+import math
+
+import numpy as np
+
+from flowsense.domain import (
+    DEFAULT_ANALYSIS_POLICY,
+    AnalysisPolicy,
+    DriftResult,
+    InsufficientHistoryError,
+    Severity,
+)
+
+
+def calculate_drift(
+    task_id: str,
+    durations: list[float],
+    policy: AnalysisPolicy = DEFAULT_ANALYSIS_POLICY,
+) -> DriftResult:
+    if len(durations) < policy.minimum_history:
+        raise InsufficientHistoryError(
+            subject_id=task_id,
+            required=policy.minimum_history,
+            actual=len(durations),
+        )
+
+    baseline_durations = durations[:-1]
+    if policy.baseline_window is not None:
+        baseline_durations = baseline_durations[-policy.baseline_window :]
+
+    baseline_values = np.array(
+        baseline_durations,
+        dtype=float,
+    )
+
+    current = float(durations[-1])
+
+    median = float(np.median(baseline_values))
+
+    absolute_deviations = np.abs(baseline_values - median)
+
+    mad = float(np.median(absolute_deviations))
+
+    if mad == 0:
+        if math.isclose(current, median):
+            robust_z_score = 0.0
+        else:
+            robust_z_score = math.copysign(
+                policy.critical_threshold,
+                current - median,
+            )
+    else:
+        robust_z_score = 0.6745 * (current - median) / mad
+
+    if median == 0:
+        deviation_percent = 0.0
+    else:
+        deviation_percent = (current - median) / median * 100
+
+    absolute_z = abs(robust_z_score)
+
+    if absolute_z >= policy.critical_threshold:
+        severity = Severity.CRITICAL
+    elif absolute_z >= policy.high_threshold:
+        severity = Severity.HIGH
+    elif absolute_z >= policy.medium_threshold:
+        severity = Severity.MEDIUM
+    else:
+        severity = Severity.NORMAL
+
+    return DriftResult(
+        task_id=task_id,
+        baseline=median,
+        current=current,
+        mad=mad,
+        robust_z_score=robust_z_score,
+        deviation_percent=deviation_percent,
+        severity=severity,
+    )

flowsense/engine/history.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from collections import defaultdict
+
+from flowsense.domain import MappedTaskAggregation, TaskRun
+
+
+def build_duration_history(
+    task_runs: list[TaskRun],
+    aggregation: MappedTaskAggregation = MappedTaskAggregation.MAX,
+) -> dict[str, list[float]]:
+    """Build logical-task history using the slowest mapped instance per DAG run."""
+    grouped_durations: dict[tuple[str, str], list[float]] = defaultdict(list)
+
+    for task_run in task_runs:
+        if task_run.state != "success" or task_run.duration is None:
+            continue
+
+        key = (task_run.dag_run_id, task_run.task_id)
+        grouped_durations[key].append(task_run.duration)
+
+    def aggregate(values: list[float]) -> float:
+        if aggregation is MappedTaskAggregation.SUM:
+            return sum(values)
+        if aggregation is MappedTaskAggregation.MEAN:
+            return sum(values) / len(values)
+        return max(values)
+
+    history: dict[str, list[float]] = defaultdict(list)
+
+    for (_dag_run_id, task_id), durations in grouped_durations.items():
+        history[task_id].append(aggregate(durations))
+
+    return dict(history)

flowsense/engine/impact.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+from flowsense.domain import (
+    DriftResult,
+    ImpactClassification,
+    Severity,
+    TaskImpact,
+)
+from flowsense.domain.enums import SEVERITY_SCORE
+
+
+def classify_task_impact(
+    task_id: str,
+    task_drift: DriftResult,
+    upstream_handoff_drifts: list[DriftResult],
+) -> TaskImpact:
+    anomalous_handoffs = [
+        drift
+        for drift in upstream_handoff_drifts
+        if drift.severity in {Severity.MEDIUM, Severity.HIGH, Severity.CRITICAL}
+    ]
+
+    task_is_anomalous = task_drift.severity in {
+        Severity.MEDIUM,
+        Severity.HIGH,
+        Severity.CRITICAL,
+    }
+
+    if task_is_anomalous and anomalous_handoffs:
+        classification = ImpactClassification.COMBINED
+    elif task_is_anomalous:
+        classification = ImpactClassification.OWN_DRIFT
+    elif anomalous_handoffs:
+        classification = ImpactClassification.INHERITED_DELAY
+    else:
+        classification = ImpactClassification.NORMAL
+
+    upstream_handoff_severity = None
+
+    if anomalous_handoffs:
+        upstream_handoff_severity = max(
+            anomalous_handoffs,
+            key=lambda result: SEVERITY_SCORE[result.severity],
+        ).severity
+
+    return TaskImpact(
+        task_id=task_id,
+        classification=classification,
+        task_severity=task_drift.severity,
+        upstream_handoff_severity=upstream_handoff_severity,
+    )