flowsense-engine 0.2.1__py3-none-any.whl

flowsense/__init__.py

@@ -0,0 +1,77 @@
+from flowsense.application import (
+    ANALYSIS_SCHEMA_VERSION,
+    AnalysisDocument,
+    AnalysisRequest,
+    DAGDataSource,
+    analysis_json_schema,
+    analyze_dag,
+    build_analysis_document,
+    serialize_analysis,
+)
+from flowsense.domain import (
+    DEFAULT_ANALYSIS_POLICY,
+    AnalysisDiagnostic,
+    AnalysisPolicy,
+    ChangeDirection,
+    ChangePointResult,
+    ConfigurationError,
+    DAGAnalysis,
+    DAGAnalysisSummary,
+    DriftResult,
+    FlowSenseError,
+    ImpactClassification,
+    InsufficientHistoryError,
+    InvalidTaskTimingError,
+    MappedTaskAggregation,
+    PropagationResult,
+    RootCauseResult,
+    Severity,
+    TaskImpact,
+    TaskRun,
+    TrendDirection,
+    TrendResult,
+)
+from flowsense.infrastructure.airflow import (
+    AirflowApiError,
+    AirflowClient,
+    AirflowDagRunNotFoundError,
+    AirflowDataError,
+)
+from flowsense.version import __version__
+
+__all__ = [
+    "ANALYSIS_SCHEMA_VERSION",
+    "DEFAULT_ANALYSIS_POLICY",
+    "AirflowApiError",
+    "AirflowClient",
+    "AirflowDagRunNotFoundError",
+    "AirflowDataError",
+    "AnalysisDiagnostic",
+    "AnalysisDocument",
+    "AnalysisPolicy",
+    "AnalysisRequest",
+    "ChangeDirection",
+    "ChangePointResult",
+    "ConfigurationError",
+    "DAGAnalysis",
+    "DAGAnalysisSummary",
+    "DAGDataSource",
+    "DriftResult",
+    "FlowSenseError",
+    "ImpactClassification",
+    "InsufficientHistoryError",
+    "InvalidTaskTimingError",
+    "MappedTaskAggregation",
+    "PropagationResult",
+    "RootCauseResult",
+    "Severity",
+    "TaskImpact",
+    "TaskRun",
+    "TrendDirection",
+    "TrendResult",
+    "__version__",
+    "analysis_json_schema",
+    "analyze_dag",
+    "build_analysis_document",
+    "serialize_analysis",
+]

flowsense/application/__init__.py

@@ -0,0 +1,20 @@
+from flowsense.application.analyzer import analyze_dag
+from flowsense.application.output import ANALYSIS_SCHEMA_VERSION, AnalysisDocument
+from flowsense.application.ports import DAGDataSource
+from flowsense.application.request import AnalysisRequest
+from flowsense.application.serialization import (
+    analysis_json_schema,
+    build_analysis_document,
+    serialize_analysis,
+)
+
+__all__ = [
+    "ANALYSIS_SCHEMA_VERSION",
+    "AnalysisDocument",
+    "AnalysisRequest",
+    "DAGDataSource",
+    "analysis_json_schema",
+    "analyze_dag",
+    "build_analysis_document",
+    "serialize_analysis",
+]

flowsense/application/analyzer.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+from flowsense.application.pipeline import (
+    analyze_handoff_histories,
+    analyze_task_histories,
+    classify_task_impacts,
+    determine_overall_severity,
+)
+from flowsense.application.ports import DAGDataSource
+from flowsense.domain import DEFAULT_ANALYSIS_POLICY, AnalysisPolicy, DAGAnalysis
+from flowsense.engine.history import build_duration_history
+from flowsense.engine.propagation import analyze_propagation
+from flowsense.engine.root_cause import select_primary_origin
+
+
+def analyze_dag(
+    dag_id: str,
+    source: DAGDataSource,
+    policy: AnalysisPolicy = DEFAULT_ANALYSIS_POLICY,
+) -> DAGAnalysis:
+    task_runs = source.collect_task_runs(dag_id)
+    dependencies = source.get_dag_dependencies(dag_id)
+    duration_history = build_duration_history(
+        task_runs,
+        aggregation=policy.mapped_task_aggregation,
+    )
+
+    task_analysis = analyze_task_histories(duration_history, policy)
+    handoff_analysis = analyze_handoff_histories(task_runs, dependencies, policy)
+    task_impacts = classify_task_impacts(
+        task_analysis.drift_results,
+        handoff_analysis.drift_results,
+    )
+    propagation_results = analyze_propagation(
+        drift_results=task_analysis.drift_results,
+        dependencies=dependencies,
+    )
+    primary_origin = select_primary_origin(
+        drift_results=task_analysis.drift_results,
+        task_impacts=task_impacts,
+        dependencies=dependencies,
+        propagation_results=propagation_results,
+    )
+
+    return DAGAnalysis(
+        dag_id=dag_id,
+        runs_analyzed=len({run.dag_run_id for run in task_runs}),
+        current_dag_run_id=(task_runs[-1].dag_run_id if task_runs else None),
+        overall_severity=determine_overall_severity(
+            task_analysis.drift_results,
+            handoff_analysis.drift_results,
+        ),
+        primary_origin=primary_origin,
+        drift_results=task_analysis.drift_results,
+        handoff_drift_results=handoff_analysis.drift_results,
+        task_impacts=task_impacts,
+        propagation_results=propagation_results,
+        dependencies=dependencies,
+        diagnostics=[
+            *task_analysis.diagnostics,
+            *handoff_analysis.diagnostics,
+        ],
+        policy=policy,
+        change_point_results=task_analysis.change_point_results,
+        handoff_change_point_results=handoff_analysis.change_point_results,
+        trend_results=task_analysis.trend_results,
+        handoff_trend_results=handoff_analysis.trend_results,
+    )

flowsense/application/output.py

@@ -0,0 +1,128 @@
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict
+
+from flowsense.domain import (
+    ChangeDirection,
+    ImpactClassification,
+    MappedTaskAggregation,
+    Severity,
+    TrendDirection,
+)
+
+ANALYSIS_SCHEMA_VERSION = "1.1"
+
+
+class OutputModel(BaseModel):
+    """Base model for the versioned public analysis contract."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+
+class AnalysisSummaryOutput(OutputModel):
+    total_tasks: int
+    analyzed_tasks: int
+    analysis_coverage_percent: float
+    normal_tasks: int
+    medium_tasks: int
+    high_tasks: int
+    critical_tasks: int
+    anomalous_tasks: int
+    anomalous_handoffs: int
+    affected_tasks: int
+    change_points: int
+    trends: int
+    diagnostics: int
+
+
+class AnalysisPolicyOutput(OutputModel):
+    minimum_history: int
+    baseline_window: int | None
+    medium_threshold: float
+    high_threshold: float
+    critical_threshold: float
+    mapped_task_aggregation: MappedTaskAggregation
+    change_point_detection_enabled: bool
+    change_point_minimum_segment_size: int
+    change_point_score_threshold: float
+    trend_detection_enabled: bool
+    trend_minimum_observations: int
+    trend_score_threshold: float
+    trend_minimum_directional_consistency: float
+
+
+class RootCauseOutput(OutputModel):
+    task_id: str
+    classification: ImpactClassification
+    severity: Severity
+    propagation_score: float
+
+
+class DriftOutput(OutputModel):
+    baseline: float
+    current: float
+    mad: float
+    robust_z_score: float
+    deviation_percent: float
+    severity: Severity
+
+
+class ChangePointOutput(OutputModel):
+    change_index: int
+    before_median: float
+    after_median: float
+    change_percent: float | None
+    score: float
+    direction: ChangeDirection
+
+
+class TrendOutput(OutputModel):
+    direction: TrendDirection
+    slope_per_observation: float
+    estimated_change: float
+    change_percent: float | None
+    score: float
+    directional_consistency: float
+    observations: int
+
+
+class TaskImpactOutput(OutputModel):
+    classification: ImpactClassification
+    task_severity: Severity
+    upstream_handoff_severity: Severity | None
+
+
+class PropagationOutput(OutputModel):
+    origin_task: str
+    affected_tasks: list[str]
+    path: list[str]
+    propagation_score: float
+
+
+class DiagnosticOutput(OutputModel):
+    code: str
+    subject_id: str
+    message: str
+
+
+class AnalysisDocument(OutputModel):
+    """Typed representation of the FlowSense analysis output schema."""
+
+    schema_version: Literal["1.1"] = ANALYSIS_SCHEMA_VERSION
+    dag_id: str
+    current_dag_run_id: str | None
+    runs_analyzed: int
+    overall_severity: Severity
+    summary: AnalysisSummaryOutput
+    policy: AnalysisPolicyOutput
+    primary_origin: RootCauseOutput | None
+    drift_results: dict[str, DriftOutput]
+    change_point_results: dict[str, ChangePointOutput]
+    trend_results: dict[str, TrendOutput]
+    handoff_drift_results: dict[str, DriftOutput]
+    handoff_change_point_results: dict[str, ChangePointOutput]
+    handoff_trend_results: dict[str, TrendOutput]
+    task_impacts: dict[str, TaskImpactOutput]
+    propagation_results: list[PropagationOutput]
+    dependencies: dict[str, list[str]]
+    diagnostics: list[DiagnosticOutput]

flowsense/application/pipeline.py

@@ -0,0 +1,210 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from flowsense.domain import (
+    AnalysisDiagnostic,
+    AnalysisPolicy,
+    ChangePointResult,
+    DriftResult,
+    InsufficientHistoryError,
+    Severity,
+    TaskImpact,
+    TaskRun,
+    TrendResult,
+)
+from flowsense.domain.enums import SEVERITY_SCORE
+from flowsense.engine.change_point import detect_change_point
+from flowsense.engine.drift import calculate_drift
+from flowsense.engine.impact import classify_task_impact
+from flowsense.engine.timing import (
+    build_handoff_history_with_diagnostics,
+    calculate_handoff_drift,
+)
+from flowsense.engine.trend import detect_trend
+
+TaskId = str
+Edge = tuple[str, str]
+
+
+@dataclass(frozen=True)
+class TaskAnalysisStageResult:
+    drift_results: dict[TaskId, DriftResult]
+    change_point_results: dict[TaskId, ChangePointResult]
+    trend_results: dict[TaskId, TrendResult]
+    diagnostics: list[AnalysisDiagnostic]
+
+
+@dataclass(frozen=True)
+class HandoffAnalysisStageResult:
+    drift_results: dict[Edge, DriftResult]
+    change_point_results: dict[Edge, ChangePointResult]
+    trend_results: dict[Edge, TrendResult]
+    diagnostics: list[AnalysisDiagnostic]
+
+
+def _detect_change_point(
+    subject_id: str,
+    values: list[float],
+    policy: AnalysisPolicy,
+) -> ChangePointResult | None:
+    if not policy.change_point_detection_enabled:
+        return None
+
+    return detect_change_point(
+        subject_id,
+        values,
+        minimum_segment_size=policy.change_point_minimum_segment_size,
+        score_threshold=policy.change_point_score_threshold,
+    )
+
+
+def _detect_trend(
+    subject_id: str,
+    values: list[float],
+    policy: AnalysisPolicy,
+) -> TrendResult | None:
+    if not policy.trend_detection_enabled:
+        return None
+
+    return detect_trend(
+        subject_id,
+        values,
+        minimum_observations=policy.trend_minimum_observations,
+        score_threshold=policy.trend_score_threshold,
+        minimum_directional_consistency=(policy.trend_minimum_directional_consistency),
+    )
+
+
+def analyze_task_histories(
+    duration_history: dict[TaskId, list[float]],
+    policy: AnalysisPolicy,
+) -> TaskAnalysisStageResult:
+    drift_results: dict[TaskId, DriftResult] = {}
+    change_point_results: dict[TaskId, ChangePointResult] = {}
+    trend_results: dict[TaskId, TrendResult] = {}
+    diagnostics: list[AnalysisDiagnostic] = []
+
+    for task_id, durations in duration_history.items():
+        change_point = _detect_change_point(task_id, durations, policy)
+        if change_point is not None:
+            change_point_results[task_id] = change_point
+
+        trend = _detect_trend(task_id, durations, policy)
+        if trend is not None:
+            trend_results[task_id] = trend
+
+        try:
+            drift_results[task_id] = calculate_drift(
+                task_id=task_id,
+                durations=durations,
+                policy=policy,
+            )
+        except InsufficientHistoryError as exc:
+            diagnostics.append(
+                AnalysisDiagnostic(
+                    code="INSUFFICIENT_TASK_HISTORY",
+                    subject_id=task_id,
+                    message=str(exc),
+                )
+            )
+
+    return TaskAnalysisStageResult(
+        drift_results=drift_results,
+        change_point_results=change_point_results,
+        trend_results=trend_results,
+        diagnostics=diagnostics,
+    )
+
+
+def analyze_handoff_histories(
+    task_runs: list[TaskRun],
+    dependencies: dict[TaskId, list[TaskId]],
+    policy: AnalysisPolicy,
+) -> HandoffAnalysisStageResult:
+    history_result = build_handoff_history_with_diagnostics(
+        task_runs=task_runs,
+        dependencies=dependencies,
+    )
+    drift_results: dict[Edge, DriftResult] = {}
+    change_point_results: dict[Edge, ChangePointResult] = {}
+    trend_results: dict[Edge, TrendResult] = {}
+    diagnostics = [
+        AnalysisDiagnostic(
+            code=diagnostic.code,
+            subject_id=f"{diagnostic.upstream_task}->{diagnostic.downstream_task}",
+            message=diagnostic.message,
+        )
+        for diagnostic in history_result.diagnostics
+    ]
+
+    for edge, delays in history_result.history.items():
+        upstream_task, downstream_task = edge
+        subject_id = f"{upstream_task}->{downstream_task}"
+
+        change_point = _detect_change_point(subject_id, delays, policy)
+        if change_point is not None:
+            change_point_results[edge] = change_point
+
+        trend = _detect_trend(subject_id, delays, policy)
+        if trend is not None:
+            trend_results[edge] = trend
+
+        try:
+            drift_results[edge] = calculate_handoff_drift(
+                upstream_task=upstream_task,
+                downstream_task=downstream_task,
+                handoff_delays=delays,
+                policy=policy,
+            )
+        except InsufficientHistoryError as exc:
+            diagnostics.append(
+                AnalysisDiagnostic(
+                    code="INSUFFICIENT_HANDOFF_HISTORY",
+                    subject_id=subject_id,
+                    message=str(exc),
+                )
+            )
+
+    return HandoffAnalysisStageResult(
+        drift_results=drift_results,
+        change_point_results=change_point_results,
+        trend_results=trend_results,
+        diagnostics=diagnostics,
+    )
+
+
+def classify_task_impacts(
+    task_drift_results: dict[TaskId, DriftResult],
+    handoff_drift_results: dict[Edge, DriftResult],
+) -> dict[TaskId, TaskImpact]:
+    return {
+        task_id: classify_task_impact(
+            task_id=task_id,
+            task_drift=task_drift,
+            upstream_handoff_drifts=[
+                drift
+                for (_upstream, downstream), drift in handoff_drift_results.items()
+                if downstream == task_id
+            ],
+        )
+        for task_id, task_drift in task_drift_results.items()
+    }
+
+
+def determine_overall_severity(
+    task_drift_results: dict[TaskId, DriftResult],
+    handoff_drift_results: dict[Edge, DriftResult],
+) -> Severity:
+    drift_results = [
+        *task_drift_results.values(),
+        *handoff_drift_results.values(),
+    ]
+
+    if not drift_results:
+        return Severity.NORMAL
+
+    return max(
+        drift_results,
+        key=lambda result: SEVERITY_SCORE[result.severity],
+    ).severity

flowsense/application/ports.py

@@ -0,0 +1,11 @@
+from typing import Protocol
+
+from flowsense.domain import TaskRun
+
+
+class DAGDataSource(Protocol):
+    def collect_task_runs(self, dag_id: str) -> list[TaskRun]:
+        """Return task runs ordered from oldest DAG run to newest."""
+        ...
+
+    def get_dag_dependencies(self, dag_id: str) -> dict[str, list[str]]: ...

flowsense/application/request.py

@@ -0,0 +1,32 @@
+from dataclasses import dataclass
+
+from flowsense.domain import (
+    DEFAULT_ANALYSIS_POLICY,
+    AnalysisPolicy,
+    ConfigurationError,
+)
+
+
+@dataclass(frozen=True)
+class AnalysisRequest:
+    """Validated input contract shared by FlowSense delivery adapters."""
+
+    dag_id: str
+    policy: AnalysisPolicy = DEFAULT_ANALYSIS_POLICY
+    history_run_limit: int | None = None
+    dag_run_id: str | None = None
+
+    def __post_init__(self) -> None:
+        if not self.dag_id.strip():
+            raise ConfigurationError("dag_id must not be empty.")
+
+        if self.dag_run_id is not None and not self.dag_run_id.strip():
+            raise ConfigurationError("dag_run_id must not be empty.")
+
+        if (
+            self.history_run_limit is not None
+            and self.history_run_limit < self.policy.minimum_history
+        ):
+            raise ConfigurationError(
+                "history_run_limit must be at least minimum_history."
+            )