flowsense-engine 0.2.1__py3-none-any.whl

flowsense/application/serialization.py

@@ -0,0 +1,177 @@
+from __future__ import annotations
+
+from flowsense.application.output import (
+    AnalysisDocument,
+    AnalysisPolicyOutput,
+    AnalysisSummaryOutput,
+    ChangePointOutput,
+    DiagnosticOutput,
+    DriftOutput,
+    PropagationOutput,
+    RootCauseOutput,
+    TaskImpactOutput,
+    TrendOutput,
+)
+from flowsense.domain import DAGAnalysis, DriftResult
+
+
+def _drift_output(result: DriftResult) -> DriftOutput:
+    return DriftOutput(
+        baseline=result.baseline,
+        current=result.current,
+        mad=result.mad,
+        robust_z_score=result.robust_z_score,
+        deviation_percent=result.deviation_percent,
+        severity=result.severity,
+    )
+
+
+def build_analysis_document(analysis: DAGAnalysis) -> AnalysisDocument:
+    """Build and validate the typed public output document."""
+    summary = analysis.summary
+
+    return AnalysisDocument(
+        dag_id=analysis.dag_id,
+        current_dag_run_id=analysis.current_dag_run_id,
+        runs_analyzed=analysis.runs_analyzed,
+        overall_severity=analysis.overall_severity,
+        summary=AnalysisSummaryOutput(
+            total_tasks=summary.total_tasks,
+            analyzed_tasks=summary.analyzed_tasks,
+            analysis_coverage_percent=summary.analysis_coverage_percent,
+            normal_tasks=summary.normal_tasks,
+            medium_tasks=summary.medium_tasks,
+            high_tasks=summary.high_tasks,
+            critical_tasks=summary.critical_tasks,
+            anomalous_tasks=summary.anomalous_tasks,
+            anomalous_handoffs=summary.anomalous_handoffs,
+            affected_tasks=summary.affected_tasks,
+            change_points=summary.change_points,
+            trends=summary.trends,
+            diagnostics=summary.diagnostics,
+        ),
+        policy=AnalysisPolicyOutput(
+            minimum_history=analysis.policy.minimum_history,
+            baseline_window=analysis.policy.baseline_window,
+            medium_threshold=analysis.policy.medium_threshold,
+            high_threshold=analysis.policy.high_threshold,
+            critical_threshold=analysis.policy.critical_threshold,
+            mapped_task_aggregation=analysis.policy.mapped_task_aggregation,
+            change_point_detection_enabled=(
+                analysis.policy.change_point_detection_enabled
+            ),
+            change_point_minimum_segment_size=(
+                analysis.policy.change_point_minimum_segment_size
+            ),
+            change_point_score_threshold=(analysis.policy.change_point_score_threshold),
+            trend_detection_enabled=analysis.policy.trend_detection_enabled,
+            trend_minimum_observations=analysis.policy.trend_minimum_observations,
+            trend_score_threshold=analysis.policy.trend_score_threshold,
+            trend_minimum_directional_consistency=(
+                analysis.policy.trend_minimum_directional_consistency
+            ),
+        ),
+        primary_origin=(
+            RootCauseOutput(
+                task_id=analysis.primary_origin.task_id,
+                classification=analysis.primary_origin.classification,
+                severity=analysis.primary_origin.severity,
+                propagation_score=analysis.primary_origin.propagation_score,
+            )
+            if analysis.primary_origin
+            else None
+        ),
+        drift_results={
+            task_id: _drift_output(result)
+            for task_id, result in analysis.drift_results.items()
+        },
+        change_point_results={
+            task_id: ChangePointOutput(
+                change_index=result.change_index,
+                before_median=result.before_median,
+                after_median=result.after_median,
+                change_percent=result.change_percent,
+                score=result.score,
+                direction=result.direction,
+            )
+            for task_id, result in analysis.change_point_results.items()
+        },
+        trend_results={
+            task_id: TrendOutput(
+                direction=result.direction,
+                slope_per_observation=result.slope_per_observation,
+                estimated_change=result.estimated_change,
+                change_percent=result.change_percent,
+                score=result.score,
+                directional_consistency=result.directional_consistency,
+                observations=result.observations,
+            )
+            for task_id, result in analysis.trend_results.items()
+        },
+        handoff_drift_results={
+            f"{upstream}->{downstream}": _drift_output(result)
+            for (upstream, downstream), result in analysis.handoff_drift_results.items()
+        },
+        handoff_change_point_results={
+            f"{upstream}->{downstream}": ChangePointOutput(
+                change_index=result.change_index,
+                before_median=result.before_median,
+                after_median=result.after_median,
+                change_percent=result.change_percent,
+                score=result.score,
+                direction=result.direction,
+            )
+            for (
+                upstream,
+                downstream,
+            ), result in analysis.handoff_change_point_results.items()
+        },
+        handoff_trend_results={
+            f"{upstream}->{downstream}": TrendOutput(
+                direction=result.direction,
+                slope_per_observation=result.slope_per_observation,
+                estimated_change=result.estimated_change,
+                change_percent=result.change_percent,
+                score=result.score,
+                directional_consistency=result.directional_consistency,
+                observations=result.observations,
+            )
+            for (upstream, downstream), result in analysis.handoff_trend_results.items()
+        },
+        task_impacts={
+            task_id: TaskImpactOutput(
+                classification=impact.classification,
+                task_severity=impact.task_severity,
+                upstream_handoff_severity=impact.upstream_handoff_severity,
+            )
+            for task_id, impact in analysis.task_impacts.items()
+        },
+        propagation_results=[
+            PropagationOutput(
+                origin_task=result.origin_task,
+                affected_tasks=result.affected_tasks,
+                path=result.path,
+                propagation_score=result.propagation_score,
+            )
+            for result in analysis.propagation_results
+        ],
+        dependencies=analysis.dependencies,
+        diagnostics=[
+            DiagnosticOutput(
+                code=diagnostic.code,
+                subject_id=diagnostic.subject_id,
+                message=diagnostic.message,
+            )
+            for diagnostic in analysis.diagnostics
+        ],
+    )
+
+
+def serialize_analysis(analysis: DAGAnalysis) -> dict[str, object]:
+    """Serialize a DAG analysis to the versioned public output schema."""
+    return build_analysis_document(analysis).model_dump(mode="json")
+
+
+def analysis_json_schema() -> dict[str, object]:
+    """Return the JSON Schema for the current analysis output contract."""
+    return AnalysisDocument.model_json_schema()

flowsense/cli/main.py

@@ -0,0 +1,195 @@
+from __future__ import annotations
+
+import json
+from enum import StrEnum
+from typing import Annotated
+
+import typer
+from rich.console import Console
+
+from flowsense.application import (
+    AnalysisRequest,
+    analysis_json_schema,
+    analyze_dag,
+    serialize_analysis,
+)
+from flowsense.cli.report import render_analysis
+from flowsense.domain import (
+    AnalysisPolicy,
+    FlowSenseError,
+    MappedTaskAggregation,
+    Severity,
+    severity_meets_threshold,
+)
+from flowsense.infrastructure.airflow import AirflowApiError, AirflowClient
+from flowsense.version import __version__
+
+app = typer.Typer(
+    name="flowsense",
+    help="Temporal drift and anomaly detection for Apache Airflow.",
+    no_args_is_help=True,
+)
+
+console = Console()
+ANALYSIS_THRESHOLD_EXIT_CODE = 2
+
+
+def _version_callback(value: bool) -> None:
+    if value:
+        typer.echo(__version__)
+        raise typer.Exit()
+
+
+class OutputFormat(StrEnum):
+    TABLE = "table"
+    JSON = "json"
+
+
+class FailureThreshold(StrEnum):
+    MEDIUM = "medium"
+    HIGH = "high"
+    CRITICAL = "critical"
+
+
+@app.callback()
+def main(
+    version: Annotated[
+        bool | None,
+        typer.Option(
+            "--version",
+            callback=_version_callback,
+            is_eager=True,
+            help="Show the installed FlowSense version and exit.",
+        ),
+    ] = None,
+) -> None:
+    """FlowSense CLI."""
+
+
+@app.command("schema")
+def show_schema() -> None:
+    """Print the versioned analysis output JSON Schema."""
+    typer.echo(
+        json.dumps(
+            analysis_json_schema(),
+            indent=2,
+            ensure_ascii=False,
+            sort_keys=True,
+        )
+    )
+
+
+@app.command()
+def analyze(
+    dag_id: str = typer.Argument(
+        ...,
+        help="Airflow DAG id to analyze.",
+    ),
+    minimum_history: int = typer.Option(5, min=2),
+    baseline_window: int | None = typer.Option(None, min=1),
+    medium_threshold: float = typer.Option(2.0, min=0.0),
+    high_threshold: float = typer.Option(3.5, min=0.0),
+    critical_threshold: float = typer.Option(5.0, min=0.0),
+    change_point_detection: bool = typer.Option(
+        True,
+        "--change-point-detection/--no-change-point-detection",
+    ),
+    change_point_minimum_segment_size: int = typer.Option(3, min=2),
+    change_point_score_threshold: float = typer.Option(3.5, min=0.0),
+    trend_detection: bool = typer.Option(
+        True,
+        "--trend-detection/--no-trend-detection",
+    ),
+    trend_minimum_observations: int = typer.Option(5, min=3),
+    trend_score_threshold: float = typer.Option(3.5, min=0.0),
+    trend_minimum_directional_consistency: float = typer.Option(
+        0.6,
+        min=0.0,
+        max=1.0,
+    ),
+    mapped_task_aggregation: Annotated[
+        MappedTaskAggregation,
+        typer.Option(),
+    ] = MappedTaskAggregation.MAX,
+    output: Annotated[
+        OutputFormat,
+        typer.Option("--output", "-o"),
+    ] = OutputFormat.TABLE,
+    fail_on: Annotated[
+        FailureThreshold | None,
+        typer.Option(
+            "--fail-on",
+            help="Exit with code 2 when severity reaches this threshold.",
+        ),
+    ] = None,
+    history_run_limit: int | None = typer.Option(
+        None,
+        min=2,
+        help="Limit collection to the most recent successful DAG runs.",
+    ),
+    dag_run_id: str | None = typer.Option(
+        None,
+        "--dag-run-id",
+        help="Analyze this successful DAG run using only its preceding history.",
+    ),
+) -> None:
+    try:
+        policy = AnalysisPolicy(
+            minimum_history=minimum_history,
+            baseline_window=baseline_window,
+            medium_threshold=medium_threshold,
+            high_threshold=high_threshold,
+            critical_threshold=critical_threshold,
+            mapped_task_aggregation=mapped_task_aggregation,
+            change_point_detection_enabled=change_point_detection,
+            change_point_minimum_segment_size=change_point_minimum_segment_size,
+            change_point_score_threshold=change_point_score_threshold,
+            trend_detection_enabled=trend_detection,
+            trend_minimum_observations=trend_minimum_observations,
+            trend_score_threshold=trend_score_threshold,
+            trend_minimum_directional_consistency=(
+                trend_minimum_directional_consistency
+            ),
+        )
+    except ValueError as exc:
+        raise typer.BadParameter(str(exc)) from exc
+
+    try:
+        request = AnalysisRequest(
+            dag_id=dag_id,
+            policy=policy,
+            history_run_limit=history_run_limit,
+            dag_run_id=dag_run_id,
+        )
+        with AirflowClient(
+            history_run_limit=history_run_limit,
+            target_dag_run_id=request.dag_run_id,
+        ) as source:
+            analysis = analyze_dag(
+                dag_id=request.dag_id,
+                source=source,
+                policy=request.policy,
+            )
+    except AirflowApiError as exc:
+        console.print(f"[bold red]Airflow request failed:[/bold red] {exc}")
+        raise typer.Exit(code=1) from exc
+    except FlowSenseError as exc:
+        console.print(f"[bold red]Analysis failed:[/bold red] {exc}")
+        raise typer.Exit(code=1) from exc
+
+    if output is OutputFormat.JSON:
+        typer.echo(
+            json.dumps(
+                serialize_analysis(analysis),
+                indent=2,
+                ensure_ascii=False,
+            )
+        )
+    else:
+        render_analysis(console, analysis)
+
+    if fail_on is not None and severity_meets_threshold(
+        analysis.overall_severity,
+        Severity(fail_on.value.upper()),
+    ):
+        raise typer.Exit(code=ANALYSIS_THRESHOLD_EXIT_CODE)

flowsense/cli/report.py

@@ -0,0 +1,233 @@
+from __future__ import annotations
+
+from rich.console import Console
+from rich.panel import Panel
+from rich.table import Table
+from rich.text import Text
+
+from flowsense.domain import DAGAnalysis, Severity
+from flowsense.domain.enums import SEVERITY_SCORE
+
+_SEVERITY_STYLES = {
+    Severity.NORMAL: "green",
+    Severity.MEDIUM: "yellow",
+    Severity.HIGH: "bright_red",
+    Severity.CRITICAL: "bold red",
+}
+
+
+def _severity_text(severity: Severity) -> Text:
+    return Text(str(severity), style=_SEVERITY_STYLES[severity])
+
+
+def _percent(value: float | None) -> str:
+    return f"{value:+.1f}%" if value is not None else "n/a"
+
+
+def _render_summary(console: Console, analysis: DAGAnalysis) -> None:
+    dag_summary = analysis.summary
+    summary = Table.grid(padding=(0, 2))
+    summary.add_column(style="bold")
+    summary.add_column()
+    summary.add_row("DAG", analysis.dag_id)
+    if analysis.current_dag_run_id is not None:
+        summary.add_row("Current DAG run", analysis.current_dag_run_id)
+    summary.add_row("Runs analyzed", str(analysis.runs_analyzed))
+    summary.add_row("Overall severity", _severity_text(analysis.overall_severity))
+    summary.add_row(
+        "Task coverage",
+        f"{dag_summary.analyzed_tasks}/{dag_summary.total_tasks} "
+        f"({dag_summary.analysis_coverage_percent:.1f}%)",
+    )
+    summary.add_row("Anomalous tasks", str(dag_summary.anomalous_tasks))
+    summary.add_row("Anomalous handoffs", str(dag_summary.anomalous_handoffs))
+    summary.add_row("Affected tasks", str(dag_summary.affected_tasks))
+    summary.add_row(
+        "Structural signals",
+        f"{dag_summary.change_points} change points, {dag_summary.trends} trends",
+    )
+
+    if analysis.primary_origin is not None:
+        summary.add_row("Primary origin", analysis.primary_origin.task_id)
+        summary.add_row("Classification", str(analysis.primary_origin.classification))
+        summary.add_row(
+            "Propagation score",
+            f"{analysis.primary_origin.propagation_score:.2f}",
+        )
+
+    console.print(Panel(summary, title="FlowSense Analysis", expand=False))
+
+
+def _render_task_drift(console: Console, analysis: DAGAnalysis) -> None:
+    table = Table(title="Task Drift")
+    table.add_column("Task")
+    table.add_column("Baseline", justify="right")
+    table.add_column("Current", justify="right")
+    table.add_column("Deviation", justify="right")
+    table.add_column("Z-Score", justify="right")
+    table.add_column("Severity")
+    table.add_column("Impact")
+
+    ordered_results = sorted(
+        analysis.drift_results.items(),
+        key=lambda item: (-SEVERITY_SCORE[item[1].severity], item[0]),
+    )
+
+    for task_id, result in ordered_results:
+        impact = analysis.task_impacts.get(task_id)
+        table.add_row(
+            task_id,
+            f"{result.baseline:.2f}s",
+            f"{result.current:.2f}s",
+            f"{result.deviation_percent:+.1f}%",
+            f"{result.robust_z_score:.2f}",
+            _severity_text(result.severity),
+            str(impact.classification) if impact else "-",
+        )
+
+    console.print(table)
+
+
+def _render_handoff_drift(console: Console, analysis: DAGAnalysis) -> None:
+    if not analysis.handoff_drift_results:
+        return
+
+    table = Table(title="Handoff Drift")
+    table.add_column("Edge")
+    table.add_column("Baseline", justify="right")
+    table.add_column("Current", justify="right")
+    table.add_column("Deviation", justify="right")
+    table.add_column("Z-Score", justify="right")
+    table.add_column("Severity")
+
+    ordered_results = sorted(
+        analysis.handoff_drift_results.items(),
+        key=lambda item: (-SEVERITY_SCORE[item[1].severity], item[0]),
+    )
+
+    for (upstream, downstream), result in ordered_results:
+        table.add_row(
+            f"{upstream} -> {downstream}",
+            f"{result.baseline:.2f}s",
+            f"{result.current:.2f}s",
+            f"{result.deviation_percent:+.1f}%",
+            f"{result.robust_z_score:.2f}",
+            _severity_text(result.severity),
+        )
+
+    console.print(table)
+
+
+def _render_change_points(console: Console, analysis: DAGAnalysis) -> None:
+    results = [
+        *analysis.change_point_results.values(),
+        *analysis.handoff_change_point_results.values(),
+    ]
+    if not results:
+        return
+
+    table = Table(title="Change Points")
+    table.add_column("Subject")
+    table.add_column("Direction")
+    table.add_column("Observation", justify="right")
+    table.add_column("Before", justify="right")
+    table.add_column("After", justify="right")
+    table.add_column("Change", justify="right")
+    table.add_column("Score", justify="right")
+
+    for result in sorted(results, key=lambda item: item.subject_id):
+        table.add_row(
+            result.subject_id,
+            str(result.direction),
+            str(result.change_index + 1),
+            f"{result.before_median:.2f}s",
+            f"{result.after_median:.2f}s",
+            _percent(result.change_percent),
+            f"{result.score:.2f}",
+        )
+
+    console.print(table)
+
+
+def _render_trends(console: Console, analysis: DAGAnalysis) -> None:
+    results = [
+        *analysis.trend_results.values(),
+        *analysis.handoff_trend_results.values(),
+    ]
+    if not results:
+        return
+
+    table = Table(title="Trends")
+    table.add_column("Subject")
+    table.add_column("Direction")
+    table.add_column("Slope / run", justify="right")
+    table.add_column("Est. change", justify="right")
+    table.add_column("Change", justify="right")
+    table.add_column("Consistency", justify="right")
+    table.add_column("Score", justify="right")
+
+    for result in sorted(results, key=lambda item: item.subject_id):
+        table.add_row(
+            result.subject_id,
+            str(result.direction),
+            f"{result.slope_per_observation:+.2f}s",
+            f"{result.estimated_change:+.2f}s",
+            _percent(result.change_percent),
+            f"{result.directional_consistency:.0%}",
+            f"{result.score:.2f}",
+        )
+
+    console.print(table)
+
+
+def _render_propagation(console: Console, analysis: DAGAnalysis) -> None:
+    if not analysis.propagation_results:
+        return
+
+    table = Table(title="Propagation")
+    table.add_column("Origin")
+    table.add_column("Path")
+    table.add_column("Affected", justify="right")
+    table.add_column("Score", justify="right")
+
+    for result in sorted(
+        analysis.propagation_results,
+        key=lambda item: (item.origin_task, item.path),
+    ):
+        table.add_row(
+            result.origin_task,
+            " -> ".join(result.path),
+            str(len(result.affected_tasks)),
+            f"{result.propagation_score:.2f}",
+        )
+
+    console.print(table)
+
+
+def _render_diagnostics(console: Console, analysis: DAGAnalysis) -> None:
+    if not analysis.diagnostics:
+        return
+
+    table = Table(title="Diagnostics", title_style="bold yellow")
+    table.add_column("Code", style="yellow")
+    table.add_column("Subject")
+    table.add_column("Message")
+
+    for diagnostic in sorted(
+        analysis.diagnostics,
+        key=lambda item: (item.code, item.subject_id),
+    ):
+        table.add_row(diagnostic.code, diagnostic.subject_id, diagnostic.message)
+
+    console.print(table)
+
+
+def render_analysis(console: Console, analysis: DAGAnalysis) -> None:
+    """Render a complete human-readable analysis report."""
+    _render_summary(console, analysis)
+    _render_task_drift(console, analysis)
+    _render_handoff_drift(console, analysis)
+    _render_change_points(console, analysis)
+    _render_trends(console, analysis)
+    _render_propagation(console, analysis)
+    _render_diagnostics(console, analysis)

flowsense/collector/airflow_client.py

@@ -0,0 +1,5 @@
+"""Backward-compatible imports for the Airflow infrastructure adapter."""
+
+from flowsense.infrastructure.airflow.client import PAGE_SIZE, AirflowClient
+
+__all__ = ["PAGE_SIZE", "AirflowClient"]

flowsense/config.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+
+from flowsense.domain import ConfigurationError
+
+
+@dataclass(frozen=True)
+class AirflowConfig:
+    base_url: str
+    username: str
+    password: str
+    api_version: str = "v2"
+    auth_mode: str = "token"
+    connect_timeout: float = 10.0
+    read_timeout: float = 10.0
+    max_retries: int = 2
+    retry_backoff: float = 0.5
+    history_run_limit: int = 100
+
+
+def get_airflow_config() -> AirflowConfig:
+    base_url = os.getenv(
+        "AIRFLOW_BASE_URL",
+        "http://localhost:8080",
+    )
+
+    username = os.getenv("AIRFLOW_USERNAME")
+    password = os.getenv("AIRFLOW_PASSWORD")
+    api_version = os.getenv("AIRFLOW_API_VERSION", "v2")
+    auth_mode = os.getenv("AIRFLOW_AUTH_MODE", "token")
+    connect_timeout = _read_float("AIRFLOW_CONNECT_TIMEOUT", "10")
+    read_timeout = _read_float("AIRFLOW_READ_TIMEOUT", "10")
+    max_retries = _read_int("AIRFLOW_MAX_RETRIES", "2")
+    retry_backoff = _read_float("AIRFLOW_RETRY_BACKOFF", "0.5")
+    history_run_limit = _read_int("AIRFLOW_HISTORY_RUN_LIMIT", "100")
+
+    if not username:
+        raise ConfigurationError("AIRFLOW_USERNAME environment variable is required.")
+
+    if not password:
+        raise ConfigurationError("AIRFLOW_PASSWORD environment variable is required.")
+
+    return AirflowConfig(
+        base_url=base_url,
+        username=username,
+        password=password,
+        api_version=api_version,
+        auth_mode=auth_mode,
+        connect_timeout=connect_timeout,
+        read_timeout=read_timeout,
+        max_retries=max_retries,
+        retry_backoff=retry_backoff,
+        history_run_limit=history_run_limit,
+    )
+
+
+def _read_float(name: str, default: str) -> float:
+    value = os.getenv(name, default)
+    try:
+        return float(value)
+    except ValueError as exc:
+        raise ConfigurationError(f"{name} must be a number.") from exc
+
+
+def _read_int(name: str, default: str) -> int:
+    value = os.getenv(name, default)
+    try:
+        return int(value)
+    except ValueError as exc:
+        raise ConfigurationError(f"{name} must be an integer.") from exc