pyairflowtester 0.3.0__py3-none-any.whl

pyairflowtester/__init__.py

@@ -0,0 +1,72 @@
+"""
+PyAirflowTester: Unified Airflow + dbt Reliability Platform
+
+A correlation-first reliability and quality platform that combines static analysis,
+runtime monitoring, and intelligent correlation to prevent data pipeline failures.
+"""
+
+__version__ = "0.3.0"
+__author__ = "PyAirflowTester Contributors"
+__license__ = "SSPL-1.0"
+
+try:
+    from pyairflowtester._core import (
+        PyCategory,
+        PyDagDefinition,
+        PyDagParser,
+        PyDbtModel,
+        PyDbtParser,
+        PyDbtProject,
+        PyDbtTest,
+        PyExecutionMode,
+        PyRule,
+        PyRuleContext,
+        PyRuleEngine,
+        PyRuleViolation,
+        PyScorer,
+        PySeverity,
+        evaluate_rules,
+    )
+except ImportError:
+    # Fallback if Rust extension not available
+    PyRule = None
+    PyRuleViolation = None
+    evaluate_rules = None
+    PySeverity = None
+    PyCategory = None
+    PyExecutionMode = None
+    PyRuleContext = None
+    PyRuleEngine = None
+    PyDagDefinition = None
+    PyDagParser = None
+    PyDbtModel = None
+    PyDbtTest = None
+    PyDbtProject = None
+    PyDbtParser = None
+    PyScorer = None
+
+from pyairflowtester.analyzer import Analyzer, AnalyzerNotImplementedError
+from pyairflowtester.report import ReportGenerator
+from pyairflowtester.scanner import Scanner
+
+__all__ = [
+    "Scanner",
+    "Analyzer",
+    "AnalyzerNotImplementedError",
+    "ReportGenerator",
+    "PyRule",
+    "PyRuleViolation",
+    "evaluate_rules",
+    "PySeverity",
+    "PyCategory",
+    "PyExecutionMode",
+    "PyRuleContext",
+    "PyRuleEngine",
+    "PyDagDefinition",
+    "PyDagParser",
+    "PyDbtModel",
+    "PyDbtTest",
+    "PyDbtProject",
+    "PyDbtParser",
+    "PyScorer",
+]

pyairflowtester/analyzer.py

@@ -0,0 +1,130 @@
+"""
+Analyzer module for runtime analysis.
+
+INTENTIONALLY UNIMPLEMENTED. This subsystem is meant to connect to live
+Airflow instances and dbt systems to analyze execution patterns, failures,
+and correlations. Building real runtime correlation isn't achievable
+without an actual live Airflow/dbt instance to develop and test against,
+so every method below raises AnalyzerNotImplementedError rather than
+silently returning an empty list/dict that could be mistaken for "analyzed,
+found nothing." See README for status.
+"""
+
+import logging
+from typing import Any, Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+
+class AnalyzerNotImplementedError(NotImplementedError):
+    """Raised by Analyzer methods: runtime correlation is not yet built.
+
+    The Analyzer subsystem requires a live Airflow metadata database (and,
+    for dbt methods, a live dbt run history) to implement and validate
+    against. That is planned future work, not something safe to fake with
+    stub logic. Use the static `Scanner` (via `pyairflowtester scan`) for
+    the analysis that is actually implemented today.
+    """
+
+    def __init__(self, method_name: str):
+        super().__init__(
+            f"Analyzer.{method_name}() is not implemented. Runtime "
+            "correlation against live Airflow/dbt instances is a planned "
+            "feature, not yet built. Use `pyairflowtester scan` (static "
+            "analysis) for functionality that exists today."
+        )
+
+
+class Analyzer:
+    """Runtime analyzer for production pipelines. NOT YET IMPLEMENTED.
+
+    Every method on this class raises AnalyzerNotImplementedError. This
+    class is a placeholder for a future feature: correlating rule findings
+    against real DAG-run/task-instance history from a live Airflow metadata
+    database and real dbt test-run history. That requires a live
+    Airflow/dbt instance to build and validate against.
+    """
+
+    def __init__(self, airflow_home: Optional[str] = None, airflow_db: Optional[str] = None):
+        """
+        Initialize analyzer.
+
+        Args:
+            airflow_home: Airflow home directory
+            airflow_db: Airflow database connection string
+        """
+        self.airflow_home = airflow_home
+        self.airflow_db = airflow_db
+        self.db_connection = None
+
+    def connect(self) -> bool:
+        """
+        Connect to Airflow metadata database.
+
+        Raises:
+            AnalyzerNotImplementedError: always; not yet implemented.
+        """
+        raise AnalyzerNotImplementedError("connect")
+
+    def analyze_dag_failures(self, dag_id: str) -> List[Dict[str, Any]]:
+        """
+        Analyze failure patterns for a DAG.
+
+        Raises:
+            AnalyzerNotImplementedError: always; not yet implemented.
+        """
+        raise AnalyzerNotImplementedError("analyze_dag_failures")
+
+    def analyze_task_failures(self, dag_id: str, task_id: str) -> List[Dict[str, Any]]:
+        """
+        Analyze failure patterns for a task.
+
+        Raises:
+            AnalyzerNotImplementedError: always; not yet implemented.
+        """
+        raise AnalyzerNotImplementedError("analyze_task_failures")
+
+    def detect_hotspots(self) -> List[Dict[str, Any]]:
+        """
+        Detect task hotspots (frequently failing tasks).
+
+        Raises:
+            AnalyzerNotImplementedError: always; not yet implemented.
+        """
+        raise AnalyzerNotImplementedError("detect_hotspots")
+
+    def analyze_cascade_failures(self) -> List[Dict[str, Any]]:
+        """
+        Analyze cascading failure patterns.
+
+        Raises:
+            AnalyzerNotImplementedError: always; not yet implemented.
+        """
+        raise AnalyzerNotImplementedError("analyze_cascade_failures")
+
+    def get_dbt_test_failures(self) -> List[Dict[str, Any]]:
+        """
+        Get dbt test failure history.
+
+        Raises:
+            AnalyzerNotImplementedError: always; not yet implemented.
+        """
+        raise AnalyzerNotImplementedError("get_dbt_test_failures")
+
+    def detect_flaky_tests(self) -> List[Dict[str, Any]]:
+        """
+        Detect flaky dbt tests.
+
+        Raises:
+            AnalyzerNotImplementedError: always; not yet implemented.
+        """
+        raise AnalyzerNotImplementedError("detect_flaky_tests")
+
+    def calculate_blast_radius(self, source: str, source_type: str = "dag") -> Dict[str, Any]:
+        """
+        Calculate blast radius for a failure source.
+
+        Raises:
+            AnalyzerNotImplementedError: always; not yet implemented.
+        """
+        raise AnalyzerNotImplementedError("calculate_blast_radius")

pyairflowtester/cli.py

@@ -0,0 +1,299 @@
+"""
+PyAirflowTester CLI: Command-line interface for artifact and runtime analysis.
+"""
+
+import logging
+from pathlib import Path
+
+import click
+from rich.console import Console
+from rich.table import Table
+
+from pyairflowtester.analyzer import Analyzer, AnalyzerNotImplementedError
+from pyairflowtester.report import ReportGenerator
+from pyairflowtester.scanner import Scanner
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+)
+logger = logging.getLogger(__name__)
+
+console = Console()
+
+
+@click.group()
+@click.version_option()
+def main():
+    """PyAirflowTester: Unified Airflow + dbt Reliability Platform"""
+    pass
+
+
+@main.command()
+@click.argument("path", type=click.Path(exists=True), default=".")
+@click.option(
+    "--dags",
+    type=click.Path(exists=True),
+    help="Path to Airflow DAGs directory",
+)
+@click.option(
+    "--dbt",
+    type=click.Path(exists=True),
+    help="Path to dbt project directory",
+)
+@click.option(
+    "--airflow-cfg",
+    type=click.Path(exists=True),
+    help="Path to airflow.cfg for configuration auditing",
+)
+@click.option(
+    "--format",
+    type=click.Choice(["json", "html", "markdown", "sarif"]),
+    default="json",
+    help="Output format",
+)
+@click.option(
+    "--output",
+    type=click.Path(),
+    default=None,
+    help="Output file path",
+)
+@click.option(
+    "--severity",
+    type=click.Choice(["critical", "high", "medium", "low", "info"]),
+    default="medium",
+    help="Minimum severity level to report",
+)
+def scan(path, dags, dbt, airflow_cfg, format, output, severity):
+    """
+    Scan Airflow DAGs and dbt projects for issues.
+
+    Performs static analysis to detect configuration issues, anti-patterns,
+    and potential reliability risks before deployment.
+    """
+    console.print("[bold cyan]PyAirflowTester Scanner[/bold cyan]")
+    console.print(f"Target: {path}\n")
+
+    scanner = Scanner()
+
+    # Scan DAGs if path provided
+    if dags:
+        console.print("[yellow]Scanning Airflow DAGs...[/yellow]")
+        dag_violations = scanner.scan_dags(Path(dags))
+        console.print(f"  Found {len(dag_violations)} violations\n")
+    else:
+        dag_violations = []
+
+    # Scan dbt if path provided
+    if dbt:
+        console.print("[yellow]Scanning dbt project...[/yellow]")
+        dbt_violations = scanner.scan_dbt(Path(dbt))
+        console.print(f"  Found {len(dbt_violations)} violations\n")
+    else:
+        dbt_violations = []
+
+    # Scan airflow.cfg if path provided
+    if airflow_cfg:
+        console.print("[yellow]Scanning Airflow configuration...[/yellow]")
+        config_violations = scanner.scan_config(Path(airflow_cfg))
+        console.print(f"  Found {len(config_violations)} violations\n")
+    else:
+        config_violations = []
+
+    all_violations = dag_violations + dbt_violations + config_violations
+
+    if not all_violations:
+        console.print("[green]✓ No violations found![/green]")
+        return
+
+    # Filter by severity
+    from pyairflowtester.models import SEVERITY_WEIGHTS
+    min_weight = SEVERITY_WEIGHTS.get(severity, 0)
+    filtered = [
+        v for v in all_violations
+        if SEVERITY_WEIGHTS.get(v.get("severity", "info"), 0) >= min_weight
+    ]
+
+    # Display results
+    _display_violations(filtered)
+
+    # Generate report if format specified
+    if format and output:
+        generator = ReportGenerator()
+        report_path = generator.generate(format, filtered, Path(output))
+        console.print(f"\n[green]Report generated:[/green] {report_path}")
+
+
+@main.command()
+@click.argument("path", type=click.Path(exists=True), default=".")
+@click.option(
+    "--compare",
+    type=str,
+    default=None,
+    help="Compare to baseline (git branch)",
+)
+@click.option(
+    "--format",
+    type=click.Choice(["json", "markdown", "html"]),
+    default="markdown",
+    help="Output format",
+)
+def score(path, compare, format):
+    """
+    Calculate risk scores for DAGs and dbt models.
+
+    Generates multi-dimensional risk scores based on:
+    - Failure history and patterns
+    - Configuration quality
+    - Test coverage
+    - Performance metrics
+    """
+    console.print("[bold cyan]PyAirflowTester Scoring[/bold cyan]")
+    console.print(f"Target: {path}\n")
+
+    scanner = Scanner()
+    scorer = scanner._get_scorer()
+
+    # Scan for violations
+    dag_violations = scanner.scan_dags(Path(path) / "dags" if (Path(path) / "dags").exists() else Path(path))
+    dbt_violations = scanner.scan_dbt(Path(path) / "dbt" if (Path(path) / "dbt").exists() else Path(path))
+
+    all_violations = dag_violations + dbt_violations
+
+    # Calculate score
+    risk_score = scorer.calculate_risk_score(all_violations)
+    risk_level = scorer.categorize_risk(risk_score)
+
+    # Display scorecard
+    table = Table(title="Risk Scorecard")
+    table.add_column("Metric", style="cyan")
+    table.add_column("Score", style="magenta")
+    table.add_row("Overall Risk", f"{risk_score:.1f}/100")
+    table.add_row("Risk Level", risk_level)
+    table.add_row("Total Violations", str(len(all_violations)))
+
+    console.print(table)
+
+    if compare:
+        console.print(f"\n[yellow]Comparing to baseline: {compare}[/yellow]")
+
+
+@main.command()
+@click.option("--category", type=str, default=None, help="Filter by category")
+@click.option("--severity", type=str, default=None, help="Filter by severity")
+@click.option("--execution-mode", type=str, default=None, help="Filter by execution mode")
+def rules(category, severity, execution_mode):
+    """
+    List all available rules.
+
+    Rules are organized by category (reliability, performance, etc.) and
+    execution mode (static, runtime, correlation).
+    """
+    console.print("[bold cyan]PyAirflowTester Rules[/bold cyan]\n")
+
+    from pyairflowtester.rules import get_all_rules
+
+    all_rules = get_all_rules()
+
+    # Filter
+    filtered_rules = all_rules
+    if category:
+        filtered_rules = [r for r in filtered_rules if r.get("category") == category]
+    if severity:
+        filtered_rules = [r for r in filtered_rules if r.get("severity") == severity]
+    if execution_mode:
+        filtered_rules = [r for r in filtered_rules if r.get("execution_mode") == execution_mode]
+
+    # Display
+    table = Table(title=f"Available Rules ({len(filtered_rules)})")
+    table.add_column("ID", style="cyan")
+    table.add_column("Name", style="green")
+    table.add_column("Severity", style="magenta")
+    table.add_column("Category", style="yellow")
+    table.add_column("Mode")
+
+    for rule in filtered_rules:
+        table.add_row(
+            rule.get("id", ""),
+            rule.get("name", ""),
+            rule.get("severity", ""),
+            rule.get("category", ""),
+            rule.get("execution_mode", ""),
+        )
+
+    console.print(table)
+
+
+@main.command()
+@click.option(
+    "--airflow-home",
+    type=click.Path(exists=True),
+    help="Airflow home directory",
+)
+@click.option(
+    "--airflow-db",
+    type=str,
+    help="Airflow database connection string",
+)
+def connect(airflow_home, airflow_db):
+    """
+    Connect to Airflow instance for runtime analysis.
+
+    NOTE: Runtime analysis (the Analyzer subsystem) is not yet implemented.
+    This command reports that plainly instead of a fake success message.
+    """
+    console.print("[bold cyan]PyAirflowTester Runtime Connection[/bold cyan]\n")
+
+    analyzer = Analyzer(airflow_home=airflow_home, airflow_db=airflow_db)
+    try:
+        analyzer.connect()
+    except AnalyzerNotImplementedError as e:
+        console.print(f"[red]✗ {e}[/red]")
+        raise SystemExit(1)
+
+
+def _display_violations(violations):
+    """Display violations in rich table format."""
+    if not violations:
+        return
+
+    table = Table(title=f"Violations Found ({len(violations)})")
+    table.add_column("Rule", style="cyan")
+    table.add_column("Severity", style="magenta")
+    table.add_column("Resource", style="green")
+    table.add_column("Message")
+
+    for v in violations[:20]:  # Show first 20
+        severity = v.get("severity", "info")
+        severity_style = {
+            "critical": "red",
+            "high": "red",
+            "medium": "yellow",
+            "low": "blue",
+            "info": "white",
+        }.get(severity, "white")
+
+        table.add_row(
+            v.get("rule_id", ""),
+            f"[{severity_style}]{severity}[/{severity_style}]",
+            v.get("affected_resource", ""),
+            v.get("message", "")[:50],
+        )
+
+    if len(violations) > 20:
+        table.add_row("...", "...", "...", f"... and {len(violations) - 20} more")
+
+    console.print(table)
+
+
+# Register the dependency-intelligence command group (graph build/impact/
+# blast-radius/etc.). This was previously defined but never attached to the
+# main CLI group, so `pyairflowtester dependency ...` did not work.
+from pyairflowtester.dependency_intelligence.cli import register_dependency_cli  # noqa: E402
+
+register_dependency_cli(main)
+
+
+if __name__ == "__main__":
+    main()

pyairflowtester/dependency_intelligence/__init__.py

@@ -0,0 +1,110 @@
+"""Dependency Intelligence Engine for PyAirflowTester."""
+
+from .analytics import (
+    OwnershipAnalysis,
+    OwnershipAnalyzer,
+    SchemaEvolution,
+    SchemaEvolutionTracker,
+    SLAValidation,
+    SLAValidator,
+    TestCoverageAnalysis,
+    TestCoverageAnalyzer,
+)
+from .analyzers import (
+    BlastRadiusEngine,
+    DriftDetectionEngine,
+    ImpactAnalysisEngine,
+    RiskScoringEngine,
+)
+from .graph import DependencyGraphEngine
+from .intelligence import (
+    AnomalyDetection,
+    AnomalyDetector,
+    FailurePrediction,
+    FailurePredictionEngine,
+    HealthScore,
+    HealthScoreCalculator,
+    Recommendation,
+    RecommendationEngine,
+)
+from .models import (
+    BlastRadiusResult,
+    DependencyGraph,
+    DriftDetectionResult,
+    Edge,
+    ImpactResult,
+    Node,
+    NodeSeverity,
+    NodeType,
+    RiskScoreResult,
+)
+from .observability import (
+    Alert,
+    AlertManager,
+    DashboardBuilder,
+    EventLogger,
+    ExecutionEvent,
+    Metric,
+    MetricsCollector,
+    MetricType,
+)
+from .parsers import (
+    AirflowDAGParser,
+    AirflowDatasetParser,
+    UnifiedGraphBuilder,
+    dbtManifestParser,
+)
+
+__all__ = [
+    # Models
+    "NodeType",
+    "NodeSeverity",
+    "Node",
+    "Edge",
+    "DependencyGraph",
+    "ImpactResult",
+    "BlastRadiusResult",
+    "RiskScoreResult",
+    "DriftDetectionResult",
+    # Graph
+    "DependencyGraphEngine",
+    # Parsers
+    "AirflowDAGParser",
+    "dbtManifestParser",
+    "AirflowDatasetParser",
+    "UnifiedGraphBuilder",
+    # Phase 1 Analyzers
+    "ImpactAnalysisEngine",
+    "BlastRadiusEngine",
+    "RiskScoringEngine",
+    "DriftDetectionEngine",
+    # Phase 2 Analytics
+    "OwnershipAnalyzer",
+    "SchemaEvolutionTracker",
+    "SLAValidator",
+    "TestCoverageAnalyzer",
+    "OwnershipAnalysis",
+    "SchemaEvolution",
+    "SLAValidation",
+    "TestCoverageAnalysis",
+    # Phase 3 Intelligence
+    "FailurePredictionEngine",
+    "AnomalyDetector",
+    "RecommendationEngine",
+    "HealthScoreCalculator",
+    "FailurePrediction",
+    "AnomalyDetection",
+    "Recommendation",
+    "HealthScore",
+    # Phase 4 Observability
+    "MetricsCollector",
+    "AlertManager",
+    "EventLogger",
+    "DashboardBuilder",
+    "MetricType",
+    "Metric",
+    "Alert",
+    "ExecutionEvent",
+]
+
+__version__ = "0.4.0"  # Phase 1-4 MVP complete