testmind 0.1.0__py3-none-any.whl

testmind/analysis/flaky.py

@@ -0,0 +1,96 @@
+"""
+Flaky test detection.
+
+A test is FLAKY when it produces mixed pass/fail results over recent runs
+without a clear directional trend — i.e. the failure rate is in the
+"uncertain zone" (between FLAKY_LOW and FLAKY_HIGH thresholds) AND
+consecutive results flip at a meaningful rate.
+
+Rules
+-----
+- Requires at least `min_runs` observations (default 5).
+- fail_rate  ∈ (FLAKY_LOW, FLAKY_HIGH)  → candidate
+- flip_rate  > FLIP_THRESHOLD            → confirmed flaky
+  where flip_rate = fraction of consecutive pairs whose outcome differs.
+"""
+
+from datetime import datetime
+
+from testmind.domain.models import TestResult, TestStatus
+from testmind.analysis.models import FlakyResult
+
+# Thresholds (all tunable via constructor kwargs)
+_FLAKY_LOW = 0.10    # below this → consistently passing
+_FLAKY_HIGH = 0.90   # above this → consistently failing
+_FLIP_THRESHOLD = 0.15
+_MIN_RUNS = 5
+
+
+def _is_failure(status: TestStatus) -> bool:
+    return status in (TestStatus.FAILED, TestStatus.ERROR)
+
+
+def _flip_rate(outcomes: list[bool]) -> float:
+    if len(outcomes) < 2:
+        return 0.0
+    flips = sum(a != b for a, b in zip(outcomes, outcomes[1:]))
+    return flips / (len(outcomes) - 1)
+
+
+class FlakyDetector:
+    def __init__(
+        self,
+        min_runs: int = _MIN_RUNS,
+        flaky_low: float = _FLAKY_LOW,
+        flaky_high: float = _FLAKY_HIGH,
+        flip_threshold: float = _FLIP_THRESHOLD,
+    ) -> None:
+        self.min_runs = min_runs
+        self.flaky_low = flaky_low
+        self.flaky_high = flaky_high
+        self.flip_threshold = flip_threshold
+
+    def analyze(
+        self,
+        test_name: str,
+        history: list[tuple[datetime, TestResult]],
+    ) -> FlakyResult:
+        """
+        Parameters
+        ----------
+        history : list of (timestamp, TestResult), oldest → newest.
+        """
+        if len(history) < self.min_runs:
+            return FlakyResult(
+                test_name=test_name,
+                is_flaky=False,
+                flip_rate=0.0,
+                pass_rate=0.0,
+                fail_rate=0.0,
+                run_count=len(history),
+                insufficient_data=True,
+            )
+
+        # Oldest → newest order (history may arrive newest-first from store)
+        ordered = sorted(history, key=lambda x: x[0])
+        outcomes = [_is_failure(r.status) for _, r in ordered]
+
+        total = len(outcomes)
+        failures = sum(outcomes)
+        fail_rate = failures / total
+        pass_rate = 1.0 - fail_rate
+        fr = _flip_rate(outcomes)
+
+        is_flaky = (
+            self.flaky_low < fail_rate < self.flaky_high
+            and fr > self.flip_threshold
+        )
+
+        return FlakyResult(
+            test_name=test_name,
+            is_flaky=is_flaky,
+            flip_rate=fr,
+            pass_rate=pass_rate,
+            fail_rate=fail_rate,
+            run_count=total,
+        )

testmind/analysis/models.py

@@ -0,0 +1,62 @@
+from dataclasses import dataclass
+from enum import StrEnum
+
+
+class Trend(StrEnum):
+    IMPROVING = "improving"
+    DEGRADING = "degrading"
+    STABLE = "stable"
+
+
+@dataclass(frozen=True)
+class FlakyResult:
+    test_name: str
+    is_flaky: bool
+    flip_rate: float      # fraction of consecutive pairs with different outcomes
+    pass_rate: float
+    fail_rate: float
+    run_count: int
+    # True when there are too few runs to decide
+    insufficient_data: bool = False
+
+
+@dataclass(frozen=True)
+class RegressionResult:
+    test_name: str
+    is_regression: bool
+    # pass rate in the reference (older) window
+    reference_pass_rate: float
+    # fail rate in the recent window
+    recent_fail_rate: float
+    insufficient_data: bool = False
+
+
+@dataclass(frozen=True)
+class SpikeResult:
+    """Suite-level: was there a sudden failure-rate spike in the latest run?"""
+    is_spike: bool
+    current_fail_rate: float
+    baseline_mean: float
+    baseline_std: float
+    z_score: float
+    insufficient_data: bool = False
+
+
+@dataclass(frozen=True)
+class StabilityResult:
+    test_name: str
+    score: float            # 0–100
+    pass_rate: float
+    duration_consistency: float   # 0–1  (1 = perfectly consistent)
+    flip_rate: float
+    run_count: int
+    insufficient_data: bool = False
+
+
+@dataclass(frozen=True)
+class PredictionResult:
+    test_name: str
+    failure_probability: float   # 0–1
+    trend: Trend
+    confidence: float            # 0–1  (grows with run count)
+    insufficient_data: bool = False

testmind/analysis/predictor.py

@@ -0,0 +1,101 @@
+"""
+Failure probability predictor.
+
+Approach
+--------
+1. Sort history oldest → newest.
+2. Compute a per-run binary outcome (1 = fail, 0 = pass).
+3. Fit a linear trend to those outcomes using OLS (no external deps).
+4. Predict the next value = last_outcome + slope.
+5. Clamp prediction to [0, 1] and interpret trend.
+
+Confidence scales with the number of runs up to MAX_CONF_RUNS.
+
+Trend thresholds
+----------------
+  slope > +SLOPE_THRESHOLD  → DEGRADING
+  slope < -SLOPE_THRESHOLD  → IMPROVING
+  otherwise                 → STABLE
+"""
+
+from datetime import datetime
+
+from testmind.domain.models import TestResult, TestStatus
+from testmind.analysis.models import PredictionResult, Trend
+
+_MIN_RUNS = 3
+_MAX_CONF_RUNS = 20
+_SLOPE_THRESHOLD = 0.05
+
+
+def _is_failure(status: TestStatus) -> bool:
+    return status in (TestStatus.FAILED, TestStatus.ERROR)
+
+
+def _ols_slope(ys: list[float]) -> float:
+    """Slope of OLS regression of ys against integer indices 0, 1, …, n-1."""
+    n = len(ys)
+    if n < 2:
+        return 0.0
+    xs = list(range(n))
+    mean_x = (n - 1) / 2.0      # exact for 0..n-1
+    mean_y = sum(ys) / n
+    num = sum((x - mean_x) * (y - mean_y) for x, y in zip(xs, ys))
+    den = sum((x - mean_x) ** 2 for x in xs)
+    return num / den if den != 0.0 else 0.0
+
+
+class FailurePredictor:
+    def __init__(
+        self,
+        min_runs: int = _MIN_RUNS,
+        slope_threshold: float = _SLOPE_THRESHOLD,
+        max_conf_runs: int = _MAX_CONF_RUNS,
+    ) -> None:
+        self.min_runs = min_runs
+        self.slope_threshold = slope_threshold
+        self.max_conf_runs = max_conf_runs
+
+    def analyze(
+        self,
+        test_name: str,
+        history: list[tuple[datetime, TestResult]],
+    ) -> PredictionResult:
+        """
+        Parameters
+        ----------
+        history : list of (timestamp, TestResult), any order.
+        """
+        if len(history) < self.min_runs:
+            return PredictionResult(
+                test_name=test_name,
+                failure_probability=0.0,
+                trend=Trend.STABLE,
+                confidence=0.0,
+                insufficient_data=True,
+            )
+
+        ordered = sorted(history, key=lambda x: x[0])
+        outcomes = [1.0 if _is_failure(r.status) else 0.0 for _, r in ordered]
+
+        slope = _ols_slope(outcomes)
+
+        last_rate = sum(outcomes[-3:]) / min(3, len(outcomes))
+        raw_prediction = last_rate + slope
+        failure_prob = max(0.0, min(1.0, raw_prediction))
+
+        if slope > self.slope_threshold:
+            trend = Trend.DEGRADING
+        elif slope < -self.slope_threshold:
+            trend = Trend.IMPROVING
+        else:
+            trend = Trend.STABLE
+
+        confidence = min(len(outcomes) / self.max_conf_runs, 1.0)
+
+        return PredictionResult(
+            test_name=test_name,
+            failure_probability=round(failure_prob, 4),
+            trend=trend,
+            confidence=round(confidence, 4),
+        )

testmind/analysis/regression.py

@@ -0,0 +1,153 @@
+"""
+Regression and spike detection.
+
+Regression (test level)
+-----------------------
+A test is a REGRESSION when it was stable (mostly passing) in an older
+reference window but has started failing in a recent window.
+
+Rules
+-----
+- Requires at least `min_runs` total observations (default 6).
+- Reference window  : all runs except the last `recent_window` (default 3).
+- Recent window     : the last `recent_window` runs.
+- Is regression when:
+    reference_pass_rate >= STABLE_THRESHOLD   (was stable)
+    AND recent_fail_rate >= RECENT_FAIL_THRESHOLD  (now failing)
+
+Spike (suite level)
+-------------------
+A SPIKE occurs when the most recent report's failure rate is significantly
+higher than the rolling baseline of previous reports.
+
+Rules
+-----
+- Requires at least `min_baseline` previous reports (default 3).
+- Baseline = fail_rate of the N reports preceding the latest.
+- z_score = (current_fail_rate - baseline_mean) / baseline_std
+- Spike when z_score >= SPIKE_Z_THRESHOLD (default 2.0)
+  AND current_fail_rate > baseline_mean  (one-tailed).
+"""
+
+import math
+from datetime import datetime
+
+from testmind.domain.models import TestReport, TestResult, TestStatus
+from testmind.analysis.models import RegressionResult, SpikeResult
+
+_STABLE_THRESHOLD = 0.90
+_RECENT_FAIL_THRESHOLD = 0.60
+_RECENT_WINDOW = 3
+_MIN_RUNS = 6
+_MIN_BASELINE = 3
+_SPIKE_Z = 2.0
+
+
+def _is_failure(status: TestStatus) -> bool:
+    return status in (TestStatus.FAILED, TestStatus.ERROR)
+
+
+class RegressionDetector:
+    def __init__(
+        self,
+        recent_window: int = _RECENT_WINDOW,
+        min_runs: int = _MIN_RUNS,
+        stable_threshold: float = _STABLE_THRESHOLD,
+        recent_fail_threshold: float = _RECENT_FAIL_THRESHOLD,
+    ) -> None:
+        self.recent_window = recent_window
+        self.min_runs = min_runs
+        self.stable_threshold = stable_threshold
+        self.recent_fail_threshold = recent_fail_threshold
+
+    def analyze(
+        self,
+        test_name: str,
+        history: list[tuple[datetime, TestResult]],
+    ) -> RegressionResult:
+        """
+        Parameters
+        ----------
+        history : list of (timestamp, TestResult), any order — will be sorted.
+        """
+        if len(history) < self.min_runs:
+            return RegressionResult(
+                test_name=test_name,
+                is_regression=False,
+                reference_pass_rate=0.0,
+                recent_fail_rate=0.0,
+                insufficient_data=True,
+            )
+
+        ordered = sorted(history, key=lambda x: x[0])
+        recent = ordered[-self.recent_window :]
+        reference = ordered[: -self.recent_window]
+
+        ref_failures = sum(_is_failure(r.status) for _, r in reference)
+        ref_pass_rate = 1.0 - ref_failures / len(reference)
+
+        rec_failures = sum(_is_failure(r.status) for _, r in recent)
+        rec_fail_rate = rec_failures / len(recent)
+
+        is_regression = (
+            ref_pass_rate >= self.stable_threshold
+            and rec_fail_rate >= self.recent_fail_threshold
+        )
+
+        return RegressionResult(
+            test_name=test_name,
+            is_regression=is_regression,
+            reference_pass_rate=ref_pass_rate,
+            recent_fail_rate=rec_fail_rate,
+        )
+
+
+class SpikeDetector:
+    def __init__(
+        self,
+        min_baseline: int = _MIN_BASELINE,
+        z_threshold: float = _SPIKE_Z,
+    ) -> None:
+        self.min_baseline = min_baseline
+        self.z_threshold = z_threshold
+
+    def analyze(self, reports: list[TestReport]) -> SpikeResult:
+        """
+        Parameters
+        ----------
+        reports : ordered oldest → newest; the last entry is the current run.
+        """
+        if len(reports) < self.min_baseline + 1:
+            return SpikeResult(
+                is_spike=False,
+                current_fail_rate=0.0,
+                baseline_mean=0.0,
+                baseline_std=0.0,
+                z_score=0.0,
+                insufficient_data=True,
+            )
+
+        ordered = sorted(reports, key=lambda r: r.timestamp)
+        current = ordered[-1]
+        baseline = ordered[:-1]
+
+        current_fail_rate = current.fail_rate
+        baseline_rates = [r.fail_rate for r in baseline]
+        mean = sum(baseline_rates) / len(baseline_rates)
+        variance = sum((x - mean) ** 2 for x in baseline_rates) / len(baseline_rates)
+        std = math.sqrt(variance)
+
+        if std == 0.0:
+            z_score = 0.0 if current_fail_rate == mean else float("inf")
+        else:
+            z_score = (current_fail_rate - mean) / std
+
+        is_spike = z_score >= self.z_threshold and current_fail_rate > mean
+
+        return SpikeResult(
+            is_spike=is_spike,
+            current_fail_rate=current_fail_rate,
+            baseline_mean=mean,
+            baseline_std=std,
+            z_score=z_score,
+        )

testmind/analysis/stability.py

@@ -0,0 +1,99 @@
+"""
+Stability Index  (0 – 100).
+
+Formula
+-------
+score = pass_rate_score + consistency_score + non_flakiness_score
+
+Where:
+  pass_rate_score    = pass_rate  × 60
+  consistency_score  = duration_consistency × 20
+  non_flakiness_score = (1 - flip_rate) × 20
+
+duration_consistency = 1 - min(CV, 1)
+  CV (coefficient of variation) = std(durations) / mean(durations)
+  → 0 when wildly variable, 1 when perfectly consistent.
+  If all durations are zero or there is only one run, consistency = 1.
+
+flip_rate = fraction of consecutive pairs with different outcomes.
+"""
+
+import math
+from datetime import datetime
+
+from testmind.domain.models import TestResult, TestStatus
+from testmind.analysis.models import StabilityResult
+
+_MIN_RUNS = 3
+
+
+def _is_failure(status: TestStatus) -> bool:
+    return status in (TestStatus.FAILED, TestStatus.ERROR)
+
+
+def _flip_rate(outcomes: list[bool]) -> float:
+    if len(outcomes) < 2:
+        return 0.0
+    flips = sum(a != b for a, b in zip(outcomes, outcomes[1:]))
+    return flips / (len(outcomes) - 1)
+
+
+def _duration_consistency(durations: list[float]) -> float:
+    if len(durations) < 2:
+        return 1.0
+    mean = sum(durations) / len(durations)
+    if mean == 0.0:
+        return 1.0
+    variance = sum((d - mean) ** 2 for d in durations) / len(durations)
+    std = math.sqrt(variance)
+    cv = std / mean
+    return 1.0 - min(cv, 1.0)
+
+
+class StabilityAnalyzer:
+    def __init__(self, min_runs: int = _MIN_RUNS) -> None:
+        self.min_runs = min_runs
+
+    def analyze(
+        self,
+        test_name: str,
+        history: list[tuple[datetime, TestResult]],
+    ) -> StabilityResult:
+        """
+        Parameters
+        ----------
+        history : list of (timestamp, TestResult), any order.
+        """
+        if len(history) < self.min_runs:
+            return StabilityResult(
+                test_name=test_name,
+                score=0.0,
+                pass_rate=0.0,
+                duration_consistency=0.0,
+                flip_rate=0.0,
+                run_count=len(history),
+                insufficient_data=True,
+            )
+
+        ordered = sorted(history, key=lambda x: x[0])
+        outcomes = [_is_failure(r.status) for _, r in ordered]
+        durations = [r.duration for _, r in ordered]
+
+        total = len(outcomes)
+        failures = sum(outcomes)
+        fail_rate = failures / total
+        pass_rate = 1.0 - fail_rate
+
+        consistency = _duration_consistency(durations)
+        fr = _flip_rate(outcomes)
+
+        score = pass_rate * 60.0 + consistency * 20.0 + (1.0 - fr) * 20.0
+
+        return StabilityResult(
+            test_name=test_name,
+            score=round(score, 2),
+            pass_rate=pass_rate,
+            duration_consistency=consistency,
+            flip_rate=fr,
+            run_count=total,
+        )