model-eval-toolkit 0.1.0__py3-none-any.whl

evalreport/regression/report.py

@@ -0,0 +1,173 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Iterable, List, Optional
+
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.metrics import (
+    mean_absolute_error,
+    mean_squared_error,
+    median_absolute_error,
+    r2_score,
+)
+
+from ..core.base_report import BaseReport
+
+
+def _as_float_array(x: Optional[Iterable[Any]]) -> Optional[np.ndarray]:
+    if x is None:
+        return None
+    arr = np.asarray(list(x), dtype=float)
+    return arr
+
+
+def _safe_float(x: Any) -> Any:
+    try:
+        if isinstance(x, (np.floating, np.integer)):
+            return x.item()
+        if isinstance(x, float) and (np.isnan(x) or np.isinf(x)):
+            return None
+        return float(x)
+    except Exception:
+        return x
+
+
+@dataclass
+class RegressionReport(BaseReport):
+    y_true: Optional[Iterable[Any]] = None
+    y_pred: Optional[Iterable[Any]] = None
+
+    def _compute_metrics(self) -> None:
+        y_true = _as_float_array(self.y_true)
+        y_pred = _as_float_array(self.y_pred)
+        if y_true is None or y_pred is None:
+            raise ValueError("RegressionReport requires y_true and y_pred.")
+
+        mse = mean_squared_error(y_true, y_pred)
+        rmse = float(np.sqrt(mse))
+
+        self.metrics["mae"] = _safe_float(mean_absolute_error(y_true, y_pred))
+        self.metrics["mse"] = _safe_float(mse)
+        self.metrics["rmse"] = _safe_float(rmse)
+        self.metrics["r2"] = _safe_float(r2_score(y_true, y_pred))
+        self.metrics["median_ae"] = _safe_float(median_absolute_error(y_true, y_pred))
+
+        # MAPE (skip samples where |y_true| ~ 0; undefined if none left)
+        valid = np.abs(y_true) >= 1e-12
+        if valid.any():
+            rel = np.abs((y_true[valid] - y_pred[valid]) / np.abs(y_true[valid]))
+            self.metrics["mape"] = _safe_float(float(np.mean(rel) * 100.0))
+        else:
+            self.metrics["mape"] = None
+
+        # Mean error (bias)
+        self.metrics["mean_error"] = _safe_float(float(np.mean(y_pred - y_true)))
+
+    def _generate_plots(self) -> None:
+        y_true = _as_float_array(self.y_true)
+        y_pred = _as_float_array(self.y_pred)
+        if y_true is None or y_pred is None:
+            self.plots = {}
+            return
+
+        root = self.output_dir or Path("reports")
+        plot_dir = root / "evalreport_plots"
+        plot_dir.mkdir(parents=True, exist_ok=True)
+        plots: dict[str, str] = {}
+
+        # Residual plot (residual vs predicted)
+        try:
+            residuals = y_pred - y_true
+            plt.figure(figsize=(4, 3))
+            plt.scatter(y_pred, residuals, alpha=0.7)
+            plt.axhline(0, color="red", linestyle="--")
+            plt.xlabel("Predicted")
+            plt.ylabel("Residual (y_pred - y_true)")
+            plt.title("Residual Plot")
+            path = plot_dir / "regression_residuals.png"
+            plt.tight_layout()
+            plt.savefig(path)
+            plt.close()
+            plots["residual_plot"] = str(path)
+        except Exception:
+            pass
+
+        # Predicted vs Actual
+        try:
+            plt.figure(figsize=(4, 3))
+            plt.scatter(y_true, y_pred, alpha=0.7)
+            min_v = float(min(np.min(y_true), np.min(y_pred)))
+            max_v = float(max(np.max(y_true), np.max(y_pred)))
+            plt.plot([min_v, max_v], [min_v, max_v], "k--", label="Ideal")
+            plt.xlabel("Actual")
+            plt.ylabel("Predicted")
+            plt.title("Predicted vs Actual")
+            plt.legend()
+            path = plot_dir / "regression_pred_vs_actual.png"
+            plt.tight_layout()
+            plt.savefig(path)
+            plt.close()
+            plots["predicted_vs_actual"] = str(path)
+        except Exception:
+            pass
+
+        # Error distribution
+        try:
+            residuals = y_pred - y_true
+            plt.figure(figsize=(4, 3))
+            plt.hist(residuals, bins=20, alpha=0.8)
+            plt.xlabel("Residual")
+            plt.ylabel("Count")
+            plt.title("Error Distribution")
+            path = plot_dir / "regression_error_distribution.png"
+            plt.tight_layout()
+            plt.savefig(path)
+            plt.close()
+            plots["error_distribution"] = str(path)
+        except Exception:
+            pass
+
+        self.plots = plots
+
+    def _generate_insights(self) -> None:
+        y_true = _as_float_array(self.y_true)
+        y_pred = _as_float_array(self.y_pred)
+        if y_true is None or y_pred is None:
+            return
+
+        insights: List[str] = []
+
+        err = y_pred - y_true
+        mean_err = float(np.mean(err))
+        if abs(mean_err) > 0:
+            direction = "positive" if mean_err > 0 else "negative"
+            insights.append(f"Predictions show {direction} bias (mean error={mean_err:.4g}).")
+
+        # Outlier influence heuristic: large 95th percentile absolute error
+        abs_err = np.abs(err)
+        p95 = float(np.percentile(abs_err, 95))
+        med = float(np.median(abs_err))
+        if med > 0 and (p95 / med) >= 5:
+            insights.append(
+                f"Errors have heavy tail (p95 abs error≈{p95:.4g} vs median≈{med:.4g}); check outliers."
+            )
+
+        self.insights = insights
+
+        # Descriptions for key metrics shown in HTML/PDF
+        self.metric_descriptions.update(
+            {
+                "mae": "Mean absolute error; average absolute deviation between predictions and true values.",
+                "mse": "Mean squared error; penalizes larger errors more strongly.",
+                "rmse": "Root mean squared error; error in the same units as the target.",
+                "r2": "Coefficient of determination; proportion of variance explained by the model.",
+                "median_ae": "Median absolute error; robust to outliers in the error distribution.",
+                "mape": "Mean absolute percentage error; average relative error as a percentage.",
+                "mean_error": "Signed average error; indicates systematic over- or under-prediction.",
+            }
+        )
+

evalreport/timeseries/__init__.py

@@ -0,0 +1,4 @@
+from .report import TimeSeriesReport
+
+__all__ = ["TimeSeriesReport"]
+

evalreport/timeseries/report.py

@@ -0,0 +1,211 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Iterable, List, Optional
+
+import matplotlib
+
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+import numpy as np
+
+from ..core.base_report import BaseReport
+
+
+def _as_float_1d(x: Optional[Iterable[Any]]) -> Optional[np.ndarray]:
+    if x is None:
+        return None
+    arr = np.asarray(list(x), dtype=float).reshape(-1)
+    return arr
+
+
+def _as_1d(x: Optional[Iterable[Any]]) -> Optional[np.ndarray]:
+    if x is None:
+        return None
+    return np.asarray(list(x))
+
+
+@dataclass
+class TimeSeriesReport(BaseReport):
+    y_true: Optional[Iterable[Any]] = None
+    y_pred: Optional[Iterable[Any]] = None
+    timestamps: Optional[Iterable[Any]] = None
+    rolling_window: int = 5
+
+    def _compute_metrics(self) -> None:
+        y_true = _as_float_1d(self.y_true)
+        y_pred = _as_float_1d(self.y_pred)
+        ts = _as_1d(self.timestamps)
+        if y_true is None or y_pred is None or ts is None:
+            raise ValueError("TimeSeriesReport requires y_true, y_pred, and timestamps.")
+        if not (len(y_true) == len(y_pred) == len(ts)):
+            raise ValueError("TimeSeriesReport requires y_true, y_pred, timestamps with equal length.")
+        if len(y_true) < 2:
+            raise ValueError("TimeSeriesReport requires at least 2 samples.")
+
+        err = y_pred - y_true
+        abs_err = np.abs(err)
+
+        mae = float(np.mean(abs_err))
+        mse = float(np.mean(err**2))
+        rmse = float(np.sqrt(mse))
+
+        denom = np.where(np.abs(y_true) < 1e-12, np.nan, np.abs(y_true))
+        mape = float(np.nanmean(abs_err / denom) * 100.0)
+        if np.isnan(mape):
+            mape = None  # undefined when all targets are ~0
+
+        smape_denom = np.abs(y_true) + np.abs(y_pred)
+        smape = float(np.mean(np.where(smape_denom < 1e-12, np.nan, 2.0 * abs_err / smape_denom)) * 100.0)
+        if np.isnan(smape):
+            smape = None
+
+        mean_forecast_error = float(np.mean(err))
+
+        # Rolling RMSE
+        w = int(self.rolling_window)
+        w = max(2, min(w, len(y_true)))
+        rolling = []
+        for i in range(w - 1, len(y_true)):
+            window_err = err[i - w + 1 : i + 1]
+            rolling.append(float(np.sqrt(np.mean(window_err**2))))
+
+        # Cache the rolling window actually used so plots align with metrics.
+        self._cached_rolling_window = w
+
+        rolling_rmse_last = float(rolling[-1]) if rolling else None
+        rolling_rmse_mean = float(np.mean(rolling)) if rolling else None
+
+        self.metrics.update(
+            {
+                "mae": mae,
+                "mse": mse,
+                "rmse": rmse,
+                "mape": mape,
+                "smape": smape,
+                "mean_forecast_error": mean_forecast_error,
+                "rolling_rmse_mean": rolling_rmse_mean,
+                "rolling_rmse_last": rolling_rmse_last,
+            }
+        )
+
+
+        # Store for plotting
+        self._cached_ts = ts
+        self._cached_y_true = y_true
+        self._cached_y_pred = y_pred
+        self._cached_rolling_rmse = rolling
+
+    def _generate_plots(self) -> None:
+        y_true = _as_float_1d(self.y_true)
+        y_pred = _as_float_1d(self.y_pred)
+        ts = _as_1d(self.timestamps)
+        if y_true is None or y_pred is None or ts is None:
+            self.plots = {}
+            return
+        if len(y_true) != len(y_pred) or len(y_true) != len(ts):
+            self.plots = {}
+            return
+
+        root = self.output_dir or Path("reports")
+        plot_dir = root / "evalreport_plots"
+        plot_dir.mkdir(parents=True, exist_ok=True)
+
+        # Ensure chronological order
+        order = np.argsort(ts)
+        ts_sorted = ts[order]
+        y_true_sorted = y_true[order]
+        y_pred_sorted = y_pred[order]
+        residuals = y_pred_sorted - y_true_sorted
+
+        # Rolling RMSE aligns with timestamps after (w-1) offset.
+        rolling = getattr(self, "_cached_rolling_rmse", None)
+        w = getattr(self, "_cached_rolling_window", max(2, int(self.rolling_window)))
+        if len(ts_sorted) >= w:
+            rolling_ts = ts_sorted[w - 1 :]
+        else:
+            # Extremely short series; align lengths defensively.
+            rolling_ts = ts_sorted[: len(rolling)] if rolling is not None else ts_sorted
+
+        plots: dict[str, str] = {}
+
+        # Actual vs Forecast
+        plt.figure(figsize=(6, 4))
+        plt.plot(ts_sorted, y_true_sorted, label="Actual", linewidth=2)
+        plt.plot(ts_sorted, y_pred_sorted, label="Forecast", linewidth=2, linestyle="--")
+        plt.xlabel("Time")
+        plt.ylabel("Value")
+        plt.title("Actual vs Forecast")
+        plt.legend()
+        path = plot_dir / "timeseries_actual_vs_forecast.png"
+        plt.tight_layout()
+        plt.savefig(path)
+        plt.close()
+        plots["actual_vs_forecast"] = str(path)
+
+        # Residuals over time
+        plt.figure(figsize=(6, 3.8))
+        plt.plot(ts_sorted, residuals, label="Residual", color="#D55E00")
+        plt.axhline(0, color="black", linewidth=1)
+        plt.xlabel("Time")
+        plt.ylabel("Residual (y_pred - y_true)")
+        plt.title("Residuals over time")
+        plt.legend()
+        path = plot_dir / "timeseries_residuals_over_time.png"
+        plt.tight_layout()
+        plt.savefig(path)
+        plt.close()
+        plots["residuals_over_time"] = str(path)
+
+        # Rolling RMSE
+        if rolling is not None and len(rolling) > 0:
+            plt.figure(figsize=(6, 3.8))
+            plt.plot(rolling_ts, rolling, label="Rolling RMSE", color="#0072B2")
+            plt.xlabel("Time")
+            plt.ylabel("RMSE")
+            plt.title(f"Rolling RMSE (window={self.rolling_window})")
+            plt.legend()
+            path = plot_dir / "timeseries_rolling_rmse.png"
+            plt.tight_layout()
+            plt.savefig(path)
+            plt.close()
+            plots["rolling_rmse"] = str(path)
+
+        self.plots = plots
+
+    def _generate_insights(self) -> None:
+        y_true = _as_float_1d(self.y_true)
+        y_pred = _as_float_1d(self.y_pred)
+        if y_true is None or y_pred is None:
+            self.insights = []
+            return
+        err = y_pred - y_true
+        mean_err = float(np.mean(err))
+        insights: List[str] = []
+        if abs(mean_err) > 0:
+            direction = "over" if mean_err > 0 else "under"
+            insights.append(f"Forecasts show systematic {direction}-prediction bias (mean error={mean_err:.4g}).")
+
+        rolling_mean = self.metrics.get("rolling_rmse_mean")
+        rolling_last = self.metrics.get("rolling_rmse_last")
+        if isinstance(rolling_mean, (int, float)) and isinstance(rolling_last, (int, float)) and rolling_mean:
+            if rolling_last > 1.2 * rolling_mean:
+                insights.append("Recent errors increased vs earlier windows; model may be drifting.")
+            elif rolling_last < 0.8 * rolling_mean:
+                insights.append("Recent errors improved vs earlier windows.")
+
+        self.insights = insights
+
+        self.metric_descriptions.update(
+            {
+                "mae": "Mean absolute error; average absolute deviation between forecast and truth.",
+                "rmse": "Root mean squared error; penalizes larger errors more.",
+                "mape": "Mean absolute percentage error; undefined if true values are all ~0.",
+                "smape": "Symmetric MAPE; more stable when scaling varies between truth and prediction.",
+                "mean_forecast_error": "Signed average error; indicates systematic over- or under-forecasting.",
+                "rolling_rmse_mean": "Average RMSE across rolling windows (overall stability).",
+                "rolling_rmse_last": "Most recent rolling RMSE (latest stability).",
+            }
+        )
+

evalreport/vision/__init__.py

@@ -0,0 +1,6 @@
+from .segmentation import SegmentationReport
+from .detection import DetectionReport
+from .image_classification import ImageClassificationReport
+
+__all__ = ["SegmentationReport", "DetectionReport", "ImageClassificationReport"]
+