tsagentkit 1.0.2__py3-none-any.whl

tsagentkit/backtest/metrics.py

@@ -0,0 +1,244 @@
+"""Metrics calculation for time series forecasting.
+
+Deprecated: use ``tsagentkit.eval.evaluate_forecasts`` (utilsforecast-backed)
+for new evaluation paths. This module is retained for backward compatibility
+and will be removed in a future phase.
+"""
+
+from __future__ import annotations
+
+import warnings
+from collections.abc import Iterable
+
+import numpy as np
+import pandas as pd
+
+from tsagentkit.eval import evaluate_forecasts
+from tsagentkit.utils import quantile_col_name
+
+_DEPRECATION_MESSAGE = (
+    "tsagentkit.backtest.metrics is deprecated; "
+    "use tsagentkit.eval.evaluate_forecasts instead."
+)
+
+
+def _warn_deprecated(name: str) -> None:
+    warnings.warn(
+        f"{name} is deprecated. {_DEPRECATION_MESSAGE}",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+
+
+def _as_array(values: Iterable[float]) -> np.ndarray:
+    return np.asarray(values, dtype=float)
+
+
+def _build_eval_frame(
+    y_true: Iterable[float],
+    y_pred: Iterable[float],
+    y_quantiles: dict[float, Iterable[float]] | None = None,
+) -> pd.DataFrame:
+    y_true_arr = _as_array(y_true)
+    y_pred_arr = _as_array(y_pred)
+    if y_true_arr.shape != y_pred_arr.shape:
+        raise ValueError("y_true and y_pred must have the same shape.")
+
+    n = y_true_arr.shape[0]
+    df = pd.DataFrame(
+        {
+            "unique_id": ["series"] * n,
+            "ds": pd.date_range("2000-01-01", periods=n, freq="D"),
+            "y": y_true_arr,
+            "yhat": y_pred_arr,
+            "model": "model",
+        }
+    )
+
+    if y_quantiles:
+        for q, values in y_quantiles.items():
+            q_values = _as_array(values)
+            if q_values.shape != y_true_arr.shape:
+                raise ValueError("Quantile predictions must match y_true shape.")
+            df[quantile_col_name(float(q))] = q_values
+
+    return df
+
+
+def _build_train_frame(
+    y_train: Iterable[float],
+) -> pd.DataFrame:
+    y_train_arr = _as_array(y_train)
+    n = y_train_arr.shape[0]
+    return pd.DataFrame(
+        {
+            "unique_id": ["series"] * n,
+            "ds": pd.date_range("1999-01-01", periods=n, freq="D"),
+            "y": y_train_arr,
+        }
+    )
+
+
+def _summary_to_metrics(summary_df: pd.DataFrame, model_name: str = "model") -> dict[str, float]:
+    if summary_df.empty:
+        return {}
+    df = summary_df
+    if "model" in df.columns:
+        df = df[df["model"] == model_name]
+    if df.empty or "metric" not in df.columns or "value" not in df.columns:
+        return {}
+    return {row["metric"]: float(row["value"]) for _, row in df.iterrows()}
+
+
+def wape(y_true: Iterable[float], y_pred: Iterable[float]) -> float:
+    """Weighted Absolute Percentage Error (utilsforecast ND)."""
+    _warn_deprecated("wape")
+    df = _build_eval_frame(y_true, y_pred)
+    _, summary = evaluate_forecasts(df)
+    metrics = _summary_to_metrics(summary.df)
+    return float(metrics.get("wape", np.nan))
+
+
+def smape(y_true: Iterable[float], y_pred: Iterable[float]) -> float:
+    """Symmetric Mean Absolute Percentage Error."""
+    _warn_deprecated("smape")
+    df = _build_eval_frame(y_true, y_pred)
+    _, summary = evaluate_forecasts(df)
+    metrics = _summary_to_metrics(summary.df)
+    return float(metrics.get("smape", np.nan))
+
+
+def mase(
+    y_true: Iterable[float],
+    y_pred: Iterable[float],
+    y_train: Iterable[float],
+    season_length: int = 1,
+) -> float:
+    """Mean Absolute Scaled Error."""
+    _warn_deprecated("mase")
+    df = _build_eval_frame(y_true, y_pred)
+    train_df = _build_train_frame(y_train)
+    _, summary = evaluate_forecasts(df, train_df=train_df, season_length=season_length)
+    metrics = _summary_to_metrics(summary.df)
+    value = float(metrics.get("mase", np.nan))
+    if np.isinf(value):
+        return float("nan")
+    return value
+
+
+def pinball_loss(
+    y_true: Iterable[float],
+    y_quantile: Iterable[float],
+    tau: float,
+) -> float:
+    """Pinball Loss for quantile forecasts."""
+    _warn_deprecated("pinball_loss")
+    df = _build_eval_frame(y_true, y_quantile, y_quantiles={tau: y_quantile})
+    _, summary = evaluate_forecasts(df)
+    metrics = _summary_to_metrics(summary.df)
+    return float(metrics.get(f"pinball_{tau:.3f}", np.nan))
+
+
+def wql(y_true: Iterable[float], y_quantiles: dict[float, Iterable[float]]) -> float:
+    """Weighted Quantile Loss (average pinball loss across quantiles)."""
+    _warn_deprecated("wql")
+    df = _build_eval_frame(y_true, y_true, y_quantiles=y_quantiles)
+    _, summary = evaluate_forecasts(df)
+    metrics = _summary_to_metrics(summary.df)
+    return float(metrics.get("wql", np.nan))
+
+
+def mae(y_true: Iterable[float], y_pred: Iterable[float]) -> float:
+    """Mean Absolute Error."""
+    _warn_deprecated("mae")
+    df = _build_eval_frame(y_true, y_pred)
+    _, summary = evaluate_forecasts(df)
+    metrics = _summary_to_metrics(summary.df)
+    return float(metrics.get("mae", np.nan))
+
+
+def rmse(y_true: Iterable[float], y_pred: Iterable[float]) -> float:
+    """Root Mean Squared Error."""
+    _warn_deprecated("rmse")
+    df = _build_eval_frame(y_true, y_pred)
+    _, summary = evaluate_forecasts(df)
+    metrics = _summary_to_metrics(summary.df)
+    return float(metrics.get("rmse", np.nan))
+
+
+def compute_all_metrics(
+    y_true: Iterable[float],
+    y_pred: Iterable[float],
+    y_train: Iterable[float] | None = None,
+    season_length: int = 1,
+    y_quantiles: dict[float, Iterable[float]] | None = None,
+) -> dict[str, float]:
+    """Compute all standard metrics via utilsforecast.evaluate."""
+    _warn_deprecated("compute_all_metrics")
+    df = _build_eval_frame(y_true, y_pred, y_quantiles=y_quantiles)
+    train_df = _build_train_frame(y_train) if y_train is not None else None
+    _, summary = evaluate_forecasts(
+        df,
+        train_df=train_df,
+        season_length=season_length if train_df is not None else None,
+    )
+    metrics = _summary_to_metrics(summary.df)
+
+    if "mase" not in metrics:
+        metrics["mase"] = float("nan")
+
+    if y_quantiles:
+        for q in y_quantiles:
+            source_key = f"pinball_{q:.3f}"
+            target_key = f"pinball_{q:.2f}"
+            if source_key in metrics and target_key not in metrics:
+                metrics[target_key] = metrics[source_key]
+
+    return metrics
+
+
+def compute_metrics_by_series(
+    df: pd.DataFrame,
+    id_col: str = "unique_id",
+    actual_col: str = "y",
+    pred_col: str = "yhat",
+) -> dict[str, dict[str, float]]:
+    """Compute metrics for each series separately via utilsforecast.evaluate."""
+    _warn_deprecated("compute_metrics_by_series")
+    if df.empty:
+        return {}
+
+    working = df.copy()
+    if "model" not in working.columns:
+        working["model"] = "model"
+    if "ds" not in working.columns:
+        working["ds"] = working.groupby(id_col).cumcount()
+
+    metric_frame, _ = evaluate_forecasts(
+        working,
+        id_col=id_col,
+        ds_col="ds",
+        target_col=actual_col,
+        model_col="model",
+        pred_col=pred_col,
+        cutoff_col=None,
+    )
+
+    metrics_df = metric_frame.df
+    if metrics_df is None or metrics_df.empty:
+        return {}
+
+    if "model" in metrics_df.columns:
+        metrics_df = metrics_df[metrics_df["model"] == "model"]
+
+    if id_col not in metrics_df.columns or "metric" not in metrics_df.columns:
+        return {}
+
+    grouped = metrics_df.groupby([id_col, "metric"])["value"].mean().reset_index()
+    result: dict[str, dict[str, float]] = {}
+    for uid, group in grouped.groupby(id_col):
+        result[str(uid)] = {
+            row["metric"]: float(row["value"]) for _, row in group.iterrows()
+        }
+
+    return result

tsagentkit/backtest/report.py

@@ -0,0 +1,342 @@
+"""Backtest report structures.
+
+Defines data classes for backtest results and diagnostics.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+import pandas as pd
+
+from tsagentkit.contracts import CVFrame
+
+
+@dataclass(frozen=True)
+class WindowResult:
+    """Results from a single backtest window.
+
+    Attributes:
+        window_index: Index of the window (0-based)
+        train_start: Start date of training set
+        train_end: End date of training set
+        test_start: Start date of test set
+        test_end: End date of test set
+        metrics: Dictionary of metrics for this window
+        num_series: Number of series evaluated
+        num_observations: Number of observations in test set
+    """
+
+    window_index: int
+    train_start: str
+    train_end: str
+    test_start: str
+    test_end: str
+    metrics: dict[str, float] = field(default_factory=dict)
+    num_series: int = 0
+    num_observations: int = 0
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary."""
+        return {
+            "window_index": self.window_index,
+            "train_start": self.train_start,
+            "train_end": self.train_end,
+            "test_start": self.test_start,
+            "test_end": self.test_end,
+            "metrics": self.metrics,
+            "num_series": self.num_series,
+            "num_observations": self.num_observations,
+        }
+
+
+@dataclass(frozen=True)
+class SeriesMetrics:
+    """Metrics aggregated by series.
+
+    Attributes:
+        series_id: Unique identifier for the series
+        metrics: Dictionary of metric name to value
+        num_windows: Number of windows this series appeared in
+    """
+
+    series_id: str
+    metrics: dict[str, float] = field(default_factory=dict)
+    num_windows: int = 0
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary."""
+        return {
+            "series_id": self.series_id,
+            "metrics": self.metrics,
+            "num_windows": self.num_windows,
+        }
+
+
+@dataclass(frozen=True)
+class SegmentMetrics:
+    """Metrics aggregated by segment (e.g., sparsity class).
+
+    Attributes:
+        segment_name: Name of the segment (e.g., "intermittent", "regular")
+        series_ids: List of series IDs in this segment
+        metrics: Dictionary of metric name to value
+        n_series: Number of series in this segment
+    """
+
+    segment_name: str
+    series_ids: list[str] = field(default_factory=list)
+    metrics: dict[str, float] = field(default_factory=dict)
+    n_series: int = 0
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary."""
+        return {
+            "segment_name": self.segment_name,
+            "series_ids": self.series_ids,
+            "metrics": self.metrics,
+            "n_series": self.n_series,
+        }
+
+
+@dataclass(frozen=True)
+class TemporalMetrics:
+    """Metrics aggregated by temporal dimension.
+
+    Attributes:
+        dimension: Temporal dimension (e.g., "hour", "dayofweek")
+        metrics_by_value: Dictionary mapping dimension value to metrics
+    """
+
+    dimension: str
+    metrics_by_value: dict[str, dict[str, float]] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary."""
+        return {
+            "dimension": self.dimension,
+            "metrics_by_value": self.metrics_by_value,
+        }
+
+
+@dataclass(frozen=True)
+class BacktestReport:
+    """Complete backtest results.
+
+    Contains window-level results, aggregate metrics, and series-level
+    diagnostics for a backtest run.
+
+    Attributes:
+        n_windows: Number of backtest windows
+        strategy: Window strategy ("expanding" or "sliding")
+        window_results: List of results per window
+        aggregate_metrics: Metrics aggregated across all windows
+        series_metrics: Metrics aggregated by series
+        segment_metrics: Metrics aggregated by segment (sparsity class)
+        temporal_metrics: Metrics aggregated by temporal dimension
+        errors: List of errors encountered during backtest
+        metadata: Additional backtest metadata
+    """
+
+    n_windows: int
+    strategy: str
+    window_results: list[WindowResult] = field(default_factory=list)
+    aggregate_metrics: dict[str, float] = field(default_factory=dict)
+    series_metrics: dict[str, SeriesMetrics] = field(default_factory=dict)
+    segment_metrics: dict[str, SegmentMetrics] = field(default_factory=dict)
+    temporal_metrics: dict[str, TemporalMetrics] = field(default_factory=dict)
+    errors: list[dict[str, Any]] = field(default_factory=list)
+    metadata: dict[str, Any] = field(default_factory=dict)
+    cv_frame: pd.DataFrame | CVFrame | None = None
+
+    def _cv_dataframe(self) -> pd.DataFrame | None:
+        if isinstance(self.cv_frame, CVFrame):
+            return self.cv_frame.df
+        return self.cv_frame
+
+    def get_metric(self, metric_name: str) -> float:
+        """Get an aggregate metric by name.
+
+        Args:
+            metric_name: Name of the metric (e.g., "wape", "smape")
+
+        Returns:
+            Metric value or NaN if not found
+        """
+        return self.aggregate_metrics.get(metric_name, float("nan"))
+
+    def get_series_metric(self, series_id: str, metric_name: str) -> float:
+        """Get a metric for a specific series.
+
+        Args:
+            series_id: Series identifier
+            metric_name: Name of the metric
+
+        Returns:
+            Metric value or NaN if not found
+        """
+        if series_id not in self.series_metrics:
+            return float("nan")
+        return self.series_metrics[series_id].metrics.get(metric_name, float("nan"))
+
+    def get_best_series(self, metric_name: str = "wape") -> str | None:
+        """Get the series with the best performance for a metric.
+
+        Args:
+            metric_name: Metric to optimize (lower is better)
+
+        Returns:
+            Series ID with lowest metric value, or None if no data
+        """
+        if not self.series_metrics:
+            return None
+
+        best_id = None
+        best_value = float("inf")
+
+        for sid, sm in self.series_metrics.items():
+            value = sm.metrics.get(metric_name, float("inf"))
+            if value < best_value:
+                best_value = value
+                best_id = sid
+
+        return best_id
+
+    def get_worst_series(self, metric_name: str = "wape") -> str | None:
+        """Get the series with the worst performance for a metric.
+
+        Args:
+            metric_name: Metric to check (higher is worse)
+
+        Returns:
+            Series ID with highest metric value, or None if no data
+        """
+        if not self.series_metrics:
+            return None
+
+        worst_id = None
+        worst_value = float("-inf")
+
+        for sid, sm in self.series_metrics.items():
+            value = sm.metrics.get(metric_name, float("-inf"))
+            if value > worst_value and not pd.isna(value):
+                worst_value = value
+                worst_id = sid
+
+        return worst_id
+
+    def get_segment_metric(self, segment_name: str, metric_name: str) -> float:
+        """Get a metric for a specific segment.
+
+        Args:
+            segment_name: Segment name (e.g., "intermittent", "regular")
+            metric_name: Name of the metric
+
+        Returns:
+            Metric value or NaN if not found
+        """
+        if segment_name not in self.segment_metrics:
+            return float("nan")
+        return self.segment_metrics[segment_name].metrics.get(metric_name, float("nan"))
+
+    def get_temporal_metric(self, dimension: str, value: str, metric_name: str) -> float:
+        """Get a metric for a specific temporal dimension value.
+
+        Args:
+            dimension: Temporal dimension ("hour", "dayofweek", etc.)
+            value: Dimension value (e.g., "0" for Sunday, "14" for 2pm)
+            metric_name: Name of the metric
+
+        Returns:
+            Metric value or NaN if not found
+        """
+        if dimension not in self.temporal_metrics:
+            return float("nan")
+        dim_metrics = self.temporal_metrics[dimension]
+        if value not in dim_metrics.metrics_by_value:
+            return float("nan")
+        return dim_metrics.metrics_by_value[value].get(metric_name, float("nan"))
+
+    def compare_segments(self, metric_name: str = "wape") -> dict[str, float]:
+        """Compare a metric across all segments.
+
+        Args:
+            metric_name: Metric to compare
+
+        Returns:
+            Dictionary mapping segment name to metric value
+        """
+        return {
+            name: sm.metrics.get(metric_name, float("nan"))
+            for name, sm in self.segment_metrics.items()
+        }
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary for serialization."""
+        return {
+            "n_windows": self.n_windows,
+            "strategy": self.strategy,
+            "aggregate_metrics": self.aggregate_metrics,
+            "series_metrics": {
+                k: v.to_dict() for k, v in self.series_metrics.items()
+            },
+            "segment_metrics": {
+                k: v.to_dict() for k, v in self.segment_metrics.items()
+            },
+            "temporal_metrics": {
+                k: v.to_dict() for k, v in self.temporal_metrics.items()
+            },
+            "window_results": [w.to_dict() for w in self.window_results],
+            "errors": self.errors,
+            "metadata": self.metadata,
+            "cv_frame": (
+                self._cv_dataframe().to_dict("records")
+                if self._cv_dataframe() is not None
+                else None
+            ),
+        }
+
+    def summary(self) -> str:
+        """Generate a human-readable summary.
+
+        Returns:
+            Summary string with key metrics
+        """
+        lines = [
+            f"Backtest Report: {self.n_windows} windows ({self.strategy})",
+            "=" * 50,
+        ]
+
+        # Aggregate metrics
+        lines.append("\nAggregate Metrics:")
+        for name, value in sorted(self.aggregate_metrics.items()):
+            lines.append(f"  {name}: {value:.4f}")
+
+        # Segment metrics
+        if self.segment_metrics:
+            lines.append("\nSegment Metrics:")
+            for name, sm in sorted(self.segment_metrics.items()):
+                metrics_str = ", ".join(
+                    f"{k}={v:.4f}" for k, v in sorted(sm.metrics.items())[:2]
+                )
+                lines.append(f"  {name} ({sm.n_series} series): {metrics_str}")
+
+        # Temporal metrics
+        if self.temporal_metrics:
+            lines.append("\nTemporal Metrics:")
+            for dim, tm in sorted(self.temporal_metrics.items()):
+                lines.append(f"  {dim}: {len(tm.metrics_by_value)} unique values")
+
+        # Best/worst series
+        best = self.get_best_series("wape")
+        worst = self.get_worst_series("wape")
+        if best and worst:
+            lines.append(f"\nBest Series: {best}")
+            lines.append(f"Worst Series: {worst}")
+
+        # Errors
+        if self.errors:
+            lines.append(f"\nErrors: {len(self.errors)}")
+
+        return "\n".join(lines)