tsagentkit 1.0.2__py3-none-any.whl

tsagentkit/eval/__init__.py

@@ -0,0 +1,285 @@
+"""Evaluation utilities for forecast metrics and summaries."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from functools import partial
+from typing import Any
+
+import numpy as np
+import pandas as pd
+
+from tsagentkit.utils import parse_quantile_column
+
+
+@dataclass(frozen=True)
+class MetricFrame:
+    """Container for metric results."""
+
+    df: pd.DataFrame
+
+
+@dataclass(frozen=True)
+class ScoreSummary:
+    """Aggregate metric summary."""
+
+    df: pd.DataFrame
+
+
+def _maybe_import_utilsforecast():
+    try:
+        from utilsforecast import evaluation as ufeval
+        from utilsforecast import losses as uflosses
+    except Exception:
+        return None, None
+    return ufeval, uflosses
+
+
+def _wide_predictions(
+    df: pd.DataFrame,
+    id_col: str,
+    ds_col: str,
+    target_col: str,
+    model_col: str,
+    pred_col: str,
+    cutoff_col: str | None,
+) -> tuple[pd.DataFrame, list[str], dict[str, dict[float, str]]]:
+    index_cols = [id_col, ds_col]
+    if cutoff_col and cutoff_col in df.columns:
+        index_cols.append(cutoff_col)
+    pivot = df.pivot_table(
+        index=index_cols,
+        columns=model_col,
+        values=pred_col,
+        aggfunc="mean",
+    )
+    wide = pivot.reset_index()
+    actuals = df[index_cols + [target_col]].drop_duplicates(subset=index_cols)
+    wide = wide.merge(actuals, on=index_cols, how="left")
+    model_cols = [c for c in wide.columns if c not in index_cols + [target_col]]
+    quantile_cols = [c for c in df.columns if parse_quantile_column(c) is not None]
+    quantile_map: dict[str, dict[float, str]] = {}
+
+    if quantile_cols:
+        for q_col in quantile_cols:
+            q = parse_quantile_column(q_col)
+            if q is None:
+                continue
+            q_pivot = df.pivot_table(
+                index=index_cols,
+                columns=model_col,
+                values=q_col,
+                aggfunc="mean",
+            ).reset_index()
+            rename_map: dict[str, str] = {}
+            for col in q_pivot.columns:
+                if col in index_cols:
+                    continue
+                new_col = f"{col}__{q_col}"
+                rename_map[col] = new_col
+                quantile_map.setdefault(col, {})[q] = new_col
+            if rename_map:
+                q_pivot = q_pivot.rename(columns=rename_map)
+                wide = wide.merge(q_pivot, on=index_cols, how="left")
+
+    return wide, model_cols, quantile_map
+
+
+def _wrap_metric_name(func: Any, name: str) -> Any:
+    func.__name__ = name
+    return func
+
+
+def _make_wape_metric(uflosses: Any, cutoff_col: str) -> Any:
+    def _metric(
+        df: pd.DataFrame,
+        models: list[str],
+        id_col: str = "unique_id",
+        target_col: str = "y",
+        **_: Any,
+    ) -> pd.DataFrame:
+        return uflosses.nd(
+            df=df,
+            models=models,
+            id_col=id_col,
+            target_col=target_col,
+            cutoff_col=cutoff_col,
+        )
+
+    return _wrap_metric_name(_metric, "wape")
+
+
+def _make_quantile_loss_metric(
+    uflosses: Any,
+    q: float,
+    quantile_models: dict[str, str],
+    cutoff_col: str,
+) -> Any:
+    def _metric(
+        df: pd.DataFrame,
+        models: list[str],
+        id_col: str = "unique_id",
+        target_col: str = "y",
+        **_: Any,
+    ) -> pd.DataFrame:
+        return uflosses.quantile_loss(
+            df=df,
+            models=quantile_models,
+            q=q,
+            id_col=id_col,
+            target_col=target_col,
+            cutoff_col=cutoff_col,
+        )
+
+    return _wrap_metric_name(_metric, f"pinball_{q:.3f}")
+
+
+def _make_wql_metric(
+    uflosses: Any,
+    quantile_models: dict[str, list[str]],
+    quantiles: np.ndarray,
+    cutoff_col: str,
+) -> Any:
+    def _metric(
+        df: pd.DataFrame,
+        models: list[str],
+        id_col: str = "unique_id",
+        target_col: str = "y",
+        **_: Any,
+    ) -> pd.DataFrame:
+        return uflosses.mqloss(
+            df=df,
+            models=quantile_models,
+            quantiles=quantiles,
+            id_col=id_col,
+            target_col=target_col,
+            cutoff_col=cutoff_col,
+        )
+
+    return _wrap_metric_name(_metric, "wql")
+
+
+def evaluate_forecasts(
+    df: pd.DataFrame,
+    train_df: pd.DataFrame | None = None,
+    season_length: int | None = None,
+    id_col: str = "unique_id",
+    ds_col: str = "ds",
+    target_col: str = "y",
+    model_col: str = "model",
+    pred_col: str = "yhat",
+    cutoff_col: str | None = "cutoff",
+) -> tuple[MetricFrame, ScoreSummary]:
+    """Compute point + quantile metrics in a stable long schema."""
+    if df.empty:
+        return MetricFrame(pd.DataFrame()), ScoreSummary(pd.DataFrame())
+
+    if model_col not in df.columns:
+        df = df.copy()
+        df[model_col] = "model"
+
+    wide, model_cols, quantile_map = _wide_predictions(
+        df,
+        id_col=id_col,
+        ds_col=ds_col,
+        target_col=target_col,
+        model_col=model_col,
+        pred_col=pred_col,
+        cutoff_col=cutoff_col,
+    )
+
+    ufeval, uflosses = _maybe_import_utilsforecast()
+    if ufeval is None or uflosses is None or not model_cols:
+        return MetricFrame(pd.DataFrame()), ScoreSummary(pd.DataFrame())
+
+    cutoff_present = cutoff_col is not None and cutoff_col in wide.columns
+    cutoff_name = cutoff_col if cutoff_col is not None else "cutoff"
+    wide_eval = wide
+
+    metrics: list[Any] = [uflosses.mae, uflosses.rmse, uflosses.smape]
+    if hasattr(uflosses, "nd"):
+        metrics.append(_make_wape_metric(uflosses, cutoff_name))
+
+    if train_df is not None and season_length and hasattr(uflosses, "mase"):
+        mase_metric = partial(uflosses.mase, seasonality=season_length)
+        metrics.append(_wrap_metric_name(mase_metric, "mase"))
+
+    if quantile_map and hasattr(uflosses, "quantile_loss"):
+        available_models = [model for model in model_cols if model in quantile_map]
+        if available_models:
+            common_quantiles = set.intersection(
+                *[
+                    set(quantile_map[model].keys())
+                    for model in available_models
+                ]
+            )
+        else:
+            common_quantiles = set()
+
+        if common_quantiles:
+            quantiles_sorted = sorted(common_quantiles)
+            for q in quantiles_sorted:
+                per_q_models = {
+                    model: quantile_map[model][q]
+                    for model in available_models
+                    if q in quantile_map[model]
+                }
+                if per_q_models:
+                    metrics.append(
+                        _make_quantile_loss_metric(
+                            uflosses=uflosses,
+                            q=q,
+                            quantile_models=per_q_models,
+                            cutoff_col=cutoff_name,
+                        )
+                    )
+
+            per_model_quantiles = {
+                model: [quantile_map[model][q] for q in quantiles_sorted]
+                for model in available_models
+                if all(q in quantile_map[model] for q in quantiles_sorted)
+            }
+            if per_model_quantiles and hasattr(uflosses, "mqloss"):
+                metrics.append(
+                    _make_wql_metric(
+                        uflosses=uflosses,
+                        quantile_models=per_model_quantiles,
+                        quantiles=np.asarray(quantiles_sorted, dtype=float),
+                        cutoff_col=cutoff_name,
+                    )
+                )
+
+    metric_df = ufeval.evaluate(
+        wide_eval,
+        metrics=metrics,
+        models=model_cols,
+        train_df=train_df,
+        id_col=id_col,
+        time_col=ds_col,
+        target_col=target_col,
+        cutoff_col=cutoff_name,
+    )
+
+    metrics_long = metric_df.copy()
+    index_cols = [id_col]
+    if cutoff_present and cutoff_col and cutoff_col in metrics_long.columns:
+        index_cols.append(cutoff_col)
+
+    metric_cols = [c for c in metrics_long.columns if c not in index_cols + ["metric"]]
+    metrics_long = metrics_long.melt(
+        id_vars=index_cols + ["metric"],
+        value_vars=metric_cols,
+        var_name="model",
+        value_name="value",
+    )
+
+    summary = (
+        metrics_long.groupby(["model", "metric"])["value"]
+        .mean()
+        .reset_index()
+    )
+
+    return MetricFrame(metrics_long), ScoreSummary(summary)
+
+
+__all__ = ["MetricFrame", "ScoreSummary", "evaluate_forecasts"]

tsagentkit/features/__init__.py

@@ -0,0 +1,20 @@
+"""Feature engineering module for time series forecasting.
+
+Provides point-in-time safe feature engineering with full versioning support.
+"""
+
+from __future__ import annotations
+
+from tsagentkit.features.covariates import CovariateManager, CovariatePolicy
+from tsagentkit.features.factory import FeatureConfig, FeatureFactory
+from tsagentkit.features.matrix import FeatureMatrix
+from tsagentkit.features.versioning import compute_feature_hash
+
+__all__ = [
+    "FeatureMatrix",
+    "FeatureFactory",
+    "FeatureConfig",
+    "CovariateManager",
+    "CovariatePolicy",
+    "compute_feature_hash",
+]

tsagentkit/features/covariates.py

@@ -0,0 +1,328 @@
+"""Covariate management for known vs observed covariates with leakage protection."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from enum import Enum
+from typing import TYPE_CHECKING
+
+import pandas as pd
+
+from tsagentkit.contracts.errors import ECovariateLeakage
+
+if TYPE_CHECKING:
+    pass
+
+
+class CovariatePolicy(Enum):
+    """Policy for handling different covariate types.
+
+    - KNOWN: Covariates known for all time steps (e.g., holidays, promotions planned in advance)
+    - OBSERVED: Covariates only observed up to current time (e.g., actual sales, weather)
+    """
+
+    KNOWN = "known"
+    OBSERVED = "observed"
+
+
+@dataclass(frozen=True)
+class CovariateConfig:
+    """Configuration specifying covariate types.
+
+    Attributes:
+        known: List of column names for known covariates
+        observed: List of column names for observed covariates
+
+    Example:
+        >>> config = CovariateConfig(
+        ...     known=["holiday", "promotion_planned"],
+        ...     observed=["competitor_price", "weather"],
+        ... )
+    """
+
+    known: list[str] = field(default_factory=list)
+    observed: list[str] = field(default_factory=list)
+
+    def __post_init__(self) -> None:
+        """Validate no overlap between known and observed."""
+        overlap = set(self.known) & set(self.observed)
+        if overlap:
+            raise ValueError(f"Covariates cannot be both known and observed: {overlap}")
+
+    def get_policy(self, column: str) -> CovariatePolicy | None:
+        """Get the policy for a specific column.
+
+        Args:
+            column: Column name to check
+
+        Returns:
+            CovariatePolicy or None if column is not a covariate
+        """
+        if column in self.known:
+            return CovariatePolicy.KNOWN
+        elif column in self.observed:
+            return CovariatePolicy.OBSERVED
+        return None
+
+    def all_covariates(self) -> list[str]:
+        """Return all covariate column names."""
+        return self.known + self.observed
+
+
+def infer_covariate_config(
+    df: pd.DataFrame,
+    policy: str,
+    id_col: str = "unique_id",
+    ds_col: str = "ds",
+    target_col: str = "y",
+) -> CovariateConfig:
+    """Infer covariate configuration based on policy and data."""
+    if policy == "ignore":
+        return CovariateConfig()
+
+    covariate_cols = [
+        c for c in df.columns
+        if c not in {id_col, ds_col, target_col}
+    ]
+
+    if not covariate_cols:
+        return CovariateConfig()
+
+    if policy == "known":
+        return CovariateConfig(known=covariate_cols, observed=[])
+    if policy == "observed":
+        return CovariateConfig(known=[], observed=covariate_cols)
+
+    # Auto policy: infer based on future rows (y is null)
+    future_mask = df[target_col].isna() if target_col in df.columns else None
+    known: list[str] = []
+    observed: list[str] = []
+
+    for col in covariate_cols:
+        if future_mask is not None and future_mask.any():
+            has_future_values = df.loc[future_mask, col].notna().any()
+            if has_future_values:
+                known.append(col)
+            else:
+                observed.append(col)
+        else:
+            # Default to observed if we can't see future values
+            observed.append(col)
+
+    return CovariateConfig(known=known, observed=observed)
+
+
+class CovariateManager:
+    """Manage known vs observed covariates with leakage protection.
+
+    This class ensures that observed covariates are properly handled to prevent
+    future information from leaking into training or predictions.
+
+    Example:
+        >>> manager = CovariateManager(
+        ...     known_covariates=["holiday"],
+        ...     observed_covariates=["promotion"],
+        ... )
+        >>>
+        >>> # Validate no leakage
+        >>> manager.validate_for_prediction(
+        ...     df, forecast_start=datetime(2024, 1, 1), horizon=7
+        ... )
+        >>>
+        >>> # Mask observed covariates for training
+        >>> train_df = manager.mask_observed_for_training(df, target_col="y")
+    """
+
+    def __init__(
+        self,
+        known_covariates: list[str] | None = None,
+        observed_covariates: list[str] | None = None,
+    ):
+        """Initialize the covariate manager.
+
+        Args:
+            known_covariates: Columns known for all time steps
+            observed_covariates: Columns only observed up to current time
+        """
+        self.known_covariates = known_covariates or []
+        self.observed_covariates = observed_covariates or []
+
+        # Check for overlap
+        overlap = set(self.known_covariates) & set(self.observed_covariates)
+        if overlap:
+            raise ValueError(f"Covariates cannot be both known and observed: {overlap}")
+
+    def validate_for_prediction(
+        self,
+        df: pd.DataFrame,
+        forecast_start: datetime,
+        horizon: int,
+        ds_col: str = "ds",
+    ) -> None:
+        """Validate that observed covariates don't leak future information.
+
+        This checks that observed covariates do not have values beyond the
+        forecast start time, which would indicate future information leakage.
+
+        Args:
+            df: DataFrame with covariates
+            forecast_start: Start time of the forecast period
+            horizon: Forecast horizon
+            ds_col: Name of the timestamp column
+
+        Raises:
+            ECovariateLeakage: If observed covariates extend beyond forecast_start
+
+        Example:
+            >>> manager = CovariateManager(observed_covariates=["promo"])
+            >>> # This will raise if promo has values after 2024-01-01
+            >>> manager.validate_for_prediction(df, datetime(2024, 1, 1), horizon=7)
+        """
+        if not self.observed_covariates:
+            return
+
+        # Check each observed covariate
+        for col in self.observed_covariates:
+            if col not in df.columns:
+                continue
+
+            # Find rows where observed covariate has non-null values beyond forecast_start
+            future_mask = (df[ds_col] >= forecast_start) & df[col].notna()
+            future_count = future_mask.sum()
+
+            if future_count > 0:
+                raise ECovariateLeakage(
+                    f"Observed covariate '{col}' has {future_count} values "
+                    f"at or after forecast start time {forecast_start}. "
+                    "Observed covariates cannot be known in advance.",
+                    context={
+                        "covariate": col,
+                        "forecast_start": forecast_start.isoformat(),
+                        "future_values_count": int(future_count),
+                    },
+                )
+
+    def mask_observed_for_training(
+        self,
+        df: pd.DataFrame,
+        target_col: str = "y",
+        ds_col: str = "ds",
+        unique_id_col: str = "unique_id",
+    ) -> pd.DataFrame:
+        """Mask observed covariates at time t to prevent leakage during training.
+
+        For observed covariates, we should only use values that would be available
+        at prediction time. This means observed covariates at time t should be
+        lagged (using values from before t) to prevent leakage.
+
+        By default, this sets observed covariates to null for the target timestamp
+        to ensure proper training. The caller is responsible for creating lagged
+        versions of observed covariates before calling this method.
+
+        Args:
+            df: DataFrame with covariates
+            target_col: Name of target column
+            ds_col: Name of timestamp column
+            unique_id_col: Name of unique_id column
+
+        Returns:
+            DataFrame with observed covariates masked at target time
+        """
+        if not self.observed_covariates:
+            return df.copy()
+
+        df = df.copy()
+
+        # For training, we mask observed covariates at the prediction time
+        # since they wouldn't be known yet. The model should use lagged versions.
+        for col in self.observed_covariates:
+            if col in df.columns:
+                # Set to null - caller should create lagged features
+                df[col] = None
+
+        return df
+
+    def create_lagged_observed_features(
+        self,
+        df: pd.DataFrame,
+        lags: list[int],
+        ds_col: str = "ds",
+        unique_id_col: str = "unique_id",
+    ) -> pd.DataFrame:
+        """Create lagged versions of observed covariates.
+
+        This creates lagged features for observed covariates to ensure
+        point-in-time correctness. For a horizon h, observed covariates
+        should be lagged by at least h to prevent leakage.
+
+        Args:
+            df: DataFrame with covariates
+            lags: List of lag periods to create
+            ds_col: Name of timestamp column
+            unique_id_col: Name of unique_id column
+
+        Returns:
+            DataFrame with added lagged observed covariate columns
+        """
+        if not self.observed_covariates or not lags:
+            return df.copy()
+
+        df = df.copy()
+
+        for col in self.observed_covariates:
+            if col not in df.columns:
+                continue
+
+            for lag in lags:
+                lag_col = f"{col}_lag_{lag}"
+                df[lag_col] = (
+                    df.groupby(unique_id_col)[col]
+                    .shift(lag)
+                    .values
+                )
+
+        return df
+
+    def separate_covariates_for_prediction(
+        self,
+        df: pd.DataFrame,
+        forecast_start: datetime,
+        ds_col: str = "ds",
+    ) -> tuple[pd.DataFrame, pd.DataFrame]:
+        """Separate known and observed covariates for prediction setup.
+
+        Returns two DataFrames:
+        1. Known covariates: Can be used directly (values known for all time steps)
+        2. Observed covariates: Should be masked/handle carefully
+
+        Args:
+            df: DataFrame with covariates
+            forecast_start: Start of forecast period
+            ds_col: Name of timestamp column
+
+        Returns:
+            Tuple of (known_covariates_df, observed_covariates_df)
+        """
+        all_cols = ["unique_id", ds_col]
+
+        known_cols = all_cols + [
+            col for col in self.known_covariates if col in df.columns
+        ]
+        observed_cols = all_cols + [
+            col for col in self.observed_covariates if col in df.columns
+        ]
+
+        known_df = df[known_cols].copy() if len(known_cols) > 2 else pd.DataFrame()
+        observed_df = (
+            df[observed_cols].copy() if len(observed_cols) > 2 else pd.DataFrame()
+        )
+
+        return known_df, observed_df
+
+    def get_config(self) -> CovariateConfig:
+        """Get covariate configuration."""
+        return CovariateConfig(
+            known=self.known_covariates,
+            observed=self.observed_covariates,
+        )

tsagentkit/features/extra/__init__.py

@@ -0,0 +1,5 @@
+"""Extra (non-default) feature engineering backends."""
+
+from .native import build_native_feature_matrix
+
+__all__ = ["build_native_feature_matrix"]