autogluon.timeseries 0.8.3b20231024__tar.gz → 0.8.3b20231027__tar.gz

{autogluon.timeseries-0.8.3b20231024 → autogluon.timeseries-0.8.3b20231027}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: autogluon.timeseries
-Version: 0.8.3b20231024
+Version: 0.8.3b20231027
 Summary: AutoML for Image, Text, and Tabular Data
 Home-page: https://github.com/autogluon/autogluon
 Author: AutoGluon Community

{autogluon.timeseries-0.8.3b20231024 → autogluon.timeseries-0.8.3b20231027}/setup.py

@@ -33,7 +33,8 @@ install_requires = [
     "networkx",  # version range defined in `core/_setup_utils.py`
     # TODO: update statsforecast to v1.5.0 - resolve antlr4-python3-runtime dependency clash with multimodal
     "statsforecast>=1.4.0,<1.5",
-    "mlforecast>=0.9.3,<0.9.4",
+    "mlforecast>=0.10.0,<0.10.1",
+    "utilsforecast>=0.0.10,<0.0.11",
     "tqdm",  # version range defined in `core/_setup_utils.py`
     "ujson>=5,<6",  # needed to silence GluonTS warning
     f"autogluon.core[raytune]=={version}",

autogluon.timeseries-0.8.3b20231027/src/autogluon/timeseries/evaluator.py

@@ -0,0 +1,58 @@
+from typing import Optional
+
+from autogluon.common.utils.deprecated_utils import Deprecated
+from autogluon.timeseries import TimeSeriesDataFrame
+from autogluon.timeseries.metrics import AVAILABLE_METRICS, check_get_evaluation_metric
+
+
+@Deprecated(
+    min_version_to_warn="1.0",
+    min_version_to_error="1.1",
+    custom_warning_msg="Please use the metrics defined in autogluon.timeseries.metrics instead.",
+)
+class TimeSeriesEvaluator:
+    """This class has been deprecated in AutoGluon v1.0 and is only provided for backward compatibility!"""
+
+    METRIC_COEFFICIENTS = {metric_name: metric_cls().sign for metric_name, metric_cls in AVAILABLE_METRICS.items()}
+    AVAILABLE_METRICS = list(AVAILABLE_METRICS.keys())
+    DEFAULT_METRIC = check_get_evaluation_metric(None).name
+
+    def __init__(
+        self,
+        eval_metric: str,
+        prediction_length: int,
+        target_column: str = "target",
+        eval_metric_seasonal_period: Optional[int] = None,
+    ):
+        self.eval_metric = check_get_evaluation_metric(eval_metric)
+        self.prediction_length = prediction_length
+        self.target_column = target_column
+        self.seasonal_period = eval_metric_seasonal_period
+
+    @property
+    def coefficient(self) -> int:
+        return self.eval_metric.sign
+
+    @property
+    def higher_is_better(self) -> bool:
+        return self.eval_metric.greater_is_better_internal
+
+    @staticmethod
+    def check_get_evaluation_metric(
+        metric_name: Optional[str] = None,
+        raise_if_not_available: bool = True,
+    ):
+        return check_get_evaluation_metric(metric_name)
+
+    def __call__(self, data: TimeSeriesDataFrame, predictions: TimeSeriesDataFrame) -> float:
+        quantile_levels = [float(col) for col in predictions.columns if col != "mean"]
+        score = self.eval_metric(
+            data=data,
+            predictions=predictions,
+            prediction_length=self.prediction_length,
+            target=self.target_column,
+            seasonal_period=self.seasonal_period,
+            quantile_levels=quantile_levels,
+        )
+        # Return raw metric in lower-is-better format to match the old Evaluator API
+        return score * self.eval_metric.sign

{autogluon.timeseries-0.8.3b20231024 → autogluon.timeseries-0.8.3b20231027}/src/autogluon/timeseries/learner.py

@@ -7,7 +7,7 @@ import pandas as pd
 
 from autogluon.core.learner import AbstractLearner
 from autogluon.timeseries.dataset.ts_dataframe import TimeSeriesDataFrame
-from autogluon.timeseries.evaluator import TimeSeriesEvaluator
+from autogluon.timeseries.metrics import TimeSeriesScorer, check_get_evaluation_metric
 from autogluon.timeseries.models.abstract import AbstractTimeSeriesModel
 from autogluon.timeseries.splitter import AbstractWindowSplitter
 from autogluon.timeseries.trainer import AbstractTimeSeriesTrainer, AutoTimeSeriesTrainer
@@ -28,14 +28,14 @@ class TimeSeriesLearner(AbstractLearner):
         target: str = "target",
         known_covariates_names: Optional[List[str]] = None,
         trainer_type: Type[AbstractTimeSeriesTrainer] = AutoTimeSeriesTrainer,
-        eval_metric: Optional[str] = None,
+        eval_metric: Union[str, TimeSeriesScorer, None] = None,
         eval_metric_seasonal_period: Optional[int] = None,
         prediction_length: int = 1,
         cache_predictions: bool = True,
         **kwargs,
     ):
         super().__init__(path_context=path_context)
-        self.eval_metric: str = TimeSeriesEvaluator.check_get_evaluation_metric(eval_metric)
+        self.eval_metric: TimeSeriesScorer = check_get_evaluation_metric(eval_metric)
         self.eval_metric_seasonal_period = eval_metric_seasonal_period
         self.trainer_type = trainer_type
         self.target = target
@@ -89,7 +89,7 @@ class TimeSeriesLearner(AbstractLearner):
         logger.info(f"AutoGluon will save models to {self.path}")
 
         logger.info(f"AutoGluon will gauge predictive performance using evaluation metric: '{self.eval_metric}'")
-        if TimeSeriesEvaluator.METRIC_COEFFICIENTS[self.eval_metric] == -1:
+        if not self.eval_metric.greater_is_better_internal:
             logger.info(
                 "\tThis metric's sign has been flipped to adhere to being 'higher is better'. "
                 "The reported score can be multiplied by -1 to get the metric value.",
@@ -185,7 +185,7 @@ class TimeSeriesLearner(AbstractLearner):
         self,
         data: TimeSeriesDataFrame,
         model: AbstractTimeSeriesModel = None,
-        metric: Optional[str] = None,
+        metric: Union[str, TimeSeriesScorer, None] = None,
         use_cache: bool = True,
     ) -> float:
         data = self.feature_generator.transform(data)

autogluon.timeseries-0.8.3b20231027/src/autogluon/timeseries/metrics/__init__.py

@@ -0,0 +1,58 @@
+import json
+from typing import Type, Union
+
+from .abstract import TimeSeriesScorer
+from .point import MAE, MAPE, MASE, MSE, RMSE, RMSSE, WAPE, sMAPE
+from .quantile import WQL
+
+__all__ = [
+    "MAE",
+    "MAPE",
+    "MASE",
+    "sMAPE",
+    "MSE",
+    "RMSE",
+    "RMSSE",
+    "WAPE",
+    "WQL",
+]
+
+DEFAULT_METRIC_NAME = "WQL"
+
+AVAILABLE_METRICS = {
+    "MASE": MASE,
+    "MAPE": MAPE,
+    "SMAPE": sMAPE,
+    "RMSE": RMSE,
+    "RMSSE": RMSSE,
+    "WAPE": WAPE,
+    "WQL": WQL,
+    # Exist for compatibility
+    "MSE": MSE,
+    "MAE": MAE,
+}
+
+
+def check_get_evaluation_metric(
+    eval_metric: Union[str, TimeSeriesScorer, Type[TimeSeriesScorer], None] = None
+) -> TimeSeriesScorer:
+    if isinstance(eval_metric, TimeSeriesScorer):
+        eval_metric = eval_metric
+    elif isinstance(eval_metric, type) and issubclass(eval_metric, TimeSeriesScorer):
+        # e.g., user passed `eval_metric=CustomMetric` instead of `eval_metric=CustomMetric()`
+        eval_metric = eval_metric()
+    elif isinstance(eval_metric, str):
+        if eval_metric.upper() not in AVAILABLE_METRICS:
+            raise ValueError(
+                f"Time series metric {eval_metric} not supported. Available metrics are:\n"
+                f"{json.dumps(list(AVAILABLE_METRICS.keys()), indent=2)}"
+            )
+        eval_metric = AVAILABLE_METRICS[eval_metric.upper()]()
+    elif eval_metric is None:
+        eval_metric = AVAILABLE_METRICS[DEFAULT_METRIC_NAME]()
+    else:
+        raise ValueError(
+            f"eval_metric must be of type str, TimeSeriesScorer or None "
+            f"(received eval_metric = {eval_metric} of type {type(eval_metric)})"
+        )
+    return eval_metric

autogluon.timeseries-0.8.3b20231027/src/autogluon/timeseries/metrics/abstract.py

@@ -0,0 +1,201 @@
+from typing import Optional, Tuple
+
+import numpy as np
+import pandas as pd
+
+from autogluon.timeseries import TimeSeriesDataFrame
+from autogluon.timeseries.utils.datetime import get_seasonality
+from autogluon.timeseries.utils.warning_filters import warning_filter
+
+
+class TimeSeriesScorer:
+    """Base class for all evaluation metrics used in AutoGluon-TimeSeries.
+
+    This object always returns the metric in greater-is-better format.
+
+    Follows the design of ``autogluon.core.metrics.Scorer``.
+
+    Attributes
+    ----------
+    greater_is_better_internal : bool, default = False
+        Whether internal method :meth:`~autogluon.timeseries.metrics.TimeSeriesScorer.compute_metric` is
+        a loss function (default), meaning low is good, or a score function, meaning high is good.
+    optimum : float, default = 0.0
+        The best score achievable by the score function, i.e. maximum in case of scorer function and minimum in case of
+        loss function.
+    optimized_by_median : bool, default = False
+        Whether given point forecast metric is optimized by the median (if True) or expected value (if False). If True,
+        all models in AutoGluon-TimeSeries will attempt to paste median forecast into the "mean" column.
+    needs_quantile : bool, default = False
+        Whether the given metric uses the quantile predictions. Some models will modify the training procedure if they
+        are trained to optimize a quantile metric.
+    equivalent_tabular_regression_metric : str
+        Name of an equivalent metric used by AutoGluon-Tabular with ``problem_type="regression"``. Used by models that
+        train a TabularPredictor under the hood. This attribute should only be specified by point forecast metrics.
+    """
+
+    greater_is_better_internal: bool = False
+    optimum: float = 0.0
+    optimized_by_median: bool = False
+    needs_quantile: bool = False
+    equivalent_tabular_regression_metric: Optional[str] = None
+
+    @property
+    def sign(self) -> int:
+        return 1 if self.greater_is_better_internal else -1
+
+    @property
+    def name(self) -> str:
+        return f"{self.__class__.__name__}"
+
+    def __repr__(self) -> str:
+        return self.name
+
+    def __str__(self) -> str:
+        return self.name
+
+    @property
+    def name_with_sign(self) -> str:
+        if self.greater_is_better_internal:
+            prefix = ""
+        else:
+            prefix = "-"
+        return f"{prefix}{self.name}"
+
+    def __call__(
+        self,
+        data: TimeSeriesDataFrame,
+        predictions: TimeSeriesDataFrame,
+        prediction_length: int = 1,
+        target: str = "target",
+        seasonal_period: Optional[int] = None,
+        **kwargs,
+    ) -> float:
+        seasonal_period = get_seasonality(data.freq) if seasonal_period is None else seasonal_period
+
+        data_past = data.slice_by_timestep(None, -prediction_length)
+        data_future = data.slice_by_timestep(-prediction_length, None)
+
+        assert (predictions.num_timesteps_per_item() == prediction_length).all()
+        assert data_future.index.equals(predictions.index), "Prediction and data indices do not match."
+
+        try:
+            with warning_filter():
+                self.save_past_metrics(
+                    data_past=data_past,
+                    target=target,
+                    seasonal_period=seasonal_period,
+                    **kwargs,
+                )
+                metric_value = self.compute_metric(
+                    data_future=data_future,
+                    predictions=predictions,
+                    target=target,
+                    **kwargs,
+                )
+        finally:
+            self.clear_past_metrics()
+        return metric_value * self.sign
+
+    score = __call__
+
+    def compute_metric(
+        self,
+        data_future: TimeSeriesDataFrame,
+        predictions: TimeSeriesDataFrame,
+        target: str = "target",
+        **kwargs,
+    ) -> float:
+        """Internal method that computes the metric for given forecast & actual data.
+
+        This method should be implemented by all custom metrics.
+
+        Parameters
+        ----------
+        data_future : TimeSeriesDataFrame
+            Actual values of the time series during the forecast horizon (``prediction_length`` values for each time
+            series in the dataset). This data frame is guaranteed to have the same index as ``predictions``.
+        predictions : TimeSeriesDataFrame
+            Data frame with predictions for the forecast horizon. Contain columns "mean" (point forecast) and the
+            columns corresponding to each of the quantile levels.
+        target : str, default = "target"
+            Name of the column in ``data_future`` that contains the target time series.
+
+        Returns
+        -------
+        score : float
+            Value of the metric for given forecast and data. If self.greater_is_better_internal is True, returns score
+            in greater-is-better format, otherwise in lower-is-better format.
+
+        """
+        raise NotImplementedError
+
+    def save_past_metrics(
+        self,
+        data_past: TimeSeriesDataFrame,
+        target: str = "target",
+        seasonal_period: int = 1,
+        **kwargs,
+    ) -> None:
+        """Compute auxiliary metrics on past data (before forecast horizon), if the chosen metric requires it.
+
+        This method should only be implemented by metrics that rely on historic (in-sample) data, such as Mean Absolute
+        Scaled Error (MASE) https://en.wikipedia.org/wiki/Mean_absolute_scaled_error.
+
+        We keep this method separate from :meth:`compute_metric` to avoid redundant computations when fitting ensemble.
+        """
+        pass
+
+    def clear_past_metrics(self) -> None:
+        """Clear auxiliary metrics saved in :meth:`save_past_metrics`.
+
+        This method should only be implemented if :meth:`save_past_metrics` has been implemented.
+        """
+        pass
+
+    def error(self, *args, **kwargs):
+        """Return error in lower-is-better format."""
+        return self.optimum - self.score(*args, **kwargs)
+
+    @staticmethod
+    def _safemean(series: pd.Series) -> float:
+        """Compute mean of an pd.Series, ignoring inf, -inf and nan values."""
+        return np.nanmean(series.replace([np.inf, -np.inf], np.nan).values)
+
+    @staticmethod
+    def _get_point_forecast_score_inputs(
+        data_future: TimeSeriesDataFrame, predictions: TimeSeriesDataFrame, target: str = "target"
+    ) -> Tuple[pd.Series, pd.Series]:
+        """Get inputs necessary to compute point forecast metrics.
+
+        Returns
+        -------
+        y_true : pd.Series, shape [num_items * prediction_length]
+            Target time series values during the forecast horizon.
+        y_pred : pd.Series, shape [num_items * prediction_length]
+            Predicted time series values during the forecast horizon.
+        """
+        y_true = data_future[target]
+        y_pred = predictions["mean"]
+        return y_true, y_pred
+
+    @staticmethod
+    def _get_quantile_forecast_score_inputs(
+        data_future: TimeSeriesDataFrame, predictions: TimeSeriesDataFrame, target: str = "target"
+    ) -> Tuple[pd.Series, pd.DataFrame, np.ndarray]:
+        """Get inputs necessary to compute quantile forecast metrics.
+
+        Returns
+        -------
+        y_true : pd.Series, shape [num_items * prediction_length]
+            Target time series values during the forecast horizon.
+        q_pred : pd.DataFrame, shape [num_items * prediction_length, num_quantiles]
+            Quantile forecast for each predicted quantile level. Column order corresponds to ``quantile_levels``.
+        quantile_levels : np.ndarray, shape [num_quantiles]
+            Quantile levels for which the forecasts are generated (as floats).
+        """
+        quantile_columns = [col for col in predictions.columns if col != "mean"]
+        y_true = data_future[target]
+        q_pred = predictions[quantile_columns]
+        quantile_levels = np.array(quantile_columns, dtype=float)
+        return y_true, q_pred, quantile_levels

autogluon.timeseries-0.8.3b20231027/src/autogluon/timeseries/metrics/point.py

@@ -0,0 +1,156 @@
+import logging
+from typing import Optional
+
+import numpy as np
+import pandas as pd
+
+from autogluon.timeseries import TimeSeriesDataFrame
+from autogluon.timeseries.dataset.ts_dataframe import ITEMID
+
+from .abstract import TimeSeriesScorer
+from .utils import _in_sample_abs_seasonal_error, _in_sample_squared_seasonal_error
+
+logger = logging.getLogger(__name__)
+
+
+class RMSE(TimeSeriesScorer):
+    """Root mean squared error."""
+
+    equivalent_tabular_regression_metric = "root_mean_squared_error"
+
+    def compute_metric(
+        self, data_future: TimeSeriesDataFrame, predictions: TimeSeriesDataFrame, target: str = "target", **kwargs
+    ) -> float:
+        y_true, y_pred = self._get_point_forecast_score_inputs(data_future, predictions, target=target)
+        return np.sqrt(self._safemean((y_true - y_pred) ** 2))
+
+
+class MSE(TimeSeriesScorer):
+    """Mean squared error."""
+
+    equivalent_tabular_regression_metric = "mean_squared_error"
+
+    def compute_metric(
+        self, data_future: TimeSeriesDataFrame, predictions: TimeSeriesDataFrame, target: str = "target", **kwargs
+    ) -> float:
+        y_true, y_pred = self._get_point_forecast_score_inputs(data_future, predictions, target=target)
+        return self._safemean((y_true - y_pred) ** 2)
+
+
+class MAE(TimeSeriesScorer):
+    """Mean absolute error."""
+
+    optimized_by_median = True
+    equivalent_tabular_regression_metric = "mean_absolute_error"
+
+    def compute_metric(
+        self, data_future: TimeSeriesDataFrame, predictions: TimeSeriesDataFrame, target: str = "target", **kwargs
+    ) -> float:
+        y_true, y_pred = self._get_point_forecast_score_inputs(data_future, predictions, target=target)
+        return self._safemean((y_true - y_pred).abs())
+
+
+class WAPE(TimeSeriesScorer):
+    """Weighted absolute percentage error."""
+
+    optimized_by_median = True
+    equivalent_tabular_regression_metric = "mean_absolute_error"
+
+    def compute_metric(
+        self, data_future: TimeSeriesDataFrame, predictions: TimeSeriesDataFrame, target: str = "target", **kwargs
+    ) -> float:
+        y_true, y_pred = self._get_point_forecast_score_inputs(data_future, predictions, target=target)
+        return (y_true - y_pred).abs().sum() / y_true.abs().sum()
+
+
+class sMAPE(TimeSeriesScorer):
+    """Symmetric mean absolute percentage error."""
+
+    optimized_by_median = True
+    equivalent_tabular_regression_metric = "symmetric_mean_absolute_percentage_error"
+
+    def compute_metric(
+        self, data_future: TimeSeriesDataFrame, predictions: TimeSeriesDataFrame, target: str = "target", **kwargs
+    ) -> float:
+        y_true, y_pred = self._get_point_forecast_score_inputs(data_future, predictions, target=target)
+        return self._safemean(2 * ((y_true - y_pred).abs() / (y_true.abs() + y_pred.abs())))
+
+
+class MAPE(TimeSeriesScorer):
+    """Mean Absolute Percentage Error."""
+
+    optimized_by_median = True
+    equivalent_tabular_regression_metric = "mean_absolute_percentage_error"
+
+    def compute_metric(
+        self, data_future: TimeSeriesDataFrame, predictions: TimeSeriesDataFrame, target: str = "target", **kwargs
+    ) -> float:
+        y_true, y_pred = self._get_point_forecast_score_inputs(data_future, predictions, target=target)
+        return self._safemean((y_true - y_pred).abs() / y_true.abs())
+
+
+class MASE(TimeSeriesScorer):
+    """Mean absolute scaled error."""
+
+    optimized_by_median = True
+    equivalent_tabular_regression_metric = "mean_absolute_error"
+
+    def __init__(self):
+        self._past_abs_seasonal_error: Optional[pd.Series] = None
+
+    def save_past_metrics(
+        self,
+        data_past: TimeSeriesDataFrame,
+        target: str = "target",
+        seasonal_period: int = 1,
+        **kwargs,
+    ) -> None:
+        self._past_abs_seasonal_error = _in_sample_abs_seasonal_error(
+            y_past=data_past[target], seasonal_period=seasonal_period
+        )
+
+    def clear_past_metrics(self) -> None:
+        self._past_abs_seasonal_error = None
+
+    def compute_metric(
+        self, data_future: TimeSeriesDataFrame, predictions: TimeSeriesDataFrame, target: str = "target", **kwargs
+    ) -> float:
+        y_true, y_pred = self._get_point_forecast_score_inputs(data_future, predictions, target=target)
+        if self._past_abs_seasonal_error is None:
+            raise AssertionError("Call `save_past_metrics` before `compute_metric`")
+
+        mae_per_item = (y_true - y_pred).abs().groupby(level=ITEMID, sort=False).mean()
+        return self._safemean(mae_per_item / self._past_abs_seasonal_error)
+
+
+class RMSSE(TimeSeriesScorer):
+    """Root mean squared scaled error."""
+
+    equivalent_tabular_regression_metric = "root_mean_squared_error"
+
+    def __init__(self):
+        self._past_squared_seasonal_error: Optional[pd.Series] = None
+
+    def save_past_metrics(
+        self,
+        data_past: TimeSeriesDataFrame,
+        target: str = "target",
+        seasonal_period: int = 1,
+        **kwargs,
+    ) -> None:
+        self._past_squared_seasonal_error = _in_sample_squared_seasonal_error(
+            y_past=data_past[target], seasonal_period=seasonal_period
+        )
+
+    def clear_past_metrics(self) -> None:
+        self._past_squared_seasonal_error = None
+
+    def compute_metric(
+        self, data_future: TimeSeriesDataFrame, predictions: TimeSeriesDataFrame, target: str = "target", **kwargs
+    ) -> float:
+        y_true, y_pred = self._get_point_forecast_score_inputs(data_future, predictions, target=target)
+        if self._past_squared_seasonal_error is None:
+            raise AssertionError("Call `save_past_metrics` before `compute_metric`")
+
+        mse_per_item = (y_true - y_pred).pow(2.0).groupby(level=ITEMID, sort=False).mean()
+        return np.sqrt(self._safemean(mse_per_item / self._past_squared_seasonal_error))

autogluon.timeseries-0.8.3b20231027/src/autogluon/timeseries/metrics/quantile.py

@@ -0,0 +1,26 @@
+import numpy as np
+
+from autogluon.timeseries import TimeSeriesDataFrame
+
+from .abstract import TimeSeriesScorer
+
+
+class WQL(TimeSeriesScorer):
+    """Weighted quantile loss.
+
+    Also known as weighted pinball loss.
+    """
+
+    needs_quantile = True
+
+    def compute_metric(
+        self, data_future: TimeSeriesDataFrame, predictions: TimeSeriesDataFrame, target: str = "target", **kwargs
+    ) -> float:
+        y_true, q_pred, quantile_levels = self._get_quantile_forecast_score_inputs(data_future, predictions, target)
+        values_true = y_true.values[:, None]  # shape [N, 1]
+        values_pred = q_pred.values  # shape [N, len(quantile_levels)]
+
+        return 2 * np.mean(
+            np.abs((values_true - values_pred) * ((values_true <= values_pred) - quantile_levels)).sum(axis=0)
+            / np.abs(values_true).sum()
+        )

autogluon.timeseries-0.8.3b20231027/src/autogluon/timeseries/metrics/utils.py

@@ -0,0 +1,18 @@
+import pandas as pd
+
+from autogluon.timeseries.dataset.ts_dataframe import ITEMID
+
+
+def _get_seasonal_diffs(*, y_past: pd.Series, seasonal_period: int = 1) -> pd.Series:
+    return y_past.groupby(level=ITEMID, sort=False).diff(seasonal_period).abs()
+
+
+def _in_sample_abs_seasonal_error(*, y_past: pd.Series, seasonal_period: int = 1) -> pd.Series:
+    """Compute seasonal naive forecast error (predict value from seasonal_period steps ago) for each time series."""
+    seasonal_diffs = _get_seasonal_diffs(y_past=y_past, seasonal_period=seasonal_period)
+    return seasonal_diffs.groupby(level=ITEMID, sort=False).mean().fillna(1.0)
+
+
+def _in_sample_squared_seasonal_error(*, y_past: pd.Series, seasonal_period: int = 1) -> pd.Series:
+    seasonal_diffs = _get_seasonal_diffs(y_past=y_past, seasonal_period=seasonal_period)
+    return seasonal_diffs.pow(2.0).groupby(level=ITEMID, sort=False).mean().fillna(1.0)