autogluon.timeseries 0.8.3b20231024__py3-none-any.whl → 0.8.3b20231027__py3-none-any.whl

autogluon/timeseries/evaluator.py

@@ -1,128 +1,21 @@
-"""Functions and objects for evaluating forecasts. Adapted from gluonts.evaluation.
-See also, https://ts.gluon.ai/api/gluonts/gluonts.evaluation.html
-"""
-import logging
 from typing import Optional
 
-import numpy as np
-import pandas as pd
-
+from autogluon.common.utils.deprecated_utils import Deprecated
 from autogluon.timeseries import TimeSeriesDataFrame
-from autogluon.timeseries.dataset.ts_dataframe import ITEMID
-from autogluon.timeseries.utils.datetime import get_seasonality
-from autogluon.timeseries.utils.warning_filters import warning_filter
-
-logger = logging.getLogger(__name__)
-
-
-def get_seasonal_diffs(*, y_past: pd.Series, seasonal_period: int = 1) -> pd.Series:
-    return y_past.groupby(level=ITEMID, sort=False).diff(seasonal_period).abs()
-
-
-def in_sample_abs_seasonal_error(*, y_past: pd.Series, seasonal_period: int = 1) -> pd.Series:
-    """Compute seasonal naive forecast error (predict value from seasonal_period steps ago) for each time series."""
-    seasonal_diffs = get_seasonal_diffs(y_past=y_past, seasonal_period=seasonal_period)
-    return seasonal_diffs.groupby(level=ITEMID, sort=False).mean().fillna(1.0)
-
-
-def in_sample_squared_seasonal_error(*, y_past: pd.Series, seasonal_period: int = 1) -> pd.Series:
-    seasonal_diffs = get_seasonal_diffs(y_past=y_past, seasonal_period=seasonal_period)
-    return seasonal_diffs.dropna().pow(2.0).groupby(level=ITEMID, sort=False).mean()
-
-
-def mse_per_item(*, y_true: pd.Series, y_pred: pd.Series) -> pd.Series:
-    """Compute Mean Squared Error for each item (time series)."""
-    return (y_true - y_pred).pow(2.0).groupby(level=ITEMID, sort=False).mean()
-
-
-def mae_per_item(*, y_true: pd.Series, y_pred: pd.Series) -> pd.Series:
-    """Compute Mean Absolute Error for each item (time series)."""
-    return (y_true - y_pred).abs().groupby(level=ITEMID, sort=False).mean()
-
-
-def mape_per_item(*, y_true: pd.Series, y_pred: pd.Series) -> pd.Series:
-    """Compute Mean Absolute Percentage Error for each item (time series)."""
-    return ((y_true - y_pred) / y_true).abs().groupby(level=ITEMID, sort=False).mean()
-
-
-def rmsse_per_item(*, y_true: pd.Series, y_pred: pd.Series, past_squared_seasonal_error: pd.Series) -> pd.Series:
-    mse = mse_per_item(y_true=y_true, y_pred=y_pred)
-    return mse / past_squared_seasonal_error
-
-
-def symmetric_mape_per_item(*, y_true: pd.Series, y_pred: pd.Series) -> pd.Series:
-    """Compute symmetric Mean Absolute Percentage Error for each item (time series)."""
-    return (2 * (y_true - y_pred).abs() / (y_true.abs() + y_pred.abs())).groupby(level=ITEMID, sort=False).mean()
+from autogluon.timeseries.metrics import AVAILABLE_METRICS, check_get_evaluation_metric
 
 
+@Deprecated(
+    min_version_to_warn="1.0",
+    min_version_to_error="1.1",
+    custom_warning_msg="Please use the metrics defined in autogluon.timeseries.metrics instead.",
+)
 class TimeSeriesEvaluator:
-    """Contains functions for computing forecast accuracy metrics.
-
-    Forecast accuracy metrics measure discrepancy between forecasts and ground-truth time
-    series. After being initialized, AutoGluon ``TimeSeriesEvaluator`` expects two time
-    series data sets (``TimeSeriesDataFrame``) as input: the first of which contains the
-    ground truth time series including both the "history" and the forecast horizon. The
-    second input is the data frame of predictions corresponding only to the forecast
-    horizon.
-
-    .. warning::
-        ``TimeSeriesEvaluator`` always computes metrics by their original definition, while
-        AutoGluon-TimeSeries predictor and model objects always report their scores in
-        higher-is-better fashion. The coefficients used to "flip" the signs of metrics to
-        obey this convention are given in ``TimeSeriesEvaluator.METRIC_COEFFICIENTS``.
+    """This class has been deprecated in AutoGluon v1.0 and is only provided for backward compatibility!"""
 
-    .. warning::
-        Definitions of forecast accuracy metrics may differ from package to package.
-        For example, the definition of MASE is different between GluonTS and autogluon.
-
-    Parameters
-    ----------
-    eval_metric : str
-        Name of the metric to be computed. Available metrics are
-
-        * ``MASE``: mean absolute scaled error. See https://en.wikipedia.org/wiki/Mean_absolute_scaled_error
-        * ``MAPE``: mean absolute percentage error. See https://en.wikipedia.org/wiki/Mean_absolute_percentage_error
-        * ``sMAPE``: "symmetric" mean absolute percentage error. See https://en.wikipedia.org/wiki/Symmetric_mean_absolute_percentage_error
-        * ``WQL``: mean weighted quantile loss, i.e., average quantile loss scaled
-         by the total absolute values of the time series. See https://docs.aws.amazon.com/forecast/latest/dg/metrics.html#metrics-wQL
-        * ``MSE``: mean squared error
-        * ``RMSE``: root mean squared error
-        * ``WAPE``: weighted absolute percentage error. See https://docs.aws.amazon.com/forecast/latest/dg/metrics.html#metrics-WAPE
-        * ``RMSSE``: Root Mean Squared Scaled Error . See https://otexts.com/fpp3/accuracy.html#scaled-errors
-
-    prediction_length : int
-        Length of the forecast horizon
-    target_column : str, default = "target"
-        Name of the target column to be forecasting.
-    eval_metric_seasonal_period : int, optional
-        Seasonal period used to compute the mean absolute scaled error (MASE) evaluation metric. This parameter is only
-        used if ``eval_metric="MASE"`. See https://en.wikipedia.org/wiki/Mean_absolute_scaled_error for more details.
-        Defaults to ``None``, in which case the seasonal period is computed based on the data frequency.
-
-    Class Attributes
-    ----------------
-    AVAILABLE_METRICS
-        list of names of available metrics
-    METRIC_COEFFICIENTS
-        coefficients by which each metric should be multiplied with to obey the higher-is-better
-        convention
-    DEFAULT_METRIC
-        name of default metric returned by
-        :meth:``~autogluon.timeseries.TimeSeriesEvaluator.check_get_evaluation_metric``.
-    """
-
-    AVAILABLE_METRICS = ["MASE", "MAPE", "sMAPE", "WQL", "MSE", "RMSE", "WAPE", "RMSSE"]
-    METRIC_COEFFICIENTS = {
-        "MASE": -1,
-        "MAPE": -1,
-        "sMAPE": -1,
-        "WQL": -1,
-        "MSE": -1,
-        "RMSE": -1,
-        "WAPE": -1,
-        "RMSSE": -1,
-    }
-    DEFAULT_METRIC = "WQL"
+    METRIC_COEFFICIENTS = {metric_name: metric_cls().sign for metric_name, metric_cls in AVAILABLE_METRICS.items()}
+    AVAILABLE_METRICS = list(AVAILABLE_METRICS.keys())
+    DEFAULT_METRIC = check_get_evaluation_metric(None).name
 
     def __init__(
         self,
@@ -131,151 +24,35 @@ class TimeSeriesEvaluator:
         target_column: str = "target",
         eval_metric_seasonal_period: Optional[int] = None,
     ):
-        assert eval_metric in self.AVAILABLE_METRICS, f"Metric {eval_metric} not available"
-
+        self.eval_metric = check_get_evaluation_metric(eval_metric)
         self.prediction_length = prediction_length
-        self.eval_metric = eval_metric
         self.target_column = target_column
         self.seasonal_period = eval_metric_seasonal_period
 
-        self.metric_method = self.__getattribute__("_" + self.eval_metric.lower())
-        self._past_abs_seasonal_error: Optional[pd.Series] = None
-        self._past_squared_seasonal_error: Optional[pd.Series] = None
-
     @property
     def coefficient(self) -> int:
-        return self.METRIC_COEFFICIENTS[self.eval_metric]
+        return self.eval_metric.sign
 
     @property
     def higher_is_better(self) -> bool:
-        return self.coefficient > 0
-
-    def _safemean(self, data: pd.Series) -> float:
-        return data.replace([np.inf, -np.inf], np.nan).dropna().mean()
-
-    def _mse(self, y_true: pd.Series, predictions: TimeSeriesDataFrame) -> float:
-        y_pred = predictions["mean"]
-        return self._safemean(mse_per_item(y_true=y_true, y_pred=y_pred))
-
-    def _rmse(self, y_true: pd.Series, predictions: TimeSeriesDataFrame) -> float:
-        return np.sqrt(self._mse(y_true=y_true, predictions=predictions))
-
-    def _mase(self, y_true: pd.Series, predictions: TimeSeriesDataFrame) -> float:
-        y_pred = self._get_median_forecast(predictions)
-        mae = mae_per_item(y_true=y_true, y_pred=y_pred)
-        return self._safemean(mae / self._past_abs_seasonal_error)
-
-    def _mape(self, y_true: pd.Series, predictions: TimeSeriesDataFrame) -> float:
-        y_pred = self._get_median_forecast(predictions)
-        return self._safemean(mape_per_item(y_true=y_true, y_pred=y_pred))
-
-    def _smape(self, y_true: pd.Series, predictions: TimeSeriesDataFrame) -> float:
-        y_pred = self._get_median_forecast(predictions)
-        return self._safemean(symmetric_mape_per_item(y_true=y_true, y_pred=y_pred))
-
-    def _wql(self, y_true: pd.Series, predictions: TimeSeriesDataFrame) -> float:
-        values_true = y_true.values[:, None]  # shape [N, 1]
-        quantile_pred_columns = [col for col in predictions.columns if col != "mean"]
-        values_pred = predictions[quantile_pred_columns].values  # shape [N, len(quantile_levels)]
-        quantile_levels = np.array([float(q) for q in quantile_pred_columns], dtype=float)
-
-        return 2 * np.mean(
-            np.abs((values_true - values_pred) * ((values_true <= values_pred) - quantile_levels)).sum(axis=0)
-            / np.abs(values_true).sum()
-        )
-
-    def _wape(self, y_true: pd.Series, predictions: TimeSeriesDataFrame) -> float:
-        y_pred = self._get_median_forecast(predictions)
-        abs_error_sum = (mae_per_item(y_true=y_true, y_pred=y_pred) * self.prediction_length).sum()
-        abs_target_sum = y_true.abs().sum()
-        return abs_error_sum / abs_target_sum
-
-    def _rmsse(self, y_true: pd.Series, predictions: TimeSeriesDataFrame) -> float:
-        y_pred = predictions["mean"]
-        return np.sqrt(
-            rmsse_per_item(
-                y_true=y_true, y_pred=y_pred, past_squared_seasonal_error=self._past_squared_seasonal_error
-            ).mean()
-        )
-
-    def _get_median_forecast(self, predictions: TimeSeriesDataFrame) -> pd.Series:
-        # TODO: Median forecast doesn't actually minimize the MAPE / sMAPE losses
-        if "0.5" in predictions.columns:
-            return predictions["0.5"]
-        else:
-            logger.warning("Median forecast not found. Defaulting to mean forecasts.")
-            return predictions["mean"]
+        return self.eval_metric.greater_is_better_internal
 
     @staticmethod
     def check_get_evaluation_metric(
         metric_name: Optional[str] = None,
         raise_if_not_available: bool = True,
     ):
-        """A utility function that checks if a given evaluation metric
-        name is available in autogluon.timeseries, and optionally raises
-        a ValueError otherwise.
-
-        Parameters
-        ----------
-        metric_name: str
-            The requested metric name, currently one of the evaluation metrics available
-            in GluonTS.
-        raise_if_not_available: bool
-            if True, a ValueError will be raised if the requested metric is not yet
-            available in autogluon.timeseries. Otherwise, the default metric name will be
-            returned instead of the requested metric.
-
-        Returns
-        -------
-        checked_metric_name: str
-            The requested metric name if it is available in autogluon.timeseries.
-        """
-        metric = metric_name or TimeSeriesEvaluator.DEFAULT_METRIC
-        if metric not in TimeSeriesEvaluator.AVAILABLE_METRICS:
-            if raise_if_not_available:
-                raise ValueError(f"metric {metric} is not available yet.")
-            return TimeSeriesEvaluator.DEFAULT_METRIC
-        return metric
-
-    def save_past_metrics(self, data_past: TimeSeriesDataFrame):
-        seasonal_period = get_seasonality(data_past.freq) if self.seasonal_period is None else self.seasonal_period
-        if self.eval_metric == "MASE":
-            self._past_abs_seasonal_error = in_sample_abs_seasonal_error(
-                y_past=data_past[self.target_column], seasonal_period=seasonal_period
-            )
-
-        if self.eval_metric == "RMSSE":
-            self._past_squared_seasonal_error = in_sample_squared_seasonal_error(
-                y_past=data_past[self.target_column], seasonal_period=seasonal_period
-            )
-
-    def score_with_saved_past_metrics(
-        self, data_future: TimeSeriesDataFrame, predictions: TimeSeriesDataFrame
-    ) -> float:
-        """Compute the metric assuming that the historic metrics have already been computed.
-
-        This method should be preferred to TimeSeriesEvaluator.__call__ if the metrics are computed multiple times, as
-        it doesn't require splitting the test data into past/future portions each time (e.g., when fitting ensembles).
-        """
-        assert (predictions.num_timesteps_per_item() == self.prediction_length).all()
-
-        if self.eval_metric == "MASE" and self._past_abs_seasonal_error is None:
-            raise AssertionError("Call save_past_metrics before score_with_saved_past_metrics")
-
-        if self.eval_metric == "RMSSE" and self._past_squared_seasonal_error is None:
-            raise AssertionError("Call save_past_metrics before score_with_saved_past_metrics")
-
-        assert data_future.index.equals(predictions.index), "Prediction and data indices do not match."
-
-        with warning_filter():
-            return self.metric_method(
-                y_true=data_future[self.target_column],
-                predictions=predictions,
-            )
+        return check_get_evaluation_metric(metric_name)
 
     def __call__(self, data: TimeSeriesDataFrame, predictions: TimeSeriesDataFrame) -> float:
-        # Select entries in `data` that correspond to the forecast horizon
-        data_past = data.slice_by_timestep(None, -self.prediction_length)
-        data_future = data.slice_by_timestep(-self.prediction_length, None)
-        self.save_past_metrics(data_past=data_past)
-        return self.score_with_saved_past_metrics(data_future=data_future, predictions=predictions)
+        quantile_levels = [float(col) for col in predictions.columns if col != "mean"]
+        score = self.eval_metric(
+            data=data,
+            predictions=predictions,
+            prediction_length=self.prediction_length,
+            target=self.target_column,
+            seasonal_period=self.seasonal_period,
+            quantile_levels=quantile_levels,
+        )
+        # Return raw metric in lower-is-better format to match the old Evaluator API
+        return score * self.eval_metric.sign

autogluon/timeseries/learner.py

@@ -7,7 +7,7 @@ import pandas as pd
 
 from autogluon.core.learner import AbstractLearner
 from autogluon.timeseries.dataset.ts_dataframe import TimeSeriesDataFrame
-from autogluon.timeseries.evaluator import TimeSeriesEvaluator
+from autogluon.timeseries.metrics import TimeSeriesScorer, check_get_evaluation_metric
 from autogluon.timeseries.models.abstract import AbstractTimeSeriesModel
 from autogluon.timeseries.splitter import AbstractWindowSplitter
 from autogluon.timeseries.trainer import AbstractTimeSeriesTrainer, AutoTimeSeriesTrainer
@@ -28,14 +28,14 @@ class TimeSeriesLearner(AbstractLearner):
         target: str = "target",
         known_covariates_names: Optional[List[str]] = None,
         trainer_type: Type[AbstractTimeSeriesTrainer] = AutoTimeSeriesTrainer,
-        eval_metric: Optional[str] = None,
+        eval_metric: Union[str, TimeSeriesScorer, None] = None,
         eval_metric_seasonal_period: Optional[int] = None,
         prediction_length: int = 1,
         cache_predictions: bool = True,
         **kwargs,
     ):
         super().__init__(path_context=path_context)
-        self.eval_metric: str = TimeSeriesEvaluator.check_get_evaluation_metric(eval_metric)
+        self.eval_metric: TimeSeriesScorer = check_get_evaluation_metric(eval_metric)
         self.eval_metric_seasonal_period = eval_metric_seasonal_period
         self.trainer_type = trainer_type
         self.target = target
@@ -89,7 +89,7 @@ class TimeSeriesLearner(AbstractLearner):
         logger.info(f"AutoGluon will save models to {self.path}")
 
         logger.info(f"AutoGluon will gauge predictive performance using evaluation metric: '{self.eval_metric}'")
-        if TimeSeriesEvaluator.METRIC_COEFFICIENTS[self.eval_metric] == -1:
+        if not self.eval_metric.greater_is_better_internal:
             logger.info(
                 "\tThis metric's sign has been flipped to adhere to being 'higher is better'. "
                 "The reported score can be multiplied by -1 to get the metric value.",
@@ -185,7 +185,7 @@ class TimeSeriesLearner(AbstractLearner):
         self,
         data: TimeSeriesDataFrame,
         model: AbstractTimeSeriesModel = None,
-        metric: Optional[str] = None,
+        metric: Union[str, TimeSeriesScorer, None] = None,
         use_cache: bool = True,
     ) -> float:
         data = self.feature_generator.transform(data)

autogluon/timeseries/metrics/__init__.py

@@ -0,0 +1,58 @@
+import json
+from typing import Type, Union
+
+from .abstract import TimeSeriesScorer
+from .point import MAE, MAPE, MASE, MSE, RMSE, RMSSE, WAPE, sMAPE
+from .quantile import WQL
+
+__all__ = [
+    "MAE",
+    "MAPE",
+    "MASE",
+    "sMAPE",
+    "MSE",
+    "RMSE",
+    "RMSSE",
+    "WAPE",
+    "WQL",
+]
+
+DEFAULT_METRIC_NAME = "WQL"
+
+AVAILABLE_METRICS = {
+    "MASE": MASE,
+    "MAPE": MAPE,
+    "SMAPE": sMAPE,
+    "RMSE": RMSE,
+    "RMSSE": RMSSE,
+    "WAPE": WAPE,
+    "WQL": WQL,
+    # Exist for compatibility
+    "MSE": MSE,
+    "MAE": MAE,
+}
+
+
+def check_get_evaluation_metric(
+    eval_metric: Union[str, TimeSeriesScorer, Type[TimeSeriesScorer], None] = None
+) -> TimeSeriesScorer:
+    if isinstance(eval_metric, TimeSeriesScorer):
+        eval_metric = eval_metric
+    elif isinstance(eval_metric, type) and issubclass(eval_metric, TimeSeriesScorer):
+        # e.g., user passed `eval_metric=CustomMetric` instead of `eval_metric=CustomMetric()`
+        eval_metric = eval_metric()
+    elif isinstance(eval_metric, str):
+        if eval_metric.upper() not in AVAILABLE_METRICS:
+            raise ValueError(
+                f"Time series metric {eval_metric} not supported. Available metrics are:\n"
+                f"{json.dumps(list(AVAILABLE_METRICS.keys()), indent=2)}"
+            )
+        eval_metric = AVAILABLE_METRICS[eval_metric.upper()]()
+    elif eval_metric is None:
+        eval_metric = AVAILABLE_METRICS[DEFAULT_METRIC_NAME]()
+    else:
+        raise ValueError(
+            f"eval_metric must be of type str, TimeSeriesScorer or None "
+            f"(received eval_metric = {eval_metric} of type {type(eval_metric)})"
+        )
+    return eval_metric

autogluon/timeseries/metrics/abstract.py

@@ -0,0 +1,201 @@
+from typing import Optional, Tuple
+
+import numpy as np
+import pandas as pd
+
+from autogluon.timeseries import TimeSeriesDataFrame
+from autogluon.timeseries.utils.datetime import get_seasonality
+from autogluon.timeseries.utils.warning_filters import warning_filter
+
+
+class TimeSeriesScorer:
+    """Base class for all evaluation metrics used in AutoGluon-TimeSeries.
+
+    This object always returns the metric in greater-is-better format.
+
+    Follows the design of ``autogluon.core.metrics.Scorer``.
+
+    Attributes
+    ----------
+    greater_is_better_internal : bool, default = False
+        Whether internal method :meth:`~autogluon.timeseries.metrics.TimeSeriesScorer.compute_metric` is
+        a loss function (default), meaning low is good, or a score function, meaning high is good.
+    optimum : float, default = 0.0
+        The best score achievable by the score function, i.e. maximum in case of scorer function and minimum in case of
+        loss function.
+    optimized_by_median : bool, default = False
+        Whether given point forecast metric is optimized by the median (if True) or expected value (if False). If True,
+        all models in AutoGluon-TimeSeries will attempt to paste median forecast into the "mean" column.
+    needs_quantile : bool, default = False
+        Whether the given metric uses the quantile predictions. Some models will modify the training procedure if they
+        are trained to optimize a quantile metric.
+    equivalent_tabular_regression_metric : str
+        Name of an equivalent metric used by AutoGluon-Tabular with ``problem_type="regression"``. Used by models that
+        train a TabularPredictor under the hood. This attribute should only be specified by point forecast metrics.
+    """
+
+    greater_is_better_internal: bool = False
+    optimum: float = 0.0
+    optimized_by_median: bool = False
+    needs_quantile: bool = False
+    equivalent_tabular_regression_metric: Optional[str] = None
+
+    @property
+    def sign(self) -> int:
+        return 1 if self.greater_is_better_internal else -1
+
+    @property
+    def name(self) -> str:
+        return f"{self.__class__.__name__}"
+
+    def __repr__(self) -> str:
+        return self.name
+
+    def __str__(self) -> str:
+        return self.name
+
+    @property
+    def name_with_sign(self) -> str:
+        if self.greater_is_better_internal:
+            prefix = ""
+        else:
+            prefix = "-"
+        return f"{prefix}{self.name}"
+
+    def __call__(
+        self,
+        data: TimeSeriesDataFrame,
+        predictions: TimeSeriesDataFrame,
+        prediction_length: int = 1,
+        target: str = "target",
+        seasonal_period: Optional[int] = None,
+        **kwargs,
+    ) -> float:
+        seasonal_period = get_seasonality(data.freq) if seasonal_period is None else seasonal_period
+
+        data_past = data.slice_by_timestep(None, -prediction_length)
+        data_future = data.slice_by_timestep(-prediction_length, None)
+
+        assert (predictions.num_timesteps_per_item() == prediction_length).all()
+        assert data_future.index.equals(predictions.index), "Prediction and data indices do not match."
+
+        try:
+            with warning_filter():
+                self.save_past_metrics(
+                    data_past=data_past,
+                    target=target,
+                    seasonal_period=seasonal_period,
+                    **kwargs,
+                )
+                metric_value = self.compute_metric(
+                    data_future=data_future,
+                    predictions=predictions,
+                    target=target,
+                    **kwargs,
+                )
+        finally:
+            self.clear_past_metrics()
+        return metric_value * self.sign
+
+    score = __call__
+
+    def compute_metric(
+        self,
+        data_future: TimeSeriesDataFrame,
+        predictions: TimeSeriesDataFrame,
+        target: str = "target",
+        **kwargs,
+    ) -> float:
+        """Internal method that computes the metric for given forecast & actual data.
+
+        This method should be implemented by all custom metrics.
+
+        Parameters
+        ----------
+        data_future : TimeSeriesDataFrame
+            Actual values of the time series during the forecast horizon (``prediction_length`` values for each time
+            series in the dataset). This data frame is guaranteed to have the same index as ``predictions``.
+        predictions : TimeSeriesDataFrame
+            Data frame with predictions for the forecast horizon. Contain columns "mean" (point forecast) and the
+            columns corresponding to each of the quantile levels.
+        target : str, default = "target"
+            Name of the column in ``data_future`` that contains the target time series.
+
+        Returns
+        -------
+        score : float
+            Value of the metric for given forecast and data. If self.greater_is_better_internal is True, returns score
+            in greater-is-better format, otherwise in lower-is-better format.
+
+        """
+        raise NotImplementedError
+
+    def save_past_metrics(
+        self,
+        data_past: TimeSeriesDataFrame,
+        target: str = "target",
+        seasonal_period: int = 1,
+        **kwargs,
+    ) -> None:
+        """Compute auxiliary metrics on past data (before forecast horizon), if the chosen metric requires it.
+
+        This method should only be implemented by metrics that rely on historic (in-sample) data, such as Mean Absolute
+        Scaled Error (MASE) https://en.wikipedia.org/wiki/Mean_absolute_scaled_error.
+
+        We keep this method separate from :meth:`compute_metric` to avoid redundant computations when fitting ensemble.
+        """
+        pass
+
+    def clear_past_metrics(self) -> None:
+        """Clear auxiliary metrics saved in :meth:`save_past_metrics`.
+
+        This method should only be implemented if :meth:`save_past_metrics` has been implemented.
+        """
+        pass
+
+    def error(self, *args, **kwargs):
+        """Return error in lower-is-better format."""
+        return self.optimum - self.score(*args, **kwargs)
+
+    @staticmethod
+    def _safemean(series: pd.Series) -> float:
+        """Compute mean of an pd.Series, ignoring inf, -inf and nan values."""
+        return np.nanmean(series.replace([np.inf, -np.inf], np.nan).values)
+
+    @staticmethod
+    def _get_point_forecast_score_inputs(
+        data_future: TimeSeriesDataFrame, predictions: TimeSeriesDataFrame, target: str = "target"
+    ) -> Tuple[pd.Series, pd.Series]:
+        """Get inputs necessary to compute point forecast metrics.
+
+        Returns
+        -------
+        y_true : pd.Series, shape [num_items * prediction_length]
+            Target time series values during the forecast horizon.
+        y_pred : pd.Series, shape [num_items * prediction_length]
+            Predicted time series values during the forecast horizon.
+        """
+        y_true = data_future[target]
+        y_pred = predictions["mean"]
+        return y_true, y_pred
+
+    @staticmethod
+    def _get_quantile_forecast_score_inputs(
+        data_future: TimeSeriesDataFrame, predictions: TimeSeriesDataFrame, target: str = "target"
+    ) -> Tuple[pd.Series, pd.DataFrame, np.ndarray]:
+        """Get inputs necessary to compute quantile forecast metrics.
+
+        Returns
+        -------
+        y_true : pd.Series, shape [num_items * prediction_length]
+            Target time series values during the forecast horizon.
+        q_pred : pd.DataFrame, shape [num_items * prediction_length, num_quantiles]
+            Quantile forecast for each predicted quantile level. Column order corresponds to ``quantile_levels``.
+        quantile_levels : np.ndarray, shape [num_quantiles]
+            Quantile levels for which the forecasts are generated (as floats).
+        """
+        quantile_columns = [col for col in predictions.columns if col != "mean"]
+        y_true = data_future[target]
+        q_pred = predictions[quantile_columns]
+        quantile_levels = np.array(quantile_columns, dtype=float)
+        return y_true, q_pred, quantile_levels