autogluon.timeseries 1.0.1b20240304__py3-none-any.whl → 1.4.1b20251210__py3-none-any.whl

autogluon/timeseries/learner.py

@@ -1,18 +1,17 @@
 import logging
 import reprlib
 import time
-from typing import Any, Dict, List, Optional, Type, Union
+from typing import Any, Literal, Type
 
 import pandas as pd
 
 from autogluon.core.learner import AbstractLearner
-from autogluon.timeseries.dataset.ts_dataframe import TimeSeriesDataFrame
+from autogluon.timeseries.dataset import TimeSeriesDataFrame
 from autogluon.timeseries.metrics import TimeSeriesScorer, check_get_evaluation_metric
 from autogluon.timeseries.models.abstract import AbstractTimeSeriesModel
-from autogluon.timeseries.splitter import AbstractWindowSplitter
-from autogluon.timeseries.trainer import AbstractTimeSeriesTrainer, AutoTimeSeriesTrainer
+from autogluon.timeseries.trainer import TimeSeriesTrainer
 from autogluon.timeseries.utils.features import TimeSeriesFeatureGenerator
-from autogluon.timeseries.utils.forecast import get_forecast_horizon_index_ts_dataframe
+from autogluon.timeseries.utils.forecast import make_future_data_frame
 
 logger = logging.getLogger(__name__)
 
@@ -26,85 +25,77 @@ class TimeSeriesLearner(AbstractLearner):
         self,
         path_context: str,
         target: str = "target",
-        known_covariates_names: Optional[List[str]] = None,
-        trainer_type: Type[AbstractTimeSeriesTrainer] = AutoTimeSeriesTrainer,
-        eval_metric: Union[str, TimeSeriesScorer, None] = None,
-        eval_metric_seasonal_period: Optional[int] = None,
+        known_covariates_names: list[str] | None = None,
+        trainer_type: Type[TimeSeriesTrainer] = TimeSeriesTrainer,
+        eval_metric: str | TimeSeriesScorer | None = None,
         prediction_length: int = 1,
         cache_predictions: bool = True,
+        ensemble_model_type: Type | None = None,
         **kwargs,
     ):
         super().__init__(path_context=path_context)
-        self.eval_metric: TimeSeriesScorer = check_get_evaluation_metric(eval_metric)
-        self.eval_metric_seasonal_period = eval_metric_seasonal_period
+        self.eval_metric = check_get_evaluation_metric(eval_metric, prediction_length=prediction_length)
         self.trainer_type = trainer_type
         self.target = target
         self.known_covariates_names = [] if known_covariates_names is None else known_covariates_names
         self.prediction_length = prediction_length
         self.quantile_levels = kwargs.get("quantile_levels", [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9])
         self.cache_predictions = cache_predictions
+        self.freq: str | None = None
+        self.ensemble_model_type = ensemble_model_type
 
         self.feature_generator = TimeSeriesFeatureGenerator(
             target=self.target, known_covariates_names=self.known_covariates_names
         )
 
-    def load_trainer(self) -> AbstractTimeSeriesTrainer:
+    def load_trainer(self) -> TimeSeriesTrainer:  # type: ignore
         """Return the trainer object corresponding to the learner."""
-        return super().load_trainer()  # noqa
+        return super().load_trainer()  # type: ignore
 
     def fit(
         self,
         train_data: TimeSeriesDataFrame,
-        val_data: TimeSeriesDataFrame = None,
-        hyperparameters: Union[str, Dict] = None,
-        hyperparameter_tune_kwargs: Optional[Union[str, dict]] = None,
-        **kwargs,
-    ) -> None:
-        return self._fit(
-            train_data=train_data,
-            val_data=val_data,
-            hyperparameters=hyperparameters,
-            hyperparameter_tune_kwargs=hyperparameter_tune_kwargs,
-            **kwargs,
-        )
-
-    def _fit(
-        self,
-        train_data: TimeSeriesDataFrame,
-        val_data: Optional[TimeSeriesDataFrame] = None,
-        hyperparameters: Union[str, Dict] = None,
-        hyperparameter_tune_kwargs: Optional[Union[str, dict]] = None,
-        time_limit: Optional[int] = None,
-        val_splitter: Optional[AbstractWindowSplitter] = None,
-        refit_every_n_windows: Optional[int] = 1,
-        random_seed: Optional[int] = None,
+        hyperparameters: str | dict,
+        val_data: TimeSeriesDataFrame | None = None,
+        hyperparameter_tune_kwargs: str | dict | None = None,
+        time_limit: float | None = None,
+        num_val_windows: tuple[int, ...] = (1,),
+        val_step_size: int | None = None,
+        refit_every_n_windows: int | None = 1,
+        random_seed: int | None = None,
         **kwargs,
     ) -> None:
         self._time_limit = time_limit
         time_start = time.time()
 
-        train_data = self.feature_generator.fit_transform(train_data, data_frame_name="train_data")
+        train_data = self.feature_generator.fit_transform(train_data)
         if val_data is not None:
             val_data = self.feature_generator.transform(val_data, data_frame_name="tuning_data")
 
+        self.freq = train_data.freq
+
         trainer_init_kwargs = kwargs.copy()
         trainer_init_kwargs.update(
             dict(
                 path=self.model_context,
                 prediction_length=self.prediction_length,
                 eval_metric=self.eval_metric,
-                eval_metric_seasonal_period=self.eval_metric_seasonal_period,
                 target=self.target,
                 quantile_levels=self.quantile_levels,
                 verbosity=kwargs.get("verbosity", 2),
+                skip_model_selection=kwargs.get("skip_model_selection", False),
                 enable_ensemble=kwargs.get("enable_ensemble", True),
-                metadata=self.feature_generator.covariate_metadata,
-                val_splitter=val_splitter,
+                covariate_metadata=self.feature_generator.covariate_metadata,
+                num_val_windows=num_val_windows,
+                val_step_size=val_step_size,
                 refit_every_n_windows=refit_every_n_windows,
                 cache_predictions=self.cache_predictions,
+                ensemble_model_type=self.ensemble_model_type,
             )
         )
-        self.trainer = self.trainer_type(**trainer_init_kwargs)
+
+        assert issubclass(self.trainer_type, TimeSeriesTrainer)
+        self.trainer: TimeSeriesTrainer | None = self.trainer_type(**trainer_init_kwargs)
         self.trainer_path = self.trainer.path
         self.save()
 
@@ -131,9 +122,9 @@ class TimeSeriesLearner(AbstractLearner):
 
     def _align_covariates_with_forecast_index(
         self,
-        known_covariates: Optional[TimeSeriesDataFrame],
+        known_covariates: TimeSeriesDataFrame | None,
         data: TimeSeriesDataFrame,
-    ) -> Optional[TimeSeriesDataFrame]:
+    ) -> TimeSeriesDataFrame | None:
         """Select the relevant item_ids and timestamps from the known_covariates dataframe.
 
         If some of the item_ids or timestamps are missing, an exception is raised.
@@ -144,6 +135,7 @@ class TimeSeriesLearner(AbstractLearner):
             raise ValueError(
                 f"known_covariates {self.known_covariates_names} for the forecast horizon should be provided at prediction time."
             )
+        assert known_covariates is not None
 
         if self.target in known_covariates.columns:
             known_covariates = known_covariates.drop(self.target, axis=1)
@@ -154,23 +146,27 @@ class TimeSeriesLearner(AbstractLearner):
                 f"known_covariates are missing information for the following item_ids: {reprlib.repr(missing_item_ids.to_list())}."
             )
 
-        forecast_index = get_forecast_horizon_index_ts_dataframe(data, prediction_length=self.prediction_length)
+        forecast_index = pd.MultiIndex.from_frame(
+            make_future_data_frame(data, prediction_length=self.prediction_length, freq=self.freq)
+        )
         try:
-            known_covariates = known_covariates.loc[forecast_index]
+            known_covariates = known_covariates.loc[forecast_index]  # type: ignore
         except KeyError:
             raise ValueError(
-                f"known_covariates should include the values for prediction_length={self.prediction_length} "
-                "many time steps into the future."
+                "`known_covariates` should include the `item_id` and `timestamp` values covering the forecast horizon "
+                "(i.e., the next `prediction_length` time steps following the end of each time series in the input "
+                "data). Use `TimeSeriesPredictor.make_future_data_frame` to generate the required `item_id` and "
+                "`timestamp` combinations for the `known_covariates`."
             )
         return known_covariates
 
     def predict(
         self,
         data: TimeSeriesDataFrame,
-        known_covariates: Optional[TimeSeriesDataFrame] = None,
-        model: Optional[Union[str, AbstractTimeSeriesModel]] = None,
+        known_covariates: TimeSeriesDataFrame | None = None,
+        model: str | AbstractTimeSeriesModel | None = None,
         use_cache: bool = True,
-        random_seed: Optional[int] = None,
+        random_seed: int | None = None,
         **kwargs,
     ) -> TimeSeriesDataFrame:
         data = self.feature_generator.transform(data)
@@ -188,8 +184,8 @@ class TimeSeriesLearner(AbstractLearner):
     def score(
         self,
         data: TimeSeriesDataFrame,
-        model: AbstractTimeSeriesModel = None,
-        metric: Union[str, TimeSeriesScorer, None] = None,
+        model: str | AbstractTimeSeriesModel | None = None,
+        metric: str | TimeSeriesScorer | None = None,
         use_cache: bool = True,
     ) -> float:
         data = self.feature_generator.transform(data)
@@ -197,20 +193,97 @@ class TimeSeriesLearner(AbstractLearner):
 
     def evaluate(
         self,
-        data: Union[TimeSeriesDataFrame, pd.DataFrame, str],
-        model: Optional[str] = None,
-        metrics: Optional[Union[str, TimeSeriesScorer, List[Union[str, TimeSeriesScorer]]]] = None,
+        data: TimeSeriesDataFrame,
+        model: str | None = None,
+        metrics: str | TimeSeriesScorer | list[str | TimeSeriesScorer] | None = None,
         use_cache: bool = True,
-    ) -> Dict[str, float]:
+    ) -> dict[str, float]:
         data = self.feature_generator.transform(data)
         return self.load_trainer().evaluate(data=data, model=model, metrics=metrics, use_cache=use_cache)
 
-    def leaderboard(self, data: Optional[TimeSeriesDataFrame] = None, use_cache: bool = True) -> pd.DataFrame:
+    def get_feature_importance(
+        self,
+        data: TimeSeriesDataFrame | None = None,
+        model: str | None = None,
+        metric: str | TimeSeriesScorer | None = None,
+        features: list[str] | None = None,
+        time_limit: float | None = None,
+        method: Literal["naive", "permutation"] = "permutation",
+        subsample_size: int = 50,
+        num_iterations: int | None = None,
+        random_seed: int | None = None,
+        relative_scores: bool = False,
+        include_confidence_band: bool = True,
+        confidence_level: float = 0.99,
+    ) -> pd.DataFrame:
+        trainer = self.load_trainer()
+        if data is None:
+            data = trainer.load_val_data() or trainer.load_train_data()
+
+        # if features are provided in the dataframe, check that they are valid features in the covariate metadata
+        provided_static_columns = [] if data.static_features is None else data.static_features.columns
+        unused_features = [
+            f
+            for f in set(provided_static_columns).union(set(data.columns) - {self.target})
+            if f not in self.feature_generator.covariate_metadata.all_features
+        ]
+
+        if features is None:
+            features = self.feature_generator.covariate_metadata.all_features
+        else:
+            if len(features) == 0:
+                raise ValueError(
+                    "No features provided to compute feature importance. At least some valid features should be provided."
+                )
+            for fn in features:
+                if fn not in self.feature_generator.covariate_metadata.all_features and fn not in unused_features:
+                    raise ValueError(f"Feature {fn} not found in covariate metadata or the dataset.")
+
+        if len(set(features)) < len(features):
+            raise ValueError(
+                "Duplicate feature names provided to compute feature importance. "
+                "Please provide unique feature names across both static features and covariates."
+            )
+
+        data = self.feature_generator.transform(data)
+
+        importance_df = trainer.get_feature_importance(
+            data=data,
+            features=features,
+            model=model,
+            metric=metric,
+            time_limit=time_limit,
+            method=method,
+            subsample_size=subsample_size,
+            num_iterations=num_iterations,
+            random_seed=random_seed,
+            relative_scores=relative_scores,
+            include_confidence_band=include_confidence_band,
+            confidence_level=confidence_level,
+        )
+
+        for feature in set(features).union(unused_features):
+            if feature not in importance_df.index:
+                importance_df.loc[feature] = (
+                    [0, 0, 0] if not include_confidence_band else [0, 0, 0, float("nan"), float("nan")]
+                )
+
+        return importance_df
+
+    def leaderboard(
+        self,
+        data: TimeSeriesDataFrame | None = None,
+        extra_info: bool = False,
+        extra_metrics: list[str | TimeSeriesScorer] | None = None,
+        use_cache: bool = True,
+    ) -> pd.DataFrame:
         if data is not None:
             data = self.feature_generator.transform(data)
-        return self.load_trainer().leaderboard(data, use_cache=use_cache)
+        return self.load_trainer().leaderboard(
+            data, extra_info=extra_info, extra_metrics=extra_metrics, use_cache=use_cache
+        )
 
-    def get_info(self, include_model_info: bool = False, **kwargs) -> Dict[str, Any]:
+    def get_info(self, include_model_info: bool = False, **kwargs) -> dict[str, Any]:
         learner_info = super().get_info(include_model_info=include_model_info)
         trainer = self.load_trainer()
         trainer_info = trainer.get_info(include_model_info=include_model_info)
@@ -227,5 +300,64 @@ class TimeSeriesLearner(AbstractLearner):
         learner_info.pop("random_state", None)
         return learner_info
 
-    def refit_full(self, model: str = "all") -> Dict[str, str]:
+    def persist_trainer(
+        self, models: Literal["all", "best"] | list[str] = "all", with_ancestors: bool = False
+    ) -> list[str]:
+        """Loads models and trainer in memory so that they don't have to be
+        loaded during predictions
+
+        Returns
+        -------
+        list_of_models
+            List of models persisted in memory
+        """
+        self.trainer = self.load_trainer()
+        return self.trainer.persist(models, with_ancestors=with_ancestors)
+
+    def unpersist_trainer(self) -> list[str]:
+        """Unloads models and trainer from memory. Models will have to be reloaded from disk
+        when predicting.
+
+        Returns
+        -------
+        list_of_models
+            List of models removed from memory
+        """
+        unpersisted_models = self.load_trainer().unpersist()
+        self.trainer = None  # type: ignore
+        return unpersisted_models
+
+    def refit_full(self, model: str = "all") -> dict[str, str]:
         return self.load_trainer().refit_full(model=model)
+
+    def backtest_predictions(
+        self,
+        data: TimeSeriesDataFrame | None,
+        model_names: list[str],
+        num_val_windows: int | None = None,
+        val_step_size: int | None = None,
+        use_cache: bool = True,
+    ) -> dict[str, list[TimeSeriesDataFrame]]:
+        if data is not None:
+            data = self.feature_generator.transform(data)
+        return self.load_trainer().backtest_predictions(
+            model_names=model_names,
+            data=data,
+            num_val_windows=num_val_windows,
+            val_step_size=val_step_size,
+            use_cache=use_cache,
+        )
+
+    def backtest_targets(
+        self,
+        data: TimeSeriesDataFrame | None,
+        num_val_windows: int | None = None,
+        val_step_size: int | None = None,
+    ) -> list[TimeSeriesDataFrame]:
+        if data is not None:
+            data = self.feature_generator.transform(data)
+        return self.load_trainer().backtest_targets(
+            data=data,
+            num_val_windows=num_val_windows,
+            val_step_size=val_step_size,
+        )

autogluon/timeseries/metrics/__init__.py

@@ -1,11 +1,17 @@
+from __future__ import annotations
+
 from pprint import pformat
-from typing import Type, Union
+from typing import Any, Sequence, Type
+
+import numpy as np
 
 from .abstract import TimeSeriesScorer
-from .point import MAE, MAPE, MASE, MSE, RMSE, RMSLE, RMSSE, SMAPE, WAPE
+from .point import MAE, MAPE, MASE, MSE, RMSE, RMSLE, RMSSE, SMAPE, WAPE, WCD
 from .quantile import SQL, WQL
 
 __all__ = [
+    "TimeSeriesScorer",
+    "check_get_evaluation_metric",
     "MAE",
     "MAPE",
     "MASE",
@@ -16,12 +22,13 @@ __all__ = [
     "RMSSE",
     "SQL",
     "WAPE",
+    "WCD",
     "WQL",
 ]
 
 DEFAULT_METRIC_NAME = "WQL"
 
-AVAILABLE_METRICS = {
+AVAILABLE_METRICS: dict[str, Type[TimeSeriesScorer]] = {
     "MASE": MASE,
     "MAPE": MAPE,
     "SMAPE": SMAPE,
@@ -40,28 +47,65 @@ DEPRECATED_METRICS = {
     "mean_wQuantileLoss": "WQL",
 }
 
+# Experimental metrics that are not yet user facing
+EXPERIMENTAL_METRICS: dict[str, Type[TimeSeriesScorer]] = {
+    "WCD": WCD,
+}
+
 
 def check_get_evaluation_metric(
-    eval_metric: Union[str, TimeSeriesScorer, Type[TimeSeriesScorer], None] = None
+    eval_metric: str | TimeSeriesScorer | Type[TimeSeriesScorer] | None,
+    prediction_length: int,
+    seasonal_period: int | None = None,
+    horizon_weight: Sequence[float] | np.ndarray | None = None,
 ) -> TimeSeriesScorer:
+    """Factory method for TimeSeriesScorer objects.
+
+    Returns
+    -------
+    scorer
+        A `TimeSeriesScorer` object based on the provided `eval_metric`.
+
+        `scorer.prediction_length` is always set to the `prediction_length` provided to this method.
+
+        If `seasonal_period` is not `None`, then `scorer.seasonal_period` is set to this value. Otherwise the original
+        value of `seasonal_period` is kept.
+
+        If `horizon_weight` is not `None`, then `scorer.horizon_weight` is set to this value. Otherwise the original
+        value of `horizon_weight` is kept.
+    """
+    scorer: TimeSeriesScorer
+    metric_kwargs: dict[str, Any] = dict(
+        prediction_length=prediction_length, seasonal_period=seasonal_period, horizon_weight=horizon_weight
+    )
     if isinstance(eval_metric, TimeSeriesScorer):
-        eval_metric = eval_metric
+        scorer = eval_metric
+        scorer.prediction_length = prediction_length
+        if seasonal_period is not None:
+            scorer.seasonal_period = seasonal_period
+        if horizon_weight is not None:
+            scorer.horizon_weight = scorer.check_get_horizon_weight(
+                horizon_weight, prediction_length=prediction_length
+            )
     elif isinstance(eval_metric, type) and issubclass(eval_metric, TimeSeriesScorer):
         # e.g., user passed `eval_metric=CustomMetric` instead of `eval_metric=CustomMetric()`
-        eval_metric = eval_metric()
+        scorer = eval_metric(**metric_kwargs)
     elif isinstance(eval_metric, str):
-        eval_metric = DEPRECATED_METRICS.get(eval_metric, eval_metric)
-        if eval_metric.upper() not in AVAILABLE_METRICS:
+        metric_name = DEPRECATED_METRICS.get(eval_metric, eval_metric).upper()
+        if metric_name in AVAILABLE_METRICS:
+            scorer = AVAILABLE_METRICS[metric_name](**metric_kwargs)
+        elif metric_name in EXPERIMENTAL_METRICS:
+            scorer = EXPERIMENTAL_METRICS[metric_name](**metric_kwargs)
+        else:
             raise ValueError(
                 f"Time series metric {eval_metric} not supported. Available metrics are:\n"
                 f"{pformat(sorted(AVAILABLE_METRICS.keys()))}"
             )
-        eval_metric = AVAILABLE_METRICS[eval_metric.upper()]()
     elif eval_metric is None:
-        eval_metric = AVAILABLE_METRICS[DEFAULT_METRIC_NAME]()
+        scorer = AVAILABLE_METRICS[DEFAULT_METRIC_NAME](**metric_kwargs)
     else:
         raise ValueError(
             f"eval_metric must be of type str, TimeSeriesScorer or None "
             f"(received eval_metric = {eval_metric} of type {type(eval_metric)})"
         )
-    return eval_metric
+    return scorer