autogluon.timeseries 0.8.3b20231102__tar.gz → 0.8.3b20231103__tar.gz

{autogluon.timeseries-0.8.3b20231102 → autogluon.timeseries-0.8.3b20231103}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: autogluon.timeseries
-Version: 0.8.3b20231102
+Version: 0.8.3b20231103
 Summary: AutoML for Image, Text, and Tabular Data
 Home-page: https://github.com/autogluon/autogluon
 Author: AutoGluon Community

{autogluon.timeseries-0.8.3b20231102 → autogluon.timeseries-0.8.3b20231103}/src/autogluon/timeseries/learner.py

@@ -191,6 +191,16 @@ class TimeSeriesLearner(AbstractLearner):
         data = self.feature_generator.transform(data)
         return self.load_trainer().score(data=data, model=model, metric=metric, use_cache=use_cache)
 
+    def evaluate(
+        self,
+        data: Union[TimeSeriesDataFrame, pd.DataFrame, str],
+        model: Optional[str] = None,
+        metrics: Optional[Union[str, TimeSeriesScorer, List[Union[str, TimeSeriesScorer]]]] = None,
+        use_cache: bool = True,
+    ) -> Dict[str, float]:
+        data = self.feature_generator.transform(data)
+        return self.load_trainer().evaluate(data=data, model=model, metrics=metrics, use_cache=use_cache)
+
     def leaderboard(self, data: Optional[TimeSeriesDataFrame] = None, use_cache: bool = True) -> pd.DataFrame:
         if data is not None:
             data = self.feature_generator.transform(data)

{autogluon.timeseries-0.8.3b20231102 → autogluon.timeseries-0.8.3b20231103}/src/autogluon/timeseries/metrics/__init__.py

@@ -1,4 +1,4 @@
-import json
+from pprint import pformat
 from typing import Type, Union
 
 from .abstract import TimeSeriesScorer
@@ -29,11 +29,15 @@ AVAILABLE_METRICS = {
     "WAPE": WAPE,
     "SQL": SQL,
     "WQL": WQL,
-    # Exist for compatibility
     "MSE": MSE,
     "MAE": MAE,
 }
 
+# For backward compatibility
+DEPRECATED_METRICS = {
+    "mean_wQuantileLoss": "WQL",
+}
+
 
 def check_get_evaluation_metric(
     eval_metric: Union[str, TimeSeriesScorer, Type[TimeSeriesScorer], None] = None
@@ -44,10 +48,11 @@ def check_get_evaluation_metric(
         # e.g., user passed `eval_metric=CustomMetric` instead of `eval_metric=CustomMetric()`
         eval_metric = eval_metric()
     elif isinstance(eval_metric, str):
+        eval_metric = DEPRECATED_METRICS.get(eval_metric, eval_metric)
         if eval_metric.upper() not in AVAILABLE_METRICS:
             raise ValueError(
                 f"Time series metric {eval_metric} not supported. Available metrics are:\n"
-                f"{json.dumps(list(AVAILABLE_METRICS.keys()), indent=2)}"
+                f"{pformat(sorted(AVAILABLE_METRICS.keys()))}"
             )
         eval_metric = AVAILABLE_METRICS[eval_metric.upper()]()
     elif eval_metric is None:

{autogluon.timeseries-0.8.3b20231102 → autogluon.timeseries-0.8.3b20231103}/src/autogluon/timeseries/models/gluonts/abstract_gluonts.py

@@ -305,6 +305,7 @@ class AbstractGluonTSModel(AbstractTimeSeriesModel):
         self, time_series_df: Optional[TimeSeriesDataFrame], known_covariates: Optional[TimeSeriesDataFrame] = None
     ) -> Optional[GluonTSDataset]:
         if time_series_df is not None:
+            # TODO: Preprocess real-valued features with StdScaler?
             if self.num_feat_static_cat > 0:
                 feat_static_cat = time_series_df.static_features[self.metadata.static_features_cat]
             else:
@@ -312,6 +313,8 @@ class AbstractGluonTSModel(AbstractTimeSeriesModel):
 
             if self.num_feat_static_real > 0:
                 feat_static_real = time_series_df.static_features[self.metadata.static_features_real]
+                if feat_static_real.isna().values.any():
+                    feat_static_real = feat_static_real.fillna(feat_static_real.mean())
             else:
                 feat_static_real = None
 

{autogluon.timeseries-0.8.3b20231102 → autogluon.timeseries-0.8.3b20231103}/src/autogluon/timeseries/models/local/abstract_local_model.py

@@ -168,6 +168,8 @@ class AbstractLocalModel(AbstractTimeSeriesModel):
                 time_series=time_series,
                 local_model_args=self._local_model_args.copy(),
             )
+            if not np.isfinite(result.values).all():
+                raise RuntimeError("Forecast contains NaN or Inf values.")
             model_failed = False
         except Exception:
             if self.use_fallback_model:
@@ -195,7 +197,8 @@ def seasonal_naive_forecast(
 ) -> pd.DataFrame:
     """Generate seasonal naive forecast, predicting the last observed value from the same period."""
     forecast = {}
-    if len(target) > seasonal_period and seasonal_period > 1:
+    # At least seasonal_period + 2 values are required to compute sigma for seasonal naive
+    if len(target) > seasonal_period + 1 and seasonal_period > 1:
         indices = [len(target) - seasonal_period + k % seasonal_period for k in range(prediction_length)]
         forecast["mean"] = target[indices]
         residuals = target[seasonal_period:] - target[:-seasonal_period]

{autogluon.timeseries-0.8.3b20231102 → autogluon.timeseries-0.8.3b20231103}/src/autogluon/timeseries/models/local/statsforecast.py

@@ -255,9 +255,10 @@ class AutoETSModel(AbstractStatsForecastModel):
         time_series: pd.Series,
         local_model_args: dict,
     ) -> pd.DataFrame:
-        # Disable seasonality if time series too short for chosen season_length or season_length == 1,
-        # otherwise model will crash
-        if len(time_series) < 2 * local_model_args["season_length"] or local_model_args["season_length"] == 1:
+        # Disable seasonality if time series too short for chosen season_length, season_length is too high, or
+        # season_length == 1. Otherwise model will crash
+        season_length = local_model_args["season_length"]
+        if len(time_series) < 2 * season_length or season_length == 1 or season_length > 24:
             # changing last character to "N" disables seasonality, e.g., model="AAA" -> model="AAN"
             local_model_args["model"] = local_model_args["model"][:-1] + "N"
         return super()._predict_with_local_model(time_series=time_series, local_model_args=local_model_args)

{autogluon.timeseries-0.8.3b20231102 → autogluon.timeseries-0.8.3b20231103}/src/autogluon/timeseries/predictor.py

@@ -16,7 +16,7 @@ from autogluon.timeseries import __version__ as current_ag_version
 from autogluon.timeseries.configs import TIMESERIES_PRESETS_CONFIGS
 from autogluon.timeseries.dataset.ts_dataframe import ITEMID, TimeSeriesDataFrame
 from autogluon.timeseries.learner import AbstractLearner, TimeSeriesLearner
-from autogluon.timeseries.metrics import TimeSeriesScorer
+from autogluon.timeseries.metrics import TimeSeriesScorer, check_get_evaluation_metric
 from autogluon.timeseries.splitter import ExpandingWindowSplitter
 from autogluon.timeseries.trainer import AbstractTimeSeriesTrainer
 
@@ -168,15 +168,7 @@ class TimeSeriesPredictor:
             if std_freq != str(self.freq):
                 logger.info(f"Frequency '{self.freq}' stored as '{std_freq}'")
             self.freq = std_freq
-        # TODO: Change to DeprecationWarning, make sure it's displayed correctly https://github.com/autogluon/autogluon/issues/3465
-        if isinstance(eval_metric, str) and eval_metric == "mean_wQuantileLoss":
-            # We don't use warnings.warn since DeprecationWarning may be silenced by the Python warning filters
-            logger.warning(
-                "DeprecationWarning: Evaluation metric 'mean_wQuantileLoss' has been renamed to 'WQL'. "
-                "Support for the old name will be removed in v1.1.",
-            )
-            eval_metric = "WQL"
-        self.eval_metric = eval_metric
+        self.eval_metric = check_get_evaluation_metric(eval_metric)
         self.eval_metric_seasonal_period = eval_metric_seasonal_period
         if quantile_levels is None:
             quantile_levels = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]
@@ -758,11 +750,13 @@ class TimeSeriesPredictor:
         self,
         data: Union[TimeSeriesDataFrame, pd.DataFrame, str],
         model: Optional[str] = None,
-        metric: Union[str, TimeSeriesScorer, None] = None,
+        metrics: Optional[Union[str, TimeSeriesScorer, List[Union[str, TimeSeriesScorer]]]] = None,
         use_cache: bool = True,
-    ):
-        """Evaluate the performance for given dataset, computing the score determined by ``self.eval_metric``
-        on the given data set, and with the same ``prediction_length`` used when training models.
+    ) -> Dict[str, float]:
+        """Evaluate the forecast accuracy for given dataset.
+
+        This method measures the forecast accuracy using the last ``self.prediction_length`` time steps of each time
+        series in ``data`` as a hold-out set.
 
         Parameters
         ----------
@@ -781,25 +775,23 @@ class TimeSeriesPredictor:
         model : str, optional
             Name of the model that you would like to evaluate. By default, the best model during training
             (with highest validation score) will be used.
-        metric : str or TimeSeriesScorer, optional
-            Evaluation metric to compute scores with. Defaults to ``self.eval_metric``
+        metrics : str, TimeSeriesScorer or List[Union[str, TimeSeriesScorer]], optional
+            Metric or a list of metrics to compute scores with. Defaults to ``self.eval_metric``. Supports both
+            metric names as strings and custom metrics based on TimeSeriesScorer.
         use_cache : bool, default = True
             If True, will attempt to use the cached predictions. If False, cached predictions will be ignored.
             This argument is ignored if ``cache_predictions`` was set to False when creating the ``TimeSeriesPredictor``.
 
         Returns
         -------
-        score : float
-            A forecast accuracy score, where higher values indicate better quality. For consistency, error metrics
+        scores_dict : Dict[str, float]
+            Dictionary where keys = metrics, values = performance along each metric. For consistency, error metrics
             will have their signs flipped to obey this convention. For example, negative MAPE values will be reported.
+            To get the ``eval_metric`` score, do ``output[predictor.eval_metric.name]``.
         """
         data = self._check_and_prepare_data_frame(data)
         self._check_data_for_evaluation(data)
-        return self._learner.score(data, model=model, metric=metric, use_cache=use_cache)
-
-    def score(self, data: Union[TimeSeriesDataFrame, pd.DataFrame, str], **kwargs):
-        """See, :meth:`~autogluon.timeseries.TimeSeriesPredictor.evaluate`."""
-        return self.evaluate(data, **kwargs)
+        return self._learner.evaluate(data, model=model, metrics=metrics, use_cache=use_cache)
 
     @classmethod
     def _load_version_file(cls, path: str) -> str:

{autogluon.timeseries-0.8.3b20231102 → autogluon.timeseries-0.8.3b20231103}/src/autogluon/timeseries/trainer/abstract_trainer.py

@@ -836,11 +836,30 @@ class AbstractTimeSeriesTrainer(SimpleAbstractTrainer):
         metric: Union[str, TimeSeriesScorer, None] = None,
         use_cache: bool = True,
     ) -> float:
+        eval_metric = self.eval_metric if metric is None else check_get_evaluation_metric(metric)
+        scores_dict = self.evaluate(data=data, model=model, metrics=[eval_metric], use_cache=use_cache)
+        return scores_dict[eval_metric.name]
+
+    def evaluate(
+        self,
+        data: TimeSeriesDataFrame,
+        model: Optional[Union[str, AbstractTimeSeriesModel]] = None,
+        metrics: Optional[Union[str, TimeSeriesScorer, List[Union[str, TimeSeriesScorer]]]] = None,
+        use_cache: bool = True,
+    ) -> Dict[str, float]:
         past_data, known_covariates = data.get_model_inputs_for_scoring(
             prediction_length=self.prediction_length, known_covariates_names=self.metadata.known_covariates_real
         )
         predictions = self.predict(data=past_data, known_covariates=known_covariates, model=model, use_cache=use_cache)
-        return self._score_with_predictions(data=data, predictions=predictions, metric=metric)
+        if not isinstance(metrics, list):  # a single metric is provided
+            metrics = [metrics]
+        scores_dict = {}
+        for metric in metrics:
+            eval_metric = self.eval_metric if metric is None else check_get_evaluation_metric(metric)
+            scores_dict[eval_metric.name] = self._score_with_predictions(
+                data=data, predictions=predictions, metric=eval_metric
+            )
+        return scores_dict
 
     def _predict_model(
         self,

{autogluon.timeseries-0.8.3b20231102 → autogluon.timeseries-0.8.3b20231103}/src/autogluon/timeseries/version.py

@@ -1,3 +1,3 @@
 """This is the autogluon version file."""
-__version__ = '0.8.3b20231102'
+__version__ = '0.8.3b20231103'
 __lite__ = False

{autogluon.timeseries-0.8.3b20231102 → autogluon.timeseries-0.8.3b20231103}/src/autogluon.timeseries.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: autogluon.timeseries
-Version: 0.8.3b20231102
+Version: 0.8.3b20231103
 Summary: AutoML for Image, Text, and Tabular Data
 Home-page: https://github.com/autogluon/autogluon
 Author: AutoGluon Community

{autogluon.timeseries-0.8.3b20231102 → autogluon.timeseries-0.8.3b20231103}/src/autogluon.timeseries.egg-info/requires.txt

@@ -13,9 +13,9 @@ mlforecast<0.10.1,>=0.10.0
 utilsforecast<0.0.11,>=0.0.10
 tqdm<5,>=4.38
 ujson<6,>=5
-autogluon.core[raytune]==0.8.3b20231102
-autogluon.common==0.8.3b20231102
-autogluon.tabular[catboost,lightgbm,xgboost]==0.8.3b20231102
+autogluon.core[raytune]==0.8.3b20231103
+autogluon.common==0.8.3b20231103
+autogluon.tabular[catboost,lightgbm,xgboost]==0.8.3b20231103
 
 [all]