oracle-ads 2.10.0__py3-none-any.whl → 2.11.0__py3-none-any.whl

ads/opctl/operator/lowcode/forecast/model/autots.py

@@ -11,13 +11,13 @@ import numpy as np
 import yaml
 
 from ads.opctl import logger
-from ads.opctl.operator.lowcode.forecast import utils
+from ads.opctl.operator.lowcode.common.utils import seconds_to_datetime
 from .base_model import ForecastOperatorBaseModel
 from ..operator_config import ForecastOperatorConfig
 from ads.common.decorator.runtime_dependency import runtime_dependency
 from .forecast_datasets import ForecastDatasets, ForecastOutput
-from ..const import ForecastOutputColumns
-
+from ..const import ForecastOutputColumns, SupportedModels
+from ads.opctl.operator.lowcode.forecast.utils import _select_plot_list
 
 AUTOTS_MAX_GENERATION = 10
 AUTOTS_MODELS_TO_VALIDATE = 0.15
@@ -45,167 +45,158 @@ class AutoTSOperatorModel(ForecastOperatorBaseModel):
         # Import necessary libraries
         from autots import AutoTS, create_regressor
 
+        self.outputs = None
         models = dict()
-        outputs = dict()
-        outputs_legacy = []
         # Get the name of the datetime column
-        date_column = self.spec.datetime_column.name
-        self.datasets.datetime_col = date_column
         self.forecast_output = ForecastOutput(
-            confidence_interval_width=self.spec.confidence_interval_width
+            confidence_interval_width=self.spec.confidence_interval_width,
+            horizon=self.spec.horizon,
+            target_column=self.original_target_column,
+            dt_column=self.spec.datetime_column.name,
         )
+        model = self.loaded_models if self.loaded_models is not None else None
+        if model is None:
+            # Initialize the AutoTS model with specified parameters
+            model = AutoTS(
+                forecast_length=self.spec.horizon,
+                frequency=self.spec.model_kwargs.get(
+                    "frequency", "infer"
+                ),  # TODO: Use datasets.get_datetime_frequency ?
+                prediction_interval=self.spec.confidence_interval_width,
+                max_generations=self.spec.model_kwargs.get(
+                    "max_generations", AUTOTS_MAX_GENERATION
+                ),
+                no_negatives=self.spec.model_kwargs.get("no_negatives", False),
+                constraint=self.spec.model_kwargs.get("constraint", None),
+                ensemble=self.spec.model_kwargs.get("ensemble", "auto"),
+                initial_template=self.spec.model_kwargs.get(
+                    "initial_template", "General+Random"
+                ),
+                random_seed=self.spec.model_kwargs.get("random_seed", 2022),
+                holiday_country=self.spec.model_kwargs.get("holiday_country", "US"),
+                subset=self.spec.model_kwargs.get("subset", None),
+                aggfunc=self.spec.model_kwargs.get("aggfunc", "first"),
+                na_tolerance=self.spec.model_kwargs.get("na_tolerance", 1),
+                drop_most_recent=self.spec.model_kwargs.get("drop_most_recent", 0),
+                drop_data_older_than_periods=self.spec.model_kwargs.get(
+                    "drop_data_older_than_periods", None
+                ),
+                model_list=self.spec.model_kwargs.get("model_list", "fast_parallel"),
+                transformer_list=self.spec.model_kwargs.get("transformer_list", "auto"),
+                transformer_max_depth=self.spec.model_kwargs.get(
+                    "transformer_max_depth", 6
+                ),
+                models_mode=self.spec.model_kwargs.get("models_mode", "random"),
+                num_validations=self.spec.model_kwargs.get("num_validations", "auto"),
+                models_to_validate=self.spec.model_kwargs.get(
+                    "models_to_validate", AUTOTS_MODELS_TO_VALIDATE
+                ),
+                max_per_model_class=self.spec.model_kwargs.get(
+                    "max_per_model_class", None
+                ),
+                validation_method=self.spec.model_kwargs.get(
+                    "validation_method", "backwards"
+                ),
+                min_allowed_train_percent=self.spec.model_kwargs.get(
+                    "min_allowed_train_percent", 0.5
+                ),
+                remove_leading_zeroes=self.spec.model_kwargs.get(
+                    "remove_leading_zeroes", False
+                ),
+                prefill_na=self.spec.model_kwargs.get("prefill_na", None),
+                introduce_na=self.spec.model_kwargs.get("introduce_na", None),
+                preclean=self.spec.model_kwargs.get("preclean", None),
+                model_interrupt=self.spec.model_kwargs.get("model_interrupt", True),
+                generation_timeout=self.spec.model_kwargs.get(
+                    "generation_timeout", None
+                ),
+                current_model_file=self.spec.model_kwargs.get(
+                    "current_model_file", None
+                ),
+                verbose=-1 if logger.level > 40 else 1,
+                n_jobs=self.spec.model_kwargs.get("n_jobs", -1),
+            )
 
-        # Initialize the AutoTS model with specified parameters
-        model = AutoTS(
-            forecast_length=self.spec.horizon,
-            frequency=self.spec.model_kwargs.get("frequency", "infer"),
-            prediction_interval=self.spec.confidence_interval_width,
-            max_generations=self.spec.model_kwargs.get(
-                "max_generations", AUTOTS_MAX_GENERATION
-            ),
-            no_negatives=self.spec.model_kwargs.get("no_negatives", False),
-            constraint=self.spec.model_kwargs.get("constraint", None),
-            ensemble=self.spec.model_kwargs.get("ensemble", "auto"),
-            initial_template=self.spec.model_kwargs.get(
-                "initial_template", "General+Random"
-            ),
-            random_seed=self.spec.model_kwargs.get("random_seed", 2022),
-            holiday_country=self.spec.model_kwargs.get("holiday_country", "US"),
-            subset=self.spec.model_kwargs.get("subset", None),
-            aggfunc=self.spec.model_kwargs.get("aggfunc", "first"),
-            na_tolerance=self.spec.model_kwargs.get("na_tolerance", 1),
-            drop_most_recent=self.spec.model_kwargs.get("drop_most_recent", 0),
-            drop_data_older_than_periods=self.spec.model_kwargs.get(
-                "drop_data_older_than_periods", None
-            ),
-            model_list=self.spec.model_kwargs.get("model_list", "fast_parallel"),
-            transformer_list=self.spec.model_kwargs.get("transformer_list", "auto"),
-            transformer_max_depth=self.spec.model_kwargs.get(
-                "transformer_max_depth", 6
-            ),
-            models_mode=self.spec.model_kwargs.get("models_mode", "random"),
-            num_validations=self.spec.model_kwargs.get("num_validations", "auto"),
-            models_to_validate=self.spec.model_kwargs.get(
-                "models_to_validate", AUTOTS_MODELS_TO_VALIDATE
-            ),
-            max_per_model_class=self.spec.model_kwargs.get("max_per_model_class", None),
-            validation_method=self.spec.model_kwargs.get(
-                "validation_method", "backwards"
-            ),
-            min_allowed_train_percent=self.spec.model_kwargs.get(
-                "min_allowed_train_percent", 0.5
-            ),
-            remove_leading_zeroes=self.spec.model_kwargs.get(
-                "remove_leading_zeroes", False
-            ),
-            prefill_na=self.spec.model_kwargs.get("prefill_na", None),
-            introduce_na=self.spec.model_kwargs.get("introduce_na", None),
-            preclean=self.spec.model_kwargs.get("preclean", None),
-            model_interrupt=self.spec.model_kwargs.get("model_interrupt", True),
-            generation_timeout=self.spec.model_kwargs.get("generation_timeout", None),
-            current_model_file=self.spec.model_kwargs.get("current_model_file", None),
-            verbose=self.spec.model_kwargs.get("verbose", 1),
-            n_jobs=self.spec.model_kwargs.get("n_jobs", -1),
-        )
+        full_data_indexed = self.datasets.get_data_multi_indexed()
 
-        # Prepare the data for model training
-        full_data_dict = self.datasets.full_data_dict
-        temp_list = [full_data_dict[i] for i in full_data_dict.keys()]
-        melt_temp = [
-            temp_list[i].melt(
-                temp_list[i].columns.difference(self.target_columns),
-                var_name="series_id",
-                value_name=self.original_target_column,
-            )
-            for i in range(len(self.target_columns))
-        ]
+        dates = full_data_indexed.index.get_level_values(0).unique().tolist()
+        train_idx = dates[: -self.spec.horizon]
 
-        self.full_data_long = pd.concat(melt_temp)
+        df_train = full_data_indexed[
+            full_data_indexed.index.get_level_values(0).isin(train_idx)
+        ][[self.original_target_column]].reset_index()
 
-        if self.spec.additional_data:
-            df_temp = (
-                self.full_data_long.set_index([self.spec.target_column])
-                .reset_index(drop=True)
-                .copy()
+        # Future regressors need to be in wide format - (num_unique_dates x (num_unique_series x num_unique_cols))
+        additional_regressors = list(
+            set(full_data_indexed.columns) - {self.original_target_column}
+        )
+        if len(additional_regressors) > 1:
+            future_regressor = full_data_indexed.reset_index().pivot(
+                index=self.spec.datetime_column.name,
+                columns=ForecastOutputColumns.SERIES,
+                values=additional_regressors,
             )
-            df_temp[self.spec.datetime_column.name] = pd.to_datetime(
-                df_temp[self.spec.datetime_column.name]
+            future_reg = future_regressor[: -self.spec.horizon]
+            regr_fcst = future_regressor[-self.spec.horizon :]
+        else:
+            future_reg = None
+            regr_fcst = None
+
+        if self.loaded_models is None:
+            model = model.fit(
+                df_train,
+                future_regressor=future_reg,
+                date_col=self.spec.datetime_column.name,
+                value_col=self.original_target_column,
+                id_col=ForecastOutputColumns.SERIES,
             )
-            r_tr, _ = create_regressor(
-                df_temp.pivot(
-                    [self.spec.datetime_column.name],
-                    columns="series_id",
-                    values=list(
-                        self.original_additional_data.set_index(
-                            [
-                                self.spec.target_category_columns[0],
-                                self.spec.datetime_column.name,
-                            ]
-                        ).columns
-                    ),
-                ),
-                forecast_length=self.spec.horizon,
-            )
-
-            self.future_regressor_train = r_tr.copy()
-
-        # Fit the model to the training data
-        model = model.fit(
-            self.full_data_long.groupby("series_id")
-            .head(-self.spec.horizon)
-            .reset_index(drop=True),
-            date_col=self.spec.datetime_column.name,
-            value_col=self.original_target_column,
-            future_regressor=r_tr.head(-self.spec.horizon)
-            if self.spec.additional_data
-            else None,
-            id_col="series_id",
-        )
+            # Store the trained model and generate forecasts
+            self.models = copy.deepcopy(model)
+        else:
+            self.models = self.loaded_models
 
-        # Store the trained model and generate forecasts
-        self.models = copy.deepcopy(model)
+        self.outputs = model.predict(future_regressor=regr_fcst)
         logger.debug("===========Forecast Generated===========")
-        self.prediction = model.predict(
-            future_regressor=r_tr.tail(self.spec.horizon)
-            if self.spec.additional_data
-            else None
-        )
-
-        outputs = dict()
 
-        output_col = pd.DataFrame()
-        yhat_upper_name = ForecastOutputColumns.UPPER_BOUND
-        yhat_lower_name = ForecastOutputColumns.LOWER_BOUND
-
-        for cat in self.categories:
-            output_i = pd.DataFrame()
-            cat_target = f"{self.original_target_column}_{cat}"
-            input_data_i = full_data_dict[cat_target]
-
-            output_i["Date"] = pd.to_datetime(
-                input_data_i[self.spec.datetime_column.name],
-                format=self.spec.datetime_column.format,
+        hist_df = model.back_forecast().forecast
+
+        params = vars(model).copy()
+        for param in [
+            "ens_copy",
+            "df_wide_numeric",
+            "future_regressor_train",
+            "initial_results",
+            "score_per_series",
+            "validation_results",
+            "validation_train_indexes",
+            "validation_test_indexes",
+            "validation_indexes",
+            "best_model",
+        ]:
+            if param in params:
+                params.pop(param)
+
+        for s_id in self.datasets.list_series_ids():
+            self.forecast_output.init_series_output(
+                series_id=s_id, data_at_series=self.datasets.get_data_at_series(s_id)
             )
-            output_i["Series"] = cat
-            output_i["input_value"] = input_data_i[cat_target]
-            output_i["fitted_value"] = float("nan")
-            output_i = output_i.set_index("Date")
-
-            output_i["forecast_value"] = self.prediction.forecast[[cat_target]]
-            output_i[yhat_upper_name] = self.prediction.upper_forecast[[cat_target]]
-            output_i[yhat_lower_name] = self.prediction.lower_forecast[[cat_target]]
-
-            output_i = output_i.reset_index()
-            output_col = pd.concat([output_col, output_i])
-            self.forecast_output.add_category(
-                category=cat, target_category_column=cat_target, forecast=output_i
+
+            self.forecast_output.populate_series_output(
+                series_id=s_id,
+                fit_val=hist_df[s_id].values,
+                forecast_val=self.outputs.forecast[s_id].values,
+                upper_bound=self.outputs.upper_forecast[s_id].values,
+                lower_bound=self.outputs.lower_forecast[s_id].values,
             )
 
-        output_col = output_col.reset_index(drop=True)
+            self.model_parameters[s_id] = {
+                "framework": SupportedModels.AutoTS,
+                **params,
+            }
 
         logger.debug("===========Done===========")
 
-        return output_col
+        return self.forecast_output.get_forecast_long()
 
     def _generate_report(self) -> tuple:
         """
@@ -228,15 +219,9 @@ class AutoTSOperatorModel(ForecastOperatorBaseModel):
             "## Forecast Overview \n"
             "These plots show your forecast in the context of historical data."
         )
-        sec_1 = utils._select_plot_list(
-            lambda idx, *args: self.prediction.plot(
-                self.models.df_wide_numeric,
-                series=self.models.df_wide_numeric.columns[idx],
-                start_date=self.models.df_wide_numeric.reset_index()[
-                    self.spec.datetime_column.name
-                ].min(),
-            ),
-            target_columns=self.target_columns,
+        sec_1 = _select_plot_list(
+            lambda s_id: self.outputs.plot(self.models.df_wide_numeric, series=s_id),
+            self.datasets.list_series_ids(),
         )
 
         # Section 2: AutoTS Model Parameters
@@ -253,65 +238,7 @@ class AutoTSOperatorModel(ForecastOperatorBaseModel):
         all_sections = [sec1_text, sec_1, sec2_text, sec2]
 
         if self.spec.generate_explanations:
-            # If the key is present, call the "explain_model" method
-            try:
-                self.explain_model(
-                    datetime_col_name=self.spec.datetime_column.name,
-                    explain_predict_fn=self._custom_predict_autots,
-                )
-
-                # Create a markdown text block for the global explanation section
-                global_explanation_text = dp.Text(
-                    f"## Global Explanation of Models \n "
-                    "The following tables provide the feature attribution for the global explainability."
-                )
-
-                # Convert the global explanation data to a DataFrame
-                global_explanation_df = pd.DataFrame(self.global_explanation).drop(
-                    index=["series_id", self.spec.target_column]
-                )
-
-                self.formatted_global_explanation = (
-                    global_explanation_df / global_explanation_df.sum(axis=0) * 100
-                )
-
-                # Create a markdown section for the global explainability
-                global_explanation_section = dp.Blocks(
-                    "### Global Explainability ",
-                    dp.DataTable(self.formatted_global_explanation),
-                )
-
-                aggregate_local_explanations = pd.DataFrame()
-                for s_id, local_ex_df in self.local_explanation.items():
-                    local_ex_df_copy = local_ex_df.copy()
-                    local_ex_df_copy["Series"] = s_id
-                    aggregate_local_explanations = pd.concat(
-                        [aggregate_local_explanations, local_ex_df_copy], axis=0
-                    )
-                self.formatted_local_explanation = aggregate_local_explanations
-
-                local_explanation_text = dp.Text(f"## Local Explanation of Models \n ")
-                blocks = [
-                    dp.DataTable(
-                        local_ex_df.div(local_ex_df.abs().sum(axis=1), axis=0) * 100,
-                        label=s_id,
-                    )
-                    for s_id, local_ex_df in self.local_explanation.items()
-                ]
-                local_explanation_section = (
-                    dp.Select(blocks=blocks) if len(blocks) > 1 else blocks[0]
-                )
-
-                # Append the global explanation text and section to the "all_sections" list
-                all_sections = all_sections + [
-                    global_explanation_text,
-                    global_explanation_section,
-                    local_explanation_text,
-                    local_explanation_section,
-                ]
-            except Exception as e:
-                logger.warn(f"Failed to generate Explanations with error: {e}.")
-                logger.debug(f"Full Traceback: {traceback.format_exc()}")
+            logger.warn(f"Explanations not yet supported for the AutoTS Module")
 
         # Model Description
         model_description = dp.Text(
@@ -327,49 +254,12 @@ class AutoTSOperatorModel(ForecastOperatorBaseModel):
             other_sections,
         )
 
-    def _custom_predict_autots(self, data):
-        """
-        Predicts the future values of a time series using the AutoTS model.
-
-        Parameters
-        ----------
-            data (numpy.ndarray): The input data to be used for prediction.
-
-        Returns
-        -------
-            numpy.ndarray: The predicted future values of the time series.
-        """
-
-        data.index = pd.to_datetime(data.index)
-        temp_model = copy.deepcopy(self.models)
-
-        if data.shape[0] > 1:
-            temp_model.fit_data(
-                data[~data.index.duplicated()],
-                future_regressor=self.future_regressor_train.head(-self.spec.horizon),
-            )
-            dedup_shape = data.shape[0] - data[~data.index.duplicated()].shape[0] + 1
-            return pd.Series(0, index=np.arange(dedup_shape)).append(
-                temp_model.back_forecast(
-                    tail=data[~data.index.duplicated()].shape[0] - 1
-                )
-                .forecast[self.spec.target_column]
-                .fillna(0)
-            )
-
-        return temp_model.predict(
-            future_regressor=self.future_regressor_train.loc[
-                self.future_regressor_train.index.isin(data.index)
-            ],
-            forecast_length=1,
-        ).forecast[self.series_id]
-
-    def _generate_train_metrics(self) -> pd.DataFrame:
+    def generate_train_metrics(self) -> pd.DataFrame:
         """
         Generate Training Metrics when fitted data is not available.
         The method that needs to be implemented on the particular model level.
 
-        metrics	Sales_Store 1
+        metrics	Store 1
         sMAPE	26.19
         MAPE	2.96E+18
         RMSE	2014.192531
@@ -380,4 +270,5 @@ class AutoTSOperatorModel(ForecastOperatorBaseModel):
         scores = pd.DataFrame(
             self.models.best_model_per_series_score(), columns=["AutoTS Score"]
         ).T
-        return pd.concat([mapes, scores])
+        df = pd.concat([mapes, scores])
+        return df