oracle-ads 2.11.8__py3-none-any.whl → 2.11.10__py3-none-any.whl

ads/model/model_metadata.py

@@ -8,10 +8,10 @@ import json
 import logging
 import os
 import sys
-from abc import ABC, abstractclassmethod, abstractmethod
+from abc import ABC, abstractmethod
 from dataclasses import dataclass, field, fields
 from pathlib import Path
-from typing import Dict, List, Tuple
+from typing import Dict, List, Tuple, Union, Optional, Any
 
 import ads.dataset.factory as factory
 import fsspec
@@ -41,6 +41,8 @@ METADATA_DESCRIPTION_LENGTH_LIMIT = 255
 _METADATA_EMPTY_VALUE = "NA"
 CURRENT_WORKING_DIR = "."
 
+_sentinel = object()
+
 
 class MetadataSizeTooLarge(ValueError):
     """Maximum allowed size for model metadata has been exceeded.
@@ -727,13 +729,18 @@ class ModelMetadata(ABC):
         """Initializes Model Metadata."""
         self._items = set()
 
-    def get(self, key: str) -> ModelMetadataItem:
+    def get(
+        self, key: str, value: Optional[Any] = _sentinel
+    ) -> Union[ModelMetadataItem, Any]:
         """Returns the model metadata item by provided key.
 
         Parameters
         ----------
         key: str
             The key of model metadata item.
+        value: (str, optional)
+            A value to return if the specified key does not exist. Defaults to `object()`.
+            If default value not specified, the ValueError will be returned.
 
         Returns
         -------
@@ -750,7 +757,11 @@ class ModelMetadata(ABC):
         for item in self._items:
             if item.key.lower() == key.lower():
                 return item
-        raise ValueError(f"The metadata with {key} not found.")
+
+        if value is _sentinel:
+            raise ValueError(f"The metadata with {key} not found.")
+
+        return value
 
     def reset(self) -> None:
         """Resets all model metadata items to empty values.
@@ -952,7 +963,7 @@ class ModelMetadata(ABC):
     def __len__(self):
         return len(self._items)
 
-    @abstractclassmethod
+    @abstractmethod
     def _from_oci_metadata(cls, metadata_list):
         pass
 
@@ -967,7 +978,7 @@ class ModelMetadata(ABC):
         """
         pass
 
-    @abstractclassmethod
+    @abstractmethod
     def from_dict(cls, data: Dict) -> "ModelMetadata":
         """Constructs an instance of `ModelMetadata` from a dictionary.
 

ads/opctl/operator/lowcode/anomaly/README.md

@@ -37,8 +37,6 @@ To run anomaly detection locally, create and activate a new conda environment (`
 ```yaml
 - report-creator
 - cerberus
-- oracle-automlx==23.4.1
-- oracle-automlx[classic]==23.4.1
 - "git+https://github.com/oracle/accelerated-data-science.git@feature/anomaly#egg=oracle-ads"
 ```
 

ads/opctl/operator/lowcode/anomaly/__main__.py

@@ -40,11 +40,11 @@ def operate(operator_config: AnomalyOperatorConfig) -> None:
                 AnomalyOperatorModelFactory.get_model(
                     operator_config, datasets
                 ).generate_report()
-            except Exception as e2:
+            except Exception as ee:
                 logger.debug(
-                    f"Failed to backup forecast with error {e2.args}. Raising original error."
+                    f"Failed to backup forecast with error {ee.args}. Raising original error."
                 )
-            raise e
+            raise ee
         else:
             raise e
 

ads/opctl/operator/lowcode/anomaly/environment.yaml

@@ -7,6 +7,4 @@ dependencies:
   - pip:
       - report-creator
       - cerberus
-      - oracle-automlx==23.4.1
-      - oracle-automlx[classic]==23.4.1
       - "oracle-ads[anomaly]"

ads/opctl/operator/lowcode/anomaly/model/automlx.py

@@ -19,8 +19,8 @@ class AutoMLXOperatorModel(AnomalyOperatorBaseModel):
     @runtime_dependency(
         module="automlx",
         err_msg=(
-            "Please run `pip3 install oracle-automlx==23.4.1` and "
-            "`pip3 install oracle-automlx[classic]==23.4.1` "
+            "Please run `pip3 install oracle-automlx>=23.4.1` and "
+            "`pip3 install oracle-automlx[classic]>=23.4.1` "
             "to install the required dependencies for automlx."
         ),
     )

ads/opctl/operator/lowcode/anomaly/model/autots.py

@@ -91,7 +91,7 @@ class AutoTSOperatorModel(AnomalyOperatorBaseModel):
             ),
         ]
         model_description = rc.Text(
-            "The automlx model automatically pre-processes, selects and engineers "
+            "The autots model automatically pre-processes, selects and engineers "
             "high-quality features in your dataset, which then given to an automatically "
             "chosen and optimized machine learning model.."
         )

ads/opctl/operator/lowcode/anomaly/model/base_model.py

@@ -4,33 +4,29 @@
 # Copyright (c) 2023, 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
+import fsspec
+import numpy as np
 import os
+import pandas as pd
 import tempfile
 import time
 from abc import ABC, abstractmethod
-from typing import Tuple
-
-import fsspec
-import pandas as pd
-import numpy as np
 from sklearn import linear_model
+from typing import Tuple
 
+from ads.common.object_storage_details import ObjectStorageDetails
 from ads.opctl import logger
-
-from ..operator_config import AnomalyOperatorConfig, AnomalyOperatorSpec
-from .anomaly_dataset import AnomalyDatasets, AnomalyOutput, TestData
 from ads.opctl.operator.lowcode.anomaly.const import OutputColumns, SupportedMetrics
-from ..const import SupportedModels
+from ads.opctl.operator.lowcode.anomaly.utils import _build_metrics_df, default_signer
 from ads.opctl.operator.lowcode.common.utils import (
     human_time_friendly,
     enable_print,
     disable_print,
     write_data,
-    merge_category_columns,
-    find_output_dirname,
 )
-from ads.opctl.operator.lowcode.anomaly.utils import _build_metrics_df, default_signer
-from ads.common.object_storage_details import ObjectStorageDetails
+from .anomaly_dataset import AnomalyDatasets, AnomalyOutput, TestData
+from ..const import SupportedModels
+from ..operator_config import AnomalyOperatorConfig, AnomalyOperatorSpec
 
 
 class AnomalyOperatorBaseModel(ABC):
@@ -246,7 +242,7 @@ class AnomalyOperatorBaseModel(ABC):
         """Saves resulting reports to the given folder."""
         import report_creator as rc
 
-        unique_output_dir = find_output_dirname(self.spec.output_directory)
+        unique_output_dir = self.spec.output_directory.url
 
         if ObjectStorageDetails.is_oci_path(unique_output_dir):
             storage_options = default_signer()
@@ -320,11 +316,11 @@ class AnomalyOperatorBaseModel(ABC):
         # Iterate over the full_data_dict items
         for target, df in self.datasets.full_data_dict.items():
             est = linear_model.SGDOneClassSVM(random_state=42)
-            est.fit(df[target].values.reshape(-1, 1))
+            est.fit(df[self.spec.target_column].fillna(0).values.reshape(-1, 1))
             y_pred = np.vectorize(self.outlier_map.get)(
-                est.predict(df[target].values.reshape(-1, 1))
+                est.predict(df[self.spec.target_column].fillna(0).values.reshape(-1, 1))
             )
-            scores = est.score_samples(df[target].values.reshape(-1, 1))
+            scores = est.score_samples(df[self.spec.target_column].fillna(0).values.reshape(-1, 1))
 
             anomaly = pd.DataFrame(
                 {date_column: df[date_column], OutputColumns.ANOMALY_COL: y_pred}

ads/opctl/operator/lowcode/anomaly/operator_config.py

@@ -16,6 +16,7 @@ from ads.opctl.operator.common.operator_config import (
     InputData,
 )
 from .const import SupportedModels
+from ads.opctl.operator.lowcode.common.utils import find_output_dirname
 
 
 @dataclass(repr=True)
@@ -79,6 +80,7 @@ class AnomalyOperatorSpec(DataClassSerializable):
 
     def __post_init__(self):
         """Adjusts the specification details."""
+        self.output_directory = self.output_directory or OutputDirectory(url=find_output_dirname(self.output_directory))
         self.report_file_name = self.report_file_name or "report.html"
         self.report_theme = self.report_theme or "light"
         self.inliers_filename = self.inliers_filename or "inliers.csv"

ads/opctl/operator/lowcode/anomaly/schema.yaml

@@ -349,9 +349,8 @@ spec:
     model:
       type: string
       required: false
-      default: automlx
+      default: autots
       allowed:
-        - automlx
         - autots
         - auto
       meta:

ads/opctl/operator/lowcode/anomaly/utils.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*--
 
-# Copyright (c) 2023 Oracle and/or its affiliates.
+# Copyright (c) 2023, 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 import os
@@ -77,5 +77,6 @@ def default_signer(**kwargs):
 
     return default_signer(**kwargs)
 
+
 def select_auto_model(datasets, operator_config):
-    return SupportedModels.AutoMLX
+    return SupportedModels.AutoTS

ads/opctl/operator/lowcode/common/transformations.py

@@ -97,7 +97,8 @@ class Transformations(ABC):
                 for value in merged_values:
                     self._target_category_columns_map[value] = df[df[DataColumns.Series] == value][self.target_category_columns].drop_duplicates().iloc[0].to_dict()
 
-            df = df.drop(self.target_category_columns, axis=1)
+            if self.target_category_columns != [DataColumns.Series]:
+                df = df.drop(self.target_category_columns, axis=1)
         return df
 
     def _format_datetime_col(self, df):

ads/opctl/operator/lowcode/common/utils.py

@@ -215,7 +215,7 @@ def human_time_friendly(seconds):
 
 
 def find_output_dirname(output_dir: OutputDirectory):
-    if output_dir:
+    if output_dir and output_dir.url:
         return output_dir.url
     output_dir = "results"
 

ads/opctl/operator/lowcode/forecast/README.md

@@ -38,9 +38,7 @@ To run forecasting locally, create and activate a new conda environment (`ads-fo
 - report-creator
 - cerberus
 - sktime
-- optuna==3.1.0
-- oracle-automlx==23.4.1
-- oracle-automlx[forecasting]==23.4.1
+- optuna
 - oracle-ads>=2.9.0
 ```
 

ads/opctl/operator/lowcode/forecast/__main__.py

@@ -24,24 +24,9 @@ def operate(operator_config: ForecastOperatorConfig) -> None:
     from .model.factory import ForecastOperatorModelFactory
 
     datasets = ForecastDatasets(operator_config)
-    try:
-        ForecastOperatorModelFactory.get_model(
-            operator_config, datasets
-        ).generate_report()
-    except Exception as e:
-        if operator_config.spec.model == "auto":
-            logger.debug(
-                f"Failed to forecast with error {e.args}. Trying again with model `prophet`."
-            )
-            operator_config.spec.model = "prophet"
-            operator_config.spec.model_kwargs = dict()
-            datasets = ForecastDatasets(operator_config)
-            ForecastOperatorModelFactory.get_model(
-                operator_config, datasets
-            ).generate_report()
-        else:
-            raise
-
+    ForecastOperatorModelFactory.get_model(
+        operator_config, datasets
+    ).generate_report()
 
 def verify(spec: Dict, **kwargs: Dict) -> bool:
     """Verifies the forecasting operator config."""

ads/opctl/operator/lowcode/forecast/const.py

@@ -14,6 +14,7 @@ class SupportedModels(str, metaclass=ExtendedEnumMeta):
     Prophet = "prophet"
     Arima = "arima"
     NeuralProphet = "neuralprophet"
+    MLForecast = "mlforecast"
     AutoMLX = "automlx"
     AutoTS = "autots"
     Auto = "auto"
@@ -86,3 +87,4 @@ DEFAULT_TRIALS = 10
 SUMMARY_METRICS_HORIZON_LIMIT = 10
 PROPHET_INTERNAL_DATE_COL = "ds"
 RENDER_LIMIT = 5000
+AUTO_SELECT = "auto-select"

ads/opctl/operator/lowcode/forecast/environment.yaml

@@ -8,6 +8,7 @@ dependencies:
       - oracle-ads>=2.9.0
       - prophet
       - neuralprophet
+      - mlforecast
       - pmdarima
       - statsmodels
       - report-creator
@@ -16,6 +17,4 @@ dependencies:
       - shap
       - autots[additional]
       - optuna
-      - oracle-automlx>=23.4.1
-      - oracle-automlx[forecasting]>=23.4.1
       - fire

ads/opctl/operator/lowcode/forecast/model/arima.py

@@ -125,6 +125,7 @@ class ArimaOperatorModel(ForecastOperatorBaseModel):
             logger.debug("===========Done===========")
         except Exception as e:
             self.errors_dict[s_id] = {"model_name": self.spec.model, "error": str(e)}
+            logger.debug(f"Encountered Error: {e}. Skipping.")
 
     def _build_model(self) -> pd.DataFrame:
         full_data_dict = self.datasets.get_data_by_series()

ads/opctl/operator/lowcode/forecast/model/automlx.py

@@ -62,8 +62,8 @@ class AutoMLXOperatorModel(ForecastOperatorBaseModel):
     @runtime_dependency(
         module="automlx",
         err_msg=(
-            "Please run `pip3 install oracle-automlx==23.4.1` and "
-            "`pip3 install oracle-automlx[forecasting]==23.4.1` "
+            "Please run `pip3 install oracle-automlx>=23.4.1` and "
+            "`pip3 install oracle-automlx[forecasting]>=23.4.1` "
             "to install the required dependencies for automlx."
         ),
     )
@@ -84,7 +84,7 @@ class AutoMLXOperatorModel(ForecastOperatorBaseModel):
                 loglevel=logging.CRITICAL,
             )
         except Exception as e:
-            logger.info("Ray already initialized")
+            logger.info(f"Error. Has Ray already been initialized? Skipping. {e}")
 
         full_data_dict = self.datasets.get_data_by_series()
 
@@ -113,7 +113,9 @@ class AutoMLXOperatorModel(ForecastOperatorBaseModel):
                 data_i = self.drop_horizon(data)
                 X_pred = self.get_horizon(data).drop(target, axis=1)
 
-                logger.debug(f"Time Index Monotonic: {data_i.index.is_monotonic}")
+                logger.debug(
+                    f"Time Index Monotonic: {data_i.index.is_monotonic_increasing}"
+                )
 
                 if self.loaded_models is not None and s_id in self.loaded_models:
                     model = self.loaded_models[s_id]
@@ -166,6 +168,7 @@ class AutoMLXOperatorModel(ForecastOperatorBaseModel):
                     "model_name": self.spec.model,
                     "error": str(e),
                 }
+                logger.debug(f"Encountered Error: {e}. Skipping.")
 
         logger.debug("===========Forecast Generated===========")
 

ads/opctl/operator/lowcode/forecast/model/autots.py

@@ -209,6 +209,7 @@ class AutoTSOperatorModel(ForecastOperatorBaseModel):
                     "model_name": self.spec.model,
                     "error": str(e),
                 }
+            logger.debug(f"Encountered Error: {e}. Skipping.")
 
         logger.debug("===========Done===========")
 

ads/opctl/operator/lowcode/forecast/model/base_model.py

@@ -4,31 +4,19 @@
 # Copyright (c) 2023, 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
-import json
+import fsspec
+import numpy as np
 import os
+import pandas as pd
 import tempfile
 import time
+import traceback
 from abc import ABC, abstractmethod
 from typing import Tuple
-import traceback
-
-import fsspec
-import numpy as np
-import pandas as pd
 
-from ads.opctl.operator.lowcode.forecast.utils import (
-    default_signer,
-    evaluate_train_metrics,
-    get_forecast_plots,
-    _build_metrics_df,
-    _build_metrics_per_horizon,
-    load_pkl,
-    write_pkl,
-    _label_encode_dataframe,
-)
+from ads.common.decorator.runtime_dependency import runtime_dependency
 from ads.common.object_storage_details import ObjectStorageDetails
 from ads.opctl import logger
-
 from ads.opctl.operator.lowcode.common.utils import (
     human_time_friendly,
     enable_print,
@@ -37,18 +25,28 @@ from ads.opctl.operator.lowcode.common.utils import (
     merged_category_column_name,
     datetime_to_seconds,
     seconds_to_datetime,
-    find_output_dirname,
 )
+from ads.opctl.operator.lowcode.forecast.model.forecast_datasets import TestData
+from ads.opctl.operator.lowcode.forecast.utils import (
+    default_signer,
+    evaluate_train_metrics,
+    get_forecast_plots,
+    get_auto_select_plot,
+    _build_metrics_df,
+    _build_metrics_per_horizon,
+    load_pkl,
+    write_pkl,
+    _label_encode_dataframe,
+)
+from .forecast_datasets import ForecastDatasets
 from ..const import (
     SUMMARY_METRICS_HORIZON_LIMIT,
     SupportedMetrics,
     SupportedModels,
     SpeedAccuracyMode,
+    AUTO_SELECT
 )
 from ..operator_config import ForecastOperatorConfig, ForecastOperatorSpec
-from ads.common.decorator.runtime_dependency import runtime_dependency
-from .forecast_datasets import ForecastDatasets, ForecastOutput
-from ads.opctl.operator.lowcode.forecast.model.forecast_datasets import TestData
 
 
 class ForecastOperatorBaseModel(ABC):
@@ -250,6 +248,23 @@ class ForecastOperatorBaseModel(ABC):
                     sec9 = rc.DataTable(self.eval_metrics, index=True)
                     train_metrics_sections = [sec9_text, sec9]
 
+                backtest_sections = []
+                if self.spec.model == AUTO_SELECT:
+                    output_dir = self.spec.output_directory.url
+                    backtest_report_name = "backtest_stats.csv"
+                    backtest_stats = pd.read_csv(f"{output_dir}/{backtest_report_name}")
+                    average_dict = backtest_stats.mean().to_dict()
+                    del average_dict['backtest']
+                    best_model = min(average_dict, key=average_dict.get)
+                    backtest_text = rc.Heading("Back Testing Metrics", level=2)
+                    summary_text = rc.Text(
+                        f"Overall, the average scores for the models are {average_dict}, with {best_model}"
+                        f" being identified as the top-performing model during backtesting.")
+                    backtest_table = rc.DataTable(backtest_stats, index=True)
+                    liner_plot = get_auto_select_plot(backtest_stats)
+                    backtest_sections = [backtest_text, backtest_table, summary_text, liner_plot]
+
+
                 forecast_plots = []
                 if len(self.forecast_output.list_series_ids()) > 0:
                     forecast_text = rc.Heading(
@@ -276,6 +291,7 @@ class ForecastOperatorBaseModel(ABC):
                 yaml_appendix = rc.Yaml(self.config.to_dict())
                 report_sections = (
                     [summary]
+                    + backtest_sections
                     + forecast_plots
                     + other_sections
                     + test_metrics_sections
@@ -409,7 +425,7 @@ class ForecastOperatorBaseModel(ABC):
         """Saves resulting reports to the given folder."""
         import report_creator as rc
 
-        unique_output_dir = find_output_dirname(self.spec.output_directory)
+        unique_output_dir = self.spec.output_directory.url
 
         if ObjectStorageDetails.is_oci_path(unique_output_dir):
             storage_options = default_signer()

ads/opctl/operator/lowcode/forecast/model/factory.py

@@ -4,7 +4,7 @@
 # Copyright (c) 2023 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
-from ..const import SupportedModels
+from ..const import SupportedModels, AUTO_SELECT
 from ..operator_config import ForecastOperatorConfig
 from .arima import ArimaOperatorModel
 from .automlx import AutoMLXOperatorModel
@@ -12,8 +12,9 @@ from .autots import AutoTSOperatorModel
 from .base_model import ForecastOperatorBaseModel
 from .neuralprophet import NeuralProphetOperatorModel
 from .prophet import ProphetOperatorModel
-from ..utils import select_auto_model
 from .forecast_datasets import ForecastDatasets
+from .ml_forecast import MLForecastOperatorModel
+from ..model_evaluator import ModelEvaluator
 
 class UnSupportedModelError(Exception):
     def __init__(self, model_type: str):
@@ -32,6 +33,7 @@ class ForecastOperatorModelFactory:
         SupportedModels.Prophet: ProphetOperatorModel,
         SupportedModels.Arima: ArimaOperatorModel,
         SupportedModels.NeuralProphet: NeuralProphetOperatorModel,
+        SupportedModels.MLForecast: MLForecastOperatorModel,
         SupportedModels.AutoMLX: AutoMLXOperatorModel,
         SupportedModels.AutoTS: AutoTSOperatorModel
     }
@@ -61,8 +63,35 @@ class ForecastOperatorModelFactory:
             In case of not supported model.
         """
         model_type = operator_config.spec.model
-        if model_type == "auto":
-            model_type = select_auto_model(datasets, operator_config)
+        if model_type == AUTO_SELECT:
+            model_type = cls.auto_select_model(datasets, operator_config)
+            operator_config.spec.model_kwargs = dict()
         if model_type not in cls._MAP:
             raise UnSupportedModelError(model_type)
         return cls._MAP[model_type](config=operator_config, datasets=datasets)
+
+    @classmethod
+    def auto_select_model(
+            cls, datasets: ForecastDatasets, operator_config: ForecastOperatorConfig
+    ) -> str:
+        """
+        Selects AutoMLX or Arima model based on column count.
+
+        If the number of columns is less than or equal to the maximum allowed for AutoMLX,
+        returns 'AutoMLX'. Otherwise, returns 'Arima'.
+
+        Parameters
+        ------------
+        datasets:  ForecastDatasets
+                Datasets for predictions
+
+        Returns
+        --------
+        str
+            The type of the model.
+        """
+        all_models = operator_config.spec.model_kwargs.get("model_list", cls._MAP.keys())
+        num_backtests = operator_config.spec.model_kwargs.get("num_backtests", 5)
+        sample_ratio = operator_config.spec.model_kwargs.get("sample_ratio", 0.20)
+        model_evaluator = ModelEvaluator(all_models, num_backtests, sample_ratio)
+        return model_evaluator.find_best_model(datasets, operator_config)

ads/opctl/operator/lowcode/forecast/model/forecast_datasets.py

@@ -86,7 +86,10 @@ class AdditionalData(AbstractData):
             pd.date_range(
                 start=historical_data.get_max_time(),
                 periods=spec.horizon + 1,
-                freq=historical_data.freq,
+                freq=historical_data.freq
+                or pd.infer_freq(
+                    historical_data.data.reset_index()[spec.datetime_column.name][-5:]
+                ),
             ),
             name=spec.datetime_column.name,
         )
@@ -135,6 +138,7 @@ class ForecastDatasets:
 
         self._horizon = config.spec.horizon
         self._datetime_column_name = config.spec.datetime_column.name
+        self._target_col = config.spec.target_column
         self._load_data(config.spec)
 
     def _load_data(self, spec):
@@ -158,6 +162,16 @@ class ForecastDatasets:
             on=[self._datetime_column_name, ForecastOutputColumns.SERIES],
         ).reset_index()
 
+    def get_all_data_long_forecast_horizon(self):
+        """Returns all data in long format for the forecast horizon."""
+        test_data = pd.merge(
+            self.historical_data.data,
+            self.additional_data.data,
+            how="outer",
+            on=[self._datetime_column_name, ForecastOutputColumns.SERIES],
+        ).reset_index()
+        return test_data[test_data[self._target_col].isnull()].reset_index(drop=True)
+
     def get_data_multi_indexed(self):
         return pd.concat(
             [