oracle-ads 2.13.1rc0__py3-none-any.whl → 2.13.2rc1__py3-none-any.whl

ads/opctl/operator/lowcode/forecast/model_evaluator.py

@@ -1,20 +1,21 @@
-# -*- coding: utf-8; -*-
-
 # Copyright (c) 2023 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 
+from pathlib import Path
+
 import numpy as np
 import pandas as pd
-from pathlib import Path
 
 from ads.opctl import logger
 from ads.opctl.operator.lowcode.common.const import DataColumns
+from ads.opctl.operator.lowcode.common.errors import InsufficientDataError
 from ads.opctl.operator.lowcode.forecast.const import BACKTEST_REPORT_NAME
+from ads.opctl.operator.lowcode.forecast.model.factory import SupportedModels
+
 from .model.forecast_datasets import ForecastDatasets
 from .operator_config import ForecastOperatorConfig
-from ads.opctl.operator.lowcode.forecast.model.factory import SupportedModels
-from ads.opctl.operator.lowcode.common.errors import InsufficientDataError
+
 
 class ModelEvaluator:
     """
@@ -23,6 +24,7 @@ class ModelEvaluator:
     This class is responsible for comparing different models or frameworks based on specified evaluation
     metrics and returning the best-performing option.
     """
+
     def __init__(self, models, k=5, subsample_ratio=0.20):
         """
         Initializes the ModelEvaluator with a list of models, number of backtests and subsample ratio.
@@ -40,23 +42,33 @@ class ModelEvaluator:
 
     def generate_cutoffs(self, unique_dates, horizon):
         sorted_dates = np.sort(unique_dates)
-        train_window_size = [len(sorted_dates) - (i + 1) * horizon for i in range(self.k)]
+        train_window_size = [
+            len(sorted_dates) - (i + 1) * horizon for i in range(self.k)
+        ]
         valid_train_window_size = [ws for ws in train_window_size if ws >= horizon * 2]
         if len(valid_train_window_size) < self.k:
-            logger.warn(f"Only {valid_train_window_size} backtests can be created")
-        cut_offs = sorted_dates[-horizon - 1:-horizon * (self.k + 1):-horizon][:len(valid_train_window_size)]
+            logger.warning(f"Only {valid_train_window_size} backtests can be created")
+        cut_offs = sorted_dates[-horizon - 1 : -horizon * (self.k + 1) : -horizon][
+            : len(valid_train_window_size)
+        ]
         return cut_offs
 
-    def generate_k_fold_data(self, datasets: ForecastDatasets, operator_config: ForecastOperatorConfig):
+    def generate_k_fold_data(
+        self, datasets: ForecastDatasets, operator_config: ForecastOperatorConfig
+    ):
         date_col = operator_config.spec.datetime_column.name
         horizon = operator_config.spec.horizon
         historical_data = datasets.historical_data.data.reset_index()
         series_col = DataColumns.Series
         group_counts = historical_data[series_col].value_counts()
 
-        sample_count = max(self.minimum_sample_count, int(len(group_counts) * self.subsample_ratio))
+        sample_count = max(
+            self.minimum_sample_count, int(len(group_counts) * self.subsample_ratio)
+        )
         sampled_groups = group_counts.head(sample_count)
-        sampled_historical_data = historical_data[historical_data[series_col].isin(sampled_groups.index)]
+        sampled_historical_data = historical_data[
+            historical_data[series_col].isin(sampled_groups.index)
+        ]
 
         min_group = group_counts.idxmin()
         min_series_data = historical_data[historical_data[series_col] == min_group]
@@ -64,99 +76,167 @@ class ModelEvaluator:
 
         cut_offs = self.generate_cutoffs(unique_dates, horizon)
         if not len(cut_offs):
-            raise InsufficientDataError("Insufficient data to evaluate multiple models. Please specify a model "
-                                        "instead of using auto-select.")
-        training_datasets = [sampled_historical_data[sampled_historical_data[date_col] <= cut_off_date] for cut_off_date
-                             in cut_offs]
-        test_datasets = [sampled_historical_data[sampled_historical_data[date_col] > cut_offs[0]]]
+            raise InsufficientDataError(
+                "Insufficient data to evaluate multiple models. Please specify a model "
+                "instead of using auto-select."
+            )
+        training_datasets = [
+            sampled_historical_data[sampled_historical_data[date_col] <= cut_off_date]
+            for cut_off_date in cut_offs
+        ]
+        test_datasets = [
+            sampled_historical_data[sampled_historical_data[date_col] > cut_offs[0]]
+        ]
         for i, current in enumerate(cut_offs[1:]):
-            test_datasets.append(sampled_historical_data[(current < sampled_historical_data[date_col]) & (
-                    sampled_historical_data[date_col] <= cut_offs[i])])
+            test_datasets.append(
+                sampled_historical_data[
+                    (current < sampled_historical_data[date_col])
+                    & (sampled_historical_data[date_col] <= cut_offs[i])
+                ]
+            )
         all_additional = datasets.additional_data.data.reset_index()
-        sampled_additional_data = all_additional[all_additional[series_col].isin(sampled_groups.index)]
+        sampled_additional_data = all_additional[
+            all_additional[series_col].isin(sampled_groups.index)
+        ]
         max_historical_date = sampled_historical_data[date_col].max()
-        additional_data = [sampled_additional_data[sampled_additional_data[date_col] <= max_historical_date]]
+        additional_data = [
+            sampled_additional_data[
+                sampled_additional_data[date_col] <= max_historical_date
+            ]
+        ]
         for cut_off in cut_offs[:-1]:
-            trimmed_additional_data = sampled_additional_data[sampled_additional_data[date_col] <= cut_off]
+            trimmed_additional_data = sampled_additional_data[
+                sampled_additional_data[date_col] <= cut_off
+            ]
             additional_data.append(trimmed_additional_data)
         return cut_offs, training_datasets, additional_data, test_datasets
 
     def remove_none_values(self, obj):
         if isinstance(obj, dict):
-            return {k: self.remove_none_values(v) for k, v in obj.items() if k is not None and v is not None}
+            return {
+                k: self.remove_none_values(v)
+                for k, v in obj.items()
+                if k is not None and v is not None
+            }
         else:
             return obj
 
-    def create_operator_config(self, operator_config, backtest, model, historical_data, additional_data, test_data):
+    def create_operator_config(
+        self,
+        operator_config,
+        backtest,
+        model,
+        historical_data,
+        additional_data,
+        test_data,
+    ):
         output_dir = operator_config.spec.output_directory.url
-        output_file_path = f'{output_dir}/back_testing/{model}/{backtest}'
+        output_file_path = f"{output_dir}/back_testing/{model}/{backtest}"
         Path(output_file_path).mkdir(parents=True, exist_ok=True)
-        historical_data_url = f'{output_file_path}/historical.csv'
-        additional_data_url = f'{output_file_path}/additional.csv'
-        test_data_url = f'{output_file_path}/test.csv'
-        historical_data.to_csv(historical_data_url, index=False)
-        additional_data.to_csv(additional_data_url, index=False)
-        test_data.to_csv(test_data_url, index=False)
         backtest_op_config_draft = operator_config.to_dict()
         backtest_spec = backtest_op_config_draft["spec"]
-        backtest_spec["historical_data"]["url"] = historical_data_url
-        if backtest_spec["additional_data"]:
-            backtest_spec["additional_data"]["url"] = additional_data_url
-        backtest_spec["test_data"] = {}
-        backtest_spec["test_data"]["url"] = test_data_url
+        backtest_spec["datetime_column"]["format"] = None
+        backtest_spec.pop("test_data")
+        backtest_spec.pop("additional_data")
+        backtest_spec.pop("historical_data")
+        backtest_spec["generate_report"] = False
         backtest_spec["model"] = model
-        backtest_spec['model_kwargs'] = None
+        backtest_spec["model_kwargs"] = None
         backtest_spec["output_directory"] = {"url": output_file_path}
         backtest_spec["target_category_columns"] = [DataColumns.Series]
-        backtest_spec['generate_explanations'] = False
+        backtest_spec["generate_explanations"] = False
         cleaned_config = self.remove_none_values(backtest_op_config_draft)
 
-        backtest_op_config = ForecastOperatorConfig.from_dict(
-            obj_dict=cleaned_config)
+        backtest_op_config = ForecastOperatorConfig.from_dict(obj_dict=cleaned_config)
         return backtest_op_config
 
-    def run_all_models(self, datasets: ForecastDatasets, operator_config: ForecastOperatorConfig):
-        cut_offs, train_sets, additional_data, test_sets = self.generate_k_fold_data(datasets, operator_config)
+    def run_all_models(
+        self, datasets: ForecastDatasets, operator_config: ForecastOperatorConfig
+    ):
+        cut_offs, train_sets, additional_data, test_sets = self.generate_k_fold_data(
+            datasets, operator_config
+        )
         metrics = {}
+        date_col = operator_config.spec.datetime_column.name
         for model in self.models:
             from .model.factory import ForecastOperatorModelFactory
+
             metrics[model] = {}
             for i in range(len(cut_offs)):
                 try:
-                    backtest_historical_data = train_sets[i]
-                    backtest_additional_data = additional_data[i]
-                    backtest_test_data = test_sets[i]
-                    backtest_operator_config = self.create_operator_config(operator_config, i, model,
-                                                                           backtest_historical_data,
-                                                                           backtest_additional_data,
-                                                                           backtest_test_data)
-                    datasets = ForecastDatasets(backtest_operator_config)
+                    backtest_historical_data = train_sets[i].set_index(
+                        [date_col, DataColumns.Series]
+                    )
+                    backtest_additional_data = additional_data[i].set_index(
+                        [date_col, DataColumns.Series]
+                    )
+                    backtest_test_data = test_sets[i].set_index(
+                        [date_col, DataColumns.Series]
+                    )
+                    backtest_operator_config = self.create_operator_config(
+                        operator_config,
+                        i,
+                        model,
+                        backtest_historical_data,
+                        backtest_additional_data,
+                        backtest_test_data,
+                    )
+                    datasets = ForecastDatasets(
+                        backtest_operator_config,
+                        backtest_historical_data,
+                        backtest_additional_data,
+                        backtest_test_data,
+                    )
                     ForecastOperatorModelFactory.get_model(
                         backtest_operator_config, datasets
                     ).generate_report()
-                    test_metrics_filename = backtest_operator_config.spec.test_metrics_filename
+                    test_metrics_filename = (
+                        backtest_operator_config.spec.test_metrics_filename
+                    )
                     metrics_df = pd.read_csv(
-                        f"{backtest_operator_config.spec.output_directory.url}/{test_metrics_filename}")
-                    metrics_df["average_across_series"] = metrics_df.drop('metrics', axis=1).mean(axis=1)
-                    metrics_average_dict = dict(zip(metrics_df['metrics'].str.lower(), metrics_df['average_across_series']))
-                    metrics[model][i] = metrics_average_dict[operator_config.spec.metric]
+                        f"{backtest_operator_config.spec.output_directory.url}/{test_metrics_filename}"
+                    )
+                    metrics_df["average_across_series"] = metrics_df.drop(
+                        "metrics", axis=1
+                    ).mean(axis=1)
+                    metrics_average_dict = dict(
+                        zip(
+                            metrics_df["metrics"].str.lower(),
+                            metrics_df["average_across_series"],
+                        )
+                    )
+                    metrics[model][i] = metrics_average_dict[
+                        operator_config.spec.metric
+                    ]
                 except:
-                    logger.warn(f"Failed to calculate metrics for {model} and {i} backtest")
+                    logger.warning(
+                        f"Failed to calculate metrics for {model} and {i} backtest"
+                    )
         return metrics
 
-    def find_best_model(self, datasets: ForecastDatasets, operator_config: ForecastOperatorConfig):
+    def find_best_model(
+        self, datasets: ForecastDatasets, operator_config: ForecastOperatorConfig
+    ):
         try:
             metrics = self.run_all_models(datasets, operator_config)
         except InsufficientDataError as e:
             model = SupportedModels.Prophet
-            logger.error(f"Running {model} model as auto-select failed with the following error: {e.message}")
+            logger.error(
+                f"Running {model} model as auto-select failed with the following error: {e.message}"
+            )
             return model
-        nonempty_metrics = {model: metric for model, metric in metrics.items() if metric != {}}
-        avg_backtests_metric = {model: sum(value.values()) / len(value.values())
-                                for model, value in nonempty_metrics.items()}
+        nonempty_metrics = {
+            model: metric for model, metric in metrics.items() if metric != {}
+        }
+        avg_backtests_metric = {
+            model: sum(value.values()) / len(value.values())
+            for model, value in nonempty_metrics.items()
+        }
         best_model = min(avg_backtests_metric, key=avg_backtests_metric.get)
-        logger.info(f"Among models {self.models}, {best_model} model shows better performance during backtesting.")
-        backtest_stats = pd.DataFrame(nonempty_metrics).rename_axis('backtest')
+        logger.info(
+            f"Among models {self.models}, {best_model} model shows better performance during backtesting."
+        )
+        backtest_stats = pd.DataFrame(nonempty_metrics).rename_axis("backtest")
         backtest_stats["metric"] = operator_config.spec.metric
         backtest_stats.reset_index(inplace=True)
         output_dir = operator_config.spec.output_directory.url

ads/opctl/operator/lowcode/forecast/operator_config.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-# Copyright (c) 2023, 2024 Oracle and/or its affiliates.
+# Copyright (c) 2023, 2025 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 import os
@@ -18,9 +18,11 @@ from ads.opctl.operator.lowcode.common.utils import find_output_dirname
 
 from .const import SpeedAccuracyMode, SupportedMetrics, SupportedModels
 
+
 @dataclass
 class AutoScaling(DataClassSerializable):
     """Class representing simple autoscaling policy"""
+
     minimum_instance: int = 1
     maximum_instance: int = None
     cool_down_in_seconds: int = 600
@@ -28,9 +30,11 @@ class AutoScaling(DataClassSerializable):
     scale_out_threshold: int = 80
     scaling_metric: str = "CPU_UTILIZATION"
 
+
 @dataclass(repr=True)
 class ModelDeploymentServer(DataClassSerializable):
     """Class representing model deployment server specification for whatif-analysis."""
+
     display_name: str = None
     initial_shape: str = None
     description: str = None
@@ -42,10 +46,13 @@ class ModelDeploymentServer(DataClassSerializable):
 @dataclass(repr=True)
 class WhatIfAnalysis(DataClassSerializable):
     """Class representing operator specification for whatif-analysis."""
+
     model_display_name: str = None
     compartment_id: str = None
     project_id: str = None
-    model_deployment: ModelDeploymentServer = field(default_factory=ModelDeploymentServer)
+    model_deployment: ModelDeploymentServer = field(
+        default_factory=ModelDeploymentServer
+    )
 
 
 @dataclass(repr=True)
@@ -106,8 +113,11 @@ class ForecastOperatorSpec(DataClassSerializable):
     datetime_column: DateTimeColumn = field(default_factory=DateTimeColumn)
     target_category_columns: List[str] = field(default_factory=list)
     generate_report: bool = None
+    generate_forecast_file: bool = None
     generate_metrics: bool = None
+    generate_metrics_file: bool = None
     generate_explanations: bool = None
+    generate_explanation_files: bool = None
     explanations_accuracy_mode: str = None
     horizon: int = None
     model: str = None
@@ -126,7 +136,7 @@ class ForecastOperatorSpec(DataClassSerializable):
         self.output_directory = self.output_directory or OutputDirectory(
             url=find_output_dirname(self.output_directory)
         )
-        self.generate_model_pickle = True if self.generate_model_pickle or self.what_if_analysis else False
+        self.generate_model_pickle = self.generate_model_pickle or self.what_if_analysis
         self.metric = (self.metric or "").lower() or SupportedMetrics.SMAPE.lower()
         self.model = self.model or SupportedModels.Prophet
         self.confidence_interval_width = self.confidence_interval_width or 0.80
@@ -144,6 +154,21 @@ class ForecastOperatorSpec(DataClassSerializable):
         self.generate_metrics = (
             self.generate_metrics if self.generate_metrics is not None else True
         )
+        self.generate_metrics_file = (
+            self.generate_metrics_file
+            if self.generate_metrics_file is not None
+            else True
+        )
+        self.generate_forecast_file = (
+            self.generate_forecast_file
+            if self.generate_forecast_file is not None
+            else True
+        )
+        self.generate_explanation_files = (
+            self.generate_explanation_files
+            if self.generate_explanation_files is not None
+            else True
+        )
         # For Explanations Generation. When user doesn't specify defaults to False
         self.generate_explanations = (
             self.generate_explanations
@@ -164,6 +189,7 @@ class ForecastOperatorSpec(DataClassSerializable):
             if self.generate_model_pickle is not None
             else False
         )
+        self.report_title = self.report_title or "Forecast Report"
         self.report_theme = self.report_theme or "light"
         self.metrics_filename = self.metrics_filename or "metrics.csv"
         self.test_metrics_filename = self.test_metrics_filename or "test_metrics.csv"

ads/opctl/operator/lowcode/forecast/schema.yaml

@@ -379,7 +379,7 @@ spec:
               required: true
             log_id:
               type: string
-              required: true
+              required: false
             auto_scaling:
               type: dict
               required: false