oracle-ads 2.11.6__py3-none-any.whl → 2.11.8__py3-none-any.whl

ads/opctl/operator/lowcode/anomaly/model/base_model.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*--
 
-# Copyright (c) 2023 Oracle and/or its affiliates.
+# Copyright (c) 2023, 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 import os
@@ -57,7 +57,7 @@ class AnomalyOperatorBaseModel(ABC):
 
     def generate_report(self):
         """Generates the report."""
-        import datapane as dp
+        import report_creator as rc
         import matplotlib.pyplot as plt
 
         start_time = time.time()
@@ -79,12 +79,10 @@ class AnomalyOperatorBaseModel(ABC):
                 anomaly_output, test_data, elapsed_time
             )
         table_blocks = [
-            dp.DataTable(df, label=col)
+            rc.DataTable(df, label=col, index=True)
             for col, df in self.datasets.full_data_dict.items()
         ]
-        data_table = (
-            dp.Select(blocks=table_blocks) if len(table_blocks) > 1 else table_blocks[0]
-        )
+        data_table = rc.Select(blocks=table_blocks)
         date_column = self.spec.datetime_column.name
 
         blocks = []
@@ -106,44 +104,42 @@ class AnomalyOperatorBaseModel(ABC):
                 plt.xlabel(date_column)
                 plt.ylabel(col)
                 plt.title(f"`{col}` with reference to anomalies")
-                figure_blocks.append(ax)
-            blocks.append(dp.Group(blocks=figure_blocks, label=target))
-        plots = dp.Select(blocks=blocks) if len(blocks) > 1 else blocks[0]
+                figure_blocks.append(rc.Widget(ax))
+            blocks.append(rc.Group(*figure_blocks, label=target))
+        plots = rc.Select(blocks)
 
         report_sections = []
-        title_text = dp.Text("# Anomaly Detection Report")
-
-        yaml_appendix_title = dp.Text(f"## Reference: YAML File")
-        yaml_appendix = dp.Code(code=self.config.to_yaml(), language="yaml")
-        summary = dp.Blocks(
-            blocks=[
-                dp.Group(
-                    dp.Text(f"You selected the **`{self.spec.model}`** model."),
-                    dp.Text(
-                        "Based on your dataset, you could have also selected "
-                        f"any of the models: `{'`, `'.join(SupportedModels.keys())}`."
-                    ),
-                    dp.BigNumber(
-                        heading="Analysis was completed in ",
-                        value=human_time_friendly(elapsed_time),
-                    ),
-                    label="Summary",
-                )
-            ]
+        title_text = rc.Heading("Anomaly Detection Report", level=1)
+
+        yaml_appendix_title = rc.Heading("Reference: YAML File", level=2)
+        yaml_appendix = rc.Yaml(self.config.to_dict())
+        summary = rc.Block(
+            rc.Group(
+                rc.Text(f"You selected the **`{self.spec.model}`** model."),
+                rc.Text(
+                    "Based on your dataset, you could have also selected "
+                    f"any of the models: `{'`, `'.join(SupportedModels.keys())}`."
+                ),
+                rc.Metric(
+                    heading="Analysis was completed in ",
+                    value=human_time_friendly(elapsed_time),
+                ),
+                label="Summary",
+            )
         )
-        sec_text = dp.Text(f"## Train Evaluation Metrics")
-        sec = dp.DataTable(self._evaluation_metrics(anomaly_output))
+        sec_text = rc.Heading("Train Evaluation Metrics", level=2)
+        sec = rc.DataTable(self._evaluation_metrics(anomaly_output), index=True)
         evaluation_metrics_sec = [sec_text, sec]
 
         test_metrics_sections = []
         if total_metrics is not None and not total_metrics.empty:
-            sec_text = dp.Text(f"## Test Data Evaluation Metrics")
-            sec = dp.DataTable(total_metrics)
+            sec_text = rc.Heading("Test Data Evaluation Metrics", level=2)
+            sec = rc.DataTable(total_metrics, index=True)
             test_metrics_sections = test_metrics_sections + [sec_text, sec]
 
         if summary_metrics is not None and not summary_metrics.empty:
-            sec_text = dp.Text(f"## Test Data Summary Metrics")
-            sec = dp.DataTable(summary_metrics)
+            sec_text = rc.Heading("Test Data Summary Metrics", level=2)
+            sec = rc.DataTable(summary_metrics, index=True)
             test_metrics_sections = test_metrics_sections + [sec_text, sec]
 
         report_sections = (
@@ -248,7 +244,7 @@ class AnomalyOperatorBaseModel(ABC):
         test_metrics: pd.DataFrame,
     ):
         """Saves resulting reports to the given folder."""
-        import datapane as dp
+        import report_creator as rc
 
         unique_output_dir = find_output_dirname(self.spec.output_directory)
 
@@ -257,11 +253,12 @@ class AnomalyOperatorBaseModel(ABC):
         else:
             storage_options = dict()
 
-        # datapane html report
+        # report-creator html report
         with tempfile.TemporaryDirectory() as temp_dir:
             report_local_path = os.path.join(temp_dir, "___report.html")
             disable_print()
-            dp.save_report(report_sections, report_local_path)
+            with rc.ReportCreator("My Report") as report:
+                report.save(rc.Block(*report_sections), report_local_path)
             enable_print()
             with open(report_local_path) as f1:
                 with fsspec.open(
@@ -272,7 +269,7 @@ class AnomalyOperatorBaseModel(ABC):
                     f2.write(f1.read())
 
         if self.spec.generate_inliers:
-            inliers = anomaly_output.get_inliers(self.datasets.data)
+            inliers = anomaly_output.get_inliers(self.datasets)
             write_data(
                 data=inliers,
                 filename=os.path.join(unique_output_dir, self.spec.inliers_filename),
@@ -280,7 +277,7 @@ class AnomalyOperatorBaseModel(ABC):
                 storage_options=storage_options,
             )
 
-        outliers = anomaly_output.get_outliers(self.datasets.data)
+        outliers = anomaly_output.get_outliers(self.datasets)
         write_data(
             data=outliers,
             filename=os.path.join(unique_output_dir, self.spec.outliers_filename),

ads/opctl/operator/lowcode/anomaly/model/tods.py

@@ -1,7 +1,7 @@
 # #!/usr/bin/env python
 # # -*- coding: utf-8 -*--
 
-# # Copyright (c) 2023 Oracle and/or its affiliates.
+# # Copyright (c) 2023, 2024 Oracle and/or its affiliates.
 # # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 # import importlib
@@ -97,16 +97,16 @@
 #         return anomaly_output
 
 #     def _generate_report(self):
-#         import datapane as dp
+#         import report_creator as rc
 
 #         """The method that needs to be implemented on the particular model level."""
-#         selected_models_text = dp.Text(
+#         selected_models_text = rc.Text(
 #             f"## Selected Models Overview \n "
 #             "The following tables provide information regarding the chosen model."
 #         )
 #         all_sections = [selected_models_text]
 
-#         model_description = dp.Text(
+#         model_description = rc.Text(
 #             "The tods model is a full-stack automated machine learning system for outlier detection "
 #             "on univariate / multivariate time-series data. It provides exhaustive modules for building "
 #             "machine learning-based outlier detection systems and wide range of algorithms."

ads/opctl/operator/lowcode/anomaly/operator_config.py

@@ -36,6 +36,21 @@ class TestData(InputData):
     """Class representing operator specification test data details."""
 
 
+@dataclass(repr=True)
+class PreprocessingSteps(DataClassSerializable):
+    """Class representing preprocessing steps for operator."""
+
+    missing_value_imputation: bool = True
+    outlier_treatment: bool = False
+
+
+@dataclass(repr=True)
+class DataPreprocessor(DataClassSerializable):
+    """Class representing operator specification preprocessing details."""
+
+    enabled: bool = True
+    steps: PreprocessingSteps = field(default_factory=PreprocessingSteps)
+
 @dataclass(repr=True)
 class AnomalyOperatorSpec(DataClassSerializable):
     """Class representing operator specification."""
@@ -74,7 +89,9 @@ class AnomalyOperatorSpec(DataClassSerializable):
             self.generate_inliers if self.generate_inliers is not None else False
         )
         self.model_kwargs = self.model_kwargs or dict()
-
+        self.preprocessing = (
+            self.preprocessing if self.preprocessing is not None else DataPreprocessor(enabled=True)
+        )
 
 @dataclass(repr=True)
 class AnomalyOperatorConfig(OperatorConfig):

ads/opctl/operator/lowcode/anomaly/schema.yaml

@@ -307,11 +307,23 @@ spec:
         description: "When provided, target_category_columns [list] indexes the data into multiple related datasets for anomaly detection"
 
     preprocessing:
-      type: boolean
+      type: dict
       required: false
-      default: true
-      meta:
-        description: "preprocessing and feature engineering can be disabled using this flag, Defaults to true"
+      schema:
+        enabled:
+          type: boolean
+          required: false
+          default: true
+          meta:
+            description: "preprocessing and feature engineering can be disabled using this flag, Defaults to true"
+        steps:
+          type: dict
+          required: false
+          schema:
+            missing_value_imputation:
+              type: boolean
+              required: false
+              default: false
 
     generate_report:
       type: boolean

ads/opctl/operator/lowcode/common/data.py

@@ -16,6 +16,7 @@ from ads.opctl.operator.lowcode.common.errors import (
     DataMismatchError,
 )
 from abc import ABC
+import pandas as pd
 
 
 class AbstractData(ABC):
@@ -26,6 +27,19 @@ class AbstractData(ABC):
         self.name = name
         self.load_transform_ingest_data(spec)
 
+    def get_raw_data_by_cat(self, category):
+        mapping = self._data_transformer.get_target_category_columns_map()
+        # For given category, mapping gives the target_category_columns and it's values.
+        # condition filters raw_data based on the values of target_category_columns for the given category
+        condition = pd.Series(True, index=self.raw_data.index)
+        if category in mapping:
+            for col, val in mapping[category].items():
+                condition &= (self.raw_data[col] == val)
+        data_by_cat = self.raw_data[condition].reset_index(drop=True)
+        data_by_cat = self._data_transformer._format_datetime_col(data_by_cat)
+        return data_by_cat
+
+
     def get_dict_by_series(self):
         if not self._data_dict:
             for s_id in self.list_series_ids():
@@ -73,8 +87,8 @@ class AbstractData(ABC):
         return data
 
     def load_transform_ingest_data(self, spec):
-        raw_data = self._load_data(getattr(spec, self.name))
-        self.data = self._transform_data(spec, raw_data)
+        self.raw_data = self._load_data(getattr(spec, self.name))
+        self.data = self._transform_data(spec, self.raw_data)
         self._ingest_data(spec)
 
     def _ingest_data(self, spec):

ads/opctl/operator/lowcode/common/transformations.py

@@ -58,26 +58,33 @@ class Transformations(ABC):
         clean_df = self._format_datetime_col(clean_df)
         clean_df = self._set_multi_index(clean_df)
 
-        if self.name == "historical_data":
-            try:
-                clean_df = self._missing_value_imputation_hist(clean_df)
-            except Exception as e:
-                logger.debug(f"Missing value imputation failed with {e.args}")
-            if self.preprocessing:
-                try:
-                    clean_df = self._outlier_treatment(clean_df)
-                except Exception as e:
-                    logger.debug(f"Outlier Treatment failed with {e.args}")
-            else:
-                logger.debug("Skipping outlier treatment as preprocessing is disabled")
-        elif self.name == "additional_data":
-            clean_df = self._missing_value_imputation_add(clean_df)
+        if self.preprocessing and self.preprocessing.enabled:
+            if self.name == "historical_data":
+                if self.preprocessing.steps.missing_value_imputation:
+                    try:
+                        clean_df = self._missing_value_imputation_hist(clean_df)
+                    except Exception as e:
+                        logger.debug(f"Missing value imputation failed with {e.args}")
+                else:
+                    logger.info("Skipping missing value imputation because it is disabled")
+                if self.preprocessing.steps.outlier_treatment:
+                    try:
+                        clean_df = self._outlier_treatment(clean_df)
+                    except Exception as e:
+                        logger.debug(f"Outlier Treatment failed with {e.args}")
+                else:
+                    logger.info("Skipping outlier treatment because it is disabled")
+            elif self.name == "additional_data":
+                clean_df = self._missing_value_imputation_add(clean_df)
+        else:
+            logger.info("Skipping all preprocessing steps because preprocessing is disabled")
         return clean_df
 
     def _remove_trailing_whitespace(self, df):
         return df.apply(lambda x: x.str.strip() if x.dtype == "object" else x)
 
     def _set_series_id_column(self, df):
+        self._target_category_columns_map = dict()
         if not self.target_category_columns:
             df[DataColumns.Series] = "Series 1"
             self.has_artificial_series = True
@@ -85,6 +92,11 @@ class Transformations(ABC):
             df[DataColumns.Series] = merge_category_columns(
                 df, self.target_category_columns
             )
+            merged_values = df[DataColumns.Series].unique().tolist()
+            if self.target_category_columns:
+                for value in merged_values:
+                    self._target_category_columns_map[value] = df[df[DataColumns.Series] == value][self.target_category_columns].drop_duplicates().iloc[0].to_dict()
+
             df = df.drop(self.target_category_columns, axis=1)
         return df
 
@@ -189,3 +201,25 @@ class Transformations(ABC):
             raise DataMismatchError(
                 f"Expected {self.name} to have columns: {expected_names}, but instead found column names: {df.columns}. Is the {self.name} path correct?"
             )
+
+    """
+        Map between merged target category column values and target category column and its value
+        If target category columns are PPG_Code, Class, Num
+        Merged target category column values are Product Category 1__A__1, Product Category 2__A__2
+        Then target_category_columns_map would be
+        {
+            "Product Category 1__A__1": {
+                "PPG_Code": "Product Category 1",
+                "Class": "A",
+                "Num": 1
+            },
+             "Product Category 2__A__2": {
+                "PPG_Code": "Product Category 2",
+                "Class": "A",
+                "Num": 2
+            },
+            
+        }
+    """
+    def get_target_category_columns_map(self):
+        return self._target_category_columns_map

ads/opctl/operator/lowcode/forecast/README.md

@@ -35,7 +35,7 @@ To run forecasting locally, create and activate a new conda environment (`ads-fo
 - neuralprophet
 - pmdarima
 - statsmodels
-- datapane
+- report-creator
 - cerberus
 - sktime
 - optuna==3.1.0

ads/opctl/operator/lowcode/forecast/environment.yaml

@@ -10,11 +10,12 @@ dependencies:
       - neuralprophet
       - pmdarima
       - statsmodels
-      - datapane
+      - report-creator
       - cerberus
       - sktime
       - shap
       - autots[additional]
-      - optuna==3.1.0
-      - oracle-automlx==23.4.1
-      - oracle-automlx[forecasting]==23.4.1
+      - optuna
+      - oracle-automlx>=23.4.1
+      - oracle-automlx[forecasting]>=23.4.1
+      - fire

ads/opctl/operator/lowcode/forecast/model/arima.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*--
 
-# Copyright (c) 2023 Oracle and/or its affiliates.
+# Copyright (c) 2023, 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 import pandas as pd
@@ -29,6 +29,7 @@ class ArimaOperatorModel(ForecastOperatorBaseModel):
         self.local_explanation = {}
         self.formatted_global_explanation = None
         self.formatted_local_explanation = None
+        self.constant_cols = {}
 
     def set_kwargs(self):
         # Extract the Confidence Interval Width and convert to arima's equivalent - alpha
@@ -64,6 +65,10 @@ class ArimaOperatorModel(ForecastOperatorBaseModel):
         try:
             target = self.original_target_column
             self.forecast_output.init_series_output(series_id=s_id, data_at_series=df)
+            # If trend is constant, remove constant columns
+            if "trend" not in model_kwargs or model_kwargs["trend"] == "c":
+                self.constant_cols[s_id] = df.columns[df.nunique() == 1]
+                df = df.drop(columns=self.constant_cols[s_id])
 
             # format the dataframe for this target. Dropping NA on target[df] will remove all future data
             data = self.preprocess(df, s_id)
@@ -74,7 +79,7 @@ class ArimaOperatorModel(ForecastOperatorBaseModel):
             X_in = data_i.drop(target, axis=1) if len(data_i.columns) > 1 else None
             X_pred = self.get_horizon(data).drop(target, axis=1)
 
-            if self.loaded_models is not None:
+            if self.loaded_models is not None and s_id in self.loaded_models:
                 model = self.loaded_models[s_id]
             else:
                 # Build and fit model
@@ -142,28 +147,25 @@ class ArimaOperatorModel(ForecastOperatorBaseModel):
 
     def _generate_report(self):
         """The method that needs to be implemented on the particular model level."""
-        import datapane as dp
-
-        sec5_text = dp.Text(f"## ARIMA Model Parameters")
-        blocks = [
-            dp.HTML(
-                m.summary().as_html(),
-                label=s_id,
-            )
-            for i, (s_id, m) in enumerate(self.models.items())
-        ]
-        sec5 = dp.Select(blocks=blocks) if len(blocks) > 1 else blocks[0]
-        all_sections = [sec5_text, sec5]
+        import report_creator as rc
+
+        all_sections = []
+        if len(self.models) > 0:
+            sec5_text = rc.Heading("ARIMA Model Parameters", level=2)
+            blocks = [
+                rc.Html(
+                    m.summary().as_html(),
+                    label=s_id,
+                )
+                for i, (s_id, m) in enumerate(self.models.items())
+            ]
+            sec5 = rc.Select(blocks=blocks)
+            all_sections = [sec5_text, sec5]
 
         if self.spec.generate_explanations:
             try:
                 # If the key is present, call the "explain_model" method
                 self.explain_model()
-                # Create a markdown text block for the global explanation section
-                global_explanation_text = dp.Text(
-                    f"## Global Explanation of Models \n "
-                    "The following tables provide the feature attribution for the global explainability."
-                )
 
                 # Convert the global explanation data to a DataFrame
                 global_explanation_df = pd.DataFrame(self.global_explanation)
@@ -179,9 +181,12 @@ class ArimaOperatorModel(ForecastOperatorBaseModel):
                 )
 
                 # Create a markdown section for the global explainability
-                global_explanation_section = dp.Blocks(
-                    "### Global Explainability ",
-                    dp.DataTable(self.formatted_global_explanation),
+                global_explanation_section = rc.Block(
+                    rc.Heading("Global Explanation of Models", level=2),
+                    rc.Text(
+                        "The following tables provide the feature attribution for the global explainability."
+                    ),
+                    rc.DataTable(self.formatted_global_explanation, index=True),
                 )
 
                 aggregate_local_explanations = pd.DataFrame()
@@ -193,30 +198,29 @@ class ArimaOperatorModel(ForecastOperatorBaseModel):
                     )
                 self.formatted_local_explanation = aggregate_local_explanations
 
-                local_explanation_text = dp.Text(f"## Local Explanation of Models \n ")
                 blocks = [
-                    dp.DataTable(
+                    rc.DataTable(
                         local_ex_df.div(local_ex_df.abs().sum(axis=1), axis=0) * 100,
                         label=s_id,
+                        index=True,
                     )
                     for s_id, local_ex_df in self.local_explanation.items()
                 ]
-                local_explanation_section = (
-                    dp.Select(blocks=blocks) if len(blocks) > 1 else blocks[0]
+                local_explanation_section = rc.Block(
+                    rc.Heading("Local Explanation of Models", level=2),
+                    rc.Select(blocks=blocks),
                 )
 
                 # Append the global explanation text and section to the "all_sections" list
                 all_sections = all_sections + [
-                    global_explanation_text,
                     global_explanation_section,
-                    local_explanation_text,
                     local_explanation_section,
                 ]
             except Exception as e:
                 logger.warn(f"Failed to generate Explanations with error: {e}.")
                 logger.debug(f"Full Traceback: {traceback.format_exc()}")
 
-        model_description = dp.Text(
+        model_description = rc.Text(
             "An autoregressive integrated moving average, or ARIMA, is a statistical "
             "analysis model that uses time series data to either better understand the "
             "data set or to predict future trends. A statistical model is autoregressive if "
@@ -239,6 +243,9 @@ class ArimaOperatorModel(ForecastOperatorBaseModel):
             """
             data: ForecastDatasets.get_data_at_series(s_id)
             """
+            if series_id in self.constant_cols:
+                data = data.drop(columns=self.constant_cols[series_id])
+
             data = data.drop([target_col], axis=1)
             data[dt_column_name] = seconds_to_datetime(
                 data[dt_column_name], dt_format=self.spec.datetime_column.format