oracle-ads 2.11.14__py3-none-any.whl → 2.11.16__py3-none-any.whl

ads/opctl/operator/lowcode/anomaly/model/oneclasssvm.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2023, 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import numpy as np
+import pandas as pd
+
+from ads.common.decorator.runtime_dependency import runtime_dependency
+
+from .base_model import AnomalyOperatorBaseModel
+from .anomaly_dataset import AnomalyOutput
+from ads.opctl.operator.lowcode.anomaly.const import OutputColumns
+
+
+class OneClassSVMOperatorModel(AnomalyOperatorBaseModel):
+    """Class representing OneClassSVM Anomaly Detection operator model."""
+
+    @runtime_dependency(
+        module="sklearn",
+        err_msg=(
+            "Please run `pip3 install scikit-learn` to "
+            "install the required dependencies for OneClassSVM."
+        ),
+    )
+    def _build_model(self) -> AnomalyOutput:
+        from sklearn.svm import OneClassSVM
+
+        model_kwargs = self.spec.model_kwargs
+        # map the output as per anomaly dataset class, 1: outlier, 0: inlier
+        self.outlier_map = {1: 0, -1: 1}
+
+        anomaly_output = AnomalyOutput(date_column="index")
+
+        for target, df in self.datasets.full_data_dict.items():
+            model = OneClassSVM(**model_kwargs)
+            model.fit(df)
+            y_pred = np.vectorize(self.outlier_map.get)(
+                model.predict(df)
+            )
+
+            scores = model.score_samples(
+                df
+            )
+
+            index_col = df.columns[0]
+
+            anomaly = pd.DataFrame(
+                {index_col: df[index_col], OutputColumns.ANOMALY_COL: y_pred}
+            ).reset_index(drop=True)
+            score = pd.DataFrame(
+                {"index": df[index_col], OutputColumns.SCORE_COL: scores}
+            ).reset_index(drop=True)
+
+            anomaly_output.add_output(target, anomaly, score)
+            
+        return anomaly_output
+
+    def _generate_report(self):
+        """Generates the report."""
+        import report_creator as rc
+
+        other_sections = [
+            rc.Heading("Selected Models Overview", level=2),
+            rc.Text(
+                "The following tables provide information regarding the chosen model."
+            ),
+        ]
+
+        model_description = rc.Text(
+            "The oneclasssvm model is a full-stack automated machine learning system for outlier detection. "
+            "It is best suited for novelty detection when the training set is not contaminated by outliers"
+        )
+
+        return (
+            model_description,
+            other_sections,
+        )

ads/opctl/operator/lowcode/anomaly/operator_config.py

@@ -77,6 +77,7 @@ class AnomalyOperatorSpec(DataClassSerializable):
     model: str = None
     model_kwargs: Dict = field(default_factory=dict)
     contamination: float = None
+    subsample_report_data: bool = None
 
     def __post_init__(self):
         """Adjusts the specification details."""

ads/opctl/operator/lowcode/anomaly/schema.yaml

@@ -29,7 +29,7 @@ spec:
     input_data:
       required: true
       type: dict
-      default: {"url": "data.csv"}
+      default: { "url": "data.csv" }
       meta:
         description: "The payload that the detector should evaluate."
       schema:
@@ -78,6 +78,9 @@ spec:
         limit:
           required: false
           type: integer
+        vault_secret_id:
+          required: false
+          type: string
 
     validation_data:
       required: false
@@ -130,10 +133,15 @@ spec:
         limit:
           required: false
           type: integer
+        vault_secret_id:
+          required: false
+          type: string
 
     datetime_column:
       type: dict
-      required: true
+      required: false
+      meta:
+        description: "`datetime_column` is required for time series anomaly detection, only non time-based anomaly detection models can be run without `datetime_column`"
       schema:
         name:
           type: string
@@ -353,6 +361,8 @@ spec:
       allowed:
         - autots
         - auto
+        - oneclasssvm
+        - isolationforest
       meta:
         description: "The model to be used for anomaly detection"
 
@@ -367,4 +377,8 @@ spec:
       type: dict
       required: false
 
+    subsample_report_data:
+      type: boolean
+      required: false
+
   type: dict

ads/opctl/operator/lowcode/anomaly/utils.py

@@ -1,31 +1,32 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*--
 
 # Copyright (c) 2023, 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 import os
+
 import pandas as pd
-import fsspec
-from .operator_config import AnomalyOperatorSpec
-from .const import SupportedMetrics, SupportedModels
+
 from ads.opctl import logger
 
+from .const import NonTimeADSupportedModels, SupportedMetrics, SupportedModels
+from .operator_config import AnomalyOperatorSpec
+
 
 def _build_metrics_df(y_true, y_pred, column_name):
     from sklearn.metrics import (
-        recall_score,
-        precision_score,
         accuracy_score,
-        f1_score,
-        confusion_matrix,
-        roc_auc_score,
-        precision_recall_curve,
         auc,
+        confusion_matrix,
+        f1_score,
         matthews_corrcoef,
+        precision_recall_curve,
+        precision_score,
+        recall_score,
+        roc_auc_score,
     )
 
-    metrics = dict()
+    metrics = {}
     metrics[SupportedMetrics.RECALL] = recall_score(y_true, y_pred)
     metrics[SupportedMetrics.PRECISION] = precision_score(y_true, y_pred)
     metrics[SupportedMetrics.ACCURACY] = accuracy_score(y_true, y_pred)
@@ -78,5 +79,7 @@ def default_signer(**kwargs):
     return default_signer(**kwargs)
 
 
-def select_auto_model(datasets, operator_config):
-    return SupportedModels.AutoTS
+def select_auto_model(operator_config):
+    if operator_config.spec.datetime_column is not None:
+        return SupportedModels.AutoTS
+    return NonTimeADSupportedModels.IsolationForest

ads/opctl/operator/lowcode/common/data.py

@@ -25,6 +25,7 @@ class AbstractData(ABC):
         self.data = None
         self._data_dict = dict()
         self.name = name
+        self.spec = spec
         self.load_transform_ingest_data(spec)
 
     def get_raw_data_by_cat(self, category):
@@ -36,7 +37,7 @@ class AbstractData(ABC):
             for col, val in mapping[category].items():
                 condition &= (self.raw_data[col] == val)
         data_by_cat = self.raw_data[condition].reset_index(drop=True)
-        data_by_cat = self._data_transformer._format_datetime_col(data_by_cat)
+        data_by_cat = self._data_transformer._format_datetime_col(data_by_cat) if self.spec.datetime_column else data_by_cat
         return data_by_cat
 
 

ads/opctl/operator/lowcode/common/errors.py

@@ -39,3 +39,9 @@ class PermissionsError(Exception):
             "complies with the required schema for the operator. \n"
             f"{error}"
         )
+
+
+class InsufficientDataError(Exception):
+    def __init__(self, message: str):
+        self.message = message
+        super().__init__(message)

ads/opctl/operator/lowcode/common/transformations.py

@@ -32,8 +32,14 @@ class Transformations(ABC):
         self.dataset_info = dataset_info
         self.target_category_columns = dataset_info.target_category_columns
         self.target_column_name = dataset_info.target_column
-        self.dt_column_name = dataset_info.datetime_column.name
-        self.dt_column_format = dataset_info.datetime_column.format
+        self.dt_column_name = (
+            dataset_info.datetime_column.name if dataset_info.datetime_column else None
+        )
+        self.dt_column_format = (
+            dataset_info.datetime_column.format
+            if dataset_info.datetime_column
+            else None
+        )
         self.preprocessing = dataset_info.preprocessing
 
     def run(self, data):
@@ -55,8 +61,10 @@ class Transformations(ABC):
         if self.name == "historical_data":
             self._check_historical_dataset(clean_df)
         clean_df = self._set_series_id_column(clean_df)
-        clean_df = self._format_datetime_col(clean_df)
+        if self.dt_column_name:
+            clean_df = self._format_datetime_col(clean_df)
         clean_df = self._set_multi_index(clean_df)
+        clean_df = self._fill_na(clean_df) if not self.dt_column_name else clean_df
 
         if self.preprocessing and self.preprocessing.enabled:
             if self.name == "historical_data":
@@ -66,7 +74,9 @@ class Transformations(ABC):
                     except Exception as e:
                         logger.debug(f"Missing value imputation failed with {e.args}")
                 else:
-                    logger.info("Skipping missing value imputation because it is disabled")
+                    logger.info(
+                        "Skipping missing value imputation because it is disabled"
+                    )
                 if self.preprocessing.steps.outlier_treatment:
                     try:
                         clean_df = self._outlier_treatment(clean_df)
@@ -77,7 +87,9 @@ class Transformations(ABC):
             elif self.name == "additional_data":
                 clean_df = self._missing_value_imputation_add(clean_df)
         else:
-            logger.info("Skipping all preprocessing steps because preprocessing is disabled")
+            logger.info(
+                "Skipping all preprocessing steps because preprocessing is disabled"
+            )
         return clean_df
 
     def _remove_trailing_whitespace(self, df):
@@ -95,7 +107,14 @@ class Transformations(ABC):
             merged_values = df[DataColumns.Series].unique().tolist()
             if self.target_category_columns:
                 for value in merged_values:
-                    self._target_category_columns_map[value] = df[df[DataColumns.Series] == value][self.target_category_columns].drop_duplicates().iloc[0].to_dict()
+                    self._target_category_columns_map[value] = (
+                        df[df[DataColumns.Series] == value][
+                            self.target_category_columns
+                        ]
+                        .drop_duplicates()
+                        .iloc[0]
+                        .to_dict()
+                    )
 
             if self.target_category_columns != [DataColumns.Series]:
                 df = df.drop(self.target_category_columns, axis=1)
@@ -124,8 +143,12 @@ class Transformations(ABC):
         -------
             A new Pandas DataFrame with sorted dates for each series
         """
-        df = df.set_index([self.dt_column_name, DataColumns.Series])
-        return df.sort_values([self.dt_column_name, DataColumns.Series], ascending=True)
+        if self.dt_column_name:
+            df = df.set_index([self.dt_column_name, DataColumns.Series])
+            return df.sort_values(
+                [self.dt_column_name, DataColumns.Series], ascending=True
+            )
+        return df.set_index([df.index, DataColumns.Series])
 
     def _missing_value_imputation_hist(self, df):
         """
@@ -222,5 +245,10 @@ class Transformations(ABC):
             
         }
     """
+
     def get_target_category_columns_map(self):
-        return self._target_category_columns_map
+        return self._target_category_columns_map
+
+    def _fill_na(self, df: pd.DataFrame, na_value=0) -> pd.DataFrame:
+        """Fill nans in dataframe"""
+        return df.fillna(value=na_value)

ads/opctl/operator/lowcode/common/utils.py

@@ -7,7 +7,9 @@
 import argparse
 import logging
 import os
+import shutil
 import sys
+import tempfile
 import time
 from string import Template
 from typing import Any, Dict, List, Tuple
@@ -28,6 +30,7 @@ from ads.opctl.operator.lowcode.common.errors import (
 )
 from ads.opctl.operator.common.operator_config import OutputDirectory
 from ads.common.object_storage_details import ObjectStorageDetails
+from ads.secrets import ADBSecretKeeper
 
 
 def call_pandas_fsspec(pd_fn, filename, storage_options, **kwargs):
@@ -53,10 +56,12 @@ def load_data(data_spec, storage_options=None, **kwargs):
     sql = data_spec.sql
     table_name = data_spec.table_name
     limit = data_spec.limit
-
+    vault_secret_id = data_spec.vault_secret_id
     storage_options = storage_options or (
         default_signer() if ObjectStorageDetails.is_oci_path(filename) else {}
     )
+    if vault_secret_id is not None and connect_args is None:
+        connect_args = dict()
 
     if filename is not None:
         if not format:
@@ -76,15 +81,32 @@ def load_data(data_spec, storage_options=None, **kwargs):
                 f"The format {format} is not currently supported for reading data. Please reformat the data source: {filename} ."
             )
     elif connect_args is not None:
-        con = oracledb.connect(**connect_args)
-        if table_name is not None:
-            data = pd.read_sql_table(table_name, con)
-        elif sql is not None:
-            data = pd.read_sql(sql, con)
-        else:
-            raise InvalidParameterError(
-                f"Database `connect_args` provided without sql query or table name. Please specify either `sql` or `table_name`."
-            )
+        with tempfile.TemporaryDirectory() as temp_dir:
+            if vault_secret_id is not None:
+                try:
+                    with ADBSecretKeeper.load_secret(vault_secret_id, wallet_dir=temp_dir) as adwsecret:
+                        if 'wallet_location' in adwsecret and 'wallet_location' not in connect_args:
+                            shutil.unpack_archive(adwsecret["wallet_location"], temp_dir)
+                            connect_args['wallet_location'] = temp_dir
+                        if 'user_name' in adwsecret and 'user' not in connect_args:
+                            connect_args['user'] = adwsecret['user_name']
+                        if 'password' in adwsecret and 'password' not in connect_args:
+                            connect_args['password'] = adwsecret['password']
+                        if 'service_name' in adwsecret and 'service_name' not in connect_args:
+                            connect_args['service_name'] = adwsecret['service_name']
+
+                except Exception as e:
+                    raise Exception(f"Could not retrieve database credentials from vault {vault_secret_id}: {e}")
+
+            con = oracledb.connect(**connect_args)
+            if table_name is not None:
+                data = pd.read_sql(f"SELECT * FROM {table_name}", con)
+            elif sql is not None:
+                data = pd.read_sql(sql, con)
+            else:
+                raise InvalidParameterError(
+                    f"Database `connect_args` provided without sql query or table name. Please specify either `sql` or `table_name`."
+                )
     else:
         raise InvalidParameterError(
             f"No filename/url provided, and no connect_args provided. Please specify one of these if you want to read data from a file or a database respectively."

ads/opctl/operator/lowcode/forecast/model/base_model.py

@@ -249,20 +249,28 @@ class ForecastOperatorBaseModel(ABC):
                     train_metrics_sections = [sec9_text, sec9]
 
                 backtest_sections = []
+                output_dir = self.spec.output_directory.url
+                backtest_report_name = "backtest_stats.csv"
+                file_path = f"{output_dir}/{backtest_report_name}"
                 if self.spec.model == AUTO_SELECT:
-                    output_dir = self.spec.output_directory.url
-                    backtest_report_name = "backtest_stats.csv"
-                    backtest_stats = pd.read_csv(f"{output_dir}/{backtest_report_name}")
-                    average_dict = backtest_stats.mean().to_dict()
-                    del average_dict['backtest']
-                    best_model = min(average_dict, key=average_dict.get)
-                    backtest_text = rc.Heading("Back Testing Metrics", level=2)
-                    summary_text = rc.Text(
-                        f"Overall, the average scores for the models are {average_dict}, with {best_model}"
-                        f" being identified as the top-performing model during backtesting.")
-                    backtest_table = rc.DataTable(backtest_stats, index=True)
-                    liner_plot = get_auto_select_plot(backtest_stats)
-                    backtest_sections = [backtest_text, backtest_table, summary_text, liner_plot]
+                    backtest_sections.append(rc.Heading("Auto-select statistics", level=2))
+                    if not os.path.exists(file_path):
+                        failure_msg = rc.Text("auto-select could not be executed. Please check the "
+                                              "logs for more details.")
+                        backtest_sections.append(failure_msg)
+                    else:
+                        backtest_stats = pd.read_csv(file_path)
+                        average_dict = backtest_stats.mean().to_dict()
+                        del average_dict['backtest']
+                        best_model = min(average_dict, key=average_dict.get)
+                        backtest_text = rc.Heading("Back Testing Metrics", level=3)
+                        summary_text = rc.Text(
+                            f"Overall, the average scores for the models are {average_dict}, with {best_model}"
+                            f" being identified as the top-performing model during backtesting.")
+                        backtest_table = rc.DataTable(backtest_stats, index=True)
+                        liner_plot = get_auto_select_plot(backtest_stats)
+                        backtest_sections.extend([backtest_text, backtest_table, summary_text,
+                                                                      liner_plot])
 
 
                 forecast_plots = []

ads/opctl/operator/lowcode/forecast/model/ml_forecast.py

@@ -61,6 +61,18 @@ class MLForecastOperatorModel(ForecastOperatorBaseModel):
                 "verbosity": -1,
                 "num_leaves": 512,
             }
+            additional_data_params = {}
+            if len(self.datasets.get_additional_data_column_names()) > 0:
+                additional_data_params = {
+                    "target_transforms": [Differences([12])],
+                    "lags": model_kwargs.get("lags", [1, 6, 12]),
+                    "lag_transforms": (
+                        {
+                            1: [ExpandingMean()],
+                            12: [RollingMean(window_size=24)],
+                        }
+                    ),
+                }
 
             fcst = MLForecast(
                 models={
@@ -80,24 +92,7 @@ class MLForecastOperatorModel(ForecastOperatorBaseModel):
                 },
                 freq=pd.infer_freq(data_train[self.date_col].drop_duplicates())
                 or pd.infer_freq(data_train[self.date_col].drop_duplicates()[-5:]),
-                target_transforms=[Differences([12])],
-                lags=model_kwargs.get(
-                    "lags",
-                    (
-                        [1, 6, 12]
-                        if len(self.datasets.get_additional_data_column_names()) > 0
-                        else []
-                    ),
-                ),
-                lag_transforms=(
-                    {
-                        1: [ExpandingMean()],
-                        12: [RollingMean(window_size=24)],
-                    }
-                    if len(self.datasets.get_additional_data_column_names()) > 0
-                    else {}
-                ),
-                # date_features=[hour_index],
+                **additional_data_params,
             )
 
             num_models = model_kwargs.get("recursive_models", False)
@@ -164,6 +159,7 @@ class MLForecastOperatorModel(ForecastOperatorBaseModel):
                 "error": str(e),
             }
             logger.debug(f"Encountered Error: {e}. Skipping.")
+            raise e
 
     def _build_model(self) -> pd.DataFrame:
         data_train = self.datasets.get_all_data_long(include_horizon=False)

ads/opctl/operator/lowcode/forecast/model_evaluator.py

@@ -12,7 +12,8 @@ from ads.opctl import logger
 from ads.opctl.operator.lowcode.common.const import DataColumns
 from .model.forecast_datasets import ForecastDatasets
 from .operator_config import ForecastOperatorConfig
-
+from ads.opctl.operator.lowcode.forecast.model.factory import SupportedModels
+from ads.opctl.operator.lowcode.common.errors import InsufficientDataError
 
 class ModelEvaluator:
     """
@@ -39,7 +40,7 @@ class ModelEvaluator:
     def generate_cutoffs(self, unique_dates, horizon):
         sorted_dates = np.sort(unique_dates)
         train_window_size = [len(sorted_dates) - (i + 1) * horizon for i in range(self.k)]
-        valid_train_window_size = [ws for ws in train_window_size if ws >= horizon * 3]
+        valid_train_window_size = [ws for ws in train_window_size if ws >= horizon * 2]
         if len(valid_train_window_size) < self.k:
             logger.warn(f"Only {valid_train_window_size} backtests can be created")
         cut_offs = sorted_dates[-horizon - 1:-horizon * (self.k + 1):-horizon][:len(valid_train_window_size)]
@@ -61,6 +62,9 @@ class ModelEvaluator:
         unique_dates = min_series_data[date_col].unique()
 
         cut_offs = self.generate_cutoffs(unique_dates, horizon)
+        if not len(cut_offs):
+            raise InsufficientDataError("Insufficient data to evaluate multiple models. Please specify a model "
+                                        "instead of using auto-select.")
         training_datasets = [sampled_historical_data[sampled_historical_data[date_col] <= cut_off_date] for cut_off_date
                              in cut_offs]
         test_datasets = [sampled_historical_data[sampled_historical_data[date_col] > cut_offs[0]]]
@@ -95,7 +99,9 @@ class ModelEvaluator:
         backtest_op_config_draft = operator_config.to_dict()
         backtest_spec = backtest_op_config_draft["spec"]
         backtest_spec["historical_data"]["url"] = historical_data_url
-        backtest_spec["additional_data"]["url"] = additional_data_url
+        if backtest_spec["additional_data"]:
+            backtest_spec["additional_data"]["url"] = additional_data_url
+        backtest_spec["test_data"] = {}
         backtest_spec["test_data"]["url"] = test_data_url
         backtest_spec["model"] = model
         backtest_spec['model_kwargs'] = None
@@ -135,7 +141,12 @@ class ModelEvaluator:
         return metrics
 
     def find_best_model(self, datasets: ForecastDatasets, operator_config: ForecastOperatorConfig):
-        metrics = self.run_all_models(datasets, operator_config)
+        try:
+            metrics = self.run_all_models(datasets, operator_config)
+        except InsufficientDataError as e:
+            model = SupportedModels.Prophet
+            logger.error(f"Running {model} model as auto-select failed with the following error: {e.message}")
+            return model
         avg_backtests_metrics = {key: sum(value.values()) / len(value.values()) for key, value in metrics.items()}
         best_model = min(avg_backtests_metrics, key=avg_backtests_metrics.get)
         logger.info(f"Among models {self.models}, {best_model} model shows better performance during backtesting.")

ads/opctl/operator/lowcode/forecast/schema.yaml

@@ -78,6 +78,9 @@ spec:
         limit:
           required: false
           type: integer
+        vault_secret_id:
+          required: false
+          type: string
 
     additional_data:
       required: false
@@ -130,6 +133,9 @@ spec:
         limit:
           required: false
           type: integer
+        vault_secret_id:
+          required: false
+          type: string
 
     test_data:
       required: false
@@ -181,6 +187,9 @@ spec:
         limit:
           required: false
           type: integer
+        vault_secret_id:
+          required: false
+          type: string
       type: dict
 
     output_directory:

ads/opctl/operator/lowcode/recommender/MLoperator

@@ -0,0 +1,16 @@
+type: recommender
+version: v1
+conda_type: service
+name: Recommender Operator
+gpu: no
+keywords:
+  - Recommender
+backends:
+  - job
+  - operator.local
+description: |
+  Recommender Systems are designed to suggest relevant items, products, or content to users based on their
+  preferences and behaviors. These systems are widely used in various industries such as e-commerce, entertainment,
+  and social media to enhance user experience by providing personalized recommendations. They help in increasing user
+  engagement, satisfaction, and sales by predicting what users might like or need based on their past interactions
+  and the preferences of similar users.