oracle-ads 2.10.0__py3-none-any.whl → 2.11.0__py3-none-any.whl

ads/opctl/operator/lowcode/anomaly/__main__.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import json
+import os
+import sys
+from typing import Dict, List
+
+import yaml
+
+from ads.opctl import logger
+from ads.opctl.operator.common.const import ENV_OPERATOR_ARGS
+from ads.opctl.operator.common.utils import _parse_input_args
+
+from .model.anomaly_dataset import AnomalyDatasets, AnomalyData
+from .operator_config import AnomalyOperatorConfig
+
+
+def operate(operator_config: AnomalyOperatorConfig) -> None:
+    """Runs the anomaly detection operator."""
+    from .model.factory import AnomalyOperatorModelFactory
+
+    datasets = AnomalyDatasets(operator_config.spec)
+    try:
+        AnomalyOperatorModelFactory.get_model(
+            operator_config, datasets
+        ).generate_report()
+    except Exception as e:
+        if operator_config.spec.model == "auto":
+            logger.debug(
+                f"Failed to forecast with error {e.args}. Trying again with model `autots`."
+            )
+            operator_config.spec.model = "autots"
+            operator_config.spec.model_kwargs = dict()
+            datasets = AnomalyDatasets(operator_config.spec)
+            try:
+                AnomalyOperatorModelFactory.get_model(
+                    operator_config, datasets
+                ).generate_report()
+            except Exception as e2:
+                logger.debug(
+                    f"Failed to backup forecast with error {e2.args}. Raising original error."
+                )
+            raise e
+        else:
+            raise e
+
+
+def verify(spec: Dict, **kwargs: Dict) -> bool:
+    """Verifies the anomaly detection operator config."""
+    operator = AnomalyOperatorConfig.from_dict(spec)
+    msg_header = (
+        f"{'*' * 50} The operator config has been successfully verified {'*' * 50}"
+    )
+    print(msg_header)
+    print(operator.to_yaml())
+    print("*" * len(msg_header))
+
+
+def main(raw_args: List[str]):
+    """The entry point of the anomaly the operator."""
+    args, _ = _parse_input_args(raw_args)
+    if not args.file and not args.spec and not os.environ.get(ENV_OPERATOR_ARGS):
+        logger.info(
+            "Please specify -f[--file] or -s[--spec] or "
+            f"pass operator's arguments via {ENV_OPERATOR_ARGS} environment variable."
+        )
+        return
+
+    logger.info("-" * 100)
+    logger.info(
+        f"{'Running' if not args.verify else 'Verifying'} the anomaly detection operator."
+    )
+
+    # if spec provided as input string, then convert the string into YAML
+    yaml_string = ""
+    if args.spec or os.environ.get(ENV_OPERATOR_ARGS):
+        operator_spec_str = args.spec or os.environ.get(ENV_OPERATOR_ARGS)
+        try:
+            yaml_string = yaml.safe_dump(json.loads(operator_spec_str))
+        except json.JSONDecodeError:
+            yaml_string = yaml.safe_dump(yaml.safe_load(operator_spec_str))
+        except:
+            yaml_string = operator_spec_str
+
+    operator_config = AnomalyOperatorConfig.from_yaml(
+        uri=args.file,
+        yaml_string=yaml_string,
+    )
+
+    logger.info(operator_config.to_yaml())
+
+    # run operator
+    if args.verify:
+        verify(operator_config)
+    else:
+        operate(operator_config)
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

ads/opctl/operator/lowcode/anomaly/cmd.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from typing import Dict
+
+from ads.opctl import logger
+from ads.opctl.operator.common.utils import _load_yaml_from_uri
+from ads.opctl.operator.common.operator_yaml_generator import YamlGenerator
+
+
+def init(**kwargs: Dict) -> str:
+    """
+    Generates operator config by the schema.
+
+    Properties
+    ----------
+    kwargs: (Dict, optional).
+        Additional key value arguments.
+
+        - type: str
+            The type of the operator.
+
+    Returns
+    -------
+    str
+        The YAML specification generated based on the schema.
+    """
+    logger.info("==== Generating anomaly detection configuration yaml file ====")
+
+    return YamlGenerator(
+        schema=_load_yaml_from_uri(__file__.replace("cmd.py", "schema.yaml"))
+    ).generate_example_dict(values={"type": kwargs.get("type")})

ads/opctl/operator/lowcode/anomaly/const.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from ads.common.extended_enum import ExtendedEnumMeta
+from ads.opctl.operator.lowcode.common.const import DataColumns
+
+
+class SupportedModels(str, metaclass=ExtendedEnumMeta):
+    """Supported anomaly models."""
+
+    AutoMLX = "automlx"
+    AutoTS = "autots"
+    Auto = "auto"
+    # TODS = "tods"
+
+
+class TODSSubModels(str, metaclass=ExtendedEnumMeta):
+    """Supported TODS sub models."""
+
+    OCSVM = "ocsvm"
+    DeepLog = "deeplog"
+    Telemanom = "telemanom"
+    IsolationForest = "isolationforest"
+    LSTMODetector = "lstmodetector"
+    KNN = "knn"
+
+
+TODS_IMPORT_MODEL_MAP = {
+    TODSSubModels.OCSVM: ".OCSVM_skinterface",
+    TODSSubModels.DeepLog: ".DeepLog_skinterface",
+    TODSSubModels.Telemanom: ".Telemanom_skinterface",
+    TODSSubModels.IsolationForest: ".IsolationForest_skinterface",
+    TODSSubModels.LSTMODetector: ".LSTMODetector_skinterface",
+    TODSSubModels.KNN: ".KNN_skinterface",
+}
+
+TODS_MODEL_MAP = {
+    TODSSubModels.OCSVM: "OCSVMSKI",
+    TODSSubModels.DeepLog: "DeepLogSKI",
+    TODSSubModels.Telemanom: "TelemanomSKI",
+    TODSSubModels.IsolationForest: "IsolationForestSKI",
+    TODSSubModels.LSTMODetector: "LSTMODetectorSKI",
+    TODSSubModels.KNN: "KNNSKI",
+}
+
+
+class SupportedMetrics(str, metaclass=ExtendedEnumMeta):
+    UNSUPERVISED_UNIFY95 = "unsupervised_unify95"
+    UNSUPERVISED_UNIFY95_LOG_LOSS = "unsupervised_unify95_log_loss"
+    UNSUPERVISED_N1_EXPERTS = "unsupervised_n-1_experts"
+    RECALL = "Recall"
+    PRECISION = "Precision"
+    ACCURACY = "Accuracy"
+    F1_SCORE = "f1_score"
+    FP = "False Positive"
+    FN = "False Negative"
+    TP = "True Positive"
+    TN = "True Negative"
+    ROC_AUC = "ROC_AUC"
+    PRC_AUC = "PRC_AUC"
+    MCC = "MCC"
+    MEAN_RECALL = "Mean Recall"
+    MEAN_PRECISION = "Mean Precision"
+    MEAN_ACCURACY = "Mean Accuracy"
+    MEAN_F1_SCORE = "Mean f1_score"
+    MEAN_ROC_AUC = "Mean ROC_AUC"
+    MEAN_PRC_AUC = "Mean PRC_AUC"
+    MEAN_MCC = "Mean MCC"
+    MEDIAN_RECALL = "Median Recall"
+    MEDIAN_PRECISION = "Median Precision"
+    MEDIAN_ACCURACY = "Median Accuracy"
+    MEDIAN_F1_SCORE = "Median f1_score"
+    MEDIAN_ROC_AUC = "Median ROC_AUC"
+    MEDIAN_PRC_AUC = "Median PRC_AUC"
+    MEDIAN_MCC = "Median MCC"
+    ELAPSED_TIME = "Elapsed Time"
+
+
+class OutputColumns(str, metaclass=ExtendedEnumMeta):
+    ANOMALY_COL = "anomaly"
+    SCORE_COL = "score"
+    Series = DataColumns.Series
+
+
+TODS_DEFAULT_MODEL = "ocsvm"

ads/opctl/operator/lowcode/anomaly/environment.yaml

@@ -0,0 +1,12 @@
+name: Anomaly
+channels:
+  - conda-forge
+dependencies:
+  - python=3.8
+  - pip
+  - pip:
+      - datapane
+      - cerberus
+      - oracle-automlx==23.4.1
+      - oracle-automlx[classic]==23.4.1
+      - "oracle-ads[anomaly]"

ads/opctl/operator/lowcode/anomaly/model/__init__.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/

ads/opctl/operator/lowcode/anomaly/model/anomaly_dataset.py

@@ -0,0 +1,147 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from ..operator_config import AnomalyOperatorSpec
+from ads.opctl.operator.lowcode.common.utils import (
+    default_signer,
+    merge_category_columns,
+)
+from ads.opctl.operator.lowcode.common.data import AbstractData
+from ads.opctl.operator.lowcode.common.data import AbstractData
+from ads.opctl.operator.lowcode.anomaly.utils import get_frequency_of_datetime
+from ads.opctl import logger
+import pandas as pd
+from ads.opctl.operator.lowcode.anomaly.const import OutputColumns
+
+
+class AnomalyData(AbstractData):
+    def __init__(self, spec: AnomalyOperatorSpec):
+        super().__init__(spec=spec, name="input_data")
+
+
+class TestData(AbstractData):
+    def __init__(self, spec: AnomalyOperatorSpec):
+        super().__init__(spec=spec, name="test_data")
+
+
+class ValidationData(AbstractData):
+    def __init__(self, spec: AnomalyOperatorSpec):
+        super().__init__(spec=spec, name="validation_data")
+
+    def _ingest_data(self, spec):
+        self.X_valid_dict = dict()
+        self.y_valid_dict = dict()
+        for s_id, df in self.get_dict_by_series().items():
+            self.X_valid_dict[s_id] = df.drop([OutputColumns.ANOMALY_COL], axis=1)
+            self.y_valid_dict[s_id] = df[OutputColumns.ANOMALY_COL]
+
+
+class AnomalyDatasets:
+    def __init__(self, spec: AnomalyOperatorSpec):
+        """Instantiates the DataIO instance.
+
+        Properties
+        ----------
+        spec: AnomalyOperatorSpec
+            The anomaly operator spec.
+        """
+        self._data = AnomalyData(spec)
+        self.data = self._data.get_data_long()
+        self.full_data_dict = self._data.get_dict_by_series()
+        if spec.validation_data is not None:
+            self.valid_data = ValidationData(spec)
+            self.X_valid_dict = self.valid_data.X_valid_dict
+            self.y_valid_dict = self.valid_data.y_valid_dict
+
+
+class AnomalyOutput:
+    def __init__(self, date_column):
+        self.category_map = dict()
+        self.date_column = date_column
+
+    def list_categories(self):
+        categories = list(self.category_map.keys())
+        categories.sort()
+        return categories
+
+    def add_output(self, category: str, anomalies: pd.DataFrame, scores: pd.DataFrame):
+        self.category_map[category] = (anomalies, scores)
+
+    def get_anomalies_by_cat(self, category: str):
+        return self.category_map[category][0]
+
+    def get_scores_by_cat(self, category: str):
+        return self.category_map[category][1]
+
+    def get_inliers_by_cat(self, category: str, data: pd.DataFrame):
+        anomaly = self.get_anomalies_by_cat(category)
+        scores = self.get_scores_by_cat(category)
+        inlier_indices = anomaly.index[anomaly[OutputColumns.ANOMALY_COL] == 0]
+        inliers = data.iloc[inlier_indices]
+        if scores is not None and not scores.empty:
+            inliers = pd.merge(inliers, scores, on=self.date_column, how="inner")
+        return inliers
+
+    def get_outliers_by_cat(self, category: str, data: pd.DataFrame):
+        anomaly = self.get_anomalies_by_cat(category)
+        scores = self.get_scores_by_cat(category)
+        outliers_indices = anomaly.index[anomaly[OutputColumns.ANOMALY_COL] == 1]
+        outliers = data.iloc[outliers_indices]
+        if scores is not None and not scores.empty:
+            outliers = pd.merge(outliers, scores, on=self.date_column, how="inner")
+        return outliers
+
+    def get_inliers(self, data):
+        inliers = pd.DataFrame()
+
+        for category in self.list_categories():
+            inliers = pd.concat(
+                [
+                    inliers,
+                    self.get_inliers_by_cat(
+                        category,
+                        data[data[OutputColumns.Series] == category]
+                        .reset_index(drop=True)
+                        .drop(OutputColumns.Series, axis=1),
+                    ),
+                ],
+                axis=0,
+                ignore_index=True,
+            )
+        return inliers
+
+    def get_outliers(self, data):
+        outliers = pd.DataFrame()
+
+        for category in self.list_categories():
+            outliers = pd.concat(
+                [
+                    outliers,
+                    self.get_outliers_by_cat(
+                        category,
+                        data[data[OutputColumns.Series] == category]
+                        .reset_index(drop=True)
+                        .drop(OutputColumns.Series, axis=1),
+                    ),
+                ],
+                axis=0,
+                ignore_index=True,
+            )
+        return outliers
+
+    def get_scores(self, target_category_columns):
+        if target_category_columns is None:
+            return self.get_scores_by_cat(self.list_categories()[0])
+
+        scores = pd.DataFrame()
+        for category in self.list_categories():
+            score = self.get_scores_by_cat(category)
+            score[target_category_columns[0]] = category
+            scores = pd.concat([scores, score], axis=0, ignore_index=True)
+        return scores
+
+    def get_num_anomalies_by_cat(self, category: str):
+        return (self.category_map[category][0][OutputColumns.ANOMALY_COL] == 1).sum()

ads/opctl/operator/lowcode/anomaly/model/automlx.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import pandas as pd
+
+from ads.common.decorator.runtime_dependency import runtime_dependency
+from .anomaly_dataset import AnomalyOutput
+
+from .base_model import AnomalyOperatorBaseModel
+from ads.opctl.operator.lowcode.anomaly.const import OutputColumns
+
+
+class AutoMLXOperatorModel(AnomalyOperatorBaseModel):
+    """Class representing AutoMLX operator model."""
+
+    @runtime_dependency(
+        module="automlx",
+        err_msg=(
+            "Please run `pip3 install oracle-automlx==23.4.1` and "
+            "`pip3 install oracle-automlx[classic]==23.4.1` "
+            "to install the required dependencies for automlx."
+        ),
+    )
+    def _build_model(self) -> pd.DataFrame:
+        from automlx import init
+        try:
+            init(engine="ray", engine_opts={"ray_setup": {"_temp_dir": "/tmp/ray-temp"}})
+        except Exception as e:
+            logger.info("Ray already initialized")
+        date_column = self.spec.datetime_column.name
+        anomaly_output = AnomalyOutput(date_column=date_column)
+        time_budget = self.spec.model_kwargs.pop("time_budget", -1)
+
+        # Iterate over the full_data_dict items
+        for target, df in self.datasets.full_data_dict.items():
+            est = automlx.Pipeline(task="anomaly_detection", **self.spec.model_kwargs)
+            est.fit(
+                X=df,
+                X_valid=self.X_valid_dict[target]
+                if self.X_valid_dict is not None
+                else None,
+                y_valid=self.y_valid_dict[target]
+                if self.y_valid_dict is not None
+                else None,
+                contamination=self.spec.contamination
+                if self.y_valid_dict is not None
+                else None,
+                time_budget=time_budget,
+            )
+            y_pred = est.predict(df)
+            scores = est.predict_proba(df)
+
+            anomaly = pd.DataFrame(
+                {date_column: df[date_column], OutputColumns.ANOMALY_COL: y_pred}
+            ).reset_index(drop=True)
+            score = pd.DataFrame(
+                {
+                    date_column: df[date_column],
+                    OutputColumns.SCORE_COL: [item[1] for item in scores],
+                }
+            ).reset_index(drop=True)
+            anomaly_output.add_output(target, anomaly, score)
+
+        return anomaly_output
+
+    def _generate_report(self):
+        import datapane as dp
+
+        """The method that needs to be implemented on the particular model level."""
+        selected_models_text = dp.Text(
+            f"## Selected Models Overview \n "
+            "The following tables provide information regarding the chosen model."
+        )
+        all_sections = [selected_models_text]
+
+        model_description = dp.Text(
+            "The automlx model automatically pre-processes, selects and engineers "
+            "high-quality features in your dataset, which then given to an automatically "
+            "chosen and optimized machine learning model.."
+        )
+        other_sections = all_sections
+
+        return (
+            model_description,
+            other_sections,
+        )

ads/opctl/operator/lowcode/anomaly/model/autots.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import pandas as pd
+
+from ads.common.decorator.runtime_dependency import runtime_dependency
+
+from .base_model import AnomalyOperatorBaseModel
+from .anomaly_dataset import AnomalyOutput
+from ads.opctl.operator.lowcode.anomaly.const import OutputColumns
+
+
+class AutoTSOperatorModel(AnomalyOperatorBaseModel):
+    """Class representing AutoTS Anomaly Detection operator model."""
+
+    @runtime_dependency(
+        module="autots",
+        err_msg=(
+            "Please run `pip3 install autots` to "
+            "install the required dependencies for AutoTS."
+        ),
+    )
+    def _build_model(self) -> AnomalyOutput:
+        from autots.evaluator.anomaly_detector import AnomalyDetector
+
+        method = self.spec.model_kwargs.get("method")
+        transform_dict = self.spec.model_kwargs.get("transform_dict", {})
+
+        if method == "random" or method == "deep" or method == "fast":
+            new_params = AnomalyDetector.get_new_params(method=method)
+            transform_dict = new_params.pop("transform_dict")
+
+            for key, value in new_params.items():
+                self.spec.model_kwargs[key] = value
+
+        if self.spec.model_kwargs.get("output") is None:
+            self.spec.model_kwargs["output"] = "univariate"
+
+        if "transform_dict" not in self.spec.model_kwargs:
+            self.spec.model_kwargs["transform_dict"] = transform_dict
+
+        if self.spec.contamination != 0.1:  # TODO: remove hard-coding
+            self.spec.model_kwargs.get("method_params", {})[
+                "contamination"
+            ] = self.spec.contamination
+
+        model = AnomalyDetector(**self.spec.model_kwargs)
+
+        date_column = self.spec.datetime_column.name
+
+        anomaly_output = AnomalyOutput(date_column=date_column)
+
+        for target, df in self.datasets.full_data_dict.items():
+            data = df.set_index(date_column)
+
+            (anomaly, score) = model.detect(data)
+
+            if len(anomaly.columns) == 1:
+                score.rename(
+                    columns={score.columns.values[0]: OutputColumns.SCORE_COL},
+                    inplace=True,
+                )
+                score = 1 - score
+                score = score.reset_index(drop=False)
+
+                col = anomaly.columns.values[0]
+                anomaly[col] = anomaly[col].replace({1: 0, -1: 1})
+                anomaly.rename(columns={col: OutputColumns.ANOMALY_COL}, inplace=True)
+                anomaly = anomaly.reset_index(drop=False)
+
+                anomaly_output.add_output(target, anomaly, score)
+
+            else:
+                raise NotImplementedError(
+                    "Multi-Output Anomaly Detection is not yet supported in autots"
+                )
+
+        return anomaly_output
+
+    def _generate_report(self):
+        import datapane as dp
+
+        """The method that needs to be implemented on the particular model level."""
+        selected_models_text = dp.Text(
+            f"## Selected Models Overview \n "
+            "The following tables provide information regarding the chosen model."
+        )
+        all_sections = [selected_models_text]
+
+        model_description = dp.Text(
+            "The automlx model automatically pre-processes, selects and engineers "
+            "high-quality features in your dataset, which then given to an automatically "
+            "chosen and optimized machine learning model.."
+        )
+        other_sections = all_sections
+
+        return (
+            model_description,
+            other_sections,
+        )