oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.10__py3-none-any.whl

ads/opctl/operator/lowcode/anomaly/schema.yaml

@@ -0,0 +1,401 @@
+kind:
+  allowed:
+    - operator
+  required: true
+  type: string
+  default: operator
+  meta:
+    description: "Which service are you trying to use? Common kinds: `operator`, `job`"
+
+version:
+  allowed:
+    - "v1"
+  required: true
+  type: string
+  default: v1
+  meta:
+    description: "Operators may change yaml file schemas from version to version, as well as implementation details. Double check the version to ensure compatibility."
+
+type:
+  required: true
+  type: string
+  default: anomaly
+  meta:
+    description: "Type should always be `anomaly` when using a anomaly detection operator"
+
+spec:
+  required: true
+  schema:
+    input_data:
+      required: true
+      type: dict
+      default: { "url": "data.csv" }
+      meta:
+        description: "The payload that the detector should evaluate."
+      schema:
+        connect_args:
+          nullable: true
+          required: false
+          type: dict
+        format:
+          allowed:
+            - csv
+            - json
+            - clipboard
+            - excel
+            - feather
+            - sql_table
+            - sql_query
+            - hdf
+            - tsv
+          required: false
+          type: string
+        columns:
+          required: false
+          type: list
+          schema:
+            type: string
+        filters:
+          required: false
+          type: list
+          schema:
+            type: string
+        options:
+          nullable: true
+          required: false
+          type: dict
+        sql:
+          required: false
+          type: string
+        table_name:
+          required: false
+          type: string
+        url:
+          required: false
+          type: string
+          meta:
+            description: "The url can be local, or remote. For example: `oci://<bucket>@<namespace>/data.csv`"
+        limit:
+          required: false
+          type: integer
+        vault_secret_id:
+          required: false
+          type: string
+
+    validation_data:
+      required: false
+      type: dict
+      meta:
+        description: "Data that has already been labeled as anomalous or not."
+      schema:
+        connect_args:
+          nullable: true
+          required: false
+          type: dict
+        format:
+          allowed:
+            - csv
+            - json
+            - clipboard
+            - excel
+            - feather
+            - sql_table
+            - sql_query
+            - hdf
+            - tsv
+          required: false
+          type: string
+        columns:
+          required: false
+          type: list
+          schema:
+            type: string
+        filters:
+          required: false
+          type: list
+          schema:
+            type: string
+        options:
+          nullable: true
+          required: false
+          type: dict
+        sql:
+          required: false
+          type: string
+        table_name:
+          required: false
+          type: string
+        url:
+          required: false
+          type: string
+          meta:
+            description: "The url can be local, or remote. For example: `oci://<bucket>@<namespace>/data.csv`"
+        limit:
+          required: false
+          type: integer
+        vault_secret_id:
+          required: false
+          type: string
+
+    datetime_column:
+      type: dict
+      required: false
+      meta:
+        description: "`datetime_column` is required for time series anomaly detection, only non time-based anomaly detection models can be run without `datetime_column`"
+      schema:
+        name:
+          type: string
+          required: true
+          default: Date
+        format:
+          type: string
+          required: false
+
+    test_data:
+      required: false
+      meta:
+        description: "Optional, only if evaluation is needed."
+      schema:
+        connect_args:
+          nullable: true
+          required: false
+          type: dict
+        format:
+          allowed:
+            - csv
+            - json
+            - clipboard
+            - excel
+            - feather
+            - sql_table
+            - sql_query
+            - hdf
+            - tsv
+          required: false
+          type: string
+        columns:
+          required: false
+          type: list
+          schema:
+            type: string
+        filters:
+          required: false
+          type: list
+          schema:
+            type: string
+        options:
+          nullable: true
+          required: false
+          type: dict
+        sql:
+          required: false
+          type: string
+        table_name:
+          required: false
+          type: string
+        url:
+          required: false
+          type: string
+          meta:
+            description: "The url can be local, or remote. For example: `oci://<bucket>@<namespace>/data.csv`"
+        limit:
+          required: false
+          type: integer
+      type: dict
+
+    output_directory:
+      required: false
+      schema:
+        connect_args:
+          nullable: true
+          required: false
+          type: dict
+        format:
+          allowed:
+            - csv
+            - json
+            - clipboard
+            - excel
+            - feather
+            - sql_table
+            - sql_query
+            - hdf
+            - tsv
+          required: false
+          type: string
+        columns:
+          required: false
+          type: list
+          schema:
+            type: string
+        filters:
+          required: false
+          type: list
+          schema:
+            type: string
+        options:
+          nullable: true
+          required: false
+          type: dict
+        sql:
+          required: false
+          type: string
+        table_name:
+          required: false
+          type: string
+        url:
+          required: false
+          type: string
+          meta:
+            description: "The url can be local, or remote. For example: `oci://<bucket>@<namespace>/data.csv`"
+        limit:
+          required: false
+          type: integer
+      type: dict
+
+    report_filename:
+      required: false
+      type: string
+      default: report.html
+      meta:
+        description: "Placed into output_directory location. Defaults to report.html"
+    report_title:
+      required: false
+      type: string
+    report_theme:
+      required: false
+      type: string
+      default: light
+      allowed:
+        - light
+        - dark
+
+    metrics_filename:
+      required: false
+      type: string
+      default: metrics.csv
+      meta:
+        description: "Placed into output_directory location. Defaults to metrics.csv"
+
+    test_metrics_filename:
+      required: false
+      type: string
+      default: test_metrics.csv
+      meta:
+        description: "Placed into output_directory location. Defaults to test_metrics.csv"
+
+    inliers_filename:
+      required: false
+      type: string
+      default: inliers.csv
+      meta:
+        description: "Placed into output_directory location. Defaults to inliers.csv"
+
+    outliers_filename:
+      required: false
+      type: string
+      default: outliers.csv
+      meta:
+        description: "Placed into output_directory location. Defaults to outliers.csv"
+
+    target_column:
+      type: string
+      required: true
+      default: target
+      meta:
+        description: "Identifier column for the series in the dataset"
+
+    target_category_columns:
+      type: list
+      required: false
+      schema:
+        type: string
+      default: ["Series ID"]
+      meta:
+        description: "When provided, target_category_columns [list] indexes the data into multiple related datasets for anomaly detection"
+
+    preprocessing:
+      type: dict
+      required: false
+      schema:
+        enabled:
+          type: boolean
+          required: false
+          default: true
+          meta:
+            description: "preprocessing and feature engineering can be disabled using this flag, Defaults to true"
+        steps:
+          type: dict
+          required: false
+          schema:
+            missing_value_imputation:
+              type: boolean
+              required: false
+              default: false
+
+    generate_report:
+      type: boolean
+      required: false
+      default: true
+      meta:
+        description: "Report file generation can be enabled using this flag. Defaults to true."
+
+    generate_metrics:
+      type: boolean
+      required: false
+      default: true
+      meta:
+        description: "Metrics files generation can be enabled using this flag. Defaults to true."
+
+    generate_inliers:
+      type: boolean
+      required: false
+      default: false
+      meta:
+        description: "Generates inliers.csv"
+
+    model:
+      type: string
+      required: false
+      default: autots
+      allowed:
+        - autots
+        - auto
+        - oneclasssvm
+        - isolationforest
+        - randomcutforest
+        - iqr
+        - lof
+        - zscore
+        - rolling_zscore
+        - mad
+        - ee
+        - dagmm
+        - deep_point_anomaly_detector
+        - lstm_ed
+        - spectral_residual
+        - vae
+        - arima
+        - ets
+        - sarima
+        - bocpd
+        - prophet
+      meta:
+        description: "The model to be used for anomaly detection"
+
+    contamination:
+      required: false
+      default: 0.01
+      type: float
+      meta:
+        description: "The proportion of outliers in the data set. The contamination should be in the range (0, 0.5]"
+
+    model_kwargs:
+      type: dict
+      required: false
+
+    subsample_report_data:
+      type: boolean
+      required: false
+
+  type: dict

ads/opctl/operator/lowcode/anomaly/utils.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2023, 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import os
+
+import numpy as np
+import pandas as pd
+
+from ads.opctl import logger
+
+from .const import NonTimeADSupportedModels, SupportedMetrics, SupportedModels
+from .operator_config import AnomalyOperatorSpec
+
+
+def _build_metrics_df(y_true, y_pred, column_name):
+    from sklearn.metrics import (
+        accuracy_score,
+        auc,
+        confusion_matrix,
+        f1_score,
+        matthews_corrcoef,
+        precision_recall_curve,
+        precision_score,
+        recall_score,
+        roc_auc_score,
+    )
+
+    metrics = {}
+    np.nan_to_num(y_true, copy=False)
+    np.nan_to_num(y_pred, copy=False)
+    metrics[SupportedMetrics.RECALL] = recall_score(y_true, y_pred)
+    metrics[SupportedMetrics.PRECISION] = precision_score(y_true, y_pred)
+    metrics[SupportedMetrics.ACCURACY] = accuracy_score(y_true, y_pred)
+    metrics[SupportedMetrics.F1_SCORE] = f1_score(y_true, y_pred)
+    tn, *fn_fp_tp = confusion_matrix(y_true, y_pred).ravel()
+    fp, fn, tp = fn_fp_tp if fn_fp_tp else (0, 0, 0)
+    metrics[SupportedMetrics.FP] = fp
+    metrics[SupportedMetrics.FN] = fn
+    metrics[SupportedMetrics.TP] = tp
+    metrics[SupportedMetrics.TN] = tn
+    try:
+        # Throws exception if y_true has only one class
+        metrics[SupportedMetrics.ROC_AUC] = roc_auc_score(y_true, y_pred)
+    except Exception as e:
+        logger.warning(f"An exception occurred: {e}")
+        metrics[SupportedMetrics.ROC_AUC] = None
+    precision, recall, thresholds = precision_recall_curve(y_true, y_pred)
+    metrics[SupportedMetrics.PRC_AUC] = auc(recall, precision)
+    metrics[SupportedMetrics.MCC] = matthews_corrcoef(y_true, y_pred)
+    return pd.DataFrame.from_dict(metrics, orient="index", columns=[column_name])
+
+
+def get_frequency_of_datetime(data: pd.DataFrame, dataset_info: AnomalyOperatorSpec):
+    """
+    Function finds the inferred freq from date time column
+
+    Parameters
+    ------------
+    data:  pd.DataFrame
+            primary dataset
+    dataset_info:  AnomalyOperatorSpec
+
+    Returns
+    --------
+    None
+
+    """
+    date_column = dataset_info.datetime_column.name
+    datetimes = pd.to_datetime(
+        data[date_column].drop_duplicates(), format=dataset_info.datetime_column.format
+    )
+    freq = pd.DatetimeIndex(datetimes).inferred_freq
+    return freq
+
+
+def default_signer(**kwargs):
+    os.environ["EXTRA_USER_AGENT_INFO"] = "Anomaly-Detection-Operator"
+    from ads.common.auth import default_signer
+
+    return default_signer(**kwargs)
+
+
+def select_auto_model(operator_config):
+    if operator_config.spec.datetime_column is not None:
+        return SupportedModels.AutoTS
+    return NonTimeADSupportedModels.IsolationForest

ads/opctl/operator/lowcode/common/__init__.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/

ads/opctl/operator/lowcode/common/const.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+
+class DataColumns:
+    Series = "Series"
+    Date = "Date"

ads/opctl/operator/lowcode/common/data.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import time
+from abc import ABC
+
+import pandas as pd
+
+from ads.opctl import logger
+from ads.opctl.operator.lowcode.common.const import DataColumns
+from ads.opctl.operator.lowcode.common.errors import (
+    InvalidParameterError,
+)
+from ads.opctl.operator.lowcode.common.utils import load_data
+
+from .transformations import Transformations
+
+
+class AbstractData(ABC):
+    def __init__(self, spec, name="input_data", data=None):
+        self.Transformations = Transformations
+        self.data = None
+        self._data_dict = dict()
+        self.name = name
+        self.spec = spec
+        if data is not None:
+            self.data = data
+        else:
+            self.load_transform_ingest_data(spec)
+
+    def get_raw_data_by_cat(self, category):
+        mapping = self._data_transformer.get_target_category_columns_map()
+        # For given category, mapping gives the target_category_columns and it's values.
+        # condition filters raw_data based on the values of target_category_columns for the given category
+        condition = pd.Series(True, index=self.raw_data.index)
+        if category in mapping:
+            for col, val in mapping[category].items():
+                condition &= self.raw_data[col] == val
+        data_by_cat = self.raw_data[condition].reset_index(drop=True)
+        data_by_cat = (
+            self._data_transformer._format_datetime_col(data_by_cat)
+            if self.spec.datetime_column
+            else data_by_cat
+        )
+        return data_by_cat
+
+    def get_dict_by_series(self):
+        if not self._data_dict:
+            for s_id in self.list_series_ids():
+                try:
+                    self._data_dict[s_id] = self.data.xs(
+                        s_id, level=DataColumns.Series
+                    ).reset_index()
+                except KeyError as ke:
+                    logger.debug(
+                        f"Unable to extract series: {s_id} from data: {self.data}. This may occur due to significant missing data. Error message: {ke.args}"
+                    )
+                    pass
+        return self._data_dict
+
+    def get_data_for_series(self, series_id):
+        data_dict = self.get_dict_by_series()
+        try:
+            return data_dict[series_id]
+        except:
+            raise InvalidParameterError(
+                f"Unable to retrieve series {series_id} from {self.name}. Available series ids are: {self.list_series_ids()}"
+            )
+
+    def _load_data(self, data_spec, **kwargs):
+        loading_start_time = time.time()
+        try:
+            raw_data = load_data(data_spec)
+        except InvalidParameterError as e:
+            e.args = e.args + (f"Invalid Parameter: {self.name}",)
+            raise e
+        loading_end_time = time.time()
+        logger.info(
+            f"{self.name} loaded in {loading_end_time - loading_start_time} seconds",
+        )
+        return raw_data
+
+    def _transform_data(self, spec, raw_data, **kwargs):
+        transformation_start_time = time.time()
+        self._data_transformer = self.Transformations(spec, name=self.name)
+        data = self._data_transformer.run(raw_data)
+        transformation_end_time = time.time()
+        logger.info(
+            f"{self.name} transformations completed in {transformation_end_time - transformation_start_time} seconds"
+        )
+        return data
+
+    def load_transform_ingest_data(self, spec):
+        self.raw_data = self._load_data(getattr(spec, self.name))
+        self.data = self._transform_data(spec, self.raw_data)
+        self._ingest_data(spec)
+
+    def _ingest_data(self, spec):
+        pass
+
+    def get_data_long(self):
+        return self.data.reset_index(drop=False)
+
+    def get_min_time(self):
+        return self.data.index.get_level_values(0).min()
+
+    def get_max_time(self):
+        return self.data.index.get_level_values(0).max()
+
+    def list_series_ids(self):
+        return self.data.index.get_level_values(1).unique().tolist()
+
+    def get_num_rows(self):
+        return self.data.shape[0]

ads/opctl/operator/lowcode/common/errors.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from ads.opctl.operator import __operators__
+from ads.opctl.operator.common.errors import InvalidParameterError
+
+
+class DataMismatchError(Exception):
+    """Exception raised when there is an issue with the schema."""
+
+    def __init__(self, error: str):
+        super().__init__(
+            "Invalid operator specification. Check the YAML structure and ensure it "
+            "complies with the required schema for the operator. \n"
+            f"{error}"
+        )
+
+
+class InputDataError(Exception):
+    """Exception raised when there is an issue with the input data."""
+
+    def __init__(self, error: str):
+        super().__init__(
+            "Invalid operator specification. Check the YAML structure and ensure it "
+            "complies with the required schema for the operator. \n"
+            f"{error}"
+        )
+
+
+class PermissionsError(Exception):
+    """Exception raised when there is an issue with the input data."""
+
+    def __init__(self, error: str):
+        super().__init__(
+            "Invalid operator specification. Check the YAML structure and ensure it "
+            "complies with the required schema for the operator. \n"
+            f"{error}"
+        )
+
+
+class InsufficientDataError(Exception):
+    def __init__(self, message: str):
+        self.message = message
+        super().__init__(message)