oracle-ads 2.13.8__py3-none-any.whl → 2.13.9rc0__py3-none-any.whl

ads/opctl/operator/lowcode/anomaly/schema.yaml

@@ -1,401 +0,0 @@
-kind:
-  allowed:
-    - operator
-  required: true
-  type: string
-  default: operator
-  meta:
-    description: "Which service are you trying to use? Common kinds: `operator`, `job`"
-
-version:
-  allowed:
-    - "v1"
-  required: true
-  type: string
-  default: v1
-  meta:
-    description: "Operators may change yaml file schemas from version to version, as well as implementation details. Double check the version to ensure compatibility."
-
-type:
-  required: true
-  type: string
-  default: anomaly
-  meta:
-    description: "Type should always be `anomaly` when using a anomaly detection operator"
-
-spec:
-  required: true
-  schema:
-    input_data:
-      required: true
-      type: dict
-      default: { "url": "data.csv" }
-      meta:
-        description: "The payload that the detector should evaluate."
-      schema:
-        connect_args:
-          nullable: true
-          required: false
-          type: dict
-        format:
-          allowed:
-            - csv
-            - json
-            - clipboard
-            - excel
-            - feather
-            - sql_table
-            - sql_query
-            - hdf
-            - tsv
-          required: false
-          type: string
-        columns:
-          required: false
-          type: list
-          schema:
-            type: string
-        filters:
-          required: false
-          type: list
-          schema:
-            type: string
-        options:
-          nullable: true
-          required: false
-          type: dict
-        sql:
-          required: false
-          type: string
-        table_name:
-          required: false
-          type: string
-        url:
-          required: false
-          type: string
-          meta:
-            description: "The url can be local, or remote. For example: `oci://<bucket>@<namespace>/data.csv`"
-        limit:
-          required: false
-          type: integer
-        vault_secret_id:
-          required: false
-          type: string
-
-    validation_data:
-      required: false
-      type: dict
-      meta:
-        description: "Data that has already been labeled as anomalous or not."
-      schema:
-        connect_args:
-          nullable: true
-          required: false
-          type: dict
-        format:
-          allowed:
-            - csv
-            - json
-            - clipboard
-            - excel
-            - feather
-            - sql_table
-            - sql_query
-            - hdf
-            - tsv
-          required: false
-          type: string
-        columns:
-          required: false
-          type: list
-          schema:
-            type: string
-        filters:
-          required: false
-          type: list
-          schema:
-            type: string
-        options:
-          nullable: true
-          required: false
-          type: dict
-        sql:
-          required: false
-          type: string
-        table_name:
-          required: false
-          type: string
-        url:
-          required: false
-          type: string
-          meta:
-            description: "The url can be local, or remote. For example: `oci://<bucket>@<namespace>/data.csv`"
-        limit:
-          required: false
-          type: integer
-        vault_secret_id:
-          required: false
-          type: string
-
-    datetime_column:
-      type: dict
-      required: false
-      meta:
-        description: "`datetime_column` is required for time series anomaly detection, only non time-based anomaly detection models can be run without `datetime_column`"
-      schema:
-        name:
-          type: string
-          required: true
-          default: Date
-        format:
-          type: string
-          required: false
-
-    test_data:
-      required: false
-      meta:
-        description: "Optional, only if evaluation is needed."
-      schema:
-        connect_args:
-          nullable: true
-          required: false
-          type: dict
-        format:
-          allowed:
-            - csv
-            - json
-            - clipboard
-            - excel
-            - feather
-            - sql_table
-            - sql_query
-            - hdf
-            - tsv
-          required: false
-          type: string
-        columns:
-          required: false
-          type: list
-          schema:
-            type: string
-        filters:
-          required: false
-          type: list
-          schema:
-            type: string
-        options:
-          nullable: true
-          required: false
-          type: dict
-        sql:
-          required: false
-          type: string
-        table_name:
-          required: false
-          type: string
-        url:
-          required: false
-          type: string
-          meta:
-            description: "The url can be local, or remote. For example: `oci://<bucket>@<namespace>/data.csv`"
-        limit:
-          required: false
-          type: integer
-      type: dict
-
-    output_directory:
-      required: false
-      schema:
-        connect_args:
-          nullable: true
-          required: false
-          type: dict
-        format:
-          allowed:
-            - csv
-            - json
-            - clipboard
-            - excel
-            - feather
-            - sql_table
-            - sql_query
-            - hdf
-            - tsv
-          required: false
-          type: string
-        columns:
-          required: false
-          type: list
-          schema:
-            type: string
-        filters:
-          required: false
-          type: list
-          schema:
-            type: string
-        options:
-          nullable: true
-          required: false
-          type: dict
-        sql:
-          required: false
-          type: string
-        table_name:
-          required: false
-          type: string
-        url:
-          required: false
-          type: string
-          meta:
-            description: "The url can be local, or remote. For example: `oci://<bucket>@<namespace>/data.csv`"
-        limit:
-          required: false
-          type: integer
-      type: dict
-
-    report_filename:
-      required: false
-      type: string
-      default: report.html
-      meta:
-        description: "Placed into output_directory location. Defaults to report.html"
-    report_title:
-      required: false
-      type: string
-    report_theme:
-      required: false
-      type: string
-      default: light
-      allowed:
-        - light
-        - dark
-
-    metrics_filename:
-      required: false
-      type: string
-      default: metrics.csv
-      meta:
-        description: "Placed into output_directory location. Defaults to metrics.csv"
-
-    test_metrics_filename:
-      required: false
-      type: string
-      default: test_metrics.csv
-      meta:
-        description: "Placed into output_directory location. Defaults to test_metrics.csv"
-
-    inliers_filename:
-      required: false
-      type: string
-      default: inliers.csv
-      meta:
-        description: "Placed into output_directory location. Defaults to inliers.csv"
-
-    outliers_filename:
-      required: false
-      type: string
-      default: outliers.csv
-      meta:
-        description: "Placed into output_directory location. Defaults to outliers.csv"
-
-    target_column:
-      type: string
-      required: true
-      default: target
-      meta:
-        description: "Identifier column for the series in the dataset"
-
-    target_category_columns:
-      type: list
-      required: false
-      schema:
-        type: string
-      default: ["Series ID"]
-      meta:
-        description: "When provided, target_category_columns [list] indexes the data into multiple related datasets for anomaly detection"
-
-    preprocessing:
-      type: dict
-      required: false
-      schema:
-        enabled:
-          type: boolean
-          required: false
-          default: true
-          meta:
-            description: "preprocessing and feature engineering can be disabled using this flag, Defaults to true"
-        steps:
-          type: dict
-          required: false
-          schema:
-            missing_value_imputation:
-              type: boolean
-              required: false
-              default: false
-
-    generate_report:
-      type: boolean
-      required: false
-      default: true
-      meta:
-        description: "Report file generation can be enabled using this flag. Defaults to true."
-
-    generate_metrics:
-      type: boolean
-      required: false
-      default: true
-      meta:
-        description: "Metrics files generation can be enabled using this flag. Defaults to true."
-
-    generate_inliers:
-      type: boolean
-      required: false
-      default: false
-      meta:
-        description: "Generates inliers.csv"
-
-    model:
-      type: string
-      required: false
-      default: autots
-      allowed:
-        - autots
-        - auto
-        - oneclasssvm
-        - isolationforest
-        - randomcutforest
-        - iqr
-        - lof
-        - zscore
-        - rolling_zscore
-        - mad
-        - ee
-        - dagmm
-        - deep_point_anomaly_detector
-        - lstm_ed
-        - spectral_residual
-        - vae
-        - arima
-        - ets
-        - sarima
-        - bocpd
-        - prophet
-      meta:
-        description: "The model to be used for anomaly detection"
-
-    contamination:
-      required: false
-      default: 0.01
-      type: float
-      meta:
-        description: "The proportion of outliers in the data set. The contamination should be in the range (0, 0.5]"
-
-    model_kwargs:
-      type: dict
-      required: false
-
-    subsample_report_data:
-      type: boolean
-      required: false
-
-  type: dict

ads/opctl/operator/lowcode/anomaly/utils.py

@@ -1,88 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright (c) 2023, 2024 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-import os
-
-import numpy as np
-import pandas as pd
-
-from ads.opctl import logger
-
-from .const import NonTimeADSupportedModels, SupportedMetrics, SupportedModels
-from .operator_config import AnomalyOperatorSpec
-
-
-def _build_metrics_df(y_true, y_pred, column_name):
-    from sklearn.metrics import (
-        accuracy_score,
-        auc,
-        confusion_matrix,
-        f1_score,
-        matthews_corrcoef,
-        precision_recall_curve,
-        precision_score,
-        recall_score,
-        roc_auc_score,
-    )
-
-    metrics = {}
-    np.nan_to_num(y_true, copy=False)
-    np.nan_to_num(y_pred, copy=False)
-    metrics[SupportedMetrics.RECALL] = recall_score(y_true, y_pred)
-    metrics[SupportedMetrics.PRECISION] = precision_score(y_true, y_pred)
-    metrics[SupportedMetrics.ACCURACY] = accuracy_score(y_true, y_pred)
-    metrics[SupportedMetrics.F1_SCORE] = f1_score(y_true, y_pred)
-    tn, *fn_fp_tp = confusion_matrix(y_true, y_pred).ravel()
-    fp, fn, tp = fn_fp_tp if fn_fp_tp else (0, 0, 0)
-    metrics[SupportedMetrics.FP] = fp
-    metrics[SupportedMetrics.FN] = fn
-    metrics[SupportedMetrics.TP] = tp
-    metrics[SupportedMetrics.TN] = tn
-    try:
-        # Throws exception if y_true has only one class
-        metrics[SupportedMetrics.ROC_AUC] = roc_auc_score(y_true, y_pred)
-    except Exception as e:
-        logger.warning(f"An exception occurred: {e}")
-        metrics[SupportedMetrics.ROC_AUC] = None
-    precision, recall, thresholds = precision_recall_curve(y_true, y_pred)
-    metrics[SupportedMetrics.PRC_AUC] = auc(recall, precision)
-    metrics[SupportedMetrics.MCC] = matthews_corrcoef(y_true, y_pred)
-    return pd.DataFrame.from_dict(metrics, orient="index", columns=[column_name])
-
-
-def get_frequency_of_datetime(data: pd.DataFrame, dataset_info: AnomalyOperatorSpec):
-    """
-    Function finds the inferred freq from date time column
-
-    Parameters
-    ------------
-    data:  pd.DataFrame
-            primary dataset
-    dataset_info:  AnomalyOperatorSpec
-
-    Returns
-    --------
-    None
-
-    """
-    date_column = dataset_info.datetime_column.name
-    datetimes = pd.to_datetime(
-        data[date_column].drop_duplicates(), format=dataset_info.datetime_column.format
-    )
-    freq = pd.DatetimeIndex(datetimes).inferred_freq
-    return freq
-
-
-def default_signer(**kwargs):
-    os.environ["EXTRA_USER_AGENT_INFO"] = "Anomaly-Detection-Operator"
-    from ads.common.auth import default_signer
-
-    return default_signer(**kwargs)
-
-
-def select_auto_model(operator_config):
-    if operator_config.spec.datetime_column is not None:
-        return SupportedModels.AutoTS
-    return NonTimeADSupportedModels.IsolationForest

ads/opctl/operator/lowcode/common/__init__.py

@@ -1,5 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*--
-
-# Copyright (c) 2024 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/

ads/opctl/operator/lowcode/common/const.py

@@ -1,10 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*--
-
-# Copyright (c) 2024 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-
-class DataColumns:
-    Series = "Series"
-    Date = "Date"

ads/opctl/operator/lowcode/common/data.py

@@ -1,116 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright (c) 2024 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-import time
-from abc import ABC
-
-import pandas as pd
-
-from ads.opctl import logger
-from ads.opctl.operator.lowcode.common.const import DataColumns
-from ads.opctl.operator.lowcode.common.errors import (
-    InvalidParameterError,
-)
-from ads.opctl.operator.lowcode.common.utils import load_data
-
-from .transformations import Transformations
-
-
-class AbstractData(ABC):
-    def __init__(self, spec, name="input_data", data=None):
-        self.Transformations = Transformations
-        self.data = None
-        self._data_dict = dict()
-        self.name = name
-        self.spec = spec
-        if data is not None:
-            self.data = data
-        else:
-            self.load_transform_ingest_data(spec)
-
-    def get_raw_data_by_cat(self, category):
-        mapping = self._data_transformer.get_target_category_columns_map()
-        # For given category, mapping gives the target_category_columns and it's values.
-        # condition filters raw_data based on the values of target_category_columns for the given category
-        condition = pd.Series(True, index=self.raw_data.index)
-        if category in mapping:
-            for col, val in mapping[category].items():
-                condition &= self.raw_data[col] == val
-        data_by_cat = self.raw_data[condition].reset_index(drop=True)
-        data_by_cat = (
-            self._data_transformer._format_datetime_col(data_by_cat)
-            if self.spec.datetime_column
-            else data_by_cat
-        )
-        return data_by_cat
-
-    def get_dict_by_series(self):
-        if not self._data_dict:
-            for s_id in self.list_series_ids():
-                try:
-                    self._data_dict[s_id] = self.data.xs(
-                        s_id, level=DataColumns.Series
-                    ).reset_index()
-                except KeyError as ke:
-                    logger.debug(
-                        f"Unable to extract series: {s_id} from data: {self.data}. This may occur due to significant missing data. Error message: {ke.args}"
-                    )
-                    pass
-        return self._data_dict
-
-    def get_data_for_series(self, series_id):
-        data_dict = self.get_dict_by_series()
-        try:
-            return data_dict[series_id]
-        except:
-            raise InvalidParameterError(
-                f"Unable to retrieve series {series_id} from {self.name}. Available series ids are: {self.list_series_ids()}"
-            )
-
-    def _load_data(self, data_spec, **kwargs):
-        loading_start_time = time.time()
-        try:
-            raw_data = load_data(data_spec)
-        except InvalidParameterError as e:
-            e.args = e.args + (f"Invalid Parameter: {self.name}",)
-            raise e
-        loading_end_time = time.time()
-        logger.info(
-            f"{self.name} loaded in {loading_end_time - loading_start_time} seconds",
-        )
-        return raw_data
-
-    def _transform_data(self, spec, raw_data, **kwargs):
-        transformation_start_time = time.time()
-        self._data_transformer = self.Transformations(spec, name=self.name)
-        data = self._data_transformer.run(raw_data)
-        transformation_end_time = time.time()
-        logger.info(
-            f"{self.name} transformations completed in {transformation_end_time - transformation_start_time} seconds"
-        )
-        return data
-
-    def load_transform_ingest_data(self, spec):
-        self.raw_data = self._load_data(getattr(spec, self.name))
-        self.data = self._transform_data(spec, self.raw_data)
-        self._ingest_data(spec)
-
-    def _ingest_data(self, spec):
-        pass
-
-    def get_data_long(self):
-        return self.data.reset_index(drop=False)
-
-    def get_min_time(self):
-        return self.data.index.get_level_values(0).min()
-
-    def get_max_time(self):
-        return self.data.index.get_level_values(0).max()
-
-    def list_series_ids(self):
-        return self.data.index.get_level_values(1).unique().tolist()
-
-    def get_num_rows(self):
-        return self.data.shape[0]

ads/opctl/operator/lowcode/common/errors.py

@@ -1,47 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*--
-
-# Copyright (c) 2024 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-from ads.opctl.operator import __operators__
-from ads.opctl.operator.common.errors import InvalidParameterError
-
-
-class DataMismatchError(Exception):
-    """Exception raised when there is an issue with the schema."""
-
-    def __init__(self, error: str):
-        super().__init__(
-            "Invalid operator specification. Check the YAML structure and ensure it "
-            "complies with the required schema for the operator. \n"
-            f"{error}"
-        )
-
-
-class InputDataError(Exception):
-    """Exception raised when there is an issue with the input data."""
-
-    def __init__(self, error: str):
-        super().__init__(
-            "Invalid operator specification. Check the YAML structure and ensure it "
-            "complies with the required schema for the operator. \n"
-            f"{error}"
-        )
-
-
-class PermissionsError(Exception):
-    """Exception raised when there is an issue with the input data."""
-
-    def __init__(self, error: str):
-        super().__init__(
-            "Invalid operator specification. Check the YAML structure and ensure it "
-            "complies with the required schema for the operator. \n"
-            f"{error}"
-        )
-
-
-class InsufficientDataError(Exception):
-    def __init__(self, message: str):
-        self.message = message
-        super().__init__(message)