oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.10__py3-none-any.whl

ads/opctl/operator/lowcode/common/transformations.py

@@ -0,0 +1,296 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2023, 2025 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from abc import ABC
+
+import numpy as np
+import pandas as pd
+
+from ads.opctl import logger
+from ads.opctl.operator.lowcode.common.const import DataColumns
+from ads.opctl.operator.lowcode.common.errors import (
+    DataMismatchError,
+    InvalidParameterError,
+)
+from ads.opctl.operator.lowcode.common.utils import merge_category_columns
+from ads.opctl.operator.lowcode.forecast.operator_config import ForecastOperatorSpec
+
+
+class Transformations(ABC):
+    """A class which implements transformation for forecast operator"""
+
+    def __init__(self, dataset_info, name="historical_data"):
+        """
+        Initializes the transformation.
+
+        Parameters
+        ----------
+            data: The Pandas DataFrame.
+            dataset_info : ForecastOperatorConfig
+        """
+        self.name = name
+        self.dataset_info = dataset_info
+        self.target_category_columns = dataset_info.target_category_columns
+        self.target_column_name = dataset_info.target_column
+        self.raw_column_names = None
+        self.dt_column_name = (
+            dataset_info.datetime_column.name if dataset_info.datetime_column else None
+        )
+        self.dt_column_format = (
+            dataset_info.datetime_column.format
+            if dataset_info.datetime_column
+            else None
+        )
+        self.preprocessing = dataset_info.preprocessing
+
+    def run(self, data):
+        """
+        The function runs all the transformation in a particular order.
+
+        Returns
+        -------
+            A new Pandas DataFrame with treated / transformed target values. Specifically:
+            - Data will be in a multiIndex with Datetime always first (level 0)
+            - whether 0, 1 or 2+, all target_category_columns will be merged into a single index column: Series
+            - All datetime columns will be formatted as such
+            - all data will be imputed (unless preprocessing disabled)
+            - all trailing whitespace will be removed
+            - the data will be sorted by Datetime then Series
+
+        """
+        clean_df = self._remove_trailing_whitespace(data)
+        if isinstance(self.dataset_info, ForecastOperatorSpec):
+            clean_df = self._clean_column_names(clean_df)
+        if self.name == "historical_data":
+            self._check_historical_dataset(clean_df)
+        clean_df = self._set_series_id_column(clean_df)
+        if self.dt_column_name:
+            clean_df = self._format_datetime_col(clean_df)
+        clean_df = self._set_multi_index(clean_df)
+        clean_df = self._fill_na(clean_df) if not self.dt_column_name else clean_df
+
+        if self.preprocessing and self.preprocessing.enabled:
+            if self.name == "historical_data":
+                if self.preprocessing.steps.missing_value_imputation:
+                    try:
+                        clean_df = self._missing_value_imputation_hist(clean_df)
+                    except Exception as e:
+                        logger.debug(f"Missing value imputation failed with {e.args}")
+                else:
+                    logger.info(
+                        "Skipping missing value imputation because it is disabled"
+                    )
+                if self.preprocessing.steps.outlier_treatment:
+                    try:
+                        clean_df = self._outlier_treatment(clean_df)
+                    except Exception as e:
+                        logger.debug(f"Outlier Treatment failed with {e.args}")
+                else:
+                    logger.info("Skipping outlier treatment because it is disabled")
+            elif self.name == "additional_data":
+                clean_df = self._missing_value_imputation_add(clean_df)
+            elif self.name == "input_data" and self.preprocessing.steps.missing_value_imputation:
+                clean_df = self._fill_na(clean_df)
+        else:
+            logger.info(
+                "Skipping all preprocessing steps because preprocessing is disabled"
+            )
+        return clean_df
+
+    def _remove_trailing_whitespace(self, df):
+        return df.apply(
+            lambda x: x.str.strip()
+            if hasattr(x, "dtype") and x.dtype == "object"
+            else x
+        )
+
+    def _clean_column_names(self, df):
+        """
+        Remove all whitespaces from column names in a DataFrame and store the original names.
+
+        Parameters:
+        df (pd.DataFrame): The DataFrame whose column names need to be cleaned.
+
+        Returns:
+        pd.DataFrame: The DataFrame with cleaned column names.
+        """
+
+        self.raw_column_names = {
+            col: col.replace(" ", "") for col in df.columns if " " in col
+        }
+        df.columns = [self.raw_column_names.get(col, col) for col in df.columns]
+
+        if self.target_column_name:
+            self.target_column_name = self.raw_column_names.get(
+                self.target_column_name, self.target_column_name
+            )
+        self.dt_column_name = self.raw_column_names.get(
+            self.dt_column_name, self.dt_column_name
+        )
+
+        if self.target_category_columns:
+            self.target_category_columns = [
+                self.raw_column_names.get(col, col)
+                for col in self.target_category_columns
+            ]
+        return df
+
+    def _set_series_id_column(self, df):
+        self._target_category_columns_map = {}
+        if not self.target_category_columns:
+            df[DataColumns.Series] = "Series 1"
+        else:
+            df[DataColumns.Series] = merge_category_columns(
+                df, self.target_category_columns
+            )
+            merged_values = df[DataColumns.Series].unique().tolist()
+            if self.target_category_columns:
+                for value in merged_values:
+                    self._target_category_columns_map[value] = (
+                        df[df[DataColumns.Series] == value][
+                            self.target_category_columns
+                        ]
+                        .drop_duplicates()
+                        .iloc[0]
+                        .to_dict()
+                    )
+
+            if self.target_category_columns != [DataColumns.Series]:
+                df = df.drop(self.target_category_columns, axis=1)
+        return df
+
+    def _format_datetime_col(self, df):
+        try:
+            df[self.dt_column_name] = pd.to_datetime(
+                df[self.dt_column_name], format=self.dt_column_format
+            )
+        except Exception as ee:
+            raise InvalidParameterError(
+                f"Unable to determine the datetime type for column: {self.dt_column_name} in dataset: {self.name}. Please specify the format explicitly. (For example adding 'format: %d/%m/%Y' underneath 'name: {self.dt_column_name}' in the datetime_column section of the yaml file if you haven't already. For reference, here is the first datetime given: {df[self.dt_column_name].values[0]}"
+            ) from ee
+        return df
+
+    def _set_multi_index(self, df):
+        """
+        Function sorts by date
+
+        Parameters
+        ----------
+            df : The Pandas DataFrame.
+
+        Returns
+        -------
+            A new Pandas DataFrame with sorted dates for each series
+        """
+        if self.dt_column_name:
+            df = df.set_index([self.dt_column_name, DataColumns.Series])
+            return df.sort_values(
+                [self.dt_column_name, DataColumns.Series], ascending=True
+            )
+        return df.set_index([df.index, DataColumns.Series])
+
+    def _missing_value_imputation_hist(self, df):
+        """
+        Function fills missing values in the pandas dataframe using liner interpolation
+
+        Parameters
+        ----------
+            df : The Pandas DataFrame.
+
+        Returns
+        -------
+            A new Pandas DataFrame without missing values.
+        """
+        # missing value imputation using linear interpolation
+        df[self.target_column_name] = (
+            df[self.target_column_name]
+            .groupby(DataColumns.Series)
+            .transform(lambda x: x.interpolate(limit_direction="both"))
+        )
+        return df
+
+    def _missing_value_imputation_add(self, df):
+        """
+        Function fills missing values with zero
+
+        Parameters
+        ----------
+            df : The Pandas DataFrame.
+
+        Returns
+        -------
+            A new Pandas DataFrame without missing values.
+        """
+        return df.fillna(0)
+
+    def _outlier_treatment(self, df):
+        """
+        Function finds outliears using z_score and treats with mean value.
+
+        Parameters
+        ----------
+            df : The Pandas DataFrame.
+
+        Returns
+        -------
+            A new Pandas DataFrame with treated outliears.
+        """
+        return df
+        df["__z_score"] = (
+            df[self.target_column_name]
+            .groupby(DataColumns.Series)
+            .transform(lambda x: (x - x.mean()) / x.std())
+        )
+        outliers_mask = df["__z_score"].abs() > 3
+
+        if df[self.target_column_name].dtype == np.int:
+            df[self.target_column_name].astype(np.float)
+
+        df.loc[outliers_mask, self.target_column_name] = (
+            df[self.target_column_name]
+            .groupby(DataColumns.Series)
+            .transform(lambda x: np.median(x))
+        )
+        df_ret = df.drop("__z_score", axis=1)
+        return df_ret
+
+    def _check_historical_dataset(self, df):
+        expected_names = [self.target_column_name, self.dt_column_name] + (
+            self.target_category_columns if self.target_category_columns else []
+        )
+
+        if self.raw_column_names:
+            expected_names.extend(list(self.raw_column_names.values()))
+
+        if set(df.columns) != set(expected_names):
+            raise DataMismatchError(
+                f"Expected {self.name} to have columns: {expected_names}, but instead found column names: {df.columns}. Is the {self.name} path correct?"
+            )
+
+    """
+        Map between merged target category column values and target category column and its value
+        If target category columns are PPG_Code, Class, Num
+        Merged target category column values are Product Category 1__A__1, Product Category 2__A__2
+        Then target_category_columns_map would be
+        {
+            "Product Category 1__A__1": {
+                "PPG_Code": "Product Category 1",
+                "Class": "A",
+                "Num": 1
+            },
+             "Product Category 2__A__2": {
+                "PPG_Code": "Product Category 2",
+                "Class": "A",
+                "Num": 2
+            },
+        }
+    """
+
+    def get_target_category_columns_map(self):
+        return self._target_category_columns_map
+
+    def _fill_na(self, df: pd.DataFrame, na_value=0) -> pd.DataFrame:
+        """Fill nans in dataframe"""
+        return df.fillna(value=na_value)

ads/opctl/operator/lowcode/common/utils.py

@@ -0,0 +1,384 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2024, 2025 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import json
+import logging
+import os
+import shutil
+import sys
+import tempfile
+from typing import List, Union
+
+import cloudpickle
+import fsspec
+import oracledb
+import pandas as pd
+
+from ads.common.object_storage_details import ObjectStorageDetails
+from ads.opctl import logger
+from ads.opctl.operator.common.operator_config import OutputDirectory
+from ads.opctl.operator.lowcode.common.errors import (
+    InvalidParameterError,
+)
+from ads.secrets import ADBSecretKeeper
+
+
+def call_pandas_fsspec(pd_fn, filename, storage_options, **kwargs):
+    if fsspec.utils.get_protocol(filename) == "file" or fsspec.utils.get_protocol(
+        filename
+    ) in ["http", "https"]:
+        return pd_fn(filename, **kwargs)
+
+    storage_options = storage_options or (
+        default_signer() if ObjectStorageDetails.is_oci_path(filename) else {}
+    )
+
+    return pd_fn(filename, storage_options=storage_options, **kwargs)
+
+
+def load_data(data_spec, storage_options=None, **kwargs):
+    if data_spec is None:
+        raise InvalidParameterError("No details provided for this data source.")
+    filename = data_spec.url
+    data = data_spec.data
+    format = data_spec.format
+    columns = data_spec.columns
+    connect_args = data_spec.connect_args
+    sql = data_spec.sql
+    table_name = data_spec.table_name
+    limit = data_spec.limit
+    vault_secret_id = data_spec.vault_secret_id
+    storage_options = storage_options or (
+        default_signer() if ObjectStorageDetails.is_oci_path(filename) else {}
+    )
+    if vault_secret_id is not None and connect_args is None:
+        connect_args = {}
+
+    if data is not None:
+        if format == "spark":
+            data = data.toPandas()
+    elif filename is not None:
+        if not format:
+            _, format = os.path.splitext(filename)
+            format = format[1:]
+        if format in ["json", "clipboard", "excel", "csv", "feather", "hdf", "parquet"]:
+            read_fn = getattr(pd, f"read_{format}")
+            data = call_pandas_fsspec(
+                read_fn, filename, storage_options=storage_options
+            )
+        elif format in ["tsv"]:
+            data = call_pandas_fsspec(
+                pd.read_csv, filename, storage_options=storage_options, sep="\t"
+            )
+        else:
+            raise InvalidParameterError(
+                f"The format {format} is not currently supported for reading data. Please reformat the data source: {filename} ."
+            )
+    elif connect_args is not None:
+        with tempfile.TemporaryDirectory() as temp_dir:
+            if vault_secret_id is not None:
+                try:
+                    with ADBSecretKeeper.load_secret(
+                        vault_secret_id, wallet_dir=temp_dir
+                    ) as adwsecret:
+                        if (
+                            "wallet_location" in adwsecret
+                            and "wallet_location" not in connect_args
+                        ):
+                            shutil.unpack_archive(
+                                adwsecret["wallet_location"], temp_dir
+                            )
+                            connect_args["wallet_location"] = temp_dir
+                        if "user_name" in adwsecret and "user" not in connect_args:
+                            connect_args["user"] = adwsecret["user_name"]
+                        if "password" in adwsecret and "password" not in connect_args:
+                            connect_args["password"] = adwsecret["password"]
+                        if (
+                            "service_name" in adwsecret
+                            and "service_name" not in connect_args
+                        ):
+                            connect_args["service_name"] = adwsecret["service_name"]
+
+                except Exception as e:
+                    raise Exception(
+                        f"Could not retrieve database credentials from vault {vault_secret_id}: {e}"
+                    ) from e
+
+            con = oracledb.connect(**connect_args)
+            if table_name is not None:
+                data = pd.read_sql(f"SELECT * FROM {table_name}", con)
+            elif sql is not None:
+                data = pd.read_sql(sql, con)
+            else:
+                raise InvalidParameterError(
+                    "Database `connect_args` provided without sql query or table name. Please specify either `sql` or `table_name`."
+                )
+    else:
+        raise InvalidParameterError(
+            "No filename/url provided, and no connect_args provided. Please specify one of these if you want to read data from a file or a database respectively."
+        )
+    if columns:
+        # keep only these columns, done after load because only CSV supports stream filtering
+        data = data[columns]
+    if limit:
+        data = data[:limit]
+    return data
+
+
+def _safe_write(fn, **kwargs):
+    try:
+        fn(**kwargs)
+    except Exception:
+        logger.warning(f'Failed to write file {kwargs.get("filename", "UNKNOWN")}')
+
+
+def write_data(data, filename, format, storage_options=None, index=False, **kwargs):
+    return _safe_write(
+        fn=_write_data,
+        data=data,
+        filename=filename,
+        format=format,
+        storage_options=storage_options,
+        index=index,
+        **kwargs,
+    )
+
+
+def _write_data(data, filename, format, storage_options=None, index=False, **kwargs):
+    disable_print()
+    if not format:
+        _, format = os.path.splitext(filename)
+        format = format[1:]
+    if format in ["json", "clipboard", "excel", "csv", "feather", "hdf"]:
+        write_fn = getattr(data, f"to_{format}")
+        return call_pandas_fsspec(
+            write_fn, filename, index=index, storage_options=storage_options, **kwargs
+        )
+    enable_print()
+    raise InvalidParameterError(
+        f"The format {format} is not currently supported for writing data. Please change the format parameter for the data output: {filename} ."
+    )
+
+
+def write_json(json_dict, filename, storage_options=None):
+    return _safe_write(
+        fn=_write_json,
+        json_dict=json_dict,
+        filename=filename,
+        storage_options=storage_options,
+    )
+
+
+def _write_json(json_dict, filename, storage_options=None):
+    with fsspec.open(filename, mode="w", **storage_options) as f:
+        f.write(json.dumps(json_dict))
+
+
+def write_simple_json(data, path):
+    return _safe_write(fn=_write_simple_json, data=data, path=path)
+
+
+def _write_simple_json(data, path):
+    if ObjectStorageDetails.is_oci_path(path):
+        storage_options = default_signer()
+    else:
+        storage_options = {}
+    with fsspec.open(path, mode="w", **storage_options) as f:
+        json.dump(data, f, indent=4)
+
+
+def write_file(local_filename, remote_filename, storage_options, **kwargs):
+    return _safe_write(
+        fn=_write_file,
+        local_filename=local_filename,
+        remote_filename=remote_filename,
+        storage_options=storage_options,
+        **kwargs,
+    )
+
+
+def _write_file(local_filename, remote_filename, storage_options, **kwargs):
+    with open(local_filename) as f1:
+        with fsspec.open(
+            remote_filename,
+            "w",
+            **storage_options,
+        ) as f2:
+            f2.write(f1.read())
+
+
+def load_pkl(filepath):
+    return _safe_write(fn=_load_pkl, filepath=filepath)
+
+
+def _load_pkl(filepath):
+    storage_options = {}
+    if ObjectStorageDetails.is_oci_path(filepath):
+        storage_options = default_signer()
+
+    with fsspec.open(filepath, "rb", **storage_options) as f:
+        return cloudpickle.load(f)
+    return None
+
+
+def write_pkl(obj, filename, output_dir, storage_options):
+    return _safe_write(
+        fn=_write_pkl,
+        obj=obj,
+        filename=filename,
+        output_dir=output_dir,
+        storage_options=storage_options,
+    )
+
+
+def _write_pkl(obj, filename, output_dir, storage_options):
+    pkl_path = os.path.join(output_dir, filename)
+    with fsspec.open(
+        pkl_path,
+        "wb",
+        **storage_options,
+    ) as f:
+        cloudpickle.dump(obj, f)
+
+
+def merge_category_columns(data, target_category_columns):
+    result = data.apply(
+        lambda x: "__".join([str(x[col]) for col in target_category_columns]), axis=1
+    )
+    return result if not result.empty else pd.Series([], dtype=str)
+
+
+def merged_category_column_name(target_category_columns: Union[List, None]):
+    if not target_category_columns or len(target_category_columns) == 0:
+        return None
+    return "__".join([str(x) for x in target_category_columns])
+
+
+def datetime_to_seconds(s: pd.Series):
+    """
+    Method converts a datetime column into an integer number of seconds.
+    This method has many uses, most notably for enabling libraries like shap
+        to read datetime columns
+    ------------
+    s: pd.Series
+        A Series of type datetime
+    Returns
+    pd.Series of type int
+    """
+    return s.apply(lambda x: x.timestamp())
+
+
+def seconds_to_datetime(s: pd.Series, dt_format=None):
+    """
+    Inverse of `datetime_to_second`
+    ------------
+    s: pd.Series
+        A Series of type int
+    Returns
+    pd.Series of type datetime
+    """
+    s = pd.to_datetime(s, unit="s")
+    if dt_format is not None:
+        return pd.to_datetime(s, format=dt_format)
+    return s
+
+
+def default_signer(**kwargs):
+    os.environ["EXTRA_USER_AGENT_INFO"] = "Operator"
+    from ads.common.auth import default_signer
+
+    return default_signer(**kwargs)
+
+
+def get_frequency_in_seconds(s: pd.Series, sample_size=100, ignore_duplicates=True):
+    """
+    Returns frequency of data in seconds
+
+    Parameters
+    ------------
+    dt_col:  pd.Series  Datetime column
+    ignore_duplicates: bool if True, duplicates will be dropped before computing frequency
+
+    Returns
+    --------
+    int   Minimum difference in seconds
+    """
+    s1 = pd.Series(s).drop_duplicates() if ignore_duplicates else s
+    return s1.tail(20).diff().min().total_seconds()
+
+
+def get_frequency_of_datetime(dt_col: pd.Series, ignore_duplicates=True):
+    """
+    Returns string frequency of data
+
+    Parameters
+    ------------
+    dt_col:  pd.Series  Datetime column
+    ignore_duplicates: bool if True, duplicates will be dropped before computing frequency
+
+    Returns
+    --------
+    str  Pandas Datetime Frequency
+    """
+    s = pd.Series(dt_col).drop_duplicates() if ignore_duplicates else dt_col
+    return pd.infer_freq(s)
+
+
+def human_time_friendly(seconds):
+    TIME_DURATION_UNITS = (
+        ("week", 60 * 60 * 24 * 7),
+        ("day", 60 * 60 * 24),
+        ("hour", 60 * 60),
+        ("min", 60),
+    )
+    if seconds == 0:
+        return "inf"
+    accumulator = []
+    for unit, div in TIME_DURATION_UNITS:
+        amount, seconds = divmod(float(seconds), div)
+        if amount > 0:
+            accumulator.append(
+                "{} {}{}".format(int(amount), unit, "" if amount == 1 else "s")
+            )
+    accumulator.append(f"{round(seconds, 2)} secs")
+    return ", ".join(accumulator)
+
+
+def find_output_dirname(output_dir: OutputDirectory):
+    if output_dir and output_dir.url:
+        return output_dir.url
+    output_dir = "results"
+
+    # If the directory exists, find the next unique directory name by appending an incrementing suffix
+    counter = 1
+    unique_output_dir = f"{output_dir}"
+    while os.path.exists(unique_output_dir):
+        unique_output_dir = f"{output_dir}_{counter}"
+        counter += 1
+    logger.warning(
+        f"Since the output directory was not specified, the output will be saved to {unique_output_dir} directory."
+    )
+    return unique_output_dir
+
+
+def set_log_level(pkg_name: str, level: int):
+    pkg_logger = logging.getLogger(pkg_name)
+    pkg_logger.addHandler(logging.NullHandler())
+    pkg_logger.propagate = False
+    pkg_logger.setLevel(level)
+
+
+# Disable
+def disable_print():
+    sys.stdout = open(os.devnull, "w")
+
+
+# Restore
+def enable_print():
+    try:
+        sys.stdout.close()
+    except Exception:
+        pass
+    sys.stdout = sys.__stdout__

ads/opctl/operator/lowcode/feature_store_marketplace/MLoperator

@@ -0,0 +1,13 @@
+type: feature_store_marketplace
+version: v1
+name: Feature store deployment Operator
+gpu: no
+keywords:
+  - Helm
+  - marketplace
+  - Feature store
+backends:
+  - marketplace
+description: |
+  Feature store deployment operator helps ease the operational burden of deploying feature store helm charts from marketplace
+  to OKE cluster of your choice!!