oracle-ads 2.13.8__py3-none-any.whl → 2.13.9rc0__py3-none-any.whl

ads/opctl/operator/lowcode/common/transformations.py

@@ -1,296 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright (c) 2023, 2025 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-from abc import ABC
-
-import numpy as np
-import pandas as pd
-
-from ads.opctl import logger
-from ads.opctl.operator.lowcode.common.const import DataColumns
-from ads.opctl.operator.lowcode.common.errors import (
-    DataMismatchError,
-    InvalidParameterError,
-)
-from ads.opctl.operator.lowcode.common.utils import merge_category_columns
-from ads.opctl.operator.lowcode.forecast.operator_config import ForecastOperatorSpec
-
-
-class Transformations(ABC):
-    """A class which implements transformation for forecast operator"""
-
-    def __init__(self, dataset_info, name="historical_data"):
-        """
-        Initializes the transformation.
-
-        Parameters
-        ----------
-            data: The Pandas DataFrame.
-            dataset_info : ForecastOperatorConfig
-        """
-        self.name = name
-        self.dataset_info = dataset_info
-        self.target_category_columns = dataset_info.target_category_columns
-        self.target_column_name = dataset_info.target_column
-        self.raw_column_names = None
-        self.dt_column_name = (
-            dataset_info.datetime_column.name if dataset_info.datetime_column else None
-        )
-        self.dt_column_format = (
-            dataset_info.datetime_column.format
-            if dataset_info.datetime_column
-            else None
-        )
-        self.preprocessing = dataset_info.preprocessing
-
-    def run(self, data):
-        """
-        The function runs all the transformation in a particular order.
-
-        Returns
-        -------
-            A new Pandas DataFrame with treated / transformed target values. Specifically:
-            - Data will be in a multiIndex with Datetime always first (level 0)
-            - whether 0, 1 or 2+, all target_category_columns will be merged into a single index column: Series
-            - All datetime columns will be formatted as such
-            - all data will be imputed (unless preprocessing disabled)
-            - all trailing whitespace will be removed
-            - the data will be sorted by Datetime then Series
-
-        """
-        clean_df = self._remove_trailing_whitespace(data)
-        if isinstance(self.dataset_info, ForecastOperatorSpec):
-            clean_df = self._clean_column_names(clean_df)
-        if self.name == "historical_data":
-            self._check_historical_dataset(clean_df)
-        clean_df = self._set_series_id_column(clean_df)
-        if self.dt_column_name:
-            clean_df = self._format_datetime_col(clean_df)
-        clean_df = self._set_multi_index(clean_df)
-        clean_df = self._fill_na(clean_df) if not self.dt_column_name else clean_df
-
-        if self.preprocessing and self.preprocessing.enabled:
-            if self.name == "historical_data":
-                if self.preprocessing.steps.missing_value_imputation:
-                    try:
-                        clean_df = self._missing_value_imputation_hist(clean_df)
-                    except Exception as e:
-                        logger.debug(f"Missing value imputation failed with {e.args}")
-                else:
-                    logger.info(
-                        "Skipping missing value imputation because it is disabled"
-                    )
-                if self.preprocessing.steps.outlier_treatment:
-                    try:
-                        clean_df = self._outlier_treatment(clean_df)
-                    except Exception as e:
-                        logger.debug(f"Outlier Treatment failed with {e.args}")
-                else:
-                    logger.info("Skipping outlier treatment because it is disabled")
-            elif self.name == "additional_data":
-                clean_df = self._missing_value_imputation_add(clean_df)
-            elif self.name == "input_data" and self.preprocessing.steps.missing_value_imputation:
-                clean_df = self._fill_na(clean_df)
-        else:
-            logger.info(
-                "Skipping all preprocessing steps because preprocessing is disabled"
-            )
-        return clean_df
-
-    def _remove_trailing_whitespace(self, df):
-        return df.apply(
-            lambda x: x.str.strip()
-            if hasattr(x, "dtype") and x.dtype == "object"
-            else x
-        )
-
-    def _clean_column_names(self, df):
-        """
-        Remove all whitespaces from column names in a DataFrame and store the original names.
-
-        Parameters:
-        df (pd.DataFrame): The DataFrame whose column names need to be cleaned.
-
-        Returns:
-        pd.DataFrame: The DataFrame with cleaned column names.
-        """
-
-        self.raw_column_names = {
-            col: col.replace(" ", "") for col in df.columns if " " in col
-        }
-        df.columns = [self.raw_column_names.get(col, col) for col in df.columns]
-
-        if self.target_column_name:
-            self.target_column_name = self.raw_column_names.get(
-                self.target_column_name, self.target_column_name
-            )
-        self.dt_column_name = self.raw_column_names.get(
-            self.dt_column_name, self.dt_column_name
-        )
-
-        if self.target_category_columns:
-            self.target_category_columns = [
-                self.raw_column_names.get(col, col)
-                for col in self.target_category_columns
-            ]
-        return df
-
-    def _set_series_id_column(self, df):
-        self._target_category_columns_map = {}
-        if not self.target_category_columns:
-            df[DataColumns.Series] = "Series 1"
-        else:
-            df[DataColumns.Series] = merge_category_columns(
-                df, self.target_category_columns
-            )
-            merged_values = df[DataColumns.Series].unique().tolist()
-            if self.target_category_columns:
-                for value in merged_values:
-                    self._target_category_columns_map[value] = (
-                        df[df[DataColumns.Series] == value][
-                            self.target_category_columns
-                        ]
-                        .drop_duplicates()
-                        .iloc[0]
-                        .to_dict()
-                    )
-
-            if self.target_category_columns != [DataColumns.Series]:
-                df = df.drop(self.target_category_columns, axis=1)
-        return df
-
-    def _format_datetime_col(self, df):
-        try:
-            df[self.dt_column_name] = pd.to_datetime(
-                df[self.dt_column_name], format=self.dt_column_format
-            )
-        except Exception as ee:
-            raise InvalidParameterError(
-                f"Unable to determine the datetime type for column: {self.dt_column_name} in dataset: {self.name}. Please specify the format explicitly. (For example adding 'format: %d/%m/%Y' underneath 'name: {self.dt_column_name}' in the datetime_column section of the yaml file if you haven't already. For reference, here is the first datetime given: {df[self.dt_column_name].values[0]}"
-            ) from ee
-        return df
-
-    def _set_multi_index(self, df):
-        """
-        Function sorts by date
-
-        Parameters
-        ----------
-            df : The Pandas DataFrame.
-
-        Returns
-        -------
-            A new Pandas DataFrame with sorted dates for each series
-        """
-        if self.dt_column_name:
-            df = df.set_index([self.dt_column_name, DataColumns.Series])
-            return df.sort_values(
-                [self.dt_column_name, DataColumns.Series], ascending=True
-            )
-        return df.set_index([df.index, DataColumns.Series])
-
-    def _missing_value_imputation_hist(self, df):
-        """
-        Function fills missing values in the pandas dataframe using liner interpolation
-
-        Parameters
-        ----------
-            df : The Pandas DataFrame.
-
-        Returns
-        -------
-            A new Pandas DataFrame without missing values.
-        """
-        # missing value imputation using linear interpolation
-        df[self.target_column_name] = (
-            df[self.target_column_name]
-            .groupby(DataColumns.Series)
-            .transform(lambda x: x.interpolate(limit_direction="both"))
-        )
-        return df
-
-    def _missing_value_imputation_add(self, df):
-        """
-        Function fills missing values with zero
-
-        Parameters
-        ----------
-            df : The Pandas DataFrame.
-
-        Returns
-        -------
-            A new Pandas DataFrame without missing values.
-        """
-        return df.fillna(0)
-
-    def _outlier_treatment(self, df):
-        """
-        Function finds outliears using z_score and treats with mean value.
-
-        Parameters
-        ----------
-            df : The Pandas DataFrame.
-
-        Returns
-        -------
-            A new Pandas DataFrame with treated outliears.
-        """
-        return df
-        df["__z_score"] = (
-            df[self.target_column_name]
-            .groupby(DataColumns.Series)
-            .transform(lambda x: (x - x.mean()) / x.std())
-        )
-        outliers_mask = df["__z_score"].abs() > 3
-
-        if df[self.target_column_name].dtype == np.int:
-            df[self.target_column_name].astype(np.float)
-
-        df.loc[outliers_mask, self.target_column_name] = (
-            df[self.target_column_name]
-            .groupby(DataColumns.Series)
-            .transform(lambda x: np.median(x))
-        )
-        df_ret = df.drop("__z_score", axis=1)
-        return df_ret
-
-    def _check_historical_dataset(self, df):
-        expected_names = [self.target_column_name, self.dt_column_name] + (
-            self.target_category_columns if self.target_category_columns else []
-        )
-
-        if self.raw_column_names:
-            expected_names.extend(list(self.raw_column_names.values()))
-
-        if set(df.columns) != set(expected_names):
-            raise DataMismatchError(
-                f"Expected {self.name} to have columns: {expected_names}, but instead found column names: {df.columns}. Is the {self.name} path correct?"
-            )
-
-    """
-        Map between merged target category column values and target category column and its value
-        If target category columns are PPG_Code, Class, Num
-        Merged target category column values are Product Category 1__A__1, Product Category 2__A__2
-        Then target_category_columns_map would be
-        {
-            "Product Category 1__A__1": {
-                "PPG_Code": "Product Category 1",
-                "Class": "A",
-                "Num": 1
-            },
-             "Product Category 2__A__2": {
-                "PPG_Code": "Product Category 2",
-                "Class": "A",
-                "Num": 2
-            },
-        }
-    """
-
-    def get_target_category_columns_map(self):
-        return self._target_category_columns_map
-
-    def _fill_na(self, df: pd.DataFrame, na_value=0) -> pd.DataFrame:
-        """Fill nans in dataframe"""
-        return df.fillna(value=na_value)

ads/opctl/operator/lowcode/common/utils.py

@@ -1,293 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright (c) 2024, 2025 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
-
-import json
-import logging
-import os
-import shutil
-import sys
-import tempfile
-from typing import List, Union
-
-import fsspec
-import oracledb
-import pandas as pd
-
-from ads.common.object_storage_details import ObjectStorageDetails
-from ads.opctl import logger
-from ads.opctl.operator.common.operator_config import OutputDirectory
-from ads.opctl.operator.lowcode.common.errors import (
-    InvalidParameterError,
-)
-from ads.secrets import ADBSecretKeeper
-
-
-def call_pandas_fsspec(pd_fn, filename, storage_options, **kwargs):
-    if fsspec.utils.get_protocol(filename) == "file" or fsspec.utils.get_protocol(
-        filename
-    ) in ["http", "https"]:
-        return pd_fn(filename, **kwargs)
-
-    storage_options = storage_options or (
-        default_signer() if ObjectStorageDetails.is_oci_path(filename) else {}
-    )
-
-    return pd_fn(filename, storage_options=storage_options, **kwargs)
-
-
-def load_data(data_spec, storage_options=None, **kwargs):
-    if data_spec is None:
-        raise InvalidParameterError("No details provided for this data source.")
-    filename = data_spec.url
-    data = data_spec.data
-    format = data_spec.format
-    columns = data_spec.columns
-    connect_args = data_spec.connect_args
-    sql = data_spec.sql
-    table_name = data_spec.table_name
-    limit = data_spec.limit
-    vault_secret_id = data_spec.vault_secret_id
-    storage_options = storage_options or (
-        default_signer() if ObjectStorageDetails.is_oci_path(filename) else {}
-    )
-    if vault_secret_id is not None and connect_args is None:
-        connect_args = {}
-
-    if data is not None:
-        if format == "spark":
-            data = data.toPandas()
-    elif filename is not None:
-        if not format:
-            _, format = os.path.splitext(filename)
-            format = format[1:]
-        if format in ["json", "clipboard", "excel", "csv", "feather", "hdf", "parquet"]:
-            read_fn = getattr(pd, f"read_{format}")
-            data = call_pandas_fsspec(
-                read_fn, filename, storage_options=storage_options
-            )
-        elif format in ["tsv"]:
-            data = call_pandas_fsspec(
-                pd.read_csv, filename, storage_options=storage_options, sep="\t"
-            )
-        else:
-            raise InvalidParameterError(
-                f"The format {format} is not currently supported for reading data. Please reformat the data source: {filename} ."
-            )
-    elif connect_args is not None:
-        with tempfile.TemporaryDirectory() as temp_dir:
-            if vault_secret_id is not None:
-                try:
-                    with ADBSecretKeeper.load_secret(
-                        vault_secret_id, wallet_dir=temp_dir
-                    ) as adwsecret:
-                        if (
-                            "wallet_location" in adwsecret
-                            and "wallet_location" not in connect_args
-                        ):
-                            shutil.unpack_archive(
-                                adwsecret["wallet_location"], temp_dir
-                            )
-                            connect_args["wallet_location"] = temp_dir
-                        if "user_name" in adwsecret and "user" not in connect_args:
-                            connect_args["user"] = adwsecret["user_name"]
-                        if "password" in adwsecret and "password" not in connect_args:
-                            connect_args["password"] = adwsecret["password"]
-                        if (
-                            "service_name" in adwsecret
-                            and "service_name" not in connect_args
-                        ):
-                            connect_args["service_name"] = adwsecret["service_name"]
-
-                except Exception as e:
-                    raise Exception(
-                        f"Could not retrieve database credentials from vault {vault_secret_id}: {e}"
-                    ) from e
-
-            con = oracledb.connect(**connect_args)
-            if table_name is not None:
-                data = pd.read_sql(f"SELECT * FROM {table_name}", con)
-            elif sql is not None:
-                data = pd.read_sql(sql, con)
-            else:
-                raise InvalidParameterError(
-                    "Database `connect_args` provided without sql query or table name. Please specify either `sql` or `table_name`."
-                )
-    else:
-        raise InvalidParameterError(
-            "No filename/url provided, and no connect_args provided. Please specify one of these if you want to read data from a file or a database respectively."
-        )
-    if columns:
-        # keep only these columns, done after load because only CSV supports stream filtering
-        data = data[columns]
-    if limit:
-        data = data[:limit]
-    return data
-
-
-def write_data(data, filename, format, storage_options=None, index=False, **kwargs):
-    disable_print()
-    if not format:
-        _, format = os.path.splitext(filename)
-        format = format[1:]
-    if format in ["json", "clipboard", "excel", "csv", "feather", "hdf"]:
-        write_fn = getattr(data, f"to_{format}")
-        return call_pandas_fsspec(
-            write_fn, filename, index=index, storage_options=storage_options, **kwargs
-        )
-    enable_print()
-    raise InvalidParameterError(
-        f"The format {format} is not currently supported for writing data. Please change the format parameter for the data output: {filename} ."
-    )
-
-
-def write_json(json_dict, filename, storage_options=None):
-    with fsspec.open(filename, mode="w", **storage_options) as f:
-        f.write(json.dumps(json_dict))
-
-
-def write_simple_json(data, path):
-    if ObjectStorageDetails.is_oci_path(path):
-        storage_options = default_signer()
-    else:
-        storage_options = {}
-    with fsspec.open(path, mode="w", **storage_options) as f:
-        json.dump(data, f, indent=4)
-
-
-def merge_category_columns(data, target_category_columns):
-    result = data.apply(
-        lambda x: "__".join([str(x[col]) for col in target_category_columns]), axis=1
-    )
-    return result if not result.empty else pd.Series([], dtype=str)
-
-
-def merged_category_column_name(target_category_columns: Union[List, None]):
-    if not target_category_columns or len(target_category_columns) == 0:
-        return None
-    return "__".join([str(x) for x in target_category_columns])
-
-
-def datetime_to_seconds(s: pd.Series):
-    """
-    Method converts a datetime column into an integer number of seconds.
-    This method has many uses, most notably for enabling libraries like shap
-        to read datetime columns
-    ------------
-    s: pd.Series
-        A Series of type datetime
-    Returns
-    pd.Series of type int
-    """
-    return s.apply(lambda x: x.timestamp())
-
-
-def seconds_to_datetime(s: pd.Series, dt_format=None):
-    """
-    Inverse of `datetime_to_second`
-    ------------
-    s: pd.Series
-        A Series of type int
-    Returns
-    pd.Series of type datetime
-    """
-    s = pd.to_datetime(s, unit="s")
-    if dt_format is not None:
-        return pd.to_datetime(s, format=dt_format)
-    return s
-
-
-def default_signer(**kwargs):
-    os.environ["EXTRA_USER_AGENT_INFO"] = "Operator"
-    from ads.common.auth import default_signer
-
-    return default_signer(**kwargs)
-
-
-def get_frequency_in_seconds(s: pd.Series, sample_size=100, ignore_duplicates=True):
-    """
-    Returns frequency of data in seconds
-
-    Parameters
-    ------------
-    dt_col:  pd.Series  Datetime column
-    ignore_duplicates: bool if True, duplicates will be dropped before computing frequency
-
-    Returns
-    --------
-    int   Minimum difference in seconds
-    """
-    s1 = pd.Series(s).drop_duplicates() if ignore_duplicates else s
-    return s1.tail(20).diff().min().total_seconds()
-
-
-def get_frequency_of_datetime(dt_col: pd.Series, ignore_duplicates=True):
-    """
-    Returns string frequency of data
-
-    Parameters
-    ------------
-    dt_col:  pd.Series  Datetime column
-    ignore_duplicates: bool if True, duplicates will be dropped before computing frequency
-
-    Returns
-    --------
-    str  Pandas Datetime Frequency
-    """
-    s = pd.Series(dt_col).drop_duplicates() if ignore_duplicates else dt_col
-    return pd.infer_freq(s)
-
-
-def human_time_friendly(seconds):
-    TIME_DURATION_UNITS = (
-        ("week", 60 * 60 * 24 * 7),
-        ("day", 60 * 60 * 24),
-        ("hour", 60 * 60),
-        ("min", 60),
-    )
-    if seconds == 0:
-        return "inf"
-    accumulator = []
-    for unit, div in TIME_DURATION_UNITS:
-        amount, seconds = divmod(float(seconds), div)
-        if amount > 0:
-            accumulator.append(
-                "{} {}{}".format(int(amount), unit, "" if amount == 1 else "s")
-            )
-    accumulator.append(f"{round(seconds, 2)} secs")
-    return ", ".join(accumulator)
-
-
-def find_output_dirname(output_dir: OutputDirectory):
-    if output_dir and output_dir.url:
-        return output_dir.url
-    output_dir = "results"
-
-    # If the directory exists, find the next unique directory name by appending an incrementing suffix
-    counter = 1
-    unique_output_dir = f"{output_dir}"
-    while os.path.exists(unique_output_dir):
-        unique_output_dir = f"{output_dir}_{counter}"
-        counter += 1
-    logger.warning(
-        f"Since the output directory was not specified, the output will be saved to {unique_output_dir} directory."
-    )
-    return unique_output_dir
-
-
-def set_log_level(pkg_name: str, level: int):
-    pkg_logger = logging.getLogger(pkg_name)
-    pkg_logger.addHandler(logging.NullHandler())
-    pkg_logger.propagate = False
-    pkg_logger.setLevel(level)
-
-
-# Disable
-def disable_print():
-    sys.stdout = open(os.devnull, "w")
-
-
-# Restore
-def enable_print():
-    sys.stdout = sys.__stdout__

ads/opctl/operator/lowcode/feature_store_marketplace/MLoperator

@@ -1,13 +0,0 @@
-type: feature_store_marketplace
-version: v1
-name: Feature store deployment Operator
-gpu: no
-keywords:
-  - Helm
-  - marketplace
-  - Feature store
-backends:
-  - marketplace
-description: |
-  Feature store deployment operator helps ease the operational burden of deploying feature store helm charts from marketplace
-  to OKE cluster of your choice!!

ads/opctl/operator/lowcode/feature_store_marketplace/README.md

@@ -1,30 +0,0 @@
-# Feature Store Marketplace Operator
-The Feature Store Marketplace Operator simplifies the process of deploying Feature Store stack from Marketplace to your existing OKE cluster.
-
-Below are the steps to configure and run the Marketplace Operator:
-## 1. Prerequisites
-Follow the [CLI Configuration](https://accelerated-data-science.readthedocs.io/en/latest/user_guide/cli/opctl/configure.html) steps from the ADS documentation.
-
-## 2. Generating configs
-
-To generate starter configs, run the command below. This will create a list of YAML configs and place them in the `output` folder.
-
-```bash
-ads operator init -t feature_store_marketplace --overwrite --output ~/marketplace
-```
-
-The most important files expected to be generated are:
-
-- `feature_store_marketplace.yaml`: Contains feature store marketplace OKE related configuration.
-
-All generated configurations should be ready to use without the need for any additional adjustments. However, they are provided as starter kit configurations that can be customized as needed.
-
-## 3. Running feature store marketplace on the local environment
-
-Use the following command to run the forecasting on local environment.
-
-```bash
-ads operator run -f ~/marketplace/feature_store_marketplace.yaml -b marketplace
-```
-
-The operator will run in your local environment without requiring any additional modifications.

ads/opctl/operator/lowcode/feature_store_marketplace/__init__.py

@@ -1,5 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*--
-
-# Copyright (c) 2024 Oracle and/or its affiliates.
-# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/