oracle-ads 2.10.0__py3-none-any.whl → 2.11.0__py3-none-any.whl

ads/opctl/operator/lowcode/common/transformations.py

@@ -0,0 +1,191 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from ads.opctl import logger
+from ads.opctl.operator.lowcode.common.errors import (
+    InvalidParameterError,
+    DataMismatchError,
+)
+from ads.opctl.operator.lowcode.common.const import DataColumns
+from ads.opctl.operator.lowcode.common.utils import merge_category_columns
+import pandas as pd
+from abc import ABC
+
+
+class Transformations(ABC):
+    """A class which implements transformation for forecast operator"""
+
+    def __init__(self, dataset_info, name="historical_data"):
+        """
+        Initializes the transformation.
+
+        Parameters
+        ----------
+            data: The Pandas DataFrame.
+            dataset_info : ForecastOperatorConfig
+        """
+        self.name = name
+        self.has_artificial_series = False
+        self.dataset_info = dataset_info
+        self.target_category_columns = dataset_info.target_category_columns
+        self.target_column_name = dataset_info.target_column
+        self.dt_column_name = dataset_info.datetime_column.name
+        self.dt_column_format = dataset_info.datetime_column.format
+        self.preprocessing = dataset_info.preprocessing
+
+    def run(self, data):
+        """
+        The function runs all the transformation in a particular order.
+
+        Returns
+        -------
+            A new Pandas DataFrame with treated / transformed target values. Specifically:
+            - Data will be in a multiIndex with Datetime always first (level 0)
+            - whether 0, 1 or 2+, all target_category_columns will be merged into a single index column: Series
+            - All datetime columns will be formatted as such
+            - all data will be imputed (unless preprocessing disabled)
+            - all trailing whitespace will be removed
+            - the data will be sorted by Datetime then Series
+
+        """
+        clean_df = self._remove_trailing_whitespace(data)
+        if self.name == "historical_data":
+            self._check_historical_dataset(clean_df)
+        clean_df = self._set_series_id_column(clean_df)
+        clean_df = self._format_datetime_col(clean_df)
+        clean_df = self._set_multi_index(clean_df)
+
+        if self.name == "historical_data":
+            try:
+                clean_df = self._missing_value_imputation_hist(clean_df)
+            except Exception as e:
+                logger.debug(f"Missing value imputation failed with {e.args}")
+            if self.preprocessing:
+                try:
+                    clean_df = self._outlier_treatment(clean_df)
+                except Exception as e:
+                    logger.debug(f"Outlier Treatment failed with {e.args}")
+            else:
+                logger.debug("Skipping outlier treatment as preprocessing is disabled")
+        elif self.name == "additional_data":
+            clean_df = self._missing_value_imputation_add(clean_df)
+        return clean_df
+
+    def _remove_trailing_whitespace(self, df):
+        return df.apply(lambda x: x.str.strip() if x.dtype == "object" else x)
+
+    def _set_series_id_column(self, df):
+        if not self.target_category_columns:
+            df[DataColumns.Series] = "Series 1"
+            self.has_artificial_series = True
+        else:
+            df[DataColumns.Series] = merge_category_columns(
+                df, self.target_category_columns
+            )
+            df = df.drop(self.target_category_columns, axis=1)
+        return df
+
+    def _format_datetime_col(self, df):
+        try:
+            df[self.dt_column_name] = pd.to_datetime(
+                df[self.dt_column_name], format=self.dt_column_format
+            )
+        except:
+            raise InvalidParameterError(
+                f"Unable to determine the datetime type for column: {self.dt_column_name} in dataset: {self.name}. Please specify the format explicitly. (For example adding 'format: %d/%m/%Y' underneath 'name: {self.dt_column_name}' in the datetime_column section of the yaml file if you haven't already. For reference, here is the first datetime given: {df[self.dt_column_name].values[0]}"
+            )
+        return df
+
+    def _set_multi_index(self, df):
+        """
+        Function sorts by date
+
+        Parameters
+        ----------
+            df : The Pandas DataFrame.
+
+        Returns
+        -------
+            A new Pandas DataFrame with sorted dates for each series
+        """
+        df = df.set_index([self.dt_column_name, DataColumns.Series])
+        return df.sort_values([self.dt_column_name, DataColumns.Series], ascending=True)
+
+    def _missing_value_imputation_hist(self, df):
+        """
+        Function fills missing values in the pandas dataframe using liner interpolation
+
+        Parameters
+        ----------
+            df : The Pandas DataFrame.
+
+        Returns
+        -------
+            A new Pandas DataFrame without missing values.
+        """
+        # missing value imputation using linear interpolation
+        df[self.target_column_name] = (
+            df[self.target_column_name]
+            .groupby(DataColumns.Series)
+            .transform(lambda x: x.interpolate(limit_direction="both"))
+        )
+        return df
+
+    def _missing_value_imputation_add(self, df):
+        """
+        Function fills missing values in the pandas dataframe using liner interpolation
+
+        Parameters
+        ----------
+            df : The Pandas DataFrame.
+
+        Returns
+        -------
+            A new Pandas DataFrame without missing values.
+        """
+        # find columns that all all NA and replace with 0
+        for col in df.columns:
+            # find next int not in list
+            i = 0
+            vals = df[col].unique()
+            while i in vals:
+                i = i + 1
+            df[col] = df[col].fillna(0)
+        return df
+
+    def _outlier_treatment(self, df):
+        """
+        Function finds outliears using z_score and treats with mean value.
+
+        Parameters
+        ----------
+            df : The Pandas DataFrame.
+
+        Returns
+        -------
+            A new Pandas DataFrame with treated outliears.
+        """
+        df["z_score"] = (
+            df[self.target_column_name]
+            .groupby(DataColumns.Series)
+            .transform(lambda x: (x - x.mean()) / x.std())
+        )
+        outliers_mask = df["z_score"].abs() > 3
+        df.loc[outliers_mask, self.target_column_name] = (
+            df[self.target_column_name]
+            .groupby(DataColumns.Series)
+            .transform(lambda x: x.mean())
+        )
+        return df.drop("z_score", axis=1)
+
+    def _check_historical_dataset(self, df):
+        expected_names = [self.target_column_name, self.dt_column_name] + (
+            self.target_category_columns if self.target_category_columns else []
+        )
+        if set(df.columns) != set(expected_names):
+            raise DataMismatchError(
+                f"Expected {self.name} to have columns: {expected_names}, but instead found column names: {df.columns}. Is the {self.name} path correct?"
+            )

ads/opctl/operator/lowcode/common/utils.py

@@ -0,0 +1,250 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import argparse
+import logging
+import os
+import sys
+import time
+from string import Template
+from typing import Any, Dict, List, Tuple
+import pandas as pd
+from ads.opctl import logger
+import oracledb
+
+import fsspec
+import yaml
+from typing import Union
+
+from ads.opctl import logger
+from ads.opctl.operator.lowcode.common.errors import (
+    InputDataError,
+    InvalidParameterError,
+    PermissionsError,
+    DataMismatchError,
+)
+from ads.opctl.operator.common.operator_config import OutputDirectory
+from ads.common.object_storage_details import ObjectStorageDetails
+
+
+def call_pandas_fsspec(pd_fn, filename, storage_options, **kwargs):
+    if fsspec.utils.get_protocol(filename) == "file":
+        return pd_fn(filename, **kwargs)
+    elif fsspec.utils.get_protocol(filename) in ["http", "https"]:
+        return pd_fn(filename, **kwargs)
+
+    storage_options = storage_options or (
+        default_signer() if ObjectStorageDetails.is_oci_path(filename) else {}
+    )
+
+    return pd_fn(filename, storage_options=storage_options, **kwargs)
+
+
+def load_data(data_spec, storage_options=None, **kwargs):
+    if data_spec is None:
+        raise InvalidParameterError(f"No details provided for this data source.")
+    filename = data_spec.url
+    format = data_spec.format
+    columns = data_spec.columns
+    connect_args = data_spec.connect_args
+    sql = data_spec.sql
+    table_name = data_spec.table_name
+    limit = data_spec.limit
+
+    storage_options = storage_options or (
+        default_signer() if ObjectStorageDetails.is_oci_path(filename) else {}
+    )
+
+    if filename is not None:
+        if not format:
+            _, format = os.path.splitext(filename)
+            format = format[1:]
+        if format in ["json", "clipboard", "excel", "csv", "feather", "hdf"]:
+            read_fn = getattr(pd, f"read_{format}")
+            data = call_pandas_fsspec(
+                read_fn, filename, storage_options=storage_options
+            )
+        elif format in ["tsv"]:
+            data = call_pandas_fsspec(
+                pd.read_csv, filename, storage_options=storage_options, sep="\t"
+            )
+        else:
+            raise InvalidParameterError(
+                f"The format {format} is not currently supported for reading data. Please reformat the data source: {filename} ."
+            )
+    elif connect_args is not None:
+        con = oracledb.connect(**connect_args)
+        if table_name is not None:
+            data = pd.read_sql_table(table_name, con)
+        elif sql is not None:
+            data = pd.read_sql(sql, con)
+        else:
+            raise InvalidParameterError(
+                f"Database `connect_args` provided without sql query or table name. Please specify either `sql` or `table_name`."
+            )
+    else:
+        raise InvalidParameterError(
+            f"No filename/url provided, and no connect_args provided. Please specify one of these if you want to read data from a file or a database respectively."
+        )
+    if columns:
+        # keep only these columns, done after load because only CSV supports stream filtering
+        data = data[columns]
+    if limit:
+        data = data[:limit]
+    return data
+
+
+def write_data(data, filename, format, storage_options, index=False, **kwargs):
+    if not format:
+        _, format = os.path.splitext(filename)
+        format = format[1:]
+    if format in ["json", "clipboard", "excel", "csv", "feather", "hdf"]:
+        write_fn = getattr(data, f"to_{format}")
+        return call_pandas_fsspec(
+            write_fn, filename, index=index, storage_options=storage_options, **kwargs
+        )
+    raise OperatorYamlContentError(
+        f"The format {format} is not currently supported for writing data. Please change the format parameter for the data output: {filename} ."
+    )
+
+
+def merge_category_columns(data, target_category_columns):
+    result = data.apply(
+        lambda x: "__".join([str(x[col]) for col in target_category_columns]), axis=1
+    )
+    return result if not result.empty else pd.Series([], dtype=str)
+
+
+def merged_category_column_name(target_category_columns: Union[List, None]):
+    if not target_category_columns or len(target_category_columns) == 0:
+        return None
+    return "__".join([str(x) for x in target_category_columns])
+
+
+def datetime_to_seconds(s: pd.Series):
+    """
+    Method converts a datetime column into an integer number of seconds.
+    This method has many uses, most notably for enabling libraries like shap
+        to read datetime columns
+    ------------
+    s: pd.Series
+        A Series of type datetime
+    Returns
+    pd.Series of type int
+    """
+    return s.apply(lambda x: x.timestamp())
+
+
+def seconds_to_datetime(s: pd.Series, dt_format=None):
+    """
+    Inverse of `datetime_to_second`
+    ------------
+    s: pd.Series
+        A Series of type int
+    Returns
+    pd.Series of type datetime
+    """
+    s = pd.to_datetime(s, unit="s")
+    if dt_format is not None:
+        return pd.to_datetime(s, format=dt_format)
+    return s
+
+
+def default_signer(**kwargs):
+    os.environ["EXTRA_USER_AGENT_INFO"] = "Operator"
+    from ads.common.auth import default_signer
+
+    return default_signer(**kwargs)
+
+
+def get_frequency_in_seconds(s: pd.Series, sample_size=100, ignore_duplicates=True):
+    """
+    Returns frequency of data in seconds
+
+    Parameters
+    ------------
+    dt_col:  pd.Series  Datetime column
+    ignore_duplicates: bool if True, duplicates will be dropped before computing frequency
+
+    Returns
+    --------
+    int   Minimum difference in seconds
+    """
+    s1 = pd.Series(s).drop_duplicates() if ignore_duplicates else s
+    return s1.tail(20).diff().min().total_seconds()
+
+
+def get_frequency_of_datetime(dt_col: pd.Series, ignore_duplicates=True):
+    """
+    Returns string frequency of data
+
+    Parameters
+    ------------
+    dt_col:  pd.Series  Datetime column
+    ignore_duplicates: bool if True, duplicates will be dropped before computing frequency
+
+    Returns
+    --------
+    str  Pandas Datetime Frequency
+    """
+    s = pd.Series(dt_col).drop_duplicates() if ignore_duplicates else dt_col
+    return pd.infer_freq(s)
+
+
+def human_time_friendly(seconds):
+    TIME_DURATION_UNITS = (
+        ("week", 60 * 60 * 24 * 7),
+        ("day", 60 * 60 * 24),
+        ("hour", 60 * 60),
+        ("min", 60),
+    )
+    if seconds == 0:
+        return "inf"
+    accumulator = []
+    for unit, div in TIME_DURATION_UNITS:
+        amount, seconds = divmod(float(seconds), div)
+        if amount > 0:
+            accumulator.append(
+                "{} {}{}".format(int(amount), unit, "" if amount == 1 else "s")
+            )
+    accumulator.append("{} secs".format(round(seconds, 2)))
+    return ", ".join(accumulator)
+
+
+def find_output_dirname(output_dir: OutputDirectory):
+    if output_dir:
+        return output_dir.url
+    output_dir = "results"
+
+    # If the directory exists, find the next unique directory name by appending an incrementing suffix
+    counter = 1
+    unique_output_dir = f"{output_dir}"
+    while os.path.exists(unique_output_dir):
+        unique_output_dir = f"{output_dir}_{counter}"
+        counter += 1
+    logger.warn(
+        "Since the output directory was not specified, the output will be saved to {} directory.".format(
+            unique_output_dir
+        )
+    )
+    return unique_output_dir
+
+
+def set_log_level(pkg_name: str, level: int):
+    pkg_logger = logging.getLogger(pkg_name)
+    pkg_logger.addHandler(logging.NullHandler())
+    pkg_logger.propagate = False
+    pkg_logger.setLevel(level)
+
+
+# Disable
+def disable_print():
+    sys.stdout = open(os.devnull, "w")
+
+
+# Restore
+def enable_print():
+    sys.stdout = sys.__stdout__

ads/opctl/operator/lowcode/forecast/README.md

@@ -38,8 +38,9 @@ To run forecasting locally, create and activate a new conda environment (`ads-fo
 - datapane
 - cerberus
 - sktime
-- optuna==2.9.0
-- oracle-automlx==23.2.3
+- optuna==3.1.0
+- oracle-automlx==23.4.1
+- oracle-automlx[forecasting]==23.4.1
 - oracle-ads>=2.9.0
 ```
 

ads/opctl/operator/lowcode/forecast/__main__.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*--
 
-# Copyright (c) 2023 Oracle and/or its affiliates.
+# Copyright (c) 2023, 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 import json
@@ -24,7 +24,23 @@ def operate(operator_config: ForecastOperatorConfig) -> None:
     from .model.factory import ForecastOperatorModelFactory
 
     datasets = ForecastDatasets(operator_config)
-    ForecastOperatorModelFactory.get_model(operator_config, datasets).generate_report()
+    try:
+        ForecastOperatorModelFactory.get_model(
+            operator_config, datasets
+        ).generate_report()
+    except Exception as e:
+        if operator_config.spec.model == "auto":
+            logger.debug(
+                f"Failed to forecast with error {e.args}. Trying again with model `prophet`."
+            )
+            operator_config.spec.model = "prophet"
+            operator_config.spec.model_kwargs = dict()
+            datasets = ForecastDatasets(operator_config)
+            ForecastOperatorModelFactory.get_model(
+                operator_config, datasets
+            ).generate_report()
+        else:
+            raise
 
 
 def verify(spec: Dict, **kwargs: Dict) -> bool:

ads/opctl/operator/lowcode/forecast/cmd.py

@@ -32,13 +32,14 @@ def init(**kwargs: Dict) -> str:
     str
         The YAML specification generated based on the schema.
     """
-    logger.info("==== Forecasting related options ====")
-
-    model_type = click.prompt(
-        "Provide a model type:",
-        type=click.Choice(SupportedModels.values()),
-        default=SupportedModels.Auto,
-    )
+    # logger.info("==== Forecasting related options ====")
+
+    # model_type = click.prompt(
+    #     "Provide a model type:",
+    #     type=click.Choice(SupportedModels.values()),
+    #     default=SupportedModels.Auto,
+    # )
+    model_type = "auto"
 
     return YamlGenerator(
         schema=_load_yaml_from_uri(__file__.replace("cmd.py", "schema.yaml"))

ads/opctl/operator/lowcode/forecast/const.py

@@ -5,6 +5,7 @@
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 from ads.common.extended_enum import ExtendedEnumMeta
+from ads.opctl.operator.lowcode.common.const import DataColumns
 
 
 class SupportedModels(str, metaclass=ExtendedEnumMeta):
@@ -18,6 +19,20 @@ class SupportedModels(str, metaclass=ExtendedEnumMeta):
     Auto = "auto"
 
 
+class SpeedAccuracyMode(str, metaclass=ExtendedEnumMeta):
+    """
+    Enum representing different modes based on time taken and accuracy for explainability.
+    """
+
+    HIGH_ACCURACY = "HIGH_ACCURACY"
+    BALANCED = "BALANCED"
+    FAST_APPROXIMATE = "FAST_APPROXIMATE"
+    ratio = dict()
+    ratio[HIGH_ACCURACY] = 1  # 100 % data used for generating explanations
+    ratio[BALANCED] = 0.5  # 50 % data used for generating explanations
+    ratio[FAST_APPROXIMATE] = 0  # constant
+
+
 class SupportedMetrics(str, metaclass=ExtendedEnumMeta):
     """Supported forecast metrics."""
 
@@ -49,7 +64,7 @@ class ForecastOutputColumns(str, metaclass=ExtendedEnumMeta):
     """The column names for the forecast.csv output file"""
 
     DATE = "Date"
-    SERIES = "Series"
+    SERIES = DataColumns.Series
     INPUT_VALUE = "input_value"
     FITTED_VALUE = "fitted_value"
     FORECAST_VALUE = "forecast_value"
@@ -70,3 +85,4 @@ MAX_COLUMNS_AUTOMLX = 15
 DEFAULT_TRIALS = 10
 SUMMARY_METRICS_HORIZON_LIMIT = 10
 PROPHET_INTERNAL_DATE_COL = "ds"
+RENDER_LIMIT = 5000

ads/opctl/operator/lowcode/forecast/environment.yaml

@@ -15,5 +15,6 @@ dependencies:
       - sktime
       - shap
       - autots[additional]
-      - optuna==2.9.0
-      - oracle-automlx==23.2.3
+      - optuna==3.1.0
+      - oracle-automlx==23.4.1
+      - oracle-automlx[forecasting]==23.4.1