oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.10rc0__py3-none-any.whl

ads/feature_store/common/exceptions.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+
+class NotMaterializedError(ValueError):
+    """
+    Exception raised when a resource is not materialized.
+
+    Attributes:
+        resource_name (str): The name of the resource that is not materialized.
+    """
+
+    def __init__(self, resource_type: str, resource_name: str):
+        self.resource_name = resource_name
+        super().__init__(f"{resource_type} {resource_name} is not materialized.")

ads/feature_store/common/spark_session_singleton.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+import copy
+
+import ads
+
+# Copyright (c) 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from ads.common.decorator.runtime_dependency import OptionalDependency
+import os
+from ads.common.oci_client import OCIClientFactory
+
+try:
+    from delta import configure_spark_with_delta_pip
+except ModuleNotFoundError:
+    raise ModuleNotFoundError(
+        f"The `pyspark` module was not found. Please run `pip install "
+        f"{OptionalDependency.SPARK}`."
+    )
+except Exception as e:
+    raise
+try:
+    from pyspark.sql import SparkSession
+except ModuleNotFoundError:
+    raise ModuleNotFoundError(
+        f"The `pyspark` module was not found. Please run `pip install "
+        f"{OptionalDependency.SPARK}`."
+    )
+except Exception as e:
+    raise
+
+
+def get_env_bool(env_var: str, default: bool = False) -> bool:
+    """
+    :param env_var: Environment variable name
+    :param default: Default environment variable value
+    :return: Value of the boolean env variable
+    """
+    env_val = os.getenv(env_var)
+    if env_val is None:
+        env_val = default
+    else:
+        env_val = env_val.lower()
+        if env_val == "true":
+            env_val = True
+        elif env_val == "false":
+            env_val = False
+        else:
+            raise ValueError(
+                "For environment variable: {0} only string values T/true or F/false are allowed but: \
+                {1} was provided.".format(
+                    env_var, env_val
+                )
+            )
+    return env_val
+
+
+def developer_enabled():
+    return get_env_bool("DEVELOPER_MODE", False)
+
+
+class SingletonMeta(type):
+    _instances = {}
+
+    def __call__(cls, *args, **kwargs):
+        if cls not in cls._instances:
+            cls._instances[cls] = super().__call__(*args, **kwargs)
+        return cls._instances[cls]
+
+
+class SparkSessionSingleton(metaclass=SingletonMeta):
+    """Class provides the spark session."""
+
+    def __init__(self, metastore_id: str = None):
+        """Virtually private constructor."""
+
+        spark_builder = (
+            SparkSession.builder.appName("FeatureStore")
+            .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension")
+            .config(
+                "spark.sql.catalog.spark_catalog",
+                "org.apache.spark.sql.delta.catalog.DeltaCatalog",
+            )
+            .enableHiveSupport()
+        )
+        _managed_table_location = None
+
+        if not developer_enabled() and metastore_id:
+            # Get the authentication credentials for the OCI data catalog service
+            auth = copy.copy(ads.auth.default_signer())
+
+            # Remove the "client_kwargs" key from the authentication credentials (if present)
+            auth.pop("client_kwargs", None)
+
+            data_catalog_client = OCIClientFactory(**auth).data_catalog
+            metastore = data_catalog_client.get_metastore(metastore_id).data
+            _managed_table_location = metastore.default_managed_table_location
+            # Configure the Spark session builder object to use the specified metastore
+            spark_builder.config(
+                "spark.hadoop.oracle.dcat.metastore.id", metastore_id
+            ).config("spark.sql.warehouse.dir", _managed_table_location).config(
+                "spark.driver.memory", "16G"
+            )
+
+        if developer_enabled():
+            # Configure spark session with delta jars only in developer mode. In other cases,
+            # jars should be part of the conda pack
+            self.spark_session = configure_spark_with_delta_pip(
+                spark_builder
+            ).getOrCreate()
+        else:
+            self.spark_session = spark_builder.getOrCreate()
+
+        self.spark_session.conf.set("spark.sql.execution.arrow.pyspark.enabled", "true")
+        self.spark_session.sparkContext.setLogLevel("OFF")
+        self.managed_table_location = _managed_table_location
+
+    def get_spark_session(self):
+        """Access method to get the spark session."""
+        return self.spark_session
+
+    def get_managed_table_location(self):
+        """Returns the managed table location for the spark"""
+        return self.managed_table_location

ads/feature_store/common/utils/base64_encoder_decoder.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import base64
+
+
+class Base64EncoderDecoder:
+    """
+    A utility class for encoding and decoding data in Base64 format.
+
+    Usage:
+    To encode a string:
+    ```
+    encoded_data = Base64EncoderDecoder.encode("Hello, World!")
+    ```
+
+    To decode an encoded string:
+    ```
+    decoded_data = Base64EncoderDecoder.decode(encoded_data)
+    ```
+
+    Attributes:
+    -----------
+    None
+
+    Methods:
+    --------
+    encode(raw_data: str) -> str:
+        Encodes the provided raw data into Base64 format and returns the encoded data as a string.
+
+    decode(encoded_data: str) -> str:
+        Decodes the provided Base64 data into its original format and returns the decoded data as a string.
+    """
+
+    @classmethod
+    def encode(cls, raw_data: str) -> str:
+        """Encodes raw data into Base64 format.
+
+        Parameters:
+        ----------
+        raw_data: str
+            The raw data that needs to be encoded.
+
+        Returns:
+        -------
+        str:
+            Base64-encoded data as a string.
+        """
+        byte_code = raw_data.encode("ascii")
+        base64_bytes = base64.b64encode(byte_code)
+        return base64_bytes.decode("ascii")
+
+    @classmethod
+    def decode(cls, encoded_data: str) -> str:
+        """Decodes Base64 data into its original format.
+
+        Parameters:
+        ----------
+        encoded_data: str
+            The Base64-encoded data that needs to be decoded.
+
+        Returns:
+        -------
+        str:
+            Decoded data as a string.
+        """
+        base64_bytes = encoded_data.encode("ascii")
+        message_bytes = base64.b64decode(base64_bytes)
+        return message_bytes.decode("ascii")

ads/feature_store/common/utils/feature_schema_mapper.py

@@ -0,0 +1,283 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import logging
+from typing import List
+
+import numpy as np
+import pandas as pd
+
+from ads.common.decorator.runtime_dependency import OptionalDependency
+from mlm_insights.constants import types
+
+from ads.feature_store.common.enums import FeatureType
+
+try:
+    from pyspark.sql.types import *
+except ModuleNotFoundError:
+    raise ModuleNotFoundError(
+        f"The `pyspark` module was not found. Please run `pip install "
+        f"{OptionalDependency.SPARK}`."
+    )
+except Exception as e:
+    raise
+
+logger = logging.getLogger(__name__)
+
+
+def map_spark_type_to_feature_type(spark_type):
+    """Returns the feature type corresponding to SparkType
+    :param spark_type:
+    :return:
+    """
+    spark_type_to_feature_type = {
+        StringType(): FeatureType.STRING,
+        IntegerType(): FeatureType.INTEGER,
+        ShortType(): FeatureType.SHORT,
+        LongType(): FeatureType.LONG,
+        FloatType(): FeatureType.FLOAT,
+        DoubleType(): FeatureType.DOUBLE,
+        BooleanType(): FeatureType.BOOLEAN,
+        DateType(): FeatureType.DATE,
+        TimestampType(): FeatureType.TIMESTAMP,
+        BinaryType(): FeatureType.BINARY,
+        ByteType(): FeatureType.BYTE,
+        ArrayType(StringType()): FeatureType.STRING_ARRAY,
+        ArrayType(IntegerType()): FeatureType.INTEGER_ARRAY,
+        ArrayType(LongType()): FeatureType.LONG_ARRAY,
+        ArrayType(FloatType()): FeatureType.FLOAT_ARRAY,
+        ArrayType(DoubleType()): FeatureType.DOUBLE_ARRAY,
+        ArrayType(BinaryType()): FeatureType.BINARY_ARRAY,
+        ArrayType(DateType()): FeatureType.DATE_ARRAY,
+        ArrayType(TimestampType()): FeatureType.TIMESTAMP_ARRAY,
+        ArrayType(ByteType()): FeatureType.BYTE_ARRAY,
+        ArrayType(BooleanType()): FeatureType.BOOLEAN_ARRAY,
+        ArrayType(ShortType()): FeatureType.SHORT_ARRAY,
+        MapType(StringType(), StringType()): FeatureType.STRING_STRING_MAP,
+        MapType(StringType(), IntegerType()): FeatureType.STRING_INTEGER_MAP,
+        MapType(StringType(), ShortType()): FeatureType.STRING_SHORT_MAP,
+        MapType(StringType(), LongType()): FeatureType.STRING_LONG_MAP,
+        MapType(StringType(), FloatType()): FeatureType.STRING_FLOAT_MAP,
+        MapType(StringType(), DoubleType()): FeatureType.STRING_DOUBLE_MAP,
+        MapType(StringType(), TimestampType()): FeatureType.STRING_TIMESTAMP_MAP,
+        MapType(StringType(), DateType()): FeatureType.STRING_DATE_MAP,
+        MapType(StringType(), BinaryType()): FeatureType.STRING_BINARY_MAP,
+        MapType(StringType(), ByteType()): FeatureType.STRING_BYTE_MAP,
+        MapType(StringType(), BooleanType()): FeatureType.STRING_BOOLEAN_MAP,
+    }
+    if spark_type in spark_type_to_feature_type:
+        return spark_type_to_feature_type.get(spark_type)
+    else:
+        return FeatureType.COMPLEX
+
+
+def map_pandas_type_to_feature_type(feature_name, values):
+    pandas_type = str(values.dtype)
+    inferred_dtype = FeatureType.UNKNOWN
+    if pandas_type is "object":
+        for row in values:
+            if isinstance(row, (list, np.ndarray)):
+                raise TypeError(f"object of type {type(row)} not supported")
+            pandas_basic_type = type(row).__name__
+            current_dtype = map_pandas_basic_type_to_feature_type(pandas_basic_type)
+            if inferred_dtype is FeatureType.UNKNOWN:
+                inferred_dtype = current_dtype
+            else:
+                if (
+                    current_dtype != inferred_dtype
+                    and current_dtype is not FeatureType.UNKNOWN
+                ):
+                    raise TypeError(
+                        f"Input feature '{feature_name}' has mixed types, {current_dtype} and {inferred_dtype}. "
+                        f"That is not allowed. "
+                    )
+    else:
+        inferred_dtype = map_pandas_basic_type_to_feature_type(pandas_type)
+    if inferred_dtype is FeatureType.UNKNOWN:
+        raise TypeError(
+            f"Input feature '{feature_name}' has type {str(pandas_type)} which is not supported"
+        )
+    else:
+        return inferred_dtype
+
+
+def map_pandas_basic_type_to_feature_type(pandas_type):
+    """Returns the feature type corresponding to pandas_type
+    :param pandas_type:
+    :return:
+    """
+    # TODO uint64 with bigger number cant be mapped to LongType
+    pandas_type_to_feature_type = {
+        "str": FeatureType.STRING,
+        "string": FeatureType.STRING,
+        "int": FeatureType.INTEGER,
+        "int8": FeatureType.INTEGER,
+        "int16": FeatureType.INTEGER,
+        "int32": FeatureType.LONG,
+        "int64": FeatureType.LONG,
+        "uint8": FeatureType.INTEGER,
+        "uint16": FeatureType.INTEGER,
+        "uint32": FeatureType.LONG,
+        "uint64": FeatureType.LONG,
+        "float": FeatureType.FLOAT,
+        "float16": FeatureType.FLOAT,
+        "float32": FeatureType.DOUBLE,
+        "float64": FeatureType.DOUBLE,
+        "datetime64[ns]": FeatureType.TIMESTAMP,
+        "datetime64[ns, UTC]": FeatureType.TIMESTAMP,
+        "timedelta64[ns]": FeatureType.LONG,
+        "bool": FeatureType.BOOLEAN,
+        "Decimal": FeatureType.DECIMAL,
+        "date": FeatureType.DATE,
+    }
+    if pandas_type in pandas_type_to_feature_type:
+        return pandas_type_to_feature_type.get(pandas_type)
+    return FeatureType.UNKNOWN
+
+
+def map_feature_type_to_spark_type(feature_type):
+    """Returns the Spark Type for a particular feature type.
+    :param feature_type:
+    :return: Spark Type
+    """
+    feature_type_in = FeatureType(feature_type)
+    spark_types = {
+        FeatureType.STRING: StringType(),
+        FeatureType.SHORT: ShortType(),
+        FeatureType.INTEGER: IntegerType(),
+        FeatureType.LONG: LongType(),
+        FeatureType.FLOAT: FloatType(),
+        FeatureType.DOUBLE: DoubleType(),
+        FeatureType.BOOLEAN: BooleanType(),
+        FeatureType.DATE: DateType(),
+        FeatureType.TIMESTAMP: TimestampType(),
+        FeatureType.DECIMAL: DecimalType(),
+        FeatureType.BINARY: BinaryType(),
+        FeatureType.STRING_ARRAY: ArrayType(StringType()),
+        FeatureType.INTEGER_ARRAY: ArrayType(IntegerType()),
+        FeatureType.SHORT_ARRAY: ArrayType(ShortType()),
+        FeatureType.LONG_ARRAY: ArrayType(LongType()),
+        FeatureType.FLOAT_ARRAY: ArrayType(FloatType()),
+        FeatureType.DOUBLE_ARRAY: ArrayType(DoubleType()),
+        FeatureType.BINARY_ARRAY: ArrayType(BinaryType()),
+        FeatureType.DATE_ARRAY: ArrayType(DateType()),
+        FeatureType.BOOLEAN_ARRAY: ArrayType(BooleanType()),
+        FeatureType.TIMESTAMP_ARRAY: ArrayType(TimestampType()),
+        FeatureType.STRING_STRING_MAP: MapType(StringType(), StringType()),
+        FeatureType.STRING_INTEGER_MAP: MapType(StringType(), IntegerType()),
+        FeatureType.STRING_SHORT_MAP: MapType(StringType(), ShortType()),
+        FeatureType.STRING_LONG_MAP: MapType(StringType(), LongType()),
+        FeatureType.STRING_FLOAT_MAP: MapType(StringType(), FloatType()),
+        FeatureType.STRING_DOUBLE_MAP: MapType(StringType(), DoubleType()),
+        FeatureType.STRING_DATE_MAP: MapType(StringType(), DateType()),
+        FeatureType.STRING_TIMESTAMP_MAP: MapType(StringType(), TimestampType()),
+        FeatureType.STRING_BOOLEAN_MAP: MapType(StringType(), BooleanType()),
+        FeatureType.BYTE: ByteType(),
+    }
+    if feature_type_in in spark_types:
+        return spark_types.get(feature_type_in)
+    else:
+        return "COMPLEX"
+
+
+def get_raw_data_source_schema(raw_feature_details: List[dict]):
+    """Converts input feature details to Spark schema.
+
+    Args:
+      raw_feature_details(List[dict]): List of input feature details.
+
+    Returns:
+      StructType: Spark schema.
+      :param raw_feature_details:
+    """
+    # Initialize the schema
+    features_schema = StructType()
+
+    for feature_details in raw_feature_details:
+        # Get the name, feature_type and is_nullable
+        feature_name = feature_details.get("name")
+        feature_type = map_feature_type_to_spark_type(
+            feature_details.get("featureType")
+        )
+        is_nullable = (
+            True
+            if feature_details.get("isNullable") is None
+            else feature_details.get("isNullable")
+        )
+
+        features_schema.add(feature_name, feature_type, is_nullable)
+
+    return features_schema
+
+
+def map_feature_type_to_pandas(feature_type):
+    feature_type_in = FeatureType(feature_type)
+    supported_feature_type = {
+        FeatureType.STRING: str,
+        FeatureType.LONG: "int64",
+        FeatureType.DOUBLE: "float64",
+        FeatureType.TIMESTAMP: "datetime64[ns]",
+        FeatureType.BOOLEAN: "bool",
+        FeatureType.FLOAT: "float32",
+        FeatureType.INTEGER: "int32",
+        FeatureType.DECIMAL: "object",
+        FeatureType.DATE: "object",
+        FeatureType.STRING_ARRAY: "object",
+        FeatureType.INTEGER_ARRAY: "object",
+        FeatureType.LONG_ARRAY: "object",
+        FeatureType.FLOAT_ARRAY: "object",
+        FeatureType.DOUBLE_ARRAY: "object",
+        FeatureType.TIMESTAMP_ARRAY: "object",
+        FeatureType.BOOLEAN_ARRAY: "object",
+        # FeatureType.DECIMAL_ARRAY: "object",
+        FeatureType.DATE_ARRAY: "object",
+    }
+    if feature_type_in in supported_feature_type:
+        return supported_feature_type.get(feature_type_in)
+    else:
+        raise TypeError(f"Feature Type {feature_type} is not supported for pandas")
+
+
+def map_spark_type_to_stats_data_type(spark_type):
+    """Maps the spark data types to MLM library data types
+    args:
+        param spark_type: Spark data type input from the feature dataframe on which we need stats
+    :return:
+        Returns the MLM data type corresponding to SparkType
+    """
+    spark_type_to_mlm_data_type = {
+        StringType(): types.DataType.STRING,
+        IntegerType(): types.DataType.INTEGER,
+        FloatType(): types.DataType.FLOAT,
+        DoubleType(): types.DataType.FLOAT,
+        BooleanType(): types.DataType.BOOLEAN,
+        DecimalType(): types.DataType.FLOAT,
+        ShortType(): types.DataType.INTEGER,
+        LongType(): types.DataType.INTEGER,
+    }
+
+    return spark_type_to_mlm_data_type.get(spark_type)
+
+
+def map_spark_type_to_stats_variable_type(spark_type):
+    """Maps the spark data types to MLM library variable types
+    args:
+        param spark_type: Spark data type input from the feature dataframe on which we need stats
+    :return:
+        Returns the MLM variable type corresponding to SparkType
+    """
+    spark_type_to_feature_type = {
+        StringType(): types.VariableType.NOMINAL,
+        IntegerType(): types.VariableType.CONTINUOUS,
+        FloatType(): types.VariableType.CONTINUOUS,
+        DoubleType(): types.VariableType.CONTINUOUS,
+        BooleanType(): types.VariableType.BINARY,
+        DecimalType(): types.VariableType.CONTINUOUS,
+        ShortType(): types.VariableType.CONTINUOUS,
+        LongType(): types.VariableType.CONTINUOUS,
+    }
+
+    return spark_type_to_feature_type.get(spark_type)

ads/feature_store/common/utils/transformation_utils.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import json
+
+# Copyright (c) 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from typing import Union
+
+from ads.common.decorator.runtime_dependency import OptionalDependency
+from ads.feature_store.common.utils.base64_encoder_decoder import Base64EncoderDecoder
+from ads.feature_store.transformation import Transformation, TransformationMode
+import pandas as pd
+
+try:
+    from pyspark.sql import SparkSession, DataFrame
+except ModuleNotFoundError:
+    raise ModuleNotFoundError(
+        f"The `pyspark` module was not found. Please run `pip install "
+        f"{OptionalDependency.SPARK}`."
+    )
+except Exception as e:
+    raise
+
+
+class TransformationUtils:
+    @staticmethod
+    def apply_transformation(
+        spark: SparkSession,
+        dataframe: Union[DataFrame, pd.DataFrame],
+        transformation: Transformation,
+        transformation_kwargs: str,
+    ):
+        """
+        Perform data transformation using either SQL or Pandas, depending on the specified transformation mode.
+
+        Args:
+            spark: A SparkSession object.
+            transformation (Transformation): A transformation object containing details of transformation to be performed.
+            dataframe (DataFrame): The input dataframe to be transformed.
+            transformation_kwargs(str): The transformation parameters as json string.
+
+        Returns:
+            DataFrame: The resulting transformed data.
+        """
+
+        # Fetch the transformation function
+        transformation_function = Base64EncoderDecoder.decode(
+            transformation.source_code_function
+        )
+
+        # Execute the function under namespace
+        execution_namespace = {}
+        exec(transformation_function, execution_namespace)
+        transformation_function_caller = execution_namespace.get(transformation.name)
+        transformed_data = None
+
+        transformation_kwargs_dict = json.loads(transformation_kwargs)
+
+        if transformation.transformation_mode == TransformationMode.SQL.value:
+            # Register the temporary table
+            temporary_table_view = "df_view"
+            dataframe.createOrReplaceTempView(temporary_table_view)
+
+            transformed_data = spark.sql(
+                transformation_function_caller(
+                    temporary_table_view, **transformation_kwargs_dict
+                )
+            )
+        elif transformation.transformation_mode in [
+            TransformationMode.PANDAS.value,
+            TransformationMode.SPARK.value,
+        ]:
+            transformed_data = transformation_function_caller(
+                dataframe, **transformation_kwargs_dict
+            )
+
+        return transformed_data