oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.9rc1__py3-none-any.whl

ads/model/framework/spark_model.py

@@ -0,0 +1,326 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2022, 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import os
+import numpy as np
+import pandas as pd
+import fsspec
+from typing import Any, Callable, Dict, List, Optional, Union, Tuple
+from ads.common import logger, utils
+from ads.common.decorator.runtime_dependency import (
+    runtime_dependency,
+    OptionalDependency,
+)
+from ads.common.object_storage_details import ObjectStorageDetails
+from ads.model.extractor.spark_extractor import SparkExtractor
+from ads.model.serde.model_input import SparkModelInputSerializerType
+from ads.model.serde.model_serializer import SparkModelSerializerType
+from ads.model.generic_model import (
+    FrameworkSpecificModel,
+    DEFAULT_MODEL_FOLDER_NAME,
+    _prepare_artifact_dir,
+)
+from ads.model.model_properties import ModelProperties
+from ads.model.serde.common import SERDE
+
+SPARK_DATAFRAME_SCHEMA_PATH = "_input_data_schema.json"
+
+
+class SparkPipelineModel(FrameworkSpecificModel):
+    """SparkPipelineModel class for estimators from the pyspark framework.
+
+    Attributes
+    ----------
+    algorithm: str
+        The algorithm of the model.
+    artifact_dir: str
+        Artifact directory to store the files needed for deployment.
+    auth: Dict
+        Default authentication is set using the `ads.set_auth` API. To override the
+        default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create
+        an authentication signer to instantiate an IdentityClient object.
+    estimator: Callable
+        A trained pyspark estimator/model using pyspark.
+    framework: str
+        "spark", the framework name of the model.
+    hyperparameter: dict
+        The hyperparameters of the estimator.
+    metadata_custom: ModelCustomMetadata
+        The model custom metadata.
+    metadata_provenance: ModelProvenanceMetadata
+        The model provenance metadata.
+    metadata_taxonomy: ModelTaxonomyMetadata
+        The model taxonomy metadata.
+    model_artifact: ModelArtifact
+        This is built by calling prepare.
+        A ModelDeployment instance.
+    model_file_name: str
+        Name of the serialized model.
+    model_id: str
+        The model ID.
+    properties: ModelProperties
+        ModelProperties object required to save and deploy model.
+        For more details, check https://accelerated-data-science.readthedocs.io/en/latest/ads.model.html#module-ads.model.model_properties.
+    runtime_info: RuntimeInfo
+        A RuntimeInfo instance.
+    schema_input: Schema
+        Schema describes the structure of the input data.
+    schema_output: Schema
+        Schema describes the structure of the output data.
+    serialize: bool
+        Whether to serialize the model to pkl file by default. If False, you need to serialize the model manually,
+        save it under artifact_dir and update the score.py manually.
+    version: str
+        The framework version of the model.
+
+    Methods
+    -------
+    delete_deployment(...)
+        Deletes the current model deployment.
+    deploy(..., **kwargs)
+        Deploys a model.
+    from_model_artifact(uri, model_file_name, artifact_dir, ..., **kwargs)
+        Loads model from the specified folder, or zip/tar archive.
+    from_model_catalog(model_id, model_file_name, artifact_dir, ..., **kwargs)
+        Loads model from model catalog.
+    introspect(...)
+        Runs model introspection.
+    predict(data, ...)
+        Returns prediction of input data run against the model deployment endpoint.
+    prepare(..., **kwargs)
+        Prepare and save the score.py, serialized model and runtime.yaml file.
+    reload(...)
+        Reloads the model artifact files: `score.py` and the `runtime.yaml`.
+    save(..., **kwargs)
+        Saves model artifacts to the model catalog.
+    summary_status(...)
+        Gets a summary table of the current status.
+    verify(data, ...)
+        Tests if deployment works in local environment.
+
+    Examples
+    --------
+    >>> import tempfile
+    >>> from ads.model.framework.spark_model import SparkPipelineModel
+    >>> from pyspark.ml.linalg import Vectors
+    >>> from pyspark.ml.classification import LogisticRegression
+
+    >>> training = spark.createDataFrame([
+    >>>     (1.0, Vectors.dense([0.0, 1.1, 0.1])),
+    >>>     (0.0, Vectors.dense([2.0, 1.0, -1.0])),
+    >>>     (0.0, Vectors.dense([2.0, 1.3, 1.0])),
+    >>>     (1.0, Vectors.dense([0.0, 1.2, -0.5]))], ["label", "features"])
+    >>> lr_estimator = LogisticRegression(maxIter=10, regParam=0.001)
+    >>> pipeline = Pipeline(stages=[lr_estimator])
+    >>> pipeline_model = pipeline.fit(training)
+
+    >>> spark_model = SparkPipelineModel(estimator=pipeline_model, artifact_dir=tempfile.mkdtemp())
+    >>> spark_model.prepare(inference_conda_env="dataexpl_p37_cpu_v3")
+    >>> spark_model.verify(training)
+    >>> spark_model.save()
+    >>> model_deployment = spark_model.deploy()
+    >>> spark_model.predict(training)
+    >>> spark_model.delete_deployment()
+    """
+
+    _PREFIX = "spark"
+    model_input_serializer_type = SparkModelInputSerializerType
+    model_save_serializer_type = SparkModelSerializerType
+
+    @runtime_dependency(
+        module="pyspark",
+        short_name="ml",
+        object="ml",
+        install_from=OptionalDependency.SPARK,
+    )
+    def __init__(
+        self,
+        estimator: Callable,
+        artifact_dir: Optional[str] = None,
+        properties: Optional[ModelProperties] = None,
+        auth: Dict = None,
+        model_save_serializer: Optional[SERDE] = model_save_serializer_type.SPARK,
+        model_input_serializer: Optional[SERDE] = model_input_serializer_type.SPARK,
+        **kwargs,
+    ):
+        """
+        Initiates a SparkPipelineModel instance.
+
+        Parameters
+        ----------
+        estimator: Callable
+            SparkPipelineModel
+        artifact_dir: str
+            The URI for the generated artifact, which can be local path or OCI object storage URI.
+        properties: (ModelProperties, optional). Defaults to None.
+            ModelProperties object required to save and deploy model.
+        auth :(Dict, optional). Defaults to None.
+            The default authetication is set using `ads.set_auth` API. If you need to override the
+            default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
+            authentication signer and kwargs required to instantiate IdentityClient object.
+        model_save_serializer: (SERDE or str, optional). Defaults to None.
+            Instance of ads.model.SERDE. Used for serialize/deserialize model.
+        model_input_serializer: (SERDE, optional). Defaults to `ads.model.serde.model_input.SparkModelInputSERDE`.
+            Instance of ads.model.SERDE. Used for serialize/deserialize data.
+
+        Returns
+        -------
+        SparkPipelineModel
+            SparkPipelineModel instance.
+
+
+        Examples
+        --------
+        >>> import tempfile
+        >>> from ads.model.framework.spark_model import SparkPipelineModel
+        >>> from pyspark.ml.linalg import Vectors
+        >>> from pyspark.ml.classification import LogisticRegression
+        >>> from pyspark.ml import Pipeline
+
+        >>> training = spark.createDataFrame([
+        >>>     (1.0, Vectors.dense([0.0, 1.1, 0.1])),
+        >>>     (0.0, Vectors.dense([2.0, 1.0, -1.0])),
+        >>>     (0.0, Vectors.dense([2.0, 1.3, 1.0])),
+        >>>     (1.0, Vectors.dense([0.0, 1.2, -0.5]))], ["label", "features"])
+        >>> lr_estimator = LogisticRegression(maxIter=10, regParam=0.001)
+        >>> pipeline = Pipeline(stages=[lr_estimator])
+        >>> pipeline_model = pipeline.fit(training)
+
+        >>> spark_model = SparkPipelineModel(estimator=pipeline_model, artifact_dir=tempfile.mkdtemp())
+        >>> spark_model.prepare(inference_conda_env="pyspark30_p37_cpu_v5")
+        >>> spark_model.verify(training)
+        >>> spark_model.save()
+        >>> model_deployment = spark_model.deploy()
+        >>> spark_model.predict(training)
+        >>> spark_model.delete_deployment()
+        """
+        if not (type(estimator) in [ml.PipelineModel]):
+            raise TypeError(
+                f"{str(type(estimator))} is not supported in `SparkPipelineModel`s."
+            )
+        super().__init__(
+            estimator=estimator,
+            artifact_dir=artifact_dir,
+            properties=properties,
+            auth=auth,
+            model_save_serializer=model_save_serializer,
+            model_input_serializer=model_input_serializer,
+            **kwargs,
+        )
+
+        self._extractor = SparkExtractor(estimator)
+        self.framework = self._extractor.framework
+        self.algorithm = self._extractor.algorithm
+        self.version = self._extractor.version
+        self.hyperparameter = self._extractor.hyperparameter
+
+    @staticmethod
+    def _handle_model_file_name(
+        as_onnx: bool, model_file_name: Optional[str] = DEFAULT_MODEL_FOLDER_NAME
+    ):
+        """
+        Process folder name for saving model.
+
+        Parameters
+        ----------
+        as_onnx: bool
+            To convert to onnx format
+        model_file_name: Optional[str]
+            File name for saving model. Default value is `model`.
+
+        Returns
+        -------
+        str
+            Processed file name. (Folder in the case of spark serialization)
+        """
+        if as_onnx:
+            raise NotImplementedError(
+                "The Spark to Onnx Conversion is not supported because it is unstable. Please set as_onnx to False (default) to perform a spark model serialization"
+            )
+        if not model_file_name:
+            return DEFAULT_MODEL_FOLDER_NAME
+        return model_file_name
+
+    def serialize_model(
+        self,
+        as_onnx: bool = False,
+        X_sample: Optional[
+            Union[
+                Dict,
+                str,
+                List,
+                np.ndarray,
+                pd.core.series.Series,
+                pd.core.frame.DataFrame,
+                "pyspark.sql.DataFrame",
+                "pyspark.pandas.DataFrame",
+            ]
+        ] = None,
+        force_overwrite: bool = False,
+        **kwargs,
+    ) -> None:
+        """
+        Serialize and save pyspark model using spark serialization.
+
+        Parameters
+        ----------
+        force_overwrite: (bool, optional). Defaults to False.
+            If set as True, overwrite serialized model if exists.
+
+        Returns
+        -------
+        None
+        """
+        if as_onnx:
+            raise NotImplementedError(
+                "The Spark to Onnx Conversion is not supported because it is unstable. Please set as_onnx to False (default) to perform a spark model serialization"
+            )
+
+        super().serialize_model(
+            as_onnx=as_onnx,
+            force_overwrite=force_overwrite,
+            X_sample=X_sample,
+            **kwargs,
+        )
+
+    @runtime_dependency(
+        module="pyspark",
+        short_name="sql",
+        object="sql",
+        install_from=OptionalDependency.SPARK,
+    )
+    def _prepare_data_for_schema(
+        self,
+        X_sample: Union[List, Tuple, pd.DataFrame, pd.Series, np.ndarray] = None,
+        y_sample: Union[List, Tuple, pd.DataFrame, pd.Series, np.ndarray] = None,
+        **kwargs,
+    ):
+        """Generate Spark Schema and format Spark DataFrame as Pandas for ADS Schema Generation"""
+        input_schema_path = os.path.join(
+            self.artifact_dir,
+            self.model_file_name + SPARK_DATAFRAME_SCHEMA_PATH,
+        )
+        (
+            X_sample,
+            data_type,
+            schema,
+        ) = self.get_data_serializer()._serialize_via_spark(X_sample)
+        if not schema:
+            raise TypeError(
+                f"Data type: {data_type} unsupported. Please use `pyspark.sql.DataFrame`, `pyspark.pandas.DataFrame`, or `pandas.DataFrame`."
+            )
+        storage_options = kwargs.get("auth", {})
+        with fsspec.open(
+            input_schema_path,
+            mode="w",
+            **(storage_options),
+        ) as f:
+            f.write(schema.json())
+
+        if isinstance(X_sample, sql.DataFrame):
+            X_sample = X_sample.toPandas()
+
+        return X_sample, y_sample

ads/model/framework/tensorflow_model.py

@@ -0,0 +1,254 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2022, 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+
+from typing import Dict, List, Optional, Tuple, Union
+
+import numpy as np
+import pandas as pd
+from ads.common import logger
+from ads.common.decorator.runtime_dependency import (
+    runtime_dependency,
+    OptionalDependency,
+)
+from ads.model.extractor.tensorflow_extractor import TensorflowExtractor
+from ads.model.generic_model import FrameworkSpecificModel
+from ads.model.model_properties import ModelProperties
+from ads.model.serde.model_serializer import TensorflowModelSerializerType
+from ads.model.common.utils import DEPRECATE_AS_ONNX_WARNING
+from ads.model.serde.common import SERDE
+
+
+class TensorFlowModel(FrameworkSpecificModel):
+    """TensorFlowModel class for estimators from Tensorflow framework.
+
+    Attributes
+    ----------
+    algorithm: str
+        The algorithm of the model.
+    artifact_dir: str
+        Directory for generate artifact.
+    auth: Dict
+        Default authentication is set using the `ads.set_auth` API. To override the
+        default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create
+        an authentication signer to instantiate an IdentityClient object.
+    estimator: Callable
+        A trained tensorflow estimator/model using Tensorflow.
+    framework: str
+        "tensorflow", the framework name of the model.
+    hyperparameter: dict
+        The hyperparameters of the estimator.
+    metadata_custom: ModelCustomMetadata
+        The model custom metadata.
+    metadata_provenance: ModelProvenanceMetadata
+        The model provenance metadata.
+    metadata_taxonomy: ModelTaxonomyMetadata
+        The model taxonomy metadata.
+    model_artifact: ModelArtifact
+        This is built by calling prepare.
+    model_deployment: ModelDeployment
+        A ModelDeployment instance.
+    model_file_name: str
+        Name of the serialized model.
+    model_id: str
+        The model ID.
+    properties: ModelProperties
+        ModelProperties object required to save and deploy model.
+        For more details, check https://accelerated-data-science.readthedocs.io/en/latest/ads.model.html#module-ads.model.model_properties.
+    runtime_info: RuntimeInfo
+        A RuntimeInfo instance.
+    schema_input: Schema
+        Schema describes the structure of the input data.
+    schema_output: Schema
+        Schema describes the structure of the output data.
+    serialize: bool
+        Whether to serialize the model to pkl file by default. If False, you need to serialize the model manually,
+        save it under artifact_dir and update the score.py manually.
+    version: str
+        The framework version of the model.
+
+    Methods
+    -------
+    delete_deployment(...)
+        Deletes the current model deployment.
+    deploy(..., **kwargs)
+        Deploys a model.
+    from_model_artifact(uri, model_file_name, artifact_dir, ..., **kwargs)
+        Loads model from the specified folder, or zip/tar archive.
+    from_model_catalog(model_id, model_file_name, artifact_dir, ..., **kwargs)
+        Loads model from model catalog.
+    introspect(...)
+        Runs model introspection.
+    predict(data, ...)
+        Returns prediction of input data run against the model deployment endpoint.
+    prepare(..., **kwargs)
+        Prepare and save the score.py, serialized model and runtime.yaml file.
+    reload(...)
+        Reloads the model artifact files: `score.py` and the `runtime.yaml`.
+    save(..., **kwargs)
+        Saves model artifacts to the model catalog.
+    summary_status(...)
+        Gets a summary table of the current status.
+    verify(data, ...)
+        Tests if deployment works in local environment.
+
+    Examples
+    --------
+    >>> from ads.model.framework.tensorflow_model import TensorFlowModel
+    >>> import tempfile
+    >>> import tensorflow as tf
+
+    >>> mnist = tf.keras.datasets.mnist
+    >>> (x_train, y_train), (x_test, y_test) = mnist.load_data()
+    >>> x_train, x_test = x_train / 255.0, x_test / 255.0
+
+    >>> tf_estimator = tf.keras.models.Sequential(
+    ...                [
+    ...                    tf.keras.layers.Flatten(input_shape=(28, 28)),
+    ...                    tf.keras.layers.Dense(128, activation="relu"),
+    ...                    tf.keras.layers.Dropout(0.2),
+    ...                    tf.keras.layers.Dense(10),
+    ...                ]
+    ...            )
+    >>> loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
+    >>> tf_estimator.compile(optimizer="adam", loss=loss_fn, metrics=["accuracy"])
+    >>> tf_estimator.fit(x_train, y_train, epochs=1)
+
+    >>> tf_model = TensorFlowModel(estimator=tf_estimator,
+    ... artifact_dir=tempfile.mkdtemp())
+    >>> inference_conda_env = "generalml_p37_cpu_v1"
+
+    >>> tf_model.prepare(inference_conda_env="generalml_p37_cpu_v1", force_overwrite=True)
+    >>> tf_model.verify(x_test[:1])
+    >>> tf_model.save()
+    >>> model_deployment = tf_model.deploy(wait_for_completion=False)
+    >>> tf_model.predict(x_test[:1])
+    """
+
+    _PREFIX = "tensorflow"
+    model_save_serializer_type = TensorflowModelSerializerType
+
+    @runtime_dependency(
+        module="tensorflow",
+        short_name="tf",
+        install_from=OptionalDependency.TENSORFLOW,
+    )
+    def __init__(
+        self,
+        estimator: callable,
+        artifact_dir: Optional[str] = None,
+        properties: Optional[ModelProperties] = None,
+        auth: Dict = None,
+        model_save_serializer: Optional[SERDE] = model_save_serializer_type.TENSORFLOW,
+        model_input_serializer: Optional[SERDE] = None,
+        **kwargs,
+    ):
+        """
+        Initiates a TensorFlowModel instance.
+
+        Parameters
+        ----------
+        estimator: callable
+            Any model object generated by tensorflow framework
+        artifact_dir: str
+            Directory for generate artifact.
+        properties: (ModelProperties, optional). Defaults to None.
+            ModelProperties object required to save and deploy model.
+        auth :(Dict, optional). Defaults to None.
+            The default authetication is set using `ads.set_auth` API. If you need to override the
+            default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
+            authentication signer and kwargs required to instantiate IdentityClient object.
+        model_save_serializer: (SERDE or str, optional). Defaults to None.
+            Instance of ads.model.SERDE. Used for serialize/deserialize model.
+        model_input_serializer: (SERDE, optional). Defaults to None.
+            Instance of ads.model.SERDE. Used for serialize/deserialize data.
+
+        Returns
+        -------
+        TensorFlowModel
+            TensorFlowModel instance.
+        """
+        super().__init__(
+            estimator=estimator,
+            artifact_dir=artifact_dir,
+            properties=properties,
+            auth=auth,
+            model_save_serializer=model_save_serializer,
+            model_input_serializer=model_input_serializer,
+            **kwargs,
+        )
+        self._extractor = TensorflowExtractor(estimator)
+        self.framework = self._extractor.framework
+        self.algorithm = self._extractor.algorithm
+        self.version = self._extractor.version
+        self.hyperparameter = self._extractor.hyperparameter
+
+        self.version = tf.version.VERSION
+
+    def serialize_model(
+        self,
+        as_onnx: bool = False,
+        X_sample: Optional[
+            Union[
+                Dict,
+                str,
+                List,
+                Tuple,
+                np.ndarray,
+                pd.core.series.Series,
+                pd.core.frame.DataFrame,
+            ]
+        ] = None,
+        force_overwrite: bool = False,
+        **kwargs,
+    ) -> None:
+        """
+        Serialize and save Tensorflow model using ONNX or model specific method.
+
+        Parameters
+        ----------
+        as_onnx: (bool, optional). Defaults to False.
+            If set as True, convert into ONNX model.
+        X_sample: Union[list, tuple, pd.Series, np.ndarray, pd.DataFrame]. Defaults to None.
+            A sample of input data that will be used to generate input schema and detect input_signature.
+        force_overwrite: (bool, optional). Defaults to False.
+            If set as True, overwrite serialized model if exists.
+        **kwargs: optional params used to serialize tensorflow model to onnx,
+        including the following:
+            input_signature: a tuple or a list of tf.TensorSpec objects). default to None.
+            Define the shape/dtype of the input so that model(input_signature) is a valid invocation of the model.
+            opset_version: int. Defaults to None. Used for the ONNX model.
+
+        Returns
+        -------
+        None
+            Nothing.
+        """
+
+        if as_onnx:
+            logger.warning(
+                "This approach supports converting tensorflow.keras models to "
+                "onnx format. If the defined model includes other tensorflow "
+                "modules (e.g., tensorflow.function), please use GenericModel instead."
+            )
+            logger.warning(DEPRECATE_AS_ONNX_WARNING)
+            self.set_model_save_serializer(self.model_save_serializer_type.ONNX)
+
+        super().serialize_model(
+            as_onnx=as_onnx,
+            force_overwrite=force_overwrite,
+            X_sample=X_sample,
+            **kwargs,
+        )
+
+    @runtime_dependency(
+        module="tensorflow",
+        short_name="tf",
+        install_from=OptionalDependency.TENSORFLOW,
+    )
+    def _to_tensor(self, data):
+        data = tf.convert_to_tensor(data)
+        return data