oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.9rc1__py3-none-any.whl

ads/model/extractor/__init__.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/

ads/model/extractor/automl_extractor.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2021, 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from ads.common.decorator.deprecate import deprecated
+from ads.model.extractor.model_info_extractor import ModelInfoExtractor
+from ads.model.model_metadata import Framework
+
+
+class AutoMLExtractor(ModelInfoExtractor):
+    """Class that extract model metadata from automl models.
+
+    Attributes
+    ----------
+    model: object
+        The model to extract metadata from.
+    estimator: object
+        The estimator to extract metadata from.
+    """
+
+    @deprecated(
+        details="Working with AutoML has moved from within ADS to working directly with the AutoMLx library. AutoMLx are preinstalled in conda pack automlx_p38_cpu_v2 and later, and can now be updated independently of ADS. AutoMLx documentation may be found at https://docs.oracle.com/en-us/iaas/tools/automlx/latest/html/multiversion/v23.1.1/index.html. Notebook examples are in Oracle's samples repository: https://github.com/oracle-samples/oci-data-science-ai-samples/tree/master/notebook_examples and a migration tutorial can be found at https://accelerated-data-science.readthedocs.io/en/latest/user_guide/model_training/automl/quick_start.html .",
+        raise_error=True,
+    )
+    def __init__(self, model):
+        self.model = model
+
+    @property
+    def framework(self):
+        """Extracts the framework of the model.
+
+        Returns
+        ----------
+        str:
+           The framework of the model.
+        """
+        return Framework.ORACLE_AUTOML
+
+    @property
+    def algorithm(self):
+        """Extracts the algorithm of the model.
+
+        Returns
+        ----------
+        object:
+           The algorithm of the model.
+        """
+        return "ensemble"
+
+    @property
+    def version(self):
+        """Extracts the framework version of the model.
+
+        Returns
+        ----------
+        str:
+           The framework version of the model.
+        """
+        import automl
+
+        return automl.__version__
+
+    @property
+    def hyperparameter(self):
+        """Extracts the hyperparameters of the model.
+
+        Returns
+        ----------
+        dict:
+           The hyperparameters of the model.
+        """
+        return None

ads/model/extractor/embedding_onnx_extractor.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2025 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from ads.common.decorator.runtime_dependency import (
+    OptionalDependency,
+    runtime_dependency,
+)
+from ads.model.extractor.model_info_extractor import ModelInfoExtractor
+from ads.model.model_metadata import Framework
+
+
+class EmbeddingONNXExtractor(ModelInfoExtractor):
+    """Class that extract model metadata from EmbeddingONNXModel models.
+
+    Attributes
+    ----------
+    model: object
+        The model to extract metadata from.
+
+    Methods
+    -------
+    framework(self) -> str
+        Returns the framework of the model.
+    algorithm(self) -> object
+        Returns the algorithm of the model.
+    version(self) -> str
+        Returns the version of framework of the model.
+    hyperparameter(self) -> dict
+        Returns the hyperparameter of the model.
+    """
+
+    def __init__(self, model=None):
+        self.model = model
+
+    @property
+    def framework(self):
+        """Extracts the framework of the model.
+
+        Returns
+        ----------
+        str:
+           The framework of the model.
+        """
+        return Framework.EMBEDDING_ONNX
+
+    @property
+    def algorithm(self):
+        """Extracts the algorithm of the model.
+
+        Returns
+        ----------
+        object:
+           The algorithm of the model.
+        """
+        return "Embedding_ONNX"
+
+    @property
+    @runtime_dependency(module="onnxruntime", install_from=OptionalDependency.ONNX)
+    def version(self):
+        """Extracts the framework version of the model.
+
+        Returns
+        ----------
+        str:
+           The framework version of the model.
+        """
+        return onnxruntime.__version__
+
+    @property
+    def hyperparameter(self):
+        """Extracts the hyperparameters of the model.
+
+        Returns
+        ----------
+        dict:
+           The hyperparameters of the model.
+        """
+        return None

ads/model/extractor/huggingface_extractor.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from ads.model.extractor.model_info_extractor import (
+    ModelInfoExtractor,
+    normalize_hyperparameter,
+)
+from ads.model.model_metadata import Framework
+from ads.common.decorator.runtime_dependency import (
+    runtime_dependency,
+    OptionalDependency,
+)
+
+
+class HuggingFaceExtractor(ModelInfoExtractor):
+    """Class that extract model metadata from HuggingFacePipeline models.
+
+    Attributes
+    ----------
+    model: object
+        The model to extract metadata from.
+    estimator: object
+        The estimator to extract metadata from.
+
+    Methods
+    -------
+    framework(self) -> str
+        Returns the framework of the model.
+    algorithm(self) -> object
+        Returns the algorithm of the model.
+    version(self) -> str
+        Returns the version of framework of the model.
+    hyperparameter(self) -> dict
+        Returns the hyperparameter of the model.
+    """
+
+    def __init__(self, model):
+        self.model = model
+
+    @property
+    def framework(self):
+        """Extracts the framework of the model.
+
+        Returns
+        ----------
+        str:
+           The framework of the model.
+        """
+        return Framework.TRANSFORMERS
+
+    @property
+    def algorithm(self):
+        """Extracts the algorithm of the model.
+
+        Returns
+        ----------
+        object:
+           The algorithm of the model.
+        """
+        return self.model.__class__.__name__
+
+    @property
+    @runtime_dependency(
+        module="transformers", install_from=OptionalDependency.HUGGINGFACE
+    )
+    def version(self):
+        """Extracts the framework version of the model.
+
+        Returns
+        ----------
+        str:
+           The framework version of the model.
+        """
+        return transformers.__version__
+
+    @property
+    def hyperparameter(self):
+        """Extracts the hyperparameters of the model.
+
+        Returns
+        ----------
+        dict:
+           The hyperparameters of the model.
+        """
+        return normalize_hyperparameter(self.model.model.config.to_dict())

ads/model/extractor/keras_extractor.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import logging
+from ads.model.extractor.model_info_extractor import ModelInfoExtractor
+from ads.model.model_metadata import Framework
+from ads.common.decorator.runtime_dependency import (
+    runtime_dependency,
+    OptionalDependency,
+)
+
+
+class KerasExtractor(ModelInfoExtractor):
+    """Class that extract model metadata from keras models.
+
+    Attributes
+    ----------
+    model: object
+        The model to extract metadata from.
+    estimator: object
+        The estimator to extract metadata from.
+    """
+
+    def __init__(self, model):
+        self.model = model
+
+    @property
+    def framework(self):
+        """Extracts the framework of the model.
+
+        Returns
+        ----------
+        str:
+           The framework of the model.
+        """
+        return Framework.KERAS
+
+    @property
+    def algorithm(self):
+        """Extracts the algorithm of the model.
+
+        Returns
+        ----------
+        object:
+           The algorithm of the model.
+        """
+        return self.model.__class__.__name__
+
+    @property
+    @runtime_dependency(
+        module="tensorflow",
+        install_from=OptionalDependency.TENSORFLOW,
+    )
+    def version(self):
+        """Extracts the framework version of the model.
+
+        Returns
+        ----------
+        str:
+           The framework version of the model.
+        """
+        from tensorflow import keras
+
+        return keras.__version__
+
+    @property
+    def hyperparameter(self):
+        """Extracts the hyperparameters of the model.
+
+        Returns
+        ----------
+        dict:
+           The hyperparameters of the model.
+        """
+        if hasattr(self.model, "get_config"):
+            return self.model.get_config()
+        else:
+            logging.warning(
+                "Cannot extract the hyperparameters from this model automatically."
+            )
+            return {}

ads/model/extractor/lightgbm_extractor.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import logging
+from ads.model.model_metadata import Framework
+from ads.model.extractor.model_info_extractor import ModelInfoExtractor
+from ads.common.decorator.runtime_dependency import (
+    runtime_dependency,
+    OptionalDependency,
+)
+
+
+class LightgbmExtractor(ModelInfoExtractor):
+    """Class that extract model metadata from lightgbm models.
+
+    Attributes
+    ----------
+    model: object
+        The model to extract metadata from.
+    estimator: object
+        The estimator to extract metadata from.
+
+    Methods
+    -------
+    framework(self) -> str
+        Returns the framework of the model.
+    algorithm(self) -> object
+        Returns the algorithm of the model.
+    version(self) -> str
+        Returns the version of framework of the model.
+    hyperparameter(self) -> dict
+        Returns the hyperparameter of the model.
+    """
+
+    def __init__(self, model):
+        self.model = model
+
+    @property
+    def framework(self):
+        """Extracts the framework of the model.
+
+        Returns
+        ----------
+        str:
+           The framework of the model.
+        """
+        return Framework.LIGHT_GBM
+
+    @property
+    def algorithm(self):
+        """Extracts the algorithm of the model.
+
+        Returns
+        ----------
+        object:
+           The algorithm of the model.
+        """
+        return self.model.__class__.__name__
+
+    @property
+    @runtime_dependency(module="lightgbm", install_from=OptionalDependency.BOOSTED)
+    def version(self):
+        """Extracts the framework version of the model.
+
+        Returns
+        ----------
+        str:
+           The framework version of the model.
+        """
+
+        return lightgbm.__version__
+
+    @property
+    def hyperparameter(self):
+        """Extracts the hyperparameters of the model.
+
+        Returns
+        ----------
+        dict:
+           The hyperparameters of the model.
+        """
+        if hasattr(self.model, "params"):
+            return self.model.params
+        elif hasattr(self.model, "get_params"):
+            return self.model.get_params()
+        else:
+            logging.warning(
+                "Cannot extract the hyperparameters from this model automatically."
+            )
+            return {}

ads/model/extractor/model_info_extractor.py

@@ -0,0 +1,114 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import abc
+from ads.model.model_metadata import MetadataTaxonomyKeys
+from typing import Dict
+
+
+def normalize_hyperparameter(data: Dict) -> dict:
+    """Converts all the fields to string to make sure it's
+    json serializable.
+
+    Parameters
+    ----------
+    data: ([Dict])
+        The hyperparameter returned by the model.
+
+    Returns
+    -------
+    Dict
+        Normalized (json serializable) dictionary.
+    """
+    if not data:
+        return None
+    new_dict = {}
+    for key, value in data.items():
+        if isinstance(value, dict):
+            new_dict[key] = normalize_hyperparameter(value)
+        else:
+            try:
+                new_dict[key] = str(value)
+            except:
+                new_dict[key] = ""
+    return new_dict
+
+
+class ModelInfoExtractor(abc.ABC):
+    """The base abstract class to extract model metadata.
+
+    Methods
+    -------
+    framework(self) -> str
+        Returns the framework of the model.
+    algorithm(self) -> object
+        Returns the algorithm of the model.
+    version(self) -> str
+        Returns the version of framework of the model.
+    hyperparameter(self) -> dict
+        Returns the hyperparameter of the model.
+    info(self) -> dict
+        Returns the model taxonomy metadata information.
+    """
+
+    @abc.abstractmethod
+    def framework(self):
+        """The abstract method to extracts the framework of the model.
+
+        Returns
+        ----------
+        str:
+           The framework of the model.
+        """
+        pass
+
+    @abc.abstractmethod
+    def algorithm(self):
+        """The abstract method to extracts the algorithm of the model.
+
+        Returns
+        ----------
+        object:
+           The algorithm of the model.
+        """
+        pass
+
+    @abc.abstractmethod
+    def version(self):
+        """The abstract method to extracts the framework version of the model.
+
+        Returns
+        ----------
+        str:
+           The framework version of the model.
+        """
+        pass
+
+    @abc.abstractmethod
+    def hyperparameter(self):
+        """The abstract method to extracts the hyperparameters of the model.
+
+        Returns
+        ----------
+        dict:
+           The hyperparameter of the model.
+        """
+        pass
+
+    def info(self):
+        """Extracts the taxonomy metadata of the model.
+
+        Returns
+        ----------
+        dict:
+           The taxonomy metadata of the model.
+        """
+        return {
+            MetadataTaxonomyKeys.FRAMEWORK: self.framework,
+            MetadataTaxonomyKeys.FRAMEWORK_VERSION: self.version,
+            MetadataTaxonomyKeys.ALGORITHM: str(self.algorithm),
+            MetadataTaxonomyKeys.HYPERPARAMETERS: self.hyperparameter,
+        }

ads/model/extractor/model_info_extractor_factory.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2021, 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from ads.model.model_metadata import Framework
+from ads.common import logger, utils
+from ads.model.extractor.sklearn_extractor import SklearnExtractor
+from ads.model.extractor.xgboost_extractor import XgboostExtractor
+from ads.model.extractor.lightgbm_extractor import LightgbmExtractor
+from ads.model.extractor.keras_extractor import KerasExtractor
+from ads.model.extractor.automl_extractor import AutoMLExtractor
+from ads.model.extractor.spark_extractor import SparkExtractor
+from ads.model.extractor.pytorch_extractor import PytorchExtractor
+from ads.model.extractor.tensorflow_extractor import TensorflowExtractor
+from ads.model.extractor.huggingface_extractor import HuggingFaceExtractor
+
+
+ORDERED_FRAMEWORKS = [
+    "lightgbm",
+    "xgboost",
+    "sklearn",
+    "keras",
+    "tensorflow",
+    "bert",
+    "transformers",
+    "torch",
+    "spark",
+    "automl",
+]
+
+
+class ModelInfoExtractorFactory:
+    """Class that extract Model Taxonomy Metadata for all supported frameworks."""
+
+    _estimator_map = {
+        Framework.SCIKIT_LEARN: SklearnExtractor,
+        Framework.XGBOOST: XgboostExtractor,
+        Framework.LIGHT_GBM: LightgbmExtractor,
+        Framework.KERAS: KerasExtractor,
+        Framework.ORACLE_AUTOML: AutoMLExtractor,
+        Framework.TENSORFLOW: TensorflowExtractor,
+        Framework.PYTORCH: PytorchExtractor,
+        Framework.SPARK: SparkExtractor,
+        Framework.TRANSFORMERS: HuggingFaceExtractor,
+    }
+
+    @staticmethod
+    def extract_info(model):
+        """Extracts model taxonomy metadata.
+
+
+        Parameters
+        ----------
+        model: [ADS model, sklearn, xgboost, lightgbm, keras, oracle_automl]
+            The model object
+
+        Returns
+        -------
+        `ModelTaxonomyMetadata`
+            A dictionary with keys of Framework, FrameworkVersion, Algorithm, Hyperparameters of the model
+
+        Examples
+        --------
+        >>> from ads.common.model_info_extractor_factory import ModelInfoExtractorFactory
+        >>> metadata_taxonomy = ModelInfoExtractorFactory.extract_info(model)
+
+        """
+        from ads.common.model import ADSModel
+
+        if isinstance(model, ADSModel):
+            model = model.est
+        model_framework = None
+        model_bases = utils.get_base_modules(model)
+        model_framework = ModelInfoExtractorFactory._get_estimator(
+            model_bases=model_bases
+        )
+        if model_framework not in ModelInfoExtractorFactory._estimator_map:
+            logger.warn(
+                f"Auto-extraction of taxonomy is not supported for the provided model. "
+                f"The supported models are {', '.join(ORDERED_FRAMEWORKS)}."
+            )
+            return None
+        return ModelInfoExtractorFactory._estimator_map[model_framework](model).info()
+
+    @staticmethod
+    def _get_estimator(model_bases):
+        mapping = {
+            "lightgbm": Framework.LIGHT_GBM,
+            "xgboost": Framework.XGBOOST,
+            "sklearn": Framework.SCIKIT_LEARN,
+            "keras": Framework.KERAS,
+            "tensorflow": Framework.TENSORFLOW,
+            "bert": Framework.BERT,
+            "transformers": Framework.TRANSFORMERS,
+            "torch": Framework.PYTORCH,
+            "spark": Framework.SPARK,
+            "automl": Framework.ORACLE_AUTOML,
+        }
+        for model_base in model_bases:
+            for framework in ORDERED_FRAMEWORKS:
+                if framework in model_base.__module__.split("."):
+                    return mapping.get(framework)
+        return None