oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.9rc1__py3-none-any.whl

ads/model/runtime/__init__.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/

ads/model/runtime/env_info.py

@@ -0,0 +1,306 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2022, 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import logging
+import warnings
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from enum import Enum
+from typing import Dict
+
+from ads.common.object_storage_details import ObjectStorageDetails
+from ads.common.serializer import DataClassSerializable
+from ads.config import CONDA_BUCKET_NAME, CONDA_BUCKET_NS
+from ads.model.runtime.utils import (
+    INFERENCE_ENV_SCHEMA_PATH,
+    TRAINING_ENV_SCHEMA_PATH,
+    SchemaValidator,
+    get_service_packs,
+)
+
+
+DEFAULT_CONDA_BUCKET_NAME = "service-conda-packs"
+
+
+class PACK_TYPE(Enum):
+    """Conda Pack Type"""
+
+    SERVICE_PACK = "data_science"
+    USER_CUSTOM_PACK = "published"
+
+
+class EnvInfo(ABC):
+    """Env Info Base class."""
+
+    @classmethod
+    @abstractmethod
+    def _populate_env_info(
+        cls, env_slug: str, env_type: str, env_path: str, python_version: str
+    ) -> "EnvInfo":
+        """Populate the EnvInfo instance.
+
+        Parameters
+        ----------
+        env_slug: (str)
+            conda pack slug.
+        env_type: (str)
+            conda pack type: data_science or published.
+        env_path: (str)
+            conda pack object storage path.
+        python_version: (str)
+            python version of the conda pack.
+
+        Returns
+        -------
+        EnvInfo
+            An EnvInfo instance.
+        """
+        pass
+
+    @classmethod
+    def from_slug(
+        cls,
+        env_slug: str,
+        namespace: str = CONDA_BUCKET_NS,
+        bucketname: str = CONDA_BUCKET_NAME,
+        auth: dict = None,
+    ) -> "EnvInfo":
+        """Initiate an EnvInfo object from a slug. Only service pack is allowed to use this method.
+
+        Parameters
+        ----------
+        env_slug: str
+            service pack slug.
+        namespace: (str, optional)
+            namespace of region.
+        bucketname: (str, optional)
+            bucketname of service pack.
+        auth: (Dict, optional). Defaults to None.
+            The default authetication is set using `ads.set_auth` API. If you need to override the
+            default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
+            authentication signer and kwargs required to instantiate IdentityClient object.
+
+        Returns
+        -------
+        EnvInfo
+            An EnvInfo instance.
+        """
+        if "oci://" not in env_slug:
+            warnings.warn("slug will be deprecated. Provide conda pack path instead.")
+
+        if not bucketname:
+            warnings.warn(
+                f"`bucketname` is not provided, defaults to `{DEFAULT_CONDA_BUCKET_NAME}`."
+            )
+            bucketname = DEFAULT_CONDA_BUCKET_NAME
+        _, service_pack_slug_mapping = get_service_packs(
+            namespace, bucketname, auth=auth
+        )
+        env_type, env_path, python_version = None, None, None
+        if service_pack_slug_mapping:
+            if env_slug in service_pack_slug_mapping:
+                env_type = PACK_TYPE.SERVICE_PACK.value
+                env_path, python_version = service_pack_slug_mapping[env_slug]
+            else:
+                warnings.warn(
+                    f"The {env_slug} is not a service pack. Use `from_path` method by passing in the object storage path."
+                )
+
+        return cls._populate_env_info(
+            env_slug=env_slug,
+            env_type=env_type,
+            env_path=env_path,
+            python_version=python_version,
+        )
+
+    @classmethod
+    def from_path(cls, env_path: str, auth: dict = None) -> "EnvInfo":
+        """Initiate an object from a conda pack path.
+
+        Parameters
+        ----------
+        env_path: str
+            conda pack path.
+        auth: (Dict, optional). Defaults to None.
+            The default authetication is set using `ads.set_auth` API. If you need to override the
+            default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate
+            authentication signer and kwargs required to instantiate IdentityClient object.
+
+        Returns
+        -------
+        EnvInfo
+            An EnvInfo instance.
+        """
+        bucketname, namespace, _ = ObjectStorageDetails.from_path(env_path).to_tuple()
+        env_type = ""
+        python_version = ""
+        env_slug = ""
+        service_pack_path_mapping = {}
+        service_pack_path_mapping, _ = get_service_packs(
+            namespace, bucketname, auth=auth
+        )
+        if env_path.startswith("oci://") and service_pack_path_mapping:
+            if env_path in service_pack_path_mapping:
+                env_type = PACK_TYPE.SERVICE_PACK.value
+                (
+                    env_slug,
+                    python_version,
+                ) = service_pack_path_mapping[env_path]
+            else:
+                env_type = PACK_TYPE.USER_CUSTOM_PACK.value
+                try:
+                    metadata_json = ObjectStorageDetails.from_path(
+                        env_path
+                    ).fetch_metadata_of_object()
+                    python_version = metadata_json.get("python", None)
+                    env_slug = metadata_json.get("slug", None)
+                    if not python_version:
+                        raise ValueError(
+                            f"The manifest metadata of {env_path} doesn't contains inforamtion for python version."
+                        )
+                except Exception as e:
+                    logging.debug(e)
+                    logging.debug(
+                        "python version and slug are not found from the manifest metadata."
+                    )
+
+        return cls._populate_env_info(
+            env_slug=env_slug,
+            env_type=env_type,
+            env_path=env_path,
+            python_version=python_version,
+        )
+
+    @staticmethod
+    def _validate(obj_dict: Dict, schema_file_path: str) -> bool:
+        """Validate the content in the ditionary format from the yaml file.
+
+        Parameters
+        ----------
+        obj_dict: (Dict)
+            yaml file content to validate.
+
+        Returns
+        -------
+        bool
+            Validation result.
+        """
+        validator = SchemaValidator(schema_file_path=schema_file_path)
+        return validator.validate(document=obj_dict)
+
+
+@dataclass(repr=False)
+class TrainingEnvInfo(EnvInfo, DataClassSerializable):
+    """Training conda environment info."""
+
+    training_env_slug: str = ""
+    training_env_type: str = ""
+    training_env_path: str = ""
+    training_python_version: str = ""
+
+    @classmethod
+    def _populate_env_info(
+        cls, env_slug: str, env_type: str, env_path: str, python_version: str
+    ) -> "TrainingEnvInfo":
+        """Populate the TrainingEnvInfo instance.
+
+        Parameters
+        ----------
+        env_slug: (str)
+            conda pack slug.
+        env_type: (str)
+            conda pack type: data_science or published.
+        env_path: (str)
+            conda pack object storage path.
+        python_version: (str)
+            python version of the conda pack.
+
+        Returns
+        -------
+        TrainingEnvInfo
+            An TrainingEnvInfo instance.
+        """
+        return cls(
+            training_env_slug=env_slug,
+            training_env_type=env_type,
+            training_env_path=env_path,
+            training_python_version=python_version,
+        )
+
+    @classmethod
+    def _validate_dict(cls, obj_dict: Dict) -> bool:
+        """Validate the content in the dictionary format from the yaml file.
+
+        Parameters
+        ----------
+        obj_dict: (Dict)
+            yaml file content to validate.
+
+        Returns
+        -------
+        bool
+            Validation result.
+        """
+        return EnvInfo._validate(
+            obj_dict=obj_dict, schema_file_path=TRAINING_ENV_SCHEMA_PATH
+        )
+
+
+@dataclass(repr=False)
+class InferenceEnvInfo(EnvInfo, DataClassSerializable):
+    """Inference conda environment info."""
+
+    inference_env_slug: str = ""
+    inference_env_type: str = ""
+    inference_env_path: str = ""
+    inference_python_version: str = ""
+
+    @classmethod
+    def _populate_env_info(
+        cls, env_slug: str, env_type: str, env_path: str, python_version: str
+    ) -> "InferenceEnvInfo":
+        """Populate the InferenceEnvInfo instance.
+
+        Parameters
+        ----------
+        env_slug: (str)
+            conda pack slug.
+        env_type: (str)
+            conda pack type: data_science or published.
+        env_path: (str)
+            conda pack object storage path.
+        python_version: (str)
+            python version of the conda pack.
+
+        Returns
+        -------
+        InferenceEnvInfo
+            An InferenceEnvInfo instance.
+        """
+        return cls(
+            inference_env_slug=env_slug,
+            inference_env_type=env_type,
+            inference_env_path=env_path,
+            inference_python_version=python_version,
+        )
+
+    @classmethod
+    def _validate_dict(cls, obj_dict: Dict) -> bool:
+        """Validate the content in the dictionary format from the yaml file.
+
+        Parameters
+        ----------
+        obj_dict: (Dict)
+            yaml file content to validate.
+
+        Returns
+        -------
+        bool
+            Validation result.
+        """
+        return EnvInfo._validate(
+            obj_dict=obj_dict, schema_file_path=INFERENCE_ENV_SCHEMA_PATH
+        )

ads/model/runtime/model_deployment_details.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from dataclasses import dataclass, field
+from typing import Dict
+
+from ads.common.serializer import DataClassSerializable
+from ads.model.runtime.env_info import InferenceEnvInfo
+
+
+@dataclass(repr=False)
+class ModelDeploymentDetails(DataClassSerializable):
+    """ModelDeploymentDetails class."""
+
+    inference_conda_env: InferenceEnvInfo = field(default_factory=InferenceEnvInfo)
+
+    @classmethod
+    def _validate_dict(cls, obj_dict: Dict) -> bool:
+        """Validate the content in the dictionary format from the yaml file.
+
+        Parameters
+        ----------
+        obj_dict: (Dict)
+            yaml file content to validate.
+
+        Returns
+        -------
+        bool
+            Validation result.
+        """
+        assert obj_dict and (
+            "INFERENCE_CONDA_ENV" in obj_dict
+        ), "`model_deployment_details` must have `INFERENCE_CONDA_ENV` field."
+        return True

ads/model/runtime/model_provenance_details.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2022 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from dataclasses import dataclass, field
+from typing import Dict
+
+from ads.common.serializer import DataClassSerializable
+from ads.model.runtime.env_info import TrainingEnvInfo
+from ads.model.runtime.utils import MODEL_PROVENANCE_SCHEMA_PATH, SchemaValidator
+
+
+@dataclass(repr=False)
+class TrainingCode(DataClassSerializable):
+    """TrainingCode class."""
+
+    artifact_directory: str = ""
+
+    @classmethod
+    def _validate_dict(cls, obj_dict: Dict) -> bool:
+        assert obj_dict and (
+            "ARTIFACT_DIRECTORY" in obj_dict
+        ), "`training_code` must have `ARTIFACT_DIRECTORY` field."
+        return True
+
+
+@dataclass(repr=False)
+class ModelProvenanceDetails(DataClassSerializable):
+    """ModelProvenanceDetails class."""
+
+    project_ocid: str = ""
+    tenancy_ocid: str = ""
+    training_code: TrainingCode = field(default_factory=TrainingCode)
+    training_compartment_ocid: str = ""
+    training_conda_env: TrainingEnvInfo = field(default_factory=TrainingEnvInfo)
+    training_region: str = ""
+    training_resource_ocid: str = ""
+    user_ocid: str = ""
+    vm_image_internal_id: str = ""
+
+    @classmethod
+    def _validate_dict(cls, obj_dict: Dict) -> bool:
+        """validate the yaml file.
+
+        Parameters
+        ----------
+        obj_dict: (Dict)
+            yaml file content to validate.
+
+        Returns
+        -------
+        bool
+            Validation result.
+        """
+        validator = SchemaValidator(schema_file_path=MODEL_PROVENANCE_SCHEMA_PATH)
+        return validator.validate(obj_dict)

ads/model/runtime/runtime_info.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2022, 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import os
+from dataclasses import dataclass, field
+from typing import Dict
+
+from ads.common.serializer import DataClassSerializable, SideEffect
+from ads.model.runtime.model_deployment_details import ModelDeploymentDetails
+from ads.model.runtime.model_provenance_details import ModelProvenanceDetails
+
+
+@dataclass(repr=False)
+class RuntimeInfo(DataClassSerializable):
+    """RuntimeInfo class which is the data class represenation of the runtime yaml file."""
+
+    model_artifact_version: str = ""
+    model_deployment: ModelDeploymentDetails = field(
+        default_factory=ModelDeploymentDetails
+    )
+    model_provenance: ModelProvenanceDetails = field(
+        default_factory=ModelProvenanceDetails
+    )
+
+    @classmethod
+    def _validate_dict(cls, obj_dict: Dict) -> bool:
+        """Validate the runtime info.
+
+        Parameters
+        ----------
+        obj_dict: (Dict)
+            runtime content in dictionary format.
+
+        Returns
+        -------
+        bool
+            the validation result.
+        """
+        assert (
+            "MODEL_ARTIFACT_VERSION" in obj_dict
+        ), "runtime.yaml must have `MODEL_ARTIFACT_VERSION` field."
+        assert (
+            "MODEL_DEPLOYMENT" in obj_dict
+        ), "runtime.yaml must have `MODEL_DEPLOYMENT` field."
+        assert (
+            "MODEL_PROVENANCE" in obj_dict
+        ), "runtime.yaml must have `MODEL_PROVENANCE` field."
+        return True
+
+    @classmethod
+    def from_env(cls) -> "RuntimeInfo":
+        """Popolate the RuntimeInfo from environment variables.
+
+        Returns
+        -------
+        RuntimeInfo
+            A RuntimeInfo instance.
+        """
+        runtime_info = cls(model_artifact_version="3.0")
+        return runtime_info
+
+    def save(self, storage_options=None):
+        """Save the RuntimeInfo object into runtime.yaml file under the artifact directory.
+
+        Returns
+        -------
+        None
+            Nothing.
+        """
+        runtime_file_path = os.path.join(
+            self.model_provenance.training_code.artifact_directory, "runtime.yaml"
+        )
+        storage_options = storage_options or {}
+        self.to_yaml(
+            uri=runtime_file_path,
+            side_effect=SideEffect.CONVERT_KEYS_TO_UPPER.value,
+            storage_options=storage_options,
+        )

ads/model/runtime/schemas/inference_env_info_schema.yaml

@@ -0,0 +1,16 @@
+INFERENCE_ENV_PATH:
+  type: string
+  required: true
+  nullable: false
+INFERENCE_ENV_SLUG:
+  type: string
+  required: false
+  nullable: true
+INFERENCE_ENV_TYPE:
+  type: string
+  required: false
+  nullable: true
+INFERENCE_PYTHON_VERSION:
+  type: string
+  required: true
+  nullable: false

ads/model/runtime/schemas/model_provenance_schema.yaml

@@ -0,0 +1,36 @@
+PROJECT_OCID:
+  type: string
+  required: true
+  nullable: false
+TENANCY_OCID:
+  type: string
+  required: true
+  nullable: true
+TRAINING_CODE:
+  type: dict
+  required: true
+  nullable: true
+TRAINING_COMPARTMENT_OCID:
+  type: string
+  required: true
+  nullable: false
+TRAINING_CONDA_ENV:
+  type: dict
+  required: true
+  nullable: true
+TRAINING_REGION:
+  type: string
+  required: true
+  nullable: true
+TRAINING_RESOURCE_OCID:
+  type: string
+  required: true
+  nullable: true
+USER_OCID:
+  type: string
+  required: true
+  nullable: true
+VM_IMAGE_INTERNAL_ID:
+  type: string
+  required: true
+  nullable: true

ads/model/runtime/schemas/training_env_info_schema.yaml

@@ -0,0 +1,16 @@
+TRAINING_ENV_PATH:
+  type: string
+  required: true
+  nullable: true
+TRAINING_ENV_SLUG:
+  type: string
+  required: true
+  nullable: true
+TRAINING_ENV_TYPE:
+  type: string
+  required: true
+  nullable: true
+TRAINING_PYTHON_VERSION:
+  type: string
+  required: true
+  nullable: true