oracle-ads 2.13.0__py3-none-any.whl → 2.13.1__py3-none-any.whl

ads/model/datascience_model.py

@@ -3,7 +3,6 @@
 # Copyright (c) 2022, 2025 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
-import cgi
 import json
 import logging
 import os
@@ -20,12 +19,14 @@ from ads.common import oci_client as oc
 from ads.common import utils
 from ads.common.extended_enum import ExtendedEnum
 from ads.common.object_storage_details import ObjectStorageDetails
+from ads.common.utils import is_path_exists
 from ads.config import (
     AQUA_SERVICE_MODELS_BUCKET as SERVICE_MODELS_BUCKET,
 )
 from ads.config import (
     COMPARTMENT_OCID,
     PROJECT_OCID,
+    USER,
 )
 from ads.feature_engineering.schema import Schema
 from ads.jobs.builders.base import Builder
@@ -34,6 +35,7 @@ from ads.model.artifact_downloader import (
     SmallArtifactDownloader,
 )
 from ads.model.artifact_uploader import LargeArtifactUploader, SmallArtifactUploader
+from ads.model.common.utils import MetadataArtifactPathType
 from ads.model.model_metadata import (
     MetadataCustomCategory,
     ModelCustomMetadata,
@@ -42,6 +44,7 @@ from ads.model.model_metadata import (
     ModelTaxonomyMetadata,
 )
 from ads.model.service.oci_datascience_model import (
+    ModelMetadataArtifactDetails,
     ModelProvenanceNotFoundError,
     OCIDataScienceModel,
 )
@@ -72,6 +75,11 @@ class BucketNotVersionedError(Exception):  # pragma: no cover
         super().__init__(msg)
 
 
+class PathNotFoundError(Exception):
+    def __init__(self, msg="The given path doesn't exist."):
+        super().__init__(msg)
+
+
 class ModelFileDescriptionError(Exception):  # pragma: no cover
     def __init__(self, msg="Model File Description file is not set up."):
         super().__init__(msg)
@@ -1591,7 +1599,11 @@ class DataScienceModel(Builder):
 
     @classmethod
     def list(
-        cls, compartment_id: str = None, project_id: str = None, **kwargs
+        cls,
+        compartment_id: str = None,
+        project_id: str = None,
+        category: str = USER,
+        **kwargs,
     ) -> List["DataScienceModel"]:
         """Lists datascience models in a given compartment.
 
@@ -1601,6 +1613,8 @@ class DataScienceModel(Builder):
             The compartment OCID.
         project_id: (str, optional). Defaults to `None`.
             The project OCID.
+        category: (str, optional). Defaults to `USER`.
+            The category of Model. Allowed values are: "USER", "SERVICE"
         kwargs
             Additional keyword arguments for filtering models.
 
@@ -1612,13 +1626,17 @@ class DataScienceModel(Builder):
         return [
             cls()._update_from_oci_dsc_model(model)
             for model in OCIDataScienceModel.list_resource(
-                compartment_id, project_id=project_id, **kwargs
+                compartment_id, project_id=project_id, category=category, **kwargs
             )
         ]
 
     @classmethod
     def list_df(
-        cls, compartment_id: str = None, project_id: str = None, **kwargs
+        cls,
+        compartment_id: str = None,
+        project_id: str = None,
+        category: str = USER,
+        **kwargs,
     ) -> "pandas.DataFrame":
         """Lists datascience models in a given compartment.
 
@@ -1628,6 +1646,8 @@ class DataScienceModel(Builder):
             The compartment OCID.
         project_id: (str, optional). Defaults to `None`.
             The project OCID.
+        category: (str, optional). Defaults to `None`.
+            The category of Model.
         kwargs
             Additional keyword arguments for filtering models.
 
@@ -1638,7 +1658,7 @@ class DataScienceModel(Builder):
         """
         records = []
         for model in OCIDataScienceModel.list_resource(
-            compartment_id, project_id=project_id, **kwargs
+            compartment_id, project_id=project_id, category=category, **kwargs
         ):
             records.append(
                 {
@@ -1776,9 +1796,11 @@ class DataScienceModel(Builder):
         # Update artifact info
         try:
             artifact_info = self.dsc_model.get_artifact_info()
-            _, file_name_info = cgi.parse_header(artifact_info["Content-Disposition"])
+            _, file_name_info = utils.parse_content_disposition(
+                artifact_info["Content-Disposition"]
+            )
 
-            if self.dsc_model._is_model_by_reference():
+            if self.dsc_model.is_model_created_by_reference():
                 _, file_extension = os.path.splitext(file_name_info["filename"])
                 if file_extension.lower() == ".json":
                     bucket_uri, _ = self._download_file_description_artifact()
@@ -1787,7 +1809,6 @@ class DataScienceModel(Builder):
                 self.set_spec(self.CONST_ARTIFACT, file_name_info["filename"])
         except:
             pass
-
         return self
 
     def to_dict(self) -> Dict:
@@ -2213,3 +2234,313 @@ class DataScienceModel(Builder):
         else:
             # model found case
             self.model_file_description["models"].pop(modelSearchIdx)
+
+    def create_custom_metadata_artifact(
+        self,
+        metadata_key_name: str,
+        artifact_path_or_content: str,
+        path_type: MetadataArtifactPathType = MetadataArtifactPathType.LOCAL,
+    ) -> ModelMetadataArtifactDetails:
+        """Creates model custom metadata artifact for specified model.
+
+        Parameters
+        ----------
+        metadata_key_name: str
+            The name of the model custom metadata key
+
+        artifact_path_or_content: str
+            The model custom metadata artifact path to be upload. It can also be the actual content of the custom metadata
+
+        path_type: MetadataArtifactPathType
+            Can be either of MetadataArtifactPathType.LOCAL , MetadataArtifactPathType.OSS , MetadataArtifactPathType.CONTENT
+            Specifies what type of path is to be provided for metadata artifact.
+            Can be either local , oss or the actual content itself
+
+        Returns
+        -------
+        ModelMetadataArtifactDetails
+            The model custom metadata artifact creation info.
+            Example:
+            {
+                'Date': 'Mon, 02 Dec 2024 06:38:24 GMT',
+                'opc-request-id': 'E4F7',
+                'ETag': '77156317-8bb9-4c4a-882b-0d85f8140d93',
+                'X-Content-Type-Options': 'nosniff',
+                'Content-Length': '4029958',
+                'Vary': 'Origin',
+                'Strict-Transport-Security': 'max-age=31536000; includeSubDomains',
+                'status': 204
+            }
+
+        """
+        return self.dsc_model.create_custom_metadata_artifact(
+            metadata_key_name=metadata_key_name,
+            artifact_path=artifact_path_or_content,
+            path_type=path_type,
+        )
+
+    def create_defined_metadata_artifact(
+        self,
+        metadata_key_name: str,
+        artifact_path_or_content: str,
+        path_type: MetadataArtifactPathType = MetadataArtifactPathType.LOCAL,
+    ) -> ModelMetadataArtifactDetails:
+        """Creates model defined metadata artifact for specified model.
+
+        Parameters
+        ----------
+        metadata_key_name: str
+            The name of the model defined metadata key
+
+        artifact_path_or_content: str
+            The model defined metadata artifact path to be upload. It can also be the actual content of the defined metadata
+
+        path_type: MetadataArtifactPathType
+            Can be either of MetadataArtifactPathType.LOCAL , MetadataArtifactPathType.OSS , MetadataArtifactPathType.CONTENT
+            Specifies what type of path is to be provided for metadata artifact.
+            Can be either local , oss or the actual content itself
+
+        Returns
+        -------
+        ModelMetadataArtifactDetails
+            The model defined metadata artifact creation info.
+            Example:
+            {
+                'Date': 'Mon, 02 Dec 2024 06:38:24 GMT',
+                'opc-request-id': 'E4F7',
+                'ETag': '77156317-8bb9-4c4a-882b-0d85f8140d93',
+                'X-Content-Type-Options': 'nosniff',
+                'Content-Length': '4029958',
+                'Vary': 'Origin',
+                'Strict-Transport-Security': 'max-age=31536000; includeSubDomains',
+                'status': 204
+            }
+
+        """
+        return self.dsc_model.create_defined_metadata_artifact(
+            metadata_key_name=metadata_key_name,
+            artifact_path=artifact_path_or_content,
+            path_type=path_type,
+        )
+
+    def update_custom_metadata_artifact(
+        self,
+        metadata_key_name: str,
+        artifact_path_or_content: str,
+        path_type: MetadataArtifactPathType = MetadataArtifactPathType.LOCAL,
+    ) -> ModelMetadataArtifactDetails:
+        """Update model custom metadata artifact for specified model.
+
+        Parameters
+        ----------
+        metadata_key_name: str
+            The name of the model custom metadata key
+
+        artifact_path_or_content: str
+            The model custom metadata artifact path. It can also be the actual content of the custom metadata
+
+        path_type: MetadataArtifactPathType
+            Can be either of MetadataArtifactPathType.LOCAL , MetadataArtifactPathType.OSS , MetadataArtifactPathType.CONTENT
+            Specifies what type of path is to be provided for metadata artifact.
+            Can be either local , oss or the actual content itself
+
+        Returns
+        -------
+        ModelMetadataArtifactDetails
+            The model custom metadata artifact update info.
+            Example:
+            {
+                'Date': 'Mon, 02 Dec 2024 06:38:24 GMT',
+                'opc-request-id': 'E4F7',
+                'ETag': '77156317-8bb9-4c4a-882b-0d85f8140d93',
+                'X-Content-Type-Options': 'nosniff',
+                'Content-Length': '4029958',
+                'Vary': 'Origin',
+                'Strict-Transport-Security': 'max-age=31536000; includeSubDomains',
+                'status': 204
+            }
+
+        """
+        return self.dsc_model.update_custom_metadata_artifact(
+            metadata_key_name=metadata_key_name,
+            artifact_path=artifact_path_or_content,
+            path_type=path_type,
+        )
+
+    def update_defined_metadata_artifact(
+        self,
+        metadata_key_name: str,
+        artifact_path_or_content: str,
+        path_type: MetadataArtifactPathType = MetadataArtifactPathType.LOCAL,
+    ) -> ModelMetadataArtifactDetails:
+        """Update model defined metadata artifact for specified model.
+
+        Parameters
+        ----------
+        metadata_key_name: str
+            The name of the model defined metadata key
+
+        artifact_path_or_content: str
+            The model defined metadata artifact path. It can also be the actual content of the defined metadata
+
+        path_type: MetadataArtifactPathType
+            Can be either of MetadataArtifactPathType.LOCAL , MetadataArtifactPathType.OSS , MetadataArtifactPathType.CONTENT
+            Specifies what type of path is to be provided for metadata artifact.
+            Can be either local , oss or the actual content itself
+
+        Returns
+        -------
+        ModelMetadataArtifactDetails
+            The model defined metadata artifact update info.
+            Example:
+            {
+                'Date': 'Mon, 02 Dec 2024 06:38:24 GMT',
+                'opc-request-id': 'E4F7',
+                'ETag': '77156317-8bb9-4c4a-882b-0d85f8140d93',
+                'X-Content-Type-Options': 'nosniff',
+                'Content-Length': '4029958',
+                'Vary': 'Origin',
+                'Strict-Transport-Security': 'max-age=31536000; includeSubDomains',
+                'status': 204
+            }
+
+        """
+        return self.dsc_model.update_defined_metadata_artifact(
+            metadata_key_name=metadata_key_name,
+            artifact_path=artifact_path_or_content,
+            path_type=path_type,
+        )
+
+    def get_custom_metadata_artifact(
+        self, metadata_key_name: str, target_dir: str, override: bool = False
+    ) -> bytes:
+        """Downloads model custom metadata artifact content for specified model metadata key.
+
+        Parameters
+        ----------
+        metadata_key_name: str
+            The name of the custom metadata key of the model
+
+        target_dir: str
+            The local file path where downloaded model custom metadata artifact will be saved.
+
+        override: bool
+            A boolean flag that controls downloaded metadata artifact file overwriting
+            - If True, overwrites the file if it already exists.
+            - If False (default), raises a `FileExistsError` if the file exists.
+        Returns
+        -------
+        bytes
+            File content of the custom metadata artifact
+
+        """
+        if not is_path_exists(target_dir):
+            raise PathNotFoundError(f"Path : {target_dir} does not exist")
+
+        file_content = self.dsc_model.get_custom_metadata_artifact(
+            metadata_key_name=metadata_key_name
+        )
+        artifact_file_path = os.path.join(target_dir, f"{metadata_key_name}")
+
+        if not override and os.path.exists(artifact_file_path):
+            raise FileExistsError(f"File already exists: {artifact_file_path}")
+
+        with open(artifact_file_path, "wb") as _file:
+            _file.write(file_content)
+            logger.debug(f"Artifact downloaded to location - {artifact_file_path}")
+        return file_content
+
+    def get_defined_metadata_artifact(
+        self, metadata_key_name: str, target_dir: str, override: bool = False
+    ) -> bytes:
+        """Downloads model defined metadata artifact content for specified model metadata key.
+
+        Parameters
+        ----------
+        metadata_key_name: str
+            The name of the model metadatum in the metadata.
+
+        target_dir: str
+            The local file path where downloaded model defined metadata artifact will be saved.
+
+        override: bool
+            A boolean flag that controls downloaded metadata artifact file overwriting
+            - If True, overwrites the file if it already exists.
+            - If False (default), raises a `FileExistsError` if the file exists.
+        Returns
+        -------
+        bytes
+            File content of the custom metadata artifact
+
+        """
+        if not is_path_exists(target_dir):
+            raise PathNotFoundError(f"Path : {target_dir} does not exist")
+
+        file_content = self.dsc_model.get_defined_metadata_artifact(
+            metadata_key_name=metadata_key_name
+        )
+        artifact_file_path = os.path.join(target_dir, f"{metadata_key_name}")
+
+        if not override and os.path.exists(artifact_file_path):
+            raise FileExistsError(f"File already exists: {artifact_file_path}")
+
+        with open(artifact_file_path, "wb") as _file:
+            _file.write(file_content)
+            logger.debug(f"Artifact downloaded to location - {artifact_file_path}")
+        return file_content
+
+    def delete_custom_metadata_artifact(
+        self, metadata_key_name: str
+    ) -> ModelMetadataArtifactDetails:
+        """Deletes model custom metadata artifact for specified model metadata key.
+
+        Parameters
+        ----------
+        metadata_key_name: str
+            The name of the model metadatum in the metadata.
+        Returns
+        -------
+        ModelMetadataArtifactDetails
+            The model custom metadata artifact delete call info.
+            Example:
+            {
+                'Date': 'Mon, 02 Dec 2024 06:38:24 GMT',
+                'opc-request-id': 'E4F7',
+                'X-Content-Type-Options': 'nosniff',
+                'Vary': 'Origin',
+                'Strict-Transport-Security': 'max-age=31536000; includeSubDomains',
+                'status': 204
+            }
+
+        """
+        return self.dsc_model.delete_custom_metadata_artifact(
+            metadata_key_name=metadata_key_name
+        )
+
+    def delete_defined_metadata_artifact(
+        self, metadata_key_name: str
+    ) -> ModelMetadataArtifactDetails:
+        """Deletes model defined metadata artifact for specified model metadata key.
+
+        Parameters
+        ----------
+        metadata_key_name: str
+            The name of the model metadatum in the metadata.
+        Returns
+        -------
+        ModelMetadataArtifactDetails
+            The model defined metadata artifact delete call info.
+            Example:
+            {
+                'Date': 'Mon, 02 Dec 2024 06:38:24 GMT',
+                'opc-request-id': 'E4F7',
+                'X-Content-Type-Options': 'nosniff',
+                'Vary': 'Origin',
+                'Strict-Transport-Security': 'max-age=31536000; includeSubDomains',
+                'status': 204
+            }
+
+        """
+        return self.dsc_model.delete_defined_metadata_artifact(
+            metadata_key_name=metadata_key_name
+        )

ads/model/model_metadata.py

@@ -24,6 +24,7 @@ from ads.common.error import ChangesNotCommitted
 from ads.common.extended_enum import ExtendedEnum
 from ads.common.object_storage_details import ObjectStorageDetails
 from ads.common.serializer import DataClassSerializable
+from ads.common.utils import parse_bool
 from ads.dataset import factory
 
 try:
@@ -86,11 +87,13 @@ class MetadataCustomPrintColumns(ExtendedEnum):
     VALUE = "Value"
     DESCRIPTION = "Description"
     CATEGORY = "Category"
+    HAS_ARTIFACT = "HasArtifact"
 
 
 class MetadataTaxonomyPrintColumns(ExtendedEnum):
     KEY = "Key"
     VALUE = "Value"
+    HAS_ARTIFACT = "HasArtifact"
 
 
 class MetadataTaxonomyKeys(ExtendedEnum):
@@ -100,6 +103,10 @@ class MetadataTaxonomyKeys(ExtendedEnum):
     ALGORITHM = "Algorithm"
     HYPERPARAMETERS = "Hyperparameters"
     ARTIFACT_TEST_RESULT = "ArtifactTestResults"
+    README = "Readme"
+    LICENSE = "License"
+    DEPLOYMENT_CONFIGURATION = "DeploymentConfiguration"
+    FINETUNE_CONFIGURATION = "FineTuneConfiguration"
 
 
 class MetadataCustomKeys(ExtendedEnum):
@@ -336,13 +343,14 @@ class ModelMetadataItem(ABC):
         """Creates a new metadata item from the OCI metadata item."""
         oci_metadata_item = to_dict(oci_metadata_item)
         key_value_map = {field: oci_metadata_item.get(field) for field in cls._FIELDS}
-
         if isinstance(key_value_map["value"], str):
             try:
                 key_value_map["value"] = json.loads(oci_metadata_item.get("value"))
+                key_value_map["has_artifact"] = parse_bool(
+                    oci_metadata_item.get("has_artifact")
+                )
             except Exception:
                 pass
-
         return cls(**key_value_map)
 
     def __hash__(self):
@@ -398,15 +406,12 @@ class ModelTaxonomyMetadataItem(ModelMetadataItem):
         Validates metadata item.
     """
 
-    _FIELDS = ["key", "value"]
+    _FIELDS = ["key", "value", "has_artifact"]
 
-    def __init__(
-        self,
-        key: str,
-        value: str = None,
-    ):
+    def __init__(self, key: str, value: str = None, has_artifact: bool = False):
         self.key = key
         self.value = value
+        self.has_artifact = has_artifact
 
     @property
     def key(self) -> str:
@@ -432,6 +437,14 @@ class ModelTaxonomyMetadataItem(ModelMetadataItem):
             raise ValueError("The key cannot be empty.")
         self._key = key
 
+    @property
+    def has_artifact(self) -> bool:
+        return self._has_artifact
+
+    @has_artifact.setter
+    def has_artifact(self, has_artifact: bool):
+        self._has_artifact = has_artifact is True
+
     @property
     def value(self) -> str:
         return self._value
@@ -471,7 +484,7 @@ class ModelTaxonomyMetadataItem(ModelMetadataItem):
         """
         self.update(value=None)
 
-    def update(self, value: str) -> None:
+    def update(self, value: str, has_artifact: bool = False) -> None:
         """Updates metadata item value.
 
         Parameters
@@ -485,6 +498,7 @@ class ModelTaxonomyMetadataItem(ModelMetadataItem):
             Nothing.
         """
         self.value = value
+        self.has_artifact = has_artifact
 
     def validate(self) -> bool:
         """Validates metadata item.
@@ -555,7 +569,7 @@ class ModelCustomMetadataItem(ModelTaxonomyMetadataItem):
         Validates metadata item.
     """
 
-    _FIELDS = ["key", "value", "description", "category"]
+    _FIELDS = ["key", "value", "description", "category", "has_artifact"]
 
     def __init__(
         self,
@@ -563,10 +577,12 @@ class ModelCustomMetadataItem(ModelTaxonomyMetadataItem):
         value: str = None,
         description: str = None,
         category: str = None,
+        has_artifact: bool = False,
     ):
         super().__init__(key=key, value=value)
         self.description = description
         self.category = category
+        self.has_artifact = has_artifact
 
     @property
     def description(self) -> str:
@@ -586,6 +602,17 @@ class ModelCustomMetadataItem(ModelTaxonomyMetadataItem):
 
         self._description = description
 
+    @property
+    def has_artifact(self) -> bool:
+        return self._has_artifact
+
+    @has_artifact.setter
+    def has_artifact(self, has_artifact: bool):
+        if not has_artifact:
+            self._has_artifact = False
+        else:
+            self._has_artifact = has_artifact
+
     @property
     def category(self) -> str:
         return self._category
@@ -631,7 +658,9 @@ class ModelCustomMetadataItem(ModelTaxonomyMetadataItem):
         """
         self.update(value=None, description=None, category=None)
 
-    def update(self, value: str, description: str, category: str) -> None:
+    def update(
+        self, value: str, description: str, category: str, has_artifact: bool = False
+    ) -> None:
         """Updates metadata item.
 
         Parameters
@@ -651,6 +680,7 @@ class ModelCustomMetadataItem(ModelTaxonomyMetadataItem):
         self.value = value
         self.description = description
         self.category = category
+        self.has_artifact = has_artifact
 
     def _to_oci_metadata(self):
         """Converts metadata item to OCI metadata item."""
@@ -659,6 +689,8 @@ class ModelCustomMetadataItem(ModelTaxonomyMetadataItem):
             oci_metadata_item.value = _METADATA_EMPTY_VALUE
         if not oci_metadata_item.category:
             oci_metadata_item.category = MetadataCustomCategory.OTHER
+        if not oci_metadata_item.has_artifact:
+            oci_metadata_item.has_artifact = False
         return oci_metadata_item
 
     def validate(self) -> bool:
@@ -1368,7 +1400,13 @@ class ModelCustomMetadata(ModelMetadata):
         return (
             pd.DataFrame(
                 (
-                    (item.key, item.value, item.description, item.category)
+                    (
+                        item.key,
+                        item.value,
+                        item.description,
+                        item.category,
+                        item.has_artifact,
+                    )
                     for item in self._items
                 ),
                 columns=[value for value in MetadataCustomPrintColumns.values()],
@@ -1510,7 +1548,9 @@ class ModelTaxonomyMetadata(ModelMetadata):
         for oci_item in metadata_list:
             item = ModelTaxonomyMetadataItem._from_oci_metadata(oci_item)
             if item.key in metadata.keys:
-                metadata[item.key].update(value=item.value)
+                metadata[item.key].update(
+                    value=item.value, has_artifact=item.has_artifact
+                )
             else:
                 metadata._items.add(item)
         return metadata
@@ -1525,7 +1565,7 @@ class ModelTaxonomyMetadata(ModelMetadata):
         """
         return (
             pd.DataFrame(
-                ((item.key, item.value) for item in self._items),
+                ((item.key, item.value, item.has_artifact) for item in self._items),
                 columns=[value for value in MetadataTaxonomyPrintColumns.values()],
             )
             .sort_values(by=[MetadataTaxonomyPrintColumns.KEY])

ads/model/model_version_set.py

@@ -14,7 +14,7 @@ from typing import Dict, List, Optional, Union
 
 import oci.data_science
 from ads.common.utils import batch_convert_case, get_value, snake_to_camel
-from ads.config import COMPARTMENT_OCID, OCI_REGION_METADATA, PROJECT_OCID
+from ads.config import COMPARTMENT_OCID, OCI_REGION_METADATA, PROJECT_OCID, USER
 from ads.jobs.builders.base import Builder
 from ads.model.datascience_model import DataScienceModel
 from ads.model.service.oci_datascience_model_version_set import (
@@ -454,7 +454,7 @@ class ModelVersionSet(Builder):
         return "modelVersionSet"
 
     @classmethod
-    def list(cls, compartment_id: str = None, **kwargs) -> List["ModelVersionSet"]:
+    def list(cls, compartment_id: str = None, category: str = USER, **kwargs) -> List["ModelVersionSet"]:
         """
         List model version sets in a given compartment.
 
@@ -462,6 +462,8 @@ class ModelVersionSet(Builder):
         ----------
         compartment_id: str
             The OCID of compartment.
+        category: (str, optional). Defaults to `USER`.
+            The category of Model. Allowed values are: "USER", "SERVICE"
         kwargs
             Additional keyword arguments for filtering model version sets.
 
@@ -473,7 +475,7 @@ class ModelVersionSet(Builder):
         return [
             cls.from_dsc_model_version_set(model_version_set)
             for model_version_set in DataScienceModelVersionSet.list_resource(
-                compartment_id, **kwargs
+                compartment_id, category=category, **kwargs
             )
         ]