oracle-ads 2.13.1rc0__py3-none-any.whl → 2.13.2rc1__py3-none-any.whl

ads/model/service/oci_datascience_model.py

@@ -4,9 +4,10 @@
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 import logging
+from dataclasses import dataclass
 from functools import wraps
 from io import BytesIO
-from typing import Callable, Dict, List, Optional
+from typing import Callable, Dict, List, Optional, Union
 
 import oci.data_science
 from oci.data_science.models import (
@@ -18,13 +19,18 @@ from oci.data_science.models import (
     UpdateModelDetails,
 )
 from oci.exceptions import ServiceError
+from requests.structures import CaseInsensitiveDict
 
+from ads.common import utils
+from ads.common.auth import default_signer
 from ads.common.object_storage_details import ObjectStorageDetails
 from ads.common.oci_datascience import OCIDataScienceMixin
 from ads.common.oci_mixin import OCIWorkRequestMixin
 from ads.common.oci_resource import SEARCH_TYPE, OCIResource
-from ads.common.utils import extract_region
+from ads.common.serializer import DataClassSerializable
+from ads.common.utils import extract_region, read_file, text_sanitizer
 from ads.common.work_request import DataScienceWorkRequest
+from ads.model.common.utils import MetadataArtifactPathType
 from ads.model.deployment import ModelDeployment
 
 logger = logging.getLogger(__name__)
@@ -54,6 +60,21 @@ class ModelWithActiveDeploymentError(Exception):  # pragma: no cover
     pass
 
 
+class ModelMetadataArtifactNotFoundError(Exception):  # pragma: no cover
+    def __init__(self, model_ocid, metadata_key: str):
+        super().__init__(
+            f"The model {model_ocid} does not contain the metadata with key {metadata_key}."
+        )
+
+
+@dataclass(repr=False)
+class ModelMetadataArtifactDetails(DataClassSerializable):
+    """Represents a details of Model Metadata ."""
+
+    headers: Union[Dict, CaseInsensitiveDict]
+    status: str
+
+
 def check_for_model_id(msg: str = MODEL_NEEDS_TO_BE_SAVED):
     """The decorator helping to check if the ID attribute sepcified for a datascience model.
 
@@ -86,6 +107,12 @@ def check_for_model_id(msg: str = MODEL_NEEDS_TO_BE_SAVED):
     return decorator
 
 
+def convert_model_metadata_response(
+    headers: Union[Dict, CaseInsensitiveDict], status: int
+) -> ModelMetadataArtifactDetails:
+    return ModelMetadataArtifactDetails(headers=headers, status=str(status))
+
+
 class OCIDataScienceModel(
     OCIDataScienceMixin,
     OCIWorkRequestMixin,
@@ -529,8 +556,6 @@ class OCIDataScienceModel(
 
         Parameters
         ----------
-        model_id: str
-            The model ID.
         config: (Dict, optional). Defaults to `None`.
             Configuration keys and values as per SDK and Tool Configuration.
             The from_file() method can be used to load configuration from a file.
@@ -579,7 +604,7 @@ class OCIDataScienceModel(
             raise ValueError("Model OCID not provided.")
         return super().from_ocid(ocid)
 
-    def _is_model_by_reference(self):
+    def is_model_created_by_reference(self):
         """Checks if model is created by reference
         Returns
         -------
@@ -594,3 +619,450 @@ class OCIDataScienceModel(
                 ):
                     return True
         return False
+
+    def get_metadata_content(
+        self, artifact_path_or_content: str, path_type: MetadataArtifactPathType
+    ):
+        """
+        returns the content of the metadata artifact
+
+        Parameters
+        ----------
+        artifact_path_or_content: str
+            The path of the file (local or oss) containing metadata artifact or content.
+        path_type: str
+            can be one of local , oss or actual content itself
+
+        Returns
+        -------
+        metadata artifact content
+        """
+
+        if path_type == MetadataArtifactPathType.CONTENT:
+            return artifact_path_or_content
+
+        elif path_type == MetadataArtifactPathType.LOCAL:
+            if not utils.is_path_exists(artifact_path_or_content):
+                raise FileNotFoundError(
+                    f"File not found:  {artifact_path_or_content} . "
+                )
+
+            with open(artifact_path_or_content, "rb") as f:
+                contents = f.read()
+                logger.info(f"The metadata artifact content - {contents}")
+
+            return contents
+
+        elif path_type == MetadataArtifactPathType.OSS:
+            if not utils.is_path_exists(artifact_path_or_content):
+                raise FileNotFoundError(f"File not found: {artifact_path_or_content}")
+
+            contents = str(
+                read_file(file_path=artifact_path_or_content, auth=default_signer())
+            )
+            logger.debug(f"The metadata artifact content - {contents}")
+
+            return contents
+
+    @check_for_model_id(
+        msg="Model needs to be saved to the Model Catalog before the creating custom metadata artifact corresponding to that model"
+    )
+    def create_custom_metadata_artifact(
+        self,
+        metadata_key_name: str,
+        artifact_path: str,
+        path_type: MetadataArtifactPathType,
+    ) -> ModelMetadataArtifactDetails:
+        """Creates model custom metadata artifact for specified model.
+
+        Parameters
+        ----------
+        metadata_key_name: str
+            The name of the model metadatum in the metadata.
+
+        artifact_path: str
+            The model custom metadata artifact path to be upload.
+
+        path_type: MetadataArtifactPathType
+            can be one of local , oss or actual content itself
+
+        Returns
+        -------
+        ModelMetadataArtifactDetails
+            The model custom metadata artifact creation info.
+            Example:
+            {
+                'Date': 'Mon, 02 Dec 2024 06:38:24 GMT',
+                'opc-request-id': 'E4F7',
+                'ETag': '77156317-8bb9-4c4a-882b-0d85f8140d93',
+                'X-Content-Type-Options': 'nosniff',
+                'Content-Length': '4029958',
+                'Vary': 'Origin',
+                'Strict-Transport-Security': 'max-age=31536000; includeSubDomains',
+                'status': 204
+            }
+
+        """
+        contents = self.get_metadata_content(
+            artifact_path_or_content=artifact_path, path_type=path_type
+        )
+        response = self.client.create_model_custom_metadatum_artifact(
+            self.id,
+            metadata_key_name,
+            text_sanitizer(contents),
+            content_disposition="form" '-data; name="file"; filename="readme.*"',
+        )
+        response_data = convert_model_metadata_response(
+            response.headers, response.status
+        )
+        return response_data
+
+    @check_for_model_id(
+        msg="Model needs to be saved to the Model Catalog before creating defined metadata artifact corresponding to that model"
+    )
+    def create_defined_metadata_artifact(
+        self,
+        metadata_key_name: str,
+        artifact_path: str,
+        path_type: MetadataArtifactPathType,
+    ) -> ModelMetadataArtifactDetails:
+        """Creates model defined metadata artifact for specified model.
+
+        Parameters
+        ----------
+        metadata_key_name: str
+            The name of the model metadatum in the metadata.
+
+        artifact_path: str
+            The model custom metadata artifact path to be upload.
+
+        path_type: MetadataArtifactPathType
+            can be one of local , oss or actual content itself.
+
+        Returns
+        -------
+        ModelMetadataArtifactDetails
+            The model defined metadata artifact creation info.
+            Example:
+            {
+                'Date': 'Mon, 02 Dec 2024 06:38:24 GMT',
+                'opc-request-id': 'E4F7',
+                'ETag': '77156317-8bb9-4c4a-882b-0d85f8140d93',
+                'X-Content-Type-Options': 'nosniff',
+                'Content-Length': '4029958',
+                'Vary': 'Origin',
+                'Strict-Transport-Security': 'max-age=31536000; includeSubDomains',
+                'status': 204
+            }
+
+        """
+        contents = self.get_metadata_content(
+            artifact_path_or_content=artifact_path, path_type=path_type
+        )
+        response = self.client.create_model_defined_metadatum_artifact(
+            self.id,
+            metadata_key_name,
+            text_sanitizer(contents),
+            content_disposition='form-data; name="file"; filename="readme.*"',
+        )
+        response_data = convert_model_metadata_response(
+            response.headers, response.status
+        )
+        return response_data
+
+    @check_for_model_id(
+        msg="Model needs to be saved to the Model Catalog before updating defined metadata artifact corresponding to that model"
+    )
+    def update_defined_metadata_artifact(
+        self,
+        metadata_key_name: str,
+        artifact_path: str,
+        path_type: MetadataArtifactPathType,
+    ) -> ModelMetadataArtifactDetails:
+        """Update model defined metadata artifact for specified model.
+
+        Parameters
+        ----------
+        metadata_key_name: str
+            The name of the model metadatum in the metadata.
+
+        artifact_path: str
+            The model defined metadata artifact path to be upload.
+
+        path_type:MetadataArtifactPathType
+            can be one of local , oss or actual content itself.
+        Returns
+        -------
+        ModelMetadataArtifactDetails
+            The model defined metadata artifact update info.
+            Example:
+            {
+                'Date': 'Mon, 02 Dec 2024 06:38:24 GMT',
+                'opc-request-id': 'E4F7',
+                'ETag': '77156317-8bb9-4c4a-882b-0d85f8140d93',
+                'X-Content-Type-Options': 'nosniff',
+                'Content-Length': '4029958',
+                'Vary': 'Origin',
+                'Strict-Transport-Security': 'max-age=31536000; includeSubDomains',
+                'status': 204
+            }
+
+        """
+        contents = self.get_metadata_content(
+            artifact_path_or_content=artifact_path, path_type=path_type
+        )
+        response = self.client.update_model_defined_metadatum_artifact(
+            self.id,
+            metadata_key_name,
+            text_sanitizer(contents),
+            content_disposition='form-data; name="file"; filename="readme.*"',
+        )
+        response_data = convert_model_metadata_response(
+            response.headers, response.status
+        )
+        return response_data
+
+    @check_for_model_id(
+        msg="Model needs to be saved to the Model Catalog before updating custom metadata artifact corresponding to that model"
+    )
+    def update_custom_metadata_artifact(
+        self,
+        metadata_key_name: str,
+        artifact_path: str,
+        path_type: MetadataArtifactPathType,
+    ) -> ModelMetadataArtifactDetails:
+        """Update model custom metadata artifact for specified model.
+
+        Parameters
+        ----------
+        metadata_key_name: str
+            The name of the model metadatum in the metadata.
+
+        artifact_path: str
+            The model custom metadata artifact path to be upload.
+
+        path_type: MetadataArtifactPathType
+            can be one of local , oss or actual content itself.
+
+        Returns
+        -------
+        ModelMetadataArtifactDetails
+            The model custom metadata artifact update info.
+            Example:
+            {
+                'Date': 'Mon, 02 Dec 2024 06:38:24 GMT',
+                'opc-request-id': 'E4F7',
+                'ETag': '77156317-8bb9-4c4a-882b-0d85f8140d93',
+                'X-Content-Type-Options': 'nosniff',
+                'Content-Length': '4029958',
+                'Vary': 'Origin',
+                'Strict-Transport-Security': 'max-age=31536000; includeSubDomains',
+                'status': 204
+            }
+
+        """
+        contents = self.get_metadata_content(
+            artifact_path_or_content=artifact_path, path_type=path_type
+        )
+        response = self.client.update_model_custom_metadatum_artifact(
+            self.id,
+            metadata_key_name,
+            text_sanitizer(contents),
+            content_disposition="form" '-data; name="file"; filename="readme.*"',
+        )
+        response_data = convert_model_metadata_response(
+            response.headers, response.status
+        )
+        return response_data
+
+    @check_for_model_id(
+        msg="Model needs to be saved to the Model Catalog before fetching custom metadata artifact corresponding to that model"
+    )
+    def get_custom_metadata_artifact(self, metadata_key_name: str) -> bytes:
+        """Downloads model custom metadata artifact content for specified model metadata key.
+
+        Parameters
+        ----------
+        metadata_key_name: str
+            The name of the model metadatum in the metadata.
+        Returns
+        -------
+        bytes
+               custom metadata artifact content
+
+        """
+        try:
+            return self.client.get_model_custom_metadatum_artifact_content(
+                self.id, metadata_key_name
+            ).data.content
+        except ServiceError as ex:
+            if ex.status == 404:
+                raise ModelMetadataArtifactNotFoundError(self.id, metadata_key_name)
+
+    @check_for_model_id(
+        msg="Model needs to be saved to the Model Catalog before fetching defined metadata artifact corresponding to that model"
+    )
+    def get_defined_metadata_artifact(self, metadata_key_name: str) -> bytes:
+        """Downloads model defined metadata artifact content for specified model metadata key.
+
+        Parameters
+        ----------
+        metadata_key_name: str
+            The name of the model metadatum in the metadata.
+        Returns
+        -------
+        bytes
+                Defined metadata artifact content
+
+        """
+        try:
+            return self.client.get_model_defined_metadatum_artifact_content(
+                self.id, metadata_key_name
+            ).data.content
+        except ServiceError as ex:
+            if ex.status == 404 or ex.status == 400:
+                raise ModelMetadataArtifactNotFoundError(self.id, metadata_key_name)
+
+    @check_for_model_id(
+        msg="Model needs to be saved to the Model Catalog before fetching custom metadata artifact corresponding to that model"
+    )
+    def head_custom_metadata_artifact(
+        self, metadata_key_name: str
+    ) -> ModelMetadataArtifactDetails:
+        """Gets custom metadata artifact metadata for specified model metadata key.
+
+        Parameters
+        ----------
+        metadata_key_name: str
+            The name of the model metadatum in the metadata.
+        Returns
+        -------
+        ModelMetadataArtifactDetails
+            The model custom metadata artifact head call info.
+            Example:
+            {
+                'Date': 'Mon, 02 Dec 2024 06:38:24 GMT',
+                'opc-request-id': 'E4F7',
+                'ETag': '77156317-8bb9-4c4a-882b-0d85f8140d93',
+                'X-Content-Type-Options': 'nosniff',
+                'Content-Length': '4029958',
+                'Vary': 'Origin',
+                'Strict-Transport-Security': 'max-age=31536000; includeSubDomains',
+                'status': 204
+            }
+
+        """
+        response = self.client.head_model_custom_metadatum_artifact(
+            self.id, metadata_key_name
+        )
+        response_data = convert_model_metadata_response(
+            response.headers, response.status
+        )
+        return response_data
+
+    @check_for_model_id(
+        msg="Model needs to be saved to the Model Catalog before fetching defined metadata artifact corresponding to that model"
+    )
+    def head_defined_metadata_artifact(
+        self, metadata_key_name: str
+    ) -> ModelMetadataArtifactDetails:
+        """Gets defined metadata artifact metadata for specified model metadata key.
+
+        Parameters
+        ----------
+        metadata_key_name: str
+            The name of the model metadatum in the metadata.
+        Returns
+        -------
+        ModelMetadataArtifactDetails
+            The model defined metadata artifact head call info.
+            Example:
+            {
+                'Date': 'Mon, 02 Dec 2024 06:38:24 GMT',
+                'opc-request-id': 'E4F7',
+                'ETag': '77156317-8bb9-4c4a-882b-0d85f8140d93',
+                'X-Content-Type-Options': 'nosniff',
+                'Content-Length': '4029958',
+                'Vary': 'Origin',
+                'Strict-Transport-Security': 'max-age=31536000; includeSubDomains',
+                'status': 204
+            }
+
+        """
+        response = self.client.head_model_defined_metadatum_artifact(
+            self.id, metadata_key_name
+        )
+        response_data = convert_model_metadata_response(
+            response.headers, response.status
+        )
+        return response_data
+
+    @check_for_model_id(
+        msg="Model needs to be saved to the Model Catalog before the deleting custom metadata artifact corresponding to that model"
+    )
+    def delete_custom_metadata_artifact(
+        self, metadata_key_name: str
+    ) -> ModelMetadataArtifactDetails:
+        """Deletes model custom metadata artifact for specified model metadata key.
+
+        Parameters
+        ----------
+        metadata_key_name: str
+            The name of the model metadatum in the metadata.
+        Returns
+        -------
+        ModelMetadataArtifactDetails
+            The model custom metadata artifact delete call info.
+            Example:
+            {
+                'Date': 'Mon, 02 Dec 2024 06:38:24 GMT',
+                'opc-request-id': 'E4F7',
+                'X-Content-Type-Options': 'nosniff',
+                'Vary': 'Origin',
+                'Strict-Transport-Security': 'max-age=31536000; includeSubDomains',
+                'status': 204
+            }
+
+        """
+        response = self.client.delete_model_custom_metadatum_artifact(
+            self.id, metadata_key_name
+        )
+        response_data = convert_model_metadata_response(
+            response.headers, response.status
+        )
+        return response_data
+
+    @check_for_model_id(
+        msg="Model needs to be saved to the Model Catalog before the deleting defined metadata artifact corresponding to that model"
+    )
+    def delete_defined_metadata_artifact(
+        self, metadata_key_name: str
+    ) -> ModelMetadataArtifactDetails:
+        """Deletes model defined metadata artifact for specified model metadata key.
+
+        Parameters
+        ----------
+        metadata_key_name: str
+            The name of the model metadatum in the metadata.
+        Returns
+        -------
+        ModelMetadataArtifactDetails
+            The model defined metadata artifact delete call info.
+            Example:
+            {
+                'Date': 'Mon, 02 Dec 2024 06:38:24 GMT',
+                'opc-request-id': 'E4F7',
+                'X-Content-Type-Options': 'nosniff',
+                'Vary': 'Origin',
+                'Strict-Transport-Security': 'max-age=31536000; includeSubDomains',
+                'status': 204
+            }
+
+        """
+        response = self.client.delete_model_defined_metadatum_artifact(
+            self.id, metadata_key_name
+        )
+        response_data = convert_model_metadata_response(
+            response.headers, response.status
+        )
+        return response_data

ads/opctl/operator/common/utils.py

@@ -9,6 +9,7 @@ import os
 import sys
 import time
 import traceback
+import uuid
 from string import Template
 from typing import Any, Dict, List, Tuple
 
@@ -17,6 +18,7 @@ import yaml
 from cerberus import Validator
 
 from ads.opctl import logger, utils
+from ads.common.oci_logging import OCILog
 
 CONTAINER_NETWORK = "CONTAINER_NETWORK"
 
@@ -190,3 +192,17 @@ def print_traceback():
     if logger.level == logging.DEBUG:
         ex_type, ex, tb = sys.exc_info()
         traceback.print_tb(tb)
+
+
+def create_log_in_log_group(compartment_id, log_group_id, auth, log_name=None):
+    """
+    Creates a log within a given log group
+    """
+    if not log_name:
+        log_name = f"log-{int(time.time())}-{uuid.uuid4()}"
+    log = OCILog(display_name=log_name,
+                 log_group_id=log_group_id,
+                 compartment_id=compartment_id,
+                 **auth).create()
+    logger.info(f"Created log with log OCID {log.id}")
+    return log.id

ads/opctl/operator/lowcode/anomaly/model/base_model.py

@@ -71,7 +71,7 @@ class AnomalyOperatorBaseModel(ABC):
         try:
             anomaly_output = self._build_model()
         except Exception as e:
-            logger.warn(f"Found exception: {e}")
+            logger.warning(f"Found exception: {e}")
             if self.spec.datetime_column:
                 anomaly_output = self._fallback_build_model()
             raise e
@@ -347,7 +347,7 @@ class AnomalyOperatorBaseModel(ABC):
                 storage_options=storage_options,
             )
 
-        logger.warn(
+        logger.warning(
             f"The report has been successfully "
             f"generated and placed to the: {unique_output_dir}."
         )
@@ -356,7 +356,7 @@ class AnomalyOperatorBaseModel(ABC):
         """
         Fallback method for the sub model _build_model method.
         """
-        logger.warn(
+        logger.warning(
             f"The build_model method has failed for the model: {self.spec.model}. "
             "A fallback model will be built."
         )

ads/opctl/operator/lowcode/anomaly/model/randomcutforest.py

@@ -95,7 +95,7 @@ class RandomCutForestOperatorModel(AnomalyOperatorBaseModel):
 
                 anomaly_output.add_output(target, anomaly, score)
             except Exception as e:
-                logger.warn(f"Encountered Error: {e}. Skipping series {target}.")
+                logger.warning(f"Encountered Error: {e}. Skipping series {target}.")
 
         return anomaly_output
 

ads/opctl/operator/lowcode/anomaly/utils.py

@@ -44,7 +44,7 @@ def _build_metrics_df(y_true, y_pred, column_name):
         # Throws exception if y_true has only one class
         metrics[SupportedMetrics.ROC_AUC] = roc_auc_score(y_true, y_pred)
     except Exception as e:
-        logger.warn(f"An exception occurred: {e}")
+        logger.warning(f"An exception occurred: {e}")
         metrics[SupportedMetrics.ROC_AUC] = None
     precision, recall, thresholds = precision_recall_curve(y_true, y_pred)
     metrics[SupportedMetrics.PRC_AUC] = auc(recall, precision)

ads/opctl/operator/lowcode/common/data.py

@@ -19,13 +19,16 @@ from .transformations import Transformations
 
 
 class AbstractData(ABC):
-    def __init__(self, spec: dict, name="input_data"):
+    def __init__(self, spec, name="input_data", data=None):
         self.Transformations = Transformations
         self.data = None
         self._data_dict = dict()
         self.name = name
         self.spec = spec
-        self.load_transform_ingest_data(spec)
+        if data is not None:
+            self.data = data
+        else:
+            self.load_transform_ingest_data(spec)
 
     def get_raw_data_by_cat(self, category):
         mapping = self._data_transformer.get_target_category_columns_map()

ads/opctl/operator/lowcode/common/transformations.py

@@ -31,7 +31,6 @@ class Transformations(ABC):
             dataset_info : ForecastOperatorConfig
         """
         self.name = name
-        self.has_artificial_series = False
         self.dataset_info = dataset_info
         self.target_category_columns = dataset_info.target_category_columns
         self.target_column_name = dataset_info.target_column
@@ -99,7 +98,11 @@ class Transformations(ABC):
         return clean_df
 
     def _remove_trailing_whitespace(self, df):
-        return df.apply(lambda x: x.str.strip() if x.dtype == "object" else x)
+        return df.apply(
+            lambda x: x.str.strip()
+            if hasattr(x, "dtype") and x.dtype == "object"
+            else x
+        )
 
     def _clean_column_names(self, df):
         """
@@ -136,7 +139,6 @@ class Transformations(ABC):
         self._target_category_columns_map = {}
         if not self.target_category_columns:
             df[DataColumns.Series] = "Series 1"
-            self.has_artificial_series = True
         else:
             df[DataColumns.Series] = merge_category_columns(
                 df, self.target_category_columns
@@ -209,7 +211,7 @@ class Transformations(ABC):
 
     def _missing_value_imputation_add(self, df):
         """
-        Function fills missing values in the pandas dataframe using liner interpolation
+        Function fills missing values with zero
 
         Parameters
         ----------
@@ -219,15 +221,7 @@ class Transformations(ABC):
         -------
             A new Pandas DataFrame without missing values.
         """
-        # find columns that all all NA and replace with 0
-        for col in df.columns:
-            # find next int not in list
-            i = 0
-            vals = df[col].unique()
-            while i in vals:
-                i = i + 1
-            df[col] = df[col].fillna(0)
-        return df
+        return df.fillna(0)
 
     def _outlier_treatment(self, df):
         """