oracle-ads 2.11.18__py3-none-any.whl → 2.12.0__py3-none-any.whl

ads/aqua/config/utils/serializer.py

@@ -0,0 +1,339 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import json
+from typing import Union
+from urllib.parse import urlparse
+
+import fsspec
+import yaml
+from pydantic import BaseModel
+from yaml import SafeLoader as Loader
+
+from ads.common.auth import default_signer
+
+
+class Serializable(BaseModel):
+    """Base class that represents a serializable item.
+
+    Methods
+    -------
+    to_json(self, uri=None, **kwargs)
+        Returns object serialized as a JSON string
+    from_json(cls, json_string=None, uri=None, **kwargs)
+        Creates an object from JSON string provided or from URI location containing JSON string
+    to_yaml(self, uri=None, **kwargs)
+        Returns object serialized as a YAML string
+    from_yaml(cls, yaml_string=None, uri=None, **kwargs)
+        Creates an object from YAML string provided or from URI location containing YAML string
+    """
+
+    @staticmethod
+    def _write_to_file(s: str, uri: str, **kwargs) -> None:
+        """Write string s into location specified by uri.
+
+        Parameters
+        ----------
+        s: (string)
+            content
+        uri: (string)
+            URI location to save string s
+        kwargs : dict
+            keyword arguments to be passed into fsspec.open().
+            For OCI object storage, this can be config="path/to/.oci/config".
+
+        Returns
+        -------
+        None
+            Nothing
+        """
+
+        overwrite = kwargs.pop("overwrite", True)
+        if not overwrite:
+            dst_path_scheme = urlparse(uri).scheme or "file"
+            if fsspec.filesystem(dst_path_scheme, **kwargs).exists(uri):
+                raise FileExistsError(
+                    f"The `{uri}` is already exists. Set `overwrite` to True "
+                    "if you wish to overwrite."
+                )
+
+        with fsspec.open(uri, "w", **kwargs) as f:
+            f.write(s)
+
+    @staticmethod
+    def _read_from_file(uri: str, **kwargs) -> str:
+        """Returns contents from location specified by URI
+
+        Parameters
+        ----------
+        uri: (string)
+            URI location
+        kwargs : dict
+            keyword arguments to be passed into fsspec.open().
+            For OCI object storage, this can be config="path/to/.oci/config".
+
+        Returns
+        -------
+        string: Contents in file specified by URI
+        """
+        # Add default signer if the uri is an object storage uri, and
+        # the user does not specify config or signer.
+        if (
+            uri.startswith("oci://")
+            and "config" not in kwargs
+            and "signer" not in kwargs
+        ):
+            kwargs.update(default_signer())
+        with fsspec.open(uri, "r", **kwargs) as f:
+            return f.read()
+
+    def to_json(
+        self,
+        uri: str = None,
+        encoder: callable = json.JSONEncoder,
+        default: callable = None,
+        **kwargs,
+    ) -> str:
+        """Returns object serialized as a JSON string
+
+        Parameters
+        ----------
+        uri: (string, optional)
+            URI location to save the JSON string. Defaults to None.
+        encoder: (callable, optional)
+            Encoder for custom data structures. Defaults to JSONEncoder.
+        default: (callable, optional)
+            A function that gets called for objects that can't otherwise be serialized.
+            It should return JSON-serializable version of the object or original object.
+
+        kwargs
+        ------
+        overwrite: (bool, optional). Defaults to True.
+            Whether to overwrite existing file or not.
+
+        keyword arguments to be passed into fsspec.open().
+        For OCI object storage, this could be config="path/to/.oci/config".
+        For other storage connections consider e.g. host, port, username, password, etc.
+
+        Returns
+        -------
+        Union[str, None]
+            Serialized version of object.
+            `None` in case when `uri` provided.
+        """
+        json_string = json.dumps(
+            self.model_dump(exclude_none=kwargs.pop("exclude_none", False)),
+            cls=encoder,
+            default=default,
+        )
+        if uri:
+            self._write_to_file(s=json_string, uri=uri, **kwargs)
+            return None
+        return json_string
+
+    def to_dict(self) -> dict:
+        """Returns object serialized as a dictionary
+
+        Returns
+        -------
+        dict
+            Serialized version of object
+        """
+        return json.loads(self.to_json())
+
+    @classmethod
+    def from_json(
+        cls,
+        json_string: str = None,
+        uri: str = None,
+        decoder: callable = json.JSONDecoder,
+        **kwargs,
+    ):
+        """Creates an object from JSON string provided or from URI location containing JSON string
+
+        Parameters
+        ----------
+        json_string: (string, optional)
+            JSON string. Defaults to None.
+        uri: (string, optional)
+            URI location of file containing JSON string. Defaults to None.
+        decoder: (callable, optional)
+            Custom decoder. Defaults to simple JSON decoder.
+        kwargs
+        ------
+        keyword arguments to be passed into fsspec.open(). For OCI object storage, this should be config="path/to/.oci/config".
+            For other storage connections consider e.g. host, port, username, password, etc.
+
+        Raises
+        ------
+        ValueError
+            Raised if neither string nor uri is provided
+
+        Returns
+        -------
+        cls
+            Returns instance of the class
+        """
+        if json_string:
+            return cls(**json.loads(json_string, cls=decoder))
+        if uri:
+            return cls(**json.loads(cls._read_from_file(uri, **kwargs), cls=decoder))
+        raise ValueError("Must provide either JSON string or URI location")
+
+    def to_yaml(
+        self, uri: str = None, dumper: callable = yaml.SafeDumper, **kwargs
+    ) -> Union[str, None]:
+        """Returns object serialized as a YAML string
+
+        Parameters
+        ----------
+        uri : str, optional
+            URI location to save the YAML string, by default None
+        dumper : callable, optional
+            Custom YAML Dumper, by default yaml.SafeDumper
+        kwargs : dict
+            overwrite: (bool, optional). Defaults to True.
+                Whether to overwrite existing file or not.
+            note: (str, optional)
+                The note that needs to be added in the beginning of the YAML.
+                It will be added as is without any formatting.
+            side_effect: Optional[SideEffect]
+                side effect to take on the dictionary. The side effect can be either
+                convert the dictionary keys to "lower" (SideEffect.CONVERT_KEYS_TO_LOWER.value)
+                or "upper"(SideEffect.CONVERT_KEYS_TO_UPPER.value) cases.
+
+            The other keyword arguments to be passed into fsspec.open().
+            For OCI object storage, this could be config="path/to/.oci/config".
+
+        Returns
+        -------
+        Union[str, None]
+            Serialized version of object.
+            `None` in case when `uri` provided.
+        """
+        note = kwargs.pop("note", "")
+
+        yaml_string = f"{note}\n" + yaml.dump(
+            self.model_dump(exclude_none=kwargs.pop("exclude_none", False)),
+            Dumper=dumper,
+        )
+        if uri:
+            self._write_to_file(s=yaml_string, uri=uri, **kwargs)
+            return None
+
+        return yaml_string
+
+    @classmethod
+    def from_yaml(
+        cls,
+        yaml_string: str = None,
+        uri: str = None,
+        loader: callable = Loader,
+        **kwargs,
+    ):
+        """Creates an object from YAML string provided or from URI location containing YAML string
+
+        Parameters
+        ----------
+        yaml_string (string, optional)
+            YAML string. Defaults to None.
+        uri (string, optional)
+            URI location of file containing YAML string. Defaults to None.
+        loader (callable, optional)
+            Custom YAML loader. Defaults to CLoader/SafeLoader.
+        kwargs (dict)
+            keyword arguments to be passed into fsspec.open().
+            For OCI object storage, this should be config="path/to/.oci/config".
+            For other storage connections consider e.g. host, port, username, password, etc.
+
+        Raises
+        ------
+        ValueError
+            Raised if neither string nor uri is provided
+
+        Returns
+        -------
+        cls
+            Returns instance of the class
+        """
+        if yaml_string:
+            return cls(**yaml.load(yaml_string, Loader=loader))
+        if uri:
+            return cls(
+                **yaml.load(cls._read_from_file(uri=uri, **kwargs), Loader=loader)
+            )
+        raise ValueError("Must provide either YAML string or URI location")
+
+    @classmethod
+    def schema_to_yaml(cls, uri: str = None, **kwargs) -> Union[str, None]:
+        """Returns the schema serialized as a YAML string
+
+        Parameters
+        ----------
+        uri : str, optional
+            URI location to save the YAML string, by default None
+        dumper : callable, optional
+            Custom YAML Dumper, by default yaml.SafeDumper
+        kwargs : dict
+            overwrite: (bool, optional). Defaults to True.
+                Whether to overwrite existing file or not.
+        Returns
+        -------
+        Union[str, None]
+            Serialized schema.
+            `None` in case when `uri` provided.
+        """
+        yaml_string = yaml.dump(cls.model_json_schema(), sort_keys=False)
+
+        if uri:
+            cls._write_to_file(s=yaml_string, uri=uri, **kwargs)
+            return None
+
+        return yaml_string
+
+    @classmethod
+    def schema_to_json(
+        cls,
+        uri: str = None,
+        encoder: callable = json.JSONEncoder,
+        default: callable = None,
+        **kwargs,
+    ) -> Union[str, None]:
+        """Returns the schema serialized as a JSON string
+
+        Parameters
+        ----------
+        uri: (string, optional)
+            URI location to save the JSON string. Defaults to None.
+        encoder: (callable, optional)
+            Encoder for custom data structures. Defaults to JSONEncoder.
+        default: (callable, optional)
+            A function that gets called for objects that can't otherwise be serialized.
+            It should return JSON-serializable version of the object or original object.
+
+        kwargs
+        ------
+        overwrite: (bool, optional). Defaults to True.
+            Whether to overwrite existing file or not.
+
+        keyword arguments to be passed into fsspec.open().
+        For OCI object storage, this could be config="path/to/.oci/config".
+        For other storage connections consider e.g. host, port, username, password, etc.
+
+        Returns
+        -------
+        Union[str, None]
+            Serialized version of object.
+            `None` in case when `uri` provided.
+        """
+        json_string = json.dumps(
+            cls.model_json_schema(),
+            cls=encoder,
+            default=default,
+        )
+        if uri:
+            cls._write_to_file(s=json_string, uri=uri, **kwargs)
+            return None
+        return json_string

ads/aqua/constants.py

@@ -24,7 +24,7 @@ DEFAULT_FT_VALIDATION_SET_SIZE = 0.1
 MAXIMUM_ALLOWED_DATASET_IN_BYTE = 52428800  # 1024 x 1024 x 50 = 50MB
 JOB_INFRASTRUCTURE_TYPE_DEFAULT_NETWORKING = "ME_STANDALONE"
 NB_SESSION_IDENTIFIER = "NB_SESSION_OCID"
-LIFECYCLE_DETAILS_MISSING_JOBRUN = "The asscociated JobRun resource has been deleted."
+LIFECYCLE_DETAILS_MISSING_JOBRUN = "The associated JobRun resource has been deleted."
 READY_TO_DEPLOY_STATUS = "ACTIVE"
 READY_TO_FINE_TUNE_STATUS = "TRUE"
 AQUA_GA_LIST = ["id19sfcrra6z"]

ads/aqua/evaluation/entities.py

@@ -102,6 +102,7 @@ class ModelParams(DataClassSerializable):
     presence_penalty: Optional[float] = 0.0
     frequency_penalty: Optional[float] = 0.0
     stop: Optional[Union[str, List[str]]] = field(default_factory=list)
+    model: Optional[str] = "odsc-llm"
 
 
 @dataclass(repr=False)

ads/aqua/evaluation/evaluation.py

@@ -11,7 +11,7 @@ from dataclasses import asdict, fields
 from datetime import datetime, timedelta
 from pathlib import Path
 from threading import Lock
-from typing import Any, Dict, List, Union
+from typing import Any, Dict, List, Optional, Union
 
 import oci
 from cachetools import TTLCache
@@ -45,6 +45,8 @@ from ads.aqua.common.utils import (
     is_valid_ocid,
     upload_local_to_os,
 )
+from ads.aqua.config.config import get_evaluation_service_config
+from ads.aqua.config.evaluation.evaluation_service_config import EvaluationServiceConfig
 from ads.aqua.constants import (
     CONSOLE_LINK_RESOURCE_TYPE_MAPPING,
     EVALUATION_REPORT,
@@ -170,8 +172,19 @@ class AquaEvaluationApp(AquaApp):
                 f"Invalid evaluation source {create_aqua_evaluation_details.evaluation_source_id}. "
                 "Specify either a model or model deployment id."
             )
+
+        # The model to evaluate
         evaluation_source = None
-        eval_inference_configuration = None
+        # The evaluation service config
+        evaluation_config: EvaluationServiceConfig = get_evaluation_service_config()
+        # The evaluation inference configuration. The inference configuration will be extracted
+        # based on the inferencing container family.
+        eval_inference_configuration: Dict = {}
+        # The evaluation inference model sampling params. The system parameters that will not be
+        # visible for user, but will be applied implicitly for evaluation. The service model params
+        # will be extracted based on the container family and version.
+        eval_inference_service_model_params: Dict = {}
+
         if (
             DataScienceResource.MODEL_DEPLOYMENT
             in create_aqua_evaluation_details.evaluation_source_id
@@ -187,17 +200,32 @@ class AquaEvaluationApp(AquaApp):
                     runtime = ModelDeploymentContainerRuntime.from_dict(
                         evaluation_source.runtime.to_dict()
                     )
-                    inference_config = AquaContainerConfig.from_container_index_json(
+                    container_config = AquaContainerConfig.from_container_index_json(
                         enable_spec=True
-                    ).inference
-                    for container in inference_config.values():
-                        if container.name == runtime.image[:runtime.image.rfind(":")]:
+                    )
+                    for (
+                        inference_container_family,
+                        inference_container_info,
+                    ) in container_config.inference.items():
+                        if (
+                            inference_container_info.name
+                            == runtime.image[: runtime.image.rfind(":")]
+                        ):
                             eval_inference_configuration = (
-                                container.spec.evaluation_configuration
+                                evaluation_config.get_merged_inference_params(
+                                    inference_container_family
+                                ).to_dict()
                             )
+                            eval_inference_service_model_params = (
+                                evaluation_config.get_merged_inference_model_params(
+                                    inference_container_family,
+                                    inference_container_info.version,
+                                )
+                            )
+
             except Exception:
                 logger.debug(
-                    f"Could not load inference config details for the evaluation id: "
+                    f"Could not load inference config details for the evaluation source id: "
                     f"{create_aqua_evaluation_details.evaluation_source_id}. Please check if the container"
                     f" runtime has the correct SMC image information."
                 )
@@ -414,11 +442,12 @@ class AquaEvaluationApp(AquaApp):
                 container_image=container_image,
                 dataset_path=evaluation_dataset_path,
                 report_path=create_aqua_evaluation_details.report_path,
-                model_parameters=create_aqua_evaluation_details.model_parameters,
+                model_parameters={
+                    **eval_inference_service_model_params,
+                    **create_aqua_evaluation_details.model_parameters,
+                },
                 metrics=create_aqua_evaluation_details.metrics,
-                inference_configuration=eval_inference_configuration.to_filtered_dict()
-                if eval_inference_configuration
-                else {},
+                inference_configuration=eval_inference_configuration or {},
             )
         ).create(**kwargs)  ## TODO: decide what parameters will be needed
         logger.debug(
@@ -901,48 +930,8 @@ class AquaEvaluationApp(AquaApp):
 
     def get_supported_metrics(self) -> dict:
         """Gets a list of supported metrics for evaluation."""
-        # TODO: implement it when starting to support more metrics.
         return [
-            {
-                "use_case": ["text_generation"],
-                "key": "bertscore",
-                "name": "bertscore",
-                "description": (
-                    "BERT Score is a metric for evaluating the quality of text "
-                    "generation models, such as machine translation or summarization. "
-                    "It utilizes pre-trained BERT contextual embeddings for both the "
-                    "generated and reference texts, and then calculates the cosine "
-                    "similarity between these embeddings."
-                ),
-                "args": {},
-            },
-            {
-                "use_case": ["text_generation"],
-                "key": "rouge",
-                "name": "rouge",
-                "description": (
-                    "ROUGE scores compare a candidate document to a collection of "
-                    "reference documents to evaluate the similarity between them. "
-                    "The metrics range from 0 to 1, with higher scores indicating "
-                    "greater similarity. ROUGE is more suitable for models that don't "
-                    "include paraphrasing and do not generate new text units that don't "
-                    "appear in the references."
-                ),
-                "args": {},
-            },
-            {
-                "use_case": ["text_generation"],
-                "key": "bleu",
-                "name": "bleu",
-                "description": (
-                    "BLEU (Bilingual Evaluation Understudy) is an algorithm for evaluating the "
-                    "quality of text which has been machine-translated from one natural language to another. "
-                    "Quality is considered to be the correspondence between a machine's output and that of a "
-                    "human: 'the closer a machine translation is to a professional human translation, "
-                    "the better it is'."
-                ),
-                "args": {},
-            },
+            item.to_dict() for item in get_evaluation_service_config().ui_config.metrics
         ]
 
     @telemetry(entry_point="plugin=evaluation&action=load_metrics", name="aqua")
@@ -1225,45 +1214,24 @@ class AquaEvaluationApp(AquaApp):
                 f"Exception message: {ex}"
             )
 
-    def load_evaluation_config(self, eval_id):
+    def load_evaluation_config(self, container: Optional[str] = None) -> Dict:
         """Loads evaluation config."""
+
+        # retrieve the evaluation config by container family name
+        evaluation_config = get_evaluation_service_config(container)
+
+        # convert the new config representation to the old one
         return {
-            "model_params": {
-                "max_tokens": 500,
-                "temperature": 0.7,
-                "top_p": 1.0,
-                "top_k": 50,
-                "presence_penalty": 0.0,
-                "frequency_penalty": 0.0,
-                "stop": [],
-            },
+            "model_params": evaluation_config.ui_config.model_params.default,
             "shape": {
-                "VM.Standard.E3.Flex": {
-                    "ocpu": 8,
-                    "memory_in_gbs": 128,
-                    "block_storage_size": 200,
-                },
-                "VM.Standard.E4.Flex": {
-                    "ocpu": 8,
-                    "memory_in_gbs": 128,
-                    "block_storage_size": 200,
-                },
-                "VM.Standard3.Flex": {
-                    "ocpu": 8,
-                    "memory_in_gbs": 128,
-                    "block_storage_size": 200,
-                },
-                "VM.Optimized3.Flex": {
-                    "ocpu": 8,
-                    "memory_in_gbs": 128,
-                    "block_storage_size": 200,
-                },
-            },
-            "default": {
-                "ocpu": 8,
-                "memory_in_gbs": 128,
-                "block_storage_size": 200,
+                shape.name: shape.to_dict()
+                for shape in evaluation_config.ui_config.shapes
             },
+            "default": (
+                evaluation_config.ui_config.shapes[0].to_dict()
+                if len(evaluation_config.ui_config.shapes) > 0
+                else {}
+            ),
         }
 
     def _get_attribute_from_model_metadata(

ads/aqua/extension/common_handler.py

@@ -11,16 +11,15 @@ from huggingface_hub import HfApi
 from huggingface_hub.utils import LocalTokenNotFoundError
 from tornado.web import HTTPError
 
-from ads.aqua import ODSC_MODEL_COMPARTMENT_OCID
 from ads.aqua.common.decorator import handle_exceptions
 from ads.aqua.common.errors import AquaResourceAccessError, AquaRuntimeError
 from ads.aqua.common.utils import (
-    fetch_service_compartment,
     get_huggingface_login_timeout,
     known_realm,
 )
 from ads.aqua.extension.base_handler import AquaAPIhandler
 from ads.aqua.extension.errors import Errors
+from ads.aqua.extension.utils import ui_compatability_check
 
 
 class ADSVersionHandler(AquaAPIhandler):
@@ -51,7 +50,7 @@ class CompatibilityCheckHandler(AquaAPIhandler):
             AquaResourceAccessError: raised when aqua is not accessible in the given session/region.
 
         """
-        if ODSC_MODEL_COMPARTMENT_OCID or fetch_service_compartment():
+        if ui_compatability_check():
             return self.finish({"status": "ok"})
         elif known_realm():
             return self.finish({"status": "compatible"})

ads/aqua/extension/common_ws_msg_handler.py

@@ -7,7 +7,6 @@ import json
 from importlib import metadata
 from typing import List, Union
 
-from ads.aqua import ODSC_MODEL_COMPARTMENT_OCID, fetch_service_compartment
 from ads.aqua.common.decorator import handle_exceptions
 from ads.aqua.common.errors import AquaResourceAccessError
 from ads.aqua.common.utils import known_realm
@@ -17,6 +16,7 @@ from ads.aqua.extension.models.ws_models import (
     CompatibilityCheckResponse,
     RequestResponseType,
 )
+from ads.aqua.extension.utils import ui_compatability_check
 
 
 class AquaCommonWsMsgHandler(AquaWSMsgHandler):
@@ -39,7 +39,7 @@ class AquaCommonWsMsgHandler(AquaWSMsgHandler):
             )
             return response
         if request.get("kind") == "CompatibilityCheck":
-            if ODSC_MODEL_COMPARTMENT_OCID or fetch_service_compartment():
+            if ui_compatability_check():
                 return CompatibilityCheckResponse(
                     message_id=request.get("message_id"),
                     kind=RequestResponseType.CompatibilityCheck,

ads/aqua/extension/evaluation_handler.py

@@ -2,6 +2,7 @@
 # Copyright (c) 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
+from typing import Optional
 from urllib.parse import urlparse
 
 from tornado.web import HTTPError
@@ -30,7 +31,7 @@ class AquaEvaluationHandler(AquaAPIhandler):
         return self.read(eval_id)
 
     @handle_exceptions
-    def post(self, *args, **kwargs):
+    def post(self, *args, **kwargs):  # noqa
         """Handles post request for the evaluation APIs
 
         Raises
@@ -117,10 +118,10 @@ class AquaEvaluationConfigHandler(AquaAPIhandler):
     """Handler for Aqua Evaluation Config REST APIs."""
 
     @handle_exceptions
-    def get(self, model_id):
+    def get(self, container: Optional[str] = None, **kwargs):  # noqa
         """Handle GET request."""
 
-        return self.finish(AquaEvaluationApp().load_evaluation_config(model_id))
+        return self.finish(AquaEvaluationApp().load_evaluation_config(container))
 
 
 __handlers__ = [

ads/aqua/extension/model_handler.py

@@ -9,7 +9,7 @@ from tornado.web import HTTPError
 
 from ads.aqua.common.decorator import handle_exceptions
 from ads.aqua.common.errors import AquaRuntimeError, AquaValueError
-from ads.aqua.common.utils import get_hf_model_info
+from ads.aqua.common.utils import get_hf_model_info, list_hf_models
 from ads.aqua.extension.base_handler import AquaAPIhandler
 from ads.aqua.extension.errors import Errors
 from ads.aqua.model import AquaModelApp
@@ -177,6 +177,31 @@ class AquaHuggingFaceHandler(AquaAPIhandler):
 
         return None
 
+
+
+    @handle_exceptions
+    def get(self,*args, **kwargs):
+        """
+        Finds a list of matching models from hugging face based on query string provided from users.
+
+        Parameters
+        ----------
+        query (str): The Hugging Face model name to search for.
+
+        Returns
+        -------
+        List[str]
+            Returns the matching model ids string
+        """
+
+        query=self.get_argument("query",default=None)
+        if not query:
+            raise HTTPError(400,Errors.MISSING_REQUIRED_PARAMETER.format("query"))
+        models=list_hf_models(query)
+        return self.finish({"models":models})
+
+
+
     @handle_exceptions
     def post(self, *args, **kwargs):
         """Handles post request for the HF Models APIs