oracle-ads 2.11.6__py3-none-any.whl → 2.11.8__py3-none-any.whl

ads/aqua/__init__.py

@@ -4,25 +4,35 @@
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 
-import logging
-import sys
 import os
+
+from ads import logger, set_auth
 from ads.aqua.utils import fetch_service_compartment
 from ads.config import OCI_RESOURCE_PRINCIPAL_VERSION
-from ads import set_auth
 
-logger = logging.getLogger(__name__)
-handler = logging.StreamHandler(sys.stdout)
-logger.setLevel(logging.INFO)
+ENV_VAR_LOG_LEVEL = "ADS_AQUA_LOG_LEVEL"
+
+
+def get_logger_level():
+    """Retrieves logging level from environment variable `ADS_AQUA_LOG_LEVEL`."""
+    level = os.environ.get(ENV_VAR_LOG_LEVEL, "INFO").upper()
+    return level
+
+
+logger.setLevel(get_logger_level())
+
+
+def set_log_level(log_level: str):
+    """Global for setting logging level."""
+
+    log_level = log_level.upper()
+    logger.setLevel(log_level.upper())
+    logger.handlers[0].setLevel(log_level)
+
 
 if OCI_RESOURCE_PRINCIPAL_VERSION:
     set_auth("resource_principal")
 
-ODSC_MODEL_COMPARTMENT_OCID = os.environ.get("ODSC_MODEL_COMPARTMENT_OCID")
-if not ODSC_MODEL_COMPARTMENT_OCID:
-    try:
-        ODSC_MODEL_COMPARTMENT_OCID = fetch_service_compartment()
-    except Exception as e:
-        logger.error(
-            f"ODSC_MODEL_COMPARTMENT_OCID environment variable is not set for Aqua, due to {e}."
-        )
+ODSC_MODEL_COMPARTMENT_OCID = (
+    os.environ.get("ODSC_MODEL_COMPARTMENT_OCID") or fetch_service_compartment()
+)

ads/aqua/base.py

@@ -19,7 +19,6 @@ from ads.aqua.utils import (
     get_artifact_path,
     is_valid_ocid,
     load_config,
-    logger,
 )
 from ads.common import oci_client as oc
 from ads.common.auth import default_signer
@@ -164,7 +163,6 @@ class AquaApp:
         tag = Tags.AQUA_FINE_TUNING.value
 
         if not model_version_set_id:
-            tag = Tags.AQUA_FINE_TUNING.value # TODO: Fix this
             try:
                 model_version_set = ModelVersionSet.from_name(
                     name=model_version_set_name,

ads/aqua/cli.py

@@ -3,17 +3,65 @@
 
 # Copyright (c) 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+import os
+import sys
 
+from ads.aqua import (
+    ENV_VAR_LOG_LEVEL,
+    set_log_level,
+    ODSC_MODEL_COMPARTMENT_OCID,
+    logger,
+)
 from ads.aqua.deployment import AquaDeploymentApp
+from ads.aqua.evaluation import AquaEvaluationApp
 from ads.aqua.finetune import AquaFineTuningApp
 from ads.aqua.model import AquaModelApp
-from ads.aqua.evaluation import AquaEvaluationApp
+from ads.config import NB_SESSION_OCID
+from ads.common.utils import LOG_LEVELS
 
 
 class AquaCommand:
-    """Contains the command groups for project Aqua."""
+    """Contains the command groups for project Aqua.
+
+    Acts as an entry point for managing different components of the Aqua
+    project including model management, fine-tuning, deployment, and
+    evaluation.
+    """
 
     model = AquaModelApp
     fine_tuning = AquaFineTuningApp
     deployment = AquaDeploymentApp
     evaluation = AquaEvaluationApp
+
+    def __init__(
+        self,
+        log_level: str = os.environ.get(ENV_VAR_LOG_LEVEL, "ERROR").upper(),
+    ):
+        """
+        Initialize the command line interface settings for the Aqua project.
+
+        FLAGS
+        -----
+        log_level (str):
+            Sets the logging level for the application.
+            Default is retrieved from environment variable `LOG_LEVEL`,
+            or 'ERROR' if not set. Example values include 'DEBUG', 'INFO',
+            'WARNING', 'ERROR', and 'CRITICAL'.
+        """
+        if log_level.upper() not in LOG_LEVELS:
+            logger.error(
+                f"Log level should be one of {LOG_LEVELS}. Setting default to ERROR."
+            )
+            log_level = "ERROR"
+        set_log_level(log_level)
+        # gracefully exit if env var is not set
+        if not ODSC_MODEL_COMPARTMENT_OCID:
+            logger.debug(
+                "ODSC_MODEL_COMPARTMENT_OCID environment variable is not set for Aqua."
+            )
+            if NB_SESSION_OCID:
+                logger.error(
+                    f"Aqua is not available for the notebook session {NB_SESSION_OCID}. For more information, "
+                    f"please refer to the documentation."
+                )
+            sys.exit(1)

ads/aqua/decorator.py

@@ -17,6 +17,7 @@ from oci.exceptions import (
     RequestException,
     ServiceError,
 )
+from tornado.web import HTTPError
 
 from ads.aqua.exception import AquaError
 from ads.aqua.extension.base_handler import AquaAPIhandler
@@ -58,6 +59,7 @@ def handle_exceptions(func):
         except ServiceError as error:
             self.write_error(
                 status_code=error.status or 500,
+                message=error.message,
                 reason=error.message,
                 service_payload=error.args[0] if error.args else None,
                 exc_info=sys.exc_info(),
@@ -91,6 +93,12 @@ def handle_exceptions(func):
                 service_payload=error.service_payload,
                 exc_info=sys.exc_info(),
             )
+        except HTTPError as e:
+            self.write_error(
+                status_code=e.status_code,
+                reason=e.log_message,
+                exc_info=sys.exc_info(),
+            )
         except Exception as ex:
             self.write_error(
                 status_code=500,

ads/aqua/deployment.py

@@ -22,6 +22,9 @@ from ads.aqua.utils import (
     UNKNOWN_DICT,
     get_resource_name,
     get_model_by_reference_paths,
+    get_ocid_substring,
+    AQUA_MODEL_TYPE_SERVICE,
+    AQUA_MODEL_TYPE_CUSTOM,
 )
 from ads.aqua.finetune import FineTuneCustomMetadata
 from ads.aqua.data import AquaResourceIdentifier
@@ -391,40 +394,27 @@ class AquaDeploymentApp(AquaApp):
             .with_runtime(container_runtime)
         ).deploy(wait_for_completion=False)
 
-        if is_fine_tuned_model:
-            # tracks unique deployments that were created in the user compartment
-            self.telemetry.record_event_async(
-                category="aqua/custom/deployment", action="create", detail=model_name
-            )
-            # tracks the shape used for deploying the custom models
-            self.telemetry.record_event_async(
-                category="aqua/custom/deployment/create",
-                action="shape",
-                detail=instance_shape,
-            )
-            # tracks the shape used for deploying the custom models by name
-            self.telemetry.record_event_async(
-                category=f"aqua/custom/{model_name}/deployment/create",
-                action="shape",
-                detail=instance_shape,
-            )
-        else:
-            # tracks unique deployments that were created in the user compartment
-            self.telemetry.record_event_async(
-                category="aqua/service/deployment", action="create", detail=model_name
-            )
-            # tracks the shape used for deploying the service models
-            self.telemetry.record_event_async(
-                category="aqua/service/deployment/create",
-                action="shape",
-                detail=instance_shape,
-            )
-            # tracks the shape used for deploying the service models by name
-            self.telemetry.record_event_async(
-                category=f"aqua/service/{model_name}/deployment/create",
-                action="shape",
-                detail=instance_shape,
-            )
+        model_type = (
+            AQUA_MODEL_TYPE_CUSTOM if is_fine_tuned_model else AQUA_MODEL_TYPE_SERVICE
+        )
+        deployment_id = deployment.dsc_model_deployment.id
+        # we arbitrarily choose last 8 characters of OCID to identify MD in telemetry
+        telemetry_kwargs = {"ocid": get_ocid_substring(deployment_id, key_len=8)}
+
+        # tracks unique deployments that were created in the user compartment
+        self.telemetry.record_event_async(
+            category=f"aqua/{model_type}/deployment",
+            action="create",
+            detail=model_name,
+            **telemetry_kwargs,
+        )
+        # tracks the shape used for deploying the custom or service models by name
+        self.telemetry.record_event_async(
+            category=f"aqua/{model_type}/deployment/create",
+            action="shape",
+            detail=instance_shape,
+            value=model_name,
+        )
 
         return AquaDeployment.from_oci_model_deployment(
             deployment.dsc_model_deployment, self.region
@@ -471,6 +461,19 @@ class AquaDeploymentApp(AquaApp):
                     )
                 )
 
+                # log telemetry if MD is in active or failed state
+                deployment_id = model_deployment.id
+                state = model_deployment.lifecycle_state.upper()
+                if state in ["ACTIVE", "FAILED"]:
+                    # tracks unique deployments that were listed in the user compartment
+                    # we arbitrarily choose last 8 characters of OCID to identify MD in telemetry
+                    self.telemetry.record_event_async(
+                        category=f"aqua/deployment",
+                        action="list",
+                        detail=get_ocid_substring(deployment_id, key_len=8),
+                        value=state,
+                    )
+
         # tracks number of times deployment listing was called
         self.telemetry.record_event_async(category="aqua/deployment", action="list")
 

ads/aqua/evaluation.py

@@ -38,6 +38,7 @@ from ads.aqua.utils import (
     JOB_INFRASTRUCTURE_TYPE_DEFAULT_NETWORKING,
     NB_SESSION_IDENTIFIER,
     UNKNOWN,
+    extract_id_and_name_from_tag,
     fire_and_forget,
     get_container_image,
     is_valid_ocid,
@@ -78,7 +79,7 @@ class EvaluationJobExitCode(Enum):
     SUCCESS = 0
     COMMON_ERROR = 1
 
-    # Configuration-related issues
+    # Configuration-related issues 10-19
     INVALID_EVALUATION_CONFIG = 10
     EVALUATION_CONFIG_NOT_PROVIDED = 11
     INVALID_OUTPUT_DIR = 12
@@ -87,7 +88,7 @@ class EvaluationJobExitCode(Enum):
     INVALID_TARGET_EVALUATION_ID = 15
     INVALID_EVALUATION_CONFIG_VALIDATION = 16
 
-    # Evaluation process issues
+    # Evaluation process issues 20-39
     OUTPUT_DIR_NOT_FOUND = 20
     INVALID_INPUT_DATASET = 21
     INPUT_DATA_NOT_FOUND = 22
@@ -100,6 +101,7 @@ class EvaluationJobExitCode(Enum):
     MODEL_INFERENCE_WRONG_RESPONSE_FORMAT = 29
     UNSUPPORTED_METRICS = 30
     METRIC_CALCULATION_FAILURE = 31
+    EVALUATION_MODEL_CATALOG_RECORD_CREATION_FAILED = 32
 
 
 EVALUATION_JOB_EXIT_CODE_MESSAGE = {
@@ -124,6 +126,11 @@ EVALUATION_JOB_EXIT_CODE_MESSAGE = {
     EvaluationJobExitCode.MODEL_INFERENCE_WRONG_RESPONSE_FORMAT.value: "Evaluation encountered unsupported, or unexpected model output, verify the target evaluation model is compatible and produces the correct format.",
     EvaluationJobExitCode.UNSUPPORTED_METRICS.value: "None of the provided metrics are supported by the framework.",
     EvaluationJobExitCode.METRIC_CALCULATION_FAILURE.value: "All attempted metric calculations were unsuccessful. Please review the metric configurations and input data.",
+    EvaluationJobExitCode.EVALUATION_MODEL_CATALOG_RECORD_CREATION_FAILED.value: (
+        "Failed to create a Model Catalog record for the evaluation. "
+        "This could be due to missing required permissions. "
+        "Please check the log for more information."
+    ),
 }
 
 
@@ -311,6 +318,8 @@ class CreateAquaEvaluationDetails(DataClassSerializable):
         The log id for the evaluation job infrastructure.
     metrics: (list, optional). Defaults to `None`.
         The metrics for the evaluation.
+    force_overwrite: (bool, optional). Defaults to `False`.
+        Whether to force overwrite the existing file in object storage.
     """
 
     evaluation_source_id: str
@@ -331,6 +340,7 @@ class CreateAquaEvaluationDetails(DataClassSerializable):
     log_group_id: Optional[str] = None
     log_id: Optional[str] = None
     metrics: Optional[List] = None
+    force_overwrite: Optional[bool] = False
 
 
 class AquaEvaluationApp(AquaApp):
@@ -434,12 +444,12 @@ class AquaEvaluationApp(AquaApp):
                     src_uri=evaluation_dataset_path,
                     dst_uri=dst_uri,
                     auth=default_signer(),
-                    force_overwrite=False,
+                    force_overwrite=create_aqua_evaluation_details.force_overwrite,
                 )
             except FileExistsError:
                 raise AquaFileExistsError(
                     f"Dataset {dataset_file} already exists in {create_aqua_evaluation_details.report_path}. "
-                    "Please use a new dataset file name or report path."
+                    "Please use a new dataset file name, report path or set `force_overwrite` as True."
                 )
             logger.debug(
                 f"Uploaded local file {evaluation_dataset_path} to object storage {dst_uri}."
@@ -673,11 +683,19 @@ class AquaEvaluationApp(AquaApp):
             ),
         )
 
+        # tracks shapes used in evaluation that were created for the given evaluation source
+        self.telemetry.record_event_async(
+            category="aqua/evaluation/create",
+            action="shape",
+            detail=create_aqua_evaluation_details.shape_name,
+            value=self._get_service_model_name(evaluation_source),
+        )
+
         # tracks unique evaluation that were created for the given evaluation source
         self.telemetry.record_event_async(
             category="aqua/evaluation",
             action="create",
-            detail=evaluation_source.display_name,
+            detail=self._get_service_model_name(evaluation_source),
         )
 
         return AquaEvaluationSummary(
@@ -768,6 +786,34 @@ class AquaEvaluationApp(AquaApp):
 
         return runtime
 
+    @staticmethod
+    def _get_service_model_name(
+        source: Union[ModelDeployment, DataScienceModel]
+    ) -> str:
+        """Gets the service model name from source. If it's ModelDeployment, needs to check
+        if its model has been fine tuned or not.
+
+        Parameters
+        ----------
+        source: Union[ModelDeployment, DataScienceModel]
+            An instance of either ModelDeployment or DataScienceModel
+
+        Returns
+        -------
+        str:
+            The service model name of source.
+        """
+        if isinstance(source, ModelDeployment):
+            fine_tuned_model_tag = source.freeform_tags.get(
+                Tags.AQUA_FINE_TUNED_MODEL_TAG.value, UNKNOWN
+            )
+            if not fine_tuned_model_tag:
+                return source.freeform_tags.get(Tags.AQUA_MODEL_NAME_TAG.value)
+            else:
+                return extract_id_and_name_from_tag(fine_tuned_model_tag)[1]
+
+        return source.display_name
+
     @staticmethod
     def _get_evaluation_container(source_id: str) -> str:
         # todo: use the source, identify if it is a model or a deployment. If latter, then fetch the base model id
@@ -824,13 +870,13 @@ class AquaEvaluationApp(AquaApp):
         logger.info(f"Fetching evaluation: {eval_id} details ...")
 
         resource = utils.query_resource(eval_id)
-        model_provenance = self.ds_client.get_model_provenance(eval_id).data
-
         if not resource:
             raise AquaRuntimeError(
                 f"Failed to retrieve evalution {eval_id}."
                 "Please check if the OCID is correct."
             )
+        model_provenance = self.ds_client.get_model_provenance(eval_id).data
+
         jobrun_id = model_provenance.training_id
         job_run_details = self._fetch_jobrun(
             resource, use_rqs=False, jobrun_id=jobrun_id
@@ -849,13 +895,17 @@ class AquaEvaluationApp(AquaApp):
             loggroup_id = ""
 
         loggroup_url = get_log_links(region=self.region, log_group_id=loggroup_id)
-        log_url = get_log_links(
-            region=self.region,
-            log_group_id=loggroup_id,
-            log_id=log_id,
-            compartment_id=job_run_details.compartment_id,
-            source_id=jobrun_id
-        ) if job_run_details else ""
+        log_url = (
+            get_log_links(
+                region=self.region,
+                log_group_id=loggroup_id,
+                log_id=log_id,
+                compartment_id=job_run_details.compartment_id,
+                source_id=jobrun_id,
+            )
+            if job_run_details
+            else ""
+        )
 
         log_name = None
         loggroup_name = None
@@ -916,6 +966,7 @@ class AquaEvaluationApp(AquaApp):
         List[AquaEvaluationSummary]:
             The list of the `ads.aqua.evalution.AquaEvaluationSummary`.
         """
+        compartment_id = compartment_id or COMPARTMENT_OCID
         logger.info(f"Fetching evaluations from compartment {compartment_id}.")
         models = utils.query_resources(
             compartment_id=compartment_id,
@@ -931,7 +982,6 @@ class AquaEvaluationApp(AquaApp):
         evaluations = []
         async_tasks = []
         for model in models:
-
             if model.identifier in self._eval_cache.keys():
                 logger.debug(f"Retrieving evaluation {model.identifier} from cache.")
                 evaluations.append(self._eval_cache.get(model.identifier))
@@ -962,7 +1012,7 @@ class AquaEvaluationApp(AquaApp):
                         self._process_evaluation_summary(model=model, jobrun=jobrun)
                     )
                 except Exception as exc:
-                    logger.error(
+                    logger.debug(
                         f"Processing evaluation: {model.identifier} generated an exception: {exc}"
                     )
                     evaluations.append(
@@ -1007,7 +1057,7 @@ class AquaEvaluationApp(AquaApp):
             return True if response.status == 200 else False
         except oci.exceptions.ServiceError as ex:
             if ex.status == 404:
-                logger.info("Evaluation artifact not found.")
+                logger.debug(f"Evaluation artifact not found for {model.identifier}.")
                 return False
 
     @telemetry(entry_point="plugin=evaluation&action=get_status", name="aqua")
@@ -1025,14 +1075,14 @@ class AquaEvaluationApp(AquaApp):
         """
         eval = utils.query_resource(eval_id)
 
-        # TODO: add job_run_id as input param to skip the query below
-        model_provenance = self.ds_client.get_model_provenance(eval_id).data
-
         if not eval:
             raise AquaRuntimeError(
                 f"Failed to retrieve evalution {eval_id}."
                 "Please check if the OCID is correct."
             )
+
+        model_provenance = self.ds_client.get_model_provenance(eval_id).data
+
         jobrun_id = model_provenance.training_id
         job_run_details = self._fetch_jobrun(eval, use_rqs=False, jobrun_id=jobrun_id)
 
@@ -1049,13 +1099,17 @@ class AquaEvaluationApp(AquaApp):
             loggroup_id = ""
 
         loggroup_url = get_log_links(region=self.region, log_group_id=loggroup_id)
-        log_url = get_log_links(
-            region=self.region,
-            log_group_id=loggroup_id,
-            log_id=log_id,
-            compartment_id=job_run_details.compartment_id,
-            source_id=jobrun_id
-        ) if job_run_details else ""
+        log_url = (
+            get_log_links(
+                region=self.region,
+                log_group_id=loggroup_id,
+                log_id=log_id,
+                compartment_id=job_run_details.compartment_id,
+                source_id=jobrun_id,
+            )
+            if job_run_details
+            else ""
+        )
 
         return dict(
             id=eval_id,
@@ -1100,6 +1154,19 @@ class AquaEvaluationApp(AquaApp):
                 ),
                 "args": {},
             },
+            {
+                "use_case": ["text_generation"],
+                "key": "bleu",
+                "name": "bleu",
+                "description": (
+                    "BLEU (Bilingual Evaluation Understudy) is an algorithm for evaluating the "
+                    "quality of text which has been machine-translated from one natural language to another. "
+                    "Quality is considered to be the correspondence between a machine's output and that of a "
+                    "human: 'the closer a machine translation is to a professional human translation, "
+                    "the better it is'."
+                ),
+                "args": {},
+            },
         ]
 
     @telemetry(entry_point="plugin=evaluation&action=load_metrics", name="aqua")
@@ -1265,7 +1332,10 @@ class AquaEvaluationApp(AquaApp):
             raise AquaRuntimeError(
                 f"Failed to get evaluation details for model {eval_id}"
             )
-        job_run_id = model.provenance_metadata.training_id
+
+        job_run_id = (
+            model.provenance_metadata.training_id if model.provenance_metadata else None
+        )
         if not job_run_id:
             raise AquaMissingKeyError(
                 "Model provenance is missing job run training_id key"
@@ -1328,7 +1398,7 @@ class AquaEvaluationApp(AquaApp):
             job_id = model.custom_metadata_list.get(
                 EvaluationCustomMetadata.EVALUATION_JOB_ID.value
             ).value
-        except ValueError:
+        except Exception:
             raise AquaMissingKeyError(
                 f"Custom metadata is missing {EvaluationCustomMetadata.EVALUATION_JOB_ID.value} key"
             )
@@ -1360,7 +1430,7 @@ class AquaEvaluationApp(AquaApp):
             )
 
     def load_evaluation_config(self, eval_id):
-        # TODO
+        """Loads evaluation config."""
         return {
             "model_params": {
                 "max_tokens": 500,
@@ -1533,25 +1603,12 @@ class AquaEvaluationApp(AquaApp):
                 ),
             )
         except Exception as e:
-            logger.error(
-                f"Failed to construct AquaResourceIdentifier from given id=`{id}`, and name=`{name}`, {str(e)}"
+            logger.debug(
+                f"Failed to construct AquaResourceIdentifier from given id=`{id}`, and name=`{name}`. "
+                f"DEBUG INFO: {str(e)}"
             )
             return AquaResourceIdentifier()
 
-    def _get_jobrun(
-        self, model: oci.resource_search.models.ResourceSummary, mapping: dict = {}
-    ) -> Union[
-        oci.resource_search.models.ResourceSummary, oci.data_science.models.JobRun
-    ]:
-        jobrun_id = self._get_attribute_from_model_metadata(
-            model, EvaluationCustomMetadata.EVALUATION_JOB_RUN_ID.value
-        )
-        job_run = mapping.get(jobrun_id)
-
-        if not job_run:
-            job_run = self._fetch_jobrun(model, use_rqs=True, jobrun_id=jobrun_id)
-        return job_run
-
     def _fetch_jobrun(
         self,
         resource: oci.resource_search.models.ResourceSummary,
@@ -1594,7 +1651,7 @@ class AquaEvaluationApp(AquaApp):
             )
             if not params.get(EvaluationConfig.PARAMS):
                 raise AquaMissingKeyError(
-                    "model parameters have not been saved in correct format in model taxonomy.",
+                    "model parameters have not been saved in correct format in model taxonomy. ",
                     service_payload={"params": params},
                 )
             # TODO: validate the format of parameters.
@@ -1626,7 +1683,7 @@ class AquaEvaluationApp(AquaApp):
 
         except Exception as e:
             logger.debug(
-                f"Failed to get job details from job_run_details: {job_run_details}"
+                f"Failed to get job details from job_run_details: {job_run_details} "
                 f"DEBUG INFO:{str(e)}"
             )
             return AquaResourceIdentifier()
@@ -1728,7 +1785,7 @@ class AquaEvaluationApp(AquaApp):
         Examples
         --------
         >>> _extract_job_lifecycle_details("Job run artifact execution failed with exit code 16")
-        'The evaluation configuration is invalid due to content validation errors.'
+        'Validation errors in the evaluation config. Exit code: 16.'
 
         >>> _extract_job_lifecycle_details("Job completed successfully.")
         'Job completed successfully.'