oracle-ads 2.11.9__py3-none-any.whl → 2.11.10__py3-none-any.whl

ads/aqua/{evaluation.py → evaluation/evaluation.py}

@@ -8,12 +8,11 @@ import os
 import re
 import tempfile
 from concurrent.futures import ThreadPoolExecutor, as_completed
-from dataclasses import asdict, dataclass, field
+from dataclasses import asdict
 from datetime import datetime, timedelta
-from enum import Enum
 from pathlib import Path
 from threading import Lock
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Union
 
 import oci
 from cachetools import TTLCache
@@ -24,29 +23,40 @@ from oci.data_science.models import (
     UpdateModelProvenanceDetails,
 )
 
-from ads.aqua import logger, utils
-from ads.aqua.base import AquaApp
-from ads.aqua.data import Tags
-from ads.aqua.exception import (
+from ads.aqua import logger
+from ads.aqua.app import AquaApp
+from ads.aqua.common import utils
+from ads.aqua.common.enums import (
+    DataScienceResource,
+    Resource,
+    RqsAdditionalDetails,
+    Tags,
+)
+from ads.aqua.common.errors import (
     AquaFileExistsError,
     AquaFileNotFoundError,
     AquaMissingKeyError,
     AquaRuntimeError,
     AquaValueError,
 )
-from ads.aqua.utils import (
-    JOB_INFRASTRUCTURE_TYPE_DEFAULT_NETWORKING,
-    NB_SESSION_IDENTIFIER,
-    UNKNOWN,
+from ads.aqua.common.utils import (
     extract_id_and_name_from_tag,
     fire_and_forget,
     get_container_image,
     is_valid_ocid,
     upload_local_to_os,
 )
+from ads.aqua.constants import (
+    JOB_INFRASTRUCTURE_TYPE_DEFAULT_NETWORKING,
+    NB_SESSION_IDENTIFIER,
+    UNKNOWN,
+    CONSOLE_LINK_RESOURCE_TYPE_MAPPING,
+)
+from ads.aqua.evaluation.constants import *
+from ads.aqua.evaluation.entities import *
+from ads.aqua.evaluation.errors import *
 from ads.common.auth import default_signer
 from ads.common.object_storage_details import ObjectStorageDetails
-from ads.common.serializer import DataClassSerializable
 from ads.common.utils import get_console_link, get_files, get_log_links, upload_to_os
 from ads.config import (
     AQUA_JOB_SUBNET_ID,
@@ -69,279 +79,6 @@ from ads.model.model_metadata import (
 from ads.model.model_version_set import ModelVersionSet
 from ads.telemetry import telemetry
 
-EVAL_TERMINATION_STATE = [
-    JobRun.LIFECYCLE_STATE_SUCCEEDED,
-    JobRun.LIFECYCLE_STATE_FAILED,
-]
-
-
-class EvaluationJobExitCode(Enum):
-    SUCCESS = 0
-    COMMON_ERROR = 1
-
-    # Configuration-related issues 10-19
-    INVALID_EVALUATION_CONFIG = 10
-    EVALUATION_CONFIG_NOT_PROVIDED = 11
-    INVALID_OUTPUT_DIR = 12
-    INVALID_INPUT_DATASET_PATH = 13
-    INVALID_EVALUATION_ID = 14
-    INVALID_TARGET_EVALUATION_ID = 15
-    INVALID_EVALUATION_CONFIG_VALIDATION = 16
-
-    # Evaluation process issues 20-39
-    OUTPUT_DIR_NOT_FOUND = 20
-    INVALID_INPUT_DATASET = 21
-    INPUT_DATA_NOT_FOUND = 22
-    EVALUATION_ID_NOT_FOUND = 23
-    EVALUATION_ALREADY_PERFORMED = 24
-    EVALUATION_TARGET_NOT_FOUND = 25
-    NO_SUCCESS_INFERENCE_RESULT = 26
-    COMPUTE_EVALUATION_ERROR = 27
-    EVALUATION_REPORT_ERROR = 28
-    MODEL_INFERENCE_WRONG_RESPONSE_FORMAT = 29
-    UNSUPPORTED_METRICS = 30
-    METRIC_CALCULATION_FAILURE = 31
-    EVALUATION_MODEL_CATALOG_RECORD_CREATION_FAILED = 32
-
-
-EVALUATION_JOB_EXIT_CODE_MESSAGE = {
-    EvaluationJobExitCode.SUCCESS.value: "Success",
-    EvaluationJobExitCode.COMMON_ERROR.value: "An error occurred during the evaluation, please check the log for more information.",
-    EvaluationJobExitCode.INVALID_EVALUATION_CONFIG.value: "The provided evaluation configuration was not in the correct format, supported formats are YAML or JSON.",
-    EvaluationJobExitCode.EVALUATION_CONFIG_NOT_PROVIDED.value: "The evaluation config was not provided.",
-    EvaluationJobExitCode.INVALID_OUTPUT_DIR.value: "The specified output directory path is invalid.",
-    EvaluationJobExitCode.INVALID_INPUT_DATASET_PATH.value: "Dataset path is invalid.",
-    EvaluationJobExitCode.INVALID_EVALUATION_ID.value: "Evaluation ID was not found in the Model Catalog.",
-    EvaluationJobExitCode.INVALID_TARGET_EVALUATION_ID.value: "Target evaluation ID was not found in the Model Deployment.",
-    EvaluationJobExitCode.INVALID_EVALUATION_CONFIG_VALIDATION.value: "Validation errors in the evaluation config.",
-    EvaluationJobExitCode.OUTPUT_DIR_NOT_FOUND.value: "Destination folder does not exist or cannot be used for writing, verify the folder's existence and permissions.",
-    EvaluationJobExitCode.INVALID_INPUT_DATASET.value: "Input dataset is in an invalid format, ensure the dataset is in jsonl format and that includes the required columns: 'prompt', 'completion' (optional 'category').",
-    EvaluationJobExitCode.INPUT_DATA_NOT_FOUND.value: "Input data file does not exist or cannot be use for reading, verify the file's existence and permissions.",
-    EvaluationJobExitCode.EVALUATION_ID_NOT_FOUND.value: "Evaluation ID does not match any resource in the Model Catalog, or access may be blocked by policies.",
-    EvaluationJobExitCode.EVALUATION_ALREADY_PERFORMED.value: "Evaluation already has an attached artifact, indicating that the evaluation has already been performed.",
-    EvaluationJobExitCode.EVALUATION_TARGET_NOT_FOUND.value: "Target evaluation ID does not match any resources in Model Deployment.",
-    EvaluationJobExitCode.NO_SUCCESS_INFERENCE_RESULT.value: "Inference process completed without producing expected outcome, verify the model parameters and config.",
-    EvaluationJobExitCode.COMPUTE_EVALUATION_ERROR.value: "Evaluation process encountered an issue while calculating metrics.",
-    EvaluationJobExitCode.EVALUATION_REPORT_ERROR.value: "Failed to save the evaluation report due to an error. Ensure the evaluation model is currently active and the specified path for the output report is valid and accessible. Verify these conditions and reinitiate the evaluation process.",
-    EvaluationJobExitCode.MODEL_INFERENCE_WRONG_RESPONSE_FORMAT.value: "Evaluation encountered unsupported, or unexpected model output, verify the target evaluation model is compatible and produces the correct format.",
-    EvaluationJobExitCode.UNSUPPORTED_METRICS.value: "None of the provided metrics are supported by the framework.",
-    EvaluationJobExitCode.METRIC_CALCULATION_FAILURE.value: "All attempted metric calculations were unsuccessful. Please review the metric configurations and input data.",
-    EvaluationJobExitCode.EVALUATION_MODEL_CATALOG_RECORD_CREATION_FAILED.value: (
-        "Failed to create a Model Catalog record for the evaluation. "
-        "This could be due to missing required permissions. "
-        "Please check the log for more information."
-    ),
-}
-
-
-class Resource(Enum):
-    JOB = "jobs"
-    MODEL = "models"
-    MODEL_DEPLOYMENT = "modeldeployments"
-    MODEL_VERSION_SET = "model-version-sets"
-
-
-class DataScienceResource(Enum):
-    MODEL_DEPLOYMENT = "datasciencemodeldeployment"
-    MODEL = "datasciencemodel"
-
-
-class EvaluationCustomMetadata(Enum):
-    EVALUATION_SOURCE = "evaluation_source"
-    EVALUATION_JOB_ID = "evaluation_job_id"
-    EVALUATION_JOB_RUN_ID = "evaluation_job_run_id"
-    EVALUATION_OUTPUT_PATH = "evaluation_output_path"
-    EVALUATION_SOURCE_NAME = "evaluation_source_name"
-    EVALUATION_ERROR = "aqua_evaluate_error"
-
-
-class EvaluationModelTags(Enum):
-    AQUA_EVALUATION = "aqua_evaluation"
-
-
-class EvaluationJobTags(Enum):
-    AQUA_EVALUATION = "aqua_evaluation"
-    EVALUATION_MODEL_ID = "evaluation_model_id"
-
-
-class EvaluationUploadStatus(Enum):
-    IN_PROGRESS = "IN_PROGRESS"
-    COMPLETED = "COMPLETED"
-
-
-@dataclass(repr=False)
-class AquaResourceIdentifier(DataClassSerializable):
-    id: str = ""
-    name: str = ""
-    url: str = ""
-
-
-@dataclass(repr=False)
-class AquaEvalReport(DataClassSerializable):
-    evaluation_id: str = ""
-    content: str = ""
-
-
-@dataclass(repr=False)
-class ModelParams(DataClassSerializable):
-    max_tokens: str = ""
-    top_p: str = ""
-    top_k: str = ""
-    temperature: str = ""
-    presence_penalty: Optional[float] = 0.0
-    frequency_penalty: Optional[float] = 0.0
-    stop: Optional[Union[str, List[str]]] = field(default_factory=list)
-
-
-@dataclass(repr=False)
-class AquaEvalParams(ModelParams, DataClassSerializable):
-    shape: str = ""
-    dataset_path: str = ""
-    report_path: str = ""
-
-
-@dataclass(repr=False)
-class AquaEvalMetric(DataClassSerializable):
-    key: str
-    name: str
-    description: str = ""
-
-
-@dataclass(repr=False)
-class AquaEvalMetricSummary(DataClassSerializable):
-    metric: str = ""
-    score: str = ""
-    grade: str = ""
-
-
-@dataclass(repr=False)
-class AquaEvalMetrics(DataClassSerializable):
-    id: str
-    report: str
-    metric_results: List[AquaEvalMetric] = field(default_factory=list)
-    metric_summary_result: List[AquaEvalMetricSummary] = field(default_factory=list)
-
-
-@dataclass(repr=False)
-class AquaEvaluationCommands(DataClassSerializable):
-    evaluation_id: str
-    evaluation_target_id: str
-    input_data: dict
-    metrics: list
-    output_dir: str
-    params: dict
-
-
-@dataclass(repr=False)
-class AquaEvaluationSummary(DataClassSerializable):
-    """Represents a summary of Aqua evalution."""
-
-    id: str
-    name: str
-    console_url: str
-    lifecycle_state: str
-    lifecycle_details: str
-    time_created: str
-    tags: dict
-    experiment: AquaResourceIdentifier = field(default_factory=AquaResourceIdentifier)
-    source: AquaResourceIdentifier = field(default_factory=AquaResourceIdentifier)
-    job: AquaResourceIdentifier = field(default_factory=AquaResourceIdentifier)
-    parameters: AquaEvalParams = field(default_factory=AquaEvalParams)
-
-
-@dataclass(repr=False)
-class AquaEvaluationDetail(AquaEvaluationSummary, DataClassSerializable):
-    """Represents a details of Aqua evalution."""
-
-    log_group: AquaResourceIdentifier = field(default_factory=AquaResourceIdentifier)
-    log: AquaResourceIdentifier = field(default_factory=AquaResourceIdentifier)
-    introspection: dict = field(default_factory=dict)
-
-
-class RqsAdditionalDetails:
-    METADATA = "metadata"
-    CREATED_BY = "createdBy"
-    DESCRIPTION = "description"
-    MODEL_VERSION_SET_ID = "modelVersionSetId"
-    MODEL_VERSION_SET_NAME = "modelVersionSetName"
-    PROJECT_ID = "projectId"
-    VERSION_LABEL = "versionLabel"
-
-
-class EvaluationConfig:
-    PARAMS = "model_params"
-
-
-@dataclass(repr=False)
-class CreateAquaEvaluationDetails(DataClassSerializable):
-    """Dataclass to create aqua model evaluation.
-
-    Fields
-    ------
-    evaluation_source_id: str
-        The evaluation source id. Must be either model or model deployment ocid.
-    evaluation_name: str
-        The name for evaluation.
-    dataset_path: str
-        The dataset path for the evaluation. Could be either a local path from notebook session
-        or an object storage path.
-    report_path: str
-        The report path for the evaluation. Must be an object storage path.
-    model_parameters: dict
-        The parameters for the evaluation.
-    shape_name: str
-        The shape name for the evaluation job infrastructure.
-    memory_in_gbs: float
-        The memory in gbs for the shape selected.
-    ocpus: float
-        The ocpu count for the shape selected.
-    block_storage_size: int
-        The storage for the evaluation job infrastructure.
-    compartment_id: (str, optional). Defaults to `None`.
-        The compartment id for the evaluation.
-    project_id: (str, optional). Defaults to `None`.
-        The project id for the evaluation.
-    evaluation_description: (str, optional). Defaults to `None`.
-        The description for evaluation
-    experiment_id: (str, optional). Defaults to `None`.
-        The evaluation model version set id. If provided,
-        evaluation model will be associated with it.
-    experiment_name: (str, optional). Defaults to `None`.
-        The evaluation model version set name. If provided,
-        the model version set with the same name will be used if exists,
-        otherwise a new model version set will be created with the name.
-    experiment_description: (str, optional). Defaults to `None`.
-        The description for the evaluation model version set.
-    log_group_id: (str, optional). Defaults to `None`.
-        The log group id for the evaluation job infrastructure.
-    log_id: (str, optional). Defaults to `None`.
-        The log id for the evaluation job infrastructure.
-    metrics: (list, optional). Defaults to `None`.
-        The metrics for the evaluation.
-    force_overwrite: (bool, optional). Defaults to `False`.
-        Whether to force overwrite the existing file in object storage.
-    """
-
-    evaluation_source_id: str
-    evaluation_name: str
-    dataset_path: str
-    report_path: str
-    model_parameters: dict
-    shape_name: str
-    block_storage_size: int
-    compartment_id: Optional[str] = None
-    project_id: Optional[str] = None
-    evaluation_description: Optional[str] = None
-    experiment_id: Optional[str] = None
-    experiment_name: Optional[str] = None
-    experiment_description: Optional[str] = None
-    memory_in_gbs: Optional[float] = None
-    ocpus: Optional[float] = None
-    log_group_id: Optional[str] = None
-    log_id: Optional[str] = None
-    metrics: Optional[List] = None
-    force_overwrite: Optional[bool] = False
-
 
 class AquaEvaluationApp(AquaApp):
     """Provides a suite of APIs to interact with Aqua evaluations within the
@@ -367,6 +104,9 @@ class AquaEvaluationApp(AquaApp):
     _report_cache = TTLCache(maxsize=10, ttl=timedelta(hours=5), timer=datetime.now)
     _metrics_cache = TTLCache(maxsize=10, ttl=timedelta(hours=5), timer=datetime.now)
     _eval_cache = TTLCache(maxsize=200, ttl=timedelta(hours=10), timer=datetime.now)
+    _deletion_cache = TTLCache(
+        maxsize=10, ttl=timedelta(minutes=10), timer=datetime.now
+    )
     _cache_lock = Lock()
 
     @telemetry(entry_point="plugin=evaluation&action=create", name="aqua")
@@ -408,14 +148,14 @@ class AquaEvaluationApp(AquaApp):
 
         evaluation_source = None
         if (
-            DataScienceResource.MODEL_DEPLOYMENT.value
+            DataScienceResource.MODEL_DEPLOYMENT
             in create_aqua_evaluation_details.evaluation_source_id
         ):
             evaluation_source = ModelDeployment.from_id(
                 create_aqua_evaluation_details.evaluation_source_id
             )
         elif (
-            DataScienceResource.MODEL.value
+            DataScienceResource.MODEL
             in create_aqua_evaluation_details.evaluation_source_id
         ):
             evaluation_source = DataScienceModel.from_id(
@@ -500,11 +240,9 @@ class AquaEvaluationApp(AquaApp):
                     name=experiment_model_version_set_name,
                     compartment_id=target_compartment,
                 )
-                if not utils._is_valid_mvs(
-                    model_version_set, Tags.AQUA_EVALUATION.value
-                ):
+                if not utils._is_valid_mvs(model_version_set, Tags.AQUA_EVALUATION):
                     raise AquaValueError(
-                        f"Invalid experiment name. Please provide an experiment with `{Tags.AQUA_EVALUATION.value}` in tags."
+                        f"Invalid experiment name. Please provide an experiment with `{Tags.AQUA_EVALUATION}` in tags."
                     )
             except:
                 logger.debug(
@@ -513,7 +251,7 @@ class AquaEvaluationApp(AquaApp):
                 )
 
                 evaluation_mvs_freeform_tags = {
-                    Tags.AQUA_EVALUATION.value: Tags.AQUA_EVALUATION.value,
+                    Tags.AQUA_EVALUATION: Tags.AQUA_EVALUATION,
                 }
 
                 model_version_set = (
@@ -534,23 +272,23 @@ class AquaEvaluationApp(AquaApp):
             experiment_model_version_set_id = model_version_set.id
         else:
             model_version_set = ModelVersionSet.from_id(experiment_model_version_set_id)
-            if not utils._is_valid_mvs(model_version_set, Tags.AQUA_EVALUATION.value):
+            if not utils._is_valid_mvs(model_version_set, Tags.AQUA_EVALUATION):
                 raise AquaValueError(
-                    f"Invalid experiment id. Please provide an experiment with `{Tags.AQUA_EVALUATION.value}` in tags."
+                    f"Invalid experiment id. Please provide an experiment with `{Tags.AQUA_EVALUATION}` in tags."
                 )
             experiment_model_version_set_name = model_version_set.name
 
         evaluation_model_custom_metadata = ModelCustomMetadata()
         evaluation_model_custom_metadata.add(
-            key=EvaluationCustomMetadata.EVALUATION_SOURCE.value,
+            key=EvaluationCustomMetadata.EVALUATION_SOURCE,
             value=create_aqua_evaluation_details.evaluation_source_id,
         )
         evaluation_model_custom_metadata.add(
-            key=EvaluationCustomMetadata.EVALUATION_OUTPUT_PATH.value,
+            key=EvaluationCustomMetadata.EVALUATION_OUTPUT_PATH,
             value=create_aqua_evaluation_details.report_path,
         )
         evaluation_model_custom_metadata.add(
-            key=EvaluationCustomMetadata.EVALUATION_SOURCE_NAME.value,
+            key=EvaluationCustomMetadata.EVALUATION_SOURCE_NAME,
             value=evaluation_source.display_name,
         )
 
@@ -588,8 +326,8 @@ class AquaEvaluationApp(AquaApp):
         # TODO: validate metrics if it's provided
 
         evaluation_job_freeform_tags = {
-            EvaluationJobTags.AQUA_EVALUATION.value: EvaluationJobTags.AQUA_EVALUATION.value,
-            EvaluationJobTags.EVALUATION_MODEL_ID.value: evaluation_model.id,
+            Tags.AQUA_EVALUATION: Tags.AQUA_EVALUATION,
+            Tags.AQUA_EVALUATION_MODEL_ID: evaluation_model.id,
         }
 
         evaluation_job = Job(name=evaluation_model.display_name).with_infrastructure(
@@ -654,11 +392,11 @@ class AquaEvaluationApp(AquaApp):
         )
 
         evaluation_model_custom_metadata.add(
-            key=EvaluationCustomMetadata.EVALUATION_JOB_ID.value,
+            key=EvaluationCustomMetadata.EVALUATION_JOB_ID,
             value=evaluation_job.id,
         )
         evaluation_model_custom_metadata.add(
-            key=EvaluationCustomMetadata.EVALUATION_JOB_RUN_ID.value,
+            key=EvaluationCustomMetadata.EVALUATION_JOB_RUN_ID,
             value=evaluation_job_run.id,
         )
         updated_custom_metadata_list = [
@@ -671,7 +409,7 @@ class AquaEvaluationApp(AquaApp):
             update_model_details=UpdateModelDetails(
                 custom_metadata_list=updated_custom_metadata_list,
                 freeform_tags={
-                    EvaluationModelTags.AQUA_EVALUATION.value: EvaluationModelTags.AQUA_EVALUATION.value,
+                    Tags.AQUA_EVALUATION: Tags.AQUA_EVALUATION,
                 },
             ),
         )
@@ -702,7 +440,7 @@ class AquaEvaluationApp(AquaApp):
             id=evaluation_model.id,
             name=evaluation_model.display_name,
             console_url=get_console_link(
-                resource=Resource.MODEL.value,
+                resource=Resource.MODEL,
                 ocid=evaluation_model.id,
                 region=self.region,
             ),
@@ -713,7 +451,7 @@ class AquaEvaluationApp(AquaApp):
                 id=experiment_model_version_set_id,
                 name=experiment_model_version_set_name,
                 url=get_console_link(
-                    resource=Resource.MODEL_VERSION_SET.value,
+                    resource=Resource.MODEL_VERSION_SET,
                     ocid=experiment_model_version_set_id,
                     region=self.region,
                 ),
@@ -723,10 +461,10 @@ class AquaEvaluationApp(AquaApp):
                 name=evaluation_source.display_name,
                 url=get_console_link(
                     resource=(
-                        Resource.MODEL_DEPLOYMENT.value
-                        if DataScienceResource.MODEL_DEPLOYMENT.value
+                        Resource.MODEL_DEPLOYMENT
+                        if DataScienceResource.MODEL_DEPLOYMENT
                         in create_aqua_evaluation_details.evaluation_source_id
-                        else Resource.MODEL.value
+                        else Resource.MODEL
                     ),
                     ocid=create_aqua_evaluation_details.evaluation_source_id,
                     region=self.region,
@@ -736,13 +474,13 @@ class AquaEvaluationApp(AquaApp):
                 id=evaluation_job.id,
                 name=evaluation_job.name,
                 url=get_console_link(
-                    resource=Resource.JOB.value,
+                    resource=Resource.JOB,
                     ocid=evaluation_job.id,
                     region=self.region,
                 ),
             ),
             tags=dict(
-                aqua_evaluation=EvaluationModelTags.AQUA_EVALUATION.value,
+                aqua_evaluation=Tags.AQUA_EVALUATION,
                 evaluation_job_id=evaluation_job.id,
                 evaluation_source=create_aqua_evaluation_details.evaluation_source_id,
                 evaluation_experiment_id=experiment_model_version_set_id,
@@ -805,10 +543,10 @@ class AquaEvaluationApp(AquaApp):
         """
         if isinstance(source, ModelDeployment):
             fine_tuned_model_tag = source.freeform_tags.get(
-                Tags.AQUA_FINE_TUNED_MODEL_TAG.value, UNKNOWN
+                Tags.AQUA_FINE_TUNED_MODEL_TAG, UNKNOWN
             )
             if not fine_tuned_model_tag:
-                return source.freeform_tags.get(Tags.AQUA_MODEL_NAME_TAG.value)
+                return source.freeform_tags.get(Tags.AQUA_MODEL_NAME_TAG)
             else:
                 return extract_id_and_name_from_tag(fine_tuned_model_tag)[1]
 
@@ -971,12 +709,10 @@ class AquaEvaluationApp(AquaApp):
         models = utils.query_resources(
             compartment_id=compartment_id,
             resource_type="datasciencemodel",
-            tag_list=[EvaluationModelTags.AQUA_EVALUATION.value],
+            tag_list=[Tags.AQUA_EVALUATION],
         )
         logger.info(f"Fetched {len(models)} evaluations.")
 
-        # TODO: add filter based on project_id if needed.
-
         mapping = self._prefetch_resources(compartment_id)
 
         evaluations = []
@@ -988,7 +724,7 @@ class AquaEvaluationApp(AquaApp):
 
             else:
                 jobrun_id = self._get_attribute_from_model_metadata(
-                    model, EvaluationCustomMetadata.EVALUATION_JOB_RUN_ID.value
+                    model, EvaluationCustomMetadata.EVALUATION_JOB_RUN_ID
                 )
                 job_run = mapping.get(jobrun_id)
 
@@ -1197,11 +933,13 @@ class AquaEvaluationApp(AquaApp):
             )
 
             files_in_artifact = get_files(temp_dir)
-            report_content = self._read_from_artifact(
+            md_report_content = self._read_from_artifact(
                 temp_dir, files_in_artifact, utils.EVALUATION_REPORT_MD
             )
+
+            # json report not availiable for failed evaluation
             try:
-                report = json.loads(
+                json_report = json.loads(
                     self._read_from_artifact(
                         temp_dir, files_in_artifact, utils.EVALUATION_REPORT_JSON
                     )
@@ -1210,27 +948,32 @@ class AquaEvaluationApp(AquaApp):
                 logger.debug(
                     "Failed to load `report.json` from evaluation artifact" f"{str(e)}"
                 )
-                report = {}
+                json_report = {}
 
-        # TODO: after finalizing the format of report.json, move the constant to class
         eval_metrics = AquaEvalMetrics(
             id=eval_id,
-            report=base64.b64encode(report_content).decode(),
+            report=base64.b64encode(md_report_content).decode(),
             metric_results=[
                 AquaEvalMetric(
-                    key=metric_key,
-                    name=metadata.get("name", utils.UNKNOWN),
-                    description=metadata.get("description", utils.UNKNOWN),
+                    key=metadata.get(EvaluationMetricResult.SHORT_NAME, utils.UNKNOWN),
+                    name=metadata.get(EvaluationMetricResult.NAME, utils.UNKNOWN),
+                    description=metadata.get(
+                        EvaluationMetricResult.DESCRIPTION, utils.UNKNOWN
+                    ),
                 )
-                for metric_key, metadata in report.get("metric_results", {}).items()
+                for _, metadata in json_report.get(
+                    EvaluationReportJson.METRIC_RESULT, {}
+                ).items()
             ],
             metric_summary_result=[
                 AquaEvalMetricSummary(**m)
-                for m in report.get("metric_summary_result", [{}])
+                for m in json_report.get(
+                    EvaluationReportJson.METRIC_SUMMARY_RESULT, [{}]
+                )
             ],
         )
 
-        if report_content:
+        if md_report_content:
             self._metrics_cache.__setitem__(key=eval_id, value=eval_metrics)
 
         return eval_metrics
@@ -1371,6 +1114,7 @@ class AquaEvaluationApp(AquaApp):
     @telemetry(entry_point="plugin=evaluation&action=delete", name="aqua")
     def delete(self, eval_id):
         """Deletes the job and the associated model for the given evaluation id.
+
         Parameters
         ----------
         eval_id: str
@@ -1383,9 +1127,9 @@ class AquaEvaluationApp(AquaApp):
         Raises
         ------
         AquaRuntimeError:
-            if a model doesn't exist for the given eval_id
+            if a model doesn't exist for the given eval_id.
         AquaMissingKeyError:
-            if training_id is missing the job run id
+            if job/jobrun id is missing.
         """
 
         model = DataScienceModel.from_id(eval_id)
@@ -1396,20 +1140,32 @@ class AquaEvaluationApp(AquaApp):
 
         try:
             job_id = model.custom_metadata_list.get(
-                EvaluationCustomMetadata.EVALUATION_JOB_ID.value
+                EvaluationCustomMetadata.EVALUATION_JOB_ID
             ).value
         except Exception:
             raise AquaMissingKeyError(
-                f"Custom metadata is missing {EvaluationCustomMetadata.EVALUATION_JOB_ID.value} key"
+                f"Custom metadata is missing {EvaluationCustomMetadata.EVALUATION_JOB_ID} key"
             )
 
         job = DataScienceJob.from_id(job_id)
 
         self._delete_job_and_model(job, model)
 
+        try:
+            jobrun_id = model.custom_metadata_list.get(
+                EvaluationCustomMetadata.EVALUATION_JOB_RUN_ID
+            ).value
+            jobrun = utils.query_resource(jobrun_id, return_all=False)
+        except Exception:
+            logger.debug("Associated Job Run OCID is missing.")
+            jobrun = None
+
+        self._eval_cache.pop(key=eval_id, default=None)
+        self._deletion_cache.__setitem__(key=eval_id, value="")
+
         status = dict(
             id=eval_id,
-            lifecycle_state="DELETING",
+            lifecycle_state=jobrun.lifecycle_state if jobrun else "DELETING",
             time_accepted=datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f%z"),
         )
         return status
@@ -1503,7 +1259,7 @@ class AquaEvaluationApp(AquaApp):
         """Returns ocid and name of the model has been evaluated."""
         source_id = self._get_attribute_from_model_metadata(
             evaluation,
-            EvaluationCustomMetadata.EVALUATION_SOURCE.value,
+            EvaluationCustomMetadata.EVALUATION_SOURCE,
         )
 
         try:
@@ -1512,20 +1268,20 @@ class AquaEvaluationApp(AquaApp):
                 source.display_name
                 if source
                 else self._get_attribute_from_model_metadata(
-                    evaluation, EvaluationCustomMetadata.EVALUATION_SOURCE_NAME.value
+                    evaluation, EvaluationCustomMetadata.EVALUATION_SOURCE_NAME
                 )
             )
 
-            if not source_name:
+            # try to resolve source_name from source id
+            if source_id and not source_name:
                 resource_type = utils.get_resource_type(source_id)
 
-                # TODO: adjust resource principal mapping
-                if resource_type == "datasciencemodel":
-                    source_name = self.ds_client.get_model(source_id).data.display_name
-                elif resource_type == "datasciencemodeldeployment":
+                if resource_type.startswith("datasciencemodeldeployment"):
                     source_name = self.ds_client.get_model_deployment(
                         source_id
                     ).data.display_name
+                elif resource_type.startswith("datasciencemodel"):
+                    source_name = self.ds_client.get_model(source_id).data.display_name
                 else:
                     raise AquaRuntimeError(
                         f"Not supported source type: {resource_type}"
@@ -1589,7 +1345,7 @@ class AquaEvaluationApp(AquaApp):
     ) -> AquaResourceIdentifier:
         """Constructs AquaResourceIdentifier based on the given ocid and display name."""
         try:
-            resource_type = utils.CONSOLE_LINK_RESOURCE_TYPE_MAPPING.get(
+            resource_type = CONSOLE_LINK_RESOURCE_TYPE_MAPPING.get(
                 utils.get_resource_type(id)
             )
 
@@ -1620,7 +1376,7 @@ class AquaEvaluationApp(AquaApp):
         """Extracts job run id from metadata, and gets related job run information."""
 
         jobrun_id = jobrun_id or self._get_attribute_from_model_metadata(
-            resource, EvaluationCustomMetadata.EVALUATION_JOB_RUN_ID.value
+            resource, EvaluationCustomMetadata.EVALUATION_JOB_RUN_ID
         )
 
         logger.info(f"Fetching associated job run: {jobrun_id}")
@@ -1654,8 +1410,6 @@ class AquaEvaluationApp(AquaApp):
                     "model parameters have not been saved in correct format in model taxonomy. ",
                     service_payload={"params": params},
                 )
-            # TODO: validate the format of parameters.
-            # self._validate_params(params)
 
             return AquaEvalParams(**params[EvaluationConfig.PARAMS])
         except Exception as e:
@@ -1688,7 +1442,6 @@ class AquaEvaluationApp(AquaApp):
             )
             return AquaResourceIdentifier()
 
-    # TODO: fix the logic for determine termination state
     def _get_status(
         self,
         model: oci.resource_search.models.ResourceSummary,
@@ -1697,30 +1450,33 @@ class AquaEvaluationApp(AquaApp):
         ] = None,
     ) -> dict:
         """Builds evaluation status based on the model status and job run status.
-        When detect `aqua_evaluation_error` in custom metadata, the jobrun is failed.
-        However, if jobrun failed before saving this meta, we need to check the existance
-        of the evaluation artifact.
+        When missing jobrun information, the status will be decided based on:
 
-        """
-        # TODO: revisit for CANCELED evaluation
-        job_run_status = (
-            JobRun.LIFECYCLE_STATE_FAILED
-            if self._get_attribute_from_model_metadata(
-                model, EvaluationCustomMetadata.EVALUATION_ERROR.value
-            )
-            else None
-        )
+            * If the evaluation just has been deleted, the jobrun status should be deleted.
+            * When detect `aqua_evaluation_error` in custom metadata, the jobrun is failed.
+            * If jobrun failed before saving this meta, we need to check the existance
+            of the evaluation artifact.
 
+        """
         model_status = model.lifecycle_state
-        job_run_status = job_run_status or (
-            jobrun.lifecycle_state
-            if jobrun and not jobrun.lifecycle_state == JobRun.LIFECYCLE_STATE_DELETED
-            else (
-                JobRun.LIFECYCLE_STATE_SUCCEEDED
-                if self._if_eval_artifact_exist(model)
-                else JobRun.LIFECYCLE_STATE_FAILED
-            )
-        )
+        job_run_status = None
+
+        if jobrun:
+            job_run_status = jobrun.lifecycle_state
+
+        if jobrun is None:
+            if model.identifier in self._deletion_cache.keys():
+                job_run_status = JobRun.LIFECYCLE_STATE_DELETED
+
+            elif self._get_attribute_from_model_metadata(
+                model, EvaluationCustomMetadata.EVALUATION_ERROR
+            ):
+                job_run_status = JobRun.LIFECYCLE_STATE_FAILED
+
+            elif self._if_eval_artifact_exist(model):
+                job_run_status = JobRun.LIFECYCLE_STATE_SUCCEEDED
+            else:
+                job_run_status = JobRun.LIFECYCLE_STATE_FAILED
 
         lifecycle_state = utils.LifecycleStatus.get_status(
             evaluation_status=model_status, job_run_status=job_run_status
@@ -1738,21 +1494,17 @@ class AquaEvaluationApp(AquaApp):
 
         return dict(
             lifecycle_state=(
-                lifecycle_state
-                if isinstance(lifecycle_state, str)
-                else lifecycle_state.value
+                lifecycle_state if isinstance(lifecycle_state, str) else lifecycle_state
             ),
             lifecycle_details=lifecycle_details,
         )
 
     def _prefetch_resources(self, compartment_id) -> dict:
         """Fetches all AQUA resources."""
-        # TODO: handle cross compartment/tenency resources
-        # TODO: add cache
         resources = utils.query_resources(
             compartment_id=compartment_id,
             resource_type="all",
-            tag_list=[EvaluationModelTags.AQUA_EVALUATION.value, "OCI_AQUA"],
+            tag_list=[Tags.AQUA_EVALUATION, "OCI_AQUA"],
             connect_by_ampersands=False,
             return_all=False,
         )