oracle-ads 2.13.12__py3-none-any.whl → 2.13.13__py3-none-any.whl

ads/aqua/model/model.py

@@ -16,7 +16,7 @@ from oci.data_science.models import JobRun, Metadata, Model, UpdateModelDetails
 
 from ads.aqua import logger
 from ads.aqua.app import AquaApp
-from ads.aqua.common.entities import AquaMultiModelRef, LoraModuleSpec
+from ads.aqua.common.entities import AquaMultiModelRef
 from ads.aqua.common.enums import (
     ConfigFolder,
     CustomInferenceContainerTypeFamily,
@@ -83,10 +83,7 @@ from ads.aqua.model.entities import (
     ModelValidationResult,
 )
 from ads.aqua.model.enums import MultiModelSupportedTaskType
-from ads.aqua.model.utils import (
-    extract_base_model_from_ft,
-    extract_fine_tune_artifacts_path,
-)
+from ads.aqua.model.utils import extract_fine_tune_artifacts_path
 from ads.common.auth import default_signer
 from ads.common.oci_resource import SEARCH_TYPE, OCIResource
 from ads.common.utils import UNKNOWN, get_console_link, is_path_exists, read_file
@@ -242,6 +239,7 @@ class AquaModelApp(AquaApp):
         compartment_id: Optional[str] = None,
         freeform_tags: Optional[Dict] = None,
         defined_tags: Optional[Dict] = None,
+        source_models: Optional[Dict[str, DataScienceModel]] = None,
         **kwargs,  # noqa: ARG002
     ) -> DataScienceModel:
         """
@@ -259,6 +257,10 @@ class AquaModelApp(AquaApp):
             Freeform tags for the model.
         defined_tags : Optional[Dict]
             Defined tags for the model.
+        source_models: Optional[Dict[str, DataScienceModel]]
+            A mapping of model OCIDs to their corresponding `DataScienceModel` objects.
+            This dictionary contains metadata for all models involved in the multi-model deployment,
+            including both base models and fine-tuned weights.
 
         Returns
         -------
@@ -295,76 +297,127 @@ class AquaModelApp(AquaApp):
 
         selected_models_deployment_containers = set()
 
+        if not source_models:
+            # Collect all unique model IDs (including fine-tuned models)
+            source_model_ids = list(
+                {model_id for model in models for model_id in model.all_model_ids()}
+            )
+            logger.debug(
+                "Fetching source model metadata for model IDs: %s", source_model_ids
+            )
+
+            # Fetch source model metadata
+            source_models = self.get_multi_source(source_model_ids) or {}
+
         # Process each model in the input list
         for model in models:
-            # Retrieve model metadata from the Model Catalog using the model ID
-            source_model = DataScienceModel.from_id(model.model_id)
-            display_name = source_model.display_name
-            model_file_description = source_model.model_file_description
-            # If model_name is not explicitly provided, use the model's display name
-            model.model_name = model.model_name or display_name
+            # Retrieve base model metadata
+            source_model: DataScienceModel = source_models.get(model.model_id)
+            if not source_model:
+                logger.error(
+                    "Failed to fetch metadata for base model ID: %s", model.model_id
+                )
+                raise AquaValueError(
+                    f"Unable to retrieve metadata for base model ID: {model.model_id}."
+                )
+
+            # Use display name as fallback if model name not provided
+            model.model_name = model.model_name or source_model.display_name
 
+            # Validate model file description
+            model_file_description = source_model.model_file_description
             if not model_file_description:
+                logger.error(
+                    "Model '%s' (%s) has no file description.",
+                    source_model.display_name,
+                    model.model_id,
+                )
                 raise AquaValueError(
                     f"Model '{source_model.display_name}' (ID: {model.model_id}) has no file description. "
-                    "Please register the model first."
+                    "Please register the model with a file description."
                 )
 
-            # Check if the model is a fine-tuned model based on its tags
-            is_fine_tuned_model = (
-                Tags.AQUA_FINE_TUNED_MODEL_TAG in source_model.freeform_tags
+            # Track model file description in a validated structure
+            model_file_description_list.append(
+                ModelFileDescription(**model_file_description)
             )
 
-            base_model_artifact_path = ""
-            fine_tune_path = ""
-
-            if is_fine_tuned_model:
-                # Extract artifact paths for the base and fine-tuned model
-                base_model_artifact_path, fine_tune_path = (
-                    extract_fine_tune_artifacts_path(source_model)
+            # Ensure base model has a valid artifact
+            if not source_model.artifact:
+                logger.error(
+                    "Base model '%s' (%s) has no artifact.",
+                    model.model_name,
+                    model.model_id,
+                )
+                raise AquaValueError(
+                    f"Model '{model.model_name}' (ID: {model.model_id}) has no registered artifacts. "
+                    "Please register the model before deployment."
                 )
 
-                # Create a single LoRA module specification for the fine-tuned model
-                # TODO: Support multiple LoRA modules in the future
-                model.fine_tune_weights = [
-                    LoraModuleSpec(
-                        model_id=model.model_id,
-                        model_name=model.model_name,
-                        model_path=fine_tune_path,
-                    )
-                ]
+            # Set base model artifact path
+            model.artifact_location = source_model.artifact
+            logger.debug(
+                "Model '%s' artifact path set to: %s",
+                model.model_name,
+                model.artifact_location,
+            )
 
-                # Use the LoRA module name as the model's display name
-                display_name = model.model_name
+            display_name_list.append(model.model_name)
 
-                # Temporarily override model ID and name with those of the base model
-                # TODO: Revisit this logic once proper base/FT model handling is implemented
-                model.model_id, model.model_name = extract_base_model_from_ft(
-                    source_model
+            # Extract model task metadata from source model
+            self._extract_model_task(model, source_model)
+
+            # Process fine-tuned weights if provided
+            for ft_model in model.fine_tune_weights or []:
+                fine_tune_source_model: DataScienceModel = source_models.get(
+                    ft_model.model_id
                 )
-            else:
-                # For base models, use the original artifact path
-                base_model_artifact_path = source_model.artifact
-                display_name = model.model_name
+                if not fine_tune_source_model:
+                    logger.error(
+                        "Failed to fetch metadata for fine-tuned model ID: %s",
+                        ft_model.model_id,
+                    )
+                    raise AquaValueError(
+                        f"Unable to retrieve metadata for fine-tuned model ID: {ft_model.model_id}."
+                    )
 
-            if not base_model_artifact_path:
-                # Fail if no artifact is found for the base model model
-                raise AquaValueError(
-                    f"Model '{model.model_name}' (ID: {model.model_id}) has no artifacts. "
-                    "Please register the model first."
+                # Validate model file description
+                ft_model_file_description = (
+                    fine_tune_source_model.model_file_description
                 )
+                if not ft_model_file_description:
+                    logger.error(
+                        "Model '%s' (%s) has no file description.",
+                        fine_tune_source_model.display_name,
+                        ft_model.model_id,
+                    )
+                    raise AquaValueError(
+                        f"Model '{fine_tune_source_model.display_name}' (ID: {ft_model.model_id}) has no file description. "
+                        "Please register the model with a file description."
+                    )
 
-            # Update the artifact path in the model configuration
-            model.artifact_location = base_model_artifact_path
-            display_name_list.append(display_name)
+                # Track model file description in a validated structure
+                model_file_description_list.append(
+                    ModelFileDescription(**ft_model_file_description)
+                )
 
-            # Extract model task metadata from source model
-            self._extract_model_task(model, source_model)
+                # Extract fine-tuned model path
+                _, fine_tune_path = extract_fine_tune_artifacts_path(
+                    fine_tune_source_model
+                )
+                logger.debug(
+                    "Resolved fine-tuned model path for '%s': %s",
+                    ft_model.model_id,
+                    fine_tune_path,
+                )
+                ft_model.model_path = fine_tune_path
 
-            # Track model file description in a validated structure
-            model_file_description_list.append(
-                ModelFileDescription(**model_file_description)
-            )
+                # Use fallback name if needed
+                ft_model.model_name = (
+                    ft_model.model_name or fine_tune_source_model.display_name
+                )
+
+                display_name_list.append(ft_model.model_name)
 
             # Validate deployment container consistency
             deployment_container = source_model.custom_metadata_list.get(
@@ -375,9 +428,15 @@ class AquaModelApp(AquaApp):
             ).value
 
             if deployment_container not in supported_container_families:
+                logger.error(
+                    "Unsupported deployment container '%s' for model '%s'. Supported: %s",
+                    deployment_container,
+                    source_model.id,
+                    supported_container_families,
+                )
                 raise AquaValueError(
                     f"Unsupported deployment container '{deployment_container}' for model '{source_model.id}'. "
-                    f"Only '{supported_container_families}' are supported for multi-model deployments."
+                    f"Only {supported_container_families} are supported for multi-model deployments."
                 )
 
             selected_models_deployment_containers.add(deployment_container)

ads/aqua/modeldeployment/deployment.py

@@ -4,6 +4,7 @@
 
 
 import json
+import re
 import shlex
 import threading
 from datetime import datetime, timedelta
@@ -47,7 +48,11 @@ from ads.aqua.constants import (
 )
 from ads.aqua.data import AquaResourceIdentifier
 from ads.aqua.model import AquaModelApp
-from ads.aqua.model.constants import AquaModelMetadataKeys, ModelCustomMetadataFields
+from ads.aqua.model.constants import (
+    AquaModelMetadataKeys,
+    ModelCustomMetadataFields,
+    ModelTask,
+)
 from ads.aqua.model.utils import (
     extract_base_model_from_ft,
     extract_fine_tune_artifacts_path,
@@ -214,20 +219,52 @@ class AquaDeploymentApp(AquaApp):
                 freeform_tags=freeform_tags,
                 defined_tags=defined_tags,
             )
+            task_tag = aqua_model.freeform_tags.get(Tags.TASK, UNKNOWN)
+            if (
+                task_tag == ModelTask.TIME_SERIES_FORECASTING
+                or task_tag == ModelTask.TIME_SERIES_FORECASTING.replace("-", "_")
+            ):
+                create_deployment_details.env_var.update(
+                    {Tags.TASK.upper(): ModelTask.TIME_SERIES_FORECASTING}
+                )
             return self._create(
                 aqua_model=aqua_model,
                 create_deployment_details=create_deployment_details,
                 container_config=container_config,
             )
         else:
-            model_ids = [model.model_id for model in create_deployment_details.models]
+            # Collect all unique model IDs (including fine-tuned models)
+            source_model_ids = list(
+                {
+                    model_id
+                    for model in create_deployment_details.models
+                    for model_id in model.all_model_ids()
+                }
+            )
+            logger.debug(
+                "Fetching source model metadata for model IDs: %s", source_model_ids
+            )
+            # Fetch source model metadata
+            source_models = self.get_multi_source(source_model_ids) or {}
+
+            try:
+                create_deployment_details.validate_input_models(
+                    model_details=source_models
+                )
+            except ConfigValidationError as err:
+                raise AquaValueError(f"{err}") from err
+
+            base_model_ids = [
+                model.model_id for model in create_deployment_details.models
+            ]
 
             try:
                 model_config_summary = self.get_multimodel_deployment_config(
-                    model_ids=model_ids, compartment_id=compartment_id
+                    model_ids=base_model_ids, compartment_id=compartment_id
                 )
                 if not model_config_summary.gpu_allocation:
                     raise AquaValueError(model_config_summary.error_message)
+
                 create_deployment_details.validate_multimodel_deployment_feasibility(
                     models_config_summary=model_config_summary
                 )
@@ -298,7 +335,7 @@ class AquaDeploymentApp(AquaApp):
                     )
 
             logger.debug(
-                f"Multi models ({model_ids}) provided. Delegating to multi model creation method."
+                f"Multi models ({source_model_ids}) provided. Delegating to multi model creation method."
             )
 
             aqua_model = model_app.create_multi(
@@ -307,6 +344,7 @@ class AquaDeploymentApp(AquaApp):
                 project_id=project_id,
                 freeform_tags=freeform_tags,
                 defined_tags=defined_tags,
+                source_models=source_models,
             )
             return self._create_multi(
                 aqua_model=aqua_model,
@@ -727,14 +765,16 @@ class AquaDeploymentApp(AquaApp):
         ).deploy(wait_for_completion=False)
 
         deployment_id = deployment.id
+
         logger.info(
             f"Aqua model deployment {deployment_id} created for model {aqua_model_id}. Work request Id is {deployment.dsc_model_deployment.workflow_req_id}"
         )
+        status_list = []
 
         progress_thread = threading.Thread(
             target=self.get_deployment_status,
             args=(
-                deployment_id,
+                deployment,
                 deployment.dsc_model_deployment.workflow_req_id,
                 model_type,
                 model_name,
@@ -822,12 +862,22 @@ class AquaDeploymentApp(AquaApp):
             )
 
             if oci_aqua:
+                # skipping the AQUA model deployments that are created from model group
+                # TODO: remove this checker after AQUA deployment is integrated with model group
+                aqua_model_id = model_deployment.freeform_tags.get(
+                    Tags.AQUA_MODEL_ID_TAG, UNKNOWN
+                )
+                if (
+                    "datasciencemodelgroup" in aqua_model_id
+                    or model_deployment.model_deployment_configuration_details.deployment_type
+                    == "UNKNOWN_ENUM_VALUE"
+                ):
+                    continue
                 results.append(
                     AquaDeployment.from_oci_model_deployment(
                         model_deployment, self.region
                     )
                 )
-
                 # log telemetry if MD is in active or failed state
                 deployment_id = model_deployment.id
                 state = model_deployment.lifecycle_state.upper()
@@ -1233,7 +1283,7 @@ class AquaDeploymentApp(AquaApp):
 
     def get_deployment_status(
         self,
-        model_deployment_id: str,
+        deployment: ModelDeployment,
         work_request_id: str,
         model_type: str,
         model_name: str,
@@ -1255,13 +1305,10 @@ class AquaDeploymentApp(AquaApp):
         AquaDeployment
             An Aqua deployment instance.
         """
-        ocid = get_ocid_substring(model_deployment_id, key_len=8)
-        telemetry_kwargs = {"ocid": ocid}
-
+        ocid = get_ocid_substring(deployment.id, key_len=8)
         data_science_work_request: DataScienceWorkRequest = DataScienceWorkRequest(
             work_request_id
         )
-
         try:
             data_science_work_request.wait_work_request(
                 progress_bar_description="Creating model deployment",
@@ -1269,23 +1316,49 @@ class AquaDeploymentApp(AquaApp):
                 poll_interval=DEFAULT_POLL_INTERVAL,
             )
         except Exception:
+            status = ""
+            logs = deployment.show_logs().sort_values(by="time", ascending=False)
+
+            if logs and len(logs) > 0:
+                status = logs.iloc[0]["message"]
+
+            status = re.sub(r"[^a-zA-Z0-9]", " ", status)
+
             if data_science_work_request._error_message:
                 error_str = ""
                 for error in data_science_work_request._error_message:
                     error_str = error_str + " " + error.message
 
-            self.telemetry.record_event(
-                category=f"aqua/{model_type}/deployment/status",
-                action="FAILED",
-                detail=error_str,
-                value=model_name,
-                **telemetry_kwargs,
-            )
+                error_str = re.sub(r"[^a-zA-Z0-9]", " ", error_str)
+                telemetry_kwargs = {
+                    "ocid": ocid,
+                    "model_name": model_name,
+                    "work_request_error": error_str,
+                    "status": status,
+                }
+
+                self.telemetry.record_event(
+                    category=f"aqua/{model_type}/deployment/status",
+                    action="FAILED",
+                    **telemetry_kwargs,
+                )
+            else:
+                telemetry_kwargs = {
+                    "ocid": ocid,
+                    "model_name": model_name,
+                    "status": status,
+                }
+
+                self.telemetry.record_event(
+                    category=f"aqua/{model_type}/deployment/status",
+                    action="FAILED",
+                    **telemetry_kwargs,
+                )
 
         else:
-            self.telemetry.record_event_async(
+            telemetry_kwargs = {"ocid": ocid, "model_name": model_name}
+            self.telemetry.record_event(
                 category=f"aqua/{model_type}/deployment/status",
                 action="SUCCEEDED",
-                value=model_name,
                 **telemetry_kwargs,
             )

ads/aqua/modeldeployment/entities.py

@@ -13,12 +13,15 @@ from ads.aqua.common.enums import Tags
 from ads.aqua.config.utils.serializer import Serializable
 from ads.aqua.constants import UNKNOWN_DICT
 from ads.aqua.data import AquaResourceIdentifier
+from ads.aqua.finetuning.constants import FineTuneCustomMetadata
 from ads.aqua.modeldeployment.config_loader import (
     ConfigurationItem,
     ModelDeploymentConfigSummary,
 )
 from ads.common.serializer import DataClassSerializable
 from ads.common.utils import UNKNOWN, get_console_link
+from ads.model.datascience_model import DataScienceModel
+from ads.model.model_metadata import ModelCustomMetadataItem
 
 
 class ConfigValidationError(Exception):
@@ -474,6 +477,149 @@ class CreateModelDeploymentDetails(BaseModel):
             logger.error(error_message)
             raise ConfigValidationError(error_message)
 
+    def validate_input_models(self, model_details: Dict[str, DataScienceModel]) -> None:
+        """
+        Validates the input models for a multi-model deployment configuration.
+
+        Validation Criteria:
+        - The base model must be explicitly provided.
+        - The base model must be in 'ACTIVE' state.
+        - Fine-tuned model IDs must refer to valid, tagged fine-tuned models.
+        - Fine-tuned models must refer back to the same base model.
+        - All model names (including fine-tuned variants) must be unique.
+
+        Parameters
+        ----------
+        model_details : Dict[str, DataScienceModel]
+            Dictionary mapping model OCIDs to DataScienceModel instances.
+            Includes the all models to validate including fine-tuned models.
+
+        Raises
+        ------
+        ConfigValidationError
+            If any of the above conditions are violated.
+        """
+        if not self.models:
+            logger.error("Validation failed: No models specified in the model group.")
+            raise ConfigValidationError(
+                "Multi-model deployment requires at least one model entry. "
+                "Please provide a base model in the `models` list."
+            )
+
+        seen_names = set()
+        duplicate_names = set()
+
+        for model in self.models:
+            base_model_id = model.model_id
+            base_model = model_details.get(base_model_id)
+
+            if not base_model:
+                logger.error(
+                    "Validation failed: Base model ID '%s' not found.", base_model_id
+                )
+                raise ConfigValidationError(f"Model not found: '{base_model_id}'.")
+
+            if Tags.AQUA_FINE_TUNED_MODEL_TAG in (base_model.freeform_tags or {}):
+                logger.error(
+                    "Validation failed: Base model ID '%s' is a fine-tuned model.",
+                    base_model_id,
+                )
+                raise ConfigValidationError(
+                    f"Invalid base model ID '{base_model_id}'. "
+                    "Specify a base model OCID in the `models` input, not a fine-tuned model."
+                )
+
+            if base_model.lifecycle_state != "ACTIVE":
+                logger.error(
+                    "Validation failed: Base model '%s' is in state '%s'.",
+                    base_model_id,
+                    base_model.lifecycle_state,
+                )
+                raise ConfigValidationError(
+                    f"Invalid base model ID '{base_model_id}': must be in ACTIVE state."
+                )
+
+            # Normalize and validate model name uniqueness
+            model_name = model.model_name or base_model.display_name
+            if model_name in seen_names:
+                duplicate_names.add(model_name)
+            else:
+                seen_names.add(model_name)
+
+            for lora_module in model.fine_tune_weights or []:
+                ft_model_id = lora_module.model_id
+                ft_model = model_details.get(ft_model_id)
+
+                if not ft_model:
+                    logger.error(
+                        "Validation failed: Fine-tuned model ID '%s' not found.",
+                        ft_model_id,
+                    )
+                    raise ConfigValidationError(
+                        f"Fine-tuned model not found: '{ft_model_id}'."
+                    )
+
+                if ft_model.lifecycle_state != "ACTIVE":
+                    logger.error(
+                        "Validation failed: Fine-tuned model '%s' is in state '%s'.",
+                        ft_model_id,
+                        ft_model.lifecycle_state,
+                    )
+                    raise ConfigValidationError(
+                        f"Invalid Fine-tuned model ID '{ft_model_id}': must be in ACTIVE state."
+                    )
+
+                if Tags.AQUA_FINE_TUNED_MODEL_TAG not in (ft_model.freeform_tags or {}):
+                    logger.error(
+                        "Validation failed: Model ID '%s' is missing tag '%s'.",
+                        ft_model_id,
+                        Tags.AQUA_FINE_TUNED_MODEL_TAG,
+                    )
+                    raise ConfigValidationError(
+                        f"Invalid fine-tuned model ID '{ft_model_id}': missing tag '{Tags.AQUA_FINE_TUNED_MODEL_TAG}'."
+                    )
+
+                ft_base_model_id = ft_model.custom_metadata_list.get(
+                    FineTuneCustomMetadata.FINE_TUNE_SOURCE,
+                    ModelCustomMetadataItem(
+                        key=FineTuneCustomMetadata.FINE_TUNE_SOURCE
+                    ),
+                ).value
+
+                if ft_base_model_id != base_model_id:
+                    logger.error(
+                        "Validation failed: Fine-tuned model '%s' is linked to base model '%s' (expected '%s').",
+                        ft_model_id,
+                        ft_base_model_id,
+                        base_model_id,
+                    )
+                    raise ConfigValidationError(
+                        f"Fine-tuned model '{ft_model_id}' belongs to base model '{ft_base_model_id}', "
+                        f"but was included under base model '{base_model_id}'."
+                    )
+
+                # Validate fine-tuned model name uniqueness
+                lora_model_name = lora_module.model_name or ft_model.display_name
+                if lora_model_name in seen_names:
+                    duplicate_names.add(lora_model_name)
+                else:
+                    seen_names.add(lora_model_name)
+
+                logger.debug(
+                    "Validated fine-tuned model '%s' under base model '%s'.",
+                    ft_model_id,
+                    base_model_id,
+                )
+
+        if duplicate_names:
+            logger.error(
+                "Duplicate model names detected: %s", ", ".join(sorted(duplicate_names))
+            )
+            raise ConfigValidationError(
+                f"The following model names are duplicated across base and fine-tuned models: "
+                f"{', '.join(sorted(duplicate_names))}. Model names must be unique for proper routing in multi-model deployments."
+            )
+
     class Config:
         extra = "allow"
         protected_namespaces = ()

ads/aqua/modeldeployment/model_group_config.py

@@ -81,14 +81,12 @@ class BaseModelSpec(BaseModel):
         unique_modules: List[LoraModuleSpec] = []
 
         for module in fine_tune_weights or []:
-            name = getattr(module, "model_name", None)
-            if not name:
-                logger.warning("Fine-tuned model in AquaMultiModelRef is missing model_name.")
-                continue
-            if name in seen:
-                logger.warning(f"Duplicate LoRA Module detected: {name!r} (skipping duplicate).")
+            if module.model_name and module.model_name in seen:
+                logger.warning(
+                    f"Duplicate LoRA Module detected: {module.model_name!r} (skipping duplicate)."
+                )
                 continue
-            seen.add(name)
+            seen.add(module.model_name)
             unique_modules.append(module)
 
         return unique_modules
@@ -169,17 +167,12 @@ class ModelGroupConfig(Serializable):
             model, container_params, container_type_key
         )
 
-        model_id = (
-            model.fine_tune_weights[0].model_id
-            if model.fine_tune_weights
-            else model.model_id
-        )
-
         deployment_config = model_config_summary.deployment_config.get(
-            model_id, AquaDeploymentConfig()
+            model.model_id, AquaDeploymentConfig()
         ).configuration.get(
             create_deployment_details.instance_shape, ConfigurationItem()
         )
+
         params_found = False
         for item in deployment_config.multi_model_deployment:
             if model.gpu_count and item.gpu_count and item.gpu_count == model.gpu_count:

ads/aqua/verify_policies/__init__.py

@@ -0,0 +1,8 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+from ads.aqua.verify_policies.verify import AquaVerifyPoliciesApp
+
+__all__ = ["AquaVerifyPoliciesApp"]