oracle-ads 2.13.11__py3-none-any.whl → 2.13.13__py3-none-any.whl

ads/aqua/modeldeployment/constants.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-# Copyright (c) 2024 Oracle and/or its affiliates.
+# Copyright (c) 2024, 2025 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 """
@@ -8,3 +8,6 @@ aqua.modeldeployment.constants
 
 This module contains constants used in Aqua Model Deployment.
 """
+
+DEFAULT_WAIT_TIME = 12000
+DEFAULT_POLL_INTERVAL = 10

ads/aqua/modeldeployment/deployment.py

@@ -2,10 +2,13 @@
 # Copyright (c) 2024, 2025 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
+
 import json
+import re
 import shlex
+import threading
 from datetime import datetime, timedelta
-from typing import Dict, List, Optional, Union
+from typing import Dict, List, Optional
 
 from cachetools import TTLCache, cached
 from oci.data_science.models import ModelDeploymentShapeSummary
@@ -17,20 +20,15 @@ from ads.aqua.common.entities import (
     ComputeShapeSummary,
     ContainerPath,
 )
-from ads.aqua.common.enums import (
-    InferenceContainerTypeFamily,
-    ModelFormat,
-    Tags,
-)
+from ads.aqua.common.enums import InferenceContainerTypeFamily, ModelFormat, Tags
 from ads.aqua.common.errors import AquaRuntimeError, AquaValueError
 from ads.aqua.common.utils import (
     DEFINED_METADATA_TO_FILE_MAP,
-    build_params_string,
     build_pydantic_error_message,
+    find_restricted_params,
     get_combined_params,
     get_container_params_type,
     get_ocid_substring,
-    get_params_dict,
     get_params_list,
     get_resource_name,
     get_restricted_params_by_container,
@@ -50,27 +48,38 @@ from ads.aqua.constants import (
 )
 from ads.aqua.data import AquaResourceIdentifier
 from ads.aqua.model import AquaModelApp
-from ads.aqua.model.constants import AquaModelMetadataKeys, ModelCustomMetadataFields
+from ads.aqua.model.constants import (
+    AquaModelMetadataKeys,
+    ModelCustomMetadataFields,
+    ModelTask,
+)
 from ads.aqua.model.utils import (
     extract_base_model_from_ft,
     extract_fine_tune_artifacts_path,
 )
+from ads.aqua.modeldeployment.config_loader import (
+    AquaDeploymentConfig,
+    ConfigurationItem,
+    ModelDeploymentConfigSummary,
+    MultiModelDeploymentConfigLoader,
+)
+from ads.aqua.modeldeployment.constants import DEFAULT_POLL_INTERVAL, DEFAULT_WAIT_TIME
 from ads.aqua.modeldeployment.entities import (
     AquaDeployment,
-    AquaDeploymentConfig,
     AquaDeploymentDetail,
-    ConfigurationItem,
     ConfigValidationError,
     CreateModelDeploymentDetails,
-    ModelDeploymentConfigSummary,
 )
-from ads.aqua.modeldeployment.utils import MultiModelDeploymentConfigLoader
+from ads.aqua.modeldeployment.model_group_config import ModelGroupConfig
 from ads.common.object_storage_details import ObjectStorageDetails
 from ads.common.utils import UNKNOWN, get_log_links
+from ads.common.work_request import DataScienceWorkRequest
 from ads.config import (
     AQUA_DEPLOYMENT_CONTAINER_CMD_VAR_METADATA_NAME,
     AQUA_DEPLOYMENT_CONTAINER_METADATA_NAME,
     AQUA_DEPLOYMENT_CONTAINER_URI_METADATA_NAME,
+    AQUA_TELEMETRY_BUCKET,
+    AQUA_TELEMETRY_BUCKET_NS,
     COMPARTMENT_OCID,
     PROJECT_OCID,
 )
@@ -210,20 +219,52 @@ class AquaDeploymentApp(AquaApp):
                 freeform_tags=freeform_tags,
                 defined_tags=defined_tags,
             )
+            task_tag = aqua_model.freeform_tags.get(Tags.TASK, UNKNOWN)
+            if (
+                task_tag == ModelTask.TIME_SERIES_FORECASTING
+                or task_tag == ModelTask.TIME_SERIES_FORECASTING.replace("-", "_")
+            ):
+                create_deployment_details.env_var.update(
+                    {Tags.TASK.upper(): ModelTask.TIME_SERIES_FORECASTING}
+                )
             return self._create(
                 aqua_model=aqua_model,
                 create_deployment_details=create_deployment_details,
                 container_config=container_config,
             )
         else:
-            model_ids = [model.model_id for model in create_deployment_details.models]
+            # Collect all unique model IDs (including fine-tuned models)
+            source_model_ids = list(
+                {
+                    model_id
+                    for model in create_deployment_details.models
+                    for model_id in model.all_model_ids()
+                }
+            )
+            logger.debug(
+                "Fetching source model metadata for model IDs: %s", source_model_ids
+            )
+            # Fetch source model metadata
+            source_models = self.get_multi_source(source_model_ids) or {}
+
+            try:
+                create_deployment_details.validate_input_models(
+                    model_details=source_models
+                )
+            except ConfigValidationError as err:
+                raise AquaValueError(f"{err}") from err
+
+            base_model_ids = [
+                model.model_id for model in create_deployment_details.models
+            ]
 
             try:
                 model_config_summary = self.get_multimodel_deployment_config(
-                    model_ids=model_ids, compartment_id=compartment_id
+                    model_ids=base_model_ids, compartment_id=compartment_id
                 )
                 if not model_config_summary.gpu_allocation:
                     raise AquaValueError(model_config_summary.error_message)
+
                 create_deployment_details.validate_multimodel_deployment_feasibility(
                     models_config_summary=model_config_summary
                 )
@@ -294,7 +335,7 @@ class AquaDeploymentApp(AquaApp):
                     )
 
             logger.debug(
-                f"Multi models ({model_ids}) provided. Delegating to multi model creation method."
+                f"Multi models ({source_model_ids}) provided. Delegating to multi model creation method."
             )
 
             aqua_model = model_app.create_multi(
@@ -303,6 +344,7 @@ class AquaDeploymentApp(AquaApp):
                 project_id=project_id,
                 freeform_tags=freeform_tags,
                 defined_tags=defined_tags,
+                source_models=source_models,
             )
             return self._create_multi(
                 aqua_model=aqua_model,
@@ -486,7 +528,7 @@ class AquaDeploymentApp(AquaApp):
                     f"with deployment without parameter overrides."
                 )
 
-            restricted_params = self._find_restricted_params(
+            restricted_params = find_restricted_params(
                 params, user_params, container_type_key
             )
             if restricted_params:
@@ -508,6 +550,9 @@ class AquaDeploymentApp(AquaApp):
                     if key not in env_var:
                         env_var.update(env)
 
+        env_var.update({"AQUA_TELEMETRY_BUCKET_NS": AQUA_TELEMETRY_BUCKET_NS})
+        env_var.update({"AQUA_TELEMETRY_BUCKET": AQUA_TELEMETRY_BUCKET})
+
         logger.info(f"Env vars used for deploying {aqua_model.id} :{env_var}")
 
         tags = {**tags, **(create_deployment_details.freeform_tags or {})}
@@ -553,7 +598,6 @@ class AquaDeploymentApp(AquaApp):
         AquaDeployment
             An Aqua deployment instance.
         """
-        model_config = []
         model_name_list = []
         env_var = {**(create_deployment_details.env_var or UNKNOWN_DICT)}
 
@@ -566,80 +610,14 @@ class AquaDeploymentApp(AquaApp):
 
         container_params = container_spec.cli_param if container_spec else UNKNOWN
 
-        for model in create_deployment_details.models:
-            user_params = build_params_string(model.env_var)
-            if user_params:
-                restricted_params = self._find_restricted_params(
-                    container_params, user_params, container_type_key
-                )
-                if restricted_params:
-                    selected_model = model.model_name or model.model_id
-                    raise AquaValueError(
-                        f"Parameters {restricted_params} are set by Aqua "
-                        f"and cannot be overridden or are invalid."
-                        f"Select other parameters for model {selected_model}."
-                    )
-
-            # replaces `--served-model-name`` with user's model name
-            container_params_dict = get_params_dict(container_params)
-            container_params_dict.update({"--served-model-name": model.model_name})
-            # replaces `--tensor-parallel-size` with model gpu count
-            container_params_dict.update({"--tensor-parallel-size": model.gpu_count})
-            params = build_params_string(container_params_dict)
-
-            deployment_config = model_config_summary.deployment_config.get(
-                model.model_id, AquaDeploymentConfig()
-            ).configuration.get(
-                create_deployment_details.instance_shape, ConfigurationItem()
-            )
-
-            # finds the corresponding deployment parameters based on the gpu count
-            # and combines them with user's parameters. Existing deployment parameters
-            # will be overriden by user's parameters.
-            params_found = False
-            for item in deployment_config.multi_model_deployment:
-                if (
-                    model.gpu_count
-                    and item.gpu_count
-                    and item.gpu_count == model.gpu_count
-                ):
-                    config_parameters = item.parameters.get(
-                        get_container_params_type(container_type_key), UNKNOWN
-                    )
-                    params = f"{params} {get_combined_params(config_parameters, user_params)}".strip()
-                    params_found = True
-                    break
-
-            if not params_found and deployment_config.parameters:
-                config_parameters = deployment_config.parameters.get(
-                    get_container_params_type(container_type_key), UNKNOWN
-                )
-                params = f"{params} {get_combined_params(config_parameters, user_params)}".strip()
-                params_found = True
-
-            # if no config parameters found, append user parameters directly.
-            if not params_found:
-                params = f"{params} {user_params}".strip()
-
-            artifact_path_prefix = model.artifact_location.rstrip("/")
-            if ObjectStorageDetails.is_oci_path(artifact_path_prefix):
-                os_path = ObjectStorageDetails.from_path(artifact_path_prefix)
-                artifact_path_prefix = os_path.filepath.rstrip("/")
-
-            # override by-default completion/ chat endpoint with other endpoint (embedding)
-            config_data = {"params": params, "model_path": artifact_path_prefix}
-            if model.model_task:
-                config_data["model_task"] = model.model_task
-
-            if model.fine_tune_weights_location:
-                config_data["fine_tune_weights_location"] = (
-                    model.fine_tune_weights_location
-                )
-
-            model_config.append(config_data)
-            model_name_list.append(model.model_name)
+        multi_model_config = ModelGroupConfig.from_create_model_deployment_details(
+            create_deployment_details,
+            model_config_summary,
+            container_type_key,
+            container_params,
+        )
 
-        env_var.update({AQUA_MULTI_MODEL_CONFIG: json.dumps({"models": model_config})})
+        env_var.update({AQUA_MULTI_MODEL_CONFIG: multi_model_config.model_dump_json()})
 
         env_vars = container_spec.env_vars if container_spec else []
         for env in env_vars:
@@ -787,9 +765,23 @@ class AquaDeploymentApp(AquaApp):
         ).deploy(wait_for_completion=False)
 
         deployment_id = deployment.id
+
         logger.info(
-            f"Aqua model deployment {deployment_id} created for model {aqua_model_id}."
+            f"Aqua model deployment {deployment_id} created for model {aqua_model_id}. Work request Id is {deployment.dsc_model_deployment.workflow_req_id}"
         )
+        status_list = []
+
+        progress_thread = threading.Thread(
+            target=self.get_deployment_status,
+            args=(
+                deployment,
+                deployment.dsc_model_deployment.workflow_req_id,
+                model_type,
+                model_name,
+            ),
+            daemon=True,
+        )
+        progress_thread.start()
 
         # we arbitrarily choose last 8 characters of OCID to identify MD in telemetry
         telemetry_kwargs = {"ocid": get_ocid_substring(deployment_id, key_len=8)}
@@ -870,12 +862,22 @@ class AquaDeploymentApp(AquaApp):
             )
 
             if oci_aqua:
+                # skipping the AQUA model deployments that are created from model group
+                # TODO: remove this checker after AQUA deployment is integrated with model group
+                aqua_model_id = model_deployment.freeform_tags.get(
+                    Tags.AQUA_MODEL_ID_TAG, UNKNOWN
+                )
+                if (
+                    "datasciencemodelgroup" in aqua_model_id
+                    or model_deployment.model_deployment_configuration_details.deployment_type
+                    == "UNKNOWN_ENUM_VALUE"
+                ):
+                    continue
                 results.append(
                     AquaDeployment.from_oci_model_deployment(
                         model_deployment, self.region
                     )
                 )
-
                 # log telemetry if MD is in active or failed state
                 deployment_id = model_deployment.id
                 state = model_deployment.lifecycle_state.upper()
@@ -1231,7 +1233,7 @@ class AquaDeploymentApp(AquaApp):
             container_spec = container_config.spec if container_config else UNKNOWN
             cli_params = container_spec.cli_param if container_spec else UNKNOWN
 
-            restricted_params = self._find_restricted_params(
+            restricted_params = find_restricted_params(
                 cli_params, params, container_type_key
             )
 
@@ -1242,41 +1244,6 @@ class AquaDeploymentApp(AquaApp):
             )
         return {"valid": True}
 
-    @staticmethod
-    def _find_restricted_params(
-        default_params: Union[str, List[str]],
-        user_params: Union[str, List[str]],
-        container_family: str,
-    ) -> List[str]:
-        """Returns a list of restricted params that user chooses to override when creating an Aqua deployment.
-        The default parameters coming from the container index json file cannot be overridden.
-
-        Parameters
-        ----------
-        default_params:
-            Inference container parameter string with default values.
-        user_params:
-            Inference container parameter string with user provided values.
-        container_family: str
-            The image family of model deployment container runtime.
-
-        Returns
-        -------
-            A list with params keys common between params1 and params2.
-
-        """
-        restricted_params = []
-        if default_params and user_params:
-            default_params_dict = get_params_dict(default_params)
-            user_params_dict = get_params_dict(user_params)
-
-            restricted_params_set = get_restricted_params_by_container(container_family)
-            for key, _items in user_params_dict.items():
-                if key in default_params_dict or key in restricted_params_set:
-                    restricted_params.append(key.lstrip("-"))
-
-        return restricted_params
-
     @telemetry(entry_point="plugin=deployment&action=list_shapes", name="aqua")
     @cached(cache=TTLCache(maxsize=1, ttl=timedelta(minutes=5), timer=datetime.now))
     def list_shapes(self, **kwargs) -> List[ComputeShapeSummary]:
@@ -1313,3 +1280,85 @@ class AquaDeploymentApp(AquaApp):
             )
             for oci_shape in oci_shapes
         ]
+
+    def get_deployment_status(
+        self,
+        deployment: ModelDeployment,
+        work_request_id: str,
+        model_type: str,
+        model_name: str,
+    ) -> None:
+        """Waits for the data science  model deployment to be completed and log its status in telemetry.
+
+        Parameters
+        ----------
+
+        model_deployment_id: str
+            The id of the deployed aqua model.
+        work_request_id: str
+            The work request Id of the model deployment.
+        model_type: str
+            The type of aqua model to be deployed. Allowed values are: `custom`, `service` and `multi_model`.
+
+        Returns
+        -------
+        AquaDeployment
+            An Aqua deployment instance.
+        """
+        ocid = get_ocid_substring(deployment.id, key_len=8)
+        data_science_work_request: DataScienceWorkRequest = DataScienceWorkRequest(
+            work_request_id
+        )
+        try:
+            data_science_work_request.wait_work_request(
+                progress_bar_description="Creating model deployment",
+                max_wait_time=DEFAULT_WAIT_TIME,
+                poll_interval=DEFAULT_POLL_INTERVAL,
+            )
+        except Exception:
+            status = ""
+            logs = deployment.show_logs().sort_values(by="time", ascending=False)
+
+            if logs and len(logs) > 0:
+                status = logs.iloc[0]["message"]
+
+            status = re.sub(r"[^a-zA-Z0-9]", " ", status)
+
+            if data_science_work_request._error_message:
+                error_str = ""
+                for error in data_science_work_request._error_message:
+                    error_str = error_str + " " + error.message
+
+                error_str = re.sub(r"[^a-zA-Z0-9]", " ", error_str)
+                telemetry_kwargs = {
+                    "ocid": ocid,
+                    "model_name": model_name,
+                    "work_request_error": error_str,
+                    "status": status,
+                }
+
+                self.telemetry.record_event(
+                    category=f"aqua/{model_type}/deployment/status",
+                    action="FAILED",
+                    **telemetry_kwargs,
+                )
+            else:
+                telemetry_kwargs = {
+                    "ocid": ocid,
+                    "model_name": model_name,
+                    "status": status,
+                }
+
+                self.telemetry.record_event(
+                    category=f"aqua/{model_type}/deployment/status",
+                    action="FAILED",
+                    **telemetry_kwargs,
+                )
+
+        else:
+            telemetry_kwargs = {"ocid": ocid, "model_name": model_name}
+            self.telemetry.record_event(
+                category=f"aqua/{model_type}/deployment/status",
+                action="SUCCEEDED",
+                **telemetry_kwargs,
+            )