oracle-ads 2.11.14__py3-none-any.whl → 2.11.16__py3-none-any.whl

ads/aqua/modeldeployment/deployment.py

@@ -1,34 +1,32 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright (c) 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
-import json
 import logging
-from typing import Dict, List, Union
-
-from oci.data_science.models import ModelDeployment
+from typing import Dict, List, Optional, Union
 
 from ads.aqua.app import AquaApp, logger
+from ads.aqua.common.entities import ContainerSpec
 from ads.aqua.common.enums import (
-    Tags,
-    InferenceContainerParamType,
-    InferenceContainerType,
     InferenceContainerTypeFamily,
+    Tags,
 )
 from ads.aqua.common.errors import AquaRuntimeError, AquaValueError
 from ads.aqua.common.utils import (
+    get_combined_params,
     get_container_config,
     get_container_image,
+    get_container_params_type,
     get_model_by_reference_paths,
     get_ocid_substring,
-    get_combined_params,
     get_params_dict,
     get_params_list,
     get_resource_name,
+    get_restricted_params_by_container,
     load_config,
 )
 from ads.aqua.constants import (
+    AQUA_MODEL_ARTIFACT_FILE,
     AQUA_MODEL_TYPE_CUSTOM,
     AQUA_MODEL_TYPE_SERVICE,
     MODEL_BY_REFERENCE_OSS_PATH_KEY,
@@ -41,12 +39,8 @@ from ads.aqua.model import AquaModelApp
 from ads.aqua.modeldeployment.entities import (
     AquaDeployment,
     AquaDeploymentDetail,
-    ContainerSpec,
-)
-from ads.aqua.modeldeployment.constants import (
-    VLLMInferenceRestrictedParams,
-    TGIInferenceRestrictedParams,
 )
+from ads.aqua.ui import ModelFormat
 from ads.common.object_storage_details import ObjectStorageDetails
 from ads.common.utils import get_log_links
 from ads.config import (
@@ -110,6 +104,8 @@ class AquaDeploymentApp(AquaApp):
         health_check_port: int = None,
         env_var: Dict = None,
         container_family: str = None,
+        memory_in_gbs: Optional[float] = None,
+        ocpus: Optional[float] = None,
     ) -> "AquaDeployment":
         """
         Creates a new Aqua deployment
@@ -150,6 +146,10 @@ class AquaDeploymentApp(AquaApp):
             Environment variable for the deployment, by default None.
         container_family: str
             The image family of model deployment container runtime. Required for unverified Aqua models.
+        memory_in_gbs: float
+            The memory in gbs for the shape selected.
+        ocpus: float
+            The ocpu count for the shape selected.
         Returns
         -------
         AquaDeployment
@@ -187,24 +187,24 @@ class AquaDeploymentApp(AquaApp):
                 model_name = aqua_model.custom_metadata_list.get(
                     FineTuneCustomMetadata.FINE_TUNE_SOURCE_NAME
                 ).value
-            except:
+            except ValueError as err:
                 raise AquaValueError(
                     f"Either {FineTuneCustomMetadata.FINE_TUNE_SOURCE} or {FineTuneCustomMetadata.FINE_TUNE_SOURCE_NAME} is missing "
                     f"from custom metadata for the model {config_source_id}"
-                )
+                ) from err
 
         # set up env vars
         if not env_var:
-            env_var = dict()
+            env_var = {}
 
         try:
             model_path_prefix = aqua_model.custom_metadata_list.get(
                 MODEL_BY_REFERENCE_OSS_PATH_KEY
             ).value.rstrip("/")
-        except ValueError:
+        except ValueError as err:
             raise AquaValueError(
                 f"{MODEL_BY_REFERENCE_OSS_PATH_KEY} key is not available in the custom metadata field."
-            )
+            ) from err
 
         if ObjectStorageDetails.is_oci_path(model_path_prefix):
             os_path = ObjectStorageDetails.from_path(model_path_prefix)
@@ -212,6 +212,21 @@ class AquaDeploymentApp(AquaApp):
 
         env_var.update({"BASE_MODEL": f"{model_path_prefix}"})
 
+        model_format = aqua_model.freeform_tags.get(
+            Tags.MODEL_FORMAT, ModelFormat.SAFETENSORS.value
+        ).upper()
+        if model_format == ModelFormat.GGUF.value:
+            try:
+                model_file = aqua_model.custom_metadata_list.get(
+                    AQUA_MODEL_ARTIFACT_FILE
+                ).value
+            except ValueError as err:
+                raise AquaValueError(
+                    f"{AQUA_MODEL_ARTIFACT_FILE} key is not available in the custom metadata field "
+                    f"for model {aqua_model.id}."
+                ) from err
+            env_var.update({"BASE_MODEL_FILE": f"{model_file}"})
+
         if is_fine_tuned_model:
             _, fine_tune_output_path = get_model_by_reference_paths(
                 aqua_model.model_file_description
@@ -219,7 +234,7 @@ class AquaDeploymentApp(AquaApp):
 
             if not fine_tune_output_path:
                 raise AquaValueError(
-                    f"Fine tuned output path is not available in the model artifact."
+                    "Fine tuned output path is not available in the model artifact."
                 )
 
             os_path = ObjectStorageDetails.from_path(fine_tune_output_path)
@@ -232,7 +247,7 @@ class AquaDeploymentApp(AquaApp):
             container_type_key = aqua_model.custom_metadata_list.get(
                 AQUA_DEPLOYMENT_CONTAINER_METADATA_NAME
             ).value
-        except ValueError:
+        except ValueError as err:
             message = (
                 f"{AQUA_DEPLOYMENT_CONTAINER_METADATA_NAME} key is not available in the custom metadata field "
                 f"for model {aqua_model.id}."
@@ -242,7 +257,7 @@ class AquaDeploymentApp(AquaApp):
                 raise AquaValueError(
                     f"{message}. For unverified Aqua models, container_family parameter should be "
                     f"set and value can be one of {', '.join(InferenceContainerTypeFamily.values())}."
-                )
+                ) from err
             container_type_key = container_family
         try:
             # Check if the container override flag is set. If set, then the user has chosen custom image
@@ -266,8 +281,10 @@ class AquaDeploymentApp(AquaApp):
             f"Aqua Image used for deploying {aqua_model.id} : {container_image}"
         )
 
+        # todo: use AquaContainerConfig.from_container_index_json instead.
         # Fetch the startup cli command for the container
-        # container_index.json will have "containerSpec" section which will provide the cli params for a given container family
+        # container_index.json will have "containerSpec" section which will provide the cli params for
+        # a given container family
         container_config = get_container_config()
         container_spec = container_config.get(ContainerSpec.CONTAINER_SPEC, {}).get(
             container_type_key, {}
@@ -282,16 +299,29 @@ class AquaDeploymentApp(AquaApp):
         )  # Give precendece to the input parameter
 
         deployment_config = self.get_deployment_config(config_source_id)
-        vllm_params = (
+
+        config_params = (
             deployment_config.get("configuration", UNKNOWN_DICT)
             .get(instance_shape, UNKNOWN_DICT)
             .get("parameters", UNKNOWN_DICT)
-            .get(InferenceContainerParamType.PARAM_TYPE_VLLM, UNKNOWN)
+            .get(get_container_params_type(container_type_key), UNKNOWN)
         )
 
         # validate user provided params
         user_params = env_var.get("PARAMS", UNKNOWN)
         if user_params:
+            # todo: remove this check in the future version, logic to be moved to container_index
+            if (
+                container_type_key.lower()
+                == InferenceContainerTypeFamily.AQUA_LLAMA_CPP_CONTAINER_FAMILY
+            ):
+                # AQUA_LLAMA_CPP_CONTAINER_FAMILY container uses uvicorn that required model/server params
+                # to be set as env vars
+                raise AquaValueError(
+                    f"Currently, parameters cannot be overridden for the container: {container_image}. Please proceed "
+                    f"with deployment without parameter overrides."
+                )
+
             restricted_params = self._find_restricted_params(
                 params, user_params, container_type_key
             )
@@ -301,15 +331,17 @@ class AquaDeploymentApp(AquaApp):
                     f"and cannot be overridden or are invalid."
                 )
 
-        deployment_params = get_combined_params(vllm_params, user_params)
+        deployment_params = get_combined_params(config_params, user_params)
 
-        if deployment_params:
-            params = f"{params} {deployment_params}"
+        params = f"{params} {deployment_params}".strip()
+        if params:
+            env_var.update({"PARAMS": params})
 
-        env_var.update({"PARAMS": params})
         for env in container_spec.get(ContainerSpec.ENV_VARS, []):
             if isinstance(env, dict):
-                env_var.update(env)
+                for key, _items in env.items():
+                    if key not in env_var:
+                        env_var.update(env)
 
         logging.info(f"Env vars used for deploying {aqua_model.id} :{env_var}")
 
@@ -332,6 +364,11 @@ class AquaDeploymentApp(AquaApp):
                 log_id=predict_log_id,
             )
         )
+        if memory_in_gbs and ocpus and infrastructure.shape_name.endswith("Flex"):
+            infrastructure.with_shape_config_details(
+                ocpus=ocpus,
+                memory_in_gbs=memory_in_gbs,
+            )
         # configure model deployment runtime
         container_runtime = (
             ModelDeploymentContainerRuntime()
@@ -345,6 +382,7 @@ class AquaDeploymentApp(AquaApp):
             .with_overwrite_existing_artifact(True)
             .with_remove_existing_artifact(True)
         )
+
         # configure model deployment and deploy model on container runtime
         deployment = (
             ModelDeployment()
@@ -429,7 +467,7 @@ class AquaDeploymentApp(AquaApp):
                     # tracks unique deployments that were listed in the user compartment
                     # we arbitrarily choose last 8 characters of OCID to identify MD in telemetry
                     self.telemetry.record_event_async(
-                        category=f"aqua/deployment",
+                        category="aqua/deployment",
                         action="list",
                         detail=get_ocid_substring(deployment_id, key_len=8),
                         value=state,
@@ -570,32 +608,27 @@ class AquaDeploymentApp(AquaApp):
                 f"{AQUA_DEPLOYMENT_CONTAINER_METADATA_NAME} key is not available in the custom metadata field for model {model_id}."
             )
 
-        if container_type_key:
-            container_type_key = container_type_key.lower()
-            if container_type_key in InferenceContainerTypeFamily.values():
-                deployment_config = self.get_deployment_config(model_id)
-                config_parameters = (
-                    deployment_config.get("configuration", UNKNOWN_DICT)
-                    .get(instance_shape, UNKNOWN_DICT)
-                    .get("parameters", UNKNOWN_DICT)
+        if (
+            container_type_key
+            and container_type_key in InferenceContainerTypeFamily.values()
+        ):
+            deployment_config = self.get_deployment_config(model_id)
+            config_params = (
+                deployment_config.get("configuration", UNKNOWN_DICT)
+                .get(instance_shape, UNKNOWN_DICT)
+                .get("parameters", UNKNOWN_DICT)
+                .get(get_container_params_type(container_type_key), UNKNOWN)
+            )
+            if config_params:
+                params_list = get_params_list(config_params)
+                restricted_params_set = get_restricted_params_by_container(
+                    container_type_key
                 )
-                if InferenceContainerType.CONTAINER_TYPE_VLLM in container_type_key:
-                    params = config_parameters.get(
-                        InferenceContainerParamType.PARAM_TYPE_VLLM, UNKNOWN
-                    )
-                elif InferenceContainerType.CONTAINER_TYPE_TGI in container_type_key:
-                    params = config_parameters.get(
-                        InferenceContainerParamType.PARAM_TYPE_TGI, UNKNOWN
-                    )
-                else:
-                    params = UNKNOWN
-                    logger.debug(
-                        f"Default inference parameters are not available for the model {model_id} and "
-                        f"instance {instance_shape}."
-                    )
-                if params:
-                    # account for param that can have --arg but no values, e.g. --trust-remote-code
-                    default_params.extend(get_params_list(params))
+
+                # remove restricted params from the list as user cannot override them during deployment
+                for params in params_list:
+                    if params.split()[0] not in restricted_params_set:
+                        default_params.append(params)
 
         return default_params
 
@@ -629,7 +662,7 @@ class AquaDeploymentApp(AquaApp):
                 container_type_key = model.custom_metadata_list.get(
                     AQUA_DEPLOYMENT_CONTAINER_METADATA_NAME
                 ).value
-            except ValueError:
+            except ValueError as err:
                 message = (
                     f"{AQUA_DEPLOYMENT_CONTAINER_METADATA_NAME} key is not available in the custom metadata field "
                     f"for model {model_id}."
@@ -640,7 +673,7 @@ class AquaDeploymentApp(AquaApp):
                     raise AquaValueError(
                         f"{message}. For unverified Aqua models, container_family parameter should be "
                         f"set and value can be one of {', '.join(InferenceContainerTypeFamily.values())}."
-                    )
+                    ) from err
                 container_type_key = container_family
 
             container_config = get_container_config()
@@ -658,7 +691,7 @@ class AquaDeploymentApp(AquaApp):
                 f"Parameters {restricted_params} are set by Aqua "
                 f"and cannot be overridden or are invalid."
             )
-        return dict(valid=True)
+        return {"valid": True}
 
     @staticmethod
     def _find_restricted_params(
@@ -667,8 +700,7 @@ class AquaDeploymentApp(AquaApp):
         container_family: str,
     ) -> List[str]:
         """Returns a list of restricted params that user chooses to override when creating an Aqua deployment.
-        The default parameters coming from the container index json file cannot be overridden. In addition to this,
-        a set of parameters maintained in
+        The default parameters coming from the container index json file cannot be overridden.
 
         Parameters
         ----------
@@ -689,18 +721,9 @@ class AquaDeploymentApp(AquaApp):
             default_params_dict = get_params_dict(default_params)
             user_params_dict = get_params_dict(user_params)
 
-            for key, items in user_params_dict.items():
-                if (
-                    key in default_params_dict
-                    or (
-                        InferenceContainerType.CONTAINER_TYPE_VLLM in container_family
-                        and key in VLLMInferenceRestrictedParams
-                    )
-                    or (
-                        InferenceContainerType.CONTAINER_TYPE_TGI in container_family
-                        and key in TGIInferenceRestrictedParams
-                    )
-                ):
-                    restricted_params.append(key.lstrip("--"))
+            restricted_params_set = get_restricted_params_by_container(container_family)
+            for key, _items in user_params_dict.items():
+                if key in default_params_dict or key in restricted_params_set:
+                    restricted_params.append(key.lstrip("-"))
 
         return restricted_params

ads/aqua/modeldeployment/entities.py

@@ -1,12 +1,14 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright (c) 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 from dataclasses import dataclass, field
 from typing import Union
 
-from oci.data_science.models import ModelDeployment, ModelDeploymentSummary
+from oci.data_science.models import (
+    ModelDeployment,
+    ModelDeploymentSummary,
+)
 
 from ads.aqua.common.enums import Tags
 from ads.aqua.constants import UNKNOWN, UNKNOWN_DICT
@@ -24,18 +26,6 @@ class ModelParams:
     model: str = None
 
 
-class ContainerSpec:
-    """
-    Class to hold to hold keys within the container spec.
-    """
-
-    CONTAINER_SPEC = "containerSpec"
-    CLI_PARM = "cliParam"
-    SERVER_PORT = "serverPort"
-    HEALTH_CHECK_PORT = "healthCheckPort"
-    ENV_VARS = "envVars"
-
-
 @dataclass
 class ShapeInfo:
     instance_shape: str = None
@@ -61,6 +51,7 @@ class AquaDeployment(DataClassSerializable):
     lifecycle_details: str = None
     shape_info: field(default_factory=ShapeInfo) = None
     tags: dict = None
+    environment_variables: dict = None
 
     @classmethod
     def from_oci_model_deployment(
@@ -83,15 +74,12 @@ class AquaDeployment(DataClassSerializable):
         AquaDeployment:
             The instance of the Aqua model deployment.
         """
-        instance_configuration = (
-            oci_model_deployment.model_deployment_configuration_details.model_configuration_details.instance_configuration
-        )
+        instance_configuration = oci_model_deployment.model_deployment_configuration_details.model_configuration_details.instance_configuration
         instance_shape_config_details = (
             instance_configuration.model_deployment_instance_shape_config_details
         )
-        instance_count = (
-            oci_model_deployment.model_deployment_configuration_details.model_configuration_details.scaling_policy.instance_count
-        )
+        instance_count = oci_model_deployment.model_deployment_configuration_details.model_configuration_details.scaling_policy.instance_count
+        environment_variables = oci_model_deployment.model_deployment_configuration_details.environment_configuration_details.environment_variables
         shape_info = ShapeInfo(
             instance_shape=instance_configuration.instance_shape_name,
             instance_count=instance_count,
@@ -131,6 +119,7 @@ class AquaDeployment(DataClassSerializable):
                 region=region,
             ),
             tags=freeform_tags,
+            environment_variables=environment_variables,
         )