oracle-ads 2.13.6__py3-none-any.whl → 2.13.8__py3-none-any.whl

ads/aqua/modeldeployment/deployment.py

@@ -16,16 +16,14 @@ from ads.aqua.common.entities import (
     AquaMultiModelRef,
     ComputeShapeSummary,
     ContainerPath,
-    ContainerSpec,
 )
 from ads.aqua.common.enums import InferenceContainerTypeFamily, ModelFormat, Tags
 from ads.aqua.common.errors import AquaRuntimeError, AquaValueError
 from ads.aqua.common.utils import (
+    DEFINED_METADATA_TO_FILE_MAP,
     build_params_string,
     build_pydantic_error_message,
     get_combined_params,
-    get_container_config,
-    get_container_image,
     get_container_params_type,
     get_model_by_reference_paths,
     get_ocid_substring,
@@ -50,7 +48,7 @@ from ads.aqua.constants import (
 from ads.aqua.data import AquaResourceIdentifier
 from ads.aqua.finetuning.finetuning import FineTuneCustomMetadata
 from ads.aqua.model import AquaModelApp
-from ads.aqua.model.constants import ModelCustomMetadataFields
+from ads.aqua.model.constants import AquaModelMetadataKeys, ModelCustomMetadataFields
 from ads.aqua.modeldeployment.entities import (
     AquaDeployment,
     AquaDeploymentConfig,
@@ -67,7 +65,6 @@ from ads.config import (
     AQUA_DEPLOYMENT_CONTAINER_CMD_VAR_METADATA_NAME,
     AQUA_DEPLOYMENT_CONTAINER_METADATA_NAME,
     AQUA_DEPLOYMENT_CONTAINER_URI_METADATA_NAME,
-    AQUA_MODEL_DEPLOYMENT_CONFIG,
     COMPARTMENT_OCID,
     PROJECT_OCID,
 )
@@ -182,7 +179,7 @@ class AquaDeploymentApp(AquaApp):
         available_shapes = [
             shape.name.lower()
             for shape in self.list_shapes(
-                compartment_id=create_deployment_details.compartment_id
+                compartment_id=compartment_id
             )
         ]
 
@@ -193,7 +190,7 @@ class AquaDeploymentApp(AquaApp):
             )
 
         # Get container config
-        container_config = get_container_config()
+        container_config = self.get_container_config()
 
         # Create an AquaModelApp instance once to perform the deployment creation.
         model_app = AquaModelApp()
@@ -228,17 +225,13 @@ class AquaDeploymentApp(AquaApp):
             except ConfigValidationError as err:
                 raise AquaValueError(f"{err}") from err
 
-            service_inference_containers = (
-                AquaContainerConfig.from_container_index_json(
-                    config=container_config
-                ).inference.values()
-            )
+            service_inference_containers = container_config.inference.values()
 
             supported_container_families = [
                 container_config_item.family
                 for container_config_item in service_inference_containers
                 if any(
-                    usage in container_config_item.usages
+                    usage.upper() in container_config_item.usages
                     for usage in [Usage.MULTI_MODEL, Usage.OTHER]
                 )
             ]
@@ -409,7 +402,7 @@ class AquaDeploymentApp(AquaApp):
 
         container_image_uri = (
             create_deployment_details.container_image_uri
-            or get_container_image(container_type=container_type_key)
+            or self.get_container_image(container_type=container_type_key)
         )
         if not container_image_uri:
             try:
@@ -475,22 +468,21 @@ class AquaDeploymentApp(AquaApp):
             env_var.update({"BASE_MODEL_FILE": f"{model_file}"})
             tags.update({Tags.MODEL_ARTIFACT_FILE: model_file})
 
-        # todo: use AquaContainerConfig.from_container_index_json instead.
         # Fetch the startup cli command for the container
         # container_index.json will have "containerSpec" section which will provide the cli params for
         # a given container family
-        container_spec = container_config.get(ContainerSpec.CONTAINER_SPEC, {}).get(
-            container_type_key, {}
-        )
+        container_config = self.get_container_config_item(container_type_key)
+
+        container_spec = container_config.spec if container_config else UNKNOWN
         # these params cannot be overridden for Aqua deployments
-        params = container_spec.get(ContainerSpec.CLI_PARM, "")
-        server_port = create_deployment_details.server_port or container_spec.get(
-            ContainerSpec.SERVER_PORT
-        )  # Give precedence to the input parameter
-        health_check_port = (
-            create_deployment_details.health_check_port
-            or container_spec.get(ContainerSpec.HEALTH_CHECK_PORT)
-        )  # Give precedence to the input parameter
+        params = container_spec.cli_param if container_spec else UNKNOWN
+        server_port = create_deployment_details.server_port or (
+            container_spec.server_port if container_spec else None
+        )
+        # Give precendece to the input parameter
+        health_check_port = create_deployment_details.health_check_port or (
+            container_spec.health_check_port if container_spec else None
+        )
 
         deployment_config = self.get_deployment_config(model_id=config_source_id)
 
@@ -527,9 +519,10 @@ class AquaDeploymentApp(AquaApp):
         params = f"{params} {deployment_params}".strip()
         if params:
             env_var.update({"PARAMS": params})
-
-        for env in container_spec.get(ContainerSpec.ENV_VARS, []):
+        env_vars = container_spec.env_vars if container_spec else []
+        for env in env_vars:
             if isinstance(env, dict):
+                env = {k: v for k, v in env.items() if v}
                 for key, _ in env.items():
                     if key not in env_var:
                         env_var.update(env)
@@ -559,7 +552,7 @@ class AquaDeploymentApp(AquaApp):
         aqua_model: DataScienceModel,
         model_config_summary: ModelDeploymentConfigSummary,
         create_deployment_details: CreateModelDeploymentDetails,
-        container_config: Dict,
+        container_config: AquaContainerConfig,
     ) -> AquaDeployment:
         """Builds the environment variables required by multi deployment container and creates the deployment.
 
@@ -587,11 +580,10 @@ class AquaDeploymentApp(AquaApp):
             model=aqua_model,
             container_family=create_deployment_details.container_family,
         )
-        container_spec = container_config.get(
-            ContainerSpec.CONTAINER_SPEC, UNKNOWN_DICT
-        ).get(container_type_key, UNKNOWN_DICT)
+        container_config = self.get_container_config_item(container_type_key)
+        container_spec = container_config.spec if container_config else UNKNOWN
 
-        container_params = container_spec.get(ContainerSpec.CLI_PARM, UNKNOWN).strip()
+        container_params = container_spec.cli_param if container_spec else UNKNOWN
 
         for model in create_deployment_details.models:
             user_params = build_params_string(model.env_var)
@@ -658,8 +650,10 @@ class AquaDeploymentApp(AquaApp):
 
         env_var.update({AQUA_MULTI_MODEL_CONFIG: json.dumps({"models": model_config})})
 
-        for env in container_spec.get(ContainerSpec.ENV_VARS, []):
+        env_vars = container_spec.env_vars if container_spec else []
+        for env in env_vars:
             if isinstance(env, dict):
+                env = {k: v for k, v in env.items() if v}
                 for key, _ in env.items():
                     if key not in env_var:
                         env_var.update(env)
@@ -668,14 +662,13 @@ class AquaDeploymentApp(AquaApp):
 
         container_image_uri = (
             create_deployment_details.container_image_uri
-            or get_container_image(container_type=container_type_key)
+            or self.get_container_image(container_type=container_type_key)
         )
-        server_port = create_deployment_details.server_port or container_spec.get(
-            ContainerSpec.SERVER_PORT
+        server_port = create_deployment_details.server_port or (
+            container_spec.server_port if container_spec else None
         )
-        health_check_port = (
-            create_deployment_details.health_check_port
-            or container_spec.get(ContainerSpec.HEALTH_CHECK_PORT)
+        health_check_port = create_deployment_details.health_check_port or (
+            container_spec.health_check_port if container_spec else None
         )
         tags = {
             Tags.AQUA_MODEL_ID_TAG: aqua_model.id,
@@ -1054,7 +1047,20 @@ class AquaDeploymentApp(AquaApp):
         AquaDeploymentConfig:
             An instance of AquaDeploymentConfig.
         """
-        config = self.get_config(model_id, AQUA_MODEL_DEPLOYMENT_CONFIG).config
+        config = self.get_config_from_metadata(
+            model_id, AquaModelMetadataKeys.DEPLOYMENT_CONFIGURATION
+        ).config
+        if config:
+            logger.info(
+                f"Fetched {AquaModelMetadataKeys.DEPLOYMENT_CONFIGURATION} from defined metadata for model: {model_id}."
+            )
+            return AquaDeploymentConfig(**(config or UNKNOWN_DICT))
+        config = self.get_config(
+            model_id,
+            DEFINED_METADATA_TO_FILE_MAP.get(
+                AquaModelMetadataKeys.DEPLOYMENT_CONFIGURATION.lower()
+            ),
+        ).config
         if not config:
             logger.debug(
                 f"Deployment config for custom model: {model_id} is not available. Use defaults."
@@ -1079,7 +1085,7 @@ class AquaDeploymentApp(AquaApp):
         https://github.com/oracle-samples/oci-data-science-ai-samples/blob/main/ai-quick-actions/multimodel-deployment-tips.md#get_multimodel_deployment_config
 
         CLI example:
-        ads aqua deployment get_multimodel_deployment_config --model_ids '["ocid1.datasciencemodel.oc1.iad.OCID"]'
+        ads aqua deployment get_multimodel_deployment_config --model_ids '["md_ocid1","md_ocid2"]'
 
         If a primary model ID is provided, GPU allocation will prioritize that model
         when selecting compatible shapes.
@@ -1230,11 +1236,9 @@ class AquaDeploymentApp(AquaApp):
                 model=model, container_family=container_family
             )
 
-            container_config = get_container_config()
-            container_spec = container_config.get(ContainerSpec.CONTAINER_SPEC, {}).get(
-                container_type_key, {}
-            )
-            cli_params = container_spec.get(ContainerSpec.CLI_PARM, "")
+            container_config = self.get_container_config_item(container_type_key)
+            container_spec = container_config.spec if container_config else UNKNOWN
+            cli_params = container_spec.cli_param if container_spec else UNKNOWN
 
             restricted_params = self._find_restricted_params(
                 cli_params, params, container_type_key

ads/aqua/modeldeployment/utils.py

@@ -11,7 +11,8 @@ from concurrent.futures import ThreadPoolExecutor
 from typing import Dict, List, Optional
 
 from ads.aqua.app import AquaApp
-from ads.aqua.common.entities import ComputeShapeSummary
+from ads.aqua.common.entities import ComputeShapeSummary, ModelConfigResult
+from ads.aqua.model.constants import AquaModelMetadataKeys
 from ads.aqua.modeldeployment.entities import (
     AquaDeploymentConfig,
     ConfigurationItem,
@@ -183,17 +184,34 @@ class MultiModelDeploymentConfigLoader:
         """Fetches deployment configurations in parallel using ThreadPoolExecutor."""
         with ThreadPoolExecutor(max_workers=self.MAX_WORKERS) as executor:
             results = executor.map(
-                lambda model_id: self.deployment_app.get_config(
-                    model_id, AQUA_MODEL_DEPLOYMENT_CONFIG
-                ).config,
+                self._fetch_deployment_config_from_metadata_and_oss,
                 model_ids,
             )
 
         return {
-            model_id: AquaDeploymentConfig(**config)
+            model_id: AquaDeploymentConfig(**config.config)
             for model_id, config in zip(model_ids, results)
         }
 
+    def _fetch_deployment_config_from_metadata_and_oss(
+        self, model_id
+    ) -> ModelConfigResult:
+        config = self.deployment_app.get_config_from_metadata(
+            model_id, AquaModelMetadataKeys.DEPLOYMENT_CONFIGURATION
+        )
+        if config:
+            logger.info(
+                f"Fetched metadata key '{AquaModelMetadataKeys.DEPLOYMENT_CONFIGURATION}' from defined metadata for model '{model_id}'"
+            )
+            return config
+        else:
+            logger.info(
+                f"Fetching '{AquaModelMetadataKeys.DEPLOYMENT_CONFIGURATION}' from object storage bucket for {model_id}'"
+            )
+            return self.deployment_app.get_config(
+                model_id, AQUA_MODEL_DEPLOYMENT_CONFIG
+            )
+
     def _extract_model_shape_gpu(
         self,
         deployment_configs: Dict[str, AquaDeploymentConfig],

ads/aqua/ui.py

@@ -13,7 +13,7 @@ from ads.aqua import logger
 from ads.aqua.app import AquaApp
 from ads.aqua.common.enums import Tags
 from ads.aqua.common.errors import AquaResourceAccessError, AquaValueError
-from ads.aqua.common.utils import get_container_config, sanitize_response
+from ads.aqua.common.utils import sanitize_response
 from ads.aqua.config.container_config import AquaContainerConfig
 from ads.aqua.constants import PRIVATE_ENDPOINT_TYPE
 from ads.common import oci_client as oc
@@ -494,7 +494,6 @@ class AquaUIApp(AquaApp):
         AquaContainerConfig
             The AQUA containers configurations.
         """
-        return AquaContainerConfig.from_container_index_json(
-            config=get_container_config(),
-            enable_spec=True,
+        return AquaContainerConfig.from_service_config(
+            service_containers=self.list_service_containers()
         )

ads/config.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-# Copyright (c) 2020, 2024 Oracle and/or its affiliates.
+# Copyright (c) 2020, 2025 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 import contextlib
@@ -56,6 +56,7 @@ AQUA_MODEL_FINETUNING_CONFIG = os.environ.get(
 AQUA_CONTAINER_INDEX_CONFIG = os.environ.get(
     "AQUA_CONTAINER_INDEX_CONFIG", "container_index.json"
 )
+AQUA_SERVICE_MODELS = "aqua-service-models-compartment"
 AQUA_DEPLOYMENT_CONTAINER_METADATA_NAME = "deployment-container"
 AQUA_FINETUNING_CONTAINER_METADATA_NAME = "finetune-container"
 AQUA_EVALUATION_CONTAINER_METADATA_NAME = "evaluation-container"
@@ -84,7 +85,6 @@ USER = "USER"
 SERVICE = "SERVICE"
 
 
-
 THREADED_DEFAULT_TIMEOUT = 5
 try:
     THREADED_DEFAULT_TIMEOUT = int(

ads/model/datascience_model.py

@@ -2240,6 +2240,35 @@ class DataScienceModel(Builder):
             # model found case
             self.model_file_description["models"].pop(modelSearchIdx)
 
+    def if_model_custom_metadata_artifact_exist(
+        self, metadata_key_name: str, **kwargs
+    ) -> bool:
+        """Checks if the custom metadata artifact exists for the model.
+
+        Parameters
+        ----------
+        metadata_key_name: str
+            Custom metadata key name
+        **kwargs :
+            Additional keyword arguments passed in head_model_artifact.
+
+        Returns
+        -------
+        bool
+            Whether the artifact exists.
+        """
+
+        try:
+            response = self.dsc_model.head_custom_metadata_artifact(
+                metadata_key_name=metadata_key_name, **kwargs
+            )
+            return int(response.status) == 200
+        except Exception as ex:
+            logger.info(
+                f"Error fetching custom metadata: {metadata_key_name} for model {self.id}. {ex}"
+            )
+            return False
+
     def create_custom_metadata_artifact(
         self,
         metadata_key_name: str,

ads/model/service/oci_datascience_model.py

@@ -110,7 +110,7 @@ def check_for_model_id(msg: str = MODEL_NEEDS_TO_BE_SAVED):
 def convert_model_metadata_response(
     headers: Union[Dict, CaseInsensitiveDict], status: int
 ) -> ModelMetadataArtifactDetails:
-    return ModelMetadataArtifactDetails(headers=headers, status=str(status))
+    return ModelMetadataArtifactDetails(headers=dict(headers), status=str(status))
 
 
 class OCIDataScienceModel(

ads/opctl/operator/lowcode/anomaly/model/anomaly_dataset.py

@@ -4,16 +4,18 @@
 # Copyright (c) 2023 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
-from ..operator_config import AnomalyOperatorSpec
+import pandas as pd
+
+from ads.opctl import logger
+from ads.opctl.operator.lowcode.anomaly.const import OutputColumns
+from ads.opctl.operator.lowcode.anomaly.utils import get_frequency_of_datetime
+from ads.opctl.operator.lowcode.common.data import AbstractData
 from ads.opctl.operator.lowcode.common.utils import (
     default_signer,
     merge_category_columns,
 )
-from ads.opctl.operator.lowcode.common.data import AbstractData
-from ads.opctl.operator.lowcode.anomaly.utils import get_frequency_of_datetime
-from ads.opctl import logger
-import pandas as pd
-from ads.opctl.operator.lowcode.anomaly.const import OutputColumns
+
+from ..operator_config import AnomalyOperatorSpec
 
 
 class AnomalyData(AbstractData):

ads/opctl/operator/lowcode/anomaly/model/base_model.py

@@ -53,6 +53,7 @@ class AnomalyOperatorBaseModel(ABC):
         self.config: AnomalyOperatorConfig = config
         self.spec: AnomalyOperatorSpec = config.spec
         self.datasets = datasets
+
         if self.spec.validation_data is not None:
             self.X_valid_dict = self.datasets.valid_data.X_valid_dict
             self.y_valid_dict = self.datasets.valid_data.y_valid_dict
@@ -74,7 +75,6 @@ class AnomalyOperatorBaseModel(ABC):
             logger.warning(f"Found exception: {e}")
             if self.spec.datetime_column:
                 anomaly_output = self._fallback_build_model()
-            raise e
 
         elapsed_time = time.time() - start_time
 

ads/opctl/operator/lowcode/anomaly/operator_config.py

@@ -9,15 +9,16 @@ from dataclasses import dataclass, field
 from typing import Dict, List
 
 from ads.common.serializer import DataClassSerializable
-from ads.opctl.operator.common.utils import _load_yaml_from_uri
 from ads.opctl.operator.common.operator_config import (
+    InputData,
     OperatorConfig,
     OutputDirectory,
-    InputData,
 )
-from .const import SupportedModels
+from ads.opctl.operator.common.utils import _load_yaml_from_uri
 from ads.opctl.operator.lowcode.common.utils import find_output_dirname
 
+from .const import SupportedModels
+
 
 @dataclass(repr=True)
 class ValidationData(InputData):
@@ -61,6 +62,7 @@ class AnomalyOperatorSpec(DataClassSerializable):
     test_data: TestData = field(default_factory=TestData)
     validation_data: ValidationData = field(default_factory=ValidationData)
     output_directory: OutputDirectory = field(default_factory=OutputDirectory)
+    preprocessing: DataPreprocessor = field(default_factory=DataPreprocessor)
     report_file_name: str = None
     report_title: str = None
     report_theme: str = None

ads/opctl/operator/lowcode/common/transformations.py

@@ -91,6 +91,8 @@ class Transformations(ABC):
                     logger.info("Skipping outlier treatment because it is disabled")
             elif self.name == "additional_data":
                 clean_df = self._missing_value_imputation_add(clean_df)
+            elif self.name == "input_data" and self.preprocessing.steps.missing_value_imputation:
+                clean_df = self._fill_na(clean_df)
         else:
             logger.info(
                 "Skipping all preprocessing steps because preprocessing is disabled"

ads/opctl/operator/lowcode/forecast/model/automlx.py

@@ -159,6 +159,7 @@ class AutoMLXOperatorModel(ForecastOperatorBaseModel):
                 self.models[s_id] = {}
                 self.models[s_id]["model"] = model
                 self.models[s_id]["le"] = self.le[s_id]
+                self.models[s_id]["score"] = self.get_validation_score_and_metric(model)
 
                 # In case of Naive model, model.forecast function call does not return confidence intervals.
                 if f"{target}_ci_upper" not in summary_frame:
@@ -511,3 +512,31 @@ class AutoMLXOperatorModel(ForecastOperatorBaseModel):
                     f"Failed to generate explanations for series {s_id} with error: {e}."
                 )
                 logger.debug(f"Full Traceback: {traceback.format_exc()}")
+
+    def get_validation_score_and_metric(self, model):
+        trials = model.completed_trials_summary_
+        model_params = model.selected_model_params_
+        if len(trials) > 0:
+            score_col = [col for col in trials.columns if "Score" in col][0]
+            validation_score = trials[trials.Hyperparameters == model_params][score_col].iloc[0]
+        else:
+            validation_score = 0
+        return -1 * validation_score
+
+    def generate_train_metrics(self) -> pd.DataFrame:
+        """
+        Generate Training Metrics when fitted data is not available.
+        """
+        total_metrics = pd.DataFrame()
+        for s_id in self.forecast_output.list_series_ids():
+            try:
+                metrics = {self.spec.metric.upper(): self.models[s_id]["score"]}
+                metrics_df = pd.DataFrame.from_dict(metrics, orient="index", columns=[s_id])
+                logger.warning("AutoMLX failed to generate training metrics. Recovering validation loss instead")
+                total_metrics = pd.concat([total_metrics, metrics_df], axis=1)
+            except Exception as e:
+                logger.debug(
+                    f"Failed to generate training metrics for target_series: {s_id}"
+                )
+                logger.debug(f"Error: {e}")
+        return total_metrics