oracle-ads 2.11.15__py3-none-any.whl → 2.11.17__py3-none-any.whl

ads/aqua/modeldeployment/deployment.py

@@ -3,9 +3,10 @@
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 import logging
-from typing import Dict, List, Union
+from typing import Dict, List, Optional, Union
 
 from ads.aqua.app import AquaApp, logger
+from ads.aqua.common.entities import ContainerSpec
 from ads.aqua.common.enums import (
     InferenceContainerTypeFamily,
     Tags,
@@ -25,6 +26,7 @@ from ads.aqua.common.utils import (
     load_config,
 )
 from ads.aqua.constants import (
+    AQUA_MODEL_ARTIFACT_FILE,
     AQUA_MODEL_TYPE_CUSTOM,
     AQUA_MODEL_TYPE_SERVICE,
     MODEL_BY_REFERENCE_OSS_PATH_KEY,
@@ -37,8 +39,8 @@ from ads.aqua.model import AquaModelApp
 from ads.aqua.modeldeployment.entities import (
     AquaDeployment,
     AquaDeploymentDetail,
-    ContainerSpec,
 )
+from ads.aqua.ui import ModelFormat
 from ads.common.object_storage_details import ObjectStorageDetails
 from ads.common.utils import get_log_links
 from ads.config import (
@@ -85,23 +87,26 @@ class AquaDeploymentApp(AquaApp):
 
     @telemetry(entry_point="plugin=deployment&action=create", name="aqua")
     def create(
-        self,
-        model_id: str,
-        instance_shape: str,
-        display_name: str,
-        instance_count: int = None,
-        log_group_id: str = None,
-        access_log_id: str = None,
-        predict_log_id: str = None,
-        compartment_id: str = None,
-        project_id: str = None,
-        description: str = None,
-        bandwidth_mbps: int = None,
-        web_concurrency: int = None,
-        server_port: int = None,
-        health_check_port: int = None,
-        env_var: Dict = None,
-        container_family: str = None,
+            self,
+            model_id: str,
+            instance_shape: str,
+            display_name: str,
+            instance_count: int = None,
+            log_group_id: str = None,
+            access_log_id: str = None,
+            predict_log_id: str = None,
+            compartment_id: str = None,
+            project_id: str = None,
+            description: str = None,
+            bandwidth_mbps: int = None,
+            web_concurrency: int = None,
+            server_port: int = None,
+            health_check_port: int = None,
+            env_var: Dict = None,
+            container_family: str = None,
+            memory_in_gbs: Optional[float] = None,
+            ocpus: Optional[float] = None,
+            model_file: Optional[str] = None,
     ) -> "AquaDeployment":
         """
         Creates a new Aqua deployment
@@ -142,6 +147,12 @@ class AquaDeploymentApp(AquaApp):
             Environment variable for the deployment, by default None.
         container_family: str
             The image family of model deployment container runtime. Required for unverified Aqua models.
+        memory_in_gbs: float
+            The memory in gbs for the shape selected.
+        ocpus: float
+            The ocpu count for the shape selected.
+        model_file: str
+            The file used for model deployment.
         Returns
         -------
         AquaDeployment
@@ -239,7 +250,7 @@ class AquaDeploymentApp(AquaApp):
         try:
             # Check if the container override flag is set. If set, then the user has chosen custom image
             if aqua_model.custom_metadata_list.get(
-                AQUA_DEPLOYMENT_CONTAINER_OVERRIDE_FLAG_METADATA_NAME
+                    AQUA_DEPLOYMENT_CONTAINER_OVERRIDE_FLAG_METADATA_NAME
             ).value:
                 is_custom_container = True
         except Exception:
@@ -258,8 +269,36 @@ class AquaDeploymentApp(AquaApp):
             f"Aqua Image used for deploying {aqua_model.id} : {container_image}"
         )
 
+        model_formats_str = aqua_model.freeform_tags.get(
+            Tags.MODEL_FORMAT, ModelFormat.SAFETENSORS.value
+        ).upper()
+        model_format = model_formats_str.split(",")
+
+        # Figure out a better way to handle this in future release
+        if ModelFormat.GGUF.value in model_format and container_type_key.lower() == InferenceContainerTypeFamily.AQUA_LLAMA_CPP_CONTAINER_FAMILY:
+            if model_file is not None:
+                logger.info(
+                    f"Overriding {model_file} as model_file for model {aqua_model.id}."
+                )
+            else:
+                try:
+                    model_file = aqua_model.custom_metadata_list.get(
+                        AQUA_MODEL_ARTIFACT_FILE
+                    ).value
+                except ValueError as err:
+                    raise AquaValueError(
+                        f"{AQUA_MODEL_ARTIFACT_FILE} key is not available in the custom metadata field "
+                        f"for model {aqua_model.id}. Either register the model with a default model_file or pass "
+                        f"as a parameter when creating a deployment."
+                    ) from err
+
+            env_var.update({"BASE_MODEL_FILE": f"{model_file}"})
+            tags.update({Tags.MODEL_ARTIFACT_FILE: model_file})
+
+        # todo: use AquaContainerConfig.from_container_index_json instead.
         # Fetch the startup cli command for the container
-        # container_index.json will have "containerSpec" section which will provide the cli params for a given container family
+        # container_index.json will have "containerSpec" section which will provide the cli params for
+        # a given container family
         container_config = get_container_config()
         container_spec = container_config.get(ContainerSpec.CONTAINER_SPEC, {}).get(
             container_type_key, {}
@@ -285,6 +324,18 @@ class AquaDeploymentApp(AquaApp):
         # validate user provided params
         user_params = env_var.get("PARAMS", UNKNOWN)
         if user_params:
+            # todo: remove this check in the future version, logic to be moved to container_index
+            if (
+                    container_type_key.lower()
+                    == InferenceContainerTypeFamily.AQUA_LLAMA_CPP_CONTAINER_FAMILY
+            ):
+                # AQUA_LLAMA_CPP_CONTAINER_FAMILY container uses uvicorn that required model/server params
+                # to be set as env vars
+                raise AquaValueError(
+                    f"Currently, parameters cannot be overridden for the container: {container_image}. Please proceed "
+                    f"with deployment without parameter overrides."
+                )
+
             restricted_params = self._find_restricted_params(
                 params, user_params, container_type_key
             )
@@ -296,13 +347,15 @@ class AquaDeploymentApp(AquaApp):
 
         deployment_params = get_combined_params(config_params, user_params)
 
-        if deployment_params:
-            params = f"{params} {deployment_params}"
+        params = f"{params} {deployment_params}".strip()
+        if params:
+            env_var.update({"PARAMS": params})
 
-        env_var.update({"PARAMS": params})
         for env in container_spec.get(ContainerSpec.ENV_VARS, []):
             if isinstance(env, dict):
-                env_var.update(env)
+                for key, _items in env.items():
+                    if key not in env_var:
+                        env_var.update(env)
 
         logging.info(f"Env vars used for deploying {aqua_model.id} :{env_var}")
 
@@ -325,6 +378,11 @@ class AquaDeploymentApp(AquaApp):
                 log_id=predict_log_id,
             )
         )
+        if memory_in_gbs and ocpus and infrastructure.shape_name.endswith("Flex"):
+            infrastructure.with_shape_config_details(
+                ocpus=ocpus,
+                memory_in_gbs=memory_in_gbs,
+            )
         # configure model deployment runtime
         container_runtime = (
             ModelDeploymentContainerRuntime()
@@ -338,6 +396,7 @@ class AquaDeploymentApp(AquaApp):
             .with_overwrite_existing_artifact(True)
             .with_remove_existing_artifact(True)
         )
+
         # configure model deployment and deploy model on container runtime
         deployment = (
             ModelDeployment()
@@ -401,8 +460,8 @@ class AquaDeploymentApp(AquaApp):
         for model_deployment in model_deployments:
             oci_aqua = (
                 (
-                    Tags.AQUA_TAG in model_deployment.freeform_tags
-                    or Tags.AQUA_TAG.lower() in model_deployment.freeform_tags
+                        Tags.AQUA_TAG in model_deployment.freeform_tags
+                        or Tags.AQUA_TAG.lower() in model_deployment.freeform_tags
                 )
                 if model_deployment.freeform_tags
                 else False
@@ -456,8 +515,8 @@ class AquaDeploymentApp(AquaApp):
 
         oci_aqua = (
             (
-                Tags.AQUA_TAG in model_deployment.freeform_tags
-                or Tags.AQUA_TAG.lower() in model_deployment.freeform_tags
+                    Tags.AQUA_TAG in model_deployment.freeform_tags
+                    or Tags.AQUA_TAG.lower() in model_deployment.freeform_tags
             )
             if model_deployment.freeform_tags
             else False
@@ -474,8 +533,8 @@ class AquaDeploymentApp(AquaApp):
         log_group_name = ""
 
         logs = (
-            model_deployment.category_log_details.access
-            or model_deployment.category_log_details.predict
+                model_deployment.category_log_details.access
+                or model_deployment.category_log_details.predict
         )
         if logs:
             log_id = logs.log_id
@@ -530,9 +589,9 @@ class AquaDeploymentApp(AquaApp):
         return config
 
     def get_deployment_default_params(
-        self,
-        model_id: str,
-        instance_shape: str,
+            self,
+            model_id: str,
+            instance_shape: str,
     ) -> List[str]:
         """Gets the default params set in the deployment configs for the given model and instance shape.
 
@@ -564,8 +623,8 @@ class AquaDeploymentApp(AquaApp):
             )
 
         if (
-            container_type_key
-            and container_type_key in InferenceContainerTypeFamily.values()
+                container_type_key
+                and container_type_key in InferenceContainerTypeFamily.values()
         ):
             deployment_config = self.get_deployment_config(model_id)
             config_params = (
@@ -588,10 +647,10 @@ class AquaDeploymentApp(AquaApp):
         return default_params
 
     def validate_deployment_params(
-        self,
-        model_id: str,
-        params: List[str] = None,
-        container_family: str = None,
+            self,
+            model_id: str,
+            params: List[str] = None,
+            container_family: str = None,
     ) -> Dict:
         """Validate if the deployment parameters passed by the user can be overridden. Parameter values are not
         validated, only param keys are validated.
@@ -650,9 +709,9 @@ class AquaDeploymentApp(AquaApp):
 
     @staticmethod
     def _find_restricted_params(
-        default_params: Union[str, List[str]],
-        user_params: Union[str, List[str]],
-        container_family: str,
+            default_params: Union[str, List[str]],
+            user_params: Union[str, List[str]],
+            container_family: str,
     ) -> List[str]:
         """Returns a list of restricted params that user chooses to override when creating an Aqua deployment.
         The default parameters coming from the container index json file cannot be overridden.

ads/aqua/modeldeployment/entities.py

@@ -1,12 +1,14 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright (c) 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 from dataclasses import dataclass, field
 from typing import Union
 
-from oci.data_science.models import ModelDeployment, ModelDeploymentSummary
+from oci.data_science.models import (
+    ModelDeployment,
+    ModelDeploymentSummary,
+)
 
 from ads.aqua.common.enums import Tags
 from ads.aqua.constants import UNKNOWN, UNKNOWN_DICT
@@ -24,18 +26,6 @@ class ModelParams:
     model: str = None
 
 
-class ContainerSpec:
-    """
-    Class to hold to hold keys within the container spec.
-    """
-
-    CONTAINER_SPEC = "containerSpec"
-    CLI_PARM = "cliParam"
-    SERVER_PORT = "serverPort"
-    HEALTH_CHECK_PORT = "healthCheckPort"
-    ENV_VARS = "envVars"
-
-
 @dataclass
 class ShapeInfo:
     instance_shape: str = None
@@ -61,6 +51,7 @@ class AquaDeployment(DataClassSerializable):
     lifecycle_details: str = None
     shape_info: field(default_factory=ShapeInfo) = None
     tags: dict = None
+    environment_variables: dict = None
 
     @classmethod
     def from_oci_model_deployment(
@@ -83,15 +74,12 @@ class AquaDeployment(DataClassSerializable):
         AquaDeployment:
             The instance of the Aqua model deployment.
         """
-        instance_configuration = (
-            oci_model_deployment.model_deployment_configuration_details.model_configuration_details.instance_configuration
-        )
+        instance_configuration = oci_model_deployment.model_deployment_configuration_details.model_configuration_details.instance_configuration
         instance_shape_config_details = (
             instance_configuration.model_deployment_instance_shape_config_details
         )
-        instance_count = (
-            oci_model_deployment.model_deployment_configuration_details.model_configuration_details.scaling_policy.instance_count
-        )
+        instance_count = oci_model_deployment.model_deployment_configuration_details.model_configuration_details.scaling_policy.instance_count
+        environment_variables = oci_model_deployment.model_deployment_configuration_details.environment_configuration_details.environment_variables
         shape_info = ShapeInfo(
             instance_shape=instance_configuration.instance_shape_name,
             instance_count=instance_count,
@@ -131,6 +119,7 @@ class AquaDeployment(DataClassSerializable):
                 region=region,
             ),
             tags=freeform_tags,
+            environment_variables=environment_variables,
         )
 
 

ads/aqua/ui.py

@@ -1,12 +1,12 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright (c) 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 import concurrent.futures
-from dataclasses import dataclass, field
+from dataclasses import dataclass, field, fields
 from datetime import datetime, timedelta
+from enum import Enum
 from threading import Lock
-from typing import Dict, List
+from typing import Dict, List, Optional
 
 from cachetools import TTLCache
 from oci.exceptions import ServiceError
@@ -14,6 +14,7 @@ from oci.identity.models import Compartment
 
 from ads.aqua import logger
 from ads.aqua.app import AquaApp
+from ads.aqua.common.entities import ContainerSpec
 from ads.aqua.common.enums import Tags
 from ads.aqua.common.errors import AquaResourceAccessError, AquaValueError
 from ads.aqua.common.utils import get_container_config, load_config, sanitize_response
@@ -31,14 +32,84 @@ from ads.config import (
 from ads.telemetry import telemetry
 
 
+class ModelFormat(Enum):
+    GGUF = "GGUF"
+    SAFETENSORS = "SAFETENSORS"
+    UNKNOWN = "UNKNOWN"
+
+    def to_dict(self):
+        return self.value
+
+
+# todo: the container config spec information is shared across ui and deployment modules, move them
+#   within ads.aqua.common.entities. In that case, check for circular imports due to usage of get_container_config.
+
+
+@dataclass(repr=False)
+class AquaContainerEvaluationConfig(DataClassSerializable):
+    """
+    Represents the evaluation configuration for the container.
+    """
+
+    inference_max_threads: Optional[int] = None
+    inference_rps: Optional[int] = None
+    inference_timeout: Optional[int] = None
+    inference_retries: Optional[int] = None
+    inference_backoff_factor: Optional[int] = None
+    inference_delay: Optional[int] = None
+
+    @classmethod
+    def from_config(cls, config: dict) -> "AquaContainerEvaluationConfig":
+        return cls(
+            inference_max_threads=config.get("inference_max_threads"),
+            inference_rps=config.get("inference_rps"),
+            inference_timeout=config.get("inference_timeout"),
+            inference_retries=config.get("inference_retries"),
+            inference_backoff_factor=config.get("inference_backoff_factor"),
+            inference_delay=config.get("inference_delay"),
+        )
+
+    def to_filtered_dict(self):
+        return {
+            field.name: getattr(self, field.name)
+            for field in fields(self)
+            if getattr(self, field.name) is not None
+        }
+
+
+@dataclass(repr=False)
+class AquaContainerConfigSpec(DataClassSerializable):
+    cli_param: str = None
+    server_port: str = None
+    health_check_port: str = None
+    env_vars: List[dict] = None
+    restricted_params: List[str] = None
+    evaluation_configuration: AquaContainerEvaluationConfig = field(
+        default_factory=AquaContainerEvaluationConfig
+    )
+
+
 @dataclass(repr=False)
 class AquaContainerConfigItem(DataClassSerializable):
     """Represents an item of the AQUA container configuration."""
 
+    class Platform(Enum):
+        ARM_CPU = "ARM_CPU"
+        NVIDIA_GPU = "NVIDIA_GPU"
+
+        def to_dict(self):
+            return self.value
+
+        def __repr__(self):
+            return repr(self.value)
+
     name: str = None
     version: str = None
     display_name: str = None
     family: str = None
+    platforms: List[Platform] = None
+    model_formats: List[ModelFormat] = None
+    spec: AquaContainerConfigSpec = field(default_factory=AquaContainerConfigSpec)
 
 
 @dataclass(repr=False)
@@ -47,12 +118,23 @@ class AquaContainerConfig(DataClassSerializable):
     Represents a configuration with AQUA containers to be returned to the client.
     """
 
-    inference: List[AquaContainerConfigItem] = field(default_factory=list)
-    finetune: List[AquaContainerConfigItem] = field(default_factory=list)
-    evaluate: List[AquaContainerConfigItem] = field(default_factory=list)
+    inference: Dict[str, AquaContainerConfigItem] = field(default_factory=dict)
+    finetune: Dict[str, AquaContainerConfigItem] = field(default_factory=dict)
+    evaluate: Dict[str, AquaContainerConfigItem] = field(default_factory=dict)
+
+    def to_dict(self):
+        return {
+            "inference": list(self.inference.values()),
+            "finetune": list(self.finetune.values()),
+            "evaluate": list(self.evaluate.values()),
+        }
 
     @classmethod
-    def from_container_index_json(cls, config: Dict) -> "AquaContainerConfig":
+    def from_container_index_json(
+        cls,
+        config: Optional[Dict] = None,
+        enable_spec: Optional[bool] = False,
+    ) -> "AquaContainerConfig":
         """
         Create an AquaContainerConfig instance from a container index JSON.
 
@@ -60,21 +142,39 @@ class AquaContainerConfig(DataClassSerializable):
         ----------
         config : Dict
             The container index JSON.
+        enable_spec: bool
+            flag to check if container specification details should be fetched.
 
         Returns
         -------
         AquaContainerConfig
             The container configuration instance.
         """
-        config = config or {}
-        inference_items = []
-        finetune_items = []
-        evaluate_items = []
+        if not config:
+            config = get_container_config()
+        inference_items = {}
+        finetune_items = {}
+        evaluate_items = {}
 
         # extract inference containers
         for container_type, containers in config.items():
             if isinstance(containers, list):
                 for container in containers:
+                    platforms = [
+                        AquaContainerConfigItem.Platform[platform]
+                        for platform in container.get("platforms", [])
+                    ]
+                    model_formats = [
+                        ModelFormat[model_format]
+                        for model_format in container.get("modelFormats", [])
+                    ]
+                    container_spec = (
+                        config.get(ContainerSpec.CONTAINER_SPEC, {}).get(
+                            container_type, {}
+                        )
+                        if enable_spec
+                        else None
+                    )
                     container_item = AquaContainerConfigItem(
                         name=container.get("name", ""),
                         version=container.get("version", ""),
@@ -82,13 +182,35 @@ class AquaContainerConfig(DataClassSerializable):
                             "displayName", container.get("version", "")
                         ),
                         family=container_type,
+                        platforms=platforms,
+                        model_formats=model_formats,
+                        spec=AquaContainerConfigSpec(
+                            cli_param=container_spec.get(ContainerSpec.CLI_PARM, ""),
+                            server_port=container_spec.get(
+                                ContainerSpec.SERVER_PORT, ""
+                            ),
+                            health_check_port=container_spec.get(
+                                ContainerSpec.HEALTH_CHECK_PORT, ""
+                            ),
+                            env_vars=container_spec.get(ContainerSpec.ENV_VARS, []),
+                            restricted_params=container_spec.get(
+                                ContainerSpec.RESTRICTED_PARAMS, []
+                            ),
+                            evaluation_configuration=AquaContainerEvaluationConfig.from_config(
+                                container_spec.get(
+                                    ContainerSpec.EVALUATION_CONFIGURATION, {}
+                                )
+                            ),
+                        )
+                        if container_spec
+                        else None,
                     )
                     if container.get("type") == "inference":
-                        inference_items.append(container_item)
+                        inference_items[container_type] = container_item
                     elif container_type == "odsc-llm-fine-tuning":
-                        finetune_items.append(container_item)
+                        finetune_items[container_type] = container_item
                     elif container_type == "odsc-llm-evaluate":
-                        evaluate_items.append(container_item)
+                        evaluate_items[container_type] = container_item
 
         return AquaContainerConfig(
             inference=inference_items, finetune=finetune_items, evaluate=evaluate_items
@@ -175,11 +297,11 @@ class AquaUIApp(AquaApp):
         try:
             if not TENANCY_OCID:
                 raise AquaValueError(
-                    f"TENANCY_OCID must be available in environment"
+                    "TENANCY_OCID must be available in environment"
                     " variables to list the sub compartments."
                 )
 
-            if TENANCY_OCID in self._compartments_cache.keys():
+            if TENANCY_OCID in self._compartments_cache:
                 logger.info(
                     f"Returning compartments list in {TENANCY_OCID} from cache."
                 )
@@ -196,7 +318,7 @@ class AquaUIApp(AquaApp):
                         access_level="ANY",
                     )
                 )
-            except ServiceError as se:
+            except ServiceError:
                 logger.error(
                     f"ERROR: Unable to list all sub compartment in tenancy {TENANCY_OCID}."
                 )
@@ -207,7 +329,7 @@ class AquaUIApp(AquaApp):
                             compartment_id=TENANCY_OCID,
                         )
                     )
-                except ServiceError as se:
+                except ServiceError:
                     logger.error(
                         f"ERROR: Unable to list all child compartment in tenancy {TENANCY_OCID}."
                     )
@@ -216,7 +338,7 @@ class AquaUIApp(AquaApp):
                     TENANCY_OCID
                 ).data
                 compartments.insert(0, root_compartment)
-            except ServiceError as se:
+            except ServiceError:
                 logger.error(
                     f"ERROR: Unable to get details of the root compartment {TENANCY_OCID}."
                 )
@@ -248,7 +370,7 @@ class AquaUIApp(AquaApp):
         """
         if not COMPARTMENT_OCID:
             logger.error("No compartment id found from environment variables.")
-        return dict(compartment_id=COMPARTMENT_OCID)
+        return {"compartment_id": COMPARTMENT_OCID}
 
     def clear_compartments_list_cache(self) -> dict:
         """Allows caller to clear compartments list cache
@@ -257,9 +379,9 @@ class AquaUIApp(AquaApp):
             dict with the key used, and True if cache has the key that needs to be deleted.
         """
         res = {}
-        logger.info(f"Clearing list_compartments cache")
+        logger.info("Clearing list_compartments cache")
         with self._cache_lock:
-            if TENANCY_OCID in self._compartments_cache.keys():
+            if TENANCY_OCID in self._compartments_cache:
                 self._compartments_cache.pop(key=TENANCY_OCID)
                 res = {
                     "key": {
@@ -332,6 +454,7 @@ class AquaUIApp(AquaApp):
         response = os_client.list_buckets(
             namespace_name=namespace_name,
             compartment_id=compartment_id,
+            limit=1000,
             **kwargs,
         ).data
 
@@ -474,16 +597,16 @@ class AquaUIApp(AquaApp):
             raise AquaResourceAccessError(
                 f"Could not check limits availability for the shape {instance_shape}. Make sure you have the necessary policy to check limits availability.",
                 service_payload=se.args[0] if se.args else None,
-            )
+            ) from None
 
         available = res.available
 
         try:
             cards = int(instance_shape.split(".")[-1])
-        except:
+        except Exception:
             cards = 1
 
-        response = dict(available_count=available)
+        response = {"available_count": available}
 
         if available < cards:
             raise AquaValueError(
@@ -516,7 +639,7 @@ class AquaUIApp(AquaApp):
             is_versioned = False
             message = f"Model artifact bucket {bucket_uri} is not versioned. Check if the path exists and enable versioning on the bucket to proceed with model creation."
 
-        return dict(is_versioned=is_versioned, message=message)
+        return {"is_versioned": is_versioned, "message": message}
 
     @telemetry(entry_point="plugin=ui&action=list_containers", name="aqua")
     def list_containers(self) -> AquaContainerConfig:
@@ -526,8 +649,9 @@ class AquaUIApp(AquaApp):
         Returns
         -------
         AquaContainerConfig
-            The AQUA containers configuration.
+            The AQUA containers configurations.
         """
         return AquaContainerConfig.from_container_index_json(
-            config=get_container_config()
+            config=get_container_config(),
+            enable_spec=True,
         )