oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.10rc0__py3-none-any.whl

ads/aqua/extension/deployment_handler.py

@@ -0,0 +1,381 @@
+#!/usr/bin/env python
+# Copyright (c) 2024, 2025 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from typing import List, Optional, Union
+from urllib.parse import urlparse
+
+from tornado.web import HTTPError
+
+from ads.aqua.app import logger
+from ads.aqua.client.client import Client, ExtendedRequestError
+from ads.aqua.common.decorator import handle_exceptions
+from ads.aqua.common.enums import PredictEndpoints
+from ads.aqua.extension.base_handler import AquaAPIhandler
+from ads.aqua.extension.errors import Errors
+from ads.aqua.modeldeployment import AquaDeploymentApp
+from ads.config import COMPARTMENT_OCID
+
+
+class AquaDeploymentHandler(AquaAPIhandler):
+    """
+    Handler for Aqua Deployment REST APIs.
+
+    Methods
+    -------
+    get(self, id: Union[str, List[str]])
+        Retrieves a list of AQUA deployments or model info or logs by ID.
+    post(self, *args, **kwargs)
+        Creates a new AQUA deployment.
+    read(self, id: str)
+        Reads the AQUA deployment information.
+    list(self)
+        Lists all the AQUA deployments.
+    get_deployment_config(self, model_id)
+        Gets the deployment config for Aqua model.
+    list_shapes(self)
+        Lists the valid model deployment shapes.
+
+    Raises
+    ------
+    HTTPError: For various failure scenarios such as invalid input format, missing data, etc.
+    """
+
+    @handle_exceptions
+    def get(self, id: Union[str, List[str]] = None):
+        """Handle GET request."""
+        url_parse = urlparse(self.request.path)
+        paths = url_parse.path.strip("/")
+        if paths.startswith("aqua/deployments/config"):
+            if not id or not isinstance(id, str):
+                raise HTTPError(
+                    400,
+                    f"Invalid request format for {self.request.path}. "
+                    "Expected a single model ID or a comma-separated list of model IDs.",
+                )
+            id = id.replace(" ", "")
+            return self.get_deployment_config(
+                model_id=id.split(",") if "," in id else id
+            )
+        elif paths.startswith("aqua/deployments/shapes"):
+            return self.list_shapes()
+        elif paths.startswith("aqua/deployments"):
+            if not id:
+                return self.list()
+            return self.read(id)
+        else:
+            raise HTTPError(400, f"The request {self.request.path} is invalid.")
+
+    @handle_exceptions
+    def delete(self, model_deployment_id):
+        return self.finish(AquaDeploymentApp().delete(model_deployment_id))
+
+    @handle_exceptions
+    def put(self, *args, **kwargs):  # noqa: ARG002
+        """
+        Handles put request for the activating and deactivating OCI datascience model deployments
+        Raises
+        ------
+        HTTPError
+            Raises HTTPError if inputs are missing or are invalid
+        """
+        url_parse = urlparse(self.request.path)
+        paths = url_parse.path.strip("/").split("/")
+        if len(paths) != 4 or paths[0] != "aqua" or paths[1] != "deployments":
+            raise HTTPError(400, f"The request {self.request.path} is invalid.")
+
+        model_deployment_id = paths[2]
+        action = paths[3]
+        if action == "activate":
+            return self.finish(AquaDeploymentApp().activate(model_deployment_id))
+        elif action == "deactivate":
+            return self.finish(AquaDeploymentApp().deactivate(model_deployment_id))
+        else:
+            raise HTTPError(400, f"The request {self.request.path} is invalid.")
+
+    @handle_exceptions
+    def post(self, *args, **kwargs):  # noqa: ARG002
+        """
+        Handles post request for the deployment APIs
+        Raises
+        ------
+        HTTPError
+            Raises HTTPError if inputs are missing or are invalid
+        """
+        try:
+            input_data = self.get_json_body()
+        except Exception as ex:
+            raise HTTPError(400, Errors.INVALID_INPUT_DATA_FORMAT) from ex
+
+        if not input_data:
+            raise HTTPError(400, Errors.NO_INPUT_DATA)
+
+        self.finish(AquaDeploymentApp().create(**input_data))
+
+    def read(self, id):
+        """Read the information of an Aqua model deployment."""
+        return self.finish(AquaDeploymentApp().get(model_deployment_id=id))
+
+    def list(self):
+        """List Aqua models."""
+        # If default is not specified,
+        # jupyterlab will raise 400 error when argument is not provided by the HTTP request.
+        compartment_id = self.get_argument("compartment_id", default=COMPARTMENT_OCID)
+        # project_id is optional.
+        project_id = self.get_argument("project_id", default=None)
+        return self.finish(
+            AquaDeploymentApp().list(
+                compartment_id=compartment_id, project_id=project_id
+            )
+        )
+
+    def get_deployment_config(self, model_id: Union[str, List[str]]):
+        """
+        Retrieves the deployment configuration for one or more Aqua models.
+
+        Parameters
+        ----------
+        model_id : Union[str, List[str]]
+            A single model ID (str) or a list of model IDs (List[str]).
+
+        Returns
+        -------
+        None
+            The function sends the deployment configuration as a response.
+        """
+        app = AquaDeploymentApp()
+
+        compartment_id = self.get_argument("compartment_id", default=COMPARTMENT_OCID)
+
+        if isinstance(model_id, list):
+            # Handle multiple model deployment
+            primary_model_id = self.get_argument("primary_model_id", default=None)
+            deployment_config = app.get_multimodel_deployment_config(
+                model_ids=model_id,
+                primary_model_id=primary_model_id,
+                compartment_id=compartment_id,
+            )
+        else:
+            # Handle single model deployment
+            deployment_config = app.get_deployment_config(model_id=model_id)
+
+        return self.finish(deployment_config)
+
+    def list_shapes(self):
+        """
+        Lists the valid model deployment shapes.
+
+        Returns
+        -------
+        List[ComputeShapeSummary]:
+            The list of the model deployment shapes.
+        """
+        compartment_id = self.get_argument("compartment_id", default=COMPARTMENT_OCID)
+
+        return self.finish(
+            AquaDeploymentApp().list_shapes(compartment_id=compartment_id)
+        )
+
+
+class AquaDeploymentStreamingInferenceHandler(AquaAPIhandler):
+    def _get_model_deployment_response(
+        self,
+        model_deployment_id: str,
+        payload: dict,
+        route_override_header: Optional[str],
+    ):
+        """
+        Returns the model deployment inference response in a streaming fashion.
+
+        This method connects to the specified model deployment endpoint and
+        streams the inference output back to the caller, handling both text
+        and chat completion endpoints depending on the route override.
+
+        Parameters
+        ----------
+        model_deployment_id : str
+            The OCID of the model deployment to invoke.
+            Example: 'ocid1.datasciencemodeldeployment.iad.oc1.xxxyz'
+
+        payload : dict
+            Dictionary containing the model inference parameters.
+            Same example for text completions:
+                {
+                    "max_tokens": 1024,
+                    "temperature": 0.5,
+                    "prompt": "what are some good skills deep learning expert. Give us some tips on how to structure interview with some coding example?",
+                    "top_p": 0.4,
+                    "top_k": 100,
+                    "model": "odsc-llm",
+                    "frequency_penalty": 1,
+                    "presence_penalty": 1,
+                    "stream": true
+                }
+
+        route_override_header : Optional[str]
+            Optional override for the inference route, used for routing between
+            different endpoint types (e.g., chat vs. text completions).
+            Example: '/v1/chat/completions'
+
+        Returns
+        -------
+        Generator[str]
+            A generator that yields strings of the model's output as they are received.
+
+        Raises
+        ------
+        HTTPError
+            If the request to the model deployment fails or if streaming cannot be established.
+        """
+
+        model_deployment = AquaDeploymentApp().get(model_deployment_id)
+        endpoint = model_deployment.endpoint + "/predictWithResponseStream"
+        endpoint_type = model_deployment.environment_variables.get(
+            "MODEL_DEPLOY_PREDICT_ENDPOINT", PredictEndpoints.TEXT_COMPLETIONS_ENDPOINT
+        )
+        aqua_client = Client(endpoint=endpoint)
+
+        if PredictEndpoints.CHAT_COMPLETIONS_ENDPOINT in (
+            endpoint_type,
+            route_override_header,
+        ):
+            try:
+                for chunk in aqua_client.chat(
+                    messages=payload.pop("messages"),
+                    payload=payload,
+                    stream=True,
+                ):
+                    try:
+                        yield chunk["choices"][0]["delta"]["content"]
+                    except Exception as e:
+                        logger.debug(
+                            f"Exception occurred while parsing streaming response: {e}"
+                        )
+            except ExtendedRequestError as ex:
+                raise HTTPError(400, str(ex))
+            except Exception as ex:
+                raise HTTPError(500, str(ex))
+
+        elif endpoint_type == PredictEndpoints.TEXT_COMPLETIONS_ENDPOINT:
+            try:
+                for chunk in aqua_client.generate(
+                    prompt=payload.pop("prompt"),
+                    payload=payload,
+                    stream=True,
+                ):
+                    try:
+                        yield chunk["choices"][0]["text"]
+                    except Exception as e:
+                        logger.debug(
+                            f"Exception occurred while parsing streaming response: {e}"
+                        )
+            except ExtendedRequestError as ex:
+                raise HTTPError(400, str(ex))
+            except Exception as ex:
+                raise HTTPError(500, str(ex))
+
+    @handle_exceptions
+    def post(self, model_deployment_id):
+        """
+        Handles streaming inference request for the Active Model Deployments
+        Raises
+        ------
+        HTTPError
+            Raises HTTPError if inputs are missing or are invalid
+        """
+        try:
+            input_data = self.get_json_body()
+        except Exception as ex:
+            raise HTTPError(400, Errors.INVALID_INPUT_DATA_FORMAT) from ex
+
+        if not input_data:
+            raise HTTPError(400, Errors.NO_INPUT_DATA)
+
+        prompt = input_data.get("prompt")
+        messages = input_data.get("messages")
+
+        if not prompt and not messages:
+            raise HTTPError(
+                400, Errors.MISSING_REQUIRED_PARAMETER.format("prompt/messages")
+            )
+        if not input_data.get("model"):
+            raise HTTPError(400, Errors.MISSING_REQUIRED_PARAMETER.format("model"))
+        route_override_header = self.request.headers.get("route", None)
+        self.set_header("Content-Type", "text/event-stream")
+        response_gen = self._get_model_deployment_response(
+            model_deployment_id, input_data, route_override_header
+        )
+        try:
+            for chunk in response_gen:
+                self.write(chunk)
+                self.flush()
+            self.finish()
+        except Exception as ex:
+            self.set_status(ex.status_code)
+            self.write({"message": "Error occurred", "reason": str(ex)})
+            self.finish()
+
+
+class AquaDeploymentParamsHandler(AquaAPIhandler):
+    """Handler for Aqua deployment params REST APIs.
+
+    Methods
+    -------
+    get(self, model_id)
+        Retrieves a list of model deployment parameters.
+    post(self, *args, **kwargs)
+        Validates parameters for the given model id.
+    """
+
+    @handle_exceptions
+    def get(self, model_id):
+        """Handle GET request."""
+        instance_shape = self.get_argument("instance_shape")
+        gpu_count = self.get_argument("gpu_count", default=None)
+        return self.finish(
+            AquaDeploymentApp().get_deployment_default_params(
+                model_id=model_id, instance_shape=instance_shape, gpu_count=gpu_count
+            )
+        )
+
+    @handle_exceptions
+    def post(self, *args, **kwargs):  # noqa: ARG002
+        """Handles post request for the deployment param handler API.
+
+        Raises
+        ------
+        HTTPError
+            Raises HTTPError if inputs are missing or are invalid.
+        """
+        try:
+            input_data = self.get_json_body()
+        except Exception as ex:
+            raise HTTPError(400, Errors.INVALID_INPUT_DATA_FORMAT) from ex
+
+        if not input_data:
+            raise HTTPError(400, Errors.NO_INPUT_DATA)
+
+        model_id = input_data.get("model_id")
+        if not model_id:
+            raise HTTPError(400, Errors.MISSING_REQUIRED_PARAMETER.format("model_id"))
+
+        params = input_data.get("params")
+        container_family = input_data.get("container_family")
+        return self.finish(
+            AquaDeploymentApp().validate_deployment_params(
+                model_id=model_id,
+                params=params,
+                container_family=container_family,
+            )
+        )
+
+
+__handlers__ = [
+    ("deployments/?([^/]*)/params", AquaDeploymentParamsHandler),
+    ("deployments/config/?([^/]*)", AquaDeploymentHandler),
+    ("deployments/shapes/?([^/]*)", AquaDeploymentHandler),
+    ("deployments/?([^/]*)", AquaDeploymentHandler),
+    ("deployments/?([^/]*)/activate", AquaDeploymentHandler),
+    ("deployments/?([^/]*)/deactivate", AquaDeploymentHandler),
+    ("inference/stream/?([^/]*)", AquaDeploymentStreamingInferenceHandler),
+]

ads/aqua/extension/deployment_ws_msg_handler.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import json
+from typing import List, Union
+
+from ads.aqua.common.decorator import handle_exceptions
+from ads.aqua.extension.aqua_ws_msg_handler import AquaWSMsgHandler
+from ads.aqua.extension.models.ws_models import (
+    ListDeploymentResponse,
+    ModelDeploymentDetailsResponse,
+    RequestResponseType,
+)
+from ads.aqua.modeldeployment import AquaDeploymentApp
+from ads.config import COMPARTMENT_OCID
+
+
+class AquaDeploymentWSMsgHandler(AquaWSMsgHandler):
+    def __init__(self, message: Union[str, bytes]):
+        super().__init__(message)
+
+    @staticmethod
+    def get_message_types() -> List[RequestResponseType]:
+        return [
+            RequestResponseType.ListDeployments,
+            RequestResponseType.DeploymentDetails,
+        ]
+
+    @handle_exceptions
+    def process(self) -> Union[ListDeploymentResponse, ModelDeploymentDetailsResponse]:
+        request = json.loads(self.message)
+        if request.get("kind") == "ListDeployments":
+            deployment_list = AquaDeploymentApp().list(
+                compartment_id=request.get("compartment_id") or COMPARTMENT_OCID,
+                project_id=request.get("project_id"),
+            )
+            response = ListDeploymentResponse(
+                message_id=request.get("message_id"),
+                kind=RequestResponseType.ListDeployments,
+                data=deployment_list,
+            )
+            return response
+        elif request.get("kind") == "DeploymentDetails":
+            deployment_details = AquaDeploymentApp().get(
+                request.get("model_deployment_id")
+            )
+            response = ModelDeploymentDetailsResponse(
+                message_id=request.get("message_id"),
+                kind=RequestResponseType.DeploymentDetails,
+                data=deployment_details,
+            )
+            return response

ads/aqua/extension/errors.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+# Copyright (c) 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+import uuid
+from typing import Any, Dict, List, Optional
+
+from pydantic import Field
+
+from ads.aqua.config.utils.serializer import Serializable
+
+from ads.aqua.constants import (
+    AQUA_TROUBLESHOOTING_LINK
+)
+
+class Errors(str):
+    INVALID_INPUT_DATA_FORMAT = "Invalid format of input data."
+    NO_INPUT_DATA = "No input data provided."
+    MISSING_REQUIRED_PARAMETER = "Missing required parameter: '{}'"
+    MISSING_ONEOF_REQUIRED_PARAMETER = "Either '{}' or '{}' is required."
+    INVALID_VALUE_OF_PARAMETER = "Invalid value of parameter: '{}'"
+
+class ReplyDetails(Serializable):
+    """Structured reply to be returned to the client."""
+    status: int
+    troubleshooting_tips: str = Field(f"For general tips on troubleshooting: {AQUA_TROUBLESHOOTING_LINK}",
+                                      description="GitHub Link for troubleshooting documentation")
+    message: str = Field("Unknown HTTP Error.", description="GitHub Link for troubleshooting documentation")
+    service_payload: Optional[Dict[str, Any]] = Field(default_factory=dict)
+    reason: str = Field("Unknown error", description="Reason for Error")
+    request_id: str = Field(str(uuid.uuid4()), description="Unique ID for tracking the error.")

ads/aqua/extension/evaluation_handler.py

@@ -0,0 +1,129 @@
+#!/usr/bin/env python
+# Copyright (c) 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from typing import Optional
+from urllib.parse import urlparse
+
+from tornado.web import HTTPError
+
+from ads.aqua.common.decorator import handle_exceptions
+from ads.aqua.evaluation import AquaEvaluationApp
+from ads.aqua.evaluation.entities import CreateAquaEvaluationDetails
+from ads.aqua.extension.base_handler import AquaAPIhandler
+from ads.aqua.extension.errors import Errors
+from ads.config import COMPARTMENT_OCID
+
+
+class AquaEvaluationHandler(AquaAPIhandler):
+    """Handler for Aqua Model Evaluation REST APIs."""
+
+    @handle_exceptions
+    def get(self, eval_id=""):
+        """Handle GET request."""
+        url_parse = urlparse(self.request.path)
+        paths = url_parse.path.strip("/")
+        if paths.startswith("aqua/evaluation/metrics"):
+            return self.get_default_metrics()
+        if not eval_id:
+            return self.list()
+        return self.read(eval_id)
+
+    @handle_exceptions
+    def post(self, *args, **kwargs):  # noqa
+        """Handles post request for the evaluation APIs
+
+        Raises
+        ------
+        HTTPError
+            Raises HTTPError if inputs are missing or are invalid.
+        """
+        try:
+            input_data = self.get_json_body()
+        except Exception as ex:
+            raise HTTPError(400, Errors.INVALID_INPUT_DATA_FORMAT) from ex
+
+        if not input_data:
+            raise HTTPError(400, Errors.NO_INPUT_DATA)
+
+        self.finish(
+            # TODO: decide what other kwargs will be needed for create aqua evaluation.
+            AquaEvaluationApp().create(
+                create_aqua_evaluation_details=(
+                    CreateAquaEvaluationDetails(**input_data)
+                )
+            )
+        )
+
+    @handle_exceptions
+    def put(self, eval_id):
+        """Handles PUT request for the evaluation APIs"""
+        eval_id = eval_id.split("/")[0]
+        return self.finish(AquaEvaluationApp().cancel(eval_id))
+
+    @handle_exceptions
+    def delete(self, eval_id):
+        return self.finish(AquaEvaluationApp().delete(eval_id))
+
+    def read(self, eval_id):
+        """Read the information of an Aqua model."""
+        return self.finish(AquaEvaluationApp().get(eval_id))
+
+    def list(self):
+        """List Aqua models."""
+        compartment_id = self.get_argument("compartment_id", default=COMPARTMENT_OCID)
+        return self.finish(AquaEvaluationApp().list(compartment_id))
+
+    def get_default_metrics(self):
+        """Lists supported metrics for evaluation."""
+        return self.finish(AquaEvaluationApp().get_supported_metrics())
+
+
+class AquaEvaluationStatusHandler(AquaAPIhandler):
+    """Handler for Aqua Evaluation status REST APIs."""
+
+    @handle_exceptions
+    def get(self, eval_id):
+        """Handle GET request."""
+        eval_id = eval_id.split("/")[0]
+        return self.finish(AquaEvaluationApp().get_status(eval_id))
+
+
+class AquaEvaluationReportHandler(AquaAPIhandler):
+    """Handler for Aqua Evaluation report REST APIs."""
+
+    @handle_exceptions
+    def get(self, eval_id):
+        """Handle GET request."""
+        eval_id = eval_id.split("/")[0]
+        return self.finish(AquaEvaluationApp().download_report(eval_id))
+
+
+class AquaEvaluationMetricsHandler(AquaAPIhandler):
+    """Handler for Aqua Evaluation metrics REST APIs."""
+
+    @handle_exceptions
+    def get(self, eval_id):
+        """Handle GET request."""
+        eval_id = eval_id.split("/")[0]
+        return self.finish(AquaEvaluationApp().load_metrics(eval_id))
+
+
+class AquaEvaluationConfigHandler(AquaAPIhandler):
+    """Handler for Aqua Evaluation Config REST APIs."""
+
+    @handle_exceptions
+    def get(self, container: Optional[str] = None, **kwargs):  # noqa
+        """Handle GET request."""
+
+        return self.finish(AquaEvaluationApp().load_evaluation_config(container))
+
+
+__handlers__ = [
+    ("evaluation/config/?([^/]*)", AquaEvaluationConfigHandler),
+    ("evaluation/?([^/]*)", AquaEvaluationHandler),
+    ("evaluation/?([^/]*/report)", AquaEvaluationReportHandler),
+    ("evaluation/?([^/]*/metrics)", AquaEvaluationMetricsHandler),
+    ("evaluation/?([^/]*/status)", AquaEvaluationStatusHandler),
+    ("evaluation/?([^/]*/cancel)", AquaEvaluationHandler),
+]

ads/aqua/extension/evaluation_ws_msg_handler.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import json
+from typing import List, Union
+
+from ads.aqua.common.decorator import handle_exceptions
+from ads.aqua.evaluation import AquaEvaluationApp
+from ads.aqua.extension.aqua_ws_msg_handler import AquaWSMsgHandler
+from ads.aqua.extension.models.ws_models import (
+    EvaluationDetailsResponse,
+    ListEvaluationsResponse,
+    RequestResponseType,
+)
+from ads.config import COMPARTMENT_OCID
+
+
+class AquaEvaluationWSMsgHandler(AquaWSMsgHandler):
+    @staticmethod
+    def get_message_types() -> List[RequestResponseType]:
+        return [
+            RequestResponseType.ListEvaluations,
+            RequestResponseType.EvaluationDetails,
+        ]
+
+    def __init__(self, message: Union[str, bytes]):
+        super().__init__(message)
+
+    @handle_exceptions
+    def process(self) -> Union[ListEvaluationsResponse, EvaluationDetailsResponse]:
+        request = json.loads(self.message)
+        if request["kind"] == "ListEvaluations":
+            return self.list_evaluations(request)
+        if request["kind"] == "EvaluationDetails":
+            return self.evaluation_details(request)
+
+    @staticmethod
+    def list_evaluations(request) -> ListEvaluationsResponse:
+        eval_list = AquaEvaluationApp().list(
+            request.get("compartment_id") or COMPARTMENT_OCID
+        )
+        response = ListEvaluationsResponse(
+            message_id=request["message_id"],
+            kind=RequestResponseType.ListEvaluations,
+            data=eval_list,
+        )
+        return response
+
+    @staticmethod
+    def evaluation_details(request) -> EvaluationDetailsResponse:
+        evaluation_details = AquaEvaluationApp().get(
+            eval_id=request.get("evaluation_id")
+        )
+        response = EvaluationDetailsResponse(
+            message_id=request.get("message_id"),
+            kind=RequestResponseType.EvaluationDetails,
+            data=evaluation_details,
+        )
+        return response