oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.10__py3-none-any.whl

ads/llm/langchain/plugins/chat_models/oci_data_science.py

@@ -0,0 +1,1027 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+"""Chat model for OCI data science model deployment endpoint."""
+
+import importlib
+import json
+import logging
+from operator import itemgetter
+from typing import (
+    Any,
+    AsyncIterator,
+    Callable,
+    Dict,
+    Iterator,
+    List,
+    Literal,
+    Optional,
+    Sequence,
+    Type,
+    Union,
+)
+
+from langchain_core.callbacks import (
+    AsyncCallbackManagerForLLMRun,
+    CallbackManagerForLLMRun,
+)
+from langchain_core.language_models import LanguageModelInput
+from langchain_core.language_models.chat_models import (
+    BaseChatModel,
+    agenerate_from_stream,
+    generate_from_stream,
+)
+from langchain_core.messages import AIMessageChunk, BaseMessage, BaseMessageChunk
+from langchain_core.output_parsers import (
+    JsonOutputParser,
+    PydanticOutputParser,
+)
+from langchain_core.outputs import ChatGeneration, ChatGenerationChunk, ChatResult
+from langchain_core.runnables import Runnable, RunnableMap, RunnablePassthrough
+from langchain_core.tools import BaseTool
+from langchain_core.utils.function_calling import convert_to_openai_tool
+from pydantic import BaseModel, Field, model_validator
+
+from ads.llm.langchain.plugins.llms.oci_data_science_model_deployment_endpoint import (
+    DEFAULT_MODEL_NAME,
+    BaseOCIModelDeployment,
+)
+
+logger = logging.getLogger(__name__)
+DEFAULT_INFERENCE_ENDPOINT_CHAT = "/v1/chat/completions"
+
+
+def _is_pydantic_class(obj: Any) -> bool:
+    return isinstance(obj, type) and issubclass(obj, BaseModel)
+
+
+class ChatOCIModelDeployment(BaseChatModel, BaseOCIModelDeployment):
+    """OCI Data Science Model Deployment chat model integration.
+
+    Setup:
+        Install ``oracle-ads`` and ``langchain-openai``.
+
+        .. code-block:: bash
+
+            pip install -U oracle-ads langchain-openai
+
+        Use `ads.set_auth()` to configure authentication.
+        For example, to use OCI resource_principal for authentication:
+
+        .. code-block:: python
+
+            import ads
+            ads.set_auth("resource_principal")
+
+        For more details on authentication, see:
+        https://accelerated-data-science.readthedocs.io/en/latest/user_guide/cli/authentication.html
+
+        Make sure to have the required policies to access the OCI Data
+        Science Model Deployment endpoint. See:
+        https://docs.oracle.com/en-us/iaas/data-science/using/model-dep-policies-auth.htm
+
+
+    Key init args - completion params:
+        endpoint: str
+            The OCI model deployment endpoint.
+        temperature: float
+            Sampling temperature.
+        max_tokens: Optional[int]
+            Max number of tokens to generate.
+
+    Key init args — client params:
+        auth: dict
+            ADS auth dictionary for OCI authentication.
+        default_headers: Optional[Dict]
+            The headers to be added to the Model Deployment request.
+
+    Instantiate:
+        .. code-block:: python
+
+            from langchain_community.chat_models import ChatOCIModelDeployment
+
+            chat = ChatOCIModelDeployment(
+                endpoint="https://modeldeployment.<region>.oci.customer-oci.com/<ocid>/predict",
+                model="odsc-llm",
+                streaming=True,
+                max_retries=3,
+                model_kwargs={
+                    "max_token": 512,
+                    "temperature": 0.2,
+                    # other model parameters ...
+                },
+                default_headers={
+                    "route": "/v1/chat/completions",
+                    # other request headers ...
+                },
+            )
+
+    Invocation:
+        .. code-block:: python
+
+            messages = [
+                ("system", "Translate the user sentence to French."),
+                ("human", "Hello World!"),
+            ]
+            chat.invoke(messages)
+
+        .. code-block:: python
+
+            AIMessage(
+                content='Bonjour le monde!',
+                response_metadata={
+                    'token_usage': {
+                        'prompt_tokens': 40,
+                        'total_tokens': 50,
+                        'completion_tokens': 10
+                    },
+                    'model_name': 'odsc-llm',
+                    'system_fingerprint': '',
+                    'finish_reason': 'stop'
+                },
+                id='run-cbed62da-e1b3-4abd-9df3-ec89d69ca012-0'
+            )
+
+    Streaming:
+        .. code-block:: python
+
+            for chunk in chat.stream(messages):
+                print(chunk)
+
+        .. code-block:: python
+
+            content='' id='run-02c6-c43f-42de'
+            content='\n' id='run-02c6-c43f-42de'
+            content='B' id='run-02c6-c43f-42de'
+            content='on' id='run-02c6-c43f-42de'
+            content='j' id='run-02c6-c43f-42de'
+            content='our' id='run-02c6-c43f-42de'
+            content=' le' id='run-02c6-c43f-42de'
+            content=' monde' id='run-02c6-c43f-42de'
+            content='!' id='run-02c6-c43f-42de'
+            content='' response_metadata={'finish_reason': 'stop'} id='run-02c6-c43f-42de'
+
+    Async:
+        .. code-block:: python
+
+            await chat.ainvoke(messages)
+
+            # stream:
+            # async for chunk in (await chat.astream(messages))
+
+        .. code-block:: python
+
+            AIMessage(
+                content='Bonjour le monde!',
+                response_metadata={'finish_reason': 'stop'},
+                id='run-8657a105-96b7-4bb6-b98e-b69ca420e5d1-0'
+            )
+
+    Structured output:
+        .. code-block:: python
+
+            from typing import Optional
+            from pydantic import BaseModel, Field
+
+            class Joke(BaseModel):
+                setup: str = Field(description="The setup of the joke")
+                punchline: str = Field(description="The punchline to the joke")
+
+            structured_llm = chat.with_structured_output(Joke, method="json_mode")
+            structured_llm.invoke(
+                "Tell me a joke about cats, "
+                "respond in JSON with `setup` and `punchline` keys"
+            )
+
+        .. code-block:: python
+
+            Joke(
+                setup='Why did the cat get stuck in the tree?',
+                punchline='Because it was chasing its tail!'
+            )
+
+        See ``ChatOCIModelDeployment.with_structured_output()`` for more.
+
+    Customized Usage:
+        You can inherit from base class and overwrite the `_process_response`,
+        `_process_stream_response`, `_construct_json_body` for customized usage.
+
+        .. code-block:: python
+
+            class MyChatModel(ChatOCIModelDeployment):
+                def _process_stream_response(self, response_json: dict) -> ChatGenerationChunk:
+                    print("My customized streaming result handler.")
+                    return GenerationChunk(...)
+
+                def _process_response(self, response_json:dict) -> ChatResult:
+                    print("My customized output handler.")
+                    return ChatResult(...)
+
+                def _construct_json_body(self, messages: list, params: dict) -> dict:
+                    print("My customized payload handler.")
+                    return {
+                        "messages": messages,
+                        **params,
+                    }
+
+            chat = MyChatModel(
+                endpoint=f"https://modeldeployment.<region>.oci.customer-oci.com/{ocid}/predict",
+                model="odsc-llm",
+            }
+
+            chat.invoke("tell me a joke")
+
+    Response metadata
+        .. code-block:: python
+
+            ai_msg = chat.invoke(messages)
+            ai_msg.response_metadata
+
+        .. code-block:: python
+
+            {
+                'token_usage': {
+                    'prompt_tokens': 40,
+                    'total_tokens': 50,
+                    'completion_tokens': 10
+                },
+                'model_name': 'odsc-llm',
+                'system_fingerprint': '',
+                'finish_reason': 'stop'
+            }
+
+    """  # noqa: E501
+
+    model_kwargs: Dict[str, Any] = Field(default_factory=dict)
+    """Keyword arguments to pass to the model."""
+
+    model: str = DEFAULT_MODEL_NAME
+    """The name of the model."""
+
+    stop: Optional[List[str]] = None
+    """Stop words to use when generating. Model output is cut off
+    at the first occurrence of any of these substrings."""
+
+    @model_validator(mode="before")
+    @classmethod
+    def validate_openai(cls, values: Any) -> Any:
+        """Checks if langchain_openai is installed."""
+        if not importlib.util.find_spec("langchain_openai"):
+            raise ImportError(
+                "Could not import langchain_openai package. "
+                "Please install it with `pip install langchain_openai`."
+            )
+        return values
+
+    @property
+    def _llm_type(self) -> str:
+        """Return type of llm."""
+        return "oci_model_depolyment_chat_endpoint"
+
+    @property
+    def _identifying_params(self) -> Dict[str, Any]:
+        """Get the identifying parameters."""
+        _model_kwargs = self.model_kwargs or {}
+        return {
+            **{"endpoint": self.endpoint, "model_kwargs": _model_kwargs},
+            **self._default_params,
+        }
+
+    @property
+    def _default_params(self) -> Dict[str, Any]:
+        """Get the default parameters."""
+        return {
+            "model": self.model,
+            "stop": self.stop,
+            "stream": self.streaming,
+        }
+
+    def _headers(
+        self, is_async: Optional[bool] = False, body: Optional[dict] = None
+    ) -> Dict:
+        """Construct and return the headers for a request.
+
+        Args:
+            is_async (bool, optional): Indicates if the request is asynchronous.
+                Defaults to `False`.
+            body (optional): The request body to be included in the headers if
+                the request is asynchronous.
+
+        Returns:
+            Dict: A dictionary containing the appropriate headers for the request.
+        """
+        return {
+            "route": DEFAULT_INFERENCE_ENDPOINT_CHAT,
+            **super()._headers(is_async=is_async, body=body),
+        }
+
+    def _generate(
+        self,
+        messages: List[BaseMessage],
+        stop: Optional[List[str]] = None,
+        run_manager: Optional[CallbackManagerForLLMRun] = None,
+        **kwargs: Any,
+    ) -> ChatResult:
+        """Call out to an OCI Model Deployment Online endpoint.
+
+        Args:
+            messages:  The messages in the conversation with the chat model.
+            stop: Optional list of stop words to use when generating.
+
+        Returns:
+            LangChain ChatResult
+
+        Raises:
+            RuntimeError:
+                Raise when invoking endpoint fails.
+
+        Example:
+
+            .. code-block:: python
+
+                messages = [
+                    (
+                        "system",
+                        "You are a helpful assistant that translates English to French. Translate the user sentence.",
+                    ),
+                    ("human", "Hello World!"),
+                ]
+
+                response = chat.invoke(messages)
+        """  # noqa: E501
+        if self.streaming:
+            stream_iter = self._stream(
+                messages, stop=stop, run_manager=run_manager, **kwargs
+            )
+            return generate_from_stream(stream_iter)
+
+        requests_kwargs = kwargs.pop("requests_kwargs", {})
+        params = self._invocation_params(stop, **kwargs)
+        body = self._construct_json_body(messages, params)
+        res = self.completion_with_retry(
+            data=body, run_manager=run_manager, **requests_kwargs
+        )
+        return self._process_response(res.json())
+
+    def _stream(
+        self,
+        messages: List[BaseMessage],
+        stop: Optional[List[str]] = None,
+        run_manager: Optional[CallbackManagerForLLMRun] = None,
+        **kwargs: Any,
+    ) -> Iterator[ChatGenerationChunk]:
+        """Stream OCI Data Science Model Deployment endpoint on given messages.
+
+        Args:
+            messages (List[BaseMessage]):
+                The messagaes to pass into the model.
+            stop (List[str], Optional):
+                List of stop words to use when generating.
+            kwargs:
+                requests_kwargs:
+                    Additional ``**kwargs`` to pass to requests.post
+
+        Returns:
+            An iterator of ChatGenerationChunk.
+
+        Raises:
+            RuntimeError:
+                Raise when invoking endpoint fails.
+
+        Example:
+
+            .. code-block:: python
+
+                messages = [
+                    (
+                        "system",
+                        "You are a helpful assistant that translates English to French. Translate the user sentence.",
+                    ),
+                    ("human", "Hello World!"),
+                ]
+
+                chunk_iter = chat.stream(messages)
+
+        """  # noqa: E501
+        requests_kwargs = kwargs.pop("requests_kwargs", {})
+        self.streaming = True
+        params = self._invocation_params(stop, **kwargs)
+        body = self._construct_json_body(messages, params)  # request json body
+
+        response = self.completion_with_retry(
+            data=body, run_manager=run_manager, stream=True, **requests_kwargs
+        )
+        default_chunk_class = AIMessageChunk
+        for line in self._parse_stream(response.iter_lines()):
+            chunk = self._handle_sse_line(line, default_chunk_class)
+            if run_manager:
+                run_manager.on_llm_new_token(chunk.text, chunk=chunk)
+            yield chunk
+
+    async def _agenerate(
+        self,
+        messages: List[BaseMessage],
+        stop: Optional[List[str]] = None,
+        run_manager: Optional[AsyncCallbackManagerForLLMRun] = None,
+        **kwargs: Any,
+    ) -> ChatResult:
+        """Asynchronously call out to OCI Data Science Model Deployment
+        endpoint on given messages.
+
+        Args:
+            messages (List[BaseMessage]):
+                The messagaes to pass into the model.
+            stop (List[str], Optional):
+                List of stop words to use when generating.
+            kwargs:
+                requests_kwargs:
+                    Additional ``**kwargs`` to pass to requests.post
+
+        Returns:
+            LangChain ChatResult.
+
+        Raises:
+            ValueError:
+                Raise when invoking endpoint fails.
+
+        Example:
+
+            .. code-block:: python
+
+                messages = [
+                    (
+                        "system",
+                        "You are a helpful assistant that translates English to French. Translate the user sentence.",
+                    ),
+                    ("human", "I love programming."),
+                ]
+
+                resp = await chat.ainvoke(messages)
+
+        """  # noqa: E501
+        if self.streaming:
+            stream_iter = self._astream(
+                messages, stop=stop, run_manager=run_manager, **kwargs
+            )
+            return await agenerate_from_stream(stream_iter)
+
+        requests_kwargs = kwargs.pop("requests_kwargs", {})
+        params = self._invocation_params(stop, **kwargs)
+        body = self._construct_json_body(messages, params)
+        response = await self.acompletion_with_retry(
+            data=body,
+            run_manager=run_manager,
+            **requests_kwargs,
+        )
+        return self._process_response(response)
+
+    async def _astream(
+        self,
+        messages: List[BaseMessage],
+        stop: Optional[List[str]] = None,
+        run_manager: Optional[AsyncCallbackManagerForLLMRun] = None,
+        **kwargs: Any,
+    ) -> AsyncIterator[ChatGenerationChunk]:
+        """Asynchronously streaming OCI Data Science Model Deployment
+        endpoint on given messages.
+
+        Args:
+            messages (List[BaseMessage]):
+                The messagaes to pass into the model.
+            stop (List[str], Optional):
+                List of stop words to use when generating.
+            kwargs:
+                requests_kwargs:
+                    Additional ``**kwargs`` to pass to requests.post
+
+        Returns:
+            An Asynciterator of ChatGenerationChunk.
+
+        Raises:
+            ValueError:
+                Raise when invoking endpoint fails.
+
+        Example:
+
+            .. code-block:: python
+
+                messages = [
+                    (
+                        "system",
+                        "You are a helpful assistant that translates English to French. Translate the user sentence.",
+                    ),
+                    ("human", "I love programming."),
+                ]
+
+                chunk_iter = await chat.astream(messages)
+
+        """  # noqa: E501
+        requests_kwargs = kwargs.pop("requests_kwargs", {})
+        self.streaming = True
+        params = self._invocation_params(stop, **kwargs)
+        body = self._construct_json_body(messages, params)  # request json body
+
+        default_chunk_class = AIMessageChunk
+        async for line in await self.acompletion_with_retry(
+            data=body, run_manager=run_manager, stream=True, **requests_kwargs
+        ):
+            chunk = self._handle_sse_line(line, default_chunk_class)
+            if run_manager:
+                await run_manager.on_llm_new_token(chunk.text, chunk=chunk)
+            yield chunk
+
+    def with_structured_output(
+        self,
+        schema: Optional[Union[Dict, Type[BaseModel]]] = None,
+        *,
+        method: Literal["json_mode"] = "json_mode",
+        include_raw: bool = False,
+        **kwargs: Any,
+    ) -> Runnable[LanguageModelInput, Union[Dict, BaseModel]]:
+        """Model wrapper that returns outputs formatted to match the given schema.
+
+        Args:
+            schema: The output schema as a dict or a Pydantic class. If a Pydantic class
+                then the model output will be an object of that class. If a dict then
+                the model output will be a dict. With a Pydantic class the returned
+                attributes will be validated, whereas with a dict they will not be. If
+                `method` is "function_calling" and `schema` is a dict, then the dict
+                must match the OpenAI function-calling spec.
+            method: The method for steering model generation, currently only support
+                for "json_mode". If "json_mode" then JSON mode will be used. Note that
+                if using "json_mode" then you must include instructions for formatting
+                the output into the desired schema into the model call.
+            include_raw: If False then only the parsed structured output is returned. If
+                an error occurs during model output parsing it will be raised. If True
+                then both the raw model response (a BaseMessage) and the parsed model
+                response will be returned. If an error occurs during output parsing it
+                will be caught and returned as well. The final output is always a dict
+                with keys "raw", "parsed", and "parsing_error".
+
+        Returns:
+            A Runnable that takes any ChatModel input and returns as output:
+
+                If include_raw is True then a dict with keys:
+                    raw: BaseMessage
+                    parsed: Optional[_DictOrPydantic]
+                    parsing_error: Optional[BaseException]
+
+                If include_raw is False then just _DictOrPydantic is returned,
+                where _DictOrPydantic depends on the schema:
+
+                If schema is a Pydantic class then _DictOrPydantic is the Pydantic
+                    class.
+
+                If schema is a dict then _DictOrPydantic is a dict.
+
+        """  # noqa: E501
+        if kwargs:
+            raise ValueError(f"Received unsupported arguments {kwargs}")
+        is_pydantic_schema = _is_pydantic_class(schema)
+        if method == "json_mode":
+            llm = self.bind(response_format={"type": "json_object"})
+            output_parser = (
+                PydanticOutputParser(pydantic_object=schema)  # type: ignore[type-var, arg-type]
+                if is_pydantic_schema
+                else JsonOutputParser()
+            )
+        else:
+            raise ValueError(
+                f"Unrecognized method argument. Expected `json_mode`."
+                f"Received: `{method}`."
+            )
+
+        if include_raw:
+            parser_assign = RunnablePassthrough.assign(
+                parsed=itemgetter("raw") | output_parser, parsing_error=lambda _: None
+            )
+            parser_none = RunnablePassthrough.assign(parsed=lambda _: None)
+            parser_with_fallback = parser_assign.with_fallbacks(
+                [parser_none], exception_key="parsing_error"
+            )
+            return RunnableMap(raw=llm) | parser_with_fallback
+        else:
+            return llm | output_parser
+
+    def _invocation_params(self, stop: Optional[List[str]], **kwargs: Any) -> dict:
+        """Combines the invocation parameters with default parameters."""
+        params = self._default_params
+        _model_kwargs = self.model_kwargs or {}
+        params["stop"] = stop or params.get("stop", [])
+        return {**params, **_model_kwargs, **kwargs}
+
+    def _handle_sse_line(
+        self, line: str, default_chunk_cls: Type[BaseMessageChunk] = AIMessageChunk
+    ) -> ChatGenerationChunk:
+        """Handle a single Server-Sent Events (SSE) line and process it into
+        a chat generation chunk.
+
+        Args:
+            line (str): A single line from the SSE stream in string format.
+            default_chunk_cls (AIMessageChunk): The default class for message
+                chunks to be used during the processing of the stream response.
+
+        Returns:
+            ChatGenerationChunk: The processed chat generation chunk. If an error
+                occurs, an empty `ChatGenerationChunk` is returned.
+        """
+        try:
+            obj = json.loads(line)
+            return self._process_stream_response(obj, default_chunk_cls)
+        except Exception as e:
+            logger.debug(f"Error occurs when processing line={line}: {str(e)}")
+            return ChatGenerationChunk(message=AIMessageChunk(content=""))
+
+    def _construct_json_body(self, messages: list, params: dict) -> dict:
+        """Constructs the request body as a dictionary (JSON).
+
+        Args:
+            messages (list): A list of message objects to be included in the
+                request body.
+            params (dict): A dictionary of additional parameters to be included
+                in the request body.
+
+        Returns:
+            dict: A dictionary representing the JSON request body, including
+                converted messages and additional parameters.
+
+        """
+        from langchain_openai.chat_models.base import _convert_message_to_dict
+
+        return {
+            "messages": [_convert_message_to_dict(m) for m in messages],
+            **params,
+        }
+
+    def _process_stream_response(
+        self,
+        response_json: dict,
+        default_chunk_cls: Type[BaseMessageChunk] = AIMessageChunk,
+    ) -> ChatGenerationChunk:
+        """Formats streaming response in OpenAI spec.
+
+        Args:
+            response_json (dict): The JSON response from the streaming endpoint.
+            default_chunk_cls (type, optional): The default class to use for
+                creating message chunks. Defaults to `AIMessageChunk`.
+
+        Returns:
+            ChatGenerationChunk: An object containing the processed message
+                chunk and any relevant generation information such as finish
+                reason and usage.
+
+        Raises:
+            ValueError: If the response JSON is not well-formed or does not
+                contain the expected structure.
+        """
+        from langchain_openai.chat_models.base import _convert_delta_to_message_chunk
+
+        try:
+            choice = response_json["choices"][0]
+            if not isinstance(choice, dict):
+                raise TypeError("Endpoint response is not well formed.")
+        except (KeyError, IndexError, TypeError) as e:
+            raise ValueError(
+                "Error while formatting response payload for chat model of type"
+            ) from e
+
+        chunk = _convert_delta_to_message_chunk(choice["delta"], default_chunk_cls)
+        default_chunk_cls = chunk.__class__
+        finish_reason = choice.get("finish_reason")
+        usage = choice.get("usage")
+        gen_info = {}
+        if finish_reason is not None:
+            gen_info.update({"finish_reason": finish_reason})
+        if usage is not None:
+            gen_info.update({"usage": usage})
+
+        return ChatGenerationChunk(
+            message=chunk, generation_info=gen_info if gen_info else None
+        )
+
+    def _process_response(self, response_json: dict) -> ChatResult:
+        """Formats response in OpenAI spec.
+
+        Args:
+            response_json (dict): The JSON response from the chat model endpoint.
+
+        Returns:
+            ChatResult: An object containing the list of `ChatGeneration` objects
+            and additional LLM output information.
+
+        Raises:
+            ValueError: If the response JSON is not well-formed or does not
+            contain the expected structure.
+
+        """
+        from langchain_openai.chat_models.base import _convert_dict_to_message
+
+        generations = []
+        try:
+            choices = response_json["choices"]
+            if not isinstance(choices, list):
+                raise TypeError("Endpoint response is not well formed.")
+        except (KeyError, TypeError) as e:
+            raise ValueError(
+                "Error while formatting response payload for chat model of type"
+            ) from e
+
+        for choice in choices:
+            message = _convert_dict_to_message(choice["message"])
+            generation_info = {"finish_reason": choice.get("finish_reason")}
+            if "logprobs" in choice:
+                generation_info["logprobs"] = choice["logprobs"]
+
+            gen = ChatGeneration(
+                message=message,
+                generation_info=generation_info,
+            )
+            generations.append(gen)
+
+        token_usage = response_json.get("usage", {})
+        llm_output = {
+            "token_usage": token_usage,
+            "model_name": self.model,
+            "system_fingerprint": response_json.get("system_fingerprint", ""),
+        }
+        return ChatResult(generations=generations, llm_output=llm_output)
+
+    def bind_tools(
+        self,
+        tools: Sequence[Union[Dict[str, Any], Type[BaseModel], Callable, BaseTool]],
+        **kwargs: Any,
+    ) -> Runnable[LanguageModelInput, BaseMessage]:
+        formatted_tools = [convert_to_openai_tool(tool) for tool in tools]
+        return super().bind(tools=formatted_tools, **kwargs)
+
+
+class ChatOCIModelDeploymentVLLM(ChatOCIModelDeployment):
+    """OCI large language chat models deployed with vLLM.
+
+    To use, you must provide the model HTTP endpoint from your deployed
+    model, e.g. https://modeldeployment.us-ashburn-1.oci.customer-oci.com/<ocid>/predict.
+
+    To authenticate, `oracle-ads` has been used to automatically load
+    credentials: https://accelerated-data-science.readthedocs.io/en/latest/user_guide/cli/authentication.html
+
+    Make sure to have the required policies to access the OCI Data
+    Science Model Deployment endpoint. See:
+    https://docs.oracle.com/en-us/iaas/data-science/using/model-dep-policies-auth.htm#model_dep_policies_auth__predict-endpoint
+
+    See https://docs.vllm.ai/en/latest/api/inference_params.html for the defaults of the parameters.
+
+    Example:
+
+        .. code-block:: python
+
+            from langchain_community.chat_models import ChatOCIModelDeploymentVLLM
+
+            chat = ChatOCIModelDeploymentVLLM(
+                endpoint="https://modeldeployment.us-ashburn-1.oci.customer-oci.com/<ocid>/predict",
+                frequency_penalty=0.1,
+                max_tokens=512,
+                temperature=0.2,
+                top_p=1.0,
+                # other model parameters...
+            )
+
+    """  # noqa: E501
+
+    frequency_penalty: Optional[float] = None
+    """Penalizes repeated tokens according to frequency. Between 0 and 1."""
+
+    logit_bias: Optional[Dict[str, float]] = None
+    """Adjust the probability of specific tokens being generated."""
+
+    max_tokens: Optional[int] = 256
+    """The maximum number of tokens to generate in the completion."""
+
+    n: int = 1
+    """Number of output sequences to return for the given prompt."""
+
+    presence_penalty: Optional[float] = None
+    """Penalizes repeated tokens. Between 0 and 1."""
+
+    temperature: float = 0.2
+    """What sampling temperature to use."""
+
+    top_p: float = 1.0
+    """Total probability mass of tokens to consider at each step."""
+
+    best_of: Optional[int] = None
+    """Generates best_of completions server-side and returns the "best"
+    (the one with the highest log probability per token).
+    """
+
+    use_beam_search: Optional[bool] = None
+    """Whether to use beam search instead of sampling."""
+
+    top_k: Optional[int] = -1
+    """Number of most likely tokens to consider at each step."""
+
+    min_p: Optional[float] = 0.0
+    """Float that represents the minimum probability for a token to be considered.
+    Must be in [0,1]. 0 to disable this."""
+
+    repetition_penalty: Optional[float] = None
+    """Float that penalizes new tokens based on their frequency in the
+    generated text. Values > 1 encourage the model to use new tokens."""
+
+    length_penalty: Optional[float] = None
+    """Float that penalizes sequences based on their length. Used only
+    when `use_beam_search` is True."""
+
+    early_stopping: Optional[bool] = None
+    """Controls the stopping condition for beam search. It accepts the
+    following values: `True`, where the generation stops as soon as there
+    are `best_of` complete candidates; `False`, where a heuristic is applied
+    to the generation stops when it is very unlikely to find better candidates;
+    `never`, where the beam search procedure only stops where there cannot be
+    better candidates (canonical beam search algorithm)."""
+
+    ignore_eos: Optional[bool] = False
+    """Whether to ignore the EOS token and continue generating tokens after
+    the EOS token is generated."""
+
+    min_tokens: Optional[int] = None
+    """Minimum number of tokens to generate per output sequence before
+    EOS or stop_token_ids can be generated"""
+
+    stop_token_ids: Optional[List[int]] = None
+    """List of tokens that stop the generation when they are generated.
+    The returned output will contain the stop tokens unless the stop tokens
+    are special tokens."""
+
+    skip_special_tokens: Optional[bool] = None
+    """Whether to skip special tokens in the output. Defaults to True."""
+
+    spaces_between_special_tokens: Optional[bool] = None
+    """Whether to add spaces between special tokens in the output."""
+
+    tool_choice: Optional[str] = None
+    """Whether to use tool calling.
+    Defaults to None, tool calling is disabled.
+    Tool calling requires model support and the vLLM to be configured
+    with `--tool-call-parser`.
+    Set this to `auto` for the model to make tool calls automatically.
+    Set this to `required` to force the model to always call one or more tools.
+    """
+
+    chat_template: Optional[str] = None
+    """Use customized chat template.
+    Defaults to None. The chat template from the tokenizer will be used.
+    """
+
+    @property
+    def _llm_type(self) -> str:
+        """Return type of llm."""
+        return "oci_model_depolyment_chat_endpoint_vllm"
+
+    @property
+    def _default_params(self) -> Dict[str, Any]:
+        """Get the default parameters."""
+        params = {
+            "model": self.model,
+            "stop": self.stop,
+            "stream": self.streaming,
+        }
+        for attr_name in self._get_model_params():
+            try:
+                value = getattr(self, attr_name)
+                if value is not None:
+                    params.update({attr_name: value})
+            except Exception:
+                pass
+
+        return params
+
+    def _get_model_params(self) -> List[str]:
+        """Gets the name of model parameters."""
+        return [
+            "best_of",
+            "early_stopping",
+            "frequency_penalty",
+            "ignore_eos",
+            "length_penalty",
+            "logit_bias",
+            "logprobs",
+            "max_tokens",
+            "min_p",
+            "min_tokens",
+            "n",
+            "presence_penalty",
+            "repetition_penalty",
+            "skip_special_tokens",
+            "spaces_between_special_tokens",
+            "stop_token_ids",
+            "temperature",
+            "top_k",
+            "top_p",
+            "use_beam_search",
+            "tool_choice",
+            "chat_template",
+        ]
+
+
+class ChatOCIModelDeploymentTGI(ChatOCIModelDeployment):
+    """OCI large language chat models deployed with Text Generation Inference.
+
+    To use, you must provide the model HTTP endpoint from your deployed
+    model, e.g. https://modeldeployment.us-ashburn-1.oci.customer-oci.com/<ocid>/predict.
+
+    To authenticate, `oracle-ads` has been used to automatically load
+    credentials: https://accelerated-data-science.readthedocs.io/en/latest/user_guide/cli/authentication.html
+
+    Make sure to have the required policies to access the OCI Data
+    Science Model Deployment endpoint. See:
+    https://docs.oracle.com/en-us/iaas/data-science/using/model-dep-policies-auth.htm#model_dep_policies_auth__predict-endpoint
+
+    Example:
+
+        .. code-block:: python
+
+            from langchain_community.chat_models import ChatOCIModelDeploymentTGI
+
+            chat = ChatOCIModelDeploymentTGI(
+                endpoint="https://modeldeployment.us-ashburn-1.oci.customer-oci.com/<ocid>/predict",
+                max_token=512,
+                temperature=0.2,
+                frequency_penalty=0.1,
+                seed=42,
+                # other model parameters...
+            )
+
+    """  # noqa: E501
+
+    frequency_penalty: Optional[float] = None
+    """Penalizes repeated tokens according to frequency. Between 0 and 1."""
+
+    logit_bias: Optional[Dict[str, float]] = None
+    """Adjust the probability of specific tokens being generated."""
+
+    logprobs: Optional[bool] = None
+    """Whether to return log probabilities of the output tokens or not."""
+
+    max_tokens: int = 256
+    """The maximum number of tokens to generate in the completion."""
+
+    n: int = 1
+    """Number of output sequences to return for the given prompt."""
+
+    presence_penalty: Optional[float] = None
+    """Penalizes repeated tokens. Between 0 and 1."""
+
+    seed: Optional[int] = None
+    """To sample deterministically,"""
+
+    temperature: float = 0.2
+    """What sampling temperature to use."""
+
+    top_p: Optional[float] = None
+    """Total probability mass of tokens to consider at each step."""
+
+    top_logprobs: Optional[int] = None
+    """An integer between 0 and 5 specifying the number of most
+    likely tokens to return at each token position, each with an
+    associated log probability. logprobs must be set to true if
+    this parameter is used."""
+
+    @property
+    def _llm_type(self) -> str:
+        """Return type of llm."""
+        return "oci_model_depolyment_chat_endpoint_tgi"
+
+    @property
+    def _default_params(self) -> Dict[str, Any]:
+        """Get the default parameters."""
+        params = {
+            "model": self.model,
+            "stop": self.stop,
+            "stream": self.streaming,
+        }
+        for attr_name in self._get_model_params():
+            try:
+                value = getattr(self, attr_name)
+                if value is not None:
+                    params.update({attr_name: value})
+            except Exception:
+                pass
+
+        return params
+
+    def _get_model_params(self) -> List[str]:
+        """Gets the name of model parameters."""
+        return [
+            "frequency_penalty",
+            "logit_bias",
+            "logprobs",
+            "max_tokens",
+            "n",
+            "presence_penalty",
+            "seed",
+            "temperature",
+            "top_k",
+            "top_p",
+            "top_logprobs",
+        ]