truefoundry 0.5.0rc6__py3-none-any.whl → 0.5.1rc1__py3-none-any.whl

truefoundry/langchain/deprecated.py

@@ -1,302 +0,0 @@
-import enum
-import warnings
-from typing import Any, Dict, List, Mapping, Optional
-from urllib.parse import urljoin
-
-import requests
-from requests.auth import HTTPBasicAuth
-
-from truefoundry.deploy.lib.auth.servicefoundry_session import ServiceFoundrySession
-from truefoundry.pydantic_v1 import root_validator
-
-try:
-    from langchain.callbacks.manager import CallbackManagerForLLMRun
-    from langchain.llms.base import LLM
-    from langchain.llms.utils import enforce_stop_tokens
-except Exception as ex:
-    raise Exception(
-        "Failed to import langchain."
-        " Please install langchain by using `pip install langchain` command"
-    ) from ex
-
-
-class _ModelServerImpl(str, enum.Enum):
-    MLSERVER = "MLSERVER"
-    TGI = "TGI"
-    VLLM = "VLLM"
-
-
-def _get_model_server_and_validate_if_mlserver(endpoint_url, auth, model_name=None):
-    try:
-        response = requests.get(urljoin(endpoint_url, "info"), json={}, auth=auth)
-        if response.status_code == 200:
-            return _ModelServerImpl.TGI, None
-        elif response.status_code == 404:
-            # We are not using TGI, try for mlserver
-            response = requests.post(
-                urljoin(endpoint_url, "v2/repository/index"), json={}, auth=auth
-            )
-            if response.status_code == 200:
-                models = response.json()
-                if len(models) == 0:
-                    raise ValueError("No model is deployed in the model server")
-                model_names = [m.get("name") for m in models]
-                if model_name and model_name not in model_names:
-                    raise ValueError(
-                        f"Model {model_name!r} is not available in the model server. "
-                        f"Available models {model_names!r}"
-                    )
-                if not model_name and len(model_names) > 1:
-                    raise ValueError(
-                        f"Please pass `model_name` while instantiating `TruefoundryLLM`. "
-                        f"Available models are {model_names!r} "
-                    )
-                if model_name:
-                    return _ModelServerImpl.MLSERVER, model_name
-                return _ModelServerImpl.MLSERVER, model_names[0]
-            if response.status_code == 404:
-                return _ModelServerImpl.VLLM, None
-        response.raise_for_status()
-    except Exception as e:
-        raise Exception(f"Error raised by inference API: {e}") from e
-
-
-# TODO (chiragjn): Refactor this into separate implementations for each model server
-
-
-class TruefoundryLLM(LLM):
-    """Wrapper around TFY model deployment.
-    To use this class, you need to have the langchain library installed.
-    Example:
-        .. code-block:: python
-            from truefoundry.langchain import TruefoundryLLM
-            endpoint_url = (
-                "https://pythia-70m-model-model-catalogue.demo2.truefoundry.tech"
-            )
-            model = TruefoundryLLM(
-                endpoint_url=endpoint_url,
-                parameters={
-                    "max_new_tokens": 100,
-                    "temperature": 0.7,
-                    "top_k": 5,
-                    "top_p": 0.9
-                }
-            )
-    """
-
-    endpoint_url: str
-    model_name: Optional[str] = None
-    auth: Optional[HTTPBasicAuth] = None
-    parameters: Optional[Dict[str, Any]] = None
-    model_server_impl: Optional[_ModelServerImpl] = None
-
-    @root_validator(pre=False)
-    def validate_model_server_and_name(cls, values: Dict):
-        warnings.warn(
-            message=f"{cls.__name__} is deprecated and will be removed soon. Please use `TrueFoundryLLM` or `TrueFoundryChat` to invoke models using the new TrueFoundry LLM Gateway",
-            category=DeprecationWarning,
-            stacklevel=2,
-        )
-        endpoint_url = values["endpoint_url"]
-        model_name = values.get("model_name")
-        auth = values.get("auth")
-        model_server_impl, model_name = _get_model_server_and_validate_if_mlserver(
-            endpoint_url=endpoint_url, model_name=model_name, auth=auth
-        )
-        values["model_server_impl"] = model_server_impl
-        if model_server_impl == _ModelServerImpl.MLSERVER:
-            values["model_name"] = model_name
-        return values
-
-    @property
-    def _identifying_params(self) -> Mapping[str, Any]:
-        """Get the identifying parameters."""
-        return {
-            "endpoint_url": self.endpoint_url,
-            "model_name": self.model_name,
-        }
-
-    @property
-    def _llm_type(self) -> str:
-        """Return type of llm."""
-        return "tfy_model_deployment"
-
-    def _call(  # noqa: C901
-        self,
-        prompt: str,
-        stop: Optional[List[str]] = None,
-        run_manager: Optional[CallbackManagerForLLMRun] = None,
-        **params: Any,
-    ) -> str:
-        """Call out to the deployed model
-        Args:
-            prompt: The prompt to pass into the model.
-            stop: Optional list of stop words to use when generating.
-        Returns:
-            The string generated by the model.
-        Example:
-            .. code-block:: python
-                response = model("Tell me a joke.")
-        """
-        _params_already_set = self.parameters or {}
-        params = {**_params_already_set, **params, "return_full_text": False}
-
-        if self.model_server_impl == _ModelServerImpl.MLSERVER:
-            generate_path = f"v2/models/{self.model_name}/infer/simple"
-            payload = {"inputs": prompt, "parameters": params}
-        elif self.model_server_impl == _ModelServerImpl.TGI:
-            generate_path = "generate"
-            payload = {"inputs": prompt, "parameters": params}
-        elif self.model_server_impl == _ModelServerImpl.VLLM:
-            generate_path = "generate"
-            payload = {**params, "prompt": prompt}
-        else:
-            raise ValueError(f"No known generate path for {self.model_server_impl}")
-        url = urljoin(self.endpoint_url, generate_path)
-
-        try:
-            response = requests.post(url, json=payload, auth=self.auth)
-            response.raise_for_status()
-        except Exception as e:
-            raise Exception(f"Error raised by inference API: {e}") from e
-        response_dict = response.json()
-        if "error" in response_dict:
-            raise ValueError(
-                f"Error raised by inference API: {response_dict['error']!r}"
-            )
-
-        if self.model_server_impl == _ModelServerImpl.MLSERVER:
-            inference_result = response_dict[0]
-        elif self.model_server_impl == _ModelServerImpl.TGI:
-            inference_result = response_dict
-        elif self.model_server_impl == _ModelServerImpl.VLLM:
-            inference_result = response_dict
-        else:
-            raise ValueError(
-                f"Unknown model server {self.model_server_impl}, cannot parse response"
-            )
-
-        if "generated_text" in inference_result:
-            text = inference_result["generated_text"]
-        elif "summarization" in inference_result:
-            text = inference_result["summary_text"]
-        elif "text" in inference_result:
-            text = inference_result["text"]
-        else:
-            raise ValueError(f"Could not parse inference response: {response_dict!r}")
-
-        if isinstance(text, list):
-            text = text[0]
-
-        if stop:
-            text = enforce_stop_tokens(text, stop)
-
-        return text
-
-
-class TruefoundryPlaygroundLLM(LLM):
-    """Wrapper around TFY Playground.
-    To use this class, you need to have the langchain library installed.
-    Example:
-        .. code-block:: python
-            from truefoundry.langchain import TruefoundryPlaygroundLLM
-            import os
-            # Note: Login using tfy login --host <https://example-domain.com>
-            model = TruefoundryPlaygroundLLM(
-                model_name="vicuna-13b",
-                parameters={
-                    "maximumLength": 100,
-                    "temperature": 0.7,
-                    "topP": 0.9,
-                    "repetitionPenalty": 1
-                }
-            )
-            response = model.predict("Enter the prompt here")
-    """
-
-    model_name: str
-    parameters: Optional[Dict[str, Any]] = None
-    provider: str = "truefoundry-public"
-
-    @root_validator(pre=False)
-    def validate_model_server_and_name(cls, values: Dict):
-        warnings.warn(
-            message=f"{cls.__name__} is deprecated and will be removed soon. Please use `TrueFoundryLLM` or `TrueFoundryChat` to invoke models using the new TrueFoundry LLM Gateway",
-            category=DeprecationWarning,
-            stacklevel=2,
-        )
-        return values
-
-    @property
-    def _get_model(self) -> str:
-        """returns the model name"""
-        return self.model_name
-
-    @property
-    def _get_provider(self) -> str:
-        """Returns the provider name"""
-        return self.provider
-
-    @property
-    def _llm_type(self) -> str:
-        """Return type of llm."""
-        return "tfy_playground"
-
-    def _call(
-        self,
-        prompt: str,
-        stop: Optional[List[str]] = None,
-        **params: Any,
-    ) -> str:
-        """Call out to the deployed model
-        Args:
-            prompt: The prompt to pass into the model.
-            stop: Optional list of stop words to use when generating.
-        Returns:
-            The string generated by the model.
-        Example:
-            .. code-block:: python
-                response = model("I have a joke for you...")
-        """
-        _params_already_set = self.parameters or {}
-        params = {**_params_already_set, **params}
-        if stop:
-            params["stopSequences"] = stop
-        session = ServiceFoundrySession()
-
-        if not session:
-            raise Exception(
-                "Unauthenticated: Please login using tfy login --host <https://example-domain.com>"
-            )
-
-        host = session.base_url
-
-        if host[-1] == "/":
-            host = host[: len(host) - 1]
-
-        url = f"{host}/llm-playground/api/inference/text"
-        headers = {"Authorization": f"Bearer {session.access_token}"}
-
-        json = {
-            "prompt": prompt,
-            "models": [
-                {
-                    "name": self.model_name,
-                    "provider": self.provider,
-                    "tag": self.model_name,
-                    "parameters": params,
-                }
-            ],
-        }
-
-        try:
-            response = requests.post(url=url, headers=headers, json=json)
-            response.raise_for_status()
-        except Exception as ex:
-            raise Exception(f"Error inferencing the model: {ex}") from ex
-
-        data = response.json()
-        text = data[0].get("text")
-        if stop:
-            text = enforce_stop_tokens(text, stop)
-        return text

truefoundry/langchain/truefoundry_chat.py

@@ -1,130 +0,0 @@
-from typing import Any, Dict, List, Optional
-
-from langchain.chat_models.base import SimpleChatModel
-from langchain.pydantic_v1 import Extra, Field, root_validator
-from langchain.schema.messages import (
-    AIMessage,
-    BaseMessage,
-    ChatMessage,
-    HumanMessage,
-    SystemMessage,
-)
-
-from truefoundry.common.request_utils import requests_retry_session
-from truefoundry.langchain.utils import (
-    validate_tfy_environment,
-)
-from truefoundry.logger import logger
-
-
-class TrueFoundryChat(SimpleChatModel):
-    """`TrueFoundry LLM Gateway` chat models API.
-
-    To use, you must have the environment variable ``TFY_API_KEY`` set with your API key and ``TFY_HOST`` set with your host or pass it as a named parameter to the constructor.
-    """
-
-    model: str = Field(description="The model to use for chat.")
-    """The model to use for chat."""
-    tfy_llm_gateway_url: Optional[str] = Field(default=None)
-    """TrueFoundry LLM Gateway endpoint URL. Automatically inferred from env var `TFY_LLM_GATEWAY_URL` if not provided."""
-    tfy_api_key: Optional[str] = Field(default=None)
-    """TrueFoundry API Key. Automatically inferred from env var `TFY_API_KEY` if not provided."""
-    model_parameters: Optional[dict] = Field(default_factory=dict)
-    """Model parameters"""
-    request_timeout: int = Field(default=30)
-    """The timeout for the request in seconds."""
-    max_retries: int = Field(default=5)
-    """The number of retries for HTTP requests."""
-    retry_backoff_factor: float = Field(default=0.3)
-    """The backoff factor for exponential backoff during retries."""
-    system_prompt: str = Field(default="You are a AI assistant")
-
-    class Config:
-        """Configuration for this pydantic object."""
-
-        extra = Extra.forbid
-        allow_population_by_field_name = True
-
-    @root_validator()
-    def validate_environment(cls, values: Dict) -> Dict:
-        values = validate_tfy_environment(values)
-        if not values["tfy_api_key"]:
-            raise ValueError(
-                "Did not find `tfy_api_key`, please add an environment variable"
-                " `TFY_API_KEY` which contains it, or pass"
-                "  `tfy_api_key` as a named parameter."
-            )
-        if not values["tfy_llm_gateway_url"]:
-            raise ValueError(
-                "Did not find `tfy_llm_gateway_url`, please add an environment variable"
-                " `TFY_LLM_GATEWAY_URL` which contains it, or pass"
-                "  `tfy_llm_gateway_url` as a named parameter."
-            )
-        return values
-
-    @property
-    def _llm_type(self) -> str:
-        """Return type of chat model."""
-        return "truefoundry-chat"
-
-    def _call(
-        self,
-        messages: List[BaseMessage],
-        stop: Optional[List[str]] = None,
-        **kwargs: Any,
-    ) -> str:
-        if len(messages) == 0:
-            raise ValueError("No messages provided to chat.")
-
-        if not isinstance(messages[0], SystemMessage):
-            messages.insert(0, SystemMessage(content=self.system_prompt))
-
-        message_dicts = [
-            TrueFoundryChat._convert_message_to_dict(message) for message in messages
-        ]
-
-        payload = {**self.model_parameters} if self.model_parameters else {}
-
-        if stop:
-            payload["stop_sequences"] = stop
-
-        payload["messages"] = message_dicts
-        payload["model"] = self.model
-
-        session = requests_retry_session(
-            retries=self.max_retries, backoff_factor=self.retry_backoff_factor
-        )
-
-        url = f"{self.tfy_llm_gateway_url}/openai/chat/completions"
-        logger.debug(f"Chat using - model: {self.model} at endpoint: {url}")
-        response = session.post(
-            url=url,
-            json=payload,
-            headers={
-                "Authorization": f"Bearer {self.tfy_api_key}",
-            },
-            timeout=self.request_timeout,
-        )
-        response.raise_for_status()
-        output = response.json()
-        return output["choices"][0]["message"]["content"]
-
-    @staticmethod
-    def _convert_message_to_dict(message: BaseMessage) -> dict:
-        if isinstance(message, ChatMessage):
-            message_dict = {"role": message.role, "content": message.content}
-        elif isinstance(message, HumanMessage):
-            message_dict = {"role": "user", "content": message.content}
-        elif isinstance(message, AIMessage):
-            message_dict = {"role": "assistant", "content": message.content}
-        elif isinstance(message, SystemMessage):
-            message_dict = {"role": "system", "content": message.content}
-        else:
-            raise ValueError(f"Got unknown message type: {message}")
-        if message.additional_kwargs:
-            logger.debug(
-                "Additional message arguments are unsupported by TrueFoundry LLM Gateway "
-                " and will be ignored: %s",
-                message.additional_kwargs,
-            )
-        return message_dict

truefoundry/langchain/truefoundry_embeddings.py

@@ -1,171 +0,0 @@
-import concurrent.futures
-import math
-from typing import Dict, List, Optional
-
-import tqdm
-from langchain.embeddings.base import Embeddings
-from langchain.pydantic_v1 import BaseModel, Extra, Field, root_validator
-
-from truefoundry.common.request_utils import requests_retry_session
-from truefoundry.langchain.utils import (
-    validate_tfy_environment,
-)
-from truefoundry.logger import logger
-
-EMBEDDER_BATCH_SIZE = 32
-PARALLEL_WORKERS = 4
-
-
-class TrueFoundryEmbeddings(BaseModel, Embeddings):
-    """`TrueFoundry LLM Gateway` embedding models API.
-
-    To use, you must have the environment variable ``TFY_API_KEY`` set with your API key and ``TFY_HOST`` set with your host or pass it
-    as a named parameter to the constructor.
-    """
-
-    model: str = Field(description="The model to use for embedding.")
-    """The model to use for embedding."""
-    tfy_llm_gateway_url: Optional[str] = Field(default=None)
-    """TrueFoundry LLM Gateway endpoint URL. Automatically inferred from env var `TFY_LLM_GATEWAY_URL` if not provided."""
-    tfy_api_key: Optional[str] = Field(default=None)
-    """TrueFoundry API Key. Automatically inferred from env var `TFY_API_KEY` if not provided."""
-    model_parameters: Optional[dict] = Field(default_factory=dict)
-    """Model parameters"""
-    request_timeout: int = Field(default=30)
-    """The timeout for the request in seconds."""
-    max_retries: int = Field(default=5)
-    """The number of retries for HTTP requests."""
-    retry_backoff_factor: float = Field(default=0.3)
-    """The backoff factor for exponential backoff during retries."""
-    batch_size: int = Field(default=EMBEDDER_BATCH_SIZE)
-    """The batch size to use for embedding."""
-    parallel_workers: int = Field(default=PARALLEL_WORKERS)
-    """The number of parallel workers to use for embedding."""
-
-    __private_attributes__ = {"_executor"}
-
-    class Config:
-        """Configuration for this pydantic object."""
-
-        extra = Extra.forbid
-        allow_population_by_field_name = True
-
-    @root_validator()
-    def validate_environment(cls, values: Dict) -> Dict:
-        values = validate_tfy_environment(values)
-        if not values["tfy_api_key"]:
-            raise ValueError(
-                "Did not find `tfy_api_key`, please add an environment variable"
-                " `TFY_API_KEY` which contains it, or pass"
-                "  `tfy_api_key` as a named parameter."
-            )
-        if not values["tfy_llm_gateway_url"]:
-            raise ValueError(
-                "Did not find `tfy_llm_gateway_url`, please add an environment variable"
-                " `TFY_LLM_GATEWAY_URL` which contains it, or pass"
-                "  `tfy_llm_gateway_url` as a named parameter."
-            )
-        return values
-
-    def _init_private_attributes(self):
-        self._executor = concurrent.futures.ThreadPoolExecutor(
-            max_workers=self.parallel_workers
-        )
-
-    @property
-    def _llm_type(self) -> str:
-        """Return type of embedding model."""
-        return "truefoundry-embeddings"
-
-    def __del__(self):
-        """
-        Destructor method to clean up the executor when the object is deleted.
-
-        Returns:
-            None
-        """
-        self._executor.shutdown()
-
-    def _remote_embed(self, texts, query_mode=False):
-        """
-        Perform remote embedding using a HTTP POST request to a designated endpoint.
-
-        Args:
-            texts (List[str]): A list of text strings to be embedded.
-            query_mode (bool): A flag to indicate if running in query mode or in embed mode (indexing).
-        Returns:
-            List[List[float]]: A list of embedded representations of the input texts.
-        """
-        session = requests_retry_session(
-            retries=self.max_retries, backoff_factor=self.retry_backoff_factor
-        )
-
-        payload = {
-            "input": texts,
-            "model": self.model,
-        }
-
-        url = f"{self.tfy_llm_gateway_url}/openai/embeddings"
-        logger.debug(
-            f"Embedding using - model: {self.model} at endpoint: {url}, for {len(texts)} texts"
-        )
-        response = session.post(
-            url=url,
-            json=payload,
-            headers={
-                "Authorization": f"Bearer {self.tfy_api_key}",
-            },
-            timeout=self.request_timeout,
-        )
-        response.raise_for_status()
-        output = response.json()
-        return [data["embedding"] for data in output["data"]]
-
-    def _embed(self, texts: List[str], query_mode: bool):
-        """
-        Perform embedding on a list of texts using remote embedding in chunks.
-
-        Args:
-            texts (List[str]): A list of text strings to be embedded.
-            query_mode (bool): A flag to indicate if running in query mode or in embed mode (indexing).
-        Returns:
-            List[List[float]]: A list of embedded representations of the input texts.
-        """
-        embeddings = []
-
-        def _feeder():
-            for i in range(0, len(texts), self.batch_size):
-                chunk = texts[i : i + self.batch_size]
-                yield chunk
-
-        embeddings = list(
-            tqdm.tqdm(
-                self._executor.map(self._remote_embed, _feeder()),
-                total=int(math.ceil(len(texts) / self.batch_size)),
-            )
-        )
-        return [item for batch in embeddings for item in batch]
-
-    def embed_documents(self, texts: List[str]) -> List[List[float]]:
-        """
-        Embed a list of text documents.
-
-        Args:
-            texts (List[str]): A list of text documents to be embedded.
-
-        Returns:
-            List[List[float]]: A list of embedded representations of the input documents.
-        """
-        return self._embed(texts, query_mode=False)
-
-    def embed_query(self, text: str) -> List[float]:
-        """
-        Embed a query text.
-
-        Args:
-            text (str): The query text to be embedded.
-
-        Returns:
-            List[float]: The embedded representation of the input query text.
-        """
-        return self._embed([text], query_mode=True)[0]