truefoundry 0.5.0rc7__py3-none-any.whl → 0.5.1rc2__py3-none-any.whl

truefoundry/deploy/function_service/route.py

@@ -1,137 +0,0 @@
-import inspect
-import json
-from typing import Any, Callable, Dict, List
-
-from truefoundry.deploy.function_service.remote import RemoteClass
-from truefoundry.deploy.function_service.utils import (
-    create_pydantic_model_from_function_signature,
-    get_qual_name,
-)
-from truefoundry.logger import logger
-from truefoundry.pydantic_v1 import BaseModel, Field, constr, validator
-
-
-def validate_we_can_create_pydantic_model_of_func_args(func: Callable):
-    qual_name = get_qual_name(func)
-    try:
-        create_pydantic_model_from_function_signature(func, get_qual_name(func))
-    except Exception as ex:
-        raise Exception(
-            f"Unable to create a route for {qual_name!r}.\n"
-            "Please ensure that in the function type signature, you have only used in-built\n"
-            "types like `int`, `float`, `str`, `bool`, `typing.Dict`, `typing.List`, typing.Optional`.\n"
-            "To temporarily resolve this error, you can remove the unsupported type signatures.\n"
-        ) from ex
-
-
-def path_pre_processor(path: str, prefix: str = "") -> str:
-    path = path.strip("/")
-
-    if not path:
-        raise ValueError("path cannot be empty")
-
-    prefix = prefix.strip("/")
-    if not prefix:
-        return f"/{path}"
-
-    return f"/{prefix}/{path}"
-
-
-class Route(BaseModel):
-    function_name: str
-    http_method: str
-    path: constr(regex=r"^[A-Za-z0-9\-_/]+$")
-
-    qual_name: str
-
-
-class FunctionRoute(Route):
-    module: str
-
-    @classmethod
-    def from_func(cls, func: Callable, path: str):
-        validate_we_can_create_pydantic_model_of_func_args(func)
-        return cls(
-            function_name=func.__name__,
-            http_method="POST",
-            path=path_pre_processor(path or func.__name__),
-            qual_name=get_qual_name(func),
-            module=func.__module__,
-        )
-
-
-class ClassRoute(BaseModel):
-    class_name: str
-    init_kwargs: Dict[str, Any] = Field(default_factory=dict)
-    module: str
-
-    routes: List[Route] = Field(default_factory=list)
-
-    @validator("init_kwargs")
-    def init_kwargs_is_json_serializable(cls, v, values):
-        try:
-            json.dumps(v)
-        except Exception as ex:
-            class_name = values.get("class_name")
-            raise ValueError(
-                f"init_kwargs {v!r} of class {class_name!r} is not JSON serializable"
-            ) from ex
-
-        return v
-
-    @classmethod
-    def from_class(cls, remote_class: RemoteClass):
-        routes = []
-        methods = inspect.getmembers(remote_class.class_, predicate=inspect.isfunction)
-
-        for method_name, method in methods:
-            if method_name.startswith("_"):
-                continue
-            validate_we_can_create_pydantic_model_of_func_args(method)
-            route = Route(
-                function_name=method_name,
-                http_method="POST",
-                path=path_pre_processor(prefix=remote_class.name, path=method_name),
-                qual_name=remote_class.get_qual_name(method),
-            )
-            routes.append(route)
-
-        return cls(
-            class_name=remote_class.class_.__name__,
-            init_kwargs=remote_class.init_kwargs,
-            routes=routes,
-            module=remote_class.class_.__module__,
-        )
-
-
-class RouteGroups(BaseModel):
-    functions: List[FunctionRoute] = Field(default_factory=list)
-    classes: Dict[str, ClassRoute] = Field(default_factory=dict)
-
-    def register_function(self, func, path):
-        function_route = FunctionRoute.from_func(func=func, path=path)
-        logger.info(
-            "Function %r from module %r will be deployed on path '%s %s'.",
-            function_route.function_name,
-            function_route.module,
-            function_route.http_method,
-            function_route.path,
-        )
-        self.functions.append(function_route)
-
-    def register_class(self, remote_class: RemoteClass):
-        if remote_class.name in self.classes:
-            raise ValueError(
-                f"name {remote_class.name!r} is already used to register a class"
-            )
-        class_route = ClassRoute.from_class(remote_class)
-        for route in class_route.routes:
-            logger.info(
-                "Method %r from `%s:%s` will be deployed on path '%s %s'.",
-                route.function_name,
-                class_route.class_name,
-                remote_class.name,
-                route.http_method,
-                route.path,
-            )
-        self.classes[remote_class.name] = class_route

truefoundry/deploy/function_service/service.py

@@ -1,113 +0,0 @@
-from threading import Thread
-from typing import Any, Callable, Dict, Optional, Union
-
-import yaml
-
-from truefoundry.deploy.auto_gen.models import Port, Resources
-from truefoundry.deploy.function_service.app import (
-    build_and_run_app_in_background_thread,
-)
-from truefoundry.deploy.function_service.build import BuildConfig
-from truefoundry.deploy.function_service.remote import RemoteClass
-from truefoundry.deploy.function_service.route import RouteGroups
-from truefoundry.deploy.v2.lib.deployable_patched_models import Service
-from truefoundry.deploy.v2.lib.patched_models import Build, LocalSource
-from truefoundry.logger import logger
-
-
-class FunctionService:
-    def __init__(
-        self,
-        name: str,
-        build_config: Optional[BuildConfig] = None,
-        resources: Optional[Resources] = None,
-        replicas: int = 1,
-        port: Union[int, Port] = 8000,
-        env: Optional[Dict[str, str]] = None,
-    ):
-        self._name = name
-        self._build_config = build_config or BuildConfig()
-        self._resources = resources or Resources()
-        self._replicas = replicas
-        if isinstance(port, int):
-            port = Port(port=port, expose=False)
-        if not port.host:
-            logger.warning(
-                "No host is set for the port. This is not an issue if you don't "
-                "want an exposed endpoint or are just testing locally.\n"
-                "However, for actual deployment it is required to pass an "
-                "instance of `truefoundry.deploy.Port` with "
-                "`host` argument defined.\n"
-                "E.g. `FunctionService(name='...', port=Port(port=8000, host='...', path='...'), ...)`"
-            )
-        self._port = port
-        self._env = env or {}
-
-        self._route_groups: RouteGroups = RouteGroups()
-
-    @property
-    def route_groups(self) -> RouteGroups:
-        return self._route_groups
-
-    def __repr__(self):
-        return yaml.dump(
-            {
-                "name": self._name,
-                "build_config": self._build_config.dict(),
-                "resources": self._resources.dict(),
-                "routes": self._route_groups.dict(),
-                "replicas": self._replicas,
-                "port": self._port.dict(),
-                "env": self._env,
-            },
-            indent=2,
-        )
-
-    def register_function(
-        self,
-        func: Callable,
-        *,
-        path: Optional[str] = None,
-    ):
-        self._route_groups.register_function(func=func, path=path)
-
-    def register_class(
-        self,
-        class_,
-        *,
-        init_kwargs: Optional[Dict[str, Any]] = None,
-        name: Optional[str] = None,
-    ):
-        # TODO: I need to rethink this `RemoteClass`.
-        #   I am mixing up multiple responsibilities here.
-        #   For now, I am removing the burden of using `remote` from the user when deploying
-        #   an instance of a class.
-        remote_class = RemoteClass(class_, init_kwargs=init_kwargs, name=name)
-        self._route_groups.register_class(remote_class=remote_class)
-
-    def run(self) -> Thread:
-        return build_and_run_app_in_background_thread(
-            route_groups=self._route_groups, port=self._port.port
-        )
-
-    def get_deployment_definition(self) -> Service:
-        # Keeping this function right now so that later,
-        # the constructor of the application call this function
-        # to get the component spec, if an object of this class
-        # is directly passed as a component
-        tfy_python_build_config = self._build_config.to_tfy_python_build_config(
-            port=self._port.port, route_groups=self._route_groups
-        )
-        service = Service(
-            name=self._name,
-            image=Build(build_source=LocalSource(), build_spec=tfy_python_build_config),
-            resources=self._resources,
-            replicas=self._replicas,
-            ports=[self._port],
-            env=self._env,
-        )
-        return service
-
-    def deploy(self, workspace_fqn: str, wait: bool = True):
-        service = self.get_deployment_definition()
-        service.deploy(workspace_fqn=workspace_fqn, wait=wait)

truefoundry/deploy/function_service/utils.py

@@ -1,53 +0,0 @@
-import inspect
-from typing import Any
-
-from truefoundry.pydantic_v1 import BaseModel
-from truefoundry.pydantic_v1 import create_model as pydantic_create_model
-
-
-def get_qual_name(obj):
-    return f"{obj.__module__}.{obj.__qualname__}"
-
-
-def create_pydantic_model_from_function_signature(func, model_name: str):
-    # https://github.com/pydantic/pydantic/issues/1391
-    (
-        args,
-        _,
-        varkw,
-        defaults,
-        kwonlyargs,
-        kwonlydefaults,
-        annotations,
-    ) = inspect.getfullargspec(func)
-    defaults = defaults or []
-    args = args or []
-    if len(args) > 0 and args[0] == "self":
-        del args[0]
-
-    non_default_args = len(args) - len(defaults)
-    defaults = [
-        ...,
-    ] * non_default_args + list(defaults)
-
-    keyword_only_params = {
-        param: kwonlydefaults.get(param, Any) for param in kwonlyargs
-    }
-    params = {
-        param: (annotations.get(param, Any), default)
-        for param, default in zip(args, defaults)
-    }
-
-    class Config:
-        extra = "allow"
-
-    # Allow extra params if there is a **kwargs parameter in the function signature
-    config = Config if varkw else None
-
-    return pydantic_create_model(
-        model_name,
-        **params,
-        **keyword_only_params,
-        __base__=BaseModel,
-        __config__=config,
-    )

truefoundry/langchain/__init__.py

@@ -1,12 +0,0 @@
-try:
-    import langchain as _
-except Exception as ex:
-    raise Exception(
-        "Failed to import langchain. "
-        "Please install langchain by using `pip install langchain` command"
-    ) from ex
-from truefoundry.langchain.deprecated import TruefoundryLLM, TruefoundryPlaygroundLLM
-from truefoundry.langchain.truefoundry_chat import TrueFoundryChat
-from truefoundry.langchain.truefoundry_embeddings import TrueFoundryEmbeddings
-from truefoundry.langchain.truefoundry_llm import TrueFoundryLLM
-from truefoundry.langchain.utils import ModelParameters

truefoundry/langchain/deprecated.py

@@ -1,302 +0,0 @@
-import enum
-import warnings
-from typing import Any, Dict, List, Mapping, Optional
-from urllib.parse import urljoin
-
-import requests
-from requests.auth import HTTPBasicAuth
-
-from truefoundry.deploy.lib.auth.servicefoundry_session import ServiceFoundrySession
-from truefoundry.pydantic_v1 import root_validator
-
-try:
-    from langchain.callbacks.manager import CallbackManagerForLLMRun
-    from langchain.llms.base import LLM
-    from langchain.llms.utils import enforce_stop_tokens
-except Exception as ex:
-    raise Exception(
-        "Failed to import langchain."
-        " Please install langchain by using `pip install langchain` command"
-    ) from ex
-
-
-class _ModelServerImpl(str, enum.Enum):
-    MLSERVER = "MLSERVER"
-    TGI = "TGI"
-    VLLM = "VLLM"
-
-
-def _get_model_server_and_validate_if_mlserver(endpoint_url, auth, model_name=None):
-    try:
-        response = requests.get(urljoin(endpoint_url, "info"), json={}, auth=auth)
-        if response.status_code == 200:
-            return _ModelServerImpl.TGI, None
-        elif response.status_code == 404:
-            # We are not using TGI, try for mlserver
-            response = requests.post(
-                urljoin(endpoint_url, "v2/repository/index"), json={}, auth=auth
-            )
-            if response.status_code == 200:
-                models = response.json()
-                if len(models) == 0:
-                    raise ValueError("No model is deployed in the model server")
-                model_names = [m.get("name") for m in models]
-                if model_name and model_name not in model_names:
-                    raise ValueError(
-                        f"Model {model_name!r} is not available in the model server. "
-                        f"Available models {model_names!r}"
-                    )
-                if not model_name and len(model_names) > 1:
-                    raise ValueError(
-                        f"Please pass `model_name` while instantiating `TruefoundryLLM`. "
-                        f"Available models are {model_names!r} "
-                    )
-                if model_name:
-                    return _ModelServerImpl.MLSERVER, model_name
-                return _ModelServerImpl.MLSERVER, model_names[0]
-            if response.status_code == 404:
-                return _ModelServerImpl.VLLM, None
-        response.raise_for_status()
-    except Exception as e:
-        raise Exception(f"Error raised by inference API: {e}") from e
-
-
-# TODO (chiragjn): Refactor this into separate implementations for each model server
-
-
-class TruefoundryLLM(LLM):
-    """Wrapper around TFY model deployment.
-    To use this class, you need to have the langchain library installed.
-    Example:
-        .. code-block:: python
-            from truefoundry.langchain import TruefoundryLLM
-            endpoint_url = (
-                "https://pythia-70m-model-model-catalogue.demo2.truefoundry.tech"
-            )
-            model = TruefoundryLLM(
-                endpoint_url=endpoint_url,
-                parameters={
-                    "max_new_tokens": 100,
-                    "temperature": 0.7,
-                    "top_k": 5,
-                    "top_p": 0.9
-                }
-            )
-    """
-
-    endpoint_url: str
-    model_name: Optional[str] = None
-    auth: Optional[HTTPBasicAuth] = None
-    parameters: Optional[Dict[str, Any]] = None
-    model_server_impl: Optional[_ModelServerImpl] = None
-
-    @root_validator(pre=False)
-    def validate_model_server_and_name(cls, values: Dict):
-        warnings.warn(
-            message=f"{cls.__name__} is deprecated and will be removed soon. Please use `TrueFoundryLLM` or `TrueFoundryChat` to invoke models using the new TrueFoundry LLM Gateway",
-            category=DeprecationWarning,
-            stacklevel=2,
-        )
-        endpoint_url = values["endpoint_url"]
-        model_name = values.get("model_name")
-        auth = values.get("auth")
-        model_server_impl, model_name = _get_model_server_and_validate_if_mlserver(
-            endpoint_url=endpoint_url, model_name=model_name, auth=auth
-        )
-        values["model_server_impl"] = model_server_impl
-        if model_server_impl == _ModelServerImpl.MLSERVER:
-            values["model_name"] = model_name
-        return values
-
-    @property
-    def _identifying_params(self) -> Mapping[str, Any]:
-        """Get the identifying parameters."""
-        return {
-            "endpoint_url": self.endpoint_url,
-            "model_name": self.model_name,
-        }
-
-    @property
-    def _llm_type(self) -> str:
-        """Return type of llm."""
-        return "tfy_model_deployment"
-
-    def _call(  # noqa: C901
-        self,
-        prompt: str,
-        stop: Optional[List[str]] = None,
-        run_manager: Optional[CallbackManagerForLLMRun] = None,
-        **params: Any,
-    ) -> str:
-        """Call out to the deployed model
-        Args:
-            prompt: The prompt to pass into the model.
-            stop: Optional list of stop words to use when generating.
-        Returns:
-            The string generated by the model.
-        Example:
-            .. code-block:: python
-                response = model("Tell me a joke.")
-        """
-        _params_already_set = self.parameters or {}
-        params = {**_params_already_set, **params, "return_full_text": False}
-
-        if self.model_server_impl == _ModelServerImpl.MLSERVER:
-            generate_path = f"v2/models/{self.model_name}/infer/simple"
-            payload = {"inputs": prompt, "parameters": params}
-        elif self.model_server_impl == _ModelServerImpl.TGI:
-            generate_path = "generate"
-            payload = {"inputs": prompt, "parameters": params}
-        elif self.model_server_impl == _ModelServerImpl.VLLM:
-            generate_path = "generate"
-            payload = {**params, "prompt": prompt}
-        else:
-            raise ValueError(f"No known generate path for {self.model_server_impl}")
-        url = urljoin(self.endpoint_url, generate_path)
-
-        try:
-            response = requests.post(url, json=payload, auth=self.auth)
-            response.raise_for_status()
-        except Exception as e:
-            raise Exception(f"Error raised by inference API: {e}") from e
-        response_dict = response.json()
-        if "error" in response_dict:
-            raise ValueError(
-                f"Error raised by inference API: {response_dict['error']!r}"
-            )
-
-        if self.model_server_impl == _ModelServerImpl.MLSERVER:
-            inference_result = response_dict[0]
-        elif self.model_server_impl == _ModelServerImpl.TGI:
-            inference_result = response_dict
-        elif self.model_server_impl == _ModelServerImpl.VLLM:
-            inference_result = response_dict
-        else:
-            raise ValueError(
-                f"Unknown model server {self.model_server_impl}, cannot parse response"
-            )
-
-        if "generated_text" in inference_result:
-            text = inference_result["generated_text"]
-        elif "summarization" in inference_result:
-            text = inference_result["summary_text"]
-        elif "text" in inference_result:
-            text = inference_result["text"]
-        else:
-            raise ValueError(f"Could not parse inference response: {response_dict!r}")
-
-        if isinstance(text, list):
-            text = text[0]
-
-        if stop:
-            text = enforce_stop_tokens(text, stop)
-
-        return text
-
-
-class TruefoundryPlaygroundLLM(LLM):
-    """Wrapper around TFY Playground.
-    To use this class, you need to have the langchain library installed.
-    Example:
-        .. code-block:: python
-            from truefoundry.langchain import TruefoundryPlaygroundLLM
-            import os
-            # Note: Login using tfy login --host <https://example-domain.com>
-            model = TruefoundryPlaygroundLLM(
-                model_name="vicuna-13b",
-                parameters={
-                    "maximumLength": 100,
-                    "temperature": 0.7,
-                    "topP": 0.9,
-                    "repetitionPenalty": 1
-                }
-            )
-            response = model.predict("Enter the prompt here")
-    """
-
-    model_name: str
-    parameters: Optional[Dict[str, Any]] = None
-    provider: str = "truefoundry-public"
-
-    @root_validator(pre=False)
-    def validate_model_server_and_name(cls, values: Dict):
-        warnings.warn(
-            message=f"{cls.__name__} is deprecated and will be removed soon. Please use `TrueFoundryLLM` or `TrueFoundryChat` to invoke models using the new TrueFoundry LLM Gateway",
-            category=DeprecationWarning,
-            stacklevel=2,
-        )
-        return values
-
-    @property
-    def _get_model(self) -> str:
-        """returns the model name"""
-        return self.model_name
-
-    @property
-    def _get_provider(self) -> str:
-        """Returns the provider name"""
-        return self.provider
-
-    @property
-    def _llm_type(self) -> str:
-        """Return type of llm."""
-        return "tfy_playground"
-
-    def _call(
-        self,
-        prompt: str,
-        stop: Optional[List[str]] = None,
-        **params: Any,
-    ) -> str:
-        """Call out to the deployed model
-        Args:
-            prompt: The prompt to pass into the model.
-            stop: Optional list of stop words to use when generating.
-        Returns:
-            The string generated by the model.
-        Example:
-            .. code-block:: python
-                response = model("I have a joke for you...")
-        """
-        _params_already_set = self.parameters or {}
-        params = {**_params_already_set, **params}
-        if stop:
-            params["stopSequences"] = stop
-        session = ServiceFoundrySession()
-
-        if not session:
-            raise Exception(
-                "Unauthenticated: Please login using tfy login --host <https://example-domain.com>"
-            )
-
-        host = session.base_url
-
-        if host[-1] == "/":
-            host = host[: len(host) - 1]
-
-        url = f"{host}/llm-playground/api/inference/text"
-        headers = {"Authorization": f"Bearer {session.access_token}"}
-
-        json = {
-            "prompt": prompt,
-            "models": [
-                {
-                    "name": self.model_name,
-                    "provider": self.provider,
-                    "tag": self.model_name,
-                    "parameters": params,
-                }
-            ],
-        }
-
-        try:
-            response = requests.post(url=url, headers=headers, json=json)
-            response.raise_for_status()
-        except Exception as ex:
-            raise Exception(f"Error inferencing the model: {ex}") from ex
-
-        data = response.json()
-        text = data[0].get("text")
-        if stop:
-            text = enforce_stop_tokens(text, stop)
-        return text