truefoundry 0.2.10__py3-none-any.whl → 0.3.0__py3-none-any.whl

truefoundry/deploy/v2/lib/source.py

@@ -0,0 +1,267 @@
+import os
+import tarfile
+import tempfile
+import time
+import warnings
+from typing import Callable, List, Optional
+
+import gitignorefile
+from tqdm import tqdm
+
+from truefoundry.deploy import builder
+from truefoundry.deploy.auto_gen import models
+from truefoundry.deploy.builder.docker_service import (
+    pull_docker_image,
+    push_docker_image,
+    push_docker_image_with_latest_tag,
+)
+from truefoundry.deploy.lib.clients.servicefoundry_client import (
+    ServiceFoundryServiceClient,
+)
+from truefoundry.deploy.lib.dao import workspace as workspace_lib
+from truefoundry.deploy.v2.lib.patched_models import Image, RemoteSource
+from truefoundry.logger import logger
+
+
+def _human_readable_size(num_bytes: float) -> str:
+    units = ["B", "KiB", "MiB", "GiB"]
+    i = 0
+    amount = num_bytes
+    while amount >= 1024 and i < len(units) - 1:
+        amount /= 1024.0
+        i += 1
+    amount = round(amount, 2)
+    return f"{amount} {units[i]}"
+
+
+def _make_tarfile(
+    output_filename: str,
+    source_dir: str,
+    additional_directories: List[str],
+    is_path_ignored: Optional[Callable[[str], bool]] = None,
+) -> None:
+    if not is_path_ignored:
+        # if no callback handler present assume that every file needs to be added
+        is_path_ignored = lambda *_: False  # noqa: E731
+
+    with tarfile.open(output_filename, "w:gz") as tar:
+        source_dirs = [source_dir]
+        source_dirs.extend(additional_directories)
+        for source_dir in source_dirs:
+            _add_files_in_tar(
+                is_path_ignored=is_path_ignored,
+                source_dir=source_dir,
+                tar=tar,
+            )
+
+
+def _add_files_in_tar(
+    is_path_ignored: Callable[[str], bool],
+    source_dir: str,
+    tar: tarfile.TarFile,
+) -> None:
+    for root, dirs, files in tqdm(
+        os.walk(source_dir, topdown=True), desc="Packaging source code"
+    ):
+        if is_path_ignored(root):
+            logger.debug("Ignoring directory %s", root)
+
+            # NOTE: we can safely ignore going through the sub-dir
+            # if root itself is excluded.
+            dirs.clear()
+            continue
+        logger.debug("Adding contents of the directory %s", root)
+        files_added_count = 0
+        for file in files:
+            file_path = os.path.join(root, file)
+            if not is_path_ignored(file_path):
+                arcname = os.path.relpath(file_path, source_dir)
+                tar.add(file_path, arcname=arcname)
+                logger.debug("Adding %s with arcname %r", file_path, arcname)
+                files_added_count += 1
+
+        if not files_added_count:
+            # If no files were added and the directory was not ignored too, then we
+            # want to add the directory as an empty dir
+            arcname = os.path.relpath(root, source_dir)
+            tar.add(root, arcname=arcname, recursive=False)
+            logger.debug("Adding empty directory %s with arcname %r", root, arcname)
+
+
+def _get_callback_handler_to_ignore_file_path(
+    source_dir: str,
+) -> Optional[Callable[[str], bool]]:
+    ignorefile_path = os.path.join(source_dir, ".tfyignore")
+    if os.path.exists(ignorefile_path):
+        logger.info(".tfyignore file found in %s", source_dir)
+        return gitignorefile.parse(path=ignorefile_path, base_path=source_dir)
+
+    ignorefile_path = os.path.join(source_dir, ".sfyignore")
+    if os.path.exists(ignorefile_path):
+        logger.info(".sfyignore file found in %s", source_dir)
+        warnings.warn(
+            "`.sfyignore` is deprecated and will be ignored in future versions. "
+            "Please rename the file to `.tfyignore`",
+            category=DeprecationWarning,
+            stacklevel=2,
+        )
+        return gitignorefile.parse(path=ignorefile_path, base_path=source_dir)
+
+    # check for valid git repo
+    try:
+        import git
+
+        repo = git.Repo(source_dir, search_parent_directories=True)
+        logger.info(
+            "Git repository detected in source. Files added in .gitignore will be ignored.\n"
+            "If you don't want to ignore these files or otherwise, "
+            "please create .tfyignore file and add file patterns to ignore."
+        )
+        return lambda file_path: bool(repo.ignored(file_path))  # noqa: E731
+    except Exception as ex:
+        logger.debug(
+            "Could not treat source %r as a git repository due to %r", source_dir, ex
+        )
+
+    logger.info(
+        "Neither `.tfyignore` file found in %s nor a valid git repository found. "
+        "We recommend you to create .tfyignore file and add file patterns to ignore",
+        source_dir,
+    )
+    return None
+
+
+def local_source_to_remote_source(
+    local_source: models.LocalSource,
+    workspace_fqn: str,
+    component_name: str,
+) -> RemoteSource:
+    with tempfile.TemporaryDirectory() as local_dir:
+        package_local_path = os.path.join(local_dir, "build.tar.gz")
+        source_dir = os.path.abspath(local_source.project_root_path)
+
+        if not os.path.exists(source_dir):
+            raise ValueError(
+                f"project root path {source_dir!r} of component {component_name!r} does not exist"
+            )
+
+        logger.info("Archiving contents of dir: %r", source_dir)
+
+        is_path_ignored = _get_callback_handler_to_ignore_file_path(source_dir)
+        _make_tarfile(
+            output_filename=package_local_path,
+            source_dir=source_dir,
+            additional_directories=[],
+            is_path_ignored=is_path_ignored,
+        )
+
+        try:
+            file_size = _human_readable_size(os.path.getsize(package_local_path))
+            logger.info("Code archive size: %r", file_size)
+        except Exception:
+            # Should not block code upload
+            logger.exception("Failed to calculate code archive size")
+
+        logger.debug("Uploading code archive.")
+        client = ServiceFoundryServiceClient()
+        remote_uri = client.upload_code_package(
+            workspace_fqn=workspace_fqn,
+            component_name=component_name,
+            package_local_path=package_local_path,
+        )
+        logger.debug("Uploaded code archive.")
+        return RemoteSource(remote_uri=remote_uri)
+
+
+def local_source_to_image(
+    build: models.Build,
+    docker_registry_fqn: Optional[str],
+    workspace_fqn: str,
+    component_name: str,
+) -> Image:
+    build = build.copy(deep=True)
+    source_dir = os.path.abspath(
+        os.path.expanduser(build.build_source.project_root_path)
+    )
+    if not os.path.exists(source_dir):
+        raise ValueError(
+            f"project root path {source_dir!r} of component {component_name!r} does not exist"
+        )
+
+    build.build_spec.build_context_path = os.path.join(
+        source_dir, build.build_spec.build_context_path
+    )
+    if not os.path.exists(build.build_spec.build_context_path):
+        raise ValueError(
+            f"Build context path {build.build_spec.build_context_path!r} "
+            f"of component {component_name!r} does not exist"
+        )
+
+    if isinstance(build.build_spec, models.DockerFileBuild):
+        build.build_spec.dockerfile_path = os.path.join(
+            source_dir, build.build_spec.dockerfile_path
+        )
+        if not os.path.exists(build.build_spec.dockerfile_path):
+            raise ValueError(
+                f"Dockerfile path {build.build_spec.dockerfile_path!r} "
+                f"of component {component_name!r} does not exist"
+            )
+
+    client = ServiceFoundryServiceClient()
+
+    workspace = workspace_lib.get_workspace_by_fqn(workspace_fqn=workspace_fqn)
+    docker_registry = client.get_docker_registry_creds(
+        docker_registry_fqn=docker_registry_fqn,
+        cluster_id=workspace.clusterId,
+    )
+
+    docker_registry_fqn = docker_registry.fqn
+    registry_url = docker_registry.registryUrl
+    registry_username = docker_registry.username
+    registry_password = docker_registry.password
+
+    create_repo_response = client.create_repo_in_registry(
+        docker_registry_fqn=docker_registry_fqn,
+        workspace_fqn=workspace_fqn,
+        application_name=component_name,
+    )
+    repo_name = create_repo_response.repoName
+
+    image_uri = f"{registry_url}/{repo_name}:{int(time.time())}"
+    cache_from = f"{registry_url}/{repo_name}:latest"
+
+    # pull cache image if it exists
+    try:
+        pull_docker_image(
+            image_uri=cache_from,
+            docker_login_username=registry_username,
+            docker_login_password=registry_password,
+        )
+    except Exception:
+        logger.info(f"Failed to pull {cache_from}. Building without cache image.")
+
+    # build
+    builder.build(
+        build.build_spec,
+        tag=image_uri,
+        extra_opts=["--cache-from", cache_from],
+    )
+
+    # push built image to registry
+    push_docker_image(
+        image_uri=image_uri,
+        docker_login_username=registry_username,
+        docker_login_password=registry_password,
+    )
+
+    # push 'latest' tag for image
+    push_docker_image_with_latest_tag(
+        image_uri=image_uri,
+        docker_login_username=registry_username,
+        docker_login_password=registry_password,
+    )
+    return Image(
+        image_uri=image_uri,
+        docker_registry=docker_registry_fqn,
+        command=build.build_spec.command,
+    )

truefoundry/langchain/__init__.py

@@ -1 +1,12 @@
-from servicefoundry.langchain import *  # noqa: F403
+try:
+    import langchain as _
+except Exception as ex:
+    raise Exception(
+        "Failed to import langchain. "
+        "Please install langchain by using `pip install langchain` command"
+    ) from ex
+from truefoundry.langchain.deprecated import TruefoundryLLM, TruefoundryPlaygroundLLM
+from truefoundry.langchain.truefoundry_chat import TrueFoundryChat
+from truefoundry.langchain.truefoundry_embeddings import TrueFoundryEmbeddings
+from truefoundry.langchain.truefoundry_llm import TrueFoundryLLM
+from truefoundry.langchain.utils import ModelParameters

truefoundry/langchain/deprecated.py

@@ -0,0 +1,302 @@
+import enum
+import warnings
+from typing import Any, Dict, List, Mapping, Optional
+from urllib.parse import urljoin
+
+import requests
+from requests.auth import HTTPBasicAuth
+
+from truefoundry.deploy.lib.auth.servicefoundry_session import ServiceFoundrySession
+from truefoundry.pydantic_v1 import root_validator
+
+try:
+    from langchain.callbacks.manager import CallbackManagerForLLMRun
+    from langchain.llms.base import LLM
+    from langchain.llms.utils import enforce_stop_tokens
+except Exception as ex:
+    raise Exception(
+        "Failed to import langchain."
+        " Please install langchain by using `pip install langchain` command"
+    ) from ex
+
+
+class _ModelServerImpl(str, enum.Enum):
+    MLSERVER = "MLSERVER"
+    TGI = "TGI"
+    VLLM = "VLLM"
+
+
+def _get_model_server_and_validate_if_mlserver(endpoint_url, auth, model_name=None):
+    try:
+        response = requests.get(urljoin(endpoint_url, "info"), json={}, auth=auth)
+        if response.status_code == 200:
+            return _ModelServerImpl.TGI, None
+        elif response.status_code == 404:
+            # We are not using TGI, try for mlserver
+            response = requests.post(
+                urljoin(endpoint_url, "v2/repository/index"), json={}, auth=auth
+            )
+            if response.status_code == 200:
+                models = response.json()
+                if len(models) == 0:
+                    raise ValueError("No model is deployed in the model server")
+                model_names = [m.get("name") for m in models]
+                if model_name and model_name not in model_names:
+                    raise ValueError(
+                        f"Model {model_name!r} is not available in the model server. "
+                        f"Available models {model_names!r}"
+                    )
+                if not model_name and len(model_names) > 1:
+                    raise ValueError(
+                        f"Please pass `model_name` while instantiating `TruefoundryLLM`. "
+                        f"Available models are {model_names!r} "
+                    )
+                if model_name:
+                    return _ModelServerImpl.MLSERVER, model_name
+                return _ModelServerImpl.MLSERVER, model_names[0]
+            if response.status_code == 404:
+                return _ModelServerImpl.VLLM, None
+        response.raise_for_status()
+    except Exception as e:
+        raise Exception(f"Error raised by inference API: {e}") from e
+
+
+# TODO (chiragjn): Refactor this into separate implementations for each model server
+
+
+class TruefoundryLLM(LLM):
+    """Wrapper around TFY model deployment.
+    To use this class, you need to have the langchain library installed.
+    Example:
+        .. code-block:: python
+            from truefoundry.langchain import TruefoundryLLM
+            endpoint_url = (
+                "https://pythia-70m-model-model-catalogue.demo2.truefoundry.tech"
+            )
+            model = TruefoundryLLM(
+                endpoint_url=endpoint_url,
+                parameters={
+                    "max_new_tokens": 100,
+                    "temperature": 0.7,
+                    "top_k": 5,
+                    "top_p": 0.9
+                }
+            )
+    """
+
+    endpoint_url: str
+    model_name: Optional[str] = None
+    auth: Optional[HTTPBasicAuth] = None
+    parameters: Optional[Dict[str, Any]] = None
+    model_server_impl: Optional[_ModelServerImpl] = None
+
+    @root_validator(pre=False)
+    def validate_model_server_and_name(cls, values: Dict):
+        warnings.warn(
+            message=f"{cls.__name__} is deprecated and will be removed soon. Please use `TrueFoundryLLM` or `TrueFoundryChat` to invoke models using the new TrueFoundry LLM Gateway",
+            category=DeprecationWarning,
+            stacklevel=2,
+        )
+        endpoint_url = values["endpoint_url"]
+        model_name = values.get("model_name")
+        auth = values.get("auth")
+        model_server_impl, model_name = _get_model_server_and_validate_if_mlserver(
+            endpoint_url=endpoint_url, model_name=model_name, auth=auth
+        )
+        values["model_server_impl"] = model_server_impl
+        if model_server_impl == _ModelServerImpl.MLSERVER:
+            values["model_name"] = model_name
+        return values
+
+    @property
+    def _identifying_params(self) -> Mapping[str, Any]:
+        """Get the identifying parameters."""
+        return {
+            "endpoint_url": self.endpoint_url,
+            "model_name": self.model_name,
+        }
+
+    @property
+    def _llm_type(self) -> str:
+        """Return type of llm."""
+        return "tfy_model_deployment"
+
+    def _call(  # noqa: C901
+        self,
+        prompt: str,
+        stop: Optional[List[str]] = None,
+        run_manager: Optional[CallbackManagerForLLMRun] = None,
+        **params: Any,
+    ) -> str:
+        """Call out to the deployed model
+        Args:
+            prompt: The prompt to pass into the model.
+            stop: Optional list of stop words to use when generating.
+        Returns:
+            The string generated by the model.
+        Example:
+            .. code-block:: python
+                response = model("Tell me a joke.")
+        """
+        _params_already_set = self.parameters or {}
+        params = {**_params_already_set, **params, "return_full_text": False}
+
+        if self.model_server_impl == _ModelServerImpl.MLSERVER:
+            generate_path = f"v2/models/{self.model_name}/infer/simple"
+            payload = {"inputs": prompt, "parameters": params}
+        elif self.model_server_impl == _ModelServerImpl.TGI:
+            generate_path = "generate"
+            payload = {"inputs": prompt, "parameters": params}
+        elif self.model_server_impl == _ModelServerImpl.VLLM:
+            generate_path = "generate"
+            payload = {**params, "prompt": prompt}
+        else:
+            raise ValueError(f"No known generate path for {self.model_server_impl}")
+        url = urljoin(self.endpoint_url, generate_path)
+
+        try:
+            response = requests.post(url, json=payload, auth=self.auth)
+            response.raise_for_status()
+        except Exception as e:
+            raise Exception(f"Error raised by inference API: {e}") from e
+        response_dict = response.json()
+        if "error" in response_dict:
+            raise ValueError(
+                f"Error raised by inference API: {response_dict['error']!r}"
+            )
+
+        if self.model_server_impl == _ModelServerImpl.MLSERVER:
+            inference_result = response_dict[0]
+        elif self.model_server_impl == _ModelServerImpl.TGI:
+            inference_result = response_dict
+        elif self.model_server_impl == _ModelServerImpl.VLLM:
+            inference_result = response_dict
+        else:
+            raise ValueError(
+                f"Unknown model server {self.model_server_impl}, cannot parse response"
+            )
+
+        if "generated_text" in inference_result:
+            text = inference_result["generated_text"]
+        elif "summarization" in inference_result:
+            text = inference_result["summary_text"]
+        elif "text" in inference_result:
+            text = inference_result["text"]
+        else:
+            raise ValueError(f"Could not parse inference response: {response_dict!r}")
+
+        if isinstance(text, list):
+            text = text[0]
+
+        if stop:
+            text = enforce_stop_tokens(text, stop)
+
+        return text
+
+
+class TruefoundryPlaygroundLLM(LLM):
+    """Wrapper around TFY Playground.
+    To use this class, you need to have the langchain library installed.
+    Example:
+        .. code-block:: python
+            from truefoundry.langchain import TruefoundryPlaygroundLLM
+            import os
+            # Note: Login using tfy login --host <https://example-domain.com>
+            model = TruefoundryPlaygroundLLM(
+                model_name="vicuna-13b",
+                parameters={
+                    "maximumLength": 100,
+                    "temperature": 0.7,
+                    "topP": 0.9,
+                    "repetitionPenalty": 1
+                }
+            )
+            response = model.predict("Enter the prompt here")
+    """
+
+    model_name: str
+    parameters: Optional[Dict[str, Any]] = None
+    provider: str = "truefoundry-public"
+
+    @root_validator(pre=False)
+    def validate_model_server_and_name(cls, values: Dict):
+        warnings.warn(
+            message=f"{cls.__name__} is deprecated and will be removed soon. Please use `TrueFoundryLLM` or `TrueFoundryChat` to invoke models using the new TrueFoundry LLM Gateway",
+            category=DeprecationWarning,
+            stacklevel=2,
+        )
+        return values
+
+    @property
+    def _get_model(self) -> str:
+        """returns the model name"""
+        return self.model_name
+
+    @property
+    def _get_provider(self) -> str:
+        """Returns the provider name"""
+        return self.provider
+
+    @property
+    def _llm_type(self) -> str:
+        """Return type of llm."""
+        return "tfy_playground"
+
+    def _call(
+        self,
+        prompt: str,
+        stop: Optional[List[str]] = None,
+        **params: Any,
+    ) -> str:
+        """Call out to the deployed model
+        Args:
+            prompt: The prompt to pass into the model.
+            stop: Optional list of stop words to use when generating.
+        Returns:
+            The string generated by the model.
+        Example:
+            .. code-block:: python
+                response = model("I have a joke for you...")
+        """
+        _params_already_set = self.parameters or {}
+        params = {**_params_already_set, **params}
+        if stop:
+            params["stopSequences"] = stop
+        session = ServiceFoundrySession()
+
+        if not session:
+            raise Exception(
+                "Unauthenticated: Please login using tfy login --host <https://example-domain.com>"
+            )
+
+        host = session.base_url
+
+        if host[-1] == "/":
+            host = host[: len(host) - 1]
+
+        url = f"{host}/llm-playground/api/inference/text"
+        headers = {"Authorization": f"Bearer {session.access_token}"}
+
+        json = {
+            "prompt": prompt,
+            "models": [
+                {
+                    "name": self.model_name,
+                    "provider": self.provider,
+                    "tag": self.model_name,
+                    "parameters": params,
+                }
+            ],
+        }
+
+        try:
+            response = requests.post(url=url, headers=headers, json=json)
+            response.raise_for_status()
+        except Exception as ex:
+            raise Exception(f"Error inferencing the model: {ex}") from ex
+
+        data = response.json()
+        text = data[0].get("text")
+        if stop:
+            text = enforce_stop_tokens(text, stop)
+        return text