truss 0.9.17__tar.gz → 0.9.17.dev2__tar.gz

{truss-0.9.17 → truss-0.9.17.dev2}/PKG-INFO

@@ -1,20 +1,19 @@
 Metadata-Version: 2.1
 Name: truss
-Version: 0.9.17
+Version: 0.9.17.dev2
 Summary: A seamless bridge from model development to model delivery
 Home-page: https://github.com/basetenlabs/truss
 License: MIT
 Keywords: MLOps,AI,Model Serving,Model Deployment,Machine Learning
 Author: Pankaj Gupta
 Author-email: pankaj@baseten.co
-Requires-Python: >=3.8,<3.13
+Requires-Python: >=3.8,<3.12
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
 Requires-Dist: Jinja2 (>=3.1.2,<4.0.0)
 Requires-Dist: PyYAML (>=6.0)
 Requires-Dist: autoflake (<=2.2)

{truss-0.9.17 → truss-0.9.17.dev2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "truss"
-version = "0.9.17"
+version = "0.9.17dev2"
 description = "A seamless bridge from model development to model delivery"
 license = "MIT"
 readme = "README.md"
@@ -50,7 +50,7 @@ psutil = ">=5.9.4"
 pydantic = ">=1.10.0"
 pytest-asyncio = "^0.23.6"
 # Note: when using chains, 3.9 will be required at runtime, but other truss functionality works with 3.8.
-python = ">=3.8,<3.13"
+python = ">=3.8,<3.12"
 python-json-logger = ">=2.0.2"
 python-on-whales = "^0.68.0"
 PyYAML = ">=6.0"

{truss-0.9.17 → truss-0.9.17.dev2}/truss/cli/cli.py

@@ -334,8 +334,8 @@ def _create_chains_table(service) -> Tuple[rich.table.Table, List[str]]:
     table.add_column("Logs URL")
     statuses = []
     # After reversing, the first one is the entrypoint (per order in service).
-    for i, chainlet in enumerate(reversed(service.get_info())):
-        displayable_status = get_displayable_status(chainlet.status)
+    for i, (name, status, logs_url) in enumerate(reversed(service.get_info())):
+        displayable_status = get_displayable_status(status)
         if displayable_status == ACTIVE_STATUS:
             spinner_name = "active"
         elif displayable_status in DEPLOYING_STATUSES:
@@ -348,13 +348,13 @@ def _create_chains_table(service) -> Tuple[rich.table.Table, List[str]]:
         else:
             spinner_name = "failed"
         spinner = rich.spinner.Spinner(spinner_name, text=displayable_status)
-        if chainlet.is_entrypoint:
-            display_name = f"{chainlet.name} (entrypoint)"
+        if i == 0:
+            display_name = f"{name} (entrypoint)"
         else:
-            display_name = f"{chainlet.name} (dep)"
+            display_name = f"{name} (dep)"
 
-        table.add_row(spinner, display_name, chainlet.logs_url)
-        if chainlet.is_entrypoint:  # Add section divider after entrypoint.
+        table.add_row(spinner, display_name, logs_url)
+        if i == 0:  # Add section divider after entrypoint.
             table.add_section()
         statuses.append(displayable_status)
     return table, statuses
@@ -480,46 +480,52 @@ def deploy(
 
 
 @chains.command(name="init")  # type: ignore
-@click.argument("directory", type=Path, required=False)
+@click.option(
+    "--target_directory",
+    type=str,
+    required=False,
+    help="Name of the chain to be deployed, if not given, the user will be prompted for a name",
+)
 @log_level_option
 @error_handling
 def chains_init(
-    directory: Optional[Path],
+    target_directory: Optional[str],
 ) -> None:
     """
-    Initializes a chains project directory.
-
-    DIRECTORY: A name of new or existing directory to create the chain in,
-      it must be empty. If not specified, the current directory is used.
-
+    Initializes a chains project with hello.py
     """
-    if not directory:
-        directory = Path.cwd()
-    if directory.exists():
-        if not directory.is_dir():
-            raise ValueError(f"The path {directory} must be a directory.")
-        if any(directory.iterdir()):
-            raise ValueError(f"Directory {directory} must be empty.")
-    else:
-        directory.mkdir()
-
-    filename = (
-        inquirer.text(
-            qmark="",
-            message="Enter the python file name for the chain.",
-            default="my_chain.py",
-        ).execute(),
-    )
-    filepath = directory / str(filename)
-    rich.print(f"Creating and populating {filepath}...\n")
+    FILENAME = "main.py"
+    if not target_directory:
+        target_directory = inquirer.text(
+            qmark="", message="Enter the target directory for the chain"
+        ).execute()
+        # Ensure that `None` is replaced to empty string. This will write a main.py
+        # file to the cwd.
+        if not target_directory:
+            target_directory = ""
+    # we do this cast to satisfy mypy when handling the output of the inquirer.text call
+    target_directory = str(target_directory)
+    cur_path = os.getcwd()
+    abs_path = os.path.join(cur_path, target_directory)
+    filename = os.path.join(abs_path, FILENAME)
+    if os.path.exists(filename):
+        raise click.UsageError(
+            f"Cannot init chains project with {filename}. Path already exists"
+        )
+    user_friendly_path = os.path.join(target_directory, FILENAME)
+    rich.print(f"Creating {user_friendly_path}...\n")
+
     source_code = _load_example_chainlet_code()
-    filepath.write_text(source_code)
+
+    if not os.path.exists(abs_path):
+        os.mkdir(abs_path)
+    with open(filename, "w") as f:
+        f.write(source_code)
+
     rich.print(
         "Next steps:\n",
-        f"💻 Run [bold green]`python {filepath}`[/bold green] for local debug "
-        "execution.\n"
-        f"🚢 Run [bold green]`truss chains deploy {filepath}`[/bold green] "
-        "to deploy the chain to Baseten.\n",
+        f"💻 Run `python {user_friendly_path}` to execute the code locally\n",
+        f"🚢 Run `truss chains deploy {user_friendly_path}` to deploy the chain to Baseten\n",
     )
 
 

{truss-0.9.17 → truss-0.9.17.dev2}/truss/config/trt_llm.py

@@ -13,7 +13,6 @@ class TrussTRTLLMModel(Enum):
     LLAMA: str = "llama"
     MISTRAL: str = "mistral"
     DEEPSEEK: str = "deepseek"
-    WHISPER: str = "whisper"
 
 
 class TrussTRTLLMQuantizationType(str, Enum):
@@ -37,8 +36,6 @@ class CheckpointSource(Enum):
     HF: str = "HF"
     GCS: str = "GCS"
     LOCAL: str = "LOCAL"
-    # REMOTE_URL is useful when the checkpoint lives on remote storage accessible via HTTP (e.g a presigned URL)
-    REMOTE_URL: str = "REMOTE_URL"
 
 
 class CheckpointRepository(BaseModel):
@@ -76,8 +73,8 @@ class TrussTRTLLMBuildConfiguration(BaseModel):
 
 
 class TrussTRTLLMServingConfiguration(BaseModel):
-    engine_repository: str
-    tokenizer_repository: str
+    engine_repository: Optional[str] = None
+    tokenizer_repository: Optional[str] = None
     tensor_parallel_count: int = 1
     pipeline_parallel_count: int = 1
 

{truss-0.9.17 → truss-0.9.17.dev2}/truss/constants.py

@@ -21,7 +21,6 @@ SERVER_CODE_DIR: pathlib.Path = TEMPLATES_DIR / "server"
 TRITON_SERVER_CODE_DIR: pathlib.Path = TEMPLATES_DIR / "triton"
 TRTLLM_BRITON_TRUSS_DIR: pathlib.Path = TEMPLATES_DIR / "trtllm-briton"
 TRTLLM_TRITON_TRUSS_DIR: pathlib.Path = TEMPLATES_DIR / "trtllm"
-AUDIO_MODEL_TRTLLM_TRUSS_DIR: pathlib.Path = TEMPLATES_DIR / "trtllm-audio"
 TRTLLM_TRUSS_DIR: pathlib.Path = (
     TRTLLM_BRITON_TRUSS_DIR if USE_BRITON else TRTLLM_TRITON_TRUSS_DIR
 )
@@ -119,26 +118,6 @@ BRITON_BASE_TRTLLM_REQUIREMENTS = [
     "grpcio==1.64.0",
     "grpcio-tools==1.64.0",
     "transformers==4.41.0",
-    "truss==0.9.17rc2",
-]
-AUDIO_MODEL_TRTLLM_REQUIREMENTS = [
-    "--extra-index-url https://pypi.nvidia.com",
-    "tensorrt_llm==0.9.0",
-    "hf_transfer",
-    "janus",
-    "kaldialign",
-    "librosa",
-    "mpi4py==3.1.4",
-    "safetensors",
-    "soundfile",
-    "tiktoken",
-    "torchaudio",
-    "async-batcher>=0.2.0",
-    "pydantic>=2.7.1",
-]
-AUDIO_MODEL_TRTLLM_SYSTEM_PACKAGES = [
-    "python3.10-venv",
-    "openmpi-bin",
-    "libopenmpi-dev",
+    "truss==0.9.12",
 ]
 OPENAI_COMPATIBLE_TAG = "openai-compatible"

{truss-0.9.17 → truss-0.9.17.dev2}/truss/contexts/image_builder/serving_image_builder.py

@@ -12,11 +12,7 @@ from botocore.client import Config
 from google.cloud import storage
 from huggingface_hub import get_hf_file_metadata, hf_hub_url, list_repo_files
 from huggingface_hub.utils import filter_repo_objects
-from truss.config.trt_llm import TrussTRTLLMModel
 from truss.constants import (
-    AUDIO_MODEL_TRTLLM_REQUIREMENTS,
-    AUDIO_MODEL_TRTLLM_SYSTEM_PACKAGES,
-    AUDIO_MODEL_TRTLLM_TRUSS_DIR,
     BASE_SERVER_REQUIREMENTS_TXT_FILENAME,
     BASE_TRTLLM_REQUIREMENTS,
     BRITON_BASE_TRTLLM_REQUIREMENTS,
@@ -345,18 +341,7 @@ class ServingImageBuilder(ImageBuilder):
         # Most of the code is pulled from upstream triton-inference-server tensorrtllm_backend
         # https://github.com/triton-inference-server/tensorrtllm_backend/tree/v0.9.0/all_models/inflight_batcher_llm
         if config.trt_llm is not None:
-            is_audio_model = (
-                config.trt_llm.build.base_model == TrussTRTLLMModel.WHISPER
-                if config.trt_llm.build is not None
-                else False
-            )
-
-            if is_audio_model:
-                copy_tree_path(
-                    AUDIO_MODEL_TRTLLM_TRUSS_DIR, build_dir, ignore_patterns=[]
-                )
-            else:
-                copy_tree_path(TRTLLM_TRUSS_DIR, build_dir, ignore_patterns=[])
+            copy_tree_path(TRTLLM_TRUSS_DIR, build_dir, ignore_patterns=[])
 
             tensor_parallel_count = (
                 config.trt_llm.build.tensor_parallel_count  # type: ignore[union-attr]
@@ -371,23 +356,17 @@ class ServingImageBuilder(ImageBuilder):
 
             config.runtime.predict_concurrency = TRTLLM_PREDICT_CONCURRENCY
 
-            if not is_audio_model:
-                config.base_image = BaseImage(
-                    image=BRITON_TRTLLM_BASE_IMAGE if USE_BRITON else TRTLLM_BASE_IMAGE,
-                    python_executable_path=TRTLLM_PYTHON_EXECUTABLE,
-                )
-
-                config.requirements.extend(
-                    BRITON_BASE_TRTLLM_REQUIREMENTS
-                    if USE_BRITON
-                    else BASE_TRTLLM_REQUIREMENTS
-                )
+            config.base_image = BaseImage(
+                image=BRITON_TRTLLM_BASE_IMAGE if USE_BRITON else TRTLLM_BASE_IMAGE,
+                python_executable_path=TRTLLM_PYTHON_EXECUTABLE,
+            )
+            config.requirements.extend(
+                BRITON_BASE_TRTLLM_REQUIREMENTS
+                if USE_BRITON
+                else BASE_TRTLLM_REQUIREMENTS
+            )
 
-                config.model_metadata["tags"] = [OPENAI_COMPATIBLE_TAG]
-            else:
-                config.requirements.extend(AUDIO_MODEL_TRTLLM_REQUIREMENTS)
-                config.system_packages.extend(AUDIO_MODEL_TRTLLM_SYSTEM_PACKAGES)
-                config.python_version = "py310"
+            config.model_metadata["tags"] = [OPENAI_COMPATIBLE_TAG]
 
         # Override config.yml
         with (build_dir / CONFIG_FILE).open("w") as config_file:

{truss-0.9.17 → truss-0.9.17.dev2}/truss/remote/baseten/api.py

@@ -275,7 +275,6 @@ class BasetenApi:
                 chainlets{{
                     name
                     id
-                    is_entrypoint
                     oracle_version {{
                         current_model_deployment_status {{
                             status

{truss-0.9.17 → truss-0.9.17.dev2}/truss/remote/baseten/types.py

@@ -3,13 +3,6 @@ from enum import Enum
 import pydantic
 
 
-class DeployedChainlet(pydantic.BaseModel):
-    name: str
-    is_entrypoint: bool
-    status: str
-    logs_url: str
-
-
 class ChainletData(pydantic.BaseModel):
     name: str
     oracle_version_id: str

{truss-0.9.17 → truss-0.9.17.dev2}/truss/remote/remote_factory.py

@@ -1,12 +1,5 @@
 import inspect
-
-try:
-    from configparser import DEFAULTSECT
-    from configparser import SafeConfigParser as ConfigParser
-except ImportError:
-    # We need to do this for py312 and onwards.
-    from configparser import DEFAULTSECT, ConfigParser  # type: ignore
-
+from configparser import DEFAULTSECT, SafeConfigParser
 from functools import partial
 from operator import is_not
 from pathlib import Path
@@ -18,13 +11,13 @@ from truss.remote.truss_remote import RemoteConfig, TrussRemote
 USER_TRUSSRC_PATH = Path("~/.trussrc").expanduser()
 
 
-def load_config() -> ConfigParser:
-    config = ConfigParser()
+def load_config() -> SafeConfigParser:
+    config = SafeConfigParser()
     config.read(USER_TRUSSRC_PATH)
     return config
 
 
-def update_config(config: ConfigParser):
+def update_config(config: SafeConfigParser):
     with open(USER_TRUSSRC_PATH, "w") as configfile:
         config.write(configfile)
 

{truss-0.9.17 → truss-0.9.17.dev2}/truss/templates/control/control/endpoints.py

@@ -51,9 +51,6 @@ async def proxy(request: Request):
             retry_if_exception_type(ConnectError)
             | retry_if_exception_type(ModelNotReady)
             | retry_if_exception_type(RemoteProtocolError)
-            | retry_if_exception_type(httpx.ReadError)
-            | retry_if_exception_type(httpx.ReadTimeout)
-            | retry_if_exception_type(httpx.ConnectTimeout)
         ),
         stop=stop_after_attempt(INFERENCE_SERVER_START_WAIT_SECS),
         wait=wait_fixed(1),

{truss-0.9.17 → truss-0.9.17.dev2}/truss/templates/control/control/helpers/truss_patch/requirement_name_identifier.py

@@ -1,12 +1,12 @@
 from typing import Dict, List
 
-from packaging import requirements  # type: ignore
+import pkg_resources
+from pkg_resources.extern.packaging import requirements  # type: ignore
 
 
 def identify_requirement_name(req: str) -> str:
     try:
-
-        parsed_req = requirements.Requirement(req)
+        parsed_req = pkg_resources.Requirement.parse(req)
         return parsed_req.name  # type: ignore
     except (requirements.InvalidRequirement, ValueError):
         # default to the whole line if we can't parse it.

{truss-0.9.17 → truss-0.9.17.dev2}/truss/templates/trtllm-briton/model/model.py

@@ -10,7 +10,10 @@ import briton_pb2
 import briton_pb2_grpc
 import grpc
 from transformers import AutoTokenizer
-from truss.config.trt_llm import TrussTRTLLMBuildConfiguration
+from truss.config.trt_llm import (
+    TrussTRTLLMBuildConfiguration,
+    TrussTRTLLMServingConfiguration,
+)
 from truss.constants import OPENAI_COMPATIBLE_TAG
 
 BRITON_PORT = 50051

{truss-0.9.17 → truss-0.9.17.dev2}/truss/util/path.py

@@ -1,9 +1,10 @@
 import os
 import random
-import shutil
 import string
 import tempfile
 from contextlib import contextmanager
+from distutils.dir_util import remove_tree
+from distutils.file_util import copy_file
 from pathlib import Path
 from typing import Iterable, Iterator, List, Optional, Set, Tuple, Union
 
@@ -32,11 +33,11 @@ def copy_tree_path(src: Path, dest: Path, ignore_patterns: List[str] = []) -> No
             dest_fp.mkdir(exist_ok=True)
         else:
             dest_fp.parent.mkdir(exist_ok=True, parents=True)
-            shutil.copy2(str(sub_path), str(dest_fp))
+            copy_file(str(sub_path), str(dest_fp), verbose=False)
 
 
 def copy_file_path(src: Path, dest: Path) -> Tuple[str, str]:
-    return shutil.copy2(str(src), str(dest))
+    return copy_file(str(src), str(dest), verbose=False)
 
 
 def copy_tree_or_file(src: Path, dest: Path) -> Union[List[str], Tuple[str, str]]:
@@ -47,7 +48,7 @@ def copy_tree_or_file(src: Path, dest: Path) -> Union[List[str], Tuple[str, str]
 
 
 def remove_tree_path(target: Path) -> None:
-    return shutil.rmtree(str(target))
+    return remove_tree(str(target), verbose=0)
 
 
 def get_max_modified_time_of_dir(path: Path) -> float:

{truss-0.9.17 → truss-0.9.17.dev2}/truss-chains/truss_chains/code_gen.py

@@ -659,7 +659,6 @@ def _make_truss_config(
             "to secrets - no need to manually add it."
         )
     config.model_cache.models = assets.cached
-    config.external_data = truss_config.ExternalData(items=assets.external_data)
     # Metadata.
     chains_metadata: definitions.TrussMetadata = definitions.TrussMetadata(
         user_config=user_config, chainlet_to_service=chainlet_to_service

{truss-0.9.17 → truss-0.9.17.dev2}/truss-chains/truss_chains/definitions.py

@@ -245,7 +245,6 @@ class AssetSpec(SafeModel):
     # TODO: this is not stable yet and might change or refer back to truss.
     secrets: dict[str, str] = pydantic.Field({})
     cached: list[truss_config.ModelRepo] = []
-    external_data: list[truss_config.ExternalDataItem] = []
 
 
 class Assets:
@@ -274,7 +273,6 @@ class Assets:
         self,
         cached: Iterable[truss_config.ModelRepo] = (),
         secret_keys: Iterable[str] = (),
-        external_data: Iterable[truss_config.ExternalDataItem] = (),
     ) -> None:
         """
         Args:
@@ -282,15 +280,9 @@ class Assets:
             secret_keys: Names of secrets stored on baseten, that the
               chainlet should have access to. You can manage secrets on baseten
               `here <https://app.baseten.co/settings/secrets>`_.
-            external_data: Data to be downloaded from public URLs and made available
-              in the deployment (via ``context.data_dir``). See
-              `here <https://docs.baseten.co/reference/config#external-data>` for
-              more details.
         """
         self._spec = AssetSpec(
-            cached=list(cached),
-            secrets={k: SECRET_DUMMY for k in secret_keys},
-            external_data=list(external_data),
+            cached=list(cached), secrets={k: SECRET_DUMMY for k in secret_keys}
         )
 
     def get_spec(self) -> AssetSpec:

{truss-0.9.17 → truss-0.9.17.dev2}/truss-chains/truss_chains/deploy.py

@@ -17,7 +17,6 @@ from typing import (
 )
 
 import truss
-from tenacity import retry, stop_after_delay, wait_fixed
 from truss.remote.baseten import remote as b10_remote
 from truss.remote.baseten import service as b10_service
 from truss.remote.baseten import types as b10_types
@@ -171,20 +170,17 @@ class RemoteChainService:
         self._chain_id = chain_id
         self._chain_deployment_id = chain_deployment_id
 
-    def get_info(self) -> list[b10_types.DeployedChainlet]:
+    def get_info(self) -> list[tuple[str, str, str]]:
         """Return list with elements (name, status, logs_url) for each chainlet."""
         chainlets = self._remote.get_chainlets(
             chain_deployment_id=self._chain_deployment_id
         )
 
         return [
-            b10_types.DeployedChainlet(
-                name=chainlet["name"],
-                is_entrypoint=chainlet["is_entrypoint"],
-                status=chainlet["oracle_version"]["current_model_deployment_status"][
-                    "status"
-                ],
-                logs_url=_chainlet_logs_url(
+            (
+                chainlet["name"],
+                chainlet["oracle_version"]["current_model_deployment_status"]["status"],
+                _chainlet_logs_url(
                     self._chain_id,
                     self._chain_deployment_id,
                     chainlet["id"],
@@ -289,8 +285,7 @@ class ChainService:
             The JSON response."""
         return self.get_entrypoint.predict(json)
 
-    @retry(stop=stop_after_delay(60), wait=wait_fixed(1), reraise=True)
-    def get_info(self) -> list[b10_types.DeployedChainlet]:
+    def get_info(self) -> list[tuple[str, str, str]]:
         """Queries the statuses of all chainlets in the chain.
 
         Returns:

{truss-0.9.17 → truss-0.9.17.dev2}/truss-chains/truss_chains/framework.py

@@ -187,7 +187,7 @@ def _validate_endpoint_params(
             raise definitions.ChainsUsageError(
                 "Inputs of endpoints must have type annotations. For "
                 f"`{cls_name}.{definitions.ENDPOINT_METHOD_NAME}` parameter "
-                f"`{param.name}` has no type annotation."
+                f"`{param.name}` has not type annotation."
             )
         _validate_io_type(param)
         type_descriptor = definitions.TypeDescriptor(raw=param.annotation)

truss-0.9.17/truss/templates/trtllm-audio/model/model.py

@@ -1,44 +0,0 @@
-import base64
-
-import requests
-from whisper_trt import WhisperModel
-from whisper_trt.types import WhisperResult
-
-
-class Model:
-    def __init__(self, **kwargs):
-        self._data_dir = kwargs["data_dir"]
-        self._secrets = kwargs["secrets"]
-        self._model = None
-
-    def load(self):
-        self._model = WhisperModel(str(self._data_dir), max_queue_time=0.050)
-
-    def preprocess(self, request: dict):
-        audio_base64 = request.get("audio")
-        audio_url = request.get("url")
-
-        if audio_base64 and audio_url:
-            return {
-                "error": "Only a base64 audio file OR a URL can be passed to the API, not both of them.",
-            }
-        if not audio_base64 and not audio_url:
-            return {
-                "error": "Please provide either an audio file in base64 string format or a URL to an audio file.",
-            }
-
-        binary_data = None
-
-        if audio_base64:
-            binary_data = base64.b64decode(audio_base64.encode("utf-8"))
-        elif audio_url:
-            resp = requests.get(audio_url)
-            binary_data = resp.content
-        return binary_data, request
-
-    async def predict(self, preprocessed_request) -> WhisperResult:
-        binary_data, request = preprocessed_request
-        waveform = self._model.preprocess_audio(binary_data)
-        return await self._model.transcribe(
-            waveform, language="english", timestamps=True
-        )