truss 0.10.0rc1__py3-none-any.whl → 0.60.0__py3-none-any.whl

truss/templates/base.Dockerfile.jinja

@@ -1,27 +1,27 @@
 ARG PYVERSION={{config.python_version}}
-FROM {{base_image_name_and_tag}} as truss_server
+FROM {{base_image_name_and_tag}} AS truss_server
 
-ENV PYTHON_EXECUTABLE {{ config.base_image.python_executable_path or 'python3' }}
-ENV JSON_LOG True
+ENV PYTHON_EXECUTABLE="{{ config.base_image.python_executable_path or 'python3' }}"
 
 {% block fail_fast %}
 RUN grep -w 'ID=debian\|ID_LIKE=debian' /etc/os-release || { echo "ERROR: Supplied base image is not a debian image"; exit 1; }
-RUN $PYTHON_EXECUTABLE -c "import sys; sys.exit(0) if sys.version_info.major == 3 and sys.version_info.minor >=8 and sys.version_info.minor <=11 else sys.exit(1)" \
-    || { echo "ERROR: Supplied base image does not have 3.8 <= python <= 3.11"; exit 1; }
+RUN $PYTHON_EXECUTABLE -c "import sys; sys.exit(0) if sys.version_info.major == {{supported_python_major_version_in_custom_base_image}} and sys.version_info.minor >={{min_supported_python_minor_version_in_custom_base_image}} and sys.version_info.minor <={{max_supported_python_minor_version_in_custom_base_image}} else sys.exit(1)" \
+    || { echo "ERROR: Supplied base image does not have {{min_supported_python_version_in_custom_base_image}} <= python <= {{max_supported_python_version_in_custom_base_image}}"; exit 1; }
 {% endblock %}
 
 RUN pip install --upgrade pip --no-cache-dir \
     && rm -rf /root/.cache/pip
 
-# Always install the truss package
-COPY ./truss/ /lib/truss_pkg/truss
-COPY ./pyproject.toml /lib/truss_pkg/
-COPY ./README.md /lib/truss_pkg/
-RUN pip install /lib/truss_pkg --no-cache-dir && rm -rf /root/.cache/pip
-
 {% block base_image_patch %}
 {% endblock %}
 
+{% if config.model_framework.value == 'huggingface_transformer' %}
+    {% if config.resources.use_gpu %}
+# HuggingFace pytorch gpu support needs mkl
+RUN pip install mkl
+    {% endif %}
+{% endif%}
+
 {% block post_base %}
 {% endblock %}
 
@@ -39,20 +39,22 @@ RUN apt-get update && apt-get install --yes --no-install-recommends $(cat {{syst
 {% block install_requirements %}
     {%- if should_install_user_requirements_file %}
 COPY ./{{user_supplied_requirements_filename}} {{user_supplied_requirements_filename}}
+RUN cat {{user_supplied_requirements_filename}}
 RUN pip install -r {{user_supplied_requirements_filename}} --no-cache-dir && rm -rf /root/.cache/pip
     {%- endif %}
     {%- if should_install_requirements %}
 COPY ./{{config_requirements_filename}} {{config_requirements_filename}}
+RUN cat {{config_requirements_filename}}
 RUN pip install -r {{config_requirements_filename}} --no-cache-dir && rm -rf /root/.cache/pip
     {%- endif %}
 {% endblock %}
 
 
 
-
-ENV APP_HOME /app
+{%- if not config.docker_server %}
+ENV APP_HOME="/app"
 WORKDIR $APP_HOME
-
+{%- endif %}
 
 {% block app_copy %}
 {% endblock %}
@@ -60,13 +62,13 @@ WORKDIR $APP_HOME
 
 {% block bundled_packages_copy %}
     {%- if bundled_packages_dir_exists %}
-COPY ./{{config.bundled_packages_dir}} /app/packages
+COPY ./{{config.bundled_packages_dir}} /packages
     {%- endif %}
 {% endblock %}
 
 
 {% for env_var_name, env_var_value in config.environment_variables.items() %}
-ENV {{ env_var_name }} {{ env_var_value }}
+ENV {{ env_var_name }}="{{ env_var_value }}"
 {% endfor %}
 
 {% block run %}

truss/templates/cache.Dockerfile.jinja

@@ -1,14 +1,14 @@
-FROM python:3.11-slim as cache_warmer
+FROM python:3.11-slim AS cache_warmer
 
 RUN mkdir -p /app/model_cache
 WORKDIR /app
 
 {% if hf_access_token %}
-ENV HUGGING_FACE_HUB_TOKEN {{hf_access_token}}
+ENV HUGGING_FACE_HUB_TOKEN="{{hf_access_token}}"
 {% endif %}
 
 RUN apt-get -y update; apt-get -y install curl; curl -s https://baseten-public.s3.us-west-2.amazonaws.com/bin/b10cp-5fe8dc7da-linux-amd64 -o /app/b10cp; chmod +x /app/b10cp
-ENV B10CP_PATH_TRUSS /app/b10cp
+ENV B10CP_PATH_TRUSS="/app/b10cp"
 COPY ./cache_requirements.txt /app/cache_requirements.txt
 RUN pip install -r /app/cache_requirements.txt --no-cache-dir && rm -rf /root/.cache/pip
 COPY ./cache_warmer.py /cache_warmer.py

truss/{server → templates/control}/control/application.py

@@ -1,50 +1,31 @@
 import asyncio
 import logging
+import logging.config
 import re
 from pathlib import Path
 from typing import Dict
 
 import httpx
+from endpoints import control_app
 from fastapi import FastAPI
 from fastapi.responses import JSONResponse
+from helpers.errors import ModelLoadFailed, PatchApplicatonError
+from helpers.inference_server_controller import InferenceServerController
+from helpers.inference_server_process_controller import InferenceServerProcessController
+from helpers.inference_server_starter import async_inference_server_startup_flow
+from helpers.truss_patch.model_container_patch_applier import ModelContainerPatchApplier
+from shared import log_config
 from starlette.datastructures import State
-from truss.server.control.endpoints import control_app
-from truss.server.control.errors import ModelLoadFailed, PatchApplicatonError
-from truss.server.control.helpers.inference_server_controller import (
-    InferenceServerController,
-)
-from truss.server.control.helpers.inference_server_process_controller import (
-    InferenceServerProcessController,
-)
-from truss.server.control.helpers.inference_server_starter import (
-    async_inference_server_startup_flow,
-)
-from truss.server.control.patch.model_container_patch_applier import (
-    ModelContainerPatchApplier,
-)
-from truss.server.shared.logging import setup_logging
 
 
 async def handle_patch_error(_, exc):
     error_type = _camel_to_snake_case(type(exc).__name__)
-    return JSONResponse(
-        content={
-            "error": {
-                "type": error_type,
-                "msg": str(exc),
-            }
-        }
-    )
+    return JSONResponse(content={"error": {"type": error_type, "msg": str(exc)}})
 
 
 async def generic_error_handler(_, exc):
     return JSONResponse(
-        content={
-            "error": {
-                "type": "unknown",
-                "msg": f"{type(exc)}: {exc}",
-            }
-        }
+        content={"error": {"type": "unknown", "msg": f"{type(exc)}: {exc}"}}
     )
 
 
@@ -55,10 +36,11 @@ async def handle_model_load_failed(_, error):
 
 def create_app(base_config: Dict):
     app_state = State()
-    setup_logging()
     app_logger = logging.getLogger(__name__)
     app_state.logger = app_logger
 
+    logging.config.dictConfig(log_config.make_log_config("INFO"))
+
     for k, v in base_config.items():
         setattr(app_state, k, v)
 
@@ -77,9 +59,7 @@ def create_app(base_config: Dict):
     pip_path = getattr(app_state, "pip_path", None)
 
     patch_applier = ModelContainerPatchApplier(
-        Path(app_state.inference_server_home),
-        app_logger,
-        pip_path,
+        Path(app_state.inference_server_home), app_logger, pip_path
     )
 
     oversee_inference_server = getattr(app_state, "oversee_inference_server", True)
@@ -94,8 +74,7 @@ def create_app(base_config: Dict):
     async def start_background_inference_startup():
         asyncio.create_task(
             async_inference_server_startup_flow(
-                app_state.inference_server_controller,
-                app_logger,
+                app_state.inference_server_controller, app_logger
             )
         )
 

truss/{server → templates/control}/control/endpoints.py

@@ -4,15 +4,14 @@ from typing import Any, Dict
 import httpx
 from fastapi import APIRouter
 from fastapi.responses import JSONResponse, StreamingResponse
+from helpers.errors import ModelLoadFailed, ModelNotReady
 from httpx import URL, ConnectError, RemoteProtocolError
 from starlette.requests import ClientDisconnect, Request
 from starlette.responses import Response
-from tenacity import Retrying, retry_if_exception_type, stop_after_attempt, wait_fixed
-from truss.server.control.errors import ModelLoadFailed, ModelNotReady
+from tenacity import RetryCallState, Retrying, retry_if_exception_type, wait_fixed
 
 INFERENCE_SERVER_START_WAIT_SECS = 60
 
-
 control_app = APIRouter()
 
 
@@ -26,7 +25,9 @@ async def proxy(request: Request):
         request.app.state.inference_server_process_controller
     )
     client: httpx.AsyncClient = request.app.state.proxy_client
-    url = URL(path=request.url.path, query=request.url.query.encode("utf-8"))
+
+    path = _reroute_if_health_check(request.url.path)
+    url = URL(path=path, query=request.url.query.encode("utf-8"))
 
     # 2 min connect timeouts, no timeout for requests.
     # We don't want requests to fail due to timeout on the proxy
@@ -51,15 +52,16 @@ async def proxy(request: Request):
             retry_if_exception_type(ConnectError)
             | retry_if_exception_type(ModelNotReady)
             | retry_if_exception_type(RemoteProtocolError)
+            | retry_if_exception_type(httpx.ReadError)
+            | retry_if_exception_type(httpx.ReadTimeout)
+            | retry_if_exception_type(httpx.ConnectTimeout)
         ),
-        stop=stop_after_attempt(INFERENCE_SERVER_START_WAIT_SECS),
+        stop=_custom_stop_strategy,
         wait=wait_fixed(1),
     ):
         with attempt:
             try:
-                if (
-                    inference_server_process_controller.is_inference_server_intentionally_stopped()
-                ):
+                if inference_server_process_controller.is_inference_server_intentionally_stopped():
                     raise ModelLoadFailed("Model load failed")
                 resp = await client.send(inf_serv_req, stream=True)
 
@@ -74,8 +76,10 @@ async def proxy(request: Request):
                     inference_server_process_controller.inference_server_ever_started()
                     and not inference_server_process_controller.is_inference_server_running()
                 ):
-                    error_msg = "It appears your model has stopped running. This often means' \
+                    error_msg = (
+                        "It appears your model has stopped running. This often means' \
                         ' it crashed and may need a fix to get it running again."
+                    )
                     return JSONResponse(error_msg, 503)
                 raise exp
 
@@ -148,3 +152,29 @@ def _is_streaming_response(resp) -> bool:
         if header_name.lower() == "transfer-encoding" and value.lower() == "chunked":
             return True
     return False
+
+
+def _reroute_if_health_check(path: str) -> str:
+    """
+    Reroutes calls from the Operator to the inference server's health check endpoint (/v1/models/model) to /v1/models/model/loaded instead.
+    This is done to avoid running custom health checks when the Operator is checking if the inference server is ready.
+    """
+    if path == "/v1/models/model":
+        path = "/v1/models/model/loaded"
+    return path
+
+
+def _custom_stop_strategy(retry_state: RetryCallState) -> bool:
+    # Stop after 10 attempts for ModelNotReady
+    if retry_state.outcome is not None and isinstance(
+        retry_state.outcome.exception(), ModelNotReady
+    ):
+        # Check if the retry limit for ModelNotReady has been reached
+        return retry_state.attempt_number >= 10
+    # For all other exceptions, stop after INFERENCE_SERVER_START_WAIT_SECS
+    seconds_since_start = (
+        retry_state.seconds_since_start
+        if retry_state.seconds_since_start is not None
+        else 0.0
+    )
+    return seconds_since_start >= INFERENCE_SERVER_START_WAIT_SECS

truss/{server/control/patch/types.py → templates/control/control/helpers/custom_types.py}

@@ -42,11 +42,7 @@ class ModelCodePatch(PatchBody):
     content: Optional[str] = None
 
     def to_dict(self):
-        return {
-            "action": self.action.value,
-            "path": self.path,
-            "content": self.content,
-        }
+        return {"action": self.action.value, "path": self.path, "content": self.content}
 
     @staticmethod
     def from_dict(patch_dict: Dict):
@@ -65,17 +61,13 @@ class PythonRequirementPatch(PatchBody):
     requirement: str
 
     def to_dict(self):
-        return {
-            "action": self.action.value,
-            "requirement": self.requirement,
-        }
+        return {"action": self.action.value, "requirement": self.requirement}
 
     @staticmethod
     def from_dict(patch_dict: Dict):
         action_str = patch_dict["action"]
         return PythonRequirementPatch(
-            action=Action[action_str],
-            requirement=patch_dict["requirement"],
+            action=Action[action_str], requirement=patch_dict["requirement"]
         )
 
 
@@ -86,17 +78,13 @@ class SystemPackagePatch(PatchBody):
     package: str
 
     def to_dict(self):
-        return {
-            "action": self.action.value,
-            "package": self.package,
-        }
+        return {"action": self.action.value, "package": self.package}
 
     @staticmethod
     def from_dict(patch_dict: Dict):
         action_str = patch_dict["action"]
         return SystemPackagePatch(
-            action=Action[action_str],
-            package=patch_dict["package"],
+            action=Action[action_str], package=patch_dict["package"]
         )
 
 
@@ -106,11 +94,7 @@ class ConfigPatch(PatchBody):
     path: str = "config.yaml"
 
     def to_dict(self):
-        return {
-            "action": self.action.value,
-            "config": self.config,
-            "path": self.path,
-        }
+        return {"action": self.action.value, "config": self.config, "path": self.path}
 
     @staticmethod
     def from_dict(patch_dict: Dict):
@@ -128,11 +112,7 @@ class DataPatch(PatchBody):
     content: Optional[str] = None
 
     def to_dict(self):
-        return {
-            "action": self.action.value,
-            "content": self.content,
-            "path": self.path,
-        }
+        return {"action": self.action.value, "content": self.content, "path": self.path}
 
     @staticmethod
     def from_dict(patch_dict: Dict):
@@ -150,11 +130,7 @@ class PackagePatch(PatchBody):
     content: Optional[str] = None
 
     def to_dict(self):
-        return {
-            "action": self.action.value,
-            "content": self.content,
-            "path": self.path,
-        }
+        return {"action": self.action.value, "content": self.content, "path": self.path}
 
     @staticmethod
     def from_dict(patch_dict: Dict):
@@ -171,18 +147,12 @@ class EnvVarPatch(PatchBody):
     item: dict
 
     def to_dict(self):
-        return {
-            "action": self.action.value,
-            "item": self.item,
-        }
+        return {"action": self.action.value, "item": self.item}
 
     @staticmethod
     def from_dict(patch_dict: Dict):
         action_str = patch_dict["action"]
-        return EnvVarPatch(
-            action=Action[action_str],
-            item=patch_dict["item"],
-        )
+        return EnvVarPatch(action=Action[action_str], item=patch_dict["item"])
 
 
 @dataclass
@@ -190,18 +160,12 @@ class ExternalDataPatch(PatchBody):
     item: Dict[str, str]
 
     def to_dict(self):
-        return {
-            "action": self.action.value,
-            "item": self.item,
-        }
+        return {"action": self.action.value, "item": self.item}
 
     @staticmethod
     def from_dict(patch_dict: Dict):
         action_str = patch_dict["action"]
-        return ExternalDataPatch(
-            action=Action[action_str],
-            item=patch_dict["item"],
-        )
+        return ExternalDataPatch(action=Action[action_str], item=patch_dict["item"])
 
 
 PATCH_BODY_BY_TYPE: Dict[
@@ -238,10 +202,7 @@ class Patch:
     body: PatchBody
 
     def to_dict(self):
-        return {
-            "type": self.type.value,
-            "body": self.body.to_dict(),
-        }
+        return {"type": self.type.value, "body": self.body.to_dict()}
 
     @staticmethod
     def from_dict(patch_dict: Dict):

truss/{server → templates/control}/control/helpers/inference_server_controller.py

@@ -4,19 +4,15 @@ import threading
 import time
 from typing import Optional
 
-from truss.server.control.errors import (
+from helpers.custom_types import Patch, PatchType
+from helpers.errors import (
     InadmissiblePatch,
     PatchFailedRecoverable,
     PatchFailedUnrecoverable,
     UnsupportedPatch,
 )
-from truss.server.control.helpers.inference_server_process_controller import (
-    InferenceServerProcessController,
-)
-from truss.server.control.patch.model_container_patch_applier import (
-    ModelContainerPatchApplier,
-)
-from truss.server.control.patch.types import Patch, PatchType
+from helpers.inference_server_process_controller import InferenceServerProcessController
+from helpers.truss_patch.model_container_patch_applier import ModelContainerPatchApplier
 
 INFERENCE_SERVER_CHECK_INTERVAL_SECS = 10
 

truss/{server → templates/control}/control/helpers/inference_server_process_controller.py

@@ -4,7 +4,7 @@ import time
 from pathlib import Path
 from typing import List, Optional
 
-from truss.server.control.helpers.context_managers import current_directory
+from helpers.context_managers import current_directory
 
 INFERENCE_SERVER_FAILED_FILE = Path("~/inference_server_crashed.txt").expanduser()
 TERMINATION_TIMEOUT_SECS = 120.0
@@ -39,8 +39,7 @@ class InferenceServerProcessController:
         with current_directory(self._inference_server_home):
             inf_env["INFERENCE_SERVER_PORT"] = str(self._inference_server_port)
             self._inference_server_process = subprocess.Popen(
-                self._inference_server_process_args,
-                env=inf_env,
+                self._inference_server_process_args, env=inf_env
             )
 
             self._inference_server_started = True
@@ -49,7 +48,6 @@ class InferenceServerProcessController:
 
     def stop(self):
         if self._inference_server_process is not None:
-
             self._inference_server_process.terminate()
             self._inference_server_process.wait()
             # Introduce delay to avoid failing to grab the port

truss/{server → templates/control}/control/helpers/inference_server_starter.py

@@ -3,15 +3,12 @@ from logging import Logger
 
 import requests
 from anyio import to_thread
+from helpers.inference_server_controller import InferenceServerController
 from tenacity import Retrying, stop_after_attempt, wait_exponential
-from truss.server.control.helpers.inference_server_controller import (
-    InferenceServerController,
-)
 
 
 def inference_server_startup_flow(
-    inference_server_controller: InferenceServerController,
-    logger: Logger,
+    inference_server_controller: InferenceServerController, logger: Logger
 ) -> None:
     """
     Perform the inference server startup flow
@@ -41,8 +38,7 @@ def inference_server_startup_flow(
     payload = {"truss_hash": truss_hash}
 
     for attempt in Retrying(
-        stop=stop_after_attempt(15),
-        wait=wait_exponential(multiplier=2, min=1, max=4),
+        stop=stop_after_attempt(15), wait=wait_exponential(multiplier=2, min=1, max=4)
     ):
         with attempt:
             try:
@@ -58,14 +54,13 @@ def inference_server_startup_flow(
                 if "is_current" in resp_body and resp_body["is_current"] is True:
                     logger.info("Hash is current, starting inference server")
                     inference_server_controller.start()
-            except Exception as exc:
+            except Exception as exc:  # noqa
                 logger.warning(f"Patch ping attempt failed with error {exc}")
                 raise exc
 
 
 async def async_inference_server_startup_flow(
-    inference_server_controller: InferenceServerController,
-    logger: Logger,
+    inference_server_controller: InferenceServerController, logger: Logger
 ) -> None:
     return await to_thread.run_sync(
         inference_server_startup_flow, inference_server_controller, logger

truss/{server/control → templates/control/control/helpers}/truss_patch/model_code_patch_applier.py

@@ -2,14 +2,16 @@ import logging
 import os
 from pathlib import Path
 
-from truss.server.control.patch.types import Action, ModelCodePatch
+# TODO(marius/TaT): remove try-except after TaT.
+# TODO(pankaj) In desparate need of refactoring into separate library
+try:
+    from helpers.custom_types import Action, Patch
+except ModuleNotFoundError as exc:
+    logging.debug(f"Importing helpers from truss core, caused by: {exc}")
+    from truss.templates.control.control.helpers.custom_types import Action, Patch
 
 
-def apply_code_patch(
-    relative_dir: Path,
-    patch: ModelCodePatch,
-    logger: logging.Logger,
-):
+def apply_code_patch(relative_dir: Path, patch: Patch, logger: logging.Logger):
     logger.debug(f"Applying code patch {patch.to_dict()}")
     filepath: Path = relative_dir / patch.path
     action = patch.action

truss/{server/control/patch → templates/control/control/helpers/truss_patch}/model_container_patch_applier.py

@@ -3,9 +3,7 @@ import subprocess
 from pathlib import Path
 from typing import Optional
 
-from truss.server.control.errors import UnsupportedPatch
-from truss.server.control.patch.model_code_patch_applier import apply_code_patch
-from truss.server.control.patch.types import (
+from helpers.custom_types import (
     Action,
     ConfigPatch,
     EnvVarPatch,
@@ -16,7 +14,19 @@ from truss.server.control.patch.types import (
     PythonRequirementPatch,
     SystemPackagePatch,
 )
-from truss.truss_config import ExternalData, ExternalDataItem, TrussConfig
+from helpers.errors import UnsupportedPatch
+from helpers.truss_patch.model_code_patch_applier import apply_code_patch
+
+# TODO(marius/TaT): remove try-except after TaT.
+try:
+    from truss.base.truss_config import ExternalData, ExternalDataItem, TrussConfig
+except ImportError:
+    from truss.truss_config import (  # type: ignore[no-redef]
+        ExternalData,
+        ExternalDataItem,
+        TrussConfig,
+    )
+
 from truss.util.download import download_external_data
 
 
@@ -36,7 +46,7 @@ class ModelContainerPatchApplier:
             self._inference_server_home / self._truss_config.model_module_dir
         )
         self._bundled_packages_dir = (
-            self._inference_server_home / self._truss_config.bundled_packages_dir
+            self._inference_server_home / ".." / self._truss_config.bundled_packages_dir
         ).resolve()
         self._data_dir = self._inference_server_home / self._truss_config.data_dir
         self._app_logger = app_logger
@@ -121,30 +131,12 @@ class ModelContainerPatchApplier:
 
         if action == Action.REMOVE:
             subprocess.run(
-                [
-                    "apt",
-                    "remove",
-                    "-y",
-                    system_package_patch.package,
-                ],
-                check=True,
+                ["apt", "remove", "-y", system_package_patch.package], check=True
             )
         elif action in [Action.ADD, Action.UPDATE]:
+            subprocess.run(["apt", "update"], check=True)
             subprocess.run(
-                [
-                    "apt",
-                    "update",
-                ],
-                check=True,
-            )
-            subprocess.run(
-                [
-                    "apt",
-                    "install",
-                    "-y",
-                    system_package_patch.package,
-                ],
-                check=True,
+                ["apt", "install", "-y", system_package_patch.package], check=True
             )
         else:
             raise ValueError(f"Unknown python requirement patch action {action}")