truss 0.11.8rc8__py3-none-any.whl → 0.11.8rc10__py3-none-any.whl

truss/templates/control/control/application.py

@@ -2,71 +2,36 @@ import asyncio
 import logging
 import logging.config
 import re
-import traceback
 from pathlib import Path
-from typing import Awaitable, Callable, Dict
+from typing import Dict
 
 import httpx
 from endpoints import control_app
-from fastapi import FastAPI, Request, Response
+from fastapi import FastAPI
 from fastapi.responses import JSONResponse
+from helpers.errors import ModelLoadFailed, PatchApplicatonError
+from helpers.inference_server_controller import InferenceServerController
+from helpers.inference_server_process_controller import InferenceServerProcessController
+from helpers.inference_server_starter import async_inference_server_startup_flow
+from helpers.truss_patch.model_container_patch_applier import ModelContainerPatchApplier
 from shared import log_config
 from starlette.datastructures import State
-from starlette.middleware.base import BaseHTTPMiddleware
-
-from truss.templates.control.control.helpers.errors import (
-    ModelLoadFailed,
-    PatchApplicatonError,
-)
-from truss.templates.control.control.helpers.inference_server_controller import (
-    InferenceServerController,
-)
-from truss.templates.control.control.helpers.inference_server_process_controller import (
-    InferenceServerProcessController,
-)
-from truss.templates.control.control.helpers.inference_server_starter import (
-    async_inference_server_startup_flow,
-)
-from truss.templates.control.control.helpers.truss_patch.model_container_patch_applier import (
-    ModelContainerPatchApplier,
-)
-
-SANITIZED_EXCEPTION_FRAMES = 2
-
-
-class SanitizedExceptionMiddleware(BaseHTTPMiddleware):
-    def __init__(self, app, num_frames: int = SANITIZED_EXCEPTION_FRAMES):
-        super().__init__(app)
-        self.num_frames = num_frames
-
-    async def dispatch(
-        self, request: Request, call_next: Callable[[Request], Awaitable[Response]]
-    ) -> Response:
-        try:
-            return await call_next(request)
-        except Exception as exc:
-            sanitized_traceback = self._create_sanitized_traceback(exc)
-            request.app.state.logger.error(sanitized_traceback)
-
-            if isinstance(exc, ModelLoadFailed):
-                return JSONResponse({"error": str(exc)}, status_code=503)
-            elif isinstance(exc, PatchApplicatonError):
-                error_type = _camel_to_snake_case(type(exc).__name__)
-                return JSONResponse(
-                    {"error": {"type": error_type, "msg": str(exc)}}, status_code=400
-                )
-            else:
-                return JSONResponse(
-                    {"error": {"type": "unknown", "msg": str(exc)}}, status_code=500
-                )
-
-    def _create_sanitized_traceback(self, error: Exception) -> str:
-        tb_lines = traceback.format_tb(error.__traceback__)
-        if tb_lines and self.num_frames > 0:
-            selected_frames = tb_lines[-self.num_frames :]
-            return "".join(selected_frames).rstrip()
-        else:
-            return f"{type(error).__name__}: {error}"
+
+
+async def handle_patch_error(_, exc):
+    error_type = _camel_to_snake_case(type(exc).__name__)
+    return JSONResponse(content={"error": {"type": error_type, "msg": str(exc)}})
+
+
+async def generic_error_handler(_, exc):
+    return JSONResponse(
+        content={"error": {"type": "unknown", "msg": f"{type(exc)}: {exc}"}}
+    )
+
+
+async def handle_model_load_failed(_, error):
+    # Model load failures should result in 503 status
+    return JSONResponse({"error": str(error)}, 503)
 
 
 def create_app(base_config: Dict):
@@ -117,10 +82,14 @@ def create_app(base_config: Dict):
     app = FastAPI(
         title="Truss Live Reload Server",
         on_startup=[start_background_inference_startup],
+        exception_handlers={
+            PatchApplicatonError: handle_patch_error,
+            ModelLoadFailed: handle_model_load_failed,
+            Exception: generic_error_handler,
+        },
     )
     app.state = app_state
     app.include_router(control_app)
-    app.add_middleware(SanitizedExceptionMiddleware)
 
     @app.on_event("shutdown")
     def on_shutdown():

truss/templates/docker_server/proxy.conf.jinja

@@ -35,6 +35,16 @@ server {
         proxy_pass http://127.0.0.1:{{server_port}};
     }
     # Predict
+    location ~ ^(/v1/models/model:predict|/v1/websocket)$ {
+        proxy_redirect off;
+        proxy_read_timeout 18030s;
+        proxy_http_version 1.1;
+
+        rewrite ^/v1/models/model:predict$ {{server_endpoint}} break;
+
+        proxy_pass http://127.0.0.1:{{server_port}};
+    }
+
     location ~ ^/v1/models/model:predict$ {
         proxy_redirect off;
         proxy_read_timeout 18030s;

{truss-0.11.8rc8.dist-info → truss-0.11.8rc10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: truss
-Version: 0.11.8rc8
+Version: 0.11.8rc10
 Summary: A seamless bridge from model development to model delivery
 Project-URL: Repository, https://github.com/basetenlabs/truss
 Project-URL: Homepage, https://truss.baseten.co

{truss-0.11.8rc8.dist-info → truss-0.11.8rc10.dist-info}/RECORD

@@ -73,7 +73,7 @@ truss/templates/copy_cache_files.Dockerfile.jinja,sha256=Os5zFdYLZ_AfCRGq4RcpVTO
 truss/templates/docker_server_requirements.txt,sha256=PyhOPKAmKW1N2vLvTfLMwsEtuGpoRrbWuNo7tT6v2Mc,18
 truss/templates/server.Dockerfile.jinja,sha256=CUYnF_hgxPGq2re7__0UPWlwzOHMoFkxp6NVKi3U16s,7071
 truss/templates/control/requirements.txt,sha256=nqqNmlTwFeV8sV4fqwItwzzd_egADBP_e-cEopXBJ4k,358
-truss/templates/control/control/application.py,sha256=bmAMh1hxEPu509O7KRUyp_Vaz92xbzgA6vRHhaNt5ts,4979
+truss/templates/control/control/application.py,sha256=jYeta6hWe1SkfLL3W4IDmdYjg3ZuKqI_UagWYs5RB_E,3793
 truss/templates/control/control/endpoints.py,sha256=VQ1lvZjFvR091yRkiFdvXw1Q7PiNGXT9rJwY7_sX6yg,11828
 truss/templates/control/control/server.py,sha256=R4Y219i1dcz0kkksN8obLoX-YXWGo9iW1igindyG50c,3128
 truss/templates/control/control/helpers/context_managers.py,sha256=W6dyFgLBhPa5meqrOb3w_phMtKfaJI-GhwUfpiycDc8,413
@@ -91,7 +91,7 @@ truss/templates/custom/examples.yaml,sha256=2UcCtEdavImWmiCtj31ckBlAKVOwNMC5AwMI
 truss/templates/custom/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 truss/templates/custom/model/model.py,sha256=J04rLxK09Pwt2F4GoKOLKL-H-CqZUdYIM-PL2CE9PoE,1079
 truss/templates/custom_python_dx/my_model.py,sha256=NG75mQ6wxzB1BYUemDFZvRLBET-UrzuUK4FuHjqI29U,910
-truss/templates/docker_server/proxy.conf.jinja,sha256=Lg-PcZzKflG85exZKHNgW_I6r0mATV8AtOIBaE40-RM,1669
+truss/templates/docker_server/proxy.conf.jinja,sha256=qgo54vFLUVBbuxYjM9GKdXUdpbVOTUCuqq73ip-RfbA,1956
 truss/templates/docker_server/supervisord.conf.jinja,sha256=dd37fwZE--cutrvOUCqEyJQQQhlp61H2IUs2huKWsSk,1808
 truss/templates/server/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 truss/templates/server/main.py,sha256=kWXrdD8z8IpamyWxc8qcvd5ck9gM1Kz2QH5qHJCnmOQ,222
@@ -368,8 +368,8 @@ truss_train/deployment.py,sha256=lWWANSuzBWu2M4oK4qD7n-oVR1JKdmw2Pn5BJQHg-Ck,307
 truss_train/loader.py,sha256=0o66EjBaHc2YY4syxxHVR4ordJWs13lNXnKjKq2wq0U,1630
 truss_train/public_api.py,sha256=9N_NstiUlmBuLUwH_fNG_1x7OhGCytZLNvqKXBlStrM,1220
 truss_train/restore_from_checkpoint.py,sha256=8hdPm-WSgkt74HDPjvCjZMBpvA9MwtoYsxVjOoa7BaM,1176
-truss-0.11.8rc8.dist-info/METADATA,sha256=sFf-bspU7seibbvhZolfFQxsX-B6dKktsUVY1-InAew,6680
-truss-0.11.8rc8.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-truss-0.11.8rc8.dist-info/entry_points.txt,sha256=-MwKfHHQHQ6j0HqIgvxrz3CehCmczDLTD-OsRHnjjuU,130
-truss-0.11.8rc8.dist-info/licenses/LICENSE,sha256=FTqGzu85i-uw1Gi8E_o0oD60bH9yQ_XIGtZbA1QUYiw,1064
-truss-0.11.8rc8.dist-info/RECORD,,
+truss-0.11.8rc10.dist-info/METADATA,sha256=fosW8ArYEmmBK4-o2StLiEgskSbAslwBX3FTie2gn0E,6681
+truss-0.11.8rc10.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+truss-0.11.8rc10.dist-info/entry_points.txt,sha256=-MwKfHHQHQ6j0HqIgvxrz3CehCmczDLTD-OsRHnjjuU,130
+truss-0.11.8rc10.dist-info/licenses/LICENSE,sha256=FTqGzu85i-uw1Gi8E_o0oD60bH9yQ_XIGtZbA1QUYiw,1064
+truss-0.11.8rc10.dist-info/RECORD,,