truss 0.10.9rc520__tar.gz → 0.10.9rc601__tar.gz

{truss-0.10.9rc520 → truss-0.10.9rc601}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: truss
-Version: 0.10.9rc520
+Version: 0.10.9rc601
 Summary: A seamless bridge from model development to model delivery
 Project-URL: Repository, https://github.com/basetenlabs/truss
 Project-URL: Homepage, https://truss.baseten.co

{truss-0.10.9rc520 → truss-0.10.9rc601}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "truss"
-version = "0.10.9rc520"
+version = "0.10.9rc601"
 description = "A seamless bridge from model development to model delivery"
 authors = [
     { name = "Pankaj Gupta", email = "no-reply@baseten.co" },

{truss-0.10.9rc520 → truss-0.10.9rc601}/truss/cli/train/deploy_checkpoints/deploy_full_checkpoints.py

@@ -14,12 +14,19 @@ from .deploy_checkpoints_helpers import (
     setup_environment_variables_and_secrets,
 )
 
+# NB(aghilan): Transformers was recently changed to save a chat_template.jinja file instead of inside the tokenizer_config.json file.
+# Old Models will not have this file, so we check for it and use it if it exists.
+# vLLM will not automatically resolve the chat_template.jinja file, so we need to pass it to the start command.
+# This logic is needed for any models trained using Transformers v4.51.3 or later
 VLLM_FULL_START_COMMAND = Template(
-    'sh -c "{%if envvars %}{{ envvars }} {% endif %}vllm serve {{ model_path }}'
-    + " --port 8000"
-    + " --tensor-parallel-size {{ specify_tensor_parallelism }}"
-    + " --dtype bfloat16"
-    + '"'
+    "sh -c '{% if envvars %}{{ envvars }} {% endif %}"
+    'HF_TOKEN="$$(cat /secrets/hf_access_token)" && export HF_TOKEN && '
+    "if [ -f {{ model_path }}/chat_template.jinja ]; then "
+    "  vllm serve {{ model_path }} --chat-template {{ model_path }}/chat_template.jinja "
+    "  --port 8000 --tensor-parallel-size {{ specify_tensor_parallelism }} --dtype bfloat16; "
+    "else "
+    "  vllm serve {{ model_path }} --port 8000 --tensor-parallel-size {{ specify_tensor_parallelism }} --dtype bfloat16; "
+    "fi'"
 )
 
 

{truss-0.10.9rc520 → truss-0.10.9rc601}/truss/contexts/image_builder/serving_image_builder.py

@@ -809,8 +809,9 @@ class ServingImageBuilder(ImageBuilder):
             data_dir_exists=data_dir.exists(),
             model_dir_exists=model_dir.exists(),
             bundled_packages_dir_exists=bundled_packages_dir.exists(),
-            # Create a sanitized copy of the truss directory for hash calculation
-            truss_hash=self._calculate_sanitized_truss_hash(build_dir),
+            truss_hash=directory_content_hash(
+                self._truss_dir, self._spec.hash_ignore_patterns
+            ),
             models=model_files,
             use_hf_secret=use_hf_secret,
             cached_files=cached_files,
@@ -831,26 +832,3 @@ class ServingImageBuilder(ImageBuilder):
         ).strip()
         docker_file_path = build_dir / MODEL_DOCKERFILE_NAME
         docker_file_path.write_text(dockerfile_contents)
-
-    def _calculate_sanitized_truss_hash(self, build_dir: Path) -> str:
-        """Calculate hash from a sanitized copy of the truss directory with runtime fields cleared."""
-        sanitized_truss_dir = build_dir / "build_model_scaffold"
-        
-        # Remove existing copy if it exists
-        if sanitized_truss_dir.exists():
-            shutil.rmtree(sanitized_truss_dir)
-        
-        # Copy the entire truss directory
-        shutil.copytree(self._truss_dir, sanitized_truss_dir)
-        
-        # Clear runtime fields from the config
-        config_file_path = sanitized_truss_dir / "config.yaml"
-        if config_file_path.exists():
-            truss_config = TrussConfig.from_yaml(config_file_path)
-            truss_config.clear_runtime_fields()
-            truss_config.write_to_yaml_file(config_file_path)
-        
-        # Calculate hash from the sanitized directory
-        return directory_content_hash(
-            sanitized_truss_dir, self._spec.hash_ignore_patterns
-        )

{truss-0.10.9rc520 → truss-0.10.9rc601}/truss/templates/server/model_wrapper.py

@@ -59,6 +59,7 @@ class MethodName(str, enum.Enum):
     CHAT_COMPLETIONS = enum.auto()
     COMPLETIONS = enum.auto()
     IS_HEALTHY = enum.auto()
+    MESSAGES = enum.auto()
     POSTPROCESS = enum.auto()
     PREDICT = enum.auto()
     PREPROCESS = enum.auto()
@@ -244,6 +245,7 @@ class ModelDescriptor:
     is_healthy: Optional[MethodDescriptor]
     completions: Optional[MethodDescriptor]
     chat_completions: Optional[MethodDescriptor]
+    messages: Optional[MethodDescriptor]
     websocket: Optional[MethodDescriptor]
 
     @cached_property
@@ -291,6 +293,7 @@ class ModelDescriptor:
         setup = cls._safe_extract_descriptor(model_cls, MethodName.SETUP_ENVIRONMENT)
         completions = cls._safe_extract_descriptor(model_cls, MethodName.COMPLETIONS)
         chats = cls._safe_extract_descriptor(model_cls, MethodName.CHAT_COMPLETIONS)
+        messages = cls._safe_extract_descriptor(model_cls, MethodName.MESSAGES)
         is_healthy = cls._safe_extract_descriptor(model_cls, MethodName.IS_HEALTHY)
         if is_healthy and is_healthy.arg_config != ArgConfig.NONE:
             raise errors.ModelDefinitionError(
@@ -359,6 +362,7 @@ class ModelDescriptor:
             is_healthy=is_healthy,
             completions=completions,
             chat_completions=chats,
+            messages=messages,
             websocket=websocket,
         )
 
@@ -925,6 +929,14 @@ class ModelWrapper:
         )
         return await self._execute_model_endpoint(inputs, request, descriptor)
 
+    async def messages(
+        self, inputs: InputType, request: starlette.requests.Request
+    ) -> OutputType:
+        descriptor = self._get_descriptor_or_raise(
+            self.model_descriptor.messages, MethodName.MESSAGES
+        )
+        return await self._execute_model_endpoint(inputs, request, descriptor)
+
     async def websocket(self, ws: WebSocket) -> None:
         descriptor = self.model_descriptor.websocket
         assert descriptor, "websocket can only be invoked if present on model."

{truss-0.10.9rc520 → truss-0.10.9rc601}/truss/templates/server/truss_server.py

@@ -231,6 +231,13 @@ class BasetenEndpoints:
             method=self._model.completions, request=request, body_raw=body_raw
         )
 
+    async def messages(
+        self, request: Request, body_raw: bytes = Depends(parse_body)
+    ) -> Response:
+        return await self._execute_request(
+            method=self._model.messages, request=request, body_raw=body_raw
+        )
+
     async def websocket(self, ws: WebSocket) -> None:
         self.check_healthy()
         trace_ctx = otel_propagate.extract(ws.headers) or None
@@ -428,6 +435,12 @@ class TrussServer:
                     methods=["POST"],
                     tags=["V1"],
                 ),
+                FastAPIRoute(
+                    r"/v1/messages",
+                    self._endpoints.messages,
+                    methods=["POST"],
+                    tags=["V1"],
+                ),
                 # Websocket endpoint
                 FastAPIWebSocketRoute(r"/v1/websocket", self._endpoints.websocket),
                 # Endpoint aliases for Sagemaker hosting

{truss-0.10.9rc520 → truss-0.10.9rc601}/truss/tests/cli/train/test_deploy_checkpoints.py

@@ -505,8 +505,16 @@ def test_render_vllm_full_truss_config():
     )
 
     result = render_vllm_full_truss_config(deploy_config)
-
-    expected_vllm_command = 'sh -c "HF_TOKEN=$(cat /secrets/hf_token) vllm serve /tmp/training_checkpoints/job123/rank-0/checkpoint-1 --port 8000 --tensor-parallel-size 2 --dtype bfloat16"'
+    expected_vllm_command = (
+        "sh -c 'HF_TOKEN=$(cat /secrets/hf_token) "
+        'HF_TOKEN="$$(cat /secrets/hf_access_token)" && export HF_TOKEN && '
+        "if [ -f /tmp/training_checkpoints/job123/rank-0/checkpoint-1/chat_template.jinja ]; then   "
+        "vllm serve /tmp/training_checkpoints/job123/rank-0/checkpoint-1 "
+        "--chat-template /tmp/training_checkpoints/job123/rank-0/checkpoint-1/chat_template.jinja   "
+        "--port 8000 --tensor-parallel-size 2 --dtype bfloat16; else   "
+        "vllm serve /tmp/training_checkpoints/job123/rank-0/checkpoint-1 "
+        "--port 8000 --tensor-parallel-size 2 --dtype bfloat16; fi'"
+    )
 
     assert isinstance(result, truss_config.TrussConfig)
     assert result.model_name == "test-full-model"

{truss-0.10.9rc520 → truss-0.10.9rc601}/truss/tests/test_data/test_openai/model/model.py

@@ -13,3 +13,6 @@ class Model:
 
     def predict(self, input: Dict) -> str:
         return "predict"
+
+    def messages(self, input: Dict) -> str:
+        return "messages"

{truss-0.10.9rc520 → truss-0.10.9rc601}/truss/truss_handle/patch/hash.py

@@ -1,6 +1,5 @@
 from pathlib import Path
 from typing import Any, List, Optional
-import traceback
 
 from blake3 import blake3
 
@@ -20,15 +19,6 @@ def directory_content_hash(
     underneath. The (root) Directory will have the same hash, even if renamed.
     """
     hasher = blake3()
-    print(f"Calculating hash for {root}, ignore_patterns: {ignore_patterns}")
-    
-    # Print caller chain
-    caller_chain = traceback.extract_stack()
-    if len(caller_chain) > 1:
-        print("Caller chain:")
-        for i, frame in enumerate(caller_chain[:-1]):  # Skip the current function
-            print(f"  {i+1}. {frame.filename}:{frame.lineno} in {frame.name}")
-    
     paths = list(get_unignored_relative_paths_from_root(root, ignore_patterns))
     paths.sort()
     for path in paths:
@@ -36,9 +26,7 @@ def directory_content_hash(
         absolute_path = root / path
         if absolute_path.is_file():
             hasher.update(file_content_hash(absolute_path))
-    hash_str = hasher.hexdigest()
-    print(f"Hash for {root}: {hash_str}")
-    return hash_str
+    return hasher.hexdigest()
 
 
 def file_content_hash(file: Path) -> bytes:

{truss-0.10.9rc520 → truss-0.10.9rc601}/truss/truss_handle/truss_handle.py

@@ -106,6 +106,7 @@ class DockerURLs:
         self.predict_url = f"{base_url}/v1/models/model:predict"
         self.completions_url = f"{base_url}/v1/completions"
         self.chat_completions_url = f"{base_url}/v1/chat/completions"
+        self.messages_url = f"{base_url}/v1/messages"
 
         self.schema_url = f"{base_url}/v1/models/model/schema"
         self.metrics_url = f"{base_url}/metrics"
@@ -938,7 +939,6 @@ class TrussHandle:
 
     def _update_config(self, **fields_to_update):
         config = self._spec.config.model_copy(update=fields_to_update)
-        print(f"Updating config: {fields_to_update}, writing to {self._spec.config_path}")
         config.write_to_yaml_file(self._spec.config_path)
         self._spec = TrussSpec(self._truss_dir)  # Reload.