truss 0.10.9rc520__py3-none-any.whl → 0.10.9rc601__py3-none-any.whl

truss/cli/train/deploy_checkpoints/deploy_full_checkpoints.py

@@ -14,12 +14,19 @@ from .deploy_checkpoints_helpers import (
     setup_environment_variables_and_secrets,
 )
 
+# NB(aghilan): Transformers was recently changed to save a chat_template.jinja file instead of inside the tokenizer_config.json file.
+# Old Models will not have this file, so we check for it and use it if it exists.
+# vLLM will not automatically resolve the chat_template.jinja file, so we need to pass it to the start command.
+# This logic is needed for any models trained using Transformers v4.51.3 or later
 VLLM_FULL_START_COMMAND = Template(
-    'sh -c "{%if envvars %}{{ envvars }} {% endif %}vllm serve {{ model_path }}'
-    + " --port 8000"
-    + " --tensor-parallel-size {{ specify_tensor_parallelism }}"
-    + " --dtype bfloat16"
-    + '"'
+    "sh -c '{% if envvars %}{{ envvars }} {% endif %}"
+    'HF_TOKEN="$$(cat /secrets/hf_access_token)" && export HF_TOKEN && '
+    "if [ -f {{ model_path }}/chat_template.jinja ]; then "
+    "  vllm serve {{ model_path }} --chat-template {{ model_path }}/chat_template.jinja "
+    "  --port 8000 --tensor-parallel-size {{ specify_tensor_parallelism }} --dtype bfloat16; "
+    "else "
+    "  vllm serve {{ model_path }} --port 8000 --tensor-parallel-size {{ specify_tensor_parallelism }} --dtype bfloat16; "
+    "fi'"
 )
 
 

truss/contexts/image_builder/serving_image_builder.py

@@ -809,8 +809,9 @@ class ServingImageBuilder(ImageBuilder):
             data_dir_exists=data_dir.exists(),
             model_dir_exists=model_dir.exists(),
             bundled_packages_dir_exists=bundled_packages_dir.exists(),
-            # Create a sanitized copy of the truss directory for hash calculation
-            truss_hash=self._calculate_sanitized_truss_hash(build_dir),
+            truss_hash=directory_content_hash(
+                self._truss_dir, self._spec.hash_ignore_patterns
+            ),
             models=model_files,
             use_hf_secret=use_hf_secret,
             cached_files=cached_files,
@@ -831,26 +832,3 @@ class ServingImageBuilder(ImageBuilder):
         ).strip()
         docker_file_path = build_dir / MODEL_DOCKERFILE_NAME
         docker_file_path.write_text(dockerfile_contents)
-
-    def _calculate_sanitized_truss_hash(self, build_dir: Path) -> str:
-        """Calculate hash from a sanitized copy of the truss directory with runtime fields cleared."""
-        sanitized_truss_dir = build_dir / "build_model_scaffold"
-        
-        # Remove existing copy if it exists
-        if sanitized_truss_dir.exists():
-            shutil.rmtree(sanitized_truss_dir)
-        
-        # Copy the entire truss directory
-        shutil.copytree(self._truss_dir, sanitized_truss_dir)
-        
-        # Clear runtime fields from the config
-        config_file_path = sanitized_truss_dir / "config.yaml"
-        if config_file_path.exists():
-            truss_config = TrussConfig.from_yaml(config_file_path)
-            truss_config.clear_runtime_fields()
-            truss_config.write_to_yaml_file(config_file_path)
-        
-        # Calculate hash from the sanitized directory
-        return directory_content_hash(
-            sanitized_truss_dir, self._spec.hash_ignore_patterns
-        )

truss/templates/server/model_wrapper.py

@@ -59,6 +59,7 @@ class MethodName(str, enum.Enum):
     CHAT_COMPLETIONS = enum.auto()
     COMPLETIONS = enum.auto()
     IS_HEALTHY = enum.auto()
+    MESSAGES = enum.auto()
     POSTPROCESS = enum.auto()
     PREDICT = enum.auto()
     PREPROCESS = enum.auto()
@@ -244,6 +245,7 @@ class ModelDescriptor:
     is_healthy: Optional[MethodDescriptor]
     completions: Optional[MethodDescriptor]
     chat_completions: Optional[MethodDescriptor]
+    messages: Optional[MethodDescriptor]
     websocket: Optional[MethodDescriptor]
 
     @cached_property
@@ -291,6 +293,7 @@ class ModelDescriptor:
         setup = cls._safe_extract_descriptor(model_cls, MethodName.SETUP_ENVIRONMENT)
         completions = cls._safe_extract_descriptor(model_cls, MethodName.COMPLETIONS)
         chats = cls._safe_extract_descriptor(model_cls, MethodName.CHAT_COMPLETIONS)
+        messages = cls._safe_extract_descriptor(model_cls, MethodName.MESSAGES)
         is_healthy = cls._safe_extract_descriptor(model_cls, MethodName.IS_HEALTHY)
         if is_healthy and is_healthy.arg_config != ArgConfig.NONE:
             raise errors.ModelDefinitionError(
@@ -359,6 +362,7 @@ class ModelDescriptor:
             is_healthy=is_healthy,
             completions=completions,
             chat_completions=chats,
+            messages=messages,
             websocket=websocket,
         )
 
@@ -925,6 +929,14 @@ class ModelWrapper:
         )
         return await self._execute_model_endpoint(inputs, request, descriptor)
 
+    async def messages(
+        self, inputs: InputType, request: starlette.requests.Request
+    ) -> OutputType:
+        descriptor = self._get_descriptor_or_raise(
+            self.model_descriptor.messages, MethodName.MESSAGES
+        )
+        return await self._execute_model_endpoint(inputs, request, descriptor)
+
     async def websocket(self, ws: WebSocket) -> None:
         descriptor = self.model_descriptor.websocket
         assert descriptor, "websocket can only be invoked if present on model."

truss/templates/server/truss_server.py

@@ -231,6 +231,13 @@ class BasetenEndpoints:
             method=self._model.completions, request=request, body_raw=body_raw
         )
 
+    async def messages(
+        self, request: Request, body_raw: bytes = Depends(parse_body)
+    ) -> Response:
+        return await self._execute_request(
+            method=self._model.messages, request=request, body_raw=body_raw
+        )
+
     async def websocket(self, ws: WebSocket) -> None:
         self.check_healthy()
         trace_ctx = otel_propagate.extract(ws.headers) or None
@@ -428,6 +435,12 @@ class TrussServer:
                     methods=["POST"],
                     tags=["V1"],
                 ),
+                FastAPIRoute(
+                    r"/v1/messages",
+                    self._endpoints.messages,
+                    methods=["POST"],
+                    tags=["V1"],
+                ),
                 # Websocket endpoint
                 FastAPIWebSocketRoute(r"/v1/websocket", self._endpoints.websocket),
                 # Endpoint aliases for Sagemaker hosting

truss/tests/cli/train/test_deploy_checkpoints.py

@@ -505,8 +505,16 @@ def test_render_vllm_full_truss_config():
     )
 
     result = render_vllm_full_truss_config(deploy_config)
-
-    expected_vllm_command = 'sh -c "HF_TOKEN=$(cat /secrets/hf_token) vllm serve /tmp/training_checkpoints/job123/rank-0/checkpoint-1 --port 8000 --tensor-parallel-size 2 --dtype bfloat16"'
+    expected_vllm_command = (
+        "sh -c 'HF_TOKEN=$(cat /secrets/hf_token) "
+        'HF_TOKEN="$$(cat /secrets/hf_access_token)" && export HF_TOKEN && '
+        "if [ -f /tmp/training_checkpoints/job123/rank-0/checkpoint-1/chat_template.jinja ]; then   "
+        "vllm serve /tmp/training_checkpoints/job123/rank-0/checkpoint-1 "
+        "--chat-template /tmp/training_checkpoints/job123/rank-0/checkpoint-1/chat_template.jinja   "
+        "--port 8000 --tensor-parallel-size 2 --dtype bfloat16; else   "
+        "vllm serve /tmp/training_checkpoints/job123/rank-0/checkpoint-1 "
+        "--port 8000 --tensor-parallel-size 2 --dtype bfloat16; fi'"
+    )
 
     assert isinstance(result, truss_config.TrussConfig)
     assert result.model_name == "test-full-model"

truss/tests/test_data/test_openai/model/model.py

@@ -13,3 +13,6 @@ class Model:
 
     def predict(self, input: Dict) -> str:
         return "predict"
+
+    def messages(self, input: Dict) -> str:
+        return "messages"

truss/truss_handle/patch/hash.py

@@ -1,6 +1,5 @@
 from pathlib import Path
 from typing import Any, List, Optional
-import traceback
 
 from blake3 import blake3
 
@@ -20,15 +19,6 @@ def directory_content_hash(
     underneath. The (root) Directory will have the same hash, even if renamed.
     """
     hasher = blake3()
-    print(f"Calculating hash for {root}, ignore_patterns: {ignore_patterns}")
-    
-    # Print caller chain
-    caller_chain = traceback.extract_stack()
-    if len(caller_chain) > 1:
-        print("Caller chain:")
-        for i, frame in enumerate(caller_chain[:-1]):  # Skip the current function
-            print(f"  {i+1}. {frame.filename}:{frame.lineno} in {frame.name}")
-    
     paths = list(get_unignored_relative_paths_from_root(root, ignore_patterns))
     paths.sort()
     for path in paths:
@@ -36,9 +26,7 @@ def directory_content_hash(
         absolute_path = root / path
         if absolute_path.is_file():
             hasher.update(file_content_hash(absolute_path))
-    hash_str = hasher.hexdigest()
-    print(f"Hash for {root}: {hash_str}")
-    return hash_str
+    return hasher.hexdigest()
 
 
 def file_content_hash(file: Path) -> bytes:

truss/truss_handle/truss_handle.py

@@ -106,6 +106,7 @@ class DockerURLs:
         self.predict_url = f"{base_url}/v1/models/model:predict"
         self.completions_url = f"{base_url}/v1/completions"
         self.chat_completions_url = f"{base_url}/v1/chat/completions"
+        self.messages_url = f"{base_url}/v1/messages"
 
         self.schema_url = f"{base_url}/v1/models/model/schema"
         self.metrics_url = f"{base_url}/metrics"
@@ -938,7 +939,6 @@ class TrussHandle:
 
     def _update_config(self, **fields_to_update):
         config = self._spec.config.model_copy(update=fields_to_update)
-        print(f"Updating config: {fields_to_update}, writing to {self._spec.config_path}")
         config.write_to_yaml_file(self._spec.config_path)
         self._spec = TrussSpec(self._truss_dir)  # Reload.
 

{truss-0.10.9rc520.dist-info → truss-0.10.9rc601.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: truss
-Version: 0.10.9rc520
+Version: 0.10.9rc601
 Summary: A seamless bridge from model development to model delivery
 Project-URL: Repository, https://github.com/basetenlabs/truss
 Project-URL: Homepage, https://truss.baseten.co

{truss-0.10.9rc520.dist-info → truss-0.10.9rc601.dist-info}/RECORD

@@ -25,7 +25,7 @@ truss/cli/train/types.py,sha256=alGtr4Q71GeB65PpGMhsoKygw4k_ncR6MKIP1ioP8rI,951
 truss/cli/train/deploy_checkpoints/__init__.py,sha256=wL-M2yu8PxO2tFvjwshXAfPnB-5TlvsBp2v_bdzimRU,99
 truss/cli/train/deploy_checkpoints/deploy_checkpoints.py,sha256=wWFCpaMjM868-f_ChECrGHyMHka1F2VS1pEKPcEw3eM,16667
 truss/cli/train/deploy_checkpoints/deploy_checkpoints_helpers.py,sha256=7CdYgsxDF7nHITyCGpjjTMeaaOvtlCYwF7NSxpKedS0,1723
-truss/cli/train/deploy_checkpoints/deploy_full_checkpoints.py,sha256=pFj7rDcnvb9C4MMfr3wc4aBXfziqFkzw0H883NtQ1Es,3245
+truss/cli/train/deploy_checkpoints/deploy_full_checkpoints.py,sha256=onY-Xk6n1M5H_QGcV47LhiCq1f9p4bOLkHLTJzH8MnI,3970
 truss/cli/train/deploy_checkpoints/deploy_lora_checkpoints.py,sha256=P91dIAzuhl2GlzmrWwCcYI7uCMT1Lm7C79JQHM_exN4,4442
 truss/cli/utils/common.py,sha256=aWnla4qMSEz57dRMTl7R-EaScsuEpnQUeziGUaIeqeU,6149
 truss/cli/utils/output.py,sha256=GNjU85ZAMp5BI6Yij5wYXcaAvpm_kmHV0nHNmdkMxb0,646
@@ -34,7 +34,7 @@ truss/contexts/docker_build_setup.py,sha256=cF4ExZgtYvrWxvyCAaUZUvV_DB_7__MqVomU
 truss/contexts/truss_context.py,sha256=uS6L-ACHxNk0BsJwESOHh1lA0OGGw0pb33aFKGsASj4,436
 truss/contexts/image_builder/cache_warmer.py,sha256=wKE5zE3efnEpjmiLqAtQZX1pb2z57aRU0uElOxxb5f4,7434
 truss/contexts/image_builder/image_builder.py,sha256=IuRgDeeoHVLzIkJvKtX3807eeqEyaroCs_KWDcIHZUg,1461
-truss/contexts/image_builder/serving_image_builder.py,sha256=zz0IkoYMsWQquOpiHR-I-Cz54MFyOtvVzkSwCdxouWc,34918
+truss/contexts/image_builder/serving_image_builder.py,sha256=NdjlQDZqKsmFDXx1UqL2HszL4ozPuNC6MFoTbXB5l0w,33886
 truss/contexts/image_builder/util.py,sha256=y2-CjUKv0XV-0w2sr1fUCflysDJLsoU4oPp6tvvoFnk,1203
 truss/contexts/local_loader/docker_build_emulator.py,sha256=rmf7I28zksSmHjwvJMx2rIa6xK4KeR5fBm5YFth_fQg,2464
 truss/contexts/local_loader/dockerfile_parser.py,sha256=GoRJ0Af_3ILyLhjovK5lrCGn1rMxz6W3l681ro17ZzI,1344
@@ -92,9 +92,9 @@ truss/templates/docker_server/proxy.conf.jinja,sha256=Lg-PcZzKflG85exZKHNgW_I6r0
 truss/templates/docker_server/supervisord.conf.jinja,sha256=CoaSLv0Lr8t1tS_q102IFufNX2lWrlbCHJLjMhYjOwM,1711
 truss/templates/server/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 truss/templates/server/main.py,sha256=kWXrdD8z8IpamyWxc8qcvd5ck9gM1Kz2QH5qHJCnmOQ,222
-truss/templates/server/model_wrapper.py,sha256=k75VVISwwlsx5EGb82UZsu8kCM_i6Yi3-Hd0-Kpm1yo,42055
+truss/templates/server/model_wrapper.py,sha256=3xrEVKdFMCEnBJ4sZB14Vt_kwGoBxUibkFlCfonBOTE,42554
 truss/templates/server/requirements.txt,sha256=iRR2BEpBQnt-YOiTEKOnaab7tlR4C23V1cuURuIt7ZY,672
-truss/templates/server/truss_server.py,sha256=ob_nceeGtFPZzKKdk_ZZGLoZrJOGE6hR52xM1sPR97A,19498
+truss/templates/server/truss_server.py,sha256=FuouBLIQZ3-lsW_FGunY-S7hJ9KUNuqjLMOwYKVx8mE,19945
 truss/templates/server/common/__init__.py,sha256=qHIqr68L5Tn4mV6S-PbORpcuJ4jmtBR8aCuRTIWDvNo,85
 truss/templates/server/common/errors.py,sha256=qWeZlmNI8ZGbZbOIp_mtS6IKvUFIzhj3QH8zp-xTp9o,8554
 truss/templates/server/common/patches.py,sha256=uEOzvDnXsHOkTSa8zygGYuR4GHhrFNVHNQc5peJcwvo,1393
@@ -136,7 +136,7 @@ truss/tests/test_truss_gatherer.py,sha256=bn288OEkC49YY0mhly4cAl410ktZPfElNdWwZy
 truss/tests/test_truss_handle.py,sha256=-xz9VXkecXDTslmQZ-dmUmQLnvD0uumRqHS2uvGlMBA,30750
 truss/tests/test_util.py,sha256=hs1bNMkXKEdoPRx4Nw-NAEdoibR92OubZuADGmbiYsQ,1344
 truss/tests/cli/test_cli.py,sha256=yfbVS5u1hnAmmA8mJ539vj3lhH-JVGUvC4Q_Mbort44,787
-truss/tests/cli/train/test_deploy_checkpoints.py,sha256=pPTqyGqvSsrsEHXguo6swLXkVObytLdupfSElqlujPc,25733
+truss/tests/cli/train/test_deploy_checkpoints.py,sha256=yCPE5S4D_92hz692QKYDaHvwye4imuz2e1pd1K1pXkE,26203
 truss/tests/cli/train/test_train_cli_core.py,sha256=T1Xa6-NRk2nTJGX6sXaA8x4qCwL3Ini72PBI2gW7rYM,7879
 truss/tests/cli/train/resources/test_deploy_from_checkpoint_config.yml,sha256=GF7r9l0KaeXiUYCPSBpeMPd2QG6PeWWyI12NdbqLOgc,1930
 truss/tests/contexts/image_builder/test_serving_image_builder.py,sha256=ycOxhGIfe5OVcgqZqJvn1Ca2368AN1KdNmIO0vSZ4ko,22368
@@ -236,7 +236,7 @@ truss/tests/test_data/test_go_custom_server_truss/docker/main.go,sha256=WR3mJU1o
 truss/tests/test_data/test_openai/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 truss/tests/test_data/test_openai/config.yaml,sha256=ByY_Smgx0lw24Yj0hqgofEmL3nrGNj7gZE5iBKlvwxk,235
 truss/tests/test_data/test_openai/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-truss/tests/test_data/test_openai/model/model.py,sha256=GEtIJnWlU1snBid2sS-bZHrjQpP8UzL8tanzyH_tdgE,319
+truss/tests/test_data/test_openai/model/model.py,sha256=NMyZH6QcJv4TMw2Cd8M02wvmwDqlWJB4ZwtD4n8QU0Y,390
 truss/tests/test_data/test_pyantic_v1/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 truss/tests/test_data/test_pyantic_v1/config.yaml,sha256=fqWpH3E4UPEnjvAw6Q9_F5oZZLy69RAfycbgtmCFsXo,270
 truss/tests/test_data/test_pyantic_v1/requirements.txt,sha256=OpG4JAdJME9VWjoNftdHYg-y94k2gbhqdM1_NwOgcT8,13
@@ -316,13 +316,13 @@ truss/truss_handle/build.py,sha256=BKFV-S57tnWcfRffvQ7SPp78BrjmRy3GhgF6ThaIrDM,3
 truss/truss_handle/decorators.py,sha256=PUR5w2rl_cvcsVtAUpcYLzNXuOml9R0-wtpXy-9hDPk,407
 truss/truss_handle/readme_generator.py,sha256=B4XbGwUjzMNOr71DWNAL8kCu5_ZHq7YOM8yVGaOZMSE,716
 truss/truss_handle/truss_gatherer.py,sha256=Xysl_UnCVhehPfZeHa8p7WFp94ENqh-VVpbuqnCui3A,2870
-truss/truss_handle/truss_handle.py,sha256=x-2W2bwJhaVvlUaza7Gt1OHKBW4ykSIbgOR-SAEE-Ss,40977
+truss/truss_handle/truss_handle.py,sha256=pGsrsfCSEa5DmrFAoA5Nxr9TBke0Dr2eaq81Ht82j9U,40940
 truss/truss_handle/patch/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 truss/truss_handle/patch/calc_patch.py,sha256=Qyk1QmacK4jy9Ia8-93L8VtAWJhw15z22DdZUkBKlys,18334
 truss/truss_handle/patch/constants.py,sha256=pCEi5Pwi8Rnqthrr3VEsWL9EP1P1VV1T8DEYuitHLmc,139
 truss/truss_handle/patch/custom_types.py,sha256=QklzhgLD_PpvNvNYQCvujAd16eYEaDGfLA1scxk6zsA,3481
 truss/truss_handle/patch/dir_signature.py,sha256=UCdZCzXkI-l-ae0I0pdmB2bavB9qzhhOKYXyLnDFQZY,921
-truss/truss_handle/patch/hash.py,sha256=3FU-eI9tHgqLovKz1-K_LbQps1RIh0VP2mIo1s0wDrc,2570
+truss/truss_handle/patch/hash.py,sha256=0LCV091DJKzkuulQQqgzqPPa2zyeHoa5faXddSKLuqM,2106
 truss/truss_handle/patch/local_truss_patch_applier.py,sha256=fOHWKt3teYnbqeRsF63FjDRu0iCxMfN5RHWtiDxXHIc,2942
 truss/truss_handle/patch/signature.py,sha256=8eas8gy6Japd1hrgdmtHmKTTxQmWsbmgKRQQGL2PVuA,858
 truss/truss_handle/patch/truss_dir_patch_applier.py,sha256=uhhHvKYHn_dpfz0xp4jwO9_qAej5sO3f8of_h-21PP4,3666
@@ -361,8 +361,8 @@ truss_train/definitions.py,sha256=yFQYJoxK2tDBeKFHR-IJz12jU1CtWRXN-ZERh9zjMHo,66
 truss_train/deployment.py,sha256=zmeJ66kg1Wc7l7bwA_cXqv85uMF77hYl7NPHuhc1NPs,2493
 truss_train/loader.py,sha256=0o66EjBaHc2YY4syxxHVR4ordJWs13lNXnKjKq2wq0U,1630
 truss_train/public_api.py,sha256=9N_NstiUlmBuLUwH_fNG_1x7OhGCytZLNvqKXBlStrM,1220
-truss-0.10.9rc520.dist-info/METADATA,sha256=gUlOXD6Xe0raLIuYQz9HDDb5mWkV_FB0sFwqlNnD6Rs,6674
-truss-0.10.9rc520.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-truss-0.10.9rc520.dist-info/entry_points.txt,sha256=-MwKfHHQHQ6j0HqIgvxrz3CehCmczDLTD-OsRHnjjuU,130
-truss-0.10.9rc520.dist-info/licenses/LICENSE,sha256=FTqGzu85i-uw1Gi8E_o0oD60bH9yQ_XIGtZbA1QUYiw,1064
-truss-0.10.9rc520.dist-info/RECORD,,
+truss-0.10.9rc601.dist-info/METADATA,sha256=lVGZYss-iZy1X7xth17tJozGiJKzPr6cpFKm9kb-7qY,6674
+truss-0.10.9rc601.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+truss-0.10.9rc601.dist-info/entry_points.txt,sha256=-MwKfHHQHQ6j0HqIgvxrz3CehCmczDLTD-OsRHnjjuU,130
+truss-0.10.9rc601.dist-info/licenses/LICENSE,sha256=FTqGzu85i-uw1Gi8E_o0oD60bH9yQ_XIGtZbA1QUYiw,1064
+truss-0.10.9rc601.dist-info/RECORD,,