truss 0.10.9rc514__tar.gz → 0.10.9rc601__tar.gz

{truss-0.10.9rc514 → truss-0.10.9rc601}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: truss
-Version: 0.10.9rc514
+Version: 0.10.9rc601
 Summary: A seamless bridge from model development to model delivery
 Project-URL: Repository, https://github.com/basetenlabs/truss
 Project-URL: Homepage, https://truss.baseten.co

{truss-0.10.9rc514 → truss-0.10.9rc601}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "truss"
-version = "0.10.9rc514"
+version = "0.10.9rc601"
 description = "A seamless bridge from model development to model delivery"
 authors = [
     { name = "Pankaj Gupta", email = "no-reply@baseten.co" },

{truss-0.10.9rc514 → truss-0.10.9rc601}/truss/base/constants.py

@@ -29,6 +29,7 @@ BEI_REQUIRED_MAX_NUM_TOKENS = 16384
 TRTLLM_MIN_MEMORY_REQUEST_GI = 10
 HF_MODELS_API_URL = "https://huggingface.co/api/models"
 HF_ACCESS_TOKEN_KEY = "hf_access_token"
+HF_ACCESS_TOKEN_FILE_NAME = "hf_access_token"
 TRUSSLESS_MAX_PAYLOAD_SIZE = "64M"
 # Alias for TEMPLATES_DIR
 SERVING_DIR: pathlib.Path = TEMPLATES_DIR

{truss-0.10.9rc514 → truss-0.10.9rc601}/truss/cli/logs/base_watcher.py

@@ -9,7 +9,7 @@ from truss.remote.baseten.api import BasetenApi
 
 POLL_INTERVAL_SEC = 2
 # NB(nikhil): This helps account for (1) log processing delays (2) clock skews
-CLOCK_SKEW_BUFFER_MS = 10000
+CLOCK_SKEW_BUFFER_MS = 60000
 
 
 class LogWatcher(ABC):

{truss-0.10.9rc514 → truss-0.10.9rc601}/truss/cli/train/deploy_checkpoints/deploy_full_checkpoints.py

@@ -14,12 +14,19 @@ from .deploy_checkpoints_helpers import (
     setup_environment_variables_and_secrets,
 )
 
+# NB(aghilan): Transformers was recently changed to save a chat_template.jinja file instead of inside the tokenizer_config.json file.
+# Old Models will not have this file, so we check for it and use it if it exists.
+# vLLM will not automatically resolve the chat_template.jinja file, so we need to pass it to the start command.
+# This logic is needed for any models trained using Transformers v4.51.3 or later
 VLLM_FULL_START_COMMAND = Template(
-    'sh -c "{%if envvars %}{{ envvars }} {% endif %}vllm serve {{ model_path }}'
-    + " --port 8000"
-    + " --tensor-parallel-size {{ specify_tensor_parallelism }}"
-    + " --dtype bfloat16"
-    + '"'
+    "sh -c '{% if envvars %}{{ envvars }} {% endif %}"
+    'HF_TOKEN="$$(cat /secrets/hf_access_token)" && export HF_TOKEN && '
+    "if [ -f {{ model_path }}/chat_template.jinja ]; then "
+    "  vllm serve {{ model_path }} --chat-template {{ model_path }}/chat_template.jinja "
+    "  --port 8000 --tensor-parallel-size {{ specify_tensor_parallelism }} --dtype bfloat16; "
+    "else "
+    "  vllm serve {{ model_path }} --port 8000 --tensor-parallel-size {{ specify_tensor_parallelism }} --dtype bfloat16; "
+    "fi'"
 )
 
 

truss-0.10.9rc601/truss/cli/train/metrics_watcher.py

@@ -0,0 +1,331 @@
+import signal
+import time
+import traceback
+from typing import Any, Dict, List, Optional, Tuple, cast
+
+from rich.columns import Columns
+from rich.layout import Layout
+from rich.live import Live
+from rich.table import Table
+from rich.text import Text
+
+from truss.cli.train.poller import TrainingPollerMixin
+from truss.cli.utils import common
+from truss.cli.utils.output import console
+from truss.remote.baseten.api import BasetenApi
+
+METRICS_POLL_INTERVAL_SEC = 30
+
+
+class MetricsWatcher(TrainingPollerMixin):
+    live: Optional[Live]
+
+    def __init__(self, api: BasetenApi, project_id: str, job_id: str):
+        super().__init__(api, project_id, job_id)
+
+        self.live = None
+        signal.signal(signal.SIGINT, self._handle_sigint)
+
+    def _handle_sigint(self, signum: int, frame: Any) -> None:
+        if self.live:
+            self.live.stop()
+        msg = f"\n\nExiting training job metrics. To stop the job, run `truss train stop --job-id {self.job_id}`"
+        console.print(msg, style="yellow")
+        raise KeyboardInterrupt()
+
+    def _format_bytes(self, bytes_val: float) -> Tuple[str, str]:
+        """Convert bytes to human readable format"""
+        color_map = {"MB": "green", "GB": "cyan", "TB": "magenta"}
+        unit = "MB"
+        if bytes_val > 1024 * 1024 * 1024 * 1024:
+            unit = "TB"
+        elif bytes_val > 1024 * 1024 * 1024:
+            unit = "GB"
+
+        if unit == "MB":
+            return f"{bytes_val / (1024 * 1024):.2f} MB", color_map[unit]
+        elif unit == "GB":
+            return f"{bytes_val / (1024 * 1024 * 1024):.2f} GB", color_map[unit]
+        return f"{bytes_val:.2f} bytes", color_map[unit]
+
+    def _format_storage_utilization(self, utilization: float) -> Tuple[str, str]:
+        percent = round(utilization * 100, 4)
+        if percent > 90:
+            return f"{percent}%", "red"
+        elif percent > 70:
+            return f"{percent}%", "yellow"
+        return f"{percent}%", "green"
+
+    def _get_latest_metric(self, metrics: List[Dict]) -> Optional[float]:
+        """Get the most recent metric value"""
+        if not metrics:
+            return None
+        return metrics[-1].get("value")
+
+    def _get_latest_storage_metrics(
+        self, storage_data: Optional[Dict[str, List[Dict]]]
+    ) -> Optional[Tuple[int, float]]:
+        if not storage_data:
+            return None
+        usage_data = storage_data.get("usage_bytes")
+        utilization_data = storage_data.get("utilization")
+        if not usage_data or not utilization_data:
+            return None
+        usage_value = usage_data[-1].get("value", None)
+        utilization_value = utilization_data[-1].get("value", None)
+        if not usage_value or not utilization_value:
+            return None
+        return cast(int, usage_value), cast(float, utilization_value)
+
+    def _maybe_format_storage_table_row(
+        self, table: Table, label: str, storage_data: Optional[Dict[str, List[Dict]]]
+    ) -> bool:
+        if not storage_data:
+            return False
+        maybe_values = self._get_latest_storage_metrics(storage_data)
+        if not maybe_values:
+            return False
+        raw_usage, raw_utilization = maybe_values
+        usage_value, usage_color = self._format_bytes(raw_usage)
+        utilization_value, utilization_color = self._format_storage_utilization(
+            raw_utilization
+        )
+        table.add_row(
+            label,
+            Text(usage_value, style=usage_color),
+            Text(utilization_value, style=utilization_color),
+        )
+        return True
+
+    def create_metrics_table(self, metrics_data: Dict) -> Layout:
+        """Create a Rich table with the metrics"""
+        tables = []
+
+        timestamp = self._get_timestamp_from_metrics(metrics_data)
+
+        node_tables = self._create_unified_node_metrics_tables(metrics_data)
+        tables.extend(node_tables)
+
+        storage_tables = self._create_storage_tables(metrics_data)
+        tables.extend(storage_tables)
+
+        columns = Columns(tables, title="Training Job Metrics")
+
+        layout = Layout()
+
+        if timestamp:
+            from rich.panel import Panel
+
+            layout.split_column(
+                Layout(
+                    Panel(
+                        f"🕐 Last Updated: {timestamp}\n💡 Press Ctrl+C to exit",
+                        style="bold cyan",
+                    ),
+                    size=4,
+                ),
+                Layout(columns),
+            )
+        else:
+            layout.split_column(Layout(columns))
+
+        return layout
+
+    def _get_timestamp_from_metrics(self, metrics_data: Dict) -> Optional[str]:
+        """Extract timestamp from metrics data for display"""
+        # Try to get timestamp from per_node_metrics first. Fall back to main metrics if not there.
+        per_node_metrics = metrics_data.get("per_node_metrics", [])
+        if per_node_metrics and len(per_node_metrics) > 0:
+            first_node_metrics = per_node_metrics[0].get("metrics", {})
+            cpu_usage_data = first_node_metrics.get("cpu_usage", [])
+            if cpu_usage_data and len(cpu_usage_data) > 0:
+                timestamp = cpu_usage_data[-1].get("timestamp")
+                if timestamp:
+                    return common.format_localized_time(timestamp)
+
+        cpu_usage_data = metrics_data.get("cpu_usage", [])
+        if cpu_usage_data and len(cpu_usage_data) > 0:
+            timestamp = cpu_usage_data[-1].get("timestamp")
+            if timestamp:
+                return common.format_localized_time(timestamp)
+
+        return None
+
+    def _create_unified_node_metrics_tables(self, metrics_data: Dict) -> List[Table]:
+        """Create tables for node metrics, handling both single and multi-node scenarios"""
+        tables = []
+
+        per_node_metrics = metrics_data.get("per_node_metrics", [])
+
+        if not per_node_metrics:
+            # Job is likely just starting up - it takes some type for the
+            # the metrics to become available after the job starts running.
+            from rich.text import Text
+
+            waiting_table = Table(title="Training Job Status")
+            waiting_table.add_column("Status")
+            waiting_table.add_column("Message")
+
+            waiting_table.add_row(
+                "Status",
+                Text("⏳ Waiting for metrics to become available...", style="yellow"),
+            )
+            waiting_table.add_row(
+                "Note",
+                Text(
+                    "Metrics will appear once the training job starts running.",
+                    style="dim",
+                ),
+            )
+
+            tables.append(waiting_table)
+            return tables
+
+        for node_metrics in per_node_metrics:
+            node_id = node_metrics.get("node_id", "Unknown")
+            metrics = node_metrics.get("metrics", {})
+
+            if not metrics:
+                continue
+
+            table = self._create_node_table(node_id, metrics)
+            tables.append(table)
+
+        return tables
+
+    def _create_node_table(self, node_id: str, metrics: Dict) -> Table:
+        """Create a table for a single node's metrics"""
+        table = Table(title=f"Node: {node_id}")
+        table.add_column("Metric")
+        table.add_column("Value")
+
+        cpu_usage = self._get_latest_metric(metrics.get("cpu_usage", []))
+        if cpu_usage is not None:
+            table.add_row("CPU usage", f"{cpu_usage:.2f} cores")
+
+        cpu_memory = self._get_latest_metric(metrics.get("cpu_memory_usage_bytes", []))
+        if cpu_memory is not None:
+            formatted_value, color = self._format_bytes(cpu_memory)
+            table.add_row("CPU memory", Text(formatted_value, style=color))
+
+        if cpu_usage is not None or cpu_memory is not None:
+            table.add_section()
+
+        gpu_utilization = metrics.get("gpu_utilization", {})
+        gpu_memory = metrics.get("gpu_memory_usage_bytes", {})
+
+        # API should return same GPU IDs for utilization and memory
+        keys = gpu_utilization.keys()
+        for idx, gpu_id in enumerate(keys):
+            latest_util = self._get_latest_metric(gpu_utilization.get(gpu_id, []))
+            if latest_util is not None:
+                table.add_row(f"GPU {gpu_id} utilization", f"{latest_util * 100:.1f}%")
+
+            latest_memory = self._get_latest_metric(gpu_memory.get(gpu_id, []))
+            if latest_memory is not None:
+                formatted_value, color = self._format_bytes(latest_memory)
+                table.add_row(
+                    f"GPU {gpu_id} memory", Text(formatted_value, style=color)
+                )
+
+            if idx != len(keys) - 1:
+                table.add_section()
+
+        ephemeral_storage = metrics.get("ephemeral_storage")
+        if ephemeral_storage:
+            if gpu_utilization or gpu_memory:
+                table.add_section()
+
+            usage_bytes = self._get_latest_metric(
+                ephemeral_storage.get("usage_bytes", [])
+            )
+            utilization = self._get_latest_metric(
+                ephemeral_storage.get("utilization", [])
+            )
+
+            if usage_bytes is not None:
+                formatted_value, color = self._format_bytes(usage_bytes)
+                table.add_row("Eph. storage usage", Text(formatted_value, style=color))
+
+            if utilization is not None:
+                utilization_percent = utilization * 100
+                if utilization_percent > 90:
+                    color = "red"
+                elif utilization_percent > 70:
+                    color = "yellow"
+                else:
+                    color = "green"
+                table.add_row(
+                    "Eph. storage utilization",
+                    Text(f"{utilization_percent:.1f}%", style=color),
+                )
+
+        return table
+
+    def _create_storage_tables(self, metrics_data: Dict) -> List[Table]:
+        """Create storage tables - only cache per job (ephemeral is now in node tables)"""
+        tables = []
+
+        # Create cache storage table (job-level, shown once)
+        cache_storage = metrics_data.get("cache")
+        if cache_storage:
+            table = self._create_cache_storage_table(cache_storage)
+            if table:
+                tables.append(table)
+
+        return tables
+
+    def _create_cache_storage_table(self, cache_storage: Dict) -> Optional[Table]:
+        """Create table for cache storage metrics (job-level)"""
+        usage_bytes = self._get_latest_metric(cache_storage.get("usage_bytes", []))
+        utilization = self._get_latest_metric(cache_storage.get("utilization", []))
+
+        if usage_bytes is None and utilization is None:
+            return None
+
+        table = Table(title="Cache storage")
+        table.add_column("Storage Type")
+        table.add_column("Usage")
+        table.add_column("Utilization")
+
+        self._maybe_format_storage_table_row(table, "Cache storage", cache_storage)
+
+        return table
+
+    def watch(self, refresh_rate: int = METRICS_POLL_INTERVAL_SEC):
+        """Display continuously updating metrics"""
+        self.before_polling()
+        with Live(auto_refresh=False) as live:
+            self.live = live
+            while True:
+                # our first instance of fetching metrics passes no explicit time range. We do this so that we can fetch metrics
+                # for inactive jobs, using the job's completion time to set the time range.
+                # Subsequent queries will fetch only the most recent data to avoid unnecessary load on VM
+                metrics = self.api.get_training_job_metrics(
+                    self.project_id, self.job_id
+                )
+                try:
+                    # range of one minute since we only want the last recording
+                    table = self.create_metrics_table(metrics)
+                    live.update(table, refresh=True)
+                    if not self.should_poll_again():
+                        live.stop()
+                        break
+                    time.sleep(refresh_rate)
+                    end_epoch_millis = int(time.time() * 1000)
+                    start_epoch_millis = end_epoch_millis - 60 * 1000
+                    metrics = self.api.get_training_job_metrics(
+                        self.project_id,
+                        self.job_id,
+                        end_epoch_millis=end_epoch_millis,
+                        start_epoch_millis=start_epoch_millis,
+                    )
+                    self.post_poll()
+                except Exception as e:
+                    live.stop()
+                    console.print(
+                        f"Error fetching metrics: {e}: {traceback.format_exc()}",
+                        style="red",
+                    )
+                    break
+        self.after_polling()

{truss-0.10.9rc514 → truss-0.10.9rc601}/truss/contexts/image_builder/cache_warmer.py

@@ -15,6 +15,8 @@ from botocore.exceptions import ClientError, NoCredentialsError
 from google.cloud import storage
 from huggingface_hub import hf_hub_download
 
+from truss.base import constants
+
 B10CP_PATH_TRUSS_ENV_VAR_NAME = "B10CP_PATH_TRUSS"
 
 GCS_CREDENTIALS = "/app/data/service_account.json"
@@ -108,7 +110,7 @@ class RepositoryFile(ABC):
 
 class HuggingFaceFile(RepositoryFile):
     def download_to_cache(self):
-        secret_path = Path("/etc/secrets/hf-access-token")
+        secret_path = Path(f"/etc/secrets/{constants.HF_ACCESS_TOKEN_FILE_NAME}")
         secret = secret_path.read_text().strip() if secret_path.exists() else None
         try:
             hf_hub_download(

{truss-0.10.9rc514 → truss-0.10.9rc601}/truss/contexts/image_builder/serving_image_builder.py

@@ -93,8 +93,6 @@ USER_TRUSS_IGNORE_FILE = ".truss_ignore"
 GCS_CREDENTIALS = "service_account.json"
 S3_CREDENTIALS = "s3_credentials.json"
 
-HF_ACCESS_TOKEN_FILE_NAME = "hf-access-token"
-
 CLOUD_BUCKET_CACHE = MODEL_CACHE_PATH
 
 HF_SOURCE_DIR = Path("./root/.cache/huggingface/hub/")
@@ -821,7 +819,7 @@ class ServingImageBuilder(ImageBuilder):
             model_cache_v1=config.model_cache.is_v1,
             model_cache_v2=config.model_cache.is_v2,
             hf_access_token=hf_access_token,
-            hf_access_token_file_name=HF_ACCESS_TOKEN_FILE_NAME,
+            hf_access_token_file_name=constants.HF_ACCESS_TOKEN_FILE_NAME,
             external_data_files=external_data_files,
             build_commands=build_commands,
             use_local_src=config.use_local_src,

{truss-0.10.9rc514 → truss-0.10.9rc601}/truss/templates/base.Dockerfile.jinja

@@ -8,30 +8,6 @@ FROM {{ base_image_name_and_tag }} AS truss_server
 {%- set python_executable = config.base_image.python_executable_path or 'python3' %}
 ENV PYTHON_EXECUTABLE="{{ python_executable }}"
 
-# Non-root user for the app.
-ENV APP_USERNAME app
-# We choose a high number for user ID to avoid conflicts with existing users.
-ENV APP_USER_UID 60000
-# Directories owned by the non-root user.
-# Directory containing inference server code.
-ENV APP_HOME /app
-# Directory containing control server code.
-ENV CONTROL_SERVER_DIR /control
-# The non-root user's home directory.
-ENV HOME /home/app
-# Create a non-root user to run model containers.
-RUN useradd -u ${APP_USER_UID} -ms /bin/bash ${APP_USERNAME} \
-    && mkdir ${APP_HOME} \
-    && mkdir ${CONTROL_SERVER_DIR} \
-    && chmod -R a=rwx ${APP_HOME} ${HOME} ${CONTROL_SERVER_DIR}
-ENV PATH=${PATH}:${HOME}/.local/bin
-# CAUTION: COPY directives use the root user, so we need to chown the directories to the non-root user.
-# If we have COPY directives after this, we need to run chown again. (or use COPY --chown ...)
-# RUN chown -R ${APP_USERNAME}:${APP_USERNAME} ${HOME} ${APP_HOME} ${CONTROL_SERVER_DIR}
-# Now switch to the non-root user early, we will switch back to root where needed.
-# USER ${APP_USERNAME}
-ENV PATH="${HOME}/.local/bin:$PATH"
-
 {%- set UV_VERSION = "0.7.19" %}
 {#
 NB(nikhil): We use a semi-complex uv installation command across the board:
@@ -63,6 +39,7 @@ RUN if ! command -v uv >/dev/null 2>&1; then \
     command -v curl >/dev/null 2>&1 || (apt update && apt install -y curl) && \
     curl -LsSf --retry 5 --retry-delay 5 https://astral.sh/uv/{{ UV_VERSION }}/install.sh | sh; \
 fi
+ENV PATH="/root/.local/bin:$PATH"
 {% endblock %}
 
 {% block base_image_patch %}

{truss-0.10.9rc514 → truss-0.10.9rc601}/truss/templates/server/model_wrapper.py

@@ -59,6 +59,7 @@ class MethodName(str, enum.Enum):
     CHAT_COMPLETIONS = enum.auto()
     COMPLETIONS = enum.auto()
     IS_HEALTHY = enum.auto()
+    MESSAGES = enum.auto()
     POSTPROCESS = enum.auto()
     PREDICT = enum.auto()
     PREPROCESS = enum.auto()
@@ -244,6 +245,7 @@ class ModelDescriptor:
     is_healthy: Optional[MethodDescriptor]
     completions: Optional[MethodDescriptor]
     chat_completions: Optional[MethodDescriptor]
+    messages: Optional[MethodDescriptor]
     websocket: Optional[MethodDescriptor]
 
     @cached_property
@@ -291,6 +293,7 @@ class ModelDescriptor:
         setup = cls._safe_extract_descriptor(model_cls, MethodName.SETUP_ENVIRONMENT)
         completions = cls._safe_extract_descriptor(model_cls, MethodName.COMPLETIONS)
         chats = cls._safe_extract_descriptor(model_cls, MethodName.CHAT_COMPLETIONS)
+        messages = cls._safe_extract_descriptor(model_cls, MethodName.MESSAGES)
         is_healthy = cls._safe_extract_descriptor(model_cls, MethodName.IS_HEALTHY)
         if is_healthy and is_healthy.arg_config != ArgConfig.NONE:
             raise errors.ModelDefinitionError(
@@ -359,6 +362,7 @@ class ModelDescriptor:
             is_healthy=is_healthy,
             completions=completions,
             chat_completions=chats,
+            messages=messages,
             websocket=websocket,
         )
 
@@ -925,6 +929,14 @@ class ModelWrapper:
         )
         return await self._execute_model_endpoint(inputs, request, descriptor)
 
+    async def messages(
+        self, inputs: InputType, request: starlette.requests.Request
+    ) -> OutputType:
+        descriptor = self._get_descriptor_or_raise(
+            self.model_descriptor.messages, MethodName.MESSAGES
+        )
+        return await self._execute_model_endpoint(inputs, request, descriptor)
+
     async def websocket(self, ws: WebSocket) -> None:
         descriptor = self.model_descriptor.websocket
         assert descriptor, "websocket can only be invoked if present on model."

{truss-0.10.9rc514 → truss-0.10.9rc601}/truss/templates/server/truss_server.py

@@ -231,6 +231,13 @@ class BasetenEndpoints:
             method=self._model.completions, request=request, body_raw=body_raw
         )
 
+    async def messages(
+        self, request: Request, body_raw: bytes = Depends(parse_body)
+    ) -> Response:
+        return await self._execute_request(
+            method=self._model.messages, request=request, body_raw=body_raw
+        )
+
     async def websocket(self, ws: WebSocket) -> None:
         self.check_healthy()
         trace_ctx = otel_propagate.extract(ws.headers) or None
@@ -428,6 +435,12 @@ class TrussServer:
                     methods=["POST"],
                     tags=["V1"],
                 ),
+                FastAPIRoute(
+                    r"/v1/messages",
+                    self._endpoints.messages,
+                    methods=["POST"],
+                    tags=["V1"],
+                ),
                 # Websocket endpoint
                 FastAPIWebSocketRoute(r"/v1/websocket", self._endpoints.websocket),
                 # Endpoint aliases for Sagemaker hosting

{truss-0.10.9rc514 → truss-0.10.9rc601}/truss/templates/server.Dockerfile.jinja

@@ -120,26 +120,16 @@ RUN mkdir -p {{ supervisor_log_dir }}
 COPY supervisord.conf {{ supervisor_config_path }}
 ENV SUPERVISOR_SERVER_URL="{{ supervisor_server_url }}"
 ENV SERVER_START_CMD="/docker_server/.venv/bin/supervisord -c {{ supervisor_config_path }}"
-RUN chown -R ${APP_USERNAME}:${APP_USERNAME} ${HOME} ${APP_HOME}
-{#- nginx writes to these directories #}
-{% set nginx_dirs = ["/var/lib/nginx", "/var/log/nginx", "/run"] %}
-RUN mkdir -p {{ nginx_dirs | join(" ") }} && \
-        chown -R ${APP_USERNAME}:${APP_USERNAME} {{ nginx_dirs | join(" ") }}
-USER ${APP_USERNAME}
 ENTRYPOINT ["/docker_server/.venv/bin/supervisord", "-c", "{{ supervisor_config_path }}"]
     {%- elif requires_live_reload %} {#- elif requires_live_reload #}
 ENV HASH_TRUSS="{{ truss_hash }}"
 ENV CONTROL_SERVER_PORT="8080"
 ENV INFERENCE_SERVER_PORT="8090"
 ENV SERVER_START_CMD="/control/.env/bin/python /control/control/server.py"
-RUN chown -R ${APP_USERNAME}:${APP_USERNAME} ${HOME} ${APP_HOME}
-USER ${APP_USERNAME}
 ENTRYPOINT ["/control/.env/bin/python", "/control/control/server.py"]
     {%- else %} {#- else (default inference server) #}
 ENV INFERENCE_SERVER_PORT="8080"
 ENV SERVER_START_CMD="{{ python_executable }} /app/main.py"
-RUN chown -R ${APP_USERNAME}:${APP_USERNAME} ${HOME} ${APP_HOME}
-USER ${APP_USERNAME}
 ENTRYPOINT ["{{ python_executable }}", "/app/main.py"]
     {%- endif %} {#- endif config.docker_server / live_reload #}
 {% endblock %} {#- endblock run #}

{truss-0.10.9rc514 → truss-0.10.9rc601}/truss/tests/cli/train/test_deploy_checkpoints.py

@@ -505,8 +505,16 @@ def test_render_vllm_full_truss_config():
     )
 
     result = render_vllm_full_truss_config(deploy_config)
-
-    expected_vllm_command = 'sh -c "HF_TOKEN=$(cat /secrets/hf_token) vllm serve /tmp/training_checkpoints/job123/rank-0/checkpoint-1 --port 8000 --tensor-parallel-size 2 --dtype bfloat16"'
+    expected_vllm_command = (
+        "sh -c 'HF_TOKEN=$(cat /secrets/hf_token) "
+        'HF_TOKEN="$$(cat /secrets/hf_access_token)" && export HF_TOKEN && '
+        "if [ -f /tmp/training_checkpoints/job123/rank-0/checkpoint-1/chat_template.jinja ]; then   "
+        "vllm serve /tmp/training_checkpoints/job123/rank-0/checkpoint-1 "
+        "--chat-template /tmp/training_checkpoints/job123/rank-0/checkpoint-1/chat_template.jinja   "
+        "--port 8000 --tensor-parallel-size 2 --dtype bfloat16; else   "
+        "vllm serve /tmp/training_checkpoints/job123/rank-0/checkpoint-1 "
+        "--port 8000 --tensor-parallel-size 2 --dtype bfloat16; fi'"
+    )
 
     assert isinstance(result, truss_config.TrussConfig)
     assert result.model_name == "test-full-model"