truss 0.10.9rc514__py3-none-any.whl → 0.10.9rc601__py3-none-any.whl

truss/base/constants.py

@@ -29,6 +29,7 @@ BEI_REQUIRED_MAX_NUM_TOKENS = 16384
 TRTLLM_MIN_MEMORY_REQUEST_GI = 10
 HF_MODELS_API_URL = "https://huggingface.co/api/models"
 HF_ACCESS_TOKEN_KEY = "hf_access_token"
+HF_ACCESS_TOKEN_FILE_NAME = "hf_access_token"
 TRUSSLESS_MAX_PAYLOAD_SIZE = "64M"
 # Alias for TEMPLATES_DIR
 SERVING_DIR: pathlib.Path = TEMPLATES_DIR

truss/cli/logs/base_watcher.py

@@ -9,7 +9,7 @@ from truss.remote.baseten.api import BasetenApi
 
 POLL_INTERVAL_SEC = 2
 # NB(nikhil): This helps account for (1) log processing delays (2) clock skews
-CLOCK_SKEW_BUFFER_MS = 10000
+CLOCK_SKEW_BUFFER_MS = 60000
 
 
 class LogWatcher(ABC):

truss/cli/train/deploy_checkpoints/deploy_full_checkpoints.py

@@ -14,12 +14,19 @@ from .deploy_checkpoints_helpers import (
     setup_environment_variables_and_secrets,
 )
 
+# NB(aghilan): Transformers was recently changed to save a chat_template.jinja file instead of inside the tokenizer_config.json file.
+# Old Models will not have this file, so we check for it and use it if it exists.
+# vLLM will not automatically resolve the chat_template.jinja file, so we need to pass it to the start command.
+# This logic is needed for any models trained using Transformers v4.51.3 or later
 VLLM_FULL_START_COMMAND = Template(
-    'sh -c "{%if envvars %}{{ envvars }} {% endif %}vllm serve {{ model_path }}'
-    + " --port 8000"
-    + " --tensor-parallel-size {{ specify_tensor_parallelism }}"
-    + " --dtype bfloat16"
-    + '"'
+    "sh -c '{% if envvars %}{{ envvars }} {% endif %}"
+    'HF_TOKEN="$$(cat /secrets/hf_access_token)" && export HF_TOKEN && '
+    "if [ -f {{ model_path }}/chat_template.jinja ]; then "
+    "  vllm serve {{ model_path }} --chat-template {{ model_path }}/chat_template.jinja "
+    "  --port 8000 --tensor-parallel-size {{ specify_tensor_parallelism }} --dtype bfloat16; "
+    "else "
+    "  vllm serve {{ model_path }} --port 8000 --tensor-parallel-size {{ specify_tensor_parallelism }} --dtype bfloat16; "
+    "fi'"
 )
 
 

truss/cli/train/metrics_watcher.py

@@ -4,6 +4,7 @@ import traceback
 from typing import Any, Dict, List, Optional, Tuple, cast
 
 from rich.columns import Columns
+from rich.layout import Layout
 from rich.live import Live
 from rich.table import Table
 from rich.text import Text
@@ -96,90 +97,200 @@ class MetricsWatcher(TrainingPollerMixin):
         )
         return True
 
-    def create_metrics_table(self, metrics_data: Dict) -> Columns:
+    def create_metrics_table(self, metrics_data: Dict) -> Layout:
         """Create a Rich table with the metrics"""
-        compute_table = self._create_compute_table(metrics_data)
-        storage_table = self._maybe_create_storage_table(metrics_data)
-        tables = [compute_table]
-        if storage_table:
-            tables.append(storage_table)
-        return Columns(tables, title="Training Job Metrics")
-
-    def _create_compute_table(self, metrics_data: Dict) -> Table:
-        table = Table(title="Compute Metrics")
-        table.add_column("Metric")
-        table.add_column("Value")
+        tables = []
+
+        timestamp = self._get_timestamp_from_metrics(metrics_data)
+
+        node_tables = self._create_unified_node_metrics_tables(metrics_data)
+        tables.extend(node_tables)
+
+        storage_tables = self._create_storage_tables(metrics_data)
+        tables.extend(storage_tables)
+
+        columns = Columns(tables, title="Training Job Metrics")
+
+        layout = Layout()
+
+        if timestamp:
+            from rich.panel import Panel
+
+            layout.split_column(
+                Layout(
+                    Panel(
+                        f"🕐 Last Updated: {timestamp}\n💡 Press Ctrl+C to exit",
+                        style="bold cyan",
+                    ),
+                    size=4,
+                ),
+                Layout(columns),
+            )
+        else:
+            layout.split_column(Layout(columns))
+
+        return layout
+
+    def _get_timestamp_from_metrics(self, metrics_data: Dict) -> Optional[str]:
+        """Extract timestamp from metrics data for display"""
+        # Try to get timestamp from per_node_metrics first. Fall back to main metrics if not there.
+        per_node_metrics = metrics_data.get("per_node_metrics", [])
+        if per_node_metrics and len(per_node_metrics) > 0:
+            first_node_metrics = per_node_metrics[0].get("metrics", {})
+            cpu_usage_data = first_node_metrics.get("cpu_usage", [])
+            if cpu_usage_data and len(cpu_usage_data) > 0:
+                timestamp = cpu_usage_data[-1].get("timestamp")
+                if timestamp:
+                    return common.format_localized_time(timestamp)
 
-        # Add timestamp if available
         cpu_usage_data = metrics_data.get("cpu_usage", [])
         if cpu_usage_data and len(cpu_usage_data) > 0:
-            latest_timestamp = cpu_usage_data[-1].get("timestamp")
-            # TODO: API result has missing timezone info.
-            if latest_timestamp:
-                table.add_row(
-                    "Timestamp", common.format_localized_time(latest_timestamp)
-                )
-                table.add_section()
+            timestamp = cpu_usage_data[-1].get("timestamp")
+            if timestamp:
+                return common.format_localized_time(timestamp)
+
+        return None
+
+    def _create_unified_node_metrics_tables(self, metrics_data: Dict) -> List[Table]:
+        """Create tables for node metrics, handling both single and multi-node scenarios"""
+        tables = []
+
+        per_node_metrics = metrics_data.get("per_node_metrics", [])
+
+        if not per_node_metrics:
+            # Job is likely just starting up - it takes some type for the
+            # the metrics to become available after the job starts running.
+            from rich.text import Text
+
+            waiting_table = Table(title="Training Job Status")
+            waiting_table.add_column("Status")
+            waiting_table.add_column("Message")
+
+            waiting_table.add_row(
+                "Status",
+                Text("⏳ Waiting for metrics to become available...", style="yellow"),
+            )
+            waiting_table.add_row(
+                "Note",
+                Text(
+                    "Metrics will appear once the training job starts running.",
+                    style="dim",
+                ),
+            )
+
+            tables.append(waiting_table)
+            return tables
+
+        for node_metrics in per_node_metrics:
+            node_id = node_metrics.get("node_id", "Unknown")
+            metrics = node_metrics.get("metrics", {})
+
+            if not metrics:
+                continue
 
-        # CPU metrics
-        cpu_usage = self._get_latest_metric(metrics_data.get("cpu_usage", []))
+            table = self._create_node_table(node_id, metrics)
+            tables.append(table)
+
+        return tables
+
+    def _create_node_table(self, node_id: str, metrics: Dict) -> Table:
+        """Create a table for a single node's metrics"""
+        table = Table(title=f"Node: {node_id}")
+        table.add_column("Metric")
+        table.add_column("Value")
+
+        cpu_usage = self._get_latest_metric(metrics.get("cpu_usage", []))
         if cpu_usage is not None:
-            table.add_row("CPU Usage", f"{cpu_usage:.2f} cores")
+            table.add_row("CPU usage", f"{cpu_usage:.2f} cores")
 
-        cpu_memory = self._get_latest_metric(
-            metrics_data.get("cpu_memory_usage_bytes", [])
-        )
+        cpu_memory = self._get_latest_metric(metrics.get("cpu_memory_usage_bytes", []))
         if cpu_memory is not None:
             formatted_value, color = self._format_bytes(cpu_memory)
-            table.add_row("CPU Memory", Text(formatted_value, style=color))
+            table.add_row("CPU memory", Text(formatted_value, style=color))
 
-        # Add separator after CPU metrics
-        table.add_section()
+        if cpu_usage is not None or cpu_memory is not None:
+            table.add_section()
 
-        # GPU metrics - grouped by GPU ID
-        gpu_metrics = metrics_data.get("gpu_utilization", {})
-        gpu_memory = metrics_data.get("gpu_memory_usage_bytes", {})
+        gpu_utilization = metrics.get("gpu_utilization", {})
+        gpu_memory = metrics.get("gpu_memory_usage_bytes", {})
 
-        for gpu_id in sorted(set(gpu_metrics.keys()) | set(gpu_memory.keys())):
-            # Add GPU utilization
-            latest_util = self._get_latest_metric(gpu_metrics.get(gpu_id, []))
+        # API should return same GPU IDs for utilization and memory
+        keys = gpu_utilization.keys()
+        for idx, gpu_id in enumerate(keys):
+            latest_util = self._get_latest_metric(gpu_utilization.get(gpu_id, []))
             if latest_util is not None:
-                table.add_row(f"GPU {gpu_id} Usage", f"{latest_util * 100:.1f}%")
+                table.add_row(f"GPU {gpu_id} utilization", f"{latest_util * 100:.1f}%")
 
-            # Add GPU memory right after its utilization
             latest_memory = self._get_latest_metric(gpu_memory.get(gpu_id, []))
             if latest_memory is not None:
                 formatted_value, color = self._format_bytes(latest_memory)
                 table.add_row(
-                    f"GPU {gpu_id} Memory", Text(formatted_value, style=color)
+                    f"GPU {gpu_id} memory", Text(formatted_value, style=color)
                 )
 
-            # Add separator after each GPU's metrics (except for the last one)
-            if gpu_id != max(set(gpu_metrics.keys()) | set(gpu_memory.keys())):
+            if idx != len(keys) - 1:
                 table.add_section()
 
-        # Add separator before storage metrics
-        if gpu_metrics or gpu_memory:
-            table.add_section()
-        return table
+        ephemeral_storage = metrics.get("ephemeral_storage")
+        if ephemeral_storage:
+            if gpu_utilization or gpu_memory:
+                table.add_section()
 
-    def _maybe_create_storage_table(self, metrics_data: Dict) -> Optional[Table]:
-        ephemeral_storage_metrics = metrics_data.get("ephemeral_storage")
-        cache_storage_metrics = metrics_data.get("cache")
-        if ephemeral_storage_metrics or cache_storage_metrics:
-            storage_table = Table(title="Storage Metrics")
-            storage_table.add_column("Storage Type")
-            storage_table.add_column("Usage")
-            storage_table.add_column("Utilization")
-            did_add_ephemeral = self._maybe_format_storage_table_row(
-                storage_table, "Ephemeral Storage", ephemeral_storage_metrics
+            usage_bytes = self._get_latest_metric(
+                ephemeral_storage.get("usage_bytes", [])
             )
-            did_add_cache = self._maybe_format_storage_table_row(
-                storage_table, "Cache Storage", cache_storage_metrics
+            utilization = self._get_latest_metric(
+                ephemeral_storage.get("utilization", [])
             )
-            if did_add_ephemeral or did_add_cache:
-                return storage_table
-        return None
+
+            if usage_bytes is not None:
+                formatted_value, color = self._format_bytes(usage_bytes)
+                table.add_row("Eph. storage usage", Text(formatted_value, style=color))
+
+            if utilization is not None:
+                utilization_percent = utilization * 100
+                if utilization_percent > 90:
+                    color = "red"
+                elif utilization_percent > 70:
+                    color = "yellow"
+                else:
+                    color = "green"
+                table.add_row(
+                    "Eph. storage utilization",
+                    Text(f"{utilization_percent:.1f}%", style=color),
+                )
+
+        return table
+
+    def _create_storage_tables(self, metrics_data: Dict) -> List[Table]:
+        """Create storage tables - only cache per job (ephemeral is now in node tables)"""
+        tables = []
+
+        # Create cache storage table (job-level, shown once)
+        cache_storage = metrics_data.get("cache")
+        if cache_storage:
+            table = self._create_cache_storage_table(cache_storage)
+            if table:
+                tables.append(table)
+
+        return tables
+
+    def _create_cache_storage_table(self, cache_storage: Dict) -> Optional[Table]:
+        """Create table for cache storage metrics (job-level)"""
+        usage_bytes = self._get_latest_metric(cache_storage.get("usage_bytes", []))
+        utilization = self._get_latest_metric(cache_storage.get("utilization", []))
+
+        if usage_bytes is None and utilization is None:
+            return None
+
+        table = Table(title="Cache storage")
+        table.add_column("Storage Type")
+        table.add_column("Usage")
+        table.add_column("Utilization")
+
+        self._maybe_format_storage_table_row(table, "Cache storage", cache_storage)
+
+        return table
 
     def watch(self, refresh_rate: int = METRICS_POLL_INTERVAL_SEC):
         """Display continuously updating metrics"""

truss/contexts/image_builder/cache_warmer.py

@@ -15,6 +15,8 @@ from botocore.exceptions import ClientError, NoCredentialsError
 from google.cloud import storage
 from huggingface_hub import hf_hub_download
 
+from truss.base import constants
+
 B10CP_PATH_TRUSS_ENV_VAR_NAME = "B10CP_PATH_TRUSS"
 
 GCS_CREDENTIALS = "/app/data/service_account.json"
@@ -108,7 +110,7 @@ class RepositoryFile(ABC):
 
 class HuggingFaceFile(RepositoryFile):
     def download_to_cache(self):
-        secret_path = Path("/etc/secrets/hf-access-token")
+        secret_path = Path(f"/etc/secrets/{constants.HF_ACCESS_TOKEN_FILE_NAME}")
         secret = secret_path.read_text().strip() if secret_path.exists() else None
         try:
             hf_hub_download(

truss/contexts/image_builder/serving_image_builder.py

@@ -93,8 +93,6 @@ USER_TRUSS_IGNORE_FILE = ".truss_ignore"
 GCS_CREDENTIALS = "service_account.json"
 S3_CREDENTIALS = "s3_credentials.json"
 
-HF_ACCESS_TOKEN_FILE_NAME = "hf-access-token"
-
 CLOUD_BUCKET_CACHE = MODEL_CACHE_PATH
 
 HF_SOURCE_DIR = Path("./root/.cache/huggingface/hub/")
@@ -821,7 +819,7 @@ class ServingImageBuilder(ImageBuilder):
             model_cache_v1=config.model_cache.is_v1,
             model_cache_v2=config.model_cache.is_v2,
             hf_access_token=hf_access_token,
-            hf_access_token_file_name=HF_ACCESS_TOKEN_FILE_NAME,
+            hf_access_token_file_name=constants.HF_ACCESS_TOKEN_FILE_NAME,
             external_data_files=external_data_files,
             build_commands=build_commands,
             use_local_src=config.use_local_src,

truss/templates/base.Dockerfile.jinja

@@ -8,30 +8,6 @@ FROM {{ base_image_name_and_tag }} AS truss_server
 {%- set python_executable = config.base_image.python_executable_path or 'python3' %}
 ENV PYTHON_EXECUTABLE="{{ python_executable }}"
 
-# Non-root user for the app.
-ENV APP_USERNAME app
-# We choose a high number for user ID to avoid conflicts with existing users.
-ENV APP_USER_UID 60000
-# Directories owned by the non-root user.
-# Directory containing inference server code.
-ENV APP_HOME /app
-# Directory containing control server code.
-ENV CONTROL_SERVER_DIR /control
-# The non-root user's home directory.
-ENV HOME /home/app
-# Create a non-root user to run model containers.
-RUN useradd -u ${APP_USER_UID} -ms /bin/bash ${APP_USERNAME} \
-    && mkdir ${APP_HOME} \
-    && mkdir ${CONTROL_SERVER_DIR} \
-    && chmod -R a=rwx ${APP_HOME} ${HOME} ${CONTROL_SERVER_DIR}
-ENV PATH=${PATH}:${HOME}/.local/bin
-# CAUTION: COPY directives use the root user, so we need to chown the directories to the non-root user.
-# If we have COPY directives after this, we need to run chown again. (or use COPY --chown ...)
-# RUN chown -R ${APP_USERNAME}:${APP_USERNAME} ${HOME} ${APP_HOME} ${CONTROL_SERVER_DIR}
-# Now switch to the non-root user early, we will switch back to root where needed.
-# USER ${APP_USERNAME}
-ENV PATH="${HOME}/.local/bin:$PATH"
-
 {%- set UV_VERSION = "0.7.19" %}
 {#
 NB(nikhil): We use a semi-complex uv installation command across the board:
@@ -63,6 +39,7 @@ RUN if ! command -v uv >/dev/null 2>&1; then \
     command -v curl >/dev/null 2>&1 || (apt update && apt install -y curl) && \
     curl -LsSf --retry 5 --retry-delay 5 https://astral.sh/uv/{{ UV_VERSION }}/install.sh | sh; \
 fi
+ENV PATH="/root/.local/bin:$PATH"
 {% endblock %}
 
 {% block base_image_patch %}

truss/templates/server/model_wrapper.py

@@ -59,6 +59,7 @@ class MethodName(str, enum.Enum):
     CHAT_COMPLETIONS = enum.auto()
     COMPLETIONS = enum.auto()
     IS_HEALTHY = enum.auto()
+    MESSAGES = enum.auto()
     POSTPROCESS = enum.auto()
     PREDICT = enum.auto()
     PREPROCESS = enum.auto()
@@ -244,6 +245,7 @@ class ModelDescriptor:
     is_healthy: Optional[MethodDescriptor]
     completions: Optional[MethodDescriptor]
     chat_completions: Optional[MethodDescriptor]
+    messages: Optional[MethodDescriptor]
     websocket: Optional[MethodDescriptor]
 
     @cached_property
@@ -291,6 +293,7 @@ class ModelDescriptor:
         setup = cls._safe_extract_descriptor(model_cls, MethodName.SETUP_ENVIRONMENT)
         completions = cls._safe_extract_descriptor(model_cls, MethodName.COMPLETIONS)
         chats = cls._safe_extract_descriptor(model_cls, MethodName.CHAT_COMPLETIONS)
+        messages = cls._safe_extract_descriptor(model_cls, MethodName.MESSAGES)
         is_healthy = cls._safe_extract_descriptor(model_cls, MethodName.IS_HEALTHY)
         if is_healthy and is_healthy.arg_config != ArgConfig.NONE:
             raise errors.ModelDefinitionError(
@@ -359,6 +362,7 @@ class ModelDescriptor:
             is_healthy=is_healthy,
             completions=completions,
             chat_completions=chats,
+            messages=messages,
             websocket=websocket,
         )
 
@@ -925,6 +929,14 @@ class ModelWrapper:
         )
         return await self._execute_model_endpoint(inputs, request, descriptor)
 
+    async def messages(
+        self, inputs: InputType, request: starlette.requests.Request
+    ) -> OutputType:
+        descriptor = self._get_descriptor_or_raise(
+            self.model_descriptor.messages, MethodName.MESSAGES
+        )
+        return await self._execute_model_endpoint(inputs, request, descriptor)
+
     async def websocket(self, ws: WebSocket) -> None:
         descriptor = self.model_descriptor.websocket
         assert descriptor, "websocket can only be invoked if present on model."

truss/templates/server/truss_server.py

@@ -231,6 +231,13 @@ class BasetenEndpoints:
             method=self._model.completions, request=request, body_raw=body_raw
         )
 
+    async def messages(
+        self, request: Request, body_raw: bytes = Depends(parse_body)
+    ) -> Response:
+        return await self._execute_request(
+            method=self._model.messages, request=request, body_raw=body_raw
+        )
+
     async def websocket(self, ws: WebSocket) -> None:
         self.check_healthy()
         trace_ctx = otel_propagate.extract(ws.headers) or None
@@ -428,6 +435,12 @@ class TrussServer:
                     methods=["POST"],
                     tags=["V1"],
                 ),
+                FastAPIRoute(
+                    r"/v1/messages",
+                    self._endpoints.messages,
+                    methods=["POST"],
+                    tags=["V1"],
+                ),
                 # Websocket endpoint
                 FastAPIWebSocketRoute(r"/v1/websocket", self._endpoints.websocket),
                 # Endpoint aliases for Sagemaker hosting

truss/templates/server.Dockerfile.jinja

@@ -120,26 +120,16 @@ RUN mkdir -p {{ supervisor_log_dir }}
 COPY supervisord.conf {{ supervisor_config_path }}
 ENV SUPERVISOR_SERVER_URL="{{ supervisor_server_url }}"
 ENV SERVER_START_CMD="/docker_server/.venv/bin/supervisord -c {{ supervisor_config_path }}"
-RUN chown -R ${APP_USERNAME}:${APP_USERNAME} ${HOME} ${APP_HOME}
-{#- nginx writes to these directories #}
-{% set nginx_dirs = ["/var/lib/nginx", "/var/log/nginx", "/run"] %}
-RUN mkdir -p {{ nginx_dirs | join(" ") }} && \
-        chown -R ${APP_USERNAME}:${APP_USERNAME} {{ nginx_dirs | join(" ") }}
-USER ${APP_USERNAME}
 ENTRYPOINT ["/docker_server/.venv/bin/supervisord", "-c", "{{ supervisor_config_path }}"]
     {%- elif requires_live_reload %} {#- elif requires_live_reload #}
 ENV HASH_TRUSS="{{ truss_hash }}"
 ENV CONTROL_SERVER_PORT="8080"
 ENV INFERENCE_SERVER_PORT="8090"
 ENV SERVER_START_CMD="/control/.env/bin/python /control/control/server.py"
-RUN chown -R ${APP_USERNAME}:${APP_USERNAME} ${HOME} ${APP_HOME}
-USER ${APP_USERNAME}
 ENTRYPOINT ["/control/.env/bin/python", "/control/control/server.py"]
     {%- else %} {#- else (default inference server) #}
 ENV INFERENCE_SERVER_PORT="8080"
 ENV SERVER_START_CMD="{{ python_executable }} /app/main.py"
-RUN chown -R ${APP_USERNAME}:${APP_USERNAME} ${HOME} ${APP_HOME}
-USER ${APP_USERNAME}
 ENTRYPOINT ["{{ python_executable }}", "/app/main.py"]
     {%- endif %} {#- endif config.docker_server / live_reload #}
 {% endblock %} {#- endblock run #}

truss/tests/cli/train/test_deploy_checkpoints.py

@@ -505,8 +505,16 @@ def test_render_vllm_full_truss_config():
     )
 
     result = render_vllm_full_truss_config(deploy_config)
-
-    expected_vllm_command = 'sh -c "HF_TOKEN=$(cat /secrets/hf_token) vllm serve /tmp/training_checkpoints/job123/rank-0/checkpoint-1 --port 8000 --tensor-parallel-size 2 --dtype bfloat16"'
+    expected_vllm_command = (
+        "sh -c 'HF_TOKEN=$(cat /secrets/hf_token) "
+        'HF_TOKEN="$$(cat /secrets/hf_access_token)" && export HF_TOKEN && '
+        "if [ -f /tmp/training_checkpoints/job123/rank-0/checkpoint-1/chat_template.jinja ]; then   "
+        "vllm serve /tmp/training_checkpoints/job123/rank-0/checkpoint-1 "
+        "--chat-template /tmp/training_checkpoints/job123/rank-0/checkpoint-1/chat_template.jinja   "
+        "--port 8000 --tensor-parallel-size 2 --dtype bfloat16; else   "
+        "vllm serve /tmp/training_checkpoints/job123/rank-0/checkpoint-1 "
+        "--port 8000 --tensor-parallel-size 2 --dtype bfloat16; fi'"
+    )
 
     assert isinstance(result, truss_config.TrussConfig)
     assert result.model_name == "test-full-model"

truss/tests/cli/train/test_train_cli_core.py

@@ -51,6 +51,38 @@ def test_view_training_job_metrics(time_sleep, capfd):
                 "0": [{"timestamp": "", "value": 4321}],
                 "1": [{"timestamp": "", "value": 2222}],
             },
+            "per_node_metrics": [
+                {
+                    "node_id": "node-0",
+                    "metrics": {
+                        "cpu_usage": [{"timestamp": "", "value": 3.2}],
+                        "cpu_memory_usage_bytes": [{"timestamp": "", "value": 1234}],
+                        "gpu_utilization": {
+                            "0": [{"timestamp": "", "value": 0.2}],
+                            "1": [{"timestamp": "", "value": 0.3}],
+                        },
+                        "gpu_memory_usage_bytes": {
+                            "0": [{"timestamp": "", "value": 4321}],
+                            "1": [{"timestamp": "", "value": 2222}],
+                        },
+                    },
+                },
+                {
+                    "node_id": "node-1",
+                    "metrics": {
+                        "cpu_usage": [{"timestamp": "", "value": 2.8}],
+                        "cpu_memory_usage_bytes": [{"timestamp": "", "value": 1000}],
+                        "gpu_utilization": {
+                            "0": [{"timestamp": "", "value": 0.15}],
+                            "1": [{"timestamp": "", "value": 0.25}],
+                        },
+                        "gpu_memory_usage_bytes": {
+                            "0": [{"timestamp": "", "value": 4000}],
+                            "1": [{"timestamp": "", "value": 2000}],
+                        },
+                    },
+                },
+            ],
         },
         {
             "training_job": {
@@ -68,6 +100,38 @@ def test_view_training_job_metrics(time_sleep, capfd):
                 "0": [{"timestamp": "", "value": 4321}],
                 "1": [{"timestamp": "", "value": 2222}],
             },
+            "per_node_metrics": [
+                {
+                    "node_id": "node-0",
+                    "metrics": {
+                        "cpu_usage": [{"timestamp": "", "value": 3.2}],
+                        "cpu_memory_usage_bytes": [{"timestamp": "", "value": 1234}],
+                        "gpu_utilization": {
+                            "0": [{"timestamp": "", "value": 0.2}],
+                            "1": [{"timestamp": "", "value": 0.3}],
+                        },
+                        "gpu_memory_usage_bytes": {
+                            "0": [{"timestamp": "", "value": 4321}],
+                            "1": [{"timestamp": "", "value": 2222}],
+                        },
+                    },
+                },
+                {
+                    "node_id": "node-1",
+                    "metrics": {
+                        "cpu_usage": [{"timestamp": "", "value": 2.8}],
+                        "cpu_memory_usage_bytes": [{"timestamp": "", "value": 1000}],
+                        "gpu_utilization": {
+                            "0": [{"timestamp": "", "value": 0.15}],
+                            "1": [{"timestamp": "", "value": 0.25}],
+                        },
+                        "gpu_memory_usage_bytes": {
+                            "0": [{"timestamp": "", "value": 4000}],
+                            "1": [{"timestamp": "", "value": 2000}],
+                        },
+                    },
+                },
+            ],
         },
         {
             "training_job": {
@@ -85,6 +149,38 @@ def test_view_training_job_metrics(time_sleep, capfd):
                 "0": [{"timestamp": "", "value": 4321}],
                 "1": [{"timestamp": "", "value": 2222}],
             },
+            "per_node_metrics": [
+                {
+                    "node_id": "node-0",
+                    "metrics": {
+                        "cpu_usage": [{"timestamp": "", "value": 3.2}],
+                        "cpu_memory_usage_bytes": [{"timestamp": "", "value": 1234}],
+                        "gpu_utilization": {
+                            "0": [{"timestamp": "", "value": 0.2}],
+                            "1": [{"timestamp": "", "value": 0.3}],
+                        },
+                        "gpu_memory_usage_bytes": {
+                            "0": [{"timestamp": "", "value": 4321}],
+                            "1": [{"timestamp": "", "value": 2222}],
+                        },
+                    },
+                },
+                {
+                    "node_id": "node-1",
+                    "metrics": {
+                        "cpu_usage": [{"timestamp": "", "value": 2.8}],
+                        "cpu_memory_usage_bytes": [{"timestamp": "", "value": 1000}],
+                        "gpu_utilization": {
+                            "0": [{"timestamp": "", "value": 0.15}],
+                            "1": [{"timestamp": "", "value": 0.25}],
+                        },
+                        "gpu_memory_usage_bytes": {
+                            "0": [{"timestamp": "", "value": 4000}],
+                            "1": [{"timestamp": "", "value": 2002}],
+                        },
+                    },
+                },
+            ],
         },
     ]
 

truss/tests/contexts/image_builder/test_serving_image_builder.py

@@ -10,6 +10,7 @@ import pytest
 import yaml
 
 from truss.base.constants import (
+    HF_ACCESS_TOKEN_FILE_NAME,
     TRTLLM_BASE_IMAGE,
     TRTLLM_PREDICT_CONCURRENCY,
     TRTLLM_PYTHON_EXECUTABLE,
@@ -17,7 +18,6 @@ from truss.base.constants import (
 )
 from truss.base.truss_config import ModelCache, ModelRepo, TrussConfig
 from truss.contexts.image_builder.serving_image_builder import (
-    HF_ACCESS_TOKEN_FILE_NAME,
     ServingImageBuilderContext,
     get_files_to_model_cache_v1,
 )

truss/tests/test_data/test_openai/model/model.py

@@ -13,3 +13,6 @@ class Model:
 
     def predict(self, input: Dict) -> str:
         return "predict"
+
+    def messages(self, input: Dict) -> str:
+        return "messages"

truss/truss_handle/truss_handle.py

@@ -106,6 +106,7 @@ class DockerURLs:
         self.predict_url = f"{base_url}/v1/models/model:predict"
         self.completions_url = f"{base_url}/v1/completions"
         self.chat_completions_url = f"{base_url}/v1/chat/completions"
+        self.messages_url = f"{base_url}/v1/messages"
 
         self.schema_url = f"{base_url}/v1/models/model/schema"
         self.metrics_url = f"{base_url}/metrics"

{truss-0.10.9rc514.dist-info → truss-0.10.9rc601.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: truss
-Version: 0.10.9rc514
+Version: 0.10.9rc601
 Summary: A seamless bridge from model development to model delivery
 Project-URL: Repository, https://github.com/basetenlabs/truss
 Project-URL: Homepage, https://truss.baseten.co

{truss-0.10.9rc514.dist-info → truss-0.10.9rc601.dist-info}/RECORD

@@ -2,7 +2,7 @@ truss/__init__.py,sha256=CoUcP6vx_pocyemRmpbCPlndkHhdMkABAlr0ZXVuPCk,1163
 truss/api/__init__.py,sha256=spBAa_m1pItiid97iDLKPmumgAkSirPkv-E8RWMZyOk,5090
 truss/api/definitions.py,sha256=QAaIBqL59Q-R7HtLcXcoeCIWBN2HqOzApdFX0PpCq2s,1604
 truss/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-truss/base/constants.py,sha256=qwNNkd9EOAuiTxYLVccJaiPCNRayBAFvyj_GisYOT3I,3488
+truss/base/constants.py,sha256=ohZalB5yu9yoCIMSEEe5GF4swNDBVj2dAjG39vktW-s,3534
 truss/base/custom_types.py,sha256=FUSIT2lPOQb6gfg6IzT63YBV8r8L6NIZ0D74Fp3e_jQ,2835
 truss/base/errors.py,sha256=zDVLEvseTChdPP0oNhBBQCtQUtZJUaof5zeWMIjqz6o,691
 truss/base/trt_llm_config.py,sha256=CRz3AqGDAyv8YpcBWXUrnfjvNAauyo3yf8ZOGVsSt6g,32782
@@ -12,29 +12,29 @@ truss/cli/chains_commands.py,sha256=y6pdIAGCcKOPG9bPuCXPfSA0onQm5x-tT_3blSBfPYg,
 truss/cli/cli.py,sha256=PaMkuwXZflkU7sa1tEoT_Zmy-iBkEZs1m4IVqcieaeo,30367
 truss/cli/remote_cli.py,sha256=G_xCKRXzgkCmkiZJhUFfsv5YSVgde1jLA5LPQitpZgI,1905
 truss/cli/train_commands.py,sha256=VPwlyfCUumRpwN1m9hkBxHKqobUmIbeQnfqnsl6_uPo,12078
-truss/cli/logs/base_watcher.py,sha256=1qu0bKLKSB7aVociYrdpJD_ItjW8C1t5P6zr1Ku87YU,2785
+truss/cli/logs/base_watcher.py,sha256=KKyd7lIrdaEeDVt8EtjMioSPGVpLyOcF0ewyzE_GGdQ,2785
 truss/cli/logs/model_log_watcher.py,sha256=NACcP-wkcaroYa2Cb9BZC7Yr0554WZa_FSM2LXOf4A8,1263
 truss/cli/logs/training_log_watcher.py,sha256=r6HRqrLnz-PiKTUXiDYYxg4ZnP8vYcXlEX1YmgHhzlo,1173
 truss/cli/logs/utils.py,sha256=z-U_FG4BUzdZLbE3BnXb4DZQ0zt3LSZ3PiQpLaDuc3o,1031
 truss/cli/train/common.py,sha256=Es1yllSYxjM9x2uBzTGbYwyd8ML66cqqge0XO8_G_X0,992
 truss/cli/train/core.py,sha256=MBOhPSVYOU7wVh09uWQrJDEVOhJQug_2Odv3u6tCVTA,13855
 truss/cli/train/deploy_from_checkpoint_config.yml,sha256=mktaVrfhN8Kjx1UveC4xr-gTW-kjwbHvq6bx_LpO-Wg,371
-truss/cli/train/metrics_watcher.py,sha256=iDD06zt7ze6thy9APE-SlyHwUdetWmbKbN7NUp1jE1U,9128
+truss/cli/train/metrics_watcher.py,sha256=ftrLQ5m7V1lAqcAvdGbMv5r0aF4D0lypfKjokCBQvLw,12798
 truss/cli/train/poller.py,sha256=TGRzELxsicga0bEXewSX1ujw6lfPmDnHd6nr8zvOFO8,3550
 truss/cli/train/types.py,sha256=alGtr4Q71GeB65PpGMhsoKygw4k_ncR6MKIP1ioP8rI,951
 truss/cli/train/deploy_checkpoints/__init__.py,sha256=wL-M2yu8PxO2tFvjwshXAfPnB-5TlvsBp2v_bdzimRU,99
 truss/cli/train/deploy_checkpoints/deploy_checkpoints.py,sha256=wWFCpaMjM868-f_ChECrGHyMHka1F2VS1pEKPcEw3eM,16667
 truss/cli/train/deploy_checkpoints/deploy_checkpoints_helpers.py,sha256=7CdYgsxDF7nHITyCGpjjTMeaaOvtlCYwF7NSxpKedS0,1723
-truss/cli/train/deploy_checkpoints/deploy_full_checkpoints.py,sha256=pFj7rDcnvb9C4MMfr3wc4aBXfziqFkzw0H883NtQ1Es,3245
+truss/cli/train/deploy_checkpoints/deploy_full_checkpoints.py,sha256=onY-Xk6n1M5H_QGcV47LhiCq1f9p4bOLkHLTJzH8MnI,3970
 truss/cli/train/deploy_checkpoints/deploy_lora_checkpoints.py,sha256=P91dIAzuhl2GlzmrWwCcYI7uCMT1Lm7C79JQHM_exN4,4442
 truss/cli/utils/common.py,sha256=aWnla4qMSEz57dRMTl7R-EaScsuEpnQUeziGUaIeqeU,6149
 truss/cli/utils/output.py,sha256=GNjU85ZAMp5BI6Yij5wYXcaAvpm_kmHV0nHNmdkMxb0,646
 truss/cli/utils/self_upgrade.py,sha256=eTJZA4Wc8uUp4Qh6viRQp6bZm--wnQp7KWe5KRRpPtg,5427
 truss/contexts/docker_build_setup.py,sha256=cF4ExZgtYvrWxvyCAaUZUvV_DB_7__MqVomUDpalvKo,3925
 truss/contexts/truss_context.py,sha256=uS6L-ACHxNk0BsJwESOHh1lA0OGGw0pb33aFKGsASj4,436
-truss/contexts/image_builder/cache_warmer.py,sha256=TGMV1Mh87n2e_dSowH0sf0rZhZraDOR-LVapZL3a5r8,7377
+truss/contexts/image_builder/cache_warmer.py,sha256=wKE5zE3efnEpjmiLqAtQZX1pb2z57aRU0uElOxxb5f4,7434
 truss/contexts/image_builder/image_builder.py,sha256=IuRgDeeoHVLzIkJvKtX3807eeqEyaroCs_KWDcIHZUg,1461
-truss/contexts/image_builder/serving_image_builder.py,sha256=21a25Dea0ZOEgs41PKmhXjzEZJr1AuvXgcQBPZi1g4s,33923
+truss/contexts/image_builder/serving_image_builder.py,sha256=NdjlQDZqKsmFDXx1UqL2HszL4ozPuNC6MFoTbXB5l0w,33886
 truss/contexts/image_builder/util.py,sha256=y2-CjUKv0XV-0w2sr1fUCflysDJLsoU4oPp6tvvoFnk,1203
 truss/contexts/local_loader/docker_build_emulator.py,sha256=rmf7I28zksSmHjwvJMx2rIa6xK4KeR5fBm5YFth_fQg,2464
 truss/contexts/local_loader/dockerfile_parser.py,sha256=GoRJ0Af_3ILyLhjovK5lrCGn1rMxz6W3l681ro17ZzI,1344
@@ -63,12 +63,12 @@ truss/remote/baseten/utils/tar.py,sha256=pMUv--YkwXDngUx1WUOK-KmAIKMcOg2E-CD5x4h
 truss/remote/baseten/utils/transfer.py,sha256=d3VptuQb6M1nyS6kz0BAfeOYDLkMKUjatJXpY-mp-As,1548
 truss/templates/README.md.jinja,sha256=N7CJdyldZuJamj5jLh47le0hFBdu9irVsTBqoxhPNPQ,2476
 truss/templates/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-truss/templates/base.Dockerfile.jinja,sha256=AXEKM5XyBW5qndMxYQpnz9ldVU1F5ox9uZCqtjkabAI,5084
+truss/templates/base.Dockerfile.jinja,sha256=vFAJH1lC9jg90-076H2DCmkXUAlpseitIN6c4UwagxA,4020
 truss/templates/cache.Dockerfile.jinja,sha256=LhsVP9F3BATKQGkgya_YT4v6ABTUkpy-Jb3N36zsw10,1030
 truss/templates/cache_requirements.txt,sha256=xoPoJ-OVnf1z6oq_RVM3vCr3ionByyqMLj7wGs61nUs,87
 truss/templates/copy_cache_files.Dockerfile.jinja,sha256=arHldnuclt7vUFHyRz6vus5NGMDkIofm-1RU37A0xZM,98
 truss/templates/docker_server_requirements.txt,sha256=PyhOPKAmKW1N2vLvTfLMwsEtuGpoRrbWuNo7tT6v2Mc,18
-truss/templates/server.Dockerfile.jinja,sha256=-WZ7a4OoV65_25gvvWAZ6_cf5vEENjSPkq59KcRyi0Q,6486
+truss/templates/server.Dockerfile.jinja,sha256=DkNyshoLWISSb9k4s1SJoT8LycPHjUI0EaKhKaCh5Rg,5994
 truss/templates/control/requirements.txt,sha256=Kk0tYID7trPk5gwX38Wrt2-YGWZAXFJCJRcqJ8ZzCjc,251
 truss/templates/control/control/application.py,sha256=jYeta6hWe1SkfLL3W4IDmdYjg3ZuKqI_UagWYs5RB_E,3793
 truss/templates/control/control/endpoints.py,sha256=FM-sgao7I3gMoUTasM3Xq_g2LDoJQe75JxIoaQxzeNo,10031
@@ -92,9 +92,9 @@ truss/templates/docker_server/proxy.conf.jinja,sha256=Lg-PcZzKflG85exZKHNgW_I6r0
 truss/templates/docker_server/supervisord.conf.jinja,sha256=CoaSLv0Lr8t1tS_q102IFufNX2lWrlbCHJLjMhYjOwM,1711
 truss/templates/server/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 truss/templates/server/main.py,sha256=kWXrdD8z8IpamyWxc8qcvd5ck9gM1Kz2QH5qHJCnmOQ,222
-truss/templates/server/model_wrapper.py,sha256=k75VVISwwlsx5EGb82UZsu8kCM_i6Yi3-Hd0-Kpm1yo,42055
+truss/templates/server/model_wrapper.py,sha256=3xrEVKdFMCEnBJ4sZB14Vt_kwGoBxUibkFlCfonBOTE,42554
 truss/templates/server/requirements.txt,sha256=iRR2BEpBQnt-YOiTEKOnaab7tlR4C23V1cuURuIt7ZY,672
-truss/templates/server/truss_server.py,sha256=ob_nceeGtFPZzKKdk_ZZGLoZrJOGE6hR52xM1sPR97A,19498
+truss/templates/server/truss_server.py,sha256=FuouBLIQZ3-lsW_FGunY-S7hJ9KUNuqjLMOwYKVx8mE,19945
 truss/templates/server/common/__init__.py,sha256=qHIqr68L5Tn4mV6S-PbORpcuJ4jmtBR8aCuRTIWDvNo,85
 truss/templates/server/common/errors.py,sha256=qWeZlmNI8ZGbZbOIp_mtS6IKvUFIzhj3QH8zp-xTp9o,8554
 truss/templates/server/common/patches.py,sha256=uEOzvDnXsHOkTSa8zygGYuR4GHhrFNVHNQc5peJcwvo,1393
@@ -136,10 +136,10 @@ truss/tests/test_truss_gatherer.py,sha256=bn288OEkC49YY0mhly4cAl410ktZPfElNdWwZy
 truss/tests/test_truss_handle.py,sha256=-xz9VXkecXDTslmQZ-dmUmQLnvD0uumRqHS2uvGlMBA,30750
 truss/tests/test_util.py,sha256=hs1bNMkXKEdoPRx4Nw-NAEdoibR92OubZuADGmbiYsQ,1344
 truss/tests/cli/test_cli.py,sha256=yfbVS5u1hnAmmA8mJ539vj3lhH-JVGUvC4Q_Mbort44,787
-truss/tests/cli/train/test_deploy_checkpoints.py,sha256=pPTqyGqvSsrsEHXguo6swLXkVObytLdupfSElqlujPc,25733
-truss/tests/cli/train/test_train_cli_core.py,sha256=ONI4Oc1R0pL-M3WuNsnnHLzqEfMTE0u9g5dE9Ii9yW8,3424
+truss/tests/cli/train/test_deploy_checkpoints.py,sha256=yCPE5S4D_92hz692QKYDaHvwye4imuz2e1pd1K1pXkE,26203
+truss/tests/cli/train/test_train_cli_core.py,sha256=T1Xa6-NRk2nTJGX6sXaA8x4qCwL3Ini72PBI2gW7rYM,7879
 truss/tests/cli/train/resources/test_deploy_from_checkpoint_config.yml,sha256=GF7r9l0KaeXiUYCPSBpeMPd2QG6PeWWyI12NdbqLOgc,1930
-truss/tests/contexts/image_builder/test_serving_image_builder.py,sha256=iJA7nxcLXhBmyjhLIKeN64ql0OI_R53l-qSt3SsENV8,22368
+truss/tests/contexts/image_builder/test_serving_image_builder.py,sha256=ycOxhGIfe5OVcgqZqJvn1Ca2368AN1KdNmIO0vSZ4ko,22368
 truss/tests/contexts/local_loader/test_load_local.py,sha256=D1qMH2IpYA2j5009v50QMgUnKdeOsX15ndkwXe10a4E,801
 truss/tests/contexts/local_loader/test_truss_module_finder.py,sha256=oN1K2lg3ATHY5yOVUTfQIaSqusTF9I2wFaYaTSo5-O4,5342
 truss/tests/local/test_local_config_handler.py,sha256=aLvcOyfppskA2MziVLy_kMcagjxMpO4mjar9zxUN6g0,2245
@@ -236,7 +236,7 @@ truss/tests/test_data/test_go_custom_server_truss/docker/main.go,sha256=WR3mJU1o
 truss/tests/test_data/test_openai/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 truss/tests/test_data/test_openai/config.yaml,sha256=ByY_Smgx0lw24Yj0hqgofEmL3nrGNj7gZE5iBKlvwxk,235
 truss/tests/test_data/test_openai/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-truss/tests/test_data/test_openai/model/model.py,sha256=GEtIJnWlU1snBid2sS-bZHrjQpP8UzL8tanzyH_tdgE,319
+truss/tests/test_data/test_openai/model/model.py,sha256=NMyZH6QcJv4TMw2Cd8M02wvmwDqlWJB4ZwtD4n8QU0Y,390
 truss/tests/test_data/test_pyantic_v1/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 truss/tests/test_data/test_pyantic_v1/config.yaml,sha256=fqWpH3E4UPEnjvAw6Q9_F5oZZLy69RAfycbgtmCFsXo,270
 truss/tests/test_data/test_pyantic_v1/requirements.txt,sha256=OpG4JAdJME9VWjoNftdHYg-y94k2gbhqdM1_NwOgcT8,13
@@ -316,7 +316,7 @@ truss/truss_handle/build.py,sha256=BKFV-S57tnWcfRffvQ7SPp78BrjmRy3GhgF6ThaIrDM,3
 truss/truss_handle/decorators.py,sha256=PUR5w2rl_cvcsVtAUpcYLzNXuOml9R0-wtpXy-9hDPk,407
 truss/truss_handle/readme_generator.py,sha256=B4XbGwUjzMNOr71DWNAL8kCu5_ZHq7YOM8yVGaOZMSE,716
 truss/truss_handle/truss_gatherer.py,sha256=Xysl_UnCVhehPfZeHa8p7WFp94ENqh-VVpbuqnCui3A,2870
-truss/truss_handle/truss_handle.py,sha256=WF2MQSly9DQ1SoAvqfi87Ulu4llTadpXoncsDjpL79E,40886
+truss/truss_handle/truss_handle.py,sha256=pGsrsfCSEa5DmrFAoA5Nxr9TBke0Dr2eaq81Ht82j9U,40940
 truss/truss_handle/patch/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 truss/truss_handle/patch/calc_patch.py,sha256=Qyk1QmacK4jy9Ia8-93L8VtAWJhw15z22DdZUkBKlys,18334
 truss/truss_handle/patch/constants.py,sha256=pCEi5Pwi8Rnqthrr3VEsWL9EP1P1VV1T8DEYuitHLmc,139
@@ -361,8 +361,8 @@ truss_train/definitions.py,sha256=yFQYJoxK2tDBeKFHR-IJz12jU1CtWRXN-ZERh9zjMHo,66
 truss_train/deployment.py,sha256=zmeJ66kg1Wc7l7bwA_cXqv85uMF77hYl7NPHuhc1NPs,2493
 truss_train/loader.py,sha256=0o66EjBaHc2YY4syxxHVR4ordJWs13lNXnKjKq2wq0U,1630
 truss_train/public_api.py,sha256=9N_NstiUlmBuLUwH_fNG_1x7OhGCytZLNvqKXBlStrM,1220
-truss-0.10.9rc514.dist-info/METADATA,sha256=6LiDk-raxa4I2VayuLXZuzc42A3VTETrJfR2W98ZlH8,6674
-truss-0.10.9rc514.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-truss-0.10.9rc514.dist-info/entry_points.txt,sha256=-MwKfHHQHQ6j0HqIgvxrz3CehCmczDLTD-OsRHnjjuU,130
-truss-0.10.9rc514.dist-info/licenses/LICENSE,sha256=FTqGzu85i-uw1Gi8E_o0oD60bH9yQ_XIGtZbA1QUYiw,1064
-truss-0.10.9rc514.dist-info/RECORD,,
+truss-0.10.9rc601.dist-info/METADATA,sha256=lVGZYss-iZy1X7xth17tJozGiJKzPr6cpFKm9kb-7qY,6674
+truss-0.10.9rc601.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+truss-0.10.9rc601.dist-info/entry_points.txt,sha256=-MwKfHHQHQ6j0HqIgvxrz3CehCmczDLTD-OsRHnjjuU,130
+truss-0.10.9rc601.dist-info/licenses/LICENSE,sha256=FTqGzu85i-uw1Gi8E_o0oD60bH9yQ_XIGtZbA1QUYiw,1064
+truss-0.10.9rc601.dist-info/RECORD,,