truss 0.10.9rc535__py3-none-any.whl → 0.10.10rc0__py3-none-any.whl

truss/remote/baseten/api.py

@@ -669,6 +669,17 @@ class BasetenApi:
         # NB(nikhil): reverse order so latest logs are at the end
         return resp_json["logs"][::-1]
 
+    def _fetch_log_batch(
+        self, project_id: str, job_id: str, query_params: Dict[str, Any]
+    ) -> List[Any]:
+        """
+        Fetch a single batch of logs from the API.
+        """
+        resp_json = self._rest_api_client.post(
+            f"v1/training_projects/{project_id}/jobs/{job_id}/logs", body=query_params
+        )
+        return resp_json["logs"]
+
     def get_training_job_checkpoint_presigned_url(
         self, project_id: str, job_id: str, page_size: int = 100
     ) -> List[Dict[str, str]]:

truss/remote/baseten/core.py

@@ -3,7 +3,9 @@ import json
 import logging
 import pathlib
 import textwrap
-from typing import IO, TYPE_CHECKING, List, NamedTuple, Optional, Tuple, Type
+from typing import IO, TYPE_CHECKING, Any, Dict, List, NamedTuple, Optional, Tuple, Type
+
+import requests
 
 from truss.base.errors import ValidationError
 
@@ -15,6 +17,7 @@ from truss.remote.baseten import custom_types as b10_types
 from truss.remote.baseten.api import BasetenApi
 from truss.remote.baseten.error import ApiError
 from truss.remote.baseten.utils.tar import create_tar_with_progress_bar
+from truss.remote.baseten.utils.time import iso_to_millis
 from truss.remote.baseten.utils.transfer import multipart_upload_boto3
 from truss.util.path import load_trussignore_patterns_from_truss_dir
 
@@ -27,6 +30,16 @@ NO_ENVIRONMENTS_EXIST_ERROR_MESSAGING = (
     "Model hasn't been deployed yet. No environments exist."
 )
 
+# Maximum number of iterations to prevent infinite loops when paginating logs
+MAX_ITERATIONS = 10_000
+MIN_BATCH_SIZE = 100
+
+# LIMIT for the number of logs to fetch per request defined by the server
+MAX_BATCH_SIZE = 1000
+
+NANOSECONDS_PER_MILLISECOND = 1_000_000
+MILLISECONDS_PER_HOUR = 60 * 60 * 1000
+
 
 class ModelIdentifier:
     value: str
@@ -465,3 +478,198 @@ def validate_truss_config_against_backend(api: BasetenApi, config: str):
             raise ValidationError(
                 f"Validation failed with the following errors:\n{error_messages}"
             )
+
+
+def _build_log_query_params(
+    start_time: Optional[int], end_time: Optional[int], batch_size: int
+) -> Dict[str, Any]:
+    """
+    Build query parameters for log fetching request.
+
+    Args:
+        start_time: Start time in milliseconds since epoch
+        end_time: End time in milliseconds since epoch
+        batch_size: Number of logs to fetch per request
+
+    Returns:
+        Dictionary of query parameters with None values removed
+    """
+    query_body = {
+        "start_epoch_millis": start_time,
+        "end_epoch_millis": end_time,
+        "limit": batch_size,
+        "direction": "asc",
+    }
+
+    return {k: v for k, v in query_body.items() if v is not None}
+
+
+def _handle_server_error_backoff(
+    error: requests.HTTPError, job_id: str, iteration: int, batch_size: int
+) -> int:
+    """
+    Slash the batch size in half and return the new batch size
+    """
+    old_batch_size = batch_size
+    new_batch_size = max(batch_size // 2, MIN_BATCH_SIZE)
+
+    logging.warning(
+        f"Server error (HTTP {error.response.status_code}) for job {job_id} at iteration {iteration}. "
+        f"Reducing batch size from {old_batch_size} to {new_batch_size}. Retrying..."
+    )
+
+    return new_batch_size
+
+
+def _process_batch_logs(
+    batch_logs: List[Any], job_id: str, iteration: int, batch_size: int
+) -> Tuple[bool, Optional[int], Optional[int]]:
+    """
+    Process a batch of logs and determine if pagination should continue.
+
+    Args:
+        batch_logs: List of logs from the current batch
+        job_id: The job ID for logging
+        iteration: Current iteration number for logging
+        batch_size: Expected batch size
+
+    Returns:
+        Tuple of (should_continue, next_start_time, next_end_time)
+    """
+
+    # If no logs returned, we're done
+    if not batch_logs:
+        logging.info(f"No logs returned for job {job_id} at iteration {iteration}")
+        return False, None, None
+
+    # If we got fewer logs than the batch size, we've reached the end
+    if len(batch_logs) == 0:
+        logging.info(f"Reached end of logs for job {job_id} at iteration {iteration}")
+        return False, None, None
+
+    # Timestamp returned in nanoseconds for the last log in this batch converted
+    # to milliseconds to use as start for next iteration
+    last_log_timestamp = int(batch_logs[-1]["timestamp"]) // NANOSECONDS_PER_MILLISECOND
+
+    # Update start time for next iteration (add 1ms to avoid overlap)
+    next_start_time_ms = last_log_timestamp + 1
+
+    # Set end time to 2 hours from next start time, maximum time delta allowed by the API
+    next_end_time_ms = next_start_time_ms + 2 * MILLISECONDS_PER_HOUR
+
+    return True, next_start_time_ms, next_end_time_ms
+
+
+class BatchedTrainingLogsFetcher:
+    """
+    Iterator for fetching training job logs in batches using time-based pagination.
+
+    This iterator handles the complexity of paginating through training job logs,
+    including error handling, batch size adjustment, and time window management.
+    """
+
+    def __init__(
+        self,
+        api: BasetenApi,
+        project_id: str,
+        job_id: str,
+        batch_size: int = MAX_BATCH_SIZE,
+    ):
+        self.api = api
+        self.project_id = project_id
+        self.job_id = job_id
+        self.batch_size = batch_size
+        self.iteration = 0
+        self.current_start_time = None
+        self.current_end_time = None
+        self._initialize_time_window()
+
+    def _initialize_time_window(self):
+        training_job = self.api.get_training_job(self.project_id, self.job_id)
+        self.current_start_time = iso_to_millis(
+            training_job["training_job"]["created_at"]
+        )
+        self.current_end_time = self.current_start_time + 2 * MILLISECONDS_PER_HOUR
+
+    def __iter__(self):
+        return self
+
+    def __next__(self) -> List[Any]:
+        if self.iteration >= MAX_ITERATIONS:
+            logging.warning(
+                f"Reached maximum iteration limit ({MAX_ITERATIONS}) while paginating "
+                f"training job logs for project_id={self.project_id}, job_id={self.job_id}."
+            )
+            raise StopIteration
+
+        query_params = _build_log_query_params(
+            self.current_start_time, self.current_end_time, self.batch_size
+        )
+
+        try:
+            batch_logs = self.api._fetch_log_batch(
+                self.project_id, self.job_id, query_params
+            )
+
+            should_continue, next_start_time, next_end_time = _process_batch_logs(
+                batch_logs, self.job_id, self.iteration, self.batch_size
+            )
+
+            if not should_continue:
+                logging.info(
+                    f"Completed pagination for job {self.job_id}. Total iterations: {self.iteration + 1}"
+                )
+                raise StopIteration
+
+            self.current_start_time = next_start_time  # type: ignore[assignment]
+            self.current_end_time = next_end_time  # type: ignore[assignment]
+            self.iteration += 1
+
+            return batch_logs
+
+        except requests.HTTPError as e:
+            if 500 <= e.response.status_code < 600:
+                if self.batch_size == MIN_BATCH_SIZE:
+                    logging.error(
+                        "Failed to fetch all training job logs due to persistent server errors. "
+                        "Please try again later or contact support if the issue persists."
+                    )
+                    raise StopIteration
+                self.batch_size = _handle_server_error_backoff(
+                    e, self.job_id, self.iteration, self.batch_size
+                )
+                # Retry the same iteration with reduced batch size
+                return self.__next__()
+            else:
+                logging.error(
+                    f"HTTP error fetching logs for job {self.job_id} at iteration {self.iteration}: {e}"
+                )
+                raise StopIteration
+        except Exception as e:
+            logging.error(
+                f"Error fetching logs for job {self.job_id} at iteration {self.iteration}: {e}"
+            )
+            raise StopIteration
+
+
+def get_training_job_logs_with_pagination(
+    api: BasetenApi, project_id: str, job_id: str, batch_size: int = MAX_BATCH_SIZE
+) -> List[Any]:
+    """
+    This method implements forward time-based pagination by starting from the earliest
+    available log and working forward in time. It uses the timestamp of the newest log in
+    each batch as the start time for the next request.
+
+    Returns:
+        List of all logs in chronological order (oldest first)
+    """
+    all_logs = []
+
+    logs_iterator = BatchedTrainingLogsFetcher(api, project_id, job_id, batch_size)
+
+    for batch_logs in logs_iterator:
+        all_logs.extend(batch_logs)
+
+    logging.info(f"Completed pagination for job {job_id}. Total logs: {len(all_logs)}")
+
+    return all_logs

truss/remote/baseten/utils/time.py

@@ -0,0 +1,15 @@
+from dateutil import parser
+
+
+def iso_to_millis(ts: str) -> int:
+    """
+    Convert ISO 8601 timestamp string to milliseconds since epoch.
+
+    Args:
+        ts: ISO 8601 timestamp string (handles Zulu/UTC (Z) automatically)
+
+    Returns:
+        Milliseconds since epoch as integer
+    """
+    dt = parser.isoparse(ts)  # handles Zulu/UTC (Z) automatically
+    return int(dt.timestamp() * 1000)

truss/templates/base.Dockerfile.jinja

@@ -8,25 +8,6 @@ FROM {{ base_image_name_and_tag }} AS truss_server
 {%- set python_executable = config.base_image.python_executable_path or 'python3' %}
 ENV PYTHON_EXECUTABLE="{{ python_executable }}"
 
-# The non-root user's home directory.
-# uv will use $HOME to install packages.
-ENV HOME=/home/{{ app_username }}
-# Directory containing inference server code.
-ENV APP_HOME=/{{ app_username }}
-RUN mkdir -p ${APP_HOME} {{ control_server_dir }}
-# Create a non-root user to run model containers.
-RUN useradd -u {{ app_user_uid }} -ms /bin/bash {{ app_username }}
-# Add the user's local bin to the path, used by uv.
-ENV PATH=${PATH}:${HOME}/.local/bin
-
-{%- if non_root_user %}
-RUN apt update -y && apt install -y sudo
-RUN echo "{{ app_username }} ALL=(root) NOPASSWD: /usr/bin/apt install *, /usr/bin/apt update *" > /etc/sudoers.d/app-packages
-RUN chmod 0440 /etc/sudoers.d/app-packages
-{#- optional but good practice: check if the sudoers file is valid #}
-RUN visudo -c
-{%- endif %}
-
 {%- set UV_VERSION = "0.7.19" %}
 {#
 NB(nikhil): We use a semi-complex uv installation command across the board:
@@ -58,6 +39,7 @@ RUN if ! command -v uv >/dev/null 2>&1; then \
     command -v curl >/dev/null 2>&1 || (apt update && apt install -y curl) && \
     curl -LsSf --retry 5 --retry-delay 5 https://astral.sh/uv/{{ UV_VERSION }}/install.sh | sh; \
 fi
+ENV PATH="/root/.local/bin:$PATH"
 {% endblock %}
 
 {% block base_image_patch %}
@@ -75,7 +57,7 @@ RUN {{ sys_pip_install_command }} install mkl
 
 {% block install_system_requirements %}
     {%- if should_install_system_requirements %}
-COPY --chown={{ default_owner }} ./{{ system_packages_filename }} {{ system_packages_filename }}
+COPY ./{{ system_packages_filename }} {{ system_packages_filename }}
 RUN apt-get update && apt-get install --yes --no-install-recommends $(cat {{ system_packages_filename }}) \
     && apt-get autoremove -y \
     && apt-get clean -y \
@@ -86,11 +68,11 @@ RUN apt-get update && apt-get install --yes --no-install-recommends $(cat {{ sys
 
 {% block install_requirements %}
     {%- if should_install_user_requirements_file %}
-COPY --chown={{ default_owner }} ./{{ user_supplied_requirements_filename }} {{ user_supplied_requirements_filename }}
+COPY ./{{ user_supplied_requirements_filename }} {{ user_supplied_requirements_filename }}
 RUN {{ sys_pip_install_command }} -r {{ user_supplied_requirements_filename }} --no-cache-dir
     {%- endif %}
     {%- if should_install_requirements %}
-COPY --chown={{ default_owner }} ./{{ config_requirements_filename }} {{ config_requirements_filename }}
+COPY ./{{ config_requirements_filename }} {{ config_requirements_filename }}
 RUN {{ sys_pip_install_command }} -r {{ config_requirements_filename }} --no-cache-dir
 {%- endif %}
 {% endblock %}
@@ -98,6 +80,7 @@ RUN {{ sys_pip_install_command }} -r {{ config_requirements_filename }} --no-cac
 
 
 {%- if not config.docker_server %}
+ENV APP_HOME="/app"
 WORKDIR $APP_HOME
 {%- endif %}
 
@@ -107,7 +90,7 @@ WORKDIR $APP_HOME
 
 {% block bundled_packages_copy %}
     {%- if bundled_packages_dir_exists %}
-COPY --chown={{ default_owner }} ./{{ config.bundled_packages_dir }} /packages
+COPY ./{{ config.bundled_packages_dir }} /packages
 {%- endif %}
 {% endblock %}
 

truss/templates/cache.Dockerfile.jinja

@@ -1,7 +1,7 @@
 FROM python:3.11-slim AS cache_warmer
 
-RUN mkdir -p ${APP_HOME}/model_cache
-WORKDIR ${APP_HOME}
+RUN mkdir -p /app/model_cache
+WORKDIR /app
 
 {% if hf_access_token %}
 ENV HUGGING_FACE_HUB_TOKEN="{{hf_access_token}}"
@@ -9,12 +9,12 @@ ENV HUGGING_FACE_HUB_TOKEN="{{hf_access_token}}"
 
 RUN apt-get -y update; apt-get -y install curl; curl -s https://baseten-public.s3.us-west-2.amazonaws.com/bin/b10cp-5fe8dc7da-linux-amd64 -o /app/b10cp; chmod +x /app/b10cp
 ENV B10CP_PATH_TRUSS="/app/b10cp"
-COPY --chown={{ default_owner }} ./cache_requirements.txt ${APP_HOME}/cache_requirements.txt
+COPY ./cache_requirements.txt /app/cache_requirements.txt
 RUN pip install -r /app/cache_requirements.txt --no-cache-dir && rm -rf /root/.cache/pip
-COPY --chown={{ default_owner }} ./cache_warmer.py /cache_warmer.py
+COPY ./cache_warmer.py /cache_warmer.py
 
 {% for credential in credentials_to_cache  %}
-COPY ./{{credential}} ${APP_HOME}/{{credential}}
+COPY ./{{credential}} /app/{{credential}}
 {% endfor %}
 
 {% for repo, hf_dir in models.items() %}

truss/templates/copy_cache_files.Dockerfile.jinja

@@ -1,3 +1,3 @@
 {% for file in cached_files %}
-COPY --chown={{ default_owner }} --from=cache_warmer {{file.source}} {{file.dst}}
+COPY --from=cache_warmer {{file.source}} {{file.dst}}
 {% endfor %}

truss/templates/docker_server/supervisord.conf.jinja

@@ -1,5 +1,4 @@
 [supervisord]
-pidfile=/tmp/supervisord.pid ; Set PID file location to /tmp to be writable by the non-root user
 nodaemon=true                ; Run supervisord in the foreground (useful for containers)
 logfile=/dev/null            ; Disable logging to file (send logs to /dev/null)
 logfile_maxbytes=0           ; No size limit on logfile (since logging is disabled)

truss/templates/server/requirements.txt

@@ -18,7 +18,7 @@ psutil>=5.9.4
 python-json-logger>=2.0.2
 pyyaml>=6.0.0
 requests>=2.31.0
-truss-transfer==0.0.27
+truss-transfer==0.0.28
 uvicorn>=0.24.0
 uvloop>=0.19.0
 websockets>=10.0

truss/templates/server.Dockerfile.jinja

@@ -15,12 +15,12 @@ ENV DEBIAN_FRONTEND="noninteractive"
 
 {# Install common dependencies #}
 RUN apt update && \
-    apt install -y bash build-essential git curl ca-certificates sudo \
+    apt install -y bash build-essential git curl ca-certificates \
     && apt-get autoremove -y \
     && apt-get clean -y \
     && rm -rf /var/lib/apt/lists/*
 
-COPY --chown={{ default_owner }} ./{{ base_server_requirements_filename }} {{ base_server_requirements_filename }}
+COPY ./{{ base_server_requirements_filename }} {{ base_server_requirements_filename }}
 RUN {{ sys_pip_install_command }} -r {{ base_server_requirements_filename }} --no-cache-dir
     {%- endif %} {#- endif not config.docker_server #}
 
@@ -38,7 +38,7 @@ RUN ln -sf {{ config.base_image.python_executable_path }} /usr/local/bin/python
 
 {% block install_requirements %}
     {%- if should_install_server_requirements %}
-COPY --chown={{ default_owner }} ./{{ server_requirements_filename }} {{ server_requirements_filename }}
+COPY ./{{ server_requirements_filename }} {{ server_requirements_filename }}
 RUN {{ sys_pip_install_command }} -r {{ server_requirements_filename }} --no-cache-dir
     {%- endif %} {#- endif should_install_server_requirements #}
 {{ super() }}
@@ -65,47 +65,47 @@ RUN {% for secret,path in config.build.secret_to_path_mapping.items() %} --mount
 
 {# Copy data before code for better caching #}
 {%- if data_dir_exists %}
-COPY --chown={{ default_owner }} ./{{ config.data_dir }} ${APP_HOME}/data
+COPY ./{{ config.data_dir }} /app/data
 {%- endif %} {#- endif data_dir_exists #}
 
 {%- if model_cache_v2 %}
 # v0.0.9, keep synced with server_requirements.txt
-RUN curl -sSL --fail --retry 5 --retry-delay 2 -o /usr/local/bin/truss-transfer-cli https://github.com/basetenlabs/truss/releases/download/v0.10.9rc0/truss-transfer-cli-v0.10.9rc0-linux-x86_64-unknown-linux-musl
+RUN curl -sSL --fail --retry 5 --retry-delay 2 -o /usr/local/bin/truss-transfer-cli https://github.com/basetenlabs/truss/releases/download/v0.10.9rc30/truss-transfer-cli-v0.10.9rc30-linux-x86_64-unknown-linux-musl
 RUN chmod +x /usr/local/bin/truss-transfer-cli
 RUN mkdir /static-bptr
 RUN echo "hash {{model_cache_hash}}"
-COPY --chown={{ default_owner }} ./bptr-manifest /static-bptr/static-bptr-manifest.json
+COPY ./bptr-manifest /static-bptr/static-bptr-manifest.json
 {%- endif %} {#- endif model_cache_v2 #}
 
 {%- if not config.docker_server %}
-COPY --chown={{ default_owner }} ./server ${APP_HOME}
+COPY ./server /app
 {%- endif %} {#- endif not config.docker_server #}
 
 {%- if use_local_src %}
 {# This path takes precedence over site-packages. #}
-COPY --chown={{ default_owner }} ./truss_chains ${APP_HOME}/truss_chains
-COPY --chown={{ default_owner }} ./truss ${APP_HOME}/truss
+COPY ./truss_chains /app/truss_chains
+COPY ./truss /app/truss
 {%- endif %} {#- endif use_local_src #}
 
-COPY --chown={{ default_owner }} ./config.yaml ${APP_HOME}/config.yaml
+COPY ./config.yaml /app/config.yaml
 {%- if requires_live_reload %}
 RUN uv python install {{ control_python_version }}
 RUN uv venv /control/.env --python {{ control_python_version }}
 
-COPY --chown={{ default_owner }} ./control /control
+COPY ./control /control
 RUN uv pip install -r /control/requirements.txt --python /control/.env/bin/python --no-cache-dir
 {%- endif %} {#- endif requires_live_reload #}
 
 {%- if model_dir_exists %}
-COPY --chown={{ default_owner }} ./{{ config.model_module_dir }} ${APP_HOME}/model
+COPY ./{{ config.model_module_dir }} /app/model
 {%- endif %} {#- endif model_dir_exists #}
 {% endblock %} {#- endblock app_copy #}
 
 {% block run %}
     {%- if config.docker_server %}
-RUN apt-get update -y && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
         curl nginx && rm -rf /var/lib/apt/lists/*
-COPY --chown={{ default_owner }} ./docker_server_requirements.txt ${APP_HOME}/docker_server_requirements.txt
+COPY ./docker_server_requirements.txt /app/docker_server_requirements.txt
 
 {# NB(nikhil): Use the same python version for custom server proxy as the control server, for consistency. #}
 RUN uv python install {{ control_python_version }}
@@ -115,38 +115,21 @@ RUN uv pip install --python /docker_server/.venv/bin/python -r /app/docker_serve
 {% set supervisor_config_path = "/etc/supervisor/supervisord.conf" %}
 {% set supervisor_log_dir = "/var/log/supervisor" %}
 {% set supervisor_server_url = "http://localhost:8080" %}
-COPY --chown={{ default_owner }} ./proxy.conf {{ proxy_config_path }}
+COPY ./proxy.conf {{ proxy_config_path }}
 RUN mkdir -p {{ supervisor_log_dir }}
-COPY --chown={{ default_owner }} ./supervisord.conf {{ supervisor_config_path }}
+COPY supervisord.conf {{ supervisor_config_path }}
 ENV SUPERVISOR_SERVER_URL="{{ supervisor_server_url }}"
 ENV SERVER_START_CMD="/docker_server/.venv/bin/supervisord -c {{ supervisor_config_path }}"
-{#- default configuration uses port 80, which requires root privileges, so we remove it #}
-RUN rm -f /etc/nginx/sites-enabled/default
-{%- if non_root_user %}
-{#- nginx writes to /var/lib/nginx, /var/log/nginx, and /run directories #}
-{% set nginx_dirs = ["/var/lib/nginx", "/var/log/nginx", "/run"] %}
-RUN chown -R {{ app_username }}:{{ app_username }} {{ nginx_dirs | join(" ") }}
-RUN chown -R {{ app_username }}:{{ app_username }} ${HOME} ${APP_HOME}
-USER {{ app_username }}
-{%- endif %} {#- endif non_root_user #}
 ENTRYPOINT ["/docker_server/.venv/bin/supervisord", "-c", "{{ supervisor_config_path }}"]
     {%- elif requires_live_reload %} {#- elif requires_live_reload #}
 ENV HASH_TRUSS="{{ truss_hash }}"
 ENV CONTROL_SERVER_PORT="8080"
 ENV INFERENCE_SERVER_PORT="8090"
 ENV SERVER_START_CMD="/control/.env/bin/python /control/control/server.py"
-{%- if non_root_user %}
-RUN chown -R {{ app_username }}:{{ app_username }} ${HOME} ${APP_HOME}
-USER {{ app_username }}
-{%- endif %} {#- endif non_root_user #}
 ENTRYPOINT ["/control/.env/bin/python", "/control/control/server.py"]
     {%- else %} {#- else (default inference server) #}
 ENV INFERENCE_SERVER_PORT="8080"
 ENV SERVER_START_CMD="{{ python_executable }} /app/main.py"
-{%- if non_root_user %}
-RUN chown -R {{ app_username }}:{{ app_username }} ${HOME} ${APP_HOME}
-USER {{ app_username }}
-{%- endif %} {#- endif non_root_user #}
 ENTRYPOINT ["{{ python_executable }}", "/app/main.py"]
     {%- endif %} {#- endif config.docker_server / live_reload #}
 {% endblock %} {#- endblock run #}