truss 0.11.9rc6__tar.gz → 0.11.10rc1__tar.gz

{truss-0.11.9rc6 → truss-0.11.10rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: truss
-Version: 0.11.9rc6
+Version: 0.11.10rc1
 Summary: A seamless bridge from model development to model delivery
 Project-URL: Repository, https://github.com/basetenlabs/truss
 Project-URL: Homepage, https://truss.baseten.co

{truss-0.11.9rc6 → truss-0.11.10rc1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "truss"
-version = "0.11.9rc6"
+version = "0.11.10rc1"
 description = "A seamless bridge from model development to model delivery"
 authors = [
     { name = "Pankaj Gupta", email = "no-reply@baseten.co" },

{truss-0.11.9rc6 → truss-0.11.10rc1}/truss/cli/logs/base_watcher.py

@@ -17,7 +17,9 @@ class LogWatcher(ABC):
     # NB(nikhil): we add buffer for clock skew, so this helps us detect duplicates.
     # TODO(nikhil): clean up hashes so this doesn't grow indefinitely.
     _log_hashes: set[str] = set()
-    _last_poll_time: Optional[int] = None
+
+    _last_poll_time_ms: Optional[int] = None
+    _last_log_time_ms: Optional[int] = None
 
     def __init__(self, api: BasetenApi):
         self.api = api
@@ -26,37 +28,54 @@ class LogWatcher(ABC):
         log_str = f"{log.timestamp}-{log.message}-{log.replica}"
         return hashlib.sha256(log_str.encode("utf-8")).hexdigest()
 
-    def _poll(self) -> Iterator[ParsedLog]:
-        start_epoch: Optional[int] = None
-        now = int(time.time() * 1000)
-        if self._last_poll_time is not None:
-            start_epoch = self._last_poll_time - CLOCK_SKEW_BUFFER_MS
+    def get_start_epoch_ms(self, now_ms: int) -> Optional[int]:
+        if self._last_poll_time_ms:
+            return self._last_poll_time_ms - CLOCK_SKEW_BUFFER_MS
+
+        return None
 
+    def fetch_and_parse_logs(
+        self, start_epoch_millis: Optional[int], end_epoch_millis: Optional[int]
+    ) -> Iterator[ParsedLog]:
         api_logs = self.fetch_logs(
-            start_epoch_millis=start_epoch, end_epoch_millis=now + CLOCK_SKEW_BUFFER_MS
+            start_epoch_millis=start_epoch_millis, end_epoch_millis=end_epoch_millis
         )
 
         parsed_logs = parse_logs(api_logs)
+
         for log in parsed_logs:
-            h = self._hash_log(log)
-            if h not in self._log_hashes:
+            if (h := self._hash_log(log)) not in self._log_hashes:
                 self._log_hashes.add(h)
+
                 yield log
 
-        self._last_poll_time = now
+    def poll(self) -> Iterator[ParsedLog]:
+        now_ms = int(time.time() * 1000)
+        start_epoch_ms = self.get_start_epoch_ms(now_ms)
+
+        for log in self.fetch_and_parse_logs(
+            start_epoch_millis=start_epoch_ms,
+            end_epoch_millis=now_ms + CLOCK_SKEW_BUFFER_MS,
+        ):
+            yield log
+
+            epoch_ns = int(log.timestamp)
+            self._last_log_time_ms = int(epoch_ns / 1e6)
+
+        self._last_poll_time_ms = now_ms
 
     def watch(self) -> Iterator[ParsedLog]:
         self.before_polling()
         with console.status("Polling logs", spinner="aesthetic"):
             while True:
-                for log in self._poll():
+                for log in self.poll():
                     yield log
                 if self._log_hashes:
                     break
                 time.sleep(POLL_INTERVAL_SEC)
 
             while self.should_poll_again():
-                for log in self._poll():
+                for log in self.poll():
                     yield log
                 time.sleep(POLL_INTERVAL_SEC)
                 self.post_poll()

{truss-0.11.9rc6 → truss-0.11.10rc1}/truss/cli/logs/model_log_watcher.py

@@ -1,9 +1,12 @@
+from functools import cached_property
 from typing import Any, List, Optional
 
 from truss.cli.logs.base_watcher import LogWatcher
 from truss.remote.baseten.api import BasetenApi
 from truss.remote.baseten.utils.status import MODEL_RUNNING_STATES
 
+MAX_LOOK_BACK_MS = 1000 * 60 * 60  # 1 hour.
+
 
 class ModelDeploymentLogWatcher(LogWatcher):
     _model_id: str
@@ -25,11 +28,31 @@ class ModelDeploymentLogWatcher(LogWatcher):
             self._model_id, self._deployment_id, start_epoch_millis, end_epoch_millis
         )
 
+    def get_start_epoch_ms(self, now_ms: int) -> Optional[int]:
+        # NOTE(Tyron): If there can be multiple replicas,
+        # we can't use a timestamp cursor to poll for logs.
+        if not self._is_development:
+            return super().get_start_epoch_ms(now_ms)
+
+        # Cursor logic.
+
+        if self._last_log_time_ms:
+            return max(self._last_log_time_ms, now_ms - MAX_LOOK_BACK_MS)
+
+        return None
+
     def should_poll_again(self) -> bool:
         return self._current_status in MODEL_RUNNING_STATES
 
+    def _get_deployment(self) -> Any:
+        return self.api.get_deployment(self._model_id, self._deployment_id)
+
     def _get_current_status(self) -> str:
-        return self.api.get_deployment(self._model_id, self._deployment_id)["status"]
+        return self._get_deployment()["status"]
+
+    @cached_property
+    def _is_development(self) -> bool:
+        return self._get_deployment()["is_development"]
 
     def post_poll(self) -> None:
         self._current_status = self._get_current_status()

{truss-0.11.9rc6 → truss-0.11.10rc1}/truss/templates/control/control/application.py

@@ -40,6 +40,7 @@ class SanitizedExceptionMiddleware(BaseHTTPMiddleware):
         except Exception as exc:
             sanitized_traceback = self._create_sanitized_traceback(exc)
             request.app.state.logger.error(sanitized_traceback)
+            request.app.state.logger.error(f"Request headers: {dict(request.headers)}")
 
             if isinstance(exc, ModelLoadFailed):
                 return JSONResponse(
@@ -84,10 +85,9 @@ def create_app(base_config: Dict):
         base_url=f"http://localhost:{app_state.inference_server_port}", limits=limits
     )
 
-    pip_path = getattr(app_state, "pip_path", None)
-
+    uv_path = getattr(app_state, "uv_path", None)
     patch_applier = ModelContainerPatchApplier(
-        Path(app_state.inference_server_home), app_logger, pip_path
+        Path(app_state.inference_server_home), app_logger, uv_path
     )
 
     oversee_inference_server = getattr(app_state, "oversee_inference_server", True)

{truss-0.11.9rc6 → truss-0.11.10rc1}/truss/tests/templates/control/control/test_server.py

@@ -53,7 +53,7 @@ def app(truss_container_fs, truss_original_hash, ports):
                 "control_server_port": ports["control_server_port"],
                 "inference_server_port": ports["inference_server_port"],
                 "oversee_inference_server": False,
-                "pip_path": "pip",
+                "uv_path": "uv",
             }
         )
         inference_server_controller = control_app.state.inference_server_controller