autoinference-utils 0.2.2__tar.gz → 0.2.4__tar.gz

{autoinference_utils-0.2.2 → autoinference_utils-0.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: autoinference-utils
-Version: 0.2.2
+Version: 0.2.4
 Summary: Shared endpoint abstractions for autoinference deployments
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown

{autoinference_utils-0.2.2 → autoinference_utils-0.2.4}/autoinference_utils/endpoint.py

@@ -11,6 +11,7 @@ from __future__ import annotations
 
 import http.server
 import json
+import math
 import os
 import shlex
 import subprocess
@@ -26,6 +27,9 @@ BENCH_MODE_ENV = "ENDPOINT_BENCH_MODE"
 DUMMY_WEIGHTS_ENV = "ENDPOINT_DUMMY"
 BENCH_MODE_PORT_OFFSET = 10000
 
+MODAL_FLASH_REQUEST_UUID_HEADER= "x-modal-flash-request-uuid"
+MODAL_SESSION_ID_HEADER = "modal-session-id"
+
 # Endpoint class names whose servers emit OAI streaming chunks that
 # sglang.bench_serving can't parse. deploy_and_bench auto-injects
 # --openai-stream-compat for deployments importing any of these.
@@ -93,6 +97,7 @@ class SGLangEndpoint(Endpoint):
         health_timeout: float = 20 * 60,
         health_poll_interval: float = 5.0,
         health_request_timeout: float = 5.0,
+        log_requests_level: int = 0,
     ):
         # In bench mode SGLang runs on worker_port + 10000 and a /bench proxy
         # listens on the original worker_port; both fall through to the same
@@ -124,6 +129,13 @@ class SGLangEndpoint(Endpoint):
         self.health_timeout = health_timeout
         self.health_poll_interval = health_poll_interval
         self.health_request_timeout = health_request_timeout
+
+        # SGLang Log request level: -1 = disabled, 0 = metadata only, 1,2,3 in increasing verbosity
+        # Refer to https://docs.sglang.io/docs/advanced_features/server_arguments#logging for more details
+        if not (log_requests_level >= -1 and log_requests_level <= 3):
+            raise ValueError("log_requests_level must be between -1 and 3")
+        self.log_requests_level = log_requests_level
+
         self._proc: Optional[subprocess.Popen] = None
         self._bench_server: Optional[http.server.ThreadingHTTPServer] = None
 
@@ -139,6 +151,32 @@ class SGLangEndpoint(Endpoint):
             ):
                 self.load_format = "dummy"
             self.extra_server_args.setdefault("--model-loader-extra-config", "{}")
+        
+        # Request logging enabled, log only metadata by default
+        if self.log_requests_level >= 0:
+            # Enable request logging    
+            if "--log-requests" not in self.extra_server_args:
+                self.extra_server_args.setdefault("--log-requests", "")
+            # Set request logging level
+            if "--log-requests-level" not in self.extra_server_args:
+                self.extra_server_args.setdefault("--log-requests-level", str(self.log_requests_level))
+            else:
+                print(f"[endpoint] --log-requests-level already set to {self.extra_server_args['--log-requests-level']} in server args")
+
+            # Add Modal-forwarded request headers to SGLang logged headers
+            sglang_log_request_headers = [
+                h.strip().lower()
+                for h in os.environ.get("SGLANG_LOG_REQUEST_HEADERS", "").split(",")
+                if h.strip()
+            ]
+            if MODAL_FLASH_REQUEST_UUID_HEADER not in sglang_log_request_headers:
+                sglang_log_request_headers.append(MODAL_FLASH_REQUEST_UUID_HEADER)
+            if MODAL_SESSION_ID_HEADER not in sglang_log_request_headers:
+                sglang_log_request_headers.append(MODAL_SESSION_ID_HEADER)
+
+            self.log_request_headers = ",".join(sglang_log_request_headers)
+            os.environ["SGLANG_LOG_REQUEST_HEADERS"] = self.log_request_headers
+
 
     def _build_cmd(self) -> list[str]:
         cmd = [
@@ -723,32 +761,86 @@ def warmup_chat_completions(
         request_headers.update(headers)
 
     for request_idx in range(successful_requests):
-        for attempt in range(max_attempts_per_request):
-            try:
-                _post_json(
-                    url,
-                    payload=payload,
-                    headers=request_headers,
-                    timeout=request_timeout,
-                )
-                break
-            except (
-                urllib.error.HTTPError,
-                urllib.error.URLError,
-                TimeoutError,
-                OSError,
-            ) as exc:
-                if attempt + 1 == max_attempts_per_request:
-                    detail = (
-                        _format_http_error(exc)
-                        if isinstance(exc, urllib.error.HTTPError)
-                        else f"{type(exc).__name__}: {exc}"
-                    )
-                    raise RuntimeError(
-                        f"warmup request {request_idx + 1}/"
-                        f"{successful_requests}: {detail}"
-                    ) from exc
-            time.sleep(retry_delay)
+        _post_json_with_retries(
+            url,
+            payload=payload,
+            headers=request_headers,
+            request_timeout=request_timeout,
+            max_attempts=max_attempts_per_request,
+            retry_delay=retry_delay,
+            description=(
+                f"warmup request {request_idx + 1}/{successful_requests}"
+            ),
+        )
+
+
+def validate_embeddings_endpoint(
+    *,
+    port: int,
+    payload: Mapping[str, Any],
+    headers: Mapping[str, str] | None = None,
+    request_timeout: float = 30.0,
+    max_attempts: int = 2,
+    retry_delay: float = 1.0,
+) -> None:
+    """Validate one OpenAI-compatible embeddings request."""
+    inputs = payload.get("input")
+    if isinstance(inputs, str):
+        expected_count = 1
+    elif isinstance(inputs, list) and inputs:
+        expected_count = (
+            1 if all(isinstance(item, int) for item in inputs) else len(inputs)
+        )
+    else:
+        raise ValueError(
+            "embedding validation payload must contain non-empty input"
+        )
+
+    request_headers = {"Content-Type": "application/json"}
+    if headers:
+        request_headers.update(headers)
+    body = _post_json_with_retries(
+        f"http://127.0.0.1:{port}/v1/embeddings",
+        payload=payload,
+        headers=request_headers,
+        request_timeout=request_timeout,
+        max_attempts=max_attempts,
+        retry_delay=retry_delay,
+        description="embedding validation request",
+    )
+    try:
+        response = json.loads(body)
+        data = response["data"]
+    except (ValueError, KeyError, TypeError) as exc:
+        raise RuntimeError("embedding validation returned an invalid response") from exc
+
+    if not isinstance(data, list) or len(data) != expected_count:
+        raise RuntimeError(
+            f"embedding validation returned {len(data) if isinstance(data, list) else 0} "
+            f"embeddings; expected {expected_count}"
+        )
+
+    dimension: int | None = None
+    for index, item in enumerate(data):
+        embedding = item.get("embedding") if isinstance(item, Mapping) else None
+        if not isinstance(embedding, list) or not embedding:
+            raise RuntimeError(
+                f"embedding validation result {index} has no embedding vector"
+            )
+        if any(
+            isinstance(value, bool)
+            or not isinstance(value, (int, float))
+            or not math.isfinite(value)
+            for value in embedding
+        ):
+            raise RuntimeError(
+                f"embedding validation result {index} contains "
+                "non-numeric or non-finite values"
+            )
+        if dimension is None:
+            dimension = len(embedding)
+        elif len(embedding) != dimension:
+            raise RuntimeError("embedding validation returned inconsistent dimensions")
 
 
 def start_heartbeat_thread(
@@ -885,13 +977,51 @@ def _post_json(
     payload: Mapping[str, Any],
     headers: Mapping[str, str],
     timeout: float | None = None,
-) -> int:
+) -> bytes:
     body = json.dumps(payload).encode("utf-8")
     req = urllib.request.Request(
         url, data=body, headers=dict(headers), method="POST"
     )
     with urllib.request.urlopen(req, timeout=timeout) as resp:
-        return resp.getcode()
+        return resp.read()
+
+
+def _post_json_with_retries(
+    url: str,
+    *,
+    payload: Mapping[str, Any],
+    headers: Mapping[str, str],
+    request_timeout: float,
+    max_attempts: int,
+    retry_delay: float,
+    description: str,
+) -> bytes:
+    if max_attempts < 1:
+        raise ValueError("max_attempts must be at least 1")
+
+    for attempt in range(max_attempts):
+        try:
+            return _post_json(
+                url,
+                payload=payload,
+                headers=headers,
+                timeout=request_timeout,
+            )
+        except (
+            urllib.error.HTTPError,
+            urllib.error.URLError,
+            TimeoutError,
+            OSError,
+        ) as exc:
+            if attempt + 1 == max_attempts:
+                detail = (
+                    _format_http_error(exc)
+                    if isinstance(exc, urllib.error.HTTPError)
+                    else f"{type(exc).__name__}: {exc}"
+                )
+                raise RuntimeError(f"{description}: {detail}") from exc
+        time.sleep(retry_delay)
+    raise AssertionError("unreachable")
 
 
 def _format_http_error(exc: urllib.error.HTTPError) -> str:

{autoinference_utils-0.2.2 → autoinference_utils-0.2.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "autoinference-utils"
-version = "0.2.2"
+version = "0.2.4"
 description = "Shared endpoint abstractions for autoinference deployments"
 readme = "README.md"
 requires-python = ">=3.10"