autoinference-utils 0.2.3__tar.gz → 0.2.5__tar.gz

{autoinference_utils-0.2.3 → autoinference_utils-0.2.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: autoinference-utils
-Version: 0.2.3
+Version: 0.2.5
 Summary: Shared endpoint abstractions for autoinference deployments
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown

{autoinference_utils-0.2.3 → autoinference_utils-0.2.5}/autoinference_utils/endpoint.py

@@ -11,6 +11,7 @@ from __future__ import annotations
 
 import http.server
 import json
+import math
 import os
 import shlex
 import subprocess
@@ -26,7 +27,7 @@ BENCH_MODE_ENV = "ENDPOINT_BENCH_MODE"
 DUMMY_WEIGHTS_ENV = "ENDPOINT_DUMMY"
 BENCH_MODE_PORT_OFFSET = 10000
 
-MODAL_FLASH_REQUEST_UUID_HEADER= "x-modal-flash-request-uuid"
+MODAL_FLASH_REQUEST_UUID_HEADER= "modal-flash-request-uuid"
 MODAL_SESSION_ID_HEADER = "modal-session-id"
 
 # Endpoint class names whose servers emit OAI streaming chunks that
@@ -760,32 +761,86 @@ def warmup_chat_completions(
         request_headers.update(headers)
 
     for request_idx in range(successful_requests):
-        for attempt in range(max_attempts_per_request):
-            try:
-                _post_json(
-                    url,
-                    payload=payload,
-                    headers=request_headers,
-                    timeout=request_timeout,
-                )
-                break
-            except (
-                urllib.error.HTTPError,
-                urllib.error.URLError,
-                TimeoutError,
-                OSError,
-            ) as exc:
-                if attempt + 1 == max_attempts_per_request:
-                    detail = (
-                        _format_http_error(exc)
-                        if isinstance(exc, urllib.error.HTTPError)
-                        else f"{type(exc).__name__}: {exc}"
-                    )
-                    raise RuntimeError(
-                        f"warmup request {request_idx + 1}/"
-                        f"{successful_requests}: {detail}"
-                    ) from exc
-            time.sleep(retry_delay)
+        _post_json_with_retries(
+            url,
+            payload=payload,
+            headers=request_headers,
+            request_timeout=request_timeout,
+            max_attempts=max_attempts_per_request,
+            retry_delay=retry_delay,
+            description=(
+                f"warmup request {request_idx + 1}/{successful_requests}"
+            ),
+        )
+
+
+def validate_embeddings_endpoint(
+    *,
+    port: int,
+    payload: Mapping[str, Any],
+    headers: Mapping[str, str] | None = None,
+    request_timeout: float = 30.0,
+    max_attempts: int = 2,
+    retry_delay: float = 1.0,
+) -> None:
+    """Validate one OpenAI-compatible embeddings request."""
+    inputs = payload.get("input")
+    if isinstance(inputs, str):
+        expected_count = 1
+    elif isinstance(inputs, list) and inputs:
+        expected_count = (
+            1 if all(isinstance(item, int) for item in inputs) else len(inputs)
+        )
+    else:
+        raise ValueError(
+            "embedding validation payload must contain non-empty input"
+        )
+
+    request_headers = {"Content-Type": "application/json"}
+    if headers:
+        request_headers.update(headers)
+    body = _post_json_with_retries(
+        f"http://127.0.0.1:{port}/v1/embeddings",
+        payload=payload,
+        headers=request_headers,
+        request_timeout=request_timeout,
+        max_attempts=max_attempts,
+        retry_delay=retry_delay,
+        description="embedding validation request",
+    )
+    try:
+        response = json.loads(body)
+        data = response["data"]
+    except (ValueError, KeyError, TypeError) as exc:
+        raise RuntimeError("embedding validation returned an invalid response") from exc
+
+    if not isinstance(data, list) or len(data) != expected_count:
+        raise RuntimeError(
+            f"embedding validation returned {len(data) if isinstance(data, list) else 0} "
+            f"embeddings; expected {expected_count}"
+        )
+
+    dimension: int | None = None
+    for index, item in enumerate(data):
+        embedding = item.get("embedding") if isinstance(item, Mapping) else None
+        if not isinstance(embedding, list) or not embedding:
+            raise RuntimeError(
+                f"embedding validation result {index} has no embedding vector"
+            )
+        if any(
+            isinstance(value, bool)
+            or not isinstance(value, (int, float))
+            or not math.isfinite(value)
+            for value in embedding
+        ):
+            raise RuntimeError(
+                f"embedding validation result {index} contains "
+                "non-numeric or non-finite values"
+            )
+        if dimension is None:
+            dimension = len(embedding)
+        elif len(embedding) != dimension:
+            raise RuntimeError("embedding validation returned inconsistent dimensions")
 
 
 def start_heartbeat_thread(
@@ -922,13 +977,51 @@ def _post_json(
     payload: Mapping[str, Any],
     headers: Mapping[str, str],
     timeout: float | None = None,
-) -> int:
+) -> bytes:
     body = json.dumps(payload).encode("utf-8")
     req = urllib.request.Request(
         url, data=body, headers=dict(headers), method="POST"
     )
     with urllib.request.urlopen(req, timeout=timeout) as resp:
-        return resp.getcode()
+        return resp.read()
+
+
+def _post_json_with_retries(
+    url: str,
+    *,
+    payload: Mapping[str, Any],
+    headers: Mapping[str, str],
+    request_timeout: float,
+    max_attempts: int,
+    retry_delay: float,
+    description: str,
+) -> bytes:
+    if max_attempts < 1:
+        raise ValueError("max_attempts must be at least 1")
+
+    for attempt in range(max_attempts):
+        try:
+            return _post_json(
+                url,
+                payload=payload,
+                headers=headers,
+                timeout=request_timeout,
+            )
+        except (
+            urllib.error.HTTPError,
+            urllib.error.URLError,
+            TimeoutError,
+            OSError,
+        ) as exc:
+            if attempt + 1 == max_attempts:
+                detail = (
+                    _format_http_error(exc)
+                    if isinstance(exc, urllib.error.HTTPError)
+                    else f"{type(exc).__name__}: {exc}"
+                )
+                raise RuntimeError(f"{description}: {detail}") from exc
+        time.sleep(retry_delay)
+    raise AssertionError("unreachable")
 
 
 def _format_http_error(exc: urllib.error.HTTPError) -> str:

{autoinference_utils-0.2.3 → autoinference_utils-0.2.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "autoinference-utils"
-version = "0.2.3"
+version = "0.2.5"
 description = "Shared endpoint abstractions for autoinference deployments"
 readme = "README.md"
 requires-python = ">=3.10"