typellm 0.2.3__tar.gz → 0.2.4__tar.gz

{typellm-0.2.3/typellm.egg-info → typellm-0.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: typellm
-Version: 0.2.3
+Version: 0.2.4
 Summary: Type-Safe Decoding for Autoregressive LLMs
 License-Expression: Apache-2.0
 Project-URL: Homepage, https://typellm.ai/

{typellm-0.2.3 → typellm-0.2.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "typellm"
-version = "0.2.3"
+version = "0.2.4"
 description = "Type-Safe Decoding for Autoregressive LLMs"
 readme = "README.md"
 license = "Apache-2.0"

{typellm-0.2.3 → typellm-0.2.4}/tests/test_concurrency.py

@@ -212,6 +212,38 @@ class DeadlineTests(unittest.TestCase):
             client._generate({"text": "x"})
         self.assertNotIsInstance(caught.exception, GenerationTimeout)
 
+    def dropping(self, error, times):
+        """A client whose server drops the first `times` requests with `error`."""
+        seen = []
+
+        def handler(request):
+            seen.append(request)
+            if len(seen) <= times:
+                raise error("connection dropped", request=request)
+            return httpx.Response(200, json={"text": "ok"})
+
+        client = SGLangClient()
+        client._http_client = httpx.Client(transport=httpx.MockTransport(handler))
+        return client, seen
+
+    def test_a_dropped_connection_is_retried_once(self):
+        for error in (httpx.ReadError, httpx.RemoteProtocolError, httpx.WriteError):
+            client, seen = self.dropping(error, 1)
+            with self.subTest(error=error.__name__):
+                self.assertEqual(client._request("/generate", {"text": "x"}), {"text": "ok"})
+                self.assertEqual(len(seen), 2)
+        client, seen = self.dropping(httpx.ReadError, 2)
+        with self.assertRaisesRegex(SGLangError, "Could not reach"):
+            client._request("/generate", {"text": "x"})
+        self.assertEqual(len(seen), 2)
+
+    def test_a_refused_connection_is_not_retried(self):
+        for error in (httpx.ConnectError, httpx.ReadTimeout):
+            client, seen = self.dropping(error, 1)
+            with self.subTest(error=error.__name__), self.assertRaises(SGLangError):
+                client._request("/generate", {"text": "x"})
+            self.assertEqual(len(seen), 1)
+
     def test_invalid_budgets_are_rejected(self):
         class Recording(FakeServer):
             def _request(self, path, payload=None, *, allow_text=False):

{typellm-0.2.3 → typellm-0.2.4}/typellm/sglang.py

@@ -252,9 +252,11 @@ class SGLangClient:
     def _http(self) -> httpx.Client:
         with self._load_lock:
             if self._http_client is None:
-                # Keep-alive connections: one call makes many small requests.
+                # Keep-alive connections: one call makes many small requests. Idle
+                # ones are dropped before SGLang closes them (5 s), so a request
+                # rarely goes out on a connection the server has just closed.
                 self._http_client = httpx.Client(limits=httpx.Limits(
-                    max_connections=None, max_keepalive_connections=64,
+                    max_connections=None, max_keepalive_connections=64, keepalive_expiry=2.0,
                 ))
             return self._http_client
 
@@ -269,24 +271,15 @@ class SGLangClient:
         scope = _call_scope.get()
         timeout = self.timeout if scope is None else scope.socket_timeout(self.timeout)
         try:
-            # Streamed so a failed body read still reports the status it follows.
-            with self._http().stream(
-                "GET" if payload is None else "POST",
-                self.base_url + path,
-                content=body,
-                headers={"Content-Type": "application/json"},
-                timeout=timeout,
-            ) as response:
-                if response.status_code >= 400:
-                    try:
-                        detail = response.read().decode("utf-8", errors="replace")
-                    except httpx.HTTPError as read_error:
-                        detail = f"Could not read error response: {read_error}"
-                    raise SGLangError(
-                        f"SGLang {path} returned HTTP {response.status_code}: {detail}",
-                        status=response.status_code,
-                    )
-                raw = response.read().decode("utf-8")
+            try:
+                raw = self._send(path, payload is None, body, timeout)
+            except (httpx.ReadError, httpx.WriteError, httpx.RemoteProtocolError):
+                # The connection dropped before an answer, most often a pooled one
+                # the server had just closed. SGLang is fine: try once on a new one.
+                if scope is not None:
+                    scope.check()
+                    timeout = scope.socket_timeout(self.timeout)
+                raw = self._send(path, payload is None, body, timeout)
         except httpx.HTTPError as exc:
             # Connection failures, timeouts, resets and truncated responses.
             raise SGLangError(
@@ -301,6 +294,26 @@ class SGLangClient:
                 f"SGLang {path} returned non-JSON data: {raw[:500]}"
             ) from exc
 
+    def _send(self, path: str, get: bool, body: bytes | None, timeout: float) -> str:
+        # Streamed so a failed body read still reports the status it follows.
+        with self._http().stream(
+            "GET" if get else "POST",
+            self.base_url + path,
+            content=body,
+            headers={"Content-Type": "application/json"},
+            timeout=timeout,
+        ) as response:
+            if response.status_code >= 400:
+                try:
+                    detail = response.read().decode("utf-8", errors="replace")
+                except httpx.HTTPError as read_error:
+                    detail = f"Could not read error response: {read_error}"
+                raise SGLangError(
+                    f"SGLang {path} returned HTTP {response.status_code}: {detail}",
+                    status=response.status_code,
+                )
+            return response.read().decode("utf-8")
+
     @contextmanager
     def images(self, images: Sequence[str]) -> Iterator[None]:
         """Attach encoded images to every /generate request made in this block."""

{typellm-0.2.3 → typellm-0.2.4/typellm.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: typellm
-Version: 0.2.3
+Version: 0.2.4
 Summary: Type-Safe Decoding for Autoregressive LLMs
 License-Expression: Apache-2.0
 Project-URL: Homepage, https://typellm.ai/