typellm 0.3.0.dev1__tar.gz → 0.3.0.dev2__tar.gz

{typellm-0.3.0.dev1/typellm.egg-info → typellm-0.3.0.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: typellm
-Version: 0.3.0.dev1
+Version: 0.3.0.dev2
 Summary: Type-Safe Decoding for Autoregressive LLMs
 License-Expression: Apache-2.0
 Project-URL: Homepage, https://typellm.ai/
@@ -79,8 +79,8 @@ client = TypeLLMClient(api_key="YOUR_API_KEY")  # calls https://api.typellm.ai
 
 `generate()` then works as in the example below, with these differences:
 
-- The service compiles and runs the schema. Use `model`, `mode`, `temperature`
-  (for sampling), `seed` and `timeout`; local compilation settings do not apply.
+- The service compiles and runs the schema. Use `model`, `temperature`, `seed`
+  and `timeout`; local compilation settings do not apply.
   A call runs for at most 300 seconds, 60 by default. Set `timeout` on the
   client for a new default, or on `generate()` to override it for one call.
 - Pass `questions`; `schema=`, `cancel` and `print_final_prompt` need your
@@ -377,19 +377,20 @@ result = client.generate(
 Only opted-in fields return `value` and `probabilities`; other fields return plain values.
 The option is not supported on open Numeric or Text fields.
 
-Argmax is the default. To enable sampling:
+`temperature` is 0 by default: each field gets its most likely answer. Above 0,
+TypeLLM samples at that temperature:
 
 ```python
 client = TypeLLMClient(
     "http://127.0.0.1:30000",
-    mode="sample",
     temperature=0.8,
     seed=42,
 )
 ```
 
 Sampling applies to finite candidates for Choice fields and to token generation
-for Numeric and Text fields. `temperature` controls sampling in each case.
+for Numeric and Text fields. `generate(temperature=...)` sets it for one call.
+The older `mode="argmax"` / `mode="sample"` still works for now but is deprecated.
 
 A seed fixes TypeLLM's own random choices. SGLang's log probabilities can shift
 with its prefix cache and batching, so the same seed may still give different

{typellm-0.3.0.dev1 → typellm-0.3.0.dev2}/README.md

@@ -64,8 +64,8 @@ client = TypeLLMClient(api_key="YOUR_API_KEY")  # calls https://api.typellm.ai
 
 `generate()` then works as in the example below, with these differences:
 
-- The service compiles and runs the schema. Use `model`, `mode`, `temperature`
-  (for sampling), `seed` and `timeout`; local compilation settings do not apply.
+- The service compiles and runs the schema. Use `model`, `temperature`, `seed`
+  and `timeout`; local compilation settings do not apply.
   A call runs for at most 300 seconds, 60 by default. Set `timeout` on the
   client for a new default, or on `generate()` to override it for one call.
 - Pass `questions`; `schema=`, `cancel` and `print_final_prompt` need your
@@ -362,19 +362,20 @@ result = client.generate(
 Only opted-in fields return `value` and `probabilities`; other fields return plain values.
 The option is not supported on open Numeric or Text fields.
 
-Argmax is the default. To enable sampling:
+`temperature` is 0 by default: each field gets its most likely answer. Above 0,
+TypeLLM samples at that temperature:
 
 ```python
 client = TypeLLMClient(
     "http://127.0.0.1:30000",
-    mode="sample",
     temperature=0.8,
     seed=42,
 )
 ```
 
 Sampling applies to finite candidates for Choice fields and to token generation
-for Numeric and Text fields. `temperature` controls sampling in each case.
+for Numeric and Text fields. `generate(temperature=...)` sets it for one call.
+The older `mode="argmax"` / `mode="sample"` still works for now but is deprecated.
 
 A seed fixes TypeLLM's own random choices. SGLang's log probabilities can shift
 with its prefix cache and batching, so the same seed may still give different

{typellm-0.3.0.dev1 → typellm-0.3.0.dev2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "typellm"
-version = "0.3.0.dev1"
+version = "0.3.0.dev2"
 description = "Type-Safe Decoding for Autoregressive LLMs"
 readme = "README.md"
 license = "Apache-2.0"

{typellm-0.3.0.dev1 → typellm-0.3.0.dev2}/tests/test_concurrency.py

@@ -212,6 +212,38 @@ class DeadlineTests(unittest.TestCase):
             client._generate({"text": "x"})
         self.assertNotIsInstance(caught.exception, GenerationTimeout)
 
+    def dropping(self, error, times):
+        """A client whose server drops the first `times` requests with `error`."""
+        seen = []
+
+        def handler(request):
+            seen.append(request)
+            if len(seen) <= times:
+                raise error("connection dropped", request=request)
+            return httpx.Response(200, json={"text": "ok"})
+
+        client = SGLangClient()
+        client._http_client = httpx.Client(transport=httpx.MockTransport(handler))
+        return client, seen
+
+    def test_a_dropped_connection_is_retried_once(self):
+        for error in (httpx.ReadError, httpx.RemoteProtocolError, httpx.WriteError):
+            client, seen = self.dropping(error, 1)
+            with self.subTest(error=error.__name__):
+                self.assertEqual(client._request("/generate", {"text": "x"}), {"text": "ok"})
+                self.assertEqual(len(seen), 2)
+        client, seen = self.dropping(httpx.ReadError, 2)
+        with self.assertRaisesRegex(SGLangError, "Could not reach"):
+            client._request("/generate", {"text": "x"})
+        self.assertEqual(len(seen), 2)
+
+    def test_a_refused_connection_is_not_retried(self):
+        for error in (httpx.ConnectError, httpx.ReadTimeout):
+            client, seen = self.dropping(error, 1)
+            with self.subTest(error=error.__name__), self.assertRaises(SGLangError):
+                client._request("/generate", {"text": "x"})
+            self.assertEqual(len(seen), 1)
+
     def test_invalid_budgets_are_rejected(self):
         class Recording(FakeServer):
             def _request(self, path, payload=None, *, allow_text=False):

{typellm-0.3.0.dev1 → typellm-0.3.0.dev2}/tests/test_grammar_fields.py

@@ -87,7 +87,7 @@ class GrammarNumberTests(unittest.TestCase):
         self.assertIn("null", patterns[3])
 
     def test_sampling_passes_the_temperature_and_no_truncation(self):
-        client = TypeLLMClient(model="fake", mode="sample", temperature=0.7, seed=1)
+        client = TypeLLMClient(model="fake", temperature=0.7, seed=1)
         client.sglang = FakeServer()
         client.generate(context="Receipt", questions={"a": {"type": "integer"}})
         [params] = regex_requests(client.sglang)[0]["sampling_params"]

{typellm-0.3.0.dev1 → typellm-0.3.0.dev2}/tests/test_permutations.py

@@ -39,7 +39,7 @@ class PermutationTests(unittest.TestCase):
 
     def test_sampling_uses_averaged_distribution(self):
         with patch('typellm.runtime._sample', return_value='B') as sample:
-            _client, result = self.run_case(False, ['one', 'two', 'three'], mode='sample', temperature=.5)
+            _client, result = self.run_case(False, ['one', 'two', 'three'], temperature=.5)
         self.assertEqual(result['roll']['value'], 'two')
         self.assertEqual(sample.call_count, 2)  # one final choice plus the boolean
         for p in sample.call_args_list[0].args[0].values():

{typellm-0.3.0.dev1 → typellm-0.3.0.dev2}/tests/test_typellm.py

@@ -956,7 +956,7 @@ class HostedApiTests(unittest.TestCase):
 
         self.client(handler, temperature=0).generate(
             context="x", questions={"a": {"type": "boolean"}})
-        self.client(handler, mode="sample", temperature=0.7).generate(
+        self.client(handler, temperature=0.7).generate(
             context="x", questions={"a": {"type": "boolean"}})
 
         self.assertNotIn("temperature", bodies[0]["options"])
@@ -1025,3 +1025,29 @@ class HostedApiTests(unittest.TestCase):
 
 if __name__ == "__main__":
     unittest.main()
+
+
+class DecodingTests(unittest.TestCase):
+    def test_temperature_alone_decides(self):
+        from typellm.runtime import _resolve_decoding
+
+        self.assertEqual(_resolve_decoding(None, None), ("argmax", 0.0))
+        self.assertEqual(_resolve_decoding(None, 0), ("argmax", 0.0))
+        self.assertEqual(_resolve_decoding(None, 0.7), ("sample", 0.7))
+        for bad in (-1, float("nan"), True, "1"):
+            with self.assertRaises(ValueError):
+                _resolve_decoding(None, bad)
+
+    def test_deprecated_mode_still_works(self):
+        from typellm.runtime import _resolve_decoding
+
+        with self.assertWarns(DeprecationWarning):
+            self.assertEqual(_resolve_decoding("sample", None), ("sample", 1.0))
+        with self.assertWarns(DeprecationWarning):
+            self.assertEqual(_resolve_decoding("argmax", 0.7)[0], "argmax")
+        with self.assertWarns(DeprecationWarning), self.assertRaises(ValueError):
+            _resolve_decoding("sample", 0)
+
+    def test_call_temperature_overrides_client(self):
+        client = TypeLLMClient(model="fake", temperature=0.7)
+        self.assertEqual((client.mode, client.temperature), ("sample", 0.7))

{typellm-0.3.0.dev1 → typellm-0.3.0.dev2}/typellm/cli.py

@@ -51,8 +51,9 @@ def main() -> None:
         default=os.environ.get("TYPELLM_TOKENIZER"),
         help="Tokenizer path or Hugging Face ID; normally discovered from SGLang",
     )
-    parser.add_argument("--mode", choices=("argmax", "sample"), default="argmax")
-    parser.add_argument("--temperature", type=float, default=1.0)
+    parser.add_argument("--temperature", type=float, default=0.0,
+                        help="0 picks the most likely answer; above 0 samples")
+    parser.add_argument("--mode", choices=("argmax", "sample"), help="deprecated; use --temperature")
     parser.add_argument("--numeric-max-digits", type=int, default=32)
     parser.add_argument("--seed", type=int, default=0)
     parser.add_argument("--probabilities", action="store_true")

{typellm-0.3.0.dev1 → typellm-0.3.0.dev2}/typellm/runtime.py

@@ -8,6 +8,7 @@ import math
 import os
 import random
 import threading
+import warnings
 from contextlib import nullcontext
 from contextvars import ContextVar
 from dataclasses import dataclass, replace
@@ -154,8 +155,8 @@ class TypeLLMClient:
         model: str | None = None,
         *,
         api_key: str | None = None,
-        mode: str = "argmax",
-        temperature: float = 1.0,
+        mode: str | None = None,
+        temperature: float | None = None,
         seed: int | None = None,
         timeout: float | None = None,
         label_pool: Sequence[str] | None = None,
@@ -168,11 +169,14 @@ class TypeLLMClient:
 
         With api_key, calls go to the hosted API instead, by default
         https://api.typellm.ai. It compiles and runs the schema itself. Model
-        chooses one of its models; mode, temperature, seed and timeout apply.
+        chooses one of its models; temperature, seed and timeout apply.
         A hosted timeout set here is the default for generate() calls; without
         one, the service uses its own default.
+
+        temperature 0 (the default) picks the most likely answer; above 0 samples
+        at that temperature. mode is deprecated: temperature alone decides.
         """
-        _validate_decoding(mode, temperature)
+        mode, temperature = _resolve_decoding(mode, temperature)
         if type(numeric_max_digits) is not int or numeric_max_digits <= 0:
             raise ValueError("numeric_max_digits must be a positive integer")
         self.api_key = api_key
@@ -412,9 +416,13 @@ class TypeLLMClient:
             if not isinstance(questions, Mapping):
                 raise SchemaError("questions must be a mapping of field names to definitions")
             schema = {"type": "object", "properties": questions}
-        active_mode = self.mode if mode is None else mode
-        active_temperature = self.temperature if temperature is None else temperature
-        _validate_decoding(active_mode, active_temperature)
+        if mode is None and temperature is None:
+            active_mode, active_temperature = self.mode, self.temperature
+        else:
+            # A call's own temperature decides for it, whatever the client's mode.
+            if temperature is None:
+                temperature = self.temperature or None
+            active_mode, active_temperature = _resolve_decoding(mode, temperature)
         if self.api_key is not None:
             if questions is None or cancel is not None or print_final_prompt:
                 raise ValueError("with api_key, generate() takes questions; schema, cancel and "
@@ -546,11 +554,30 @@ def _sample(probs: Mapping[str, float], rng: random.Random) -> str:
     return last
 
 
-def _validate_decoding(mode: str, temperature: float) -> None:
-    if mode not in {"argmax", "sample"}:
-        raise ValueError("mode must be 'argmax' or 'sample'")
-    if mode == "sample" and (not math.isfinite(temperature) or temperature <= 0):
-        raise ValueError("temperature must be finite and > 0 in sample mode")
+def _resolve_decoding(mode: str | None, temperature: float | None) -> tuple[str, float]:
+    """The (mode, temperature) that temperature, and the deprecated mode, ask for.
+
+    temperature 0 picks the most likely answer (argmax); above 0 samples. For
+    older code, mode="sample" without a temperature samples at 1.0 and
+    mode="argmax" is argmax whatever the temperature.
+    """
+    if mode is not None:
+        warnings.warn("mode is deprecated; temperature 0 picks the most likely answer and "
+                      "above 0 samples", DeprecationWarning, stacklevel=3)
+        if mode not in {"argmax", "sample"}:
+            raise ValueError("mode must be 'argmax' or 'sample'")
+        if mode == "sample" and temperature is None:
+            temperature = 1.0
+    if temperature is None:
+        temperature = 0.0
+    if (isinstance(temperature, bool) or not isinstance(temperature, (int, float))
+            or not math.isfinite(temperature) or temperature < 0):
+        raise ValueError("temperature must be a finite number >= 0")
+    if mode == "argmax":
+        return "argmax", float(temperature)
+    if mode == "sample" and temperature == 0:
+        raise ValueError("temperature must be > 0 in sample mode")
+    return ("sample" if temperature > 0 else "argmax"), float(temperature)
 
 
 def _numeric_text_is_complete(text: str, numeric_type: str) -> bool:
@@ -1006,8 +1033,8 @@ def _print_final_prompts(prefixes: Sequence[str]) -> None:
 def run_schema(
     context: str | None = None,
     schema: Mapping[str, Any] | None = None,
-    mode: str = "argmax",
-    temperature: float = 1.0,
+    mode: str | None = None,
+    temperature: float | None = None,
     *,
     state: str | None = None,
     questions: Mapping[str, Any] | None = None,

{typellm-0.3.0.dev1 → typellm-0.3.0.dev2}/typellm/sglang.py

@@ -252,9 +252,11 @@ class SGLangClient:
     def _http(self) -> httpx.Client:
         with self._load_lock:
             if self._http_client is None:
-                # Keep-alive connections: one call makes many small requests.
+                # Keep-alive connections: one call makes many small requests. Idle
+                # ones are dropped before SGLang closes them (5 s), so a request
+                # rarely goes out on a connection the server has just closed.
                 self._http_client = httpx.Client(limits=httpx.Limits(
-                    max_connections=None, max_keepalive_connections=64,
+                    max_connections=None, max_keepalive_connections=64, keepalive_expiry=2.0,
                 ))
             return self._http_client
 
@@ -269,24 +271,15 @@ class SGLangClient:
         scope = _call_scope.get()
         timeout = self.timeout if scope is None else scope.socket_timeout(self.timeout)
         try:
-            # Streamed so a failed body read still reports the status it follows.
-            with self._http().stream(
-                "GET" if payload is None else "POST",
-                self.base_url + path,
-                content=body,
-                headers={"Content-Type": "application/json"},
-                timeout=timeout,
-            ) as response:
-                if response.status_code >= 400:
-                    try:
-                        detail = response.read().decode("utf-8", errors="replace")
-                    except httpx.HTTPError as read_error:
-                        detail = f"Could not read error response: {read_error}"
-                    raise SGLangError(
-                        f"SGLang {path} returned HTTP {response.status_code}: {detail}",
-                        status=response.status_code,
-                    )
-                raw = response.read().decode("utf-8")
+            try:
+                raw = self._send(path, payload is None, body, timeout)
+            except (httpx.ReadError, httpx.WriteError, httpx.RemoteProtocolError):
+                # The connection dropped before an answer, most often a pooled one
+                # the server had just closed. SGLang is fine: try once on a new one.
+                if scope is not None:
+                    scope.check()
+                    timeout = scope.socket_timeout(self.timeout)
+                raw = self._send(path, payload is None, body, timeout)
         except httpx.HTTPError as exc:
             # Connection failures, timeouts, resets and truncated responses.
             raise SGLangError(
@@ -301,6 +294,26 @@ class SGLangClient:
                 f"SGLang {path} returned non-JSON data: {raw[:500]}"
             ) from exc
 
+    def _send(self, path: str, get: bool, body: bytes | None, timeout: float) -> str:
+        # Streamed so a failed body read still reports the status it follows.
+        with self._http().stream(
+            "GET" if get else "POST",
+            self.base_url + path,
+            content=body,
+            headers={"Content-Type": "application/json"},
+            timeout=timeout,
+        ) as response:
+            if response.status_code >= 400:
+                try:
+                    detail = response.read().decode("utf-8", errors="replace")
+                except httpx.HTTPError as read_error:
+                    detail = f"Could not read error response: {read_error}"
+                raise SGLangError(
+                    f"SGLang {path} returned HTTP {response.status_code}: {detail}",
+                    status=response.status_code,
+                )
+            return response.read().decode("utf-8")
+
     @contextmanager
     def images(self, images: Sequence[str]) -> Iterator[None]:
         """Attach encoded images to every /generate request made in this block."""

{typellm-0.3.0.dev1 → typellm-0.3.0.dev2/typellm.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: typellm
-Version: 0.3.0.dev1
+Version: 0.3.0.dev2
 Summary: Type-Safe Decoding for Autoregressive LLMs
 License-Expression: Apache-2.0
 Project-URL: Homepage, https://typellm.ai/
@@ -79,8 +79,8 @@ client = TypeLLMClient(api_key="YOUR_API_KEY")  # calls https://api.typellm.ai
 
 `generate()` then works as in the example below, with these differences:
 
-- The service compiles and runs the schema. Use `model`, `mode`, `temperature`
-  (for sampling), `seed` and `timeout`; local compilation settings do not apply.
+- The service compiles and runs the schema. Use `model`, `temperature`, `seed`
+  and `timeout`; local compilation settings do not apply.
   A call runs for at most 300 seconds, 60 by default. Set `timeout` on the
   client for a new default, or on `generate()` to override it for one call.
 - Pass `questions`; `schema=`, `cancel` and `print_final_prompt` need your
@@ -377,19 +377,20 @@ result = client.generate(
 Only opted-in fields return `value` and `probabilities`; other fields return plain values.
 The option is not supported on open Numeric or Text fields.
 
-Argmax is the default. To enable sampling:
+`temperature` is 0 by default: each field gets its most likely answer. Above 0,
+TypeLLM samples at that temperature:
 
 ```python
 client = TypeLLMClient(
     "http://127.0.0.1:30000",
-    mode="sample",
     temperature=0.8,
     seed=42,
 )
 ```
 
 Sampling applies to finite candidates for Choice fields and to token generation
-for Numeric and Text fields. `temperature` controls sampling in each case.
+for Numeric and Text fields. `generate(temperature=...)` sets it for one call.
+The older `mode="argmax"` / `mode="sample"` still works for now but is deprecated.
 
 A seed fixes TypeLLM's own random choices. SGLang's log probabilities can shift
 with its prefix cache and batching, so the same seed may still give different