typellm 0.2.3__tar.gz → 0.2.5__tar.gz

{typellm-0.2.3/typellm.egg-info → typellm-0.2.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: typellm
-Version: 0.2.3
+Version: 0.2.5
 Summary: Type-Safe Decoding for Autoregressive LLMs
 License-Expression: Apache-2.0
 Project-URL: Homepage, https://typellm.ai/
@@ -347,19 +347,20 @@ result = client.generate(
 Only opted-in fields return `value` and `probabilities`; other fields return plain values.
 The option is not supported on open Numeric or Text fields.
 
-Argmax is the default. To enable sampling:
+`temperature` is 0 by default: each field gets its most likely answer. Above 0,
+TypeLLM samples at that temperature:
 
 ```python
 client = TypeLLMClient(
     "http://127.0.0.1:30000",
-    mode="sample",
     temperature=0.8,
     seed=42,
 )
 ```
 
 Sampling applies to finite candidates for Choice fields and to token generation
-for Numeric and Text fields. `temperature` controls sampling in each case.
+for Numeric and Text fields. `generate(temperature=...)` sets it for one call.
+The older `mode="argmax"` / `mode="sample"` still works for now but is deprecated.
 
 A seed fixes TypeLLM's own random choices. SGLang's log probabilities can shift
 with its prefix cache and batching, so the same seed may still give different

{typellm-0.2.3 → typellm-0.2.5}/README.md

@@ -332,19 +332,20 @@ result = client.generate(
 Only opted-in fields return `value` and `probabilities`; other fields return plain values.
 The option is not supported on open Numeric or Text fields.
 
-Argmax is the default. To enable sampling:
+`temperature` is 0 by default: each field gets its most likely answer. Above 0,
+TypeLLM samples at that temperature:
 
 ```python
 client = TypeLLMClient(
     "http://127.0.0.1:30000",
-    mode="sample",
     temperature=0.8,
     seed=42,
 )
 ```
 
 Sampling applies to finite candidates for Choice fields and to token generation
-for Numeric and Text fields. `temperature` controls sampling in each case.
+for Numeric and Text fields. `generate(temperature=...)` sets it for one call.
+The older `mode="argmax"` / `mode="sample"` still works for now but is deprecated.
 
 A seed fixes TypeLLM's own random choices. SGLang's log probabilities can shift
 with its prefix cache and batching, so the same seed may still give different

{typellm-0.2.3 → typellm-0.2.5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "typellm"
-version = "0.2.3"
+version = "0.2.5"
 description = "Type-Safe Decoding for Autoregressive LLMs"
 readme = "README.md"
 license = "Apache-2.0"

{typellm-0.2.3 → typellm-0.2.5}/tests/test_concurrency.py

@@ -212,6 +212,38 @@ class DeadlineTests(unittest.TestCase):
             client._generate({"text": "x"})
         self.assertNotIsInstance(caught.exception, GenerationTimeout)
 
+    def dropping(self, error, times):
+        """A client whose server drops the first `times` requests with `error`."""
+        seen = []
+
+        def handler(request):
+            seen.append(request)
+            if len(seen) <= times:
+                raise error("connection dropped", request=request)
+            return httpx.Response(200, json={"text": "ok"})
+
+        client = SGLangClient()
+        client._http_client = httpx.Client(transport=httpx.MockTransport(handler))
+        return client, seen
+
+    def test_a_dropped_connection_is_retried_once(self):
+        for error in (httpx.ReadError, httpx.RemoteProtocolError, httpx.WriteError):
+            client, seen = self.dropping(error, 1)
+            with self.subTest(error=error.__name__):
+                self.assertEqual(client._request("/generate", {"text": "x"}), {"text": "ok"})
+                self.assertEqual(len(seen), 2)
+        client, seen = self.dropping(httpx.ReadError, 2)
+        with self.assertRaisesRegex(SGLangError, "Could not reach"):
+            client._request("/generate", {"text": "x"})
+        self.assertEqual(len(seen), 2)
+
+    def test_a_refused_connection_is_not_retried(self):
+        for error in (httpx.ConnectError, httpx.ReadTimeout):
+            client, seen = self.dropping(error, 1)
+            with self.subTest(error=error.__name__), self.assertRaises(SGLangError):
+                client._request("/generate", {"text": "x"})
+            self.assertEqual(len(seen), 1)
+
     def test_invalid_budgets_are_rejected(self):
         class Recording(FakeServer):
             def _request(self, path, payload=None, *, allow_text=False):

{typellm-0.2.3 → typellm-0.2.5}/tests/test_grammar_fields.py

@@ -87,7 +87,7 @@ class GrammarNumberTests(unittest.TestCase):
         self.assertIn("null", patterns[3])
 
     def test_sampling_passes_the_temperature_and_no_truncation(self):
-        client = TypeLLMClient(model="fake", mode="sample", temperature=0.7, seed=1)
+        client = TypeLLMClient(model="fake", temperature=0.7, seed=1)
         client.sglang = FakeServer()
         client.generate(context="Receipt", questions={"a": {"type": "integer"}})
         [params] = regex_requests(client.sglang)[0]["sampling_params"]

{typellm-0.2.3 → typellm-0.2.5}/tests/test_permutations.py

@@ -39,7 +39,7 @@ class PermutationTests(unittest.TestCase):
 
     def test_sampling_uses_averaged_distribution(self):
         with patch('typellm.runtime._sample', return_value='B') as sample:
-            _client, result = self.run_case(False, ['one', 'two', 'three'], mode='sample', temperature=.5)
+            _client, result = self.run_case(False, ['one', 'two', 'three'], temperature=.5)
         self.assertEqual(result['roll']['value'], 'two')
         self.assertEqual(sample.call_count, 2)  # one final choice plus the boolean
         for p in sample.call_args_list[0].args[0].values():

{typellm-0.2.3 → typellm-0.2.5}/tests/test_typellm.py

@@ -909,3 +909,29 @@ class JsonSchemaExecutionTests(unittest.TestCase):
 
 if __name__ == "__main__":
     unittest.main()
+
+
+class DecodingTests(unittest.TestCase):
+    def test_temperature_alone_decides(self):
+        from typellm.runtime import _resolve_decoding
+
+        self.assertEqual(_resolve_decoding(None, None), ("argmax", 0.0))
+        self.assertEqual(_resolve_decoding(None, 0), ("argmax", 0.0))
+        self.assertEqual(_resolve_decoding(None, 0.7), ("sample", 0.7))
+        for bad in (-1, float("nan"), True, "1"):
+            with self.assertRaises(ValueError):
+                _resolve_decoding(None, bad)
+
+    def test_deprecated_mode_still_works(self):
+        from typellm.runtime import _resolve_decoding
+
+        with self.assertWarns(DeprecationWarning):
+            self.assertEqual(_resolve_decoding("sample", None), ("sample", 1.0))
+        with self.assertWarns(DeprecationWarning):
+            self.assertEqual(_resolve_decoding("argmax", 0.7)[0], "argmax")
+        with self.assertWarns(DeprecationWarning), self.assertRaises(ValueError):
+            _resolve_decoding("sample", 0)
+
+    def test_call_temperature_overrides_client(self):
+        client = TypeLLMClient(model="fake", temperature=0.7)
+        self.assertEqual((client.mode, client.temperature), ("sample", 0.7))

{typellm-0.2.3 → typellm-0.2.5}/typellm/cli.py

@@ -51,8 +51,9 @@ def main() -> None:
         default=os.environ.get("TYPELLM_TOKENIZER"),
         help="Tokenizer path or Hugging Face ID; normally discovered from SGLang",
     )
-    parser.add_argument("--mode", choices=("argmax", "sample"), default="argmax")
-    parser.add_argument("--temperature", type=float, default=1.0)
+    parser.add_argument("--temperature", type=float, default=0.0,
+                        help="0 picks the most likely answer; above 0 samples")
+    parser.add_argument("--mode", choices=("argmax", "sample"), help="deprecated; use --temperature")
     parser.add_argument("--numeric-max-digits", type=int, default=32)
     parser.add_argument("--seed", type=int, default=0)
     parser.add_argument("--probabilities", action="store_true")

{typellm-0.2.3 → typellm-0.2.5}/typellm/runtime.py

@@ -8,6 +8,7 @@ import math
 import os
 import random
 import threading
+import warnings
 from contextlib import nullcontext
 from contextvars import ContextVar
 from dataclasses import dataclass, replace
@@ -149,8 +150,8 @@ class TypeLLMClient:
         base_url: str = "http://127.0.0.1:30000",
         model: str | None = None,
         *,
-        mode: str = "argmax",
-        temperature: float = 1.0,
+        mode: str | None = None,
+        temperature: float | None = None,
         seed: int | None = None,
         timeout: float = 120.0,
         label_pool: Sequence[str] | None = None,
@@ -159,7 +160,10 @@ class TypeLLMClient:
         thinking_budget: int | None = None,
         text_max_tokens: int = 128,
     ) -> None:
-        _validate_decoding(mode, temperature)
+        """temperature 0 (the default) picks the most likely answer; above 0
+        samples at that temperature. mode is deprecated: temperature alone decides.
+        """
+        mode, temperature = _resolve_decoding(mode, temperature)
         if type(numeric_max_digits) is not int or numeric_max_digits <= 0:
             raise ValueError("numeric_max_digits must be a positive integer")
         self.sglang = SGLangClient(
@@ -387,9 +391,13 @@ class TypeLLMClient:
             if not isinstance(questions, Mapping):
                 raise SchemaError("questions must be a mapping of field names to definitions")
             schema = {"type": "object", "properties": questions}
-        active_mode = self.mode if mode is None else mode
-        active_temperature = self.temperature if temperature is None else temperature
-        _validate_decoding(active_mode, active_temperature)
+        if mode is None and temperature is None:
+            active_mode, active_temperature = self.mode, self.temperature
+        else:
+            # A call's own temperature decides for it, whatever the client's mode.
+            if temperature is None:
+                temperature = self.temperature or None
+            active_mode, active_temperature = _resolve_decoding(mode, temperature)
         rng = self.rng if seed is None else random.Random(seed)
         # The time budget covers compiling too: labels are tokenized by SGLang.
         with call_scope(timeout, cancel) as scope:
@@ -462,11 +470,30 @@ def _sample(probs: Mapping[str, float], rng: random.Random) -> str:
     return last
 
 
-def _validate_decoding(mode: str, temperature: float) -> None:
-    if mode not in {"argmax", "sample"}:
-        raise ValueError("mode must be 'argmax' or 'sample'")
-    if mode == "sample" and (not math.isfinite(temperature) or temperature <= 0):
-        raise ValueError("temperature must be finite and > 0 in sample mode")
+def _resolve_decoding(mode: str | None, temperature: float | None) -> tuple[str, float]:
+    """The (mode, temperature) that temperature, and the deprecated mode, ask for.
+
+    temperature 0 picks the most likely answer (argmax); above 0 samples. For
+    older code, mode="sample" without a temperature samples at 1.0 and
+    mode="argmax" is argmax whatever the temperature.
+    """
+    if mode is not None:
+        warnings.warn("mode is deprecated; temperature 0 picks the most likely answer and "
+                      "above 0 samples", DeprecationWarning, stacklevel=3)
+        if mode not in {"argmax", "sample"}:
+            raise ValueError("mode must be 'argmax' or 'sample'")
+        if mode == "sample" and temperature is None:
+            temperature = 1.0
+    if temperature is None:
+        temperature = 0.0
+    if (isinstance(temperature, bool) or not isinstance(temperature, (int, float))
+            or not math.isfinite(temperature) or temperature < 0):
+        raise ValueError("temperature must be a finite number >= 0")
+    if mode == "argmax":
+        return "argmax", float(temperature)
+    if mode == "sample" and temperature == 0:
+        raise ValueError("temperature must be > 0 in sample mode")
+    return ("sample" if temperature > 0 else "argmax"), float(temperature)
 
 
 def _numeric_text_is_complete(text: str, numeric_type: str) -> bool:
@@ -922,8 +949,8 @@ def _print_final_prompts(prefixes: Sequence[str]) -> None:
 def run_schema(
     context: str | None = None,
     schema: Mapping[str, Any] | None = None,
-    mode: str = "argmax",
-    temperature: float = 1.0,
+    mode: str | None = None,
+    temperature: float | None = None,
     *,
     state: str | None = None,
     questions: Mapping[str, Any] | None = None,

{typellm-0.2.3 → typellm-0.2.5}/typellm/sglang.py

@@ -252,9 +252,11 @@ class SGLangClient:
     def _http(self) -> httpx.Client:
         with self._load_lock:
             if self._http_client is None:
-                # Keep-alive connections: one call makes many small requests.
+                # Keep-alive connections: one call makes many small requests. Idle
+                # ones are dropped before SGLang closes them (5 s), so a request
+                # rarely goes out on a connection the server has just closed.
                 self._http_client = httpx.Client(limits=httpx.Limits(
-                    max_connections=None, max_keepalive_connections=64,
+                    max_connections=None, max_keepalive_connections=64, keepalive_expiry=2.0,
                 ))
             return self._http_client
 
@@ -269,24 +271,15 @@ class SGLangClient:
         scope = _call_scope.get()
         timeout = self.timeout if scope is None else scope.socket_timeout(self.timeout)
         try:
-            # Streamed so a failed body read still reports the status it follows.
-            with self._http().stream(
-                "GET" if payload is None else "POST",
-                self.base_url + path,
-                content=body,
-                headers={"Content-Type": "application/json"},
-                timeout=timeout,
-            ) as response:
-                if response.status_code >= 400:
-                    try:
-                        detail = response.read().decode("utf-8", errors="replace")
-                    except httpx.HTTPError as read_error:
-                        detail = f"Could not read error response: {read_error}"
-                    raise SGLangError(
-                        f"SGLang {path} returned HTTP {response.status_code}: {detail}",
-                        status=response.status_code,
-                    )
-                raw = response.read().decode("utf-8")
+            try:
+                raw = self._send(path, payload is None, body, timeout)
+            except (httpx.ReadError, httpx.WriteError, httpx.RemoteProtocolError):
+                # The connection dropped before an answer, most often a pooled one
+                # the server had just closed. SGLang is fine: try once on a new one.
+                if scope is not None:
+                    scope.check()
+                    timeout = scope.socket_timeout(self.timeout)
+                raw = self._send(path, payload is None, body, timeout)
         except httpx.HTTPError as exc:
             # Connection failures, timeouts, resets and truncated responses.
             raise SGLangError(
@@ -301,6 +294,26 @@ class SGLangClient:
                 f"SGLang {path} returned non-JSON data: {raw[:500]}"
             ) from exc
 
+    def _send(self, path: str, get: bool, body: bytes | None, timeout: float) -> str:
+        # Streamed so a failed body read still reports the status it follows.
+        with self._http().stream(
+            "GET" if get else "POST",
+            self.base_url + path,
+            content=body,
+            headers={"Content-Type": "application/json"},
+            timeout=timeout,
+        ) as response:
+            if response.status_code >= 400:
+                try:
+                    detail = response.read().decode("utf-8", errors="replace")
+                except httpx.HTTPError as read_error:
+                    detail = f"Could not read error response: {read_error}"
+                raise SGLangError(
+                    f"SGLang {path} returned HTTP {response.status_code}: {detail}",
+                    status=response.status_code,
+                )
+            return response.read().decode("utf-8")
+
     @contextmanager
     def images(self, images: Sequence[str]) -> Iterator[None]:
         """Attach encoded images to every /generate request made in this block."""

{typellm-0.2.3 → typellm-0.2.5/typellm.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: typellm
-Version: 0.2.3
+Version: 0.2.5
 Summary: Type-Safe Decoding for Autoregressive LLMs
 License-Expression: Apache-2.0
 Project-URL: Homepage, https://typellm.ai/
@@ -347,19 +347,20 @@ result = client.generate(
 Only opted-in fields return `value` and `probabilities`; other fields return plain values.
 The option is not supported on open Numeric or Text fields.
 
-Argmax is the default. To enable sampling:
+`temperature` is 0 by default: each field gets its most likely answer. Above 0,
+TypeLLM samples at that temperature:
 
 ```python
 client = TypeLLMClient(
     "http://127.0.0.1:30000",
-    mode="sample",
     temperature=0.8,
     seed=42,
 )
 ```
 
 Sampling applies to finite candidates for Choice fields and to token generation
-for Numeric and Text fields. `temperature` controls sampling in each case.
+for Numeric and Text fields. `generate(temperature=...)` sets it for one call.
+The older `mode="argmax"` / `mode="sample"` still works for now but is deprecated.
 
 A seed fixes TypeLLM's own random choices. SGLang's log probabilities can shift
 with its prefix cache and batching, so the same seed may still give different