typellm 0.2.3__tar.gz → 0.3.0.dev1__tar.gz

{typellm-0.2.3/typellm.egg-info → typellm-0.3.0.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: typellm
-Version: 0.2.3
+Version: 0.3.0.dev1
 Summary: Type-Safe Decoding for Autoregressive LLMs
 License-Expression: Apache-2.0
 Project-URL: Homepage, https://typellm.ai/
@@ -66,6 +66,36 @@ Evaluated on 231 public [JevBench](https://github.com/fstandhartinger/jevbench)
 
 ## Quick start
 
+### Hosted API (early access)
+
+No GPU? [Early-access](https://typellm.ai/early-access) users get an API key
+by email, and the hosted API runs the model:
+
+```python
+from typellm import TypeLLMClient
+
+client = TypeLLMClient(api_key="YOUR_API_KEY")  # calls https://api.typellm.ai
+```
+
+`generate()` then works as in the example below, with these differences:
+
+- The service compiles and runs the schema. Use `model`, `mode`, `temperature`
+  (for sampling), `seed` and `timeout`; local compilation settings do not apply.
+  A call runs for at most 300 seconds, 60 by default. Set `timeout` on the
+  client for a new default, or on `generate()` to override it for one call.
+- Pass `questions`; `schema=`, `cancel` and `print_final_prompt` need your
+  own server. Images can be files, bytes, PIL images or data: URIs, not URLs.
+- Each key runs a limited number of calls at once, told with the key. Calls
+  beyond it fail right away with HTTP 429.
+- Errors raise `SGLangError` with the HTTP status in `.status`, and a call
+  that runs out of time raises `GenerationTimeout`. The service caps the
+  context size and the images, answering 400 past them, and stops string
+  answers at 128 tokens (`maxLength` is refused).
+- Successful calls set `last_usage` from the service's input and thinking token
+  counts. Failed hosted calls may not return usage.
+
+To run on your own GPU instead, start here:
+
 ### 1. Serve a model with SGLang
 
 Use [SGLang](https://github.com/sgl-project/sglang) to configure and serve a

{typellm-0.2.3 → typellm-0.3.0.dev1}/README.md

@@ -51,6 +51,36 @@ Evaluated on 231 public [JevBench](https://github.com/fstandhartinger/jevbench)
 
 ## Quick start
 
+### Hosted API (early access)
+
+No GPU? [Early-access](https://typellm.ai/early-access) users get an API key
+by email, and the hosted API runs the model:
+
+```python
+from typellm import TypeLLMClient
+
+client = TypeLLMClient(api_key="YOUR_API_KEY")  # calls https://api.typellm.ai
+```
+
+`generate()` then works as in the example below, with these differences:
+
+- The service compiles and runs the schema. Use `model`, `mode`, `temperature`
+  (for sampling), `seed` and `timeout`; local compilation settings do not apply.
+  A call runs for at most 300 seconds, 60 by default. Set `timeout` on the
+  client for a new default, or on `generate()` to override it for one call.
+- Pass `questions`; `schema=`, `cancel` and `print_final_prompt` need your
+  own server. Images can be files, bytes, PIL images or data: URIs, not URLs.
+- Each key runs a limited number of calls at once, told with the key. Calls
+  beyond it fail right away with HTTP 429.
+- Errors raise `SGLangError` with the HTTP status in `.status`, and a call
+  that runs out of time raises `GenerationTimeout`. The service caps the
+  context size and the images, answering 400 past them, and stops string
+  answers at 128 tokens (`maxLength` is refused).
+- Successful calls set `last_usage` from the service's input and thinking token
+  counts. Failed hosted calls may not return usage.
+
+To run on your own GPU instead, start here:
+
 ### 1. Serve a model with SGLang
 
 Use [SGLang](https://github.com/sgl-project/sglang) to configure and serve a

{typellm-0.2.3 → typellm-0.3.0.dev1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "typellm"
-version = "0.2.3"
+version = "0.3.0.dev1"
 description = "Type-Safe Decoding for Autoregressive LLMs"
 readme = "README.md"
 license = "Apache-2.0"

{typellm-0.2.3 → typellm-0.3.0.dev1}/tests/test_typellm.py

@@ -1,9 +1,12 @@
+import json
+import random
 import unittest
 
 import httpx
 
 from typellm.runtime import numeric_pattern
 from typellm import (
+    GenerationTimeout,
     SGLangClient,
     SGLangError,
     SchemaError,
@@ -907,5 +910,118 @@ class JsonSchemaExecutionTests(unittest.TestCase):
         self.assertEqual(scored[0][0], {65: -0.1, 66: -2.0})
         self.assertEqual(scored[1][0], {65: -3.0, 66: -0.2})
 
+
+class HostedApiTests(unittest.TestCase):
+    def client(self, handler, **options):
+        client = TypeLLMClient(api_key="k", **options)
+        client._transport = httpx.MockTransport(handler)
+        return client
+
+    def test_a_call_is_one_request_to_the_hosted_api(self):
+        seen = []
+
+        def handler(request):
+            seen.append(request)
+            return httpx.Response(200, json={
+                "id": "gen_1", "model": "typellm-latest", "result": {"total": 12.5},
+                "thinking": {"total": "The receipt says 12.50."},
+                "usage": {"input_tokens": 40, "thinking_tokens": 9}, "elapsed": 0.5,
+            })
+
+        client = self.client(handler, seed=7)
+        questions = {"total": {"type": "number", "thinking": True}}
+        image = "data:image/png;base64,iVBORw0KGgo="
+        result = client.generate(context="Total: 12.50", questions=questions, images=[image], timeout=90)
+
+        self.assertEqual(result, {"total": 12.5})
+        [request] = seen
+        self.assertEqual(str(request.url), "https://api.typellm.ai/v1/generate")
+        self.assertEqual(request.headers["authorization"], "Bearer k")
+        self.assertEqual(json.loads(request.content), {
+            "context": "Total: 12.50", "questions": questions, "images": [image], "timeout": 90,
+            # Each call draws its seed from the client's seeded stream.
+            "options": {"mode": "argmax", "seed": random.Random(7).randrange(2**32)},
+        })
+        self.assertEqual((client.last_usage.input_tokens, client.last_usage.thinking_tokens), (40, 9))
+        self.assertEqual(client.last_thinking, {"total": "The receipt says 12.50."})
+
+    def test_hosted_temperature_only_applies_to_sampling(self):
+        bodies = []
+
+        def handler(request):
+            bodies.append(json.loads(request.content))
+            return httpx.Response(200, json={
+                "result": {"a": True}, "usage": {"input_tokens": 1, "thinking_tokens": 0},
+            })
+
+        self.client(handler, temperature=0).generate(
+            context="x", questions={"a": {"type": "boolean"}})
+        self.client(handler, mode="sample", temperature=0.7).generate(
+            context="x", questions={"a": {"type": "boolean"}})
+
+        self.assertNotIn("temperature", bodies[0]["options"])
+        self.assertEqual(bodies[1]["options"]["temperature"], 0.7)
+
+    def test_constructor_timeout_sets_hosted_limit_with_per_call_override(self):
+        seen = []
+
+        def handler(request):
+            seen.append(request)
+            return httpx.Response(200, json={
+                "result": {"a": True}, "usage": {"input_tokens": 1, "thinking_tokens": 0},
+            })
+
+        questions = {"a": {"type": "boolean"}}
+        client = self.client(handler, timeout=240)
+        client.generate(context="x", questions=questions)
+        client.generate(context="x", questions=questions, timeout=30)
+        self.client(handler).generate(context="x", questions=questions)
+
+        self.assertEqual([json.loads(request.content).get("timeout") for request in seen],
+                         [240, 30, None])
+        self.assertEqual([request.extensions["timeout"]["read"] for request in seen],
+                         [300, 90, 180])
+        self.assertEqual(TypeLLMClient().sglang.timeout, 120)
+
+    def test_hosted_rejects_invalid_timeouts_before_sending(self):
+        sent = []
+        client = self.client(lambda request: sent.append(request))
+        questions = {"a": {"type": "boolean"}}
+
+        for timeout in (0, -1, True, "5", float("nan")):
+            with self.subTest(timeout=timeout), self.assertRaises(ValueError):
+                client.generate(context="x", questions=questions, timeout=timeout)
+        with self.assertRaises(ValueError):
+            self.client(lambda request: sent.append(request), timeout=0).generate(
+                context="x", questions=questions)
+        self.assertEqual(sent, [])
+
+    def test_api_errors_keep_their_status(self):
+        for status, error in ((401, SGLangError), (429, SGLangError), (504, GenerationTimeout)):
+            client = self.client(lambda request: httpx.Response(status, json={"error": {"message": "no"}}))
+            with self.assertRaises(error) as caught:
+                client.generate(context="x", questions={"a": {"type": "boolean"}})
+            self.assertEqual(caught.exception.status, status)
+        with self.assertRaises(ValueError):  # only local compilation takes a raw schema
+            client.generate(context="x", schema={"type": "object", "properties": {"a": {"type": "boolean"}}})
+
+    def test_hosted_compile_schema_requires_own_server(self):
+        client = TypeLLMClient(api_key="k")
+        with self.assertRaisesRegex(ValueError, "compile_schema needs your own server"):
+            client.compile_schema({"type": "object", "properties": {"a": {"type": "boolean"}}})
+
+    def test_hosted_invalid_responses_fail_clearly(self):
+        questions = {"a": {"type": "boolean"}}
+        client = TypeLLMClient(api_key="k")
+        for response in (httpx.Response(200, text="not JSON"),
+                         httpx.Response(200, json={"result": {"a": True}}),
+                         httpx.Response(302, headers={"Location": "/elsewhere"})):
+            with self.subTest(status=response.status_code):
+                client._transport = httpx.MockTransport(lambda request: response)
+                with self.assertRaises(SGLangError) as caught:
+                    client.generate(context="x", questions=questions)
+                self.assertEqual(caught.exception.status, response.status_code)
+
+
 if __name__ == "__main__":
     unittest.main()

{typellm-0.2.3 → typellm-0.3.0.dev1}/typellm/runtime.py

@@ -16,6 +16,8 @@ from itertools import permutations as all_permutations
 from string import ascii_uppercase, digits
 from typing import Any, Mapping, Sequence
 
+import httpx
+
 from .schema import (
     MAX_ENUM_CHOICES,
     SchemaError,
@@ -23,10 +25,12 @@ from .schema import (
     dependency_layers,
 )
 from .images import encode_images
-from .sglang import SGLangClient, Usage, call_scope
+from .sglang import GenerationTimeout, SGLangClient, SGLangError, Usage, call_scope
 
 
 LOG = logging.getLogger("typellm")
+HOSTED_URL = "https://api.typellm.ai"
+DEFAULT_SOCKET_TIMEOUT = 120.0
 
 
 def _closed_answer(decision: "Choice", value_json: str) -> str:
@@ -140,37 +144,54 @@ class Choice:
 
 
 class TypeLLMClient:
-    """Compile ordered schemas into constrained single-token decisions."""
+    """Generate typed answers with local SGLang or the hosted API."""
 
     DEFAULT_LABEL_POOL = tuple(ascii_uppercase + digits)
 
     def __init__(
         self,
-        base_url: str = "http://127.0.0.1:30000",
+        base_url: str | None = None,
         model: str | None = None,
         *,
+        api_key: str | None = None,
         mode: str = "argmax",
         temperature: float = 1.0,
         seed: int | None = None,
-        timeout: float = 120.0,
+        timeout: float | None = None,
         label_pool: Sequence[str] | None = None,
         numeric_max_digits: int = 32,
         tokenizer: str | None = None,
         thinking_budget: int | None = None,
         text_max_tokens: int = 128,
     ) -> None:
+        """Run on your own SGLang server, by default http://127.0.0.1:30000.
+
+        With api_key, calls go to the hosted API instead, by default
+        https://api.typellm.ai. It compiles and runs the schema itself. Model
+        chooses one of its models; mode, temperature, seed and timeout apply.
+        A hosted timeout set here is the default for generate() calls; without
+        one, the service uses its own default.
+        """
         _validate_decoding(mode, temperature)
         if type(numeric_max_digits) is not int or numeric_max_digits <= 0:
             raise ValueError("numeric_max_digits must be a positive integer")
-        self.sglang = SGLangClient(
-            base_url,
-            model,
-            timeout,
-            tokenizer=tokenizer,
-            thinking_budget=thinking_budget,
-            text_max_tokens=text_max_tokens,
-            answer_reserve_tokens=numeric_max_digits + 3,
-        )
+        self.api_key = api_key
+        if api_key is None:
+            self.sglang = SGLangClient(
+                base_url or "http://127.0.0.1:30000",
+                model,
+                DEFAULT_SOCKET_TIMEOUT if timeout is None else timeout,
+                tokenizer=tokenizer,
+                thinking_budget=thinking_budget,
+                text_max_tokens=text_max_tokens,
+                answer_reserve_tokens=numeric_max_digits + 3,
+            )
+        else:
+            self.sglang = None
+            self.base_url = (base_url or HOSTED_URL).rstrip("/")
+            self.model = model
+            self.timeout = timeout
+            self._transport: httpx.BaseTransport | None = None  # tests put a MockTransport here
         self.mode = mode
         self.temperature = temperature
         self.rng = random.Random(seed)
@@ -215,7 +236,8 @@ class TypeLLMClient:
     def last_usage(self) -> Usage | None:
         """Requests and tokens of the last generate() call made in this thread or task.
 
-        Set even when the call raised, so partial work is still counted.
+        Local calls set this even when they raise, so partial work is counted.
+        Hosted success responses set input_tokens and thinking_tokens.
         """
         return self._last_usage.get()
 
@@ -242,6 +264,8 @@ class TypeLLMClient:
 
     def compile_schema(self, schema: Mapping[str, Any]) -> list[Choice]:
         """Compile standard JSON Schema or the original ordered-list format."""
+        if self.api_key is not None:
+            raise ValueError("compile_schema needs your own server when using api_key")
         if not isinstance(schema, Mapping):
             raise SchemaError("schema must be a mapping")
         if schema.get("type", "object") != "object":
@@ -369,9 +393,10 @@ class TypeLLMClient:
         A seed fixes this call's own random choices and leaves the client's shared
         stream alone; without one, calls share that stream. The server's numerics
         can still vary with its cache and batching, so a seed does not guarantee
-        identical results. timeout caps the whole call in seconds and
-        raises GenerationTimeout; setting cancel raises GenerationCancelled.
-        Both stop the call before its next request to SGLang.
+        identical results. timeout caps the call and raises GenerationTimeout.
+        In local mode, setting cancel raises GenerationCancelled; timeout and
+        cancel stop the call before its next request to SGLang. Hosted mode
+        sends timeout to the service and does not support cancel.
         """
         self._last_usage.set(None)
         self._last_thinking.set({})
@@ -390,6 +415,12 @@ class TypeLLMClient:
         active_mode = self.mode if mode is None else mode
         active_temperature = self.temperature if temperature is None else temperature
         _validate_decoding(active_mode, active_temperature)
+        if self.api_key is not None:
+            if questions is None or cancel is not None or print_final_prompt:
+                raise ValueError("with api_key, generate() takes questions; schema, cancel and "
+                                 "print_final_prompt need your own server")
+            return self._generate_hosted(context, questions, encoded_images, active_mode,
+                                         active_temperature, seed, timeout)
         rng = self.rng if seed is None else random.Random(seed)
         # The time budget covers compiling too: labels are tokenized by SGLang.
         with call_scope(timeout, cancel) as scope:
@@ -437,6 +468,59 @@ class TypeLLMClient:
             _print_final_prompts(prompts)
         return output
 
+    def _generate_hosted(self, context: str, questions: Mapping[str, Any], images: Sequence[str],
+                         mode: str, temperature: float, seed: int | None,
+                         timeout: float | None) -> dict[str, Any]:
+        """One POST /v1/generate. HTTP errors carry their status in SGLangError."""
+        active_timeout = self.timeout if timeout is None else timeout
+        if active_timeout is not None and (type(active_timeout) not in (int, float) or
+                                           not active_timeout > 0):
+            raise ValueError("timeout must be a positive number of seconds or None")
+        options: dict[str, Any] = {
+            "mode": mode,
+            # Without a seed, calls draw theirs from the client's stream, as locally.
+            "seed": self.rng.randrange(2**32) if seed is None else seed,
+        }
+        if mode == "sample":
+            options["temperature"] = temperature
+        body: dict[str, Any] = {
+            "context": context,
+            "questions": questions,
+            "images": list(images),
+            "options": options,
+        }
+        if self.model is not None:
+            body["model"] = self.model
+        if active_timeout is not None:
+            body["timeout"] = active_timeout  # the service's own default (60 s) otherwise
+        try:
+            with httpx.Client(transport=self._transport) as http:
+                # The service times the call itself. The margin covers its queue, image
+                # scaling and the grace it gives its own workers past the timeout.
+                response = http.post(self.base_url + "/v1/generate", json=body,
+                                     headers={"Authorization": f"Bearer {self.api_key}"},
+                                     timeout=(active_timeout if active_timeout is not None
+                                              else DEFAULT_SOCKET_TIMEOUT) + 60)
+        except httpx.HTTPError as exc:
+            raise SGLangError(f"Could not reach the TypeLLM API at {self.base_url}: {exc!r}") from exc
+        if response.status_code != 200:
+            error = GenerationTimeout if response.status_code == 504 else SGLangError
+            raise error(f"TypeLLM API returned HTTP {response.status_code}: {response.text}",
+                        status=response.status_code)
+        try:
+            data = response.json()
+            usage = data["usage"]
+            result = data["result"]
+            thinking = data.get("thinking") or {}
+            input_tokens = usage["input_tokens"]
+            thinking_tokens = usage["thinking_tokens"]
+        except (ValueError, TypeError, KeyError) as exc:
+            raise SGLangError("TypeLLM API returned an invalid response",
+                              status=response.status_code) from exc
+        self._last_usage.set(Usage(input_tokens=input_tokens, thinking_tokens=thinking_tokens))
+        self._last_thinking.set(thinking)
+        return result
+
 
 def candidate_softmax(
     logprobs: Mapping[str, float], temperature: float = 1.0

{typellm-0.2.3 → typellm-0.3.0.dev1/typellm.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: typellm
-Version: 0.2.3
+Version: 0.3.0.dev1
 Summary: Type-Safe Decoding for Autoregressive LLMs
 License-Expression: Apache-2.0
 Project-URL: Homepage, https://typellm.ai/
@@ -66,6 +66,36 @@ Evaluated on 231 public [JevBench](https://github.com/fstandhartinger/jevbench)
 
 ## Quick start
 
+### Hosted API (early access)
+
+No GPU? [Early-access](https://typellm.ai/early-access) users get an API key
+by email, and the hosted API runs the model:
+
+```python
+from typellm import TypeLLMClient
+
+client = TypeLLMClient(api_key="YOUR_API_KEY")  # calls https://api.typellm.ai
+```
+
+`generate()` then works as in the example below, with these differences:
+
+- The service compiles and runs the schema. Use `model`, `mode`, `temperature`
+  (for sampling), `seed` and `timeout`; local compilation settings do not apply.
+  A call runs for at most 300 seconds, 60 by default. Set `timeout` on the
+  client for a new default, or on `generate()` to override it for one call.
+- Pass `questions`; `schema=`, `cancel` and `print_final_prompt` need your
+  own server. Images can be files, bytes, PIL images or data: URIs, not URLs.
+- Each key runs a limited number of calls at once, told with the key. Calls
+  beyond it fail right away with HTTP 429.
+- Errors raise `SGLangError` with the HTTP status in `.status`, and a call
+  that runs out of time raises `GenerationTimeout`. The service caps the
+  context size and the images, answering 400 past them, and stops string
+  answers at 128 tokens (`maxLength` is refused).
+- Successful calls set `last_usage` from the service's input and thinking token
+  counts. Failed hosted calls may not return usage.
+
+To run on your own GPU instead, start here:
+
 ### 1. Serve a model with SGLang
 
 Use [SGLang](https://github.com/sgl-project/sglang) to configure and serve a