typellm 0.2.2__tar.gz → 0.2.3__tar.gz

{typellm-0.2.2/typellm.egg-info → typellm-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: typellm
-Version: 0.2.2
+Version: 0.2.3
 Summary: Type-Safe Decoding for Autoregressive LLMs
 License-Expression: Apache-2.0
 Project-URL: Homepage, https://typellm.ai/
@@ -424,11 +424,14 @@ result = client.generate(
     cancel=cancel,   # set it from another thread; raises GenerationCancelled
 )
 print(client.last_usage)
-# Usage(requests=4, prompt_tokens=1830, cached_tokens=1504, completion_tokens=7, thinking_tokens=0)
+# Usage(requests=4, prompt_tokens=1830, cached_tokens=1504, completion_tokens=7, thinking_tokens=0, input_tokens=410)
 ```
 
-`last_usage` reports the SGLang requests and tokens of the last call made in the
-current thread, including a call that failed partway. A timeout or cancel stops
+`last_usage` reports the tokens of the last call made in the current thread,
+including a call that failed partway. `input_tokens` is what you sent, each part
+counted once: the context, the questions as JSON, and the images. The other
+counts are SGLang's for the call's requests, where every prompt carries the
+shared context. A timeout or cancel stops
 the call before its next SGLang request.
 
 ## Cost analysis

{typellm-0.2.2 → typellm-0.2.3}/README.md

@@ -409,11 +409,14 @@ result = client.generate(
     cancel=cancel,   # set it from another thread; raises GenerationCancelled
 )
 print(client.last_usage)
-# Usage(requests=4, prompt_tokens=1830, cached_tokens=1504, completion_tokens=7, thinking_tokens=0)
+# Usage(requests=4, prompt_tokens=1830, cached_tokens=1504, completion_tokens=7, thinking_tokens=0, input_tokens=410)
 ```
 
-`last_usage` reports the SGLang requests and tokens of the last call made in the
-current thread, including a call that failed partway. A timeout or cancel stops
+`last_usage` reports the tokens of the last call made in the current thread,
+including a call that failed partway. `input_tokens` is what you sent, each part
+counted once: the context, the questions as JSON, and the images. The other
+counts are SGLang's for the call's requests, where every prompt carries the
+shared context. A timeout or cancel stops
 the call before its next SGLang request.
 
 ## Cost analysis

{typellm-0.2.2 → typellm-0.2.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "typellm"
-version = "0.2.2"
+version = "0.2.3"
 description = "Type-Safe Decoding for Autoregressive LLMs"
 readme = "README.md"
 license = "Apache-2.0"

{typellm-0.2.2 → typellm-0.2.3}/tests/test_concurrency.py

@@ -1,3 +1,4 @@
+import json
 import pickle
 import sys
 import threading
@@ -104,10 +105,13 @@ class UsageTests(unittest.TestCase):
         self.assertIsNone(client.last_usage)
         client.generate(context="Receipt", questions=self.QUESTIONS)
         n = prompts_sent(client.sglang)
+        # The input counts once, though every prompt carries it.
+        sent = client.sglang.count_tokens("Receipt") + client.sglang.count_tokens(json.dumps(self.QUESTIONS))
         self.assertEqual(client.last_usage, Usage(
             requests=len(client.sglang.payloads),
-            prompt_tokens=10 * n, cached_tokens=4 * n, completion_tokens=n,
+            prompt_tokens=10 * n, cached_tokens=4 * n, completion_tokens=n, input_tokens=sent,
         ))
+        self.assertGreater(client.last_usage.requests, 1)
         client.sglang.payloads.clear()
         client.generate(context="Receipt", questions={"b": {"type": "boolean"}})
         self.assertEqual(client.last_usage.requests, len(client.sglang.payloads))
@@ -136,6 +140,7 @@ class UsageTests(unittest.TestCase):
         with self.assertRaisesRegex(SGLangError, "went away"):
             client.generate(context="Receipt", questions=self.QUESTIONS)
         self.assertEqual(client.last_usage.requests, 2)
+        self.assertGreater(client.last_usage.input_tokens, 0)
         with self.assertRaises(ValueError):
             client.generate(questions=self.QUESTIONS)
         self.assertIsNone(client.last_usage)

{typellm-0.2.2 → typellm-0.2.3}/tests/test_images.py

@@ -271,6 +271,21 @@ class ImageRequestTests(unittest.TestCase):
         self.assertEqual(rest, [])
 
 
+class ImageInputTokenTests(unittest.TestCase):
+    def test_images_count_as_the_server_expands_them(self):
+        client = TypeLLMClient(model="fake-vl")
+        client.sglang = FakeServerClient()
+        questions = {"a": {"type": "boolean"}}
+        client.generate(context="Receipt", images=[PNG], questions=questions)
+        first = client.sglang.generate_payloads[0]["text"]
+        first = first if isinstance(first, str) else first[0]
+        count = client.sglang.count_tokens
+        # The server reported 900 prompt tokens, with the placeholder expanded.
+        image = 900 - count(first) + count(VISION)
+        self.assertEqual(client.last_usage.input_tokens,
+                         count("Receipt") + count('{"a": {"type": "boolean"}}') + image)
+
+
 class RuntimeContentTests(unittest.TestCase):
     def test_fake_backends_receive_image_parts(self):
         class Recording(FakeSGLang):

{typellm-0.2.2 → typellm-0.2.3}/typellm/runtime.py

@@ -395,6 +395,11 @@ class TypeLLMClient:
         with call_scope(timeout, cancel) as scope:
             try:
                 decisions = self.compile_schema(schema)
+                count = getattr(self.sglang, "count_tokens", None)
+                if callable(count):
+                    scope.usage.input_tokens = sum(map(count, (
+                        context, json.dumps(questions if questions is not None else schema, ensure_ascii=False))))
+                    scope.unmeasured_images = len(encoded_images)
                 # Independent fields run together; depends_on turns the fields into a
                 # graph whose layers run in order.
                 run = (_execute_dependency_decisions if any(d.depends_on is not None for d in decisions)

{typellm-0.2.2 → typellm-0.2.3}/typellm/sglang.py

@@ -37,7 +37,13 @@ class GenerationCancelled(RuntimeError):
 
 @dataclass
 class Usage:
-    """What SGLang reported for the /generate requests of one generate() call."""
+    """The tokens of one generate() call.
+
+    input_tokens is what the caller sent, each part counted once: the context,
+    the questions or schema as JSON, and the images as the server expands them.
+    The other counts are what SGLang reported for the call's /generate requests,
+    where the shared context is part of every prompt.
+    """
 
     requests: int = 0
     prompt_tokens: int = 0
@@ -45,6 +51,7 @@ class Usage:
     completion_tokens: int = 0
     # The part of completion_tokens that is reasoning, not typed answers.
     thinking_tokens: int = 0
+    input_tokens: int = 0
 
     def _add(self, response: Any) -> None:
         self.requests += 1
@@ -75,6 +82,8 @@ class _CallScope:
     usage: Usage = field(default_factory=Usage)
     deadline: float | None = None  # time.monotonic() when the call runs out of time
     cancel: threading.Event | None = None
+    # Images whose tokens input_tokens still lacks; the first response measures them.
+    unmeasured_images: int = 0
 
     def expired(self) -> bool:
         return self.deadline is not None and time.monotonic() >= self.deadline
@@ -351,8 +360,27 @@ class SGLangClient:
                 raise GenerationTimeout("generate() ran out of time") from exc
             raise
         scope.usage._add(response)
+        if scope.unmeasured_images:
+            self._measure_images(scope, payload, response)
         return response
 
+    def count_tokens(self, text: str) -> int:
+        return len(self._get_chat_tokenizer().encode(text, add_special_tokens=False))
+
+    def _measure_images(self, scope: _CallScope, payload: Mapping[str, Any], response: Any) -> None:
+        """Add the images' tokens to input_tokens: the server's prompt count less the local one."""
+        text = payload.get("text")
+        prompt = text if isinstance(text, str) else text[0] if isinstance(text, list) and text else None
+        item = response[0] if isinstance(response, list) and response else response
+        meta = item.get("meta_info") if isinstance(item, Mapping) else None
+        served = meta.get("prompt_tokens") if isinstance(meta, Mapping) else None
+        if not isinstance(prompt, str) or type(served) is not int:
+            return
+        # The server replaces each placeholder with the image's tokens.
+        placeholders = scope.unmeasured_images * self.count_tokens(self.image_placeholder())
+        scope.usage.input_tokens += max(0, served - self.count_tokens(prompt) + placeholders)
+        scope.unmeasured_images = 0
+
     def _with_images(self, payload: Mapping[str, Any]) -> Mapping[str, Any]:
         images = self._active_images.get()
         if not images:

{typellm-0.2.2 → typellm-0.2.3/typellm.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: typellm
-Version: 0.2.2
+Version: 0.2.3
 Summary: Type-Safe Decoding for Autoregressive LLMs
 License-Expression: Apache-2.0
 Project-URL: Homepage, https://typellm.ai/
@@ -424,11 +424,14 @@ result = client.generate(
     cancel=cancel,   # set it from another thread; raises GenerationCancelled
 )
 print(client.last_usage)
-# Usage(requests=4, prompt_tokens=1830, cached_tokens=1504, completion_tokens=7, thinking_tokens=0)
+# Usage(requests=4, prompt_tokens=1830, cached_tokens=1504, completion_tokens=7, thinking_tokens=0, input_tokens=410)
 ```
 
-`last_usage` reports the SGLang requests and tokens of the last call made in the
-current thread, including a call that failed partway. A timeout or cancel stops
+`last_usage` reports the tokens of the last call made in the current thread,
+including a call that failed partway. `input_tokens` is what you sent, each part
+counted once: the context, the questions as JSON, and the images. The other
+counts are SGLang's for the call's requests, where every prompt carries the
+shared context. A timeout or cancel stops
 the call before its next SGLang request.
 
 ## Cost analysis