unique_toolkit 0.8.49__py3-none-any.whl → 0.8.51__py3-none-any.whl

unique_toolkit/_common/token/token_counting.py

@@ -4,11 +4,11 @@
 import json
 from typing import Any, Callable
 
-from pydantic import BaseModel
-
-from unique_toolkit._common.token.image_token_counting import (
+from _common.utils.token.image_token_counting import (
     calculate_image_tokens_from_base64,
 )
+from pydantic import BaseModel
+
 from unique_toolkit.language_model import (
     LanguageModelMessage,
     LanguageModelMessages,
@@ -71,11 +71,25 @@ def num_tokens_per_messages(
     for message in messages:
         num_tokens = 3  # extra_tokens_per_message
         for key, value in message.items():
-            if isinstance(value, str):
+            if key == "content":
+                if message.get("role") == "tool":
+                    """
+                    We have observed a general difference in the way tool response messages are handled.
+                    Specifically, if we take a list of tool responses and artificially transform them into user messages (content field stays the same)
+                    the token consumption goes does drastically, this seems to scale with the number of tokens in the tool responses.
+
+                    the json.dumps() method was found by trial and error. It will give a conservative estimate, but seems to be close to the token count
+                    returned from the openai call.
+                    """
+                    num_tokens += len(encode(json.dumps(value)))
+                elif isinstance(value, list):
+                    # NOTE: The result returned by the function below is not 100% accurate.
+                    num_tokens += handle_message_with_images(value, encode)
+                else:
+                    num_tokens += len(encode(value))
+            elif isinstance(value, str):
                 num_tokens += len(encode(value))
-            elif isinstance(value, list):
-                # NOTE: The result returned by the function below is not 100% accurate.
-                num_tokens += handle_message_with_images(value, encode)
+
             if key == "name":
                 num_tokens += 1  # extra_tokens_per_name
 
@@ -163,8 +177,14 @@ def messages_to_openai_messages(
         messages = LanguageModelMessages(messages)
 
     return [
-        {k: v for k, v in m.items() if (k in ["content", "role"] and v is not None)}
-        for m in json.loads(messages.model_dump_json())
+        {
+            k: v
+            for k, v in m.items()
+            if (
+                k in ["content", "role", "name"] and v is not None
+            )  # Ignore tool_calls for now
+        }
+        for m in messages.model_dump(mode="json")
     ]
 
 

{unique_toolkit-0.8.49.dist-info → unique_toolkit-0.8.51.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unique_toolkit
-Version: 0.8.49
+Version: 0.8.51
 Summary: 
 License: Proprietary
 Author: Cedric Klinkert
@@ -117,6 +117,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.8.51] - 2025-09-06
+- Update token counter to latest version of monorepo.
+
+## [0.8.50] - 2025-09-08
+- Minor fix in documentation
+- Updated examples
+
 ## [0.8.49] - 2025-09-05
 - Fixed token reducer now has a safety margin of 10% less did not work.
 

{unique_toolkit-0.8.49.dist-info → unique_toolkit-0.8.51.dist-info}/RECORD

@@ -14,7 +14,7 @@ unique_toolkit/_common/exception.py,sha256=caQIE1btsQnpKCHqL2cgWUSbHup06enQu_Pt7
 unique_toolkit/_common/feature_flags/schema.py,sha256=3JpTuld8kK-UQ5B0sbYTu0yqhyFPnChXG2Iv4BNqHdg,539
 unique_toolkit/_common/pydantic_helpers.py,sha256=4a8LPey31k4dCztYag1OBhYnGHREN08-l3NEjbFD1ok,743
 unique_toolkit/_common/token/image_token_counting.py,sha256=VpFfZyY0GIH27q_Wy4YNjk2algqvbCtJyzuuROoFQPw,2189
-unique_toolkit/_common/token/token_counting.py,sha256=Jo5B11GLlnaZDi4u7xCzIboLl8zn5cY_dmrogHixVdk,6132
+unique_toolkit/_common/token/token_counting.py,sha256=cg7RPmmotZWofrGMrphaRSr2ch9mT7tMKl9AJXyYGWo,7112
 unique_toolkit/_common/utils/structured_output/schema.py,sha256=Tp7kDYcmKtnUhcuRkH86TSYhylRff0ZZJYb2dLkISts,131
 unique_toolkit/_common/validate_required_values.py,sha256=Y_M1ub9gIKP9qZ45F6Zq3ZHtuIqhmOjl8Z2Vd3avg8w,588
 unique_toolkit/_common/validators.py,sha256=aZwbMho7XszN7lT5RtemaiXgC0WJ4u40oeVgsNGhF4U,2803
@@ -118,7 +118,7 @@ unique_toolkit/tools/utils/execution/execution.py,sha256=vjG2Y6awsGNtlvyQAGCTthQ
 unique_toolkit/tools/utils/source_handling/schema.py,sha256=vzAyf6ZWNexjMO0OrnB8y2glGkvAilmGGQXd6zcDaKw,870
 unique_toolkit/tools/utils/source_handling/source_formatting.py,sha256=C7uayNbdkNVJdEARA5CENnHtNY1SU6etlaqbgHNyxaQ,9152
 unique_toolkit/tools/utils/source_handling/tests/test_source_formatting.py,sha256=oM5ZxEgzROrnX1229KViCAFjRxl9wCTzWZoinYSHleM,6979
-unique_toolkit-0.8.49.dist-info/LICENSE,sha256=GlN8wHNdh53xwOPg44URnwag6TEolCjoq3YD_KrWgss,193
-unique_toolkit-0.8.49.dist-info/METADATA,sha256=4EVSBrsj4dhQW8rwBS-jWGhqXZZDe0nR30DoGFsZ1Do,31103
-unique_toolkit-0.8.49.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-unique_toolkit-0.8.49.dist-info/RECORD,,
+unique_toolkit-0.8.51.dist-info/LICENSE,sha256=GlN8wHNdh53xwOPg44URnwag6TEolCjoq3YD_KrWgss,193
+unique_toolkit-0.8.51.dist-info/METADATA,sha256=4tMl3Lf8D4sKZKgXRTZf8fPes-6qSK75P0dY5HzURgA,31257
+unique_toolkit-0.8.51.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+unique_toolkit-0.8.51.dist-info/RECORD,,