unique_toolkit 1.46.12__tar.gz → 1.46.13__tar.gz

{unique_toolkit-1.46.12 → unique_toolkit-1.46.13}/CHANGELOG.md

@@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+
+## [1.46.13] - 2026-02-23
+- Add easy access to decoder from llm info
+
 ## [1.46.12] - 2026-02-21
 - Add `litellm:gemini-3-1-pro-preview` to `info.py`
 

{unique_toolkit-1.46.12 → unique_toolkit-1.46.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unique_toolkit
-Version: 1.46.12
+Version: 1.46.13
 Summary: 
 License: Proprietary
 Author: Cedric Klinkert
@@ -127,6 +127,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+
+## [1.46.13] - 2026-02-23
+- Add easy access to decoder from llm info
+
 ## [1.46.12] - 2026-02-21
 - Add `litellm:gemini-3-1-pro-preview` to `info.py`
 

{unique_toolkit-1.46.12 → unique_toolkit-1.46.13}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "unique_toolkit"
-version = "1.46.12"
+version = "1.46.13"
 description = ""
 authors = [
     "Cedric Klinkert <cedric.klinkert@unique.ch>",

unique_toolkit-1.46.13/unique_toolkit/language_model/__init__.py

@@ -0,0 +1,79 @@
+from .constants import DOMAIN_NAME
+from .default_language_model import DEFAULT_LANGUAGE_MODEL, DEFAULT_GPT_4o
+from .functions import (
+    stream_complete_with_references_openai as stream_complete_with_references_openai,
+)
+from .infos import LanguageModel, LanguageModelName, TypeDecoder, TypeEncoder
+from .prompt import (
+    Prompt,
+)
+from .schemas import (
+    LanguageModelAssistantMessage,
+    LanguageModelCompletionChoice,
+    LanguageModelFunction,
+    LanguageModelFunctionCall,
+    LanguageModelMessage,
+    LanguageModelMessageRole,
+    LanguageModelMessages,
+    LanguageModelResponse,
+    LanguageModelStreamResponse,
+    LanguageModelStreamResponseMessage,
+    LanguageModelSystemMessage,
+    LanguageModelTokenLimits,
+    LanguageModelTool,
+    LanguageModelToolDescription,
+    LanguageModelToolMessage,
+    LanguageModelToolParameterProperty,
+    LanguageModelToolParameters,
+    LanguageModelUserMessage,
+)
+from .service import (
+    LanguageModelService,
+)
+from .stream_transform import (
+    NormalizationTransform,
+    ReferenceInjectionTransform,
+    StreamTransform,
+    TextTransformPipeline,
+)
+from .utils import (
+    convert_string_to_json,
+    find_last_json_object,
+)
+
+__all__ = [
+    "LanguageModel",
+    "LanguageModelName",
+    "Prompt",
+    "LanguageModelAssistantMessage",
+    "LanguageModelCompletionChoice",
+    "LanguageModelFunction",
+    "LanguageModelFunctionCall",
+    "LanguageModelMessage",
+    "LanguageModelMessageRole",
+    "LanguageModelMessages",
+    "LanguageModelResponse",
+    "LanguageModelStreamResponse",
+    "LanguageModelStreamResponseMessage",
+    "LanguageModelSystemMessage",
+    "LanguageModelTokenLimits",
+    "LanguageModelTool",
+    "LanguageModelToolDescription",
+    "LanguageModelToolMessage",
+    "LanguageModelToolParameterProperty",
+    "LanguageModelToolParameters",
+    "LanguageModelUserMessage",
+    "LanguageModelService",
+    "DOMAIN_NAME",
+    "DEFAULT_LANGUAGE_MODEL",
+    "DEFAULT_GPT_4o",
+    "stream_complete_with_references_openai",
+    "NormalizationTransform",
+    "ReferenceInjectionTransform",
+    "StreamTransform",
+    "TextTransformPipeline",
+    "convert_string_to_json",
+    "find_last_json_object",
+    "TypeDecoder",
+    "TypeEncoder",
+]

{unique_toolkit-1.46.12 → unique_toolkit-1.46.13}/unique_toolkit/language_model/infos.py

@@ -19,6 +19,22 @@ from unique_toolkit.language_model.schemas import LanguageModelTokenLimits
 _LOGGER = logging.getLogger(__name__)
 
 
+TypeEncoder = Callable[[str], list[int]]
+TypeDecoder = Callable[[list[int]], str]
+
+
+@lru_cache(maxsize=10)
+def get_tokenizer_from_path(path: Path) -> Tokenizer:
+    if not path.exists():
+        raise FileNotFoundError(f"Tokenizer not found: {path}")
+    return Tokenizer.from_file(str(path))
+
+
+@lru_cache(maxsize=10)
+def get_tokenizer_from_tiktoken(tokenizer_name: str) -> tiktoken.Encoding:
+    return tiktoken.get_encoding(tokenizer_name)
+
+
 class LanguageModelName(StrEnum):
     AZURE_GPT_35_TURBO_0125 = "AZURE_GPT_35_TURBO_0125"
     AZURE_GPT_4_0613 = "AZURE_GPT_4_0613"
@@ -104,21 +120,27 @@ class EncoderName(StrEnum):
     QWEN = "qwen"
     DEEPSEEK = "deepseek"
 
-    @lru_cache(maxsize=10)
-    def get_encoder(self) -> Callable[[str], list[int]]:
-        if self.value in {"cl100k_base", "o200k_base"}:
-            enc = tiktoken.get_encoding(self.value)
-            return enc.encode
-
+    @property
+    def tokenizer_path(self) -> Path:
         base_path = Path(__file__).parent.parent / "_common" / "token" / "tokenizers"
-        tokenizer_path = base_path / self.value / "tokenizer.json"
+        return base_path / self.value / "tokenizer.json"
 
-        if not tokenizer_path.exists():
-            raise FileNotFoundError(f"Tokenizer not found: {tokenizer_path}")
+    def get_encoder(self) -> TypeEncoder:
+        if self.value in {"cl100k_base", "o200k_base"}:
+            tokenizer = get_tokenizer_from_tiktoken(self.value)
+            return tokenizer.encode
 
-        tokenizer = Tokenizer.from_file(str(tokenizer_path))
+        tokenizer = get_tokenizer_from_path(self.tokenizer_path)
         return lambda text: tokenizer.encode(text).ids
 
+    def get_decoder(self) -> TypeDecoder:
+        if self.value in {"cl100k_base", "o200k_base"}:
+            tokenizer = get_tokenizer_from_tiktoken(self.value)
+            return tokenizer.decode
+
+        tokenizer = get_tokenizer_from_path(self.tokenizer_path)
+        return tokenizer.decode
+
 
 def get_encoder_name(model_name: LanguageModelName) -> EncoderName:
     LMN = LanguageModelName
@@ -198,7 +220,7 @@ class TemperatureBounds(BaseModel):
     max_temperature: float
 
 
-def _load_custom_encoder(tokenizer_name: str) -> Callable[[str], list[int]]:
+def _load_custom_encoder(tokenizer_name: str) -> TypeEncoder:
     """Load a custom tokenizer from UNIQUE_CUSTOM_TOKENIZERS_PATH."""
     custom_path = os.getenv("UNIQUE_CUSTOM_TOKENIZERS_PATH")
     if not custom_path:
@@ -207,13 +229,23 @@ def _load_custom_encoder(tokenizer_name: str) -> Callable[[str], list[int]]:
         )
 
     tokenizer_path = Path(custom_path) / tokenizer_name / "tokenizer.json"
-    if not tokenizer_path.exists():
-        raise FileNotFoundError(f"Custom tokenizer not found: {tokenizer_path}")
 
-    tokenizer = Tokenizer.from_file(str(tokenizer_path))
+    tokenizer = get_tokenizer_from_path(tokenizer_path)
     return lambda text: tokenizer.encode(text).ids
 
 
+def _load_custom_decoder(tokenizer_name: str) -> TypeDecoder:
+    """Load a custom tokenizer from UNIQUE_CUSTOM_TOKENIZERS_PATH."""
+    custom_path = os.getenv("UNIQUE_CUSTOM_TOKENIZERS_PATH")
+    if not custom_path:
+        raise ValueError(
+            "UNIQUE_CUSTOM_TOKENIZERS_PATH must be set to use custom tokenizers"
+        )
+    tokenizer_path = Path(custom_path) / tokenizer_name / "tokenizer.json"
+    tokenizer = get_tokenizer_from_path(tokenizer_path)
+    return tokenizer.decode
+
+
 class LanguageModelInfo(BaseModel):
     model_config = get_configuration_dict()
     name: (
@@ -264,12 +296,17 @@ class LanguageModelInfo(BaseModel):
 
     _ENV_VAR: ClassVar[str] = "LANGUAGE_MODEL_INFOS"
 
-    def get_encoder(self) -> Callable[[str], list[int]]:
+    def get_encoder(self) -> TypeEncoder:
         """Get an encode callable for this model's tokenizer."""
         if isinstance(self.encoder_name, EncoderName):
             return self.encoder_name.get_encoder()
         return _load_custom_encoder(self.encoder_name)
 
+    def get_decoder(self) -> TypeDecoder:
+        if isinstance(self.encoder_name, EncoderName):
+            return self.encoder_name.get_decoder()
+        return _load_custom_decoder(self.encoder_name)
+
     @classmethod
     @lru_cache(maxsize=1)
     def _load_from_env(cls) -> dict[str, dict]:

unique_toolkit-1.46.12/unique_toolkit/language_model/__init__.py

@@ -1,90 +0,0 @@
-from .constants import DOMAIN_NAME as DOMAIN_NAME
-from .default_language_model import (
-    DEFAULT_LANGUAGE_MODEL as DEFAULT_LANGUAGE_MODEL,
-)
-from .default_language_model import (
-    DEFAULT_GPT_4o as DEFAULT_GPT_4o,
-)
-from .functions import (
-    stream_complete_with_references_openai as stream_complete_with_references_openai,
-)
-from .infos import LanguageModel as LanguageModel
-from .infos import LanguageModelName as LanguageModelName
-from .prompt import (
-    Prompt as Prompt,
-)
-from .schemas import (
-    LanguageModelAssistantMessage as LanguageModelAssistantMessage,
-)
-from .schemas import (
-    LanguageModelCompletionChoice as LanguageModelCompletionChoice,
-)
-from .schemas import (
-    LanguageModelFunction as LanguageModelFunction,
-)
-from .schemas import (
-    LanguageModelFunctionCall as LanguageModelFunctionCall,
-)
-from .schemas import (
-    LanguageModelMessage as LanguageModelMessage,
-)
-from .schemas import (
-    LanguageModelMessageRole as LanguageModelMessageRole,
-)
-from .schemas import (
-    LanguageModelMessages as LanguageModelMessages,
-)
-from .schemas import (
-    LanguageModelResponse as LanguageModelResponse,
-)
-from .schemas import (
-    LanguageModelStreamResponse as LanguageModelStreamResponse,
-)
-from .schemas import (
-    LanguageModelStreamResponseMessage as LanguageModelStreamResponseMessage,
-)
-from .schemas import (
-    LanguageModelSystemMessage as LanguageModelSystemMessage,
-)
-from .schemas import (
-    LanguageModelTokenLimits as LanguageModelTokenLimits,
-)
-from .schemas import (
-    LanguageModelTool as LanguageModelTool,
-)
-from .schemas import (
-    LanguageModelToolDescription as LanguageModelToolDescription,
-)
-from .schemas import (
-    LanguageModelToolMessage as LanguageModelToolMessage,
-)
-from .schemas import (
-    LanguageModelToolParameterProperty as LanguageModelToolParameterProperty,
-)
-from .schemas import (
-    LanguageModelToolParameters as LanguageModelToolParameters,
-)
-from .schemas import (
-    LanguageModelUserMessage as LanguageModelUserMessage,
-)
-from .service import (
-    LanguageModelService as LanguageModelService,
-)
-from .stream_transform import (
-    NormalizationTransform as NormalizationTransform,
-)
-from .stream_transform import (
-    ReferenceInjectionTransform as ReferenceInjectionTransform,
-)
-from .stream_transform import (
-    StreamTransform as StreamTransform,
-)
-from .stream_transform import (
-    TextTransformPipeline as TextTransformPipeline,
-)
-from .utils import (
-    convert_string_to_json as convert_string_to_json,
-)
-from .utils import (
-    find_last_json_object as find_last_json_object,
-)