speedy-utils 1.1.34__py3-none-any.whl → 1.1.35__py3-none-any.whl

llm_utils/lm/__init__.py

@@ -7,6 +7,7 @@ from .lm_base import LMBase, get_model_name
 from .mixins import (
     ModelUtilsMixin,
     TemperatureRangeMixin,
+    TokenizationMixin,
     TwoStepPydanticMixin,
     VLLMMixin,
 )
@@ -14,19 +15,20 @@ from .signature import Input, InputField, Output, OutputField, Signature
 
 
 __all__ = [
-    "LMBase",
-    "LLM",
-    "AsyncLM",
-    "AsyncLLMTask",
-    "BasePromptBuilder",
-    "LLMSignature",
-    "Signature",
-    "InputField",
-    "OutputField",
-    "Input",
-    "Output",
-    "TemperatureRangeMixin",
-    "TwoStepPydanticMixin",
-    "VLLMMixin",
-    "ModelUtilsMixin",
+    'LMBase',
+    'LLM',
+    'AsyncLM',
+    'AsyncLLMTask',
+    'BasePromptBuilder',
+    'LLMSignature',
+    'Signature',
+    'InputField',
+    'OutputField',
+    'Input',
+    'Output',
+    'TemperatureRangeMixin',
+    'TwoStepPydanticMixin',
+    'VLLMMixin',
+    'ModelUtilsMixin',
+    'TokenizationMixin',
 ]

llm_utils/lm/llm.py

@@ -20,6 +20,7 @@ from .base_prompt_builder import BasePromptBuilder
 from .mixins import (
     ModelUtilsMixin,
     TemperatureRangeMixin,
+    TokenizationMixin,
     TwoStepPydanticMixin,
     VLLMMixin,
 )
@@ -47,6 +48,7 @@ class LLM(
     TwoStepPydanticMixin,
     VLLMMixin,
     ModelUtilsMixin,
+    TokenizationMixin,
 ):
     """LLM task with structured input/output handling."""
 

llm_utils/lm/mixins.py

@@ -396,6 +396,80 @@ class VLLMMixin:
         return _kill_vllm_on_port(port)
 
 
+class TokenizationMixin:
+    """Mixin for tokenization operations (encode/decode)."""
+
+    def encode(
+        self,
+        text: str,
+        *,
+        add_special_tokens: bool = True,
+        return_token_strs: bool = False,
+    ) -> list[int] | tuple[list[int], list[str]]:
+        """
+        Encode text to token IDs using the model's tokenizer.
+
+        Args:
+            text: Text to tokenize
+            add_special_tokens: Whether to add special tokens (e.g., BOS)
+            return_token_strs: If True, also return token strings
+
+        Returns:
+            List of token IDs, or tuple of (token IDs, token strings)
+        """
+        import requests
+
+        # Get base_url from client and remove /v1 suffix if present
+        # (tokenize endpoint is at root level, not under /v1)
+        base_url = str(self.client.base_url).rstrip('/')
+        if base_url.endswith('/v1'):
+            base_url = base_url[:-3]  # Remove '/v1'
+
+        response = requests.post(
+            f'{base_url}/tokenize',
+            json={
+                'prompt': text,
+                'add_special_tokens': add_special_tokens,
+                'return_token_strs': return_token_strs,
+            },
+        )
+        response.raise_for_status()
+        data = response.json()
+
+        if return_token_strs:
+            return data['tokens'], data.get('token_strs', [])
+        return data['tokens']
+
+    def decode(
+        self,
+        token_ids: list[int],
+    ) -> str:
+        """
+        Decode token IDs to text using the model's tokenizer.
+
+        Args:
+            token_ids: List of token IDs to decode
+
+        Returns:
+            Decoded text string
+        """
+        import requests
+
+        # Get base_url from client and remove /v1 suffix if present
+        # (detokenize endpoint is at root level, not under /v1)
+        base_url = str(self.client.base_url).rstrip('/')
+        if base_url.endswith('/v1'):
+            base_url = base_url[:-3]  # Remove '/v1'
+
+        response = requests.post(
+            f'{base_url}/detokenize',
+            json={'tokens': token_ids},
+        )
+        response.raise_for_status()
+        data = response.json()
+        return data['prompt']
+
+
 class ModelUtilsMixin:
     """Mixin for model utility methods."""
 

{speedy_utils-1.1.34.dist-info → speedy_utils-1.1.35.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: speedy-utils
-Version: 1.1.34
+Version: 1.1.35
 Summary: Fast and easy-to-use package for data science
 Project-URL: Homepage, https://github.com/anhvth/speedy
 Project-URL: Repository, https://github.com/anhvth/speedy

{speedy_utils-1.1.34.dist-info → speedy_utils-1.1.35.dist-info}/RECORD

@@ -4,12 +4,12 @@ llm_utils/chat_format/__init__.py,sha256=a7BKtBVktgLMq2Do4iNu3YfdDdTG1v9M_BkmaEo
 llm_utils/chat_format/display.py,sha256=Lffjzna9_vV3QgfiXZM2_tuVb3wqA-WxwrmoAjsJigw,17356
 llm_utils/chat_format/transform.py,sha256=PJ2g9KT1GSbWuAs7giEbTpTAffpU9QsIXyRlbfpTZUQ,5351
 llm_utils/chat_format/utils.py,sha256=M2EctZ6NeHXqFYufh26Y3CpSphN0bdZm5xoNaEJj5vg,1251
-llm_utils/lm/__init__.py,sha256=lFE2DZRpj6eRMo11kx7oRLyYOP2FuDmz08mAcq-cYew,730
+llm_utils/lm/__init__.py,sha256=4jYMy3wPH3tg-tHFyWEWOqrnmX4Tu32VZCdzRGMGQsI,778
 llm_utils/lm/base_prompt_builder.py,sha256=_TzYMsWr-SsbA_JNXptUVN56lV5RfgWWTrFi-E8LMy4,12337
-llm_utils/lm/llm.py,sha256=C8Z8l6Ljs7uVX-zabLcDCdTf3fpGxfljaYRM0patHUQ,16469
+llm_utils/lm/llm.py,sha256=yas7Khd0Djc8-GD8jL--B2oPteV9FC3PpfPbr9XCLOQ,16515
 llm_utils/lm/llm_signature.py,sha256=vV8uZgLLd6ZKqWbq0OPywWvXAfl7hrJQnbtBF-VnZRU,1244
 llm_utils/lm/lm_base.py,sha256=Bk3q34KrcCK_bC4Ryxbc3KqkiPL39zuVZaBQ1i6wJqs,9437
-llm_utils/lm/mixins.py,sha256=on83g-JO2SpZ0digOpU8mooqFBX6w7Bc-DeGzVoVCX8,14536
+llm_utils/lm/mixins.py,sha256=o0tZiaKW4u1BxBVlT_0yTwnO8h7KnY02HX5TuWipvr0,16735
 llm_utils/lm/openai_memoize.py,sha256=rYrSFPpgO7adsjK1lVdkJlhqqIw_13TCW7zU8eNwm3o,5185
 llm_utils/lm/signature.py,sha256=K1hvCAqoC5CmsQ0Y_ywnYy2fRb5JzmIK8OS-hjH-5To,9971
 llm_utils/lm/utils.py,sha256=dEKFta8S6Mm4LjIctcpFlEGL9RnmLm5DHd2TA70UWuA,12649
@@ -50,7 +50,7 @@ vision_utils/README.md,sha256=AIDZZj8jo_QNrEjFyHwd00iOO431s-js-M2dLtVTn3I,5740
 vision_utils/__init__.py,sha256=hF54sT6FAxby8kDVhOvruy4yot8O-Ateey5n96O1pQM,284
 vision_utils/io_utils.py,sha256=pI0Va6miesBysJcllK6NXCay8HpGZsaMWwlsKB2DMgA,26510
 vision_utils/plot.py,sha256=HkNj3osA3moPuupP1VguXfPPOW614dZO5tvC-EFKpKM,12028
-speedy_utils-1.1.34.dist-info/METADATA,sha256=diZ6MTVGRDDhsbxoK9eBydHrbW2I6rvYG8lXXzJnJEU,8048
-speedy_utils-1.1.34.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-speedy_utils-1.1.34.dist-info/entry_points.txt,sha256=1rrFMfqvaMUE9hvwGiD6vnVh98kmgy0TARBj-v0Lfhs,244
-speedy_utils-1.1.34.dist-info/RECORD,,
+speedy_utils-1.1.35.dist-info/METADATA,sha256=wsz89syaYNXEeGjJXV8zb0W2ZrTjpN2Lj47tE7LQeEI,8048
+speedy_utils-1.1.35.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+speedy_utils-1.1.35.dist-info/entry_points.txt,sha256=1rrFMfqvaMUE9hvwGiD6vnVh98kmgy0TARBj-v0Lfhs,244
+speedy_utils-1.1.35.dist-info/RECORD,,