speedy-utils 1.1.34__tar.gz → 1.1.35__tar.gz

{speedy_utils-1.1.34 → speedy_utils-1.1.35}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: speedy-utils
-Version: 1.1.34
+Version: 1.1.35
 Summary: Fast and easy-to-use package for data science
 Project-URL: Homepage, https://github.com/anhvth/speedy
 Project-URL: Repository, https://github.com/anhvth/speedy

speedy_utils-1.1.35/debug/test_decode_api.py

@@ -0,0 +1,19 @@
+from llm_utils.lm import LLM
+
+lm = LLM(client=8000)
+
+# Encode text to tokens
+token_ids = lm.encode('Hello, world!')
+print(f'Token IDs: {token_ids}')
+
+# Decode tokens back to text
+text = lm.decode(token_ids)
+print(f'Decoded text: {text}')
+
+# Get token strings for debugging
+ids, strs = lm.encode('Hello', return_token_strs=True)
+print(f'IDs: {ids}')
+print(f'Strings: {strs}')
+print(f'Tokens with strings:')
+for i, s in zip(ids, strs):
+    print(f'  {i:6d} -> "{s}"')

speedy_utils-1.1.35/debug/test_endpoints.py

@@ -0,0 +1,27 @@
+from llm_utils.lm import LLM
+
+lm = LLM(client=8000)
+
+# Debug: Check the base_url
+print(f"Client base_url: {lm.client.base_url}")
+
+# Try to manually check what endpoints are available
+import requests
+
+# Try different endpoint paths
+test_urls = [
+    "http://localhost:8000/tokenize",
+    "http://localhost:8000/v1/tokenize",
+]
+
+for url in test_urls:
+    try:
+        response = requests.post(
+            url,
+            json={"prompt": "test", "add_special_tokens": True}
+        )
+        print(f"✓ {url} - Status: {response.status_code}")
+        if response.status_code == 200:
+            print(f"  Response: {response.json()}")
+    except Exception as e:
+        print(f"✗ {url} - Error: {e}")

speedy_utils-1.1.35/docs/TOKENIZATION.md

@@ -0,0 +1,149 @@
+# Tokenization Support in LLM
+
+The `LLM` class now includes built-in tokenization support through the `TokenizationMixin`, providing `encode` and `decode` methods to work with token IDs.
+
+## Features
+
+- **encode()**: Convert text to token IDs
+- **decode()**: Convert token IDs back to text
+- Support for special tokens (BOS, EOS, etc.)
+- Optional token string output for debugging
+
+## API Reference
+
+### encode()
+
+```python
+def encode(
+    self,
+    text: str,
+    *,
+    add_special_tokens: bool = True,
+    return_token_strs: bool = False,
+) -> list[int] | tuple[list[int], list[str]]
+```
+
+**Parameters:**
+- `text` (str): Text to tokenize
+- `add_special_tokens` (bool): Whether to add special tokens like BOS/EOS (default: True)
+- `return_token_strs` (bool): If True, also return token strings (default: False)
+
+**Returns:**
+- `list[int]`: Token IDs (if `return_token_strs=False`)
+- `tuple[list[int], list[str]]`: Token IDs and token strings (if `return_token_strs=True`)
+
+### decode()
+
+```python
+def decode(
+    self,
+    token_ids: list[int],
+) -> str
+```
+
+**Parameters:**
+- `token_ids` (list[int]): List of token IDs to decode
+
+**Returns:**
+- `str`: Decoded text
+
+## Usage Examples
+
+### Basic Encoding/Decoding
+
+```python
+from llm_utils.lm import LLM
+
+lm = LLM(base_url='http://localhost:8000/v1')
+
+# Encode text to token IDs
+text = 'Hello, world!'
+token_ids = lm.encode(text)
+print(token_ids)  # [123, 456, 789, ...]
+
+# Decode token IDs back to text
+decoded = lm.decode(token_ids)
+print(decoded)  # 'Hello, world!'
+```
+
+### Getting Token Strings
+
+Useful for debugging and understanding tokenization:
+
+```python
+# Get both token IDs and their string representations
+token_ids, token_strs = lm.encode('Hello world', return_token_strs=True)
+
+for tid, tstr in zip(token_ids, token_strs):
+    print(f'{tid:6d} -> "{tstr}"')
+# Output:
+#    123 -> "Hello"
+#    456 -> " world"
+```
+
+### Counting Tokens
+
+Count tokens before making API calls to manage context windows:
+
+```python
+text = 'A very long document...'
+token_count = len(lm.encode(text))
+print(f'This text uses {token_count} tokens')
+
+# Check if it fits in model's context window
+MAX_TOKENS = 4096
+if token_count > MAX_TOKENS:
+    print('Text is too long!')
+```
+
+### Working Without Special Tokens
+
+```python
+# Without special tokens (useful for token manipulation)
+tokens_clean = lm.encode('Hello', add_special_tokens=False)
+
+# With special tokens (default)
+tokens_with_special = lm.encode('Hello', add_special_tokens=True)
+
+print(f'Clean: {len(tokens_clean)} tokens')
+print(f'With special: {len(tokens_with_special)} tokens')
+```
+
+### Token-Level Text Manipulation
+
+```python
+# Combine texts at token level
+sent1_tokens = lm.encode('Hello', add_special_tokens=False)
+sent2_tokens = lm.encode('world', add_special_tokens=False)
+
+# Manually combine
+combined = sent1_tokens + sent2_tokens
+result = lm.decode(combined)
+print(result)  # 'Helloworld'
+```
+
+## Requirements
+
+The tokenization functionality requires a VLLM server (or compatible API) that implements:
+- `/tokenize` endpoint (accepts `TokenizeCompletionRequest`)
+- `/detokenize` endpoint (accepts `DetokenizeRequest`)
+
+## Implementation Details
+
+The `TokenizationMixin` is automatically included in the `LLM` class. It uses the model's base URL to make HTTP requests to the tokenization endpoints.
+
+The mixin can be used standalone if needed:
+
+```python
+from llm_utils.lm.mixins import TokenizationMixin
+
+class MyCustomLM(TokenizationMixin):
+    def __init__(self, base_url):
+        self.client = MOpenAI(base_url=base_url, api_key='abc')
+```
+
+## See Also
+
+- Example script: `examples/tokenization_example.py`
+- Tests: `tests/test_tokenization.py`
+- API specification: See OpenAPI schema for endpoint details

speedy_utils-1.1.35/docs/TOKENIZATION_IMPLEMENTATION.md

@@ -0,0 +1,104 @@
+# Tokenization Feature Implementation Summary
+
+## What Was Added
+
+Added tokenization support (encode/decode methods) to the LLM class for converting between text and token IDs.
+
+## Changes Made
+
+### 1. New Mixin: `TokenizationMixin`
+**File:** `src/llm_utils/lm/mixins.py`
+
+Added a new mixin class with two methods:
+- `encode(text, add_special_tokens=True, return_token_strs=False)` - Convert text to token IDs
+- `decode(token_ids)` - Convert token IDs back to text
+
+### 2. Updated LLM Class
+**File:** `src/llm_utils/lm/llm.py`
+
+- Added `TokenizationMixin` to the LLM class inheritance
+- Imported the new mixin
+
+### 3. Updated Exports
+**File:** `src/llm_utils/lm/__init__.py`
+
+- Added `TokenizationMixin` to imports and `__all__`
+
+### 4. Documentation
+**File:** `docs/TOKENIZATION.md`
+
+- Comprehensive documentation with API reference
+- Usage examples for common scenarios
+- Implementation details
+
+### 5. Example Script
+**File:** `examples/tokenization_example.py`
+
+- Practical examples demonstrating all features
+- Shows token counting, manipulation, debugging
+
+### 6. Tests
+**File:** `tests/test_tokenization.py`
+
+- Unit tests for encode/decode functionality
+- Tests for special tokens handling
+- Manual test runner included
+
+## API Endpoints Used
+
+Based on the provided OpenAPI specification, the implementation uses:
+
+1. **POST /tokenize** - Tokenizes text input
+   - Accepts: `TokenizeCompletionRequest` with `prompt`, `add_special_tokens`, `return_token_strs`
+   - Returns: `tokens` (list of ints) and optionally `token_strs`
+
+2. **POST /detokenize** - Converts token IDs back to text
+   - Accepts: `DetokenizeRequest` with `tokens` (list of ints)
+   - Returns: `prompt` (string)
+
+## Usage
+
+```python
+from llm_utils.lm import LLM
+
+# Initialize
+lm = LLM(base_url='http://localhost:8000/v1')
+
+# Encode
+token_ids = lm.encode('Hello, world!')
+
+# Decode
+text = lm.decode(token_ids)
+
+# With token strings for debugging
+token_ids, token_strs = lm.encode('Hello', return_token_strs=True)
+```
+
+## Testing
+
+Run tests with:
+```bash
+# Using pytest
+pytest tests/test_tokenization.py
+
+# Manual test
+python tests/test_tokenization.py
+```
+
+Run example:
+```bash
+python examples/tokenization_example.py
+```
+
+## Requirements
+
+- VLLM server (or compatible) running with tokenization endpoints
+- `requests` library (already a dependency)
+
+## Benefits
+
+1. **Token Counting**: Check token count before API calls to manage context windows
+2. **Token-Level Manipulation**: Combine/split text at token boundaries
+3. **Debugging**: Inspect exact tokenization with `return_token_strs=True`
+4. **Consistency**: Use same tokenizer as the model server
+5. **No Local Tokenizer**: No need to install transformers or download tokenizer locally

speedy_utils-1.1.35/examples/tokenization_example.py

@@ -0,0 +1,70 @@
+"""Example: Using LLM encode/decode methods for tokenization.
+
+This example demonstrates how to use the tokenization functionality
+in the LLM class to encode text to token IDs and decode token IDs
+back to text.
+"""
+
+from llm_utils.lm import LLM
+
+
+def main():
+    """Demonstrate encode/decode functionality."""
+    # Initialize LLM with your VLLM server
+    lm = LLM(base_url='http://localhost:8000/v1')
+    
+    # Example text
+    text = 'The quick brown fox jumps over the lazy dog.'
+    print(f'Original text: {text}\n')
+    
+    # 1. Basic encoding
+    print('1. Basic encoding:')
+    token_ids = lm.encode(text)
+    print(f'   Token IDs: {token_ids}')
+    print(f'   Number of tokens: {len(token_ids)}\n')
+    
+    # 2. Encoding with token strings
+    print('2. Encoding with token strings:')
+    token_ids, token_strs = lm.encode(text, return_token_strs=True)
+    for tid, tstr in zip(token_ids, token_strs):
+        print(f'   {tid:6d} -> "{tstr}"')
+    print()
+    
+    # 3. Encoding without special tokens
+    print('3. Comparing with/without special tokens:')
+    tokens_with = lm.encode(text, add_special_tokens=True)
+    tokens_without = lm.encode(text, add_special_tokens=False)
+    print(f'   With special tokens:    {len(tokens_with)} tokens')
+    print(f'   Without special tokens: {len(tokens_without)} tokens\n')
+    
+    # 4. Decoding
+    print('4. Decoding:')
+    decoded = lm.decode(token_ids)
+    print(f'   Decoded text: {decoded}\n')
+    
+    # 5. Practical use case: counting tokens before API call
+    print('5. Counting tokens for API calls:')
+    long_text = ' '.join(['This is a test sentence.'] * 10)
+    token_count = len(lm.encode(long_text))
+    print(f'   Text length: {len(long_text)} characters')
+    print(f'   Token count: {token_count} tokens')
+    print(f'   Avg chars per token: {len(long_text) / token_count:.2f}\n')
+    
+    # 6. Working with custom token sequences
+    print('6. Custom token manipulation:')
+    # Encode two sentences
+    sent1 = 'Hello world'
+    sent2 = 'How are you?'
+    tokens1 = lm.encode(sent1, add_special_tokens=False)
+    tokens2 = lm.encode(sent2, add_special_tokens=False)
+    
+    # Combine tokens manually
+    combined_tokens = tokens1 + tokens2
+    combined_text = lm.decode(combined_tokens)
+    print(f'   Sentence 1: {sent1}')
+    print(f'   Sentence 2: {sent2}')
+    print(f'   Combined (token-level): {combined_text}')
+
+
+if __name__ == '__main__':
+    main()

{speedy_utils-1.1.34 → speedy_utils-1.1.35}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "speedy-utils"
-version = "1.1.34"
+version = "1.1.35"
 description = "Fast and easy-to-use package for data science"
 authors = [{ name = "AnhVTH", email = "anhvth.226@gmail.com" }]
 readme = "README.md"

{speedy_utils-1.1.34 → speedy_utils-1.1.35}/src/llm_utils/lm/__init__.py

@@ -7,6 +7,7 @@ from .lm_base import LMBase, get_model_name
 from .mixins import (
     ModelUtilsMixin,
     TemperatureRangeMixin,
+    TokenizationMixin,
     TwoStepPydanticMixin,
     VLLMMixin,
 )
@@ -14,19 +15,20 @@ from .signature import Input, InputField, Output, OutputField, Signature
 
 
 __all__ = [
-    "LMBase",
-    "LLM",
-    "AsyncLM",
-    "AsyncLLMTask",
-    "BasePromptBuilder",
-    "LLMSignature",
-    "Signature",
-    "InputField",
-    "OutputField",
-    "Input",
-    "Output",
-    "TemperatureRangeMixin",
-    "TwoStepPydanticMixin",
-    "VLLMMixin",
-    "ModelUtilsMixin",
+    'LMBase',
+    'LLM',
+    'AsyncLM',
+    'AsyncLLMTask',
+    'BasePromptBuilder',
+    'LLMSignature',
+    'Signature',
+    'InputField',
+    'OutputField',
+    'Input',
+    'Output',
+    'TemperatureRangeMixin',
+    'TwoStepPydanticMixin',
+    'VLLMMixin',
+    'ModelUtilsMixin',
+    'TokenizationMixin',
 ]

{speedy_utils-1.1.34 → speedy_utils-1.1.35}/src/llm_utils/lm/llm.py

@@ -20,6 +20,7 @@ from .base_prompt_builder import BasePromptBuilder
 from .mixins import (
     ModelUtilsMixin,
     TemperatureRangeMixin,
+    TokenizationMixin,
     TwoStepPydanticMixin,
     VLLMMixin,
 )
@@ -47,6 +48,7 @@ class LLM(
     TwoStepPydanticMixin,
     VLLMMixin,
     ModelUtilsMixin,
+    TokenizationMixin,
 ):
     """LLM task with structured input/output handling."""
 

{speedy_utils-1.1.34 → speedy_utils-1.1.35}/src/llm_utils/lm/mixins.py

@@ -396,6 +396,80 @@ class VLLMMixin:
         return _kill_vllm_on_port(port)
 
 
+class TokenizationMixin:
+    """Mixin for tokenization operations (encode/decode)."""
+
+    def encode(
+        self,
+        text: str,
+        *,
+        add_special_tokens: bool = True,
+        return_token_strs: bool = False,
+    ) -> list[int] | tuple[list[int], list[str]]:
+        """
+        Encode text to token IDs using the model's tokenizer.
+
+        Args:
+            text: Text to tokenize
+            add_special_tokens: Whether to add special tokens (e.g., BOS)
+            return_token_strs: If True, also return token strings
+
+        Returns:
+            List of token IDs, or tuple of (token IDs, token strings)
+        """
+        import requests
+
+        # Get base_url from client and remove /v1 suffix if present
+        # (tokenize endpoint is at root level, not under /v1)
+        base_url = str(self.client.base_url).rstrip('/')
+        if base_url.endswith('/v1'):
+            base_url = base_url[:-3]  # Remove '/v1'
+
+        response = requests.post(
+            f'{base_url}/tokenize',
+            json={
+                'prompt': text,
+                'add_special_tokens': add_special_tokens,
+                'return_token_strs': return_token_strs,
+            },
+        )
+        response.raise_for_status()
+        data = response.json()
+
+        if return_token_strs:
+            return data['tokens'], data.get('token_strs', [])
+        return data['tokens']
+
+    def decode(
+        self,
+        token_ids: list[int],
+    ) -> str:
+        """
+        Decode token IDs to text using the model's tokenizer.
+
+        Args:
+            token_ids: List of token IDs to decode
+
+        Returns:
+            Decoded text string
+        """
+        import requests
+
+        # Get base_url from client and remove /v1 suffix if present
+        # (detokenize endpoint is at root level, not under /v1)
+        base_url = str(self.client.base_url).rstrip('/')
+        if base_url.endswith('/v1'):
+            base_url = base_url[:-3]  # Remove '/v1'
+
+        response = requests.post(
+            f'{base_url}/detokenize',
+            json={'tokens': token_ids},
+        )
+        response.raise_for_status()
+        data = response.json()
+        return data['prompt']
+
+
 class ModelUtilsMixin:
     """Mixin for model utility methods."""
 

speedy_utils-1.1.35/tests/test_tokenization.py

@@ -0,0 +1,88 @@
+"""Test tokenization functionality for LLM."""
+
+import pytest
+from llm_utils.lm import LLM
+
+
+def test_encode_decode():
+    """Test basic encode/decode functionality."""
+    # Skip if no VLLM server is running
+    try:
+        lm = LLM(base_url='http://localhost:8000/v1')
+        
+        # Test encode
+        text = 'Hello, world!'
+        token_ids = lm.encode(text)
+        assert isinstance(token_ids, list)
+        assert all(isinstance(t, int) for t in token_ids)
+        assert len(token_ids) > 0
+        
+        # Test encode with token strings
+        token_ids_with_strs, token_strs = lm.encode(
+            text, return_token_strs=True
+        )
+        assert isinstance(token_ids_with_strs, list)
+        assert isinstance(token_strs, list)
+        assert len(token_ids_with_strs) == len(token_strs)
+        
+        # Test decode
+        decoded = lm.decode(token_ids)
+        assert isinstance(decoded, str)
+        # Note: decoded text might have slight differences due to tokenizer
+        # behavior (e.g., special tokens), so we don't assert exact match
+        
+    except Exception as e:
+        pytest.skip(f'VLLM server not available: {e}')
+
+
+def test_encode_with_special_tokens():
+    """Test encode with and without special tokens."""
+    try:
+        lm = LLM(base_url='http://localhost:8000/v1')
+        
+        text = 'Test text'
+        
+        # With special tokens (default)
+        tokens_with = lm.encode(text, add_special_tokens=True)
+        
+        # Without special tokens
+        tokens_without = lm.encode(text, add_special_tokens=False)
+        
+        # Typically tokens_with should have more tokens (BOS, EOS, etc.)
+        # but this depends on the model
+        assert isinstance(tokens_with, list)
+        assert isinstance(tokens_without, list)
+        
+    except Exception as e:
+        pytest.skip(f'VLLM server not available: {e}')
+
+
+if __name__ == '__main__':
+    # Simple manual test
+    print('Testing tokenization...')
+    try:
+        lm = LLM(base_url='http://localhost:8000/v1')
+        
+        text = 'Hello, how are you?'
+        print(f'Original text: {text}')
+        
+        # Encode
+        token_ids = lm.encode(text)
+        print(f'Token IDs: {token_ids}')
+        
+        # Encode with token strings
+        token_ids_with_strs, token_strs = lm.encode(
+            text, return_token_strs=True
+        )
+        print(f'Tokens with strings: {list(zip(token_ids_with_strs, token_strs))}')
+        
+        # Decode
+        decoded = lm.decode(token_ids)
+        print(f'Decoded text: {decoded}')
+        
+        print('✓ All tests passed!')
+        
+    except Exception as e:
+        print(f'✗ Error: {e}')
+        import traceback
+        traceback.print_exc()