saia-python 0.6.0__tar.gz → 0.7.0__tar.gz

{saia_python-0.6.0/saia_python.egg-info → saia_python-0.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: saia-python
-Version: 0.6.0
+Version: 0.7.0
 Summary: Python wrapper for the GWDG SAIA platform REST API
 Author: Friedrich Schwarz
 License-Expression: AGPL-3.0-only
@@ -31,6 +31,11 @@ Requires-Dist: tqdm>=4.60
 Requires-Dist: tomlkit>=0.12
 Provides-Extra: openai
 Requires-Dist: openai>=1.0; extra == "openai"
+Provides-Extra: tokenizer
+Requires-Dist: transformers>=4.40; extra == "tokenizer"
+Requires-Dist: huggingface-hub>=0.20; extra == "tokenizer"
+Requires-Dist: tiktoken>=0.5; extra == "tokenizer"
+Requires-Dist: sentencepiece>=0.1.99; extra == "tokenizer"
 Provides-Extra: test
 Requires-Dist: pytest>=7.0; extra == "test"
 Requires-Dist: pytest-cov>=4.0; extra == "test"
@@ -117,6 +122,7 @@ chat_completion(model="meta-llama-3.1-8b-instruct", messages=[...])
 | **ARCANA** | RAG — knowledge base management and retrieval-augmented chat | [ARCANA](https://docs.hpc.gwdg.de/services/ai-services/arcana/index.html) |
 | **Documents** | PDF/document conversion via Docling | [SAIA API](https://docs.hpc.gwdg.de/services/ai-services/saia/index.html) |
 | **Models** | List available models, probe tool-calling support | [SAIA API](https://docs.hpc.gwdg.de/services/ai-services/saia/index.html) |
+| **Tokenizers** | Download model tokenizers; count chat-template tokens, special-token overhead, and subword fertility (opt-in `[tokenizer]` extra) | [Chat AI Models](https://docs.hpc.gwdg.de/services/ai-services/chat-ai/models/index.html) |
 | **Rate Limits** | Inspect current quota and usage | [SAIA API](https://docs.hpc.gwdg.de/services/ai-services/saia/index.html) |
 
 ## Repository Structure
@@ -130,6 +136,7 @@ saia-python/
 │   ├── voice.py                  # VoiceService — transcribe + translate
 │   ├── arcana.py                 # ArcanaService — RAG / knowledge bases
 │   ├── models.py                 # ModelsService — list available models
+│   ├── tokenizer.py              # Tokenizers — download, chat-template token counting
 │   ├── documents.py              # DocumentService — Docling conversion
 │   ├── openai_compat.py          # OpenAI SDK compatibility layer
 │   ├── auth.py                   # Credential and config discovery

{saia_python-0.6.0 → saia_python-0.7.0}/README.md

@@ -65,6 +65,7 @@ chat_completion(model="meta-llama-3.1-8b-instruct", messages=[...])
 | **ARCANA** | RAG — knowledge base management and retrieval-augmented chat | [ARCANA](https://docs.hpc.gwdg.de/services/ai-services/arcana/index.html) |
 | **Documents** | PDF/document conversion via Docling | [SAIA API](https://docs.hpc.gwdg.de/services/ai-services/saia/index.html) |
 | **Models** | List available models, probe tool-calling support | [SAIA API](https://docs.hpc.gwdg.de/services/ai-services/saia/index.html) |
+| **Tokenizers** | Download model tokenizers; count chat-template tokens, special-token overhead, and subword fertility (opt-in `[tokenizer]` extra) | [Chat AI Models](https://docs.hpc.gwdg.de/services/ai-services/chat-ai/models/index.html) |
 | **Rate Limits** | Inspect current quota and usage | [SAIA API](https://docs.hpc.gwdg.de/services/ai-services/saia/index.html) |
 
 ## Repository Structure
@@ -78,6 +79,7 @@ saia-python/
 │   ├── voice.py                  # VoiceService — transcribe + translate
 │   ├── arcana.py                 # ArcanaService — RAG / knowledge bases
 │   ├── models.py                 # ModelsService — list available models
+│   ├── tokenizer.py              # Tokenizers — download, chat-template token counting
 │   ├── documents.py              # DocumentService — Docling conversion
 │   ├── openai_compat.py          # OpenAI SDK compatibility layer
 │   ├── auth.py                   # Credential and config discovery

{saia_python-0.6.0 → saia_python-0.7.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "saia-python"
-version = "0.6.0"
+version = "0.7.0"
 description = "Python wrapper for the GWDG SAIA platform REST API"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -62,6 +62,16 @@ saia_python = ["py.typed"]
 openai = [
     "openai>=1.0",
 ]
+tokenizer = [
+    # AutoTokenizer + chat-template (apply_chat_template) engine
+    "transformers>=4.40",
+    # downloads tokenizer files (snapshot_download) to the local cache
+    "huggingface-hub>=0.20",
+    # byte-pair encodings for the externally hosted OpenAI models
+    "tiktoken>=0.5",
+    # slow-tokenizer / SentencePiece backing for several of the models
+    "sentencepiece>=0.1.99",
+]
 test = [
     "pytest>=7.0",
     "pytest-cov>=4.0",

{saia_python-0.6.0 → saia_python-0.7.0}/saia_python/__init__.py

@@ -42,6 +42,29 @@ from .exceptions import APIError, AuthenticationError, RateLimitError, SAIAError
 from .openai_compat import create_openai_client
 from .rate_limits import RateLimitInfo, parse_rate_limits
 from .responses import text_of
+from .tokenizer import (
+    DEFAULT_TOKENIZER_DIR,
+    GWDG_MODEL_REPOS,
+    OPENAI_TIKTOKEN_ENCODINGS,
+    ChatTokenCount,
+    FileTokenCount,
+    GatedRepoAccessError,
+    TokenDistribution,
+    TokenizerService,
+    available_open_models,
+    chat_template_length,
+    chat_template_tokens,
+    count_tiktoken_tokens,
+    download_all_tokenizers,
+    download_tokenizer,
+    load_hf_token,
+    load_tokenizer,
+    repo_url,
+    resolve_repo,
+    special_token_overhead,
+    subword_fertility,
+    token_distribution,
+)
 
 try:
     __version__ = version("saia-python")
@@ -81,6 +104,28 @@ __all__ = [
     "parse_arcana_references",
     "parse_reference_entries",
     "is_arcana_event",
+    # Tokenizers ([tokenizer] extra)
+    "GWDG_MODEL_REPOS",
+    "OPENAI_TIKTOKEN_ENCODINGS",
+    "DEFAULT_TOKENIZER_DIR",
+    "ChatTokenCount",
+    "FileTokenCount",
+    "TokenDistribution",
+    "TokenizerService",
+    "GatedRepoAccessError",
+    "available_open_models",
+    "resolve_repo",
+    "repo_url",
+    "load_hf_token",
+    "download_tokenizer",
+    "download_all_tokenizers",
+    "load_tokenizer",
+    "chat_template_tokens",
+    "chat_template_length",
+    "special_token_overhead",
+    "subword_fertility",
+    "count_tiktoken_tokens",
+    "token_distribution",
     # Functional API
     "list_models",
     "list_model_ids",

{saia_python-0.6.0 → saia_python-0.7.0}/saia_python/client.py

@@ -12,6 +12,7 @@ from .documents import DocumentService
 from .exceptions import raise_for_status
 from .models import ModelsService
 from .rate_limits import RateLimitInfo, parse_rate_limits
+from .tokenizer import TokenizerService
 from .voice import VoiceService
 
 
@@ -80,6 +81,7 @@ class SAIAClient:
         self._models: ModelsService | None = None
         self._arcana: ArcanaService | None = None
         self._documents: DocumentService | None = None
+        self._tokenizers: TokenizerService | None = None
         self._openai = None
         self._openai_async = None
 
@@ -106,6 +108,19 @@ class SAIAClient:
             )
         return self._models
 
+    @property
+    def tokenizers(self) -> TokenizerService:
+        """Tokenizer service for the open-weight models.
+
+        Loads model tokenizers, counts chat-template tokens, and annotates the
+        live model list with Hugging Face repositories. Requires the optional
+        ``[tokenizer]`` extra (``pip install saia-python[tokenizer]``) for the
+        download/load operations; the repository annotations work without it.
+        """
+        if self._tokenizers is None:
+            self._tokenizers = TokenizerService(self.models)
+        return self._tokenizers
+
     @property
     def arcana(self) -> ArcanaService:
         """ARCANA/RAG service."""