saia-python 0.6.0__tar.gz → 0.8.0__tar.gz

{saia_python-0.6.0/saia_python.egg-info → saia_python-0.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: saia-python
-Version: 0.6.0
+Version: 0.8.0
 Summary: Python wrapper for the GWDG SAIA platform REST API
 Author: Friedrich Schwarz
 License-Expression: AGPL-3.0-only
@@ -31,6 +31,11 @@ Requires-Dist: tqdm>=4.60
 Requires-Dist: tomlkit>=0.12
 Provides-Extra: openai
 Requires-Dist: openai>=1.0; extra == "openai"
+Provides-Extra: tokenizer
+Requires-Dist: transformers>=4.40; extra == "tokenizer"
+Requires-Dist: huggingface-hub>=0.20; extra == "tokenizer"
+Requires-Dist: tiktoken>=0.5; extra == "tokenizer"
+Requires-Dist: sentencepiece>=0.1.99; extra == "tokenizer"
 Provides-Extra: test
 Requires-Dist: pytest>=7.0; extra == "test"
 Requires-Dist: pytest-cov>=4.0; extra == "test"
@@ -117,6 +122,7 @@ chat_completion(model="meta-llama-3.1-8b-instruct", messages=[...])
 | **ARCANA** | RAG — knowledge base management and retrieval-augmented chat | [ARCANA](https://docs.hpc.gwdg.de/services/ai-services/arcana/index.html) |
 | **Documents** | PDF/document conversion via Docling | [SAIA API](https://docs.hpc.gwdg.de/services/ai-services/saia/index.html) |
 | **Models** | List available models, probe tool-calling support | [SAIA API](https://docs.hpc.gwdg.de/services/ai-services/saia/index.html) |
+| **Tokenizers** | Download model tokenizers; count chat-template tokens, special-token overhead, and subword fertility (opt-in `[tokenizer]` extra) | [Chat AI Models](https://docs.hpc.gwdg.de/services/ai-services/chat-ai/models/index.html) |
 | **Rate Limits** | Inspect current quota and usage | [SAIA API](https://docs.hpc.gwdg.de/services/ai-services/saia/index.html) |
 
 ## Repository Structure
@@ -130,6 +136,7 @@ saia-python/
 │   ├── voice.py                  # VoiceService — transcribe + translate
 │   ├── arcana.py                 # ArcanaService — RAG / knowledge bases
 │   ├── models.py                 # ModelsService — list available models
+│   ├── tokenizer.py              # Tokenizers — download, chat-template token counting
 │   ├── documents.py              # DocumentService — Docling conversion
 │   ├── openai_compat.py          # OpenAI SDK compatibility layer
 │   ├── auth.py                   # Credential and config discovery

{saia_python-0.6.0 → saia_python-0.8.0}/README.md

@@ -65,6 +65,7 @@ chat_completion(model="meta-llama-3.1-8b-instruct", messages=[...])
 | **ARCANA** | RAG — knowledge base management and retrieval-augmented chat | [ARCANA](https://docs.hpc.gwdg.de/services/ai-services/arcana/index.html) |
 | **Documents** | PDF/document conversion via Docling | [SAIA API](https://docs.hpc.gwdg.de/services/ai-services/saia/index.html) |
 | **Models** | List available models, probe tool-calling support | [SAIA API](https://docs.hpc.gwdg.de/services/ai-services/saia/index.html) |
+| **Tokenizers** | Download model tokenizers; count chat-template tokens, special-token overhead, and subword fertility (opt-in `[tokenizer]` extra) | [Chat AI Models](https://docs.hpc.gwdg.de/services/ai-services/chat-ai/models/index.html) |
 | **Rate Limits** | Inspect current quota and usage | [SAIA API](https://docs.hpc.gwdg.de/services/ai-services/saia/index.html) |
 
 ## Repository Structure
@@ -78,6 +79,7 @@ saia-python/
 │   ├── voice.py                  # VoiceService — transcribe + translate
 │   ├── arcana.py                 # ArcanaService — RAG / knowledge bases
 │   ├── models.py                 # ModelsService — list available models
+│   ├── tokenizer.py              # Tokenizers — download, chat-template token counting
 │   ├── documents.py              # DocumentService — Docling conversion
 │   ├── openai_compat.py          # OpenAI SDK compatibility layer
 │   ├── auth.py                   # Credential and config discovery

{saia_python-0.6.0 → saia_python-0.8.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "saia-python"
-version = "0.6.0"
+version = "0.8.0"
 description = "Python wrapper for the GWDG SAIA platform REST API"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -62,6 +62,16 @@ saia_python = ["py.typed"]
 openai = [
     "openai>=1.0",
 ]
+tokenizer = [
+    # AutoTokenizer + chat-template (apply_chat_template) engine
+    "transformers>=4.40",
+    # downloads tokenizer files (snapshot_download) to the local cache
+    "huggingface-hub>=0.20",
+    # byte-pair encodings for the externally hosted OpenAI models
+    "tiktoken>=0.5",
+    # slow-tokenizer / SentencePiece backing for several of the models
+    "sentencepiece>=0.1.99",
+]
 test = [
     "pytest>=7.0",
     "pytest-cov>=4.0",

{saia_python-0.6.0 → saia_python-0.8.0}/saia_python/__init__.py

@@ -42,6 +42,29 @@ from .exceptions import APIError, AuthenticationError, RateLimitError, SAIAError
 from .openai_compat import create_openai_client
 from .rate_limits import RateLimitInfo, parse_rate_limits
 from .responses import text_of
+from .tokenizer import (
+    DEFAULT_TOKENIZER_DIR,
+    GWDG_MODEL_REPOS,
+    OPENAI_TIKTOKEN_ENCODINGS,
+    ChatTokenCount,
+    FileTokenCount,
+    GatedRepoAccessError,
+    TokenDistribution,
+    TokenizerService,
+    available_open_models,
+    chat_template_length,
+    chat_template_tokens,
+    count_tiktoken_tokens,
+    download_all_tokenizers,
+    download_tokenizer,
+    load_hf_token,
+    load_tokenizer,
+    repo_url,
+    resolve_repo,
+    special_token_overhead,
+    subword_fertility,
+    token_distribution,
+)
 
 try:
     __version__ = version("saia-python")
@@ -81,6 +104,28 @@ __all__ = [
     "parse_arcana_references",
     "parse_reference_entries",
     "is_arcana_event",
+    # Tokenizers ([tokenizer] extra)
+    "GWDG_MODEL_REPOS",
+    "OPENAI_TIKTOKEN_ENCODINGS",
+    "DEFAULT_TOKENIZER_DIR",
+    "ChatTokenCount",
+    "FileTokenCount",
+    "TokenDistribution",
+    "TokenizerService",
+    "GatedRepoAccessError",
+    "available_open_models",
+    "resolve_repo",
+    "repo_url",
+    "load_hf_token",
+    "download_tokenizer",
+    "download_all_tokenizers",
+    "load_tokenizer",
+    "chat_template_tokens",
+    "chat_template_length",
+    "special_token_overhead",
+    "subword_fertility",
+    "count_tiktoken_tokens",
+    "token_distribution",
     # Functional API
     "list_models",
     "list_model_ids",

{saia_python-0.6.0 → saia_python-0.8.0}/saia_python/arcana.py

@@ -415,6 +415,49 @@ class ArcanaService:
 
         return _json_or_none(resp)
 
+    def recreate(self, name: str, *, update_toml: bool = False) -> dict:
+        """Delete an arcana and recreate it **empty with the same ID**.
+
+        The minimal-call way to wipe an entire arcana: two requests
+        (:meth:`delete` + :meth:`create`) regardless of how many files it
+        holds, versus one :meth:`delete_file` per file (thousands of calls,
+        each its own read-timeout risk while the arcana is busy). The name —
+        including any UUID suffix — is preserved verbatim via
+        ``create(..., append_uuid=False)``, so a downstream pin on the full
+        ``owner/name-uuid`` ID stays valid.
+
+        Trade-offs versus emptying file-by-file (:meth:`delete_file` in a
+        loop, which keeps the container): there is a brief window between the
+        two calls where the arcana does **not exist**, and the recreated
+        arcana is brand-new — ``created_at``, sharing/permissions and any
+        other container settings reset to defaults. Only the name/ID carries
+        across.
+
+        Args:
+            name: The arcana name or full ``owner/name`` ID to recreate.
+            update_toml: Forwarded to :meth:`create`.
+
+        Returns:
+            The :meth:`create` result (``{"name", "id", "message"}``).
+
+        Raises:
+            APIError: If recreation fails *after* the delete already
+                succeeded — the arcana is then gone. The message says so
+                explicitly so the operator recreates it before any consumer
+                (adapter / manifest pin) points at the ID.
+        """
+        short = extract_arcana_name(name)
+        self.delete(name)
+        try:
+            return self.create(short, append_uuid=False, update_toml=update_toml)
+        except Exception as e:  # noqa: BLE001 — re-raise with a louder message
+            raise APIError(
+                f"arcana {short!r} was DELETED but could not be recreated: {e}. "
+                f"The arcana no longer exists — recreate it (e.g. "
+                f"create({short!r}, append_uuid=False)) before any consumer "
+                f"(adapter / manifest pin) points at its ID."
+            ) from e
+
     def list(self) -> list[dict]:
         """List all available arcanas.
 
@@ -742,6 +785,70 @@ class ArcanaService:
         raise_for_status(resp)
         return _json_or_none(resp)
 
+    def delete_files(
+        self,
+        name: str,
+        file_names: Iterable[str],
+        *,
+        verbose: bool = False,
+        on_result: Callable[[str, dict], None] | None = None,
+    ) -> list[dict]:
+        """Delete an explicit list of files from an arcana, by name.
+
+        The batch counterpart to :meth:`delete_file`: hand it the file names
+        (as returned by :meth:`list_files`) and it deletes each one,
+        capturing a per-file outcome instead of aborting on the first
+        failure. Unlike :meth:`delete_directory` no local directory is
+        consulted — the names come straight from the caller — so it can
+        target files that no longer exist on disk (e.g. a name list from a
+        CSV). Pairs with a thin CLI front-end that resolves *which* names to
+        delete; this method only does the deleting.
+
+        Args:
+            name: The arcana name or full ``owner/name`` ID.
+            file_names: The file names to delete (flat names, as listed by
+                :meth:`list_files`). Order is preserved; a repeated name is
+                attempted each time (a second delete just reports the
+                server's response / 404).
+            verbose: If ``True``, print per-file deletion status.
+            on_result: Optional callback invoked as ``on_result(file_name,
+                entry)`` after each file (``entry`` is that file's
+                ``{"file", "status", ["error"]}`` dict), for inline
+                per-file logging.
+
+        Returns:
+            A list of dicts with keys ``"file"`` (the name), ``"status"``
+            (``"deleted"`` or ``"failed"``), and ``"error"`` (only on
+            failure) — the same shape every batch op returns.
+        """
+        names = [str(n) for n in file_names]
+        # Arcana file names are flat (as listed by :meth:`list_files`). The
+        # batch executor below is Path-centric and targets ``Path(n).name``;
+        # a name containing ``/`` would silently collapse to its basename and
+        # delete the WRONG file. Refuse the whole batch up front — atomic, so
+        # nothing is deleted if any name is malformed (e.g. from a bad CSV).
+        bad = [n for n in names if "/" in n]
+        if bad:
+            raise ValueError(
+                f"ARCANA file names are flat (no '/'); refusing to delete "
+                f"{len(bad)} name(s) containing a path separator: {bad}"
+            )
+        # Reuse the shared batch executor (iteration, per-file error capture,
+        # progress bar, on_result, tally). With flat names ``Path(n).name ==
+        # n``, so the label/delete-target round-trips exactly.
+        return self._run_file_batch(
+            [Path(n) for n in names],
+            lambda fp: self.delete_file(name, fp.name),
+            default_status="deleted",
+            desc="Deleting",
+            verbose=verbose,
+            on_result=(
+                None
+                if on_result is None
+                else lambda fp, entry: on_result(fp.name, entry)
+            ),
+        )
+
     def download_file(self, name: str, file_name: str, output_path: str | Path) -> Path:
         """Download a file from an arcana to a local path.
 

{saia_python-0.6.0 → saia_python-0.8.0}/saia_python/client.py

@@ -12,6 +12,7 @@ from .documents import DocumentService
 from .exceptions import raise_for_status
 from .models import ModelsService
 from .rate_limits import RateLimitInfo, parse_rate_limits
+from .tokenizer import TokenizerService
 from .voice import VoiceService
 
 
@@ -80,6 +81,7 @@ class SAIAClient:
         self._models: ModelsService | None = None
         self._arcana: ArcanaService | None = None
         self._documents: DocumentService | None = None
+        self._tokenizers: TokenizerService | None = None
         self._openai = None
         self._openai_async = None
 
@@ -106,6 +108,19 @@ class SAIAClient:
             )
         return self._models
 
+    @property
+    def tokenizers(self) -> TokenizerService:
+        """Tokenizer service for the open-weight models.
+
+        Loads model tokenizers, counts chat-template tokens, and annotates the
+        live model list with Hugging Face repositories. Requires the optional
+        ``[tokenizer]`` extra (``pip install saia-python[tokenizer]``) for the
+        download/load operations; the repository annotations work without it.
+        """
+        if self._tokenizers is None:
+            self._tokenizers = TokenizerService(self.models)
+        return self._tokenizers
+
     @property
     def arcana(self) -> ArcanaService:
         """ARCANA/RAG service."""