triton-inference-sdk 0.1.0__tar.gz

triton_inference_sdk-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Vadim Shubin
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

triton_inference_sdk-0.1.0/PKG-INFO

@@ -0,0 +1,213 @@
+Metadata-Version: 2.4
+Name: triton-inference-sdk
+Version: 0.1.0
+Summary: Python SDK for NVIDIA Triton Inference Server focused on LLM workloads (TensorRT-LLM, vLLM)
+Author-email: Shubin Vadim <midavnibush@gmail.com>
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/Shubin-vadim/triron_sdk
+Project-URL: Repository, https://github.com/Shubin-vadim/triron_sdk
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: httpx~=0.28.1
+Requires-Dist: pydantic~=2.12.3
+Requires-Dist: pydantic-settings~=2.12.0
+Requires-Dist: tenacity>=9.0.0
+Provides-Extra: grpc
+Requires-Dist: grpcio>=1.66; extra == "grpc"
+Requires-Dist: protobuf>=5.28; extra == "grpc"
+Provides-Extra: chat
+Requires-Dist: transformers>=4.45; extra == "chat"
+Requires-Dist: jinja2~=3.1.6; extra == "chat"
+Provides-Extra: all
+Requires-Dist: triton-inference-sdk[chat,grpc]; extra == "all"
+Dynamic: license-file
+
+# Triton-sdk
+
+Python SDK for NVIDIA Triton Inference Server, focused on LLM workloads
+(TensorRT-LLM and vLLM backends). One public API — `generate` / `stream` / `chat`,
+with backend and transport (HTTP / gRPC) differences hidden inside.
+
+Key features:
+
+- **One LLM API across supported backends:** `generate`, `stream`, and `chat`
+  adapt vLLM and TensorRT-LLM model schemas, including backend-specific inputs
+  and outputs.
+- **OpenAI-style interface for native Triton endpoints:** use
+  `chat.completions` and `completions` over HTTP or gRPC without deploying
+  Triton's OpenAI frontend. Compatibility is partial; see the limitations below.
+
+## Quick start
+
+Clone the repository:
+
+```bash
+git clone https://github.com/Shubin-vadim/triron_sdk.git
+cd triron_sdk
+```
+
+Python 3.12 or newer is required. If `uv` is not installed, install it with pip:
+
+```bash
+python -m pip install uv
+```
+
+Install the SDK, optional dependencies, and development dependencies in one command:
+
+```bash
+uv sync --all-extras --dev
+```
+
+This installs the `triton-inference-sdk` distribution into the project's `.venv`; import it
+in Python as `triton_sdk`. It does **not** install Triton Inference Server or vLLM: the
+SDK is a client, and it sends requests to a Triton server where a model is already
+running. You do not need Triton or vLLM installed on the computer running the SDK.
+
+### Prepare Triton and an LLM model
+
+If you do not already have a Triton deployment, follow NVIDIA's
+[Triton Server Quickstart](https://docs.nvidia.com/deeplearning/triton-inference-server/user-guide/docs/getting_started/quickstart.html)
+to install and start the server. A running Triton process alone is not enough: it must
+also have an LLM backend installed and a compatible model configured and loaded in its
+model repository. The SDK cannot install or load these server-side components for you.
+
+Choose the backend you plan to use and follow its setup guide:
+
+- **vLLM:** NVIDIA's [Triton vLLM backend guide](https://github.com/triton-inference-server/vllm_backend)
+  covers the backend-enabled container and sample model repository.
+- **TensorRT-LLM:** NVIDIA's [TensorRT-LLM backend Getting Started guide](https://github.com/triton-inference-server/tensorrtllm_backend#getting-started)
+  covers the compatible container, model configuration, and server launch. Match the
+  container and NVIDIA driver to your GPU; gated Hugging Face models may require a token.
+
+After following the backend guide, confirm the model is loaded and ready in Triton.
+Then use its server address and Triton model name in the example below. The model's input
+schema must be supported by this SDK.
+
+### Connect to your Triton server
+
+Once a Triton server is prepared, connect to it and pass the **Triton model name**
+configured on that server. This can be a vLLM model or a TensorRT-LLM ensemble/BLS model
+with a supported text-input schema; it is not necessarily the Hugging Face model ID.
+
+```python
+from triton_sdk import SamplingParams, TritonLLMClient
+
+with TritonLLMClient("your-triton-host:8001", protocol="grpc") as client:
+    response = client.generate(
+        "The capital of France is",
+        SamplingParams(max_tokens=32),
+        model="your-triton-model-name",
+    )
+    print(response.text)
+
+    for chunk in client.stream(
+        "Tell me a short story.",
+        SamplingParams(max_tokens=64),
+        model="your-triton-model-name",
+    ):
+        print(chunk.text, end="", flush=True)
+```
+
+To run the repository's end-to-end example, set `TRITON_DEFAULT_MODEL` to your Triton
+model name and run:
+
+PowerShell:
+
+```powershell
+$env:TRITON_DEFAULT_MODEL = "your-triton-model-name"
+uv run python examples/quickstart.py
+```
+
+On Linux/macOS:
+
+```bash
+export TRITON_DEFAULT_MODEL=your-triton-model-name
+uv run python examples/quickstart.py
+```
+
+The example uses HTTP at `localhost:8000` and gRPC at `localhost:8001` by default; set
+`TRITON_URL` and `TRITON_GRPC_URL` to override them. Set `TRITON_OPENAI_URL` to the
+address of an OpenAI-compatible Triton frontend to exercise that protocol too; otherwise,
+that part of the example is skipped. The example requires both native HTTP and gRPC
+endpoints, and a model configured for streaming (HTTP streaming also requires Triton's
+generate extension). It uses the Qwen2.5-0.5B-Instruct tokenizer by default for chat;
+set `TRITON_CHAT_TEMPLATE_MODEL` to a tokenizer matching your model. The first run may
+download the tokenizer from Hugging Face.
+The SDK reads model configuration from the server to select an adapter.
+
+The SDK does not install or launch Triton, backends, or models.
+
+## OpenAI-compatible interface
+
+Code written for `openai` is ported by swapping the constructor:
+
+```python
+from triton_sdk.openai import OpenAICompatClient
+
+client = OpenAICompatClient("localhost:8001", protocol="grpc", tokenizer="Qwen/Qwen2.5-0.5B-Instruct")
+resp = client.chat.completions.create(model="vllm_model", messages=[{"role": "user", "content": "Hello"}])
+resp.choices[0].message.content, resp.usage.total_tokens
+
+for chunk in client.chat.completions.create(model="vllm_model", messages=[...], stream=True):
+    chunk.choices[0].delta.content
+```
+
+Two engines behind one interface:
+
+- `protocol="http" | "grpc"` — native KServe v2, works with any Triton; the chat template and `usage` are computed on the client.
+- `protocol="openai"` — Triton's OpenAI frontend (`/v1/chat/completions`, port 9000); the template and `usage` are handled by the server,
+  `native` is unavailable.
+  `usage` is computed by the tokenizer from `tokenizer=` (without it — `None`). `tools`, `response_format`, `n > 1` are not supported
+  and raise `UnsupportedParameterError`; `extra_body={"top_k": ...}` forwards backend-specific parameters.
+
+To run the standalone example against an already-running OpenAI-compatible Triton frontend
+(defaults to `localhost:9000`, model `vllm_model`):
+
+```bash
+uv run --with openai python examples/openai_sdk.py
+```
+
+Set `TRITON_OPENAI_URL` and `TRITON_DEFAULT_MODEL` first if your frontend or model uses a
+different address or name.
+
+## Structure
+
+| Path                           | Role                                                                                                        |
+| ------------------------------ | ----------------------------------------------------------------------------------------------------------- |
+| `triton_sdk/client/`         | Public API: `TritonLLMClient` (sync) and `AsyncTritonLLMClient`                                         |
+| `triton_sdk/transport/`      | Network layer: `TransportInterface` + `HttpTransport` / `GrpcTransport`, SSE parser, protobuf stubs   |
+| `triton_sdk/backends/`       | Input/output schema adapters: `tensorrt_llm`, `vllm`, `generic` (generate endpoint)                   |
+| `triton_sdk/chat/`           | Chat templates: message list → raw prompt, token counting                                                  |
+| `triton_sdk/openai/`         | OpenAI-compatible facade: `chat.completions`, `completions`, `models` with types mirroring `openai` |
+| `triton_sdk/core/models/`    | Pydantic models for requests, responses, metadata, tensors                                                  |
+| `triton_sdk/core/constants/` | Protocols, KServe v2 paths, error codes, retry statuses                                                     |
+| `triton_sdk/settings.py`     | `TritonSettings` (`pydantic-settings`, `TRITON_` prefix)                                              |
+| `triton_sdk/exceptions.py`   | `TritonError` hierarchy                                                                                   |
+| `triton_sdk/tests/`          | `unit/` (transport mocks), `integration/` (live Triton, `integration` marker)                         |
+| `examples/`                  | Ready-made usage scenarios                                                                                  |
+| `model_repository/`          | Example vLLM model configuration for a Triton deployment                                                     |
+
+## Local development and integration tests
+
+```bash
+uv sync --all-extras --dev
+cp triton_sdk/config/environment/env.sh.template env.sh && source env.sh
+./code_quality_check.sh                                # ruff + mypy + pytest (unit)
+```
+
+Integration tests require a separately running Triton server and a vLLM model named
+`vllm_model`, with native HTTP and gRPC endpoints. Configure the HTTP server address in
+`env.sh`, then run `pytest -m integration`. The template sets the model name to
+`vllm_model`; change it if your server uses another name. Set `TRITON_GRPC_URL` and
+`TRITON_OPENAI_URL` when those endpoints use non-default addresses. Tests for Triton's
+OpenAI frontend are skipped when the frontend is unavailable.
+
+## License
+
+This project is licensed under the [MIT License](LICENSE).

triton_inference_sdk-0.1.0/README.md

@@ -0,0 +1,183 @@
+# Triton-sdk
+
+Python SDK for NVIDIA Triton Inference Server, focused on LLM workloads
+(TensorRT-LLM and vLLM backends). One public API — `generate` / `stream` / `chat`,
+with backend and transport (HTTP / gRPC) differences hidden inside.
+
+Key features:
+
+- **One LLM API across supported backends:** `generate`, `stream`, and `chat`
+  adapt vLLM and TensorRT-LLM model schemas, including backend-specific inputs
+  and outputs.
+- **OpenAI-style interface for native Triton endpoints:** use
+  `chat.completions` and `completions` over HTTP or gRPC without deploying
+  Triton's OpenAI frontend. Compatibility is partial; see the limitations below.
+
+## Quick start
+
+Clone the repository:
+
+```bash
+git clone https://github.com/Shubin-vadim/triron_sdk.git
+cd triron_sdk
+```
+
+Python 3.12 or newer is required. If `uv` is not installed, install it with pip:
+
+```bash
+python -m pip install uv
+```
+
+Install the SDK, optional dependencies, and development dependencies in one command:
+
+```bash
+uv sync --all-extras --dev
+```
+
+This installs the `triton-inference-sdk` distribution into the project's `.venv`; import it
+in Python as `triton_sdk`. It does **not** install Triton Inference Server or vLLM: the
+SDK is a client, and it sends requests to a Triton server where a model is already
+running. You do not need Triton or vLLM installed on the computer running the SDK.
+
+### Prepare Triton and an LLM model
+
+If you do not already have a Triton deployment, follow NVIDIA's
+[Triton Server Quickstart](https://docs.nvidia.com/deeplearning/triton-inference-server/user-guide/docs/getting_started/quickstart.html)
+to install and start the server. A running Triton process alone is not enough: it must
+also have an LLM backend installed and a compatible model configured and loaded in its
+model repository. The SDK cannot install or load these server-side components for you.
+
+Choose the backend you plan to use and follow its setup guide:
+
+- **vLLM:** NVIDIA's [Triton vLLM backend guide](https://github.com/triton-inference-server/vllm_backend)
+  covers the backend-enabled container and sample model repository.
+- **TensorRT-LLM:** NVIDIA's [TensorRT-LLM backend Getting Started guide](https://github.com/triton-inference-server/tensorrtllm_backend#getting-started)
+  covers the compatible container, model configuration, and server launch. Match the
+  container and NVIDIA driver to your GPU; gated Hugging Face models may require a token.
+
+After following the backend guide, confirm the model is loaded and ready in Triton.
+Then use its server address and Triton model name in the example below. The model's input
+schema must be supported by this SDK.
+
+### Connect to your Triton server
+
+Once a Triton server is prepared, connect to it and pass the **Triton model name**
+configured on that server. This can be a vLLM model or a TensorRT-LLM ensemble/BLS model
+with a supported text-input schema; it is not necessarily the Hugging Face model ID.
+
+```python
+from triton_sdk import SamplingParams, TritonLLMClient
+
+with TritonLLMClient("your-triton-host:8001", protocol="grpc") as client:
+    response = client.generate(
+        "The capital of France is",
+        SamplingParams(max_tokens=32),
+        model="your-triton-model-name",
+    )
+    print(response.text)
+
+    for chunk in client.stream(
+        "Tell me a short story.",
+        SamplingParams(max_tokens=64),
+        model="your-triton-model-name",
+    ):
+        print(chunk.text, end="", flush=True)
+```
+
+To run the repository's end-to-end example, set `TRITON_DEFAULT_MODEL` to your Triton
+model name and run:
+
+PowerShell:
+
+```powershell
+$env:TRITON_DEFAULT_MODEL = "your-triton-model-name"
+uv run python examples/quickstart.py
+```
+
+On Linux/macOS:
+
+```bash
+export TRITON_DEFAULT_MODEL=your-triton-model-name
+uv run python examples/quickstart.py
+```
+
+The example uses HTTP at `localhost:8000` and gRPC at `localhost:8001` by default; set
+`TRITON_URL` and `TRITON_GRPC_URL` to override them. Set `TRITON_OPENAI_URL` to the
+address of an OpenAI-compatible Triton frontend to exercise that protocol too; otherwise,
+that part of the example is skipped. The example requires both native HTTP and gRPC
+endpoints, and a model configured for streaming (HTTP streaming also requires Triton's
+generate extension). It uses the Qwen2.5-0.5B-Instruct tokenizer by default for chat;
+set `TRITON_CHAT_TEMPLATE_MODEL` to a tokenizer matching your model. The first run may
+download the tokenizer from Hugging Face.
+The SDK reads model configuration from the server to select an adapter.
+
+The SDK does not install or launch Triton, backends, or models.
+
+## OpenAI-compatible interface
+
+Code written for `openai` is ported by swapping the constructor:
+
+```python
+from triton_sdk.openai import OpenAICompatClient
+
+client = OpenAICompatClient("localhost:8001", protocol="grpc", tokenizer="Qwen/Qwen2.5-0.5B-Instruct")
+resp = client.chat.completions.create(model="vllm_model", messages=[{"role": "user", "content": "Hello"}])
+resp.choices[0].message.content, resp.usage.total_tokens
+
+for chunk in client.chat.completions.create(model="vllm_model", messages=[...], stream=True):
+    chunk.choices[0].delta.content
+```
+
+Two engines behind one interface:
+
+- `protocol="http" | "grpc"` — native KServe v2, works with any Triton; the chat template and `usage` are computed on the client.
+- `protocol="openai"` — Triton's OpenAI frontend (`/v1/chat/completions`, port 9000); the template and `usage` are handled by the server,
+  `native` is unavailable.
+  `usage` is computed by the tokenizer from `tokenizer=` (without it — `None`). `tools`, `response_format`, `n > 1` are not supported
+  and raise `UnsupportedParameterError`; `extra_body={"top_k": ...}` forwards backend-specific parameters.
+
+To run the standalone example against an already-running OpenAI-compatible Triton frontend
+(defaults to `localhost:9000`, model `vllm_model`):
+
+```bash
+uv run --with openai python examples/openai_sdk.py
+```
+
+Set `TRITON_OPENAI_URL` and `TRITON_DEFAULT_MODEL` first if your frontend or model uses a
+different address or name.
+
+## Structure
+
+| Path                           | Role                                                                                                        |
+| ------------------------------ | ----------------------------------------------------------------------------------------------------------- |
+| `triton_sdk/client/`         | Public API: `TritonLLMClient` (sync) and `AsyncTritonLLMClient`                                         |
+| `triton_sdk/transport/`      | Network layer: `TransportInterface` + `HttpTransport` / `GrpcTransport`, SSE parser, protobuf stubs   |
+| `triton_sdk/backends/`       | Input/output schema adapters: `tensorrt_llm`, `vllm`, `generic` (generate endpoint)                   |
+| `triton_sdk/chat/`           | Chat templates: message list → raw prompt, token counting                                                  |
+| `triton_sdk/openai/`         | OpenAI-compatible facade: `chat.completions`, `completions`, `models` with types mirroring `openai` |
+| `triton_sdk/core/models/`    | Pydantic models for requests, responses, metadata, tensors                                                  |
+| `triton_sdk/core/constants/` | Protocols, KServe v2 paths, error codes, retry statuses                                                     |
+| `triton_sdk/settings.py`     | `TritonSettings` (`pydantic-settings`, `TRITON_` prefix)                                              |
+| `triton_sdk/exceptions.py`   | `TritonError` hierarchy                                                                                   |
+| `triton_sdk/tests/`          | `unit/` (transport mocks), `integration/` (live Triton, `integration` marker)                         |
+| `examples/`                  | Ready-made usage scenarios                                                                                  |
+| `model_repository/`          | Example vLLM model configuration for a Triton deployment                                                     |
+
+## Local development and integration tests
+
+```bash
+uv sync --all-extras --dev
+cp triton_sdk/config/environment/env.sh.template env.sh && source env.sh
+./code_quality_check.sh                                # ruff + mypy + pytest (unit)
+```
+
+Integration tests require a separately running Triton server and a vLLM model named
+`vllm_model`, with native HTTP and gRPC endpoints. Configure the HTTP server address in
+`env.sh`, then run `pytest -m integration`. The template sets the model name to
+`vllm_model`; change it if your server uses another name. Set `TRITON_GRPC_URL` and
+`TRITON_OPENAI_URL` when those endpoints use non-default addresses. Tests for Triton's
+OpenAI frontend are skipped when the frontend is unavailable.
+
+## License
+
+This project is licensed under the [MIT License](LICENSE).

triton_inference_sdk-0.1.0/pyproject.toml

@@ -0,0 +1,109 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.packages.find]
+include = ["triton_sdk*"]
+exclude = ["triton_sdk.tests*"]
+
+[tool.setuptools.package-data]
+triton_sdk = ["py.typed"]
+
+[project]
+name = "triton-inference-sdk"
+version = "0.1.0"
+description = "Python SDK for NVIDIA Triton Inference Server focused on LLM workloads (TensorRT-LLM, vLLM)"
+readme = "README.md"
+requires-python = ">=3.12"
+license = "MIT"
+authors = [
+    { name = "Shubin Vadim", email = "midavnibush@gmail.com" },
+]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+dependencies = [
+    "httpx~=0.28.1",
+    "pydantic~=2.12.3",
+    "pydantic-settings~=2.12.0",
+    "tenacity>=9.0.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/Shubin-vadim/triron_sdk"
+Repository = "https://github.com/Shubin-vadim/triron_sdk"
+
+[project.optional-dependencies]
+grpc = ["grpcio>=1.66", "protobuf>=5.28"]
+chat = ["transformers>=4.45", "jinja2~=3.1.6"]
+all = ["triton-inference-sdk[grpc,chat]"]
+
+[dependency-groups]
+dev = [
+    "colorlog~=6.10.1",
+    "grpcio-tools>=1.66",
+    "mypy>=1.18.2",
+    "pytest>=8.4.2",
+    "pytest-asyncio>=1.2.0",
+    "pytest-cov>=6.0.0",
+    "python-dotenv>=1.0.0",
+    "respx>=0.22.0",
+    "ruff>=0.14.3",
+    "types-protobuf>=5.28",
+]
+
+[tool.ruff]
+line-length = 120
+target-version = "py312"
+extend-exclude = ["triton_sdk/transport/protos"]
+
+[tool.ruff.lint]
+select = ["ALL"]
+ignore = [
+    "D", "ANN201", "ANN401", "ANN002", "ANN003", "C90", "CPY", "COM812", "ERA", "FIX002",
+    "ISC001", "PERF203", "PLR09", "RUF012", "TD002", "TD003", "TRY301", "PLC0415", "PLW1641",
+    # SDK-specific: interface stubs with unused args,
+    # explicit timeout= parameters in the public async API, no TYPE_CHECKING gymnastics for pydantic models
+    "ARG002", "ASYNC109", "TC001", "TC002", "TC003",
+]
+
+[tool.ruff.lint.per-file-ignores]
+"triton_sdk/tests/*" = ["S101", "SLF001"]
+"examples/*" = ["T201", "INP001"]
+"scripts/*" = ["T201", "INP001"]
+
+[tool.ruff.format]
+quote-style = "double"
+indent-style = "space"
+
+[tool.ruff.lint.isort]
+lines-after-imports = 2
+
+[tool.ruff.lint.flake8-tidy-imports]
+ban-relative-imports = "all"
+
+[tool.mypy]
+python_version = "3.12"
+no_implicit_optional = true
+warn_redundant_casts = true
+warn_unused_ignores = false
+pretty = true
+show_error_codes = true
+exclude = ["triton_sdk/transport/protos/"]
+
+[[tool.mypy.overrides]]
+module = ["triton_sdk.transport.protos.*", "grpc.*", "transformers.*"]
+ignore_missing_imports = true
+ignore_errors = true
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+testpaths = ["triton_sdk/tests"]
+addopts = "-m 'not integration'"
+markers = [
+    "integration: requires a running Triton Inference Server",
+]

triton_inference_sdk-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+