swarmauri_llm_nvidia_nim 0.11.0.dev2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- swarmauri_llm_nvidia_nim/NvidiaNIMChatRetryPolicy.py +34 -0
- swarmauri_llm_nvidia_nim/NvidiaNIMModel.py +97 -0
- swarmauri_llm_nvidia_nim/NvidiaNIMToolModel.py +107 -0
- swarmauri_llm_nvidia_nim/__init__.py +16 -0
- swarmauri_llm_nvidia_nim/_client.py +29 -0
- swarmauri_llm_nvidia_nim-0.11.0.dev2.dist-info/METADATA +76 -0
- swarmauri_llm_nvidia_nim-0.11.0.dev2.dist-info/RECORD +9 -0
- swarmauri_llm_nvidia_nim-0.11.0.dev2.dist-info/WHEEL +4 -0
- swarmauri_llm_nvidia_nim-0.11.0.dev2.dist-info/entry_points.txt +6 -0
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""Retry policy for NVIDIA NIM chat operations."""
|
|
2
|
+
|
|
3
|
+
from swarmauri_core.transports.i_http_response import IHttpResponse
|
|
4
|
+
from swarmauri_transport_retry import ExponentialBackoffRetryPolicy
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class NvidiaNIMChatRetryPolicy(ExponentialBackoffRetryPolicy):
|
|
8
|
+
"""Retry safe reads and rate-limited chat submissions."""
|
|
9
|
+
|
|
10
|
+
def __init__(self, **kwargs) -> None:
|
|
11
|
+
kwargs.setdefault("retryable_methods", {"GET", "POST"})
|
|
12
|
+
super().__init__(**kwargs)
|
|
13
|
+
|
|
14
|
+
def should_retry(
|
|
15
|
+
self,
|
|
16
|
+
*,
|
|
17
|
+
method: str,
|
|
18
|
+
attempt: int,
|
|
19
|
+
response: IHttpResponse | None = None,
|
|
20
|
+
error: Exception | None = None,
|
|
21
|
+
) -> bool:
|
|
22
|
+
if method.upper() == "POST":
|
|
23
|
+
return (
|
|
24
|
+
attempt < self.max_attempts
|
|
25
|
+
and error is None
|
|
26
|
+
and response is not None
|
|
27
|
+
and response.status_code == 429
|
|
28
|
+
)
|
|
29
|
+
return super().should_retry(
|
|
30
|
+
method=method,
|
|
31
|
+
attempt=attempt,
|
|
32
|
+
response=response,
|
|
33
|
+
error=error,
|
|
34
|
+
)
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
from typing import Any, ClassVar, Literal
|
|
2
|
+
|
|
3
|
+
from pydantic import PrivateAttr
|
|
4
|
+
from swarmauri_base.ComponentBase import ComponentBase
|
|
5
|
+
from swarmauri_base.llms.LLMBase import LLMBase
|
|
6
|
+
from swarmauri_core.transports.i_http_client import IHttpClient
|
|
7
|
+
from swarmauri_core.transports.i_http_retry_policy import IHttpRetryPolicy
|
|
8
|
+
from swarmauri_core.transports.i_managed_http_client import IManagedHttpClient
|
|
9
|
+
from swarmauri_standard.utils.provider_chat_transport import (
|
|
10
|
+
abatch_chat,
|
|
11
|
+
apredict_chat,
|
|
12
|
+
astream_chat,
|
|
13
|
+
batch_chat,
|
|
14
|
+
predict_chat,
|
|
15
|
+
stream_chat,
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
from ._client import build_nvidia_nim_client
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@ComponentBase.register_type()
|
|
22
|
+
class NvidiaNIMModel(LLMBase):
|
|
23
|
+
"""Chat adapter for an NVIDIA NIM for LLMs deployment."""
|
|
24
|
+
|
|
25
|
+
type: Literal["NvidiaNIMModel"] = "NvidiaNIMModel"
|
|
26
|
+
base_url: str = "http://localhost:8000"
|
|
27
|
+
name: str = "meta/llama-3.1-8b-instruct"
|
|
28
|
+
discover_models: bool = False
|
|
29
|
+
capabilities: ClassVar[frozenset[str]] = frozenset(
|
|
30
|
+
{"chat_completions", "structured_output", "streaming", "tools"}
|
|
31
|
+
)
|
|
32
|
+
_http_client: IHttpClient = PrivateAttr()
|
|
33
|
+
_retry_owned_by_transport: bool = PrivateAttr(default=True)
|
|
34
|
+
|
|
35
|
+
predict = predict_chat
|
|
36
|
+
apredict = apredict_chat
|
|
37
|
+
stream = stream_chat
|
|
38
|
+
astream = astream_chat
|
|
39
|
+
batch = batch_chat
|
|
40
|
+
abatch = abatch_chat
|
|
41
|
+
|
|
42
|
+
def __init__(
|
|
43
|
+
self,
|
|
44
|
+
*,
|
|
45
|
+
http_client: IHttpClient | None = None,
|
|
46
|
+
retry_policy: IHttpRetryPolicy | None = None,
|
|
47
|
+
**data: Any,
|
|
48
|
+
) -> None:
|
|
49
|
+
selected_name = data.get(
|
|
50
|
+
"name", self.__class__.model_fields["name"].default
|
|
51
|
+
)
|
|
52
|
+
data.setdefault("allowed_models", [selected_name])
|
|
53
|
+
super().__init__(**data)
|
|
54
|
+
self._http_client = build_nvidia_nim_client(
|
|
55
|
+
base_url=self.base_url,
|
|
56
|
+
timeout=self.timeout,
|
|
57
|
+
max_attempts=self.max_retries,
|
|
58
|
+
http_client=http_client,
|
|
59
|
+
retry_policy=retry_policy,
|
|
60
|
+
)
|
|
61
|
+
if self.discover_models:
|
|
62
|
+
self.allowed_models = self._get_models()
|
|
63
|
+
|
|
64
|
+
def _api_root(self) -> str:
|
|
65
|
+
root = self.base_url.rstrip("/")
|
|
66
|
+
return root if root.endswith("/v1") else f"{root}/v1"
|
|
67
|
+
|
|
68
|
+
def _build_endpoint(self) -> str:
|
|
69
|
+
return f"{self._api_root()}/chat/completions"
|
|
70
|
+
|
|
71
|
+
def _build_headers(self) -> dict[str, str]:
|
|
72
|
+
headers = {"Content-Type": "application/json"}
|
|
73
|
+
if self.api_key is not None:
|
|
74
|
+
headers["Authorization"] = (
|
|
75
|
+
f"Bearer {self.api_key.get_secret_value()}"
|
|
76
|
+
)
|
|
77
|
+
return headers
|
|
78
|
+
|
|
79
|
+
def _get_models(self) -> list[str]:
|
|
80
|
+
if not self.discover_models:
|
|
81
|
+
return [self.name] if self.name else []
|
|
82
|
+
response = self._http_client.request(
|
|
83
|
+
"GET",
|
|
84
|
+
f"{self._api_root()}/models",
|
|
85
|
+
headers=self._build_headers(),
|
|
86
|
+
)
|
|
87
|
+
response.raise_for_status()
|
|
88
|
+
data: list[dict[str, Any]] = response.json().get("data") or []
|
|
89
|
+
return [model["id"] for model in data if model.get("id")]
|
|
90
|
+
|
|
91
|
+
def close(self) -> None:
|
|
92
|
+
if isinstance(self._http_client, IManagedHttpClient):
|
|
93
|
+
self._http_client.close()
|
|
94
|
+
|
|
95
|
+
async def aclose(self) -> None:
|
|
96
|
+
if isinstance(self._http_client, IManagedHttpClient):
|
|
97
|
+
await self._http_client.aclose()
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
from typing import Any, ClassVar, Literal
|
|
2
|
+
|
|
3
|
+
from pydantic import PrivateAttr
|
|
4
|
+
from swarmauri_base.ComponentBase import ComponentBase
|
|
5
|
+
from swarmauri_base.tool_llms.ToolLLMBase import ToolLLMBase
|
|
6
|
+
from swarmauri_core.transports.i_http_client import IHttpClient
|
|
7
|
+
from swarmauri_core.transports.i_http_retry_policy import IHttpRetryPolicy
|
|
8
|
+
from swarmauri_core.transports.i_managed_http_client import IManagedHttpClient
|
|
9
|
+
from swarmauri_standard.schema_converters.OpenAISchemaConverter import (
|
|
10
|
+
OpenAISchemaConverter,
|
|
11
|
+
)
|
|
12
|
+
from swarmauri_standard.utils.provider_chat_transport import (
|
|
13
|
+
abatch_tools,
|
|
14
|
+
apredict_tools,
|
|
15
|
+
astream_tools,
|
|
16
|
+
batch_tools,
|
|
17
|
+
format_messages,
|
|
18
|
+
predict_tools,
|
|
19
|
+
stream_tools,
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
from ._client import build_nvidia_nim_client
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@ComponentBase.register_type()
|
|
26
|
+
class NvidiaNIMToolModel(ToolLLMBase):
|
|
27
|
+
"""Tool-calling adapter for an NVIDIA NIM for LLMs deployment."""
|
|
28
|
+
|
|
29
|
+
type: Literal["NvidiaNIMToolModel"] = "NvidiaNIMToolModel"
|
|
30
|
+
base_url: str = "http://localhost:8000"
|
|
31
|
+
name: str = "meta/llama-3.1-8b-instruct"
|
|
32
|
+
discover_models: bool = False
|
|
33
|
+
capabilities: ClassVar[frozenset[str]] = frozenset(
|
|
34
|
+
{"chat_completions", "streaming", "tools"}
|
|
35
|
+
)
|
|
36
|
+
_http_client: IHttpClient = PrivateAttr()
|
|
37
|
+
_retry_owned_by_transport: bool = PrivateAttr(default=True)
|
|
38
|
+
|
|
39
|
+
predict = predict_tools
|
|
40
|
+
apredict = apredict_tools
|
|
41
|
+
stream = stream_tools
|
|
42
|
+
astream = astream_tools
|
|
43
|
+
batch = batch_tools
|
|
44
|
+
abatch = abatch_tools
|
|
45
|
+
|
|
46
|
+
def __init__(
|
|
47
|
+
self,
|
|
48
|
+
*,
|
|
49
|
+
http_client: IHttpClient | None = None,
|
|
50
|
+
retry_policy: IHttpRetryPolicy | None = None,
|
|
51
|
+
**data: Any,
|
|
52
|
+
) -> None:
|
|
53
|
+
selected_name = data.get(
|
|
54
|
+
"name", self.__class__.model_fields["name"].default
|
|
55
|
+
)
|
|
56
|
+
data.setdefault("allowed_models", [selected_name])
|
|
57
|
+
super().__init__(**data)
|
|
58
|
+
self._http_client = build_nvidia_nim_client(
|
|
59
|
+
base_url=self.base_url,
|
|
60
|
+
timeout=self.timeout,
|
|
61
|
+
max_attempts=self.max_retries,
|
|
62
|
+
http_client=http_client,
|
|
63
|
+
retry_policy=retry_policy,
|
|
64
|
+
)
|
|
65
|
+
if self.discover_models:
|
|
66
|
+
self.allowed_models = self._get_models()
|
|
67
|
+
|
|
68
|
+
def get_schema_converter(self):
|
|
69
|
+
return OpenAISchemaConverter()
|
|
70
|
+
|
|
71
|
+
def _format_messages(self, messages):
|
|
72
|
+
return format_messages(messages)
|
|
73
|
+
|
|
74
|
+
def _api_root(self) -> str:
|
|
75
|
+
root = self.base_url.rstrip("/")
|
|
76
|
+
return root if root.endswith("/v1") else f"{root}/v1"
|
|
77
|
+
|
|
78
|
+
def _build_endpoint(self) -> str:
|
|
79
|
+
return f"{self._api_root()}/chat/completions"
|
|
80
|
+
|
|
81
|
+
def _build_headers(self) -> dict[str, str]:
|
|
82
|
+
headers = {"Content-Type": "application/json"}
|
|
83
|
+
if self.api_key is not None:
|
|
84
|
+
headers["Authorization"] = (
|
|
85
|
+
f"Bearer {self.api_key.get_secret_value()}"
|
|
86
|
+
)
|
|
87
|
+
return headers
|
|
88
|
+
|
|
89
|
+
def _get_models(self) -> list[str]:
|
|
90
|
+
if not self.discover_models:
|
|
91
|
+
return [self.name] if self.name else []
|
|
92
|
+
response = self._http_client.request(
|
|
93
|
+
"GET",
|
|
94
|
+
f"{self._api_root()}/models",
|
|
95
|
+
headers=self._build_headers(),
|
|
96
|
+
)
|
|
97
|
+
response.raise_for_status()
|
|
98
|
+
data: list[dict[str, Any]] = response.json().get("data") or []
|
|
99
|
+
return [model["id"] for model in data if model.get("id")]
|
|
100
|
+
|
|
101
|
+
def close(self) -> None:
|
|
102
|
+
if isinstance(self._http_client, IManagedHttpClient):
|
|
103
|
+
self._http_client.close()
|
|
104
|
+
|
|
105
|
+
async def aclose(self) -> None:
|
|
106
|
+
if isinstance(self._http_client, IManagedHttpClient):
|
|
107
|
+
await self._http_client.aclose()
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
2
|
+
|
|
3
|
+
from .NvidiaNIMChatRetryPolicy import NvidiaNIMChatRetryPolicy
|
|
4
|
+
from .NvidiaNIMModel import NvidiaNIMModel
|
|
5
|
+
from .NvidiaNIMToolModel import NvidiaNIMToolModel
|
|
6
|
+
|
|
7
|
+
try:
|
|
8
|
+
__version__ = version("swarmauri_llm_nvidia_nim")
|
|
9
|
+
except PackageNotFoundError:
|
|
10
|
+
__version__ = "0.0.0"
|
|
11
|
+
|
|
12
|
+
__all__ = [
|
|
13
|
+
"NvidiaNIMChatRetryPolicy",
|
|
14
|
+
"NvidiaNIMModel",
|
|
15
|
+
"NvidiaNIMToolModel",
|
|
16
|
+
]
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""HTTP client composition for NVIDIA NIM models."""
|
|
2
|
+
|
|
3
|
+
from swarmauri_core.transports.i_http_client import IHttpClient
|
|
4
|
+
from swarmauri_core.transports.i_http_retry_policy import IHttpRetryPolicy
|
|
5
|
+
from swarmauri_transport_httpx import HttpxClient
|
|
6
|
+
from swarmauri_transport_retry import RetryingHttpClient
|
|
7
|
+
|
|
8
|
+
from .NvidiaNIMChatRetryPolicy import NvidiaNIMChatRetryPolicy
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def build_nvidia_nim_client(
|
|
12
|
+
*,
|
|
13
|
+
base_url: str,
|
|
14
|
+
timeout: float,
|
|
15
|
+
max_attempts: int,
|
|
16
|
+
http_client: IHttpClient | None,
|
|
17
|
+
retry_policy: IHttpRetryPolicy | None,
|
|
18
|
+
) -> IHttpClient:
|
|
19
|
+
"""Build a persistent client while preserving injected ownership."""
|
|
20
|
+
concrete_client = http_client or HttpxClient(
|
|
21
|
+
base_url=base_url,
|
|
22
|
+
timeout=timeout,
|
|
23
|
+
)
|
|
24
|
+
return RetryingHttpClient(
|
|
25
|
+
client=concrete_client,
|
|
26
|
+
policy=retry_policy
|
|
27
|
+
or NvidiaNIMChatRetryPolicy(max_attempts=max(max_attempts, 1)),
|
|
28
|
+
own_client=http_client is None,
|
|
29
|
+
)
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: swarmauri_llm_nvidia_nim
|
|
3
|
+
Version: 0.11.0.dev2
|
|
4
|
+
Summary: Swarmauri chat and tool-calling adapters for NVIDIA NIM OpenAI-compatible deployments.
|
|
5
|
+
License-Expression: Apache-2.0
|
|
6
|
+
Keywords: swarmauri,llm,nvidia,nim,tool calling,streaming,openai compatible
|
|
7
|
+
Author: Jacob Stewart
|
|
8
|
+
Author-email: jacob@swarmauri.com
|
|
9
|
+
Requires-Python: >=3.10,<3.13
|
|
10
|
+
Classifier: Development Status :: 1 - Planning
|
|
11
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Requires-Dist: swarmauri_base (>=0.11.0.dev5)
|
|
16
|
+
Requires-Dist: swarmauri_core (>=0.11.0.dev5)
|
|
17
|
+
Requires-Dist: swarmauri_standard (>=0.11.0.dev6)
|
|
18
|
+
Requires-Dist: swarmauri_transport_httpx (>=0.1.0.dev5)
|
|
19
|
+
Requires-Dist: swarmauri_transport_retry (>=0.1.0.dev1)
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
|
|
22
|
+

|
|
23
|
+
|
|
24
|
+
<p align="center">
|
|
25
|
+
<a href="https://pepy.tech/project/swarmauri_llm_nvidia_nim"><img src="https://static.pepy.tech/badge/swarmauri_llm_nvidia_nim/month" alt="Downloads"></a>
|
|
26
|
+
<a href="https://hits.sh/github.com/swarmauri/swarmauri-sdk/tree/master/pkgs/community/swarmauri_llm_nvidia_nim/"><img src="https://hits.sh/github.com/swarmauri/swarmauri-sdk/tree/master/pkgs/community/swarmauri_llm_nvidia_nim.svg" alt="Hits"></a>
|
|
27
|
+
<a href="https://pypi.org/project/swarmauri_llm_nvidia_nim/"><img src="https://img.shields.io/badge/python-3.10%20%7C%203.11%20%7C%203.12-blue" alt="Python"></a>
|
|
28
|
+
<a href="https://pypi.org/project/swarmauri_llm_nvidia_nim/"><img src="https://img.shields.io/pypi/l/swarmauri_llm_nvidia_nim" alt="License"></a>
|
|
29
|
+
<a href="https://pypi.org/project/swarmauri_llm_nvidia_nim/"><img src="https://img.shields.io/pypi/v/swarmauri_llm_nvidia_nim" alt="Release"></a>
|
|
30
|
+
</p>
|
|
31
|
+
|
|
32
|
+
# Swarmauri NVIDIA NIM LLM
|
|
33
|
+
|
|
34
|
+
`swarmauri_llm_nvidia_nim` connects Swarmauri conversations and toolkits to an NVIDIA NIM for LLMs deployment.
|
|
35
|
+
|
|
36
|
+
## Features
|
|
37
|
+
|
|
38
|
+
- OpenAI-compatible chat completions through `NvidiaNIMModel`.
|
|
39
|
+
- Tool calling through `NvidiaNIMToolModel`.
|
|
40
|
+
- Synchronous, asynchronous, incremental streaming, and batch workflows inherited from Swarmauri's shared transports.
|
|
41
|
+
- Optional discovery from the deployment's `/v1/models` endpoint.
|
|
42
|
+
- Configurable deployment base URL for local, private, and managed NIM endpoints.
|
|
43
|
+
- Persistent, injectable Swarmauri HTTP clients with conservative rate-limit retries.
|
|
44
|
+
|
|
45
|
+
## Installation
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
uv add swarmauri_llm_nvidia_nim
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
pip install swarmauri_llm_nvidia_nim
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
## Usage
|
|
56
|
+
|
|
57
|
+
```python
|
|
58
|
+
from swarmauri_llm_nvidia_nim import NvidiaNIMModel
|
|
59
|
+
from swarmauri_standard.conversations.Conversation import Conversation
|
|
60
|
+
from swarmauri_standard.messages.HumanMessage import HumanMessage
|
|
61
|
+
|
|
62
|
+
conversation = Conversation(messages=[HumanMessage(content="What is accelerated computing?")])
|
|
63
|
+
model = NvidiaNIMModel(
|
|
64
|
+
base_url="http://localhost:8000",
|
|
65
|
+
name="meta/llama-3.1-8b-instruct",
|
|
66
|
+
)
|
|
67
|
+
model.predict(conversation)
|
|
68
|
+
print(conversation.get_last().content)
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
Pass `api_key` when the deployment requires bearer authentication. Set `discover_models=True` to populate `allowed_models` from the running deployment. For tool workflows, instantiate `NvidiaNIMToolModel` and pass a Swarmauri `Toolkit` to `predict`, `apredict`, `stream`, or `astream`.
|
|
72
|
+
|
|
73
|
+
Pass an `IHttpClient` as `http_client` to integrate a custom transport or to test without network access. Pass an `IHttpRetryPolicy` as `retry_policy` to replace the default policy, which retries safe reads and rate-limited chat submissions without replaying failed POST requests for generic server errors.
|
|
74
|
+
|
|
75
|
+
See the [NVIDIA NIM LLM API reference](https://docs.nvidia.com/nim/large-language-models/latest/reference/api-reference.html) for deployment and model-specific capability details.
|
|
76
|
+
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
swarmauri_llm_nvidia_nim/NvidiaNIMChatRetryPolicy.py,sha256=1vxcE53FY3Danq6zINznA91PHipxI398XtJXOb1BBE8,1048
|
|
2
|
+
swarmauri_llm_nvidia_nim/NvidiaNIMModel.py,sha256=JSIpbTsKYAX6esou4FUGos8ZQMObiP-FPLkdJCGNjrI,3284
|
|
3
|
+
swarmauri_llm_nvidia_nim/NvidiaNIMToolModel.py,sha256=qRXj9ZXt69NynZESesaPyLz8EVqY3xYE_TOaOGlrsFg,3595
|
|
4
|
+
swarmauri_llm_nvidia_nim/__init__.py,sha256=kpusnpkiaNwBnuCmHJKrD72UnChbQ9-bRdVLIN6NONs,429
|
|
5
|
+
swarmauri_llm_nvidia_nim/_client.py,sha256=snkFCgPbWSJvTzDDUcwcqVDvjBLrDmmxRfYW8nP0XHo,952
|
|
6
|
+
swarmauri_llm_nvidia_nim-0.11.0.dev2.dist-info/METADATA,sha256=Dk_0JUjRnKsB8ap5_A4IPpddkXSjjtFT2qh6Jev4L_E,3942
|
|
7
|
+
swarmauri_llm_nvidia_nim-0.11.0.dev2.dist-info/WHEEL,sha256=L-WvLdSBvJWHXv5zrQxDvNRnIgoBzLUpcbuiw19KCuQ,88
|
|
8
|
+
swarmauri_llm_nvidia_nim-0.11.0.dev2.dist-info/entry_points.txt,sha256=DKMH0acJLSkxZx1ABBAwFY5fIcNfZkgzdP_DMt1-xrQ,159
|
|
9
|
+
swarmauri_llm_nvidia_nim-0.11.0.dev2.dist-info/RECORD,,
|