nimstatus 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nimstatus-0.1.0.dist-info/METADATA +106 -0
- nimstatus-0.1.0.dist-info/RECORD +7 -0
- nimstatus-0.1.0.dist-info/WHEEL +5 -0
- nimstatus-0.1.0.dist-info/licenses/LICENSE +21 -0
- nimstatus-0.1.0.dist-info/top_level.txt +2 -0
- nimstatus.py +20 -0
- nimstatus_client.py +142 -0
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: nimstatus
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Resolve NVIDIA model aliases and fail over using a public health directory
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Project-URL: Repository, https://github.com/Jaykay73/free-models
|
|
7
|
+
Project-URL: Documentation, https://github.com/Jaykay73/free-models/blob/main/docs/USAGE.md
|
|
8
|
+
Requires-Python: >=3.11
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Dynamic: license-file
|
|
12
|
+
|
|
13
|
+
# nimstatus
|
|
14
|
+
|
|
15
|
+
A dependency-free Python 3.11+ library that replaces hardcoded NVIDIA model IDs
|
|
16
|
+
with aliases: `fast`, `general`, `code`, `tools`, and `json`.
|
|
17
|
+
|
|
18
|
+
## Install
|
|
19
|
+
|
|
20
|
+
After the first PyPI release:
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
python -m pip install nimstatus
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
To install from a local checkout:
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
python -m pip install ./clients/python
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
After these files have been pushed to GitHub:
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
python -m pip install "nimstatus @ git+https://github.com/Jaykay73/free-models.git@main#subdirectory=clients/python"
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
For reproducible application builds, replace `main` with a reviewed commit SHA.
|
|
39
|
+
This pins the client implementation; model selections still refresh dynamically.
|
|
40
|
+
Use the client in your application environment. Use a separate environment if you also install the monitor, which shares the `nimstatus` import name.
|
|
41
|
+
|
|
42
|
+
## Use without model IDs
|
|
43
|
+
|
|
44
|
+
```python
|
|
45
|
+
from nimstatus import ModelRouter
|
|
46
|
+
|
|
47
|
+
router = ModelRouter()
|
|
48
|
+
print(router.primary("fast"))
|
|
49
|
+
print(router.resolve("code"))
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
The default directory is `https://jaykay73.github.io/free-models/status.json`.
|
|
53
|
+
No network request occurs when constructing the router. Calls refresh its
|
|
54
|
+
process-local cache at most once every five minutes.
|
|
55
|
+
|
|
56
|
+
For inference, install the OpenAI SDK in your application and provide your
|
|
57
|
+
key only to that SDK:
|
|
58
|
+
|
|
59
|
+
```python
|
|
60
|
+
import os
|
|
61
|
+
from openai import OpenAI
|
|
62
|
+
from nimstatus import ModelRouter
|
|
63
|
+
|
|
64
|
+
api = OpenAI(
|
|
65
|
+
base_url="https://integrate.api.nvidia.com/v1",
|
|
66
|
+
api_key=os.environ["NVIDIA_API_KEY"],
|
|
67
|
+
max_retries=0,
|
|
68
|
+
timeout=30,
|
|
69
|
+
)
|
|
70
|
+
router = ModelRouter(max_attempts=3)
|
|
71
|
+
|
|
72
|
+
answer = router.call("fast", lambda model: api.chat.completions.create(
|
|
73
|
+
model=model,
|
|
74
|
+
messages=[{"role": "user", "content": "Explain what a dictionary is."}],
|
|
75
|
+
max_tokens=256,
|
|
76
|
+
))
|
|
77
|
+
print(answer.choices[0].message.content)
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
Only the alias stays in application code. Each call resolves current routing
|
|
81
|
+
through the cache; 429, 5xx and recognized connection/timeout failures try the
|
|
82
|
+
next model. Auth and bad-request errors propagate immediately. Unknown errors
|
|
83
|
+
are not retried. Override classification with `ModelRouter(retryable=...)` for
|
|
84
|
+
other SDKs. SDK retrying is disabled in the example so attempts remain bounded.
|
|
85
|
+
|
|
86
|
+
This library sends no inference traffic and accepts no API key. Your callback
|
|
87
|
+
calls NVIDIA directly. The monitor uses a separate maintainer key for probes.
|
|
88
|
+
|
|
89
|
+
## Outages and limits
|
|
90
|
+
|
|
91
|
+
Expired status and failed refreshes retain the last good in-memory document
|
|
92
|
+
with a warning. Optional `defaults={alias: [...]}` can provide emergency model
|
|
93
|
+
names, but is **not required**. Without defaults or a prior cache, a failed first
|
|
94
|
+
fetch means `resolve` returns an empty list and `primary`/`call` raise
|
|
95
|
+
`NoModelsAvailable`. The cache does not survive process restarts. No library can
|
|
96
|
+
discover a model during an outage without some available source of model names.
|
|
97
|
+
|
|
98
|
+
Only synchronous calls are supported. Fallback handles failures raised by the
|
|
99
|
+
callback, not errors while consuming a returned streaming iterator. The current
|
|
100
|
+
urllib timeout is three seconds per blocking operation, not a strict total
|
|
101
|
+
deadline. Defaults are used when no route is available, not appended after an
|
|
102
|
+
otherwise nonempty remote route. Stale routing can currently be retained
|
|
103
|
+
indefinitely. Do not depend on a successful directory fetch to start your app.
|
|
104
|
+
|
|
105
|
+
Consumers need no NVIDIA model IDs; the **monitor maintainer still classifies
|
|
106
|
+
models in `models.yml`**. New unclassified models never enter aliases until tagged.
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
nimstatus.py,sha256=5VlVoEInIlvhdKQBnm2vPP3ZJPOG3DVx2AlNsuz6t_w,376
|
|
2
|
+
nimstatus_client.py,sha256=n_SgQGvLTnqpb_uXMfjVv7aqALDqEOMDkXpE6dxTo-E,6937
|
|
3
|
+
nimstatus-0.1.0.dist-info/licenses/LICENSE,sha256=-MQZXzzuutTwvPRrHirb0Zc3fU32mMq4u1jlfEeAx1w,1086
|
|
4
|
+
nimstatus-0.1.0.dist-info/METADATA,sha256=v5vwDK9Dr7uK9AOR_D3PFSt14CVB0jLKzCLtQqtTa2w,3880
|
|
5
|
+
nimstatus-0.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
6
|
+
nimstatus-0.1.0.dist-info/top_level.txt,sha256=gNp2DA_wd6HOoX6XfTCYW1Q81VdxyXRK7qsgGnUyQ0g,27
|
|
7
|
+
nimstatus-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 nim-model-status contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
nimstatus.py
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""Public imports for the nimstatus client distribution."""
|
|
2
|
+
|
|
3
|
+
from nimstatus_client import (
|
|
4
|
+
DEFAULT_STATUS_URL,
|
|
5
|
+
ModelRouter,
|
|
6
|
+
NoModelsAvailable,
|
|
7
|
+
__version__,
|
|
8
|
+
call_with_fallback,
|
|
9
|
+
is_retryable,
|
|
10
|
+
resolve,
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
__all__ = [
|
|
14
|
+
"DEFAULT_STATUS_URL",
|
|
15
|
+
"ModelRouter",
|
|
16
|
+
"NoModelsAvailable",
|
|
17
|
+
"call_with_fallback",
|
|
18
|
+
"is_retryable",
|
|
19
|
+
"resolve",
|
|
20
|
+
]
|
nimstatus_client.py
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
"""Stdlib-only status resolution. Never accepts inference credentials."""
|
|
2
|
+
from datetime import datetime, timezone
|
|
3
|
+
import json
|
|
4
|
+
import math
|
|
5
|
+
import sys
|
|
6
|
+
import time
|
|
7
|
+
import urllib.request
|
|
8
|
+
import warnings
|
|
9
|
+
|
|
10
|
+
__version__ = "0.1.0"
|
|
11
|
+
DEFAULT_STATUS_URL = "https://jaykay73.github.io/free-models/status.json"
|
|
12
|
+
_cache = {}
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class NoModelsAvailable(RuntimeError):
|
|
16
|
+
"""Neither the directory, cache nor optional defaults supplied a model."""
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def is_retryable(error):
|
|
20
|
+
"""Retry 429/5xx and connection/timeouts; never retry auth or bad requests.
|
|
21
|
+
|
|
22
|
+
Recognizes OpenAI SDK connection/timeout exceptions if the caller has
|
|
23
|
+
imported that SDK. The SDK is not a dependency of this library.
|
|
24
|
+
"""
|
|
25
|
+
status = getattr(error, "status_code", getattr(error, "code", None))
|
|
26
|
+
if type(status) is int:
|
|
27
|
+
return status == 429 or 500 <= status <= 599
|
|
28
|
+
if isinstance(error, (TimeoutError, ConnectionError)):
|
|
29
|
+
return True
|
|
30
|
+
sdk = sys.modules.get("openai")
|
|
31
|
+
connection_error = getattr(sdk, "APIConnectionError", None)
|
|
32
|
+
return isinstance(connection_error, type) and isinstance(error, connection_error)
|
|
33
|
+
|
|
34
|
+
def _validate(data):
|
|
35
|
+
if not isinstance(data, dict) or data.get("schema_version") != 1 or not isinstance(data.get("aliases"), dict):
|
|
36
|
+
raise ValueError("Invalid status document")
|
|
37
|
+
expiry = datetime.fromisoformat(data["valid_until"].replace("Z", "+00:00"))
|
|
38
|
+
if expiry.tzinfo is None:
|
|
39
|
+
raise ValueError("Missing expiry timezone")
|
|
40
|
+
for entry in data["aliases"].values():
|
|
41
|
+
if not isinstance(entry, dict) or not isinstance(entry.get("fallbacks"), list) or type(entry.get("stale")) is not bool:
|
|
42
|
+
raise ValueError("Invalid alias")
|
|
43
|
+
if entry.get("primary") is not None and not isinstance(entry["primary"], str):
|
|
44
|
+
raise ValueError("Invalid primary")
|
|
45
|
+
if any(not isinstance(m, str) or not m for m in entry["fallbacks"]):
|
|
46
|
+
raise ValueError("Invalid fallback")
|
|
47
|
+
return data
|
|
48
|
+
|
|
49
|
+
def resolve(alias, options=None, *, status_url=DEFAULT_STATUS_URL, ttl_s=300, defaults=None):
|
|
50
|
+
"""Return ordered names; warnings flag expired, retained or fallback routing.
|
|
51
|
+
|
|
52
|
+
Accepts an options dict for parity with JavaScript, or Python keyword args.
|
|
53
|
+
Cache is per URL. Failed refreshes are also delayed by TTL to avoid a storm.
|
|
54
|
+
"""
|
|
55
|
+
options = options or {}
|
|
56
|
+
status_url = options.get("status_url", status_url)
|
|
57
|
+
ttl_s = options.get("ttl_s", ttl_s)
|
|
58
|
+
defaults = options.get("defaults", defaults) or {}
|
|
59
|
+
if not status_url or not math.isfinite(ttl_s) or ttl_s < 300:
|
|
60
|
+
raise ValueError("status_url required; ttl_s must be at least 300")
|
|
61
|
+
now = time.monotonic()
|
|
62
|
+
cached = _cache.get(status_url)
|
|
63
|
+
if cached is None or now - cached["checked"] >= ttl_s:
|
|
64
|
+
try:
|
|
65
|
+
with urllib.request.urlopen(status_url, timeout=3) as response:
|
|
66
|
+
raw = response.read(1_000_001)
|
|
67
|
+
if len(raw) > 1_000_000:
|
|
68
|
+
raise ValueError("Status document too large")
|
|
69
|
+
data = _validate(json.loads(raw))
|
|
70
|
+
cached = {"data": data, "checked": now, "failed": False}
|
|
71
|
+
except Exception:
|
|
72
|
+
cached = {"data": cached["data"] if cached else None, "checked": now, "failed": True}
|
|
73
|
+
_cache[status_url] = cached
|
|
74
|
+
data = cached["data"]
|
|
75
|
+
entry = (data or {}).get("aliases", {}).get(alias, {})
|
|
76
|
+
names = [entry.get("primary"), *entry.get("fallbacks", [])] if entry.get("primary") else []
|
|
77
|
+
if cached["failed"] or not names or entry.get("stale") or (data and datetime.now(timezone.utc) > datetime.fromisoformat(data["valid_until"].replace("Z", "+00:00"))):
|
|
78
|
+
warnings.warn("nimstatus: stale status or caller defaults in use", RuntimeWarning, stacklevel=2)
|
|
79
|
+
return list(dict.fromkeys(names or defaults.get(alias, [])))
|
|
80
|
+
|
|
81
|
+
def call_with_fallback(alias, fn, options=None, *, retryable=is_retryable, max_attempts=5, **resolve_options):
|
|
82
|
+
"""Try directory-selected models; optionally override error classification."""
|
|
83
|
+
if type(max_attempts) is not int or max_attempts < 1:
|
|
84
|
+
raise ValueError("max_attempts must be positive")
|
|
85
|
+
models = resolve(alias, options, **resolve_options)[:max_attempts]
|
|
86
|
+
if not models:
|
|
87
|
+
raise NoModelsAvailable("No models available: directory/cache/defaults have no route for the requested alias")
|
|
88
|
+
for index, model in enumerate(models):
|
|
89
|
+
try:
|
|
90
|
+
return fn(model)
|
|
91
|
+
except Exception as exc:
|
|
92
|
+
if not retryable(exc) or index == len(models)-1:
|
|
93
|
+
raise
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
class ModelRouter:
|
|
97
|
+
"""Choose aliases once; refresh model names automatically through the cache.
|
|
98
|
+
|
|
99
|
+
No API keys or inference SDK objects are accepted. `call` invokes caller
|
|
100
|
+
code with a model name, leaving inference and credentials in the app.
|
|
101
|
+
Instances share the process-local per-URL cache. Construction does no I/O.
|
|
102
|
+
"""
|
|
103
|
+
|
|
104
|
+
def __init__(self, *, status_url=DEFAULT_STATUS_URL, ttl_s=300, defaults=None,
|
|
105
|
+
max_attempts=5, retryable=is_retryable):
|
|
106
|
+
if not isinstance(status_url, str) or not status_url:
|
|
107
|
+
raise ValueError("status_url must be a nonempty string")
|
|
108
|
+
if type(ttl_s) not in (int, float) or not math.isfinite(ttl_s) or ttl_s < 300:
|
|
109
|
+
raise ValueError("ttl_s must be at least 300")
|
|
110
|
+
if type(max_attempts) is not int or max_attempts < 1:
|
|
111
|
+
raise ValueError("max_attempts must be positive")
|
|
112
|
+
if not callable(retryable):
|
|
113
|
+
raise ValueError("retryable must be callable")
|
|
114
|
+
if defaults is not None and (not isinstance(defaults, dict) or any(
|
|
115
|
+
not isinstance(alias, str) or not isinstance(models, (list, tuple)) or
|
|
116
|
+
any(not isinstance(model, str) or not model for model in models)
|
|
117
|
+
for alias, models in defaults.items()
|
|
118
|
+
)):
|
|
119
|
+
raise ValueError("defaults must map aliases to lists of model names")
|
|
120
|
+
self.options = dict(status_url=status_url, ttl_s=ttl_s,
|
|
121
|
+
defaults={a: list(m) for a, m in (defaults or {}).items()})
|
|
122
|
+
self.max_attempts, self.retryable = max_attempts, retryable
|
|
123
|
+
|
|
124
|
+
def resolve(self, alias="fast"):
|
|
125
|
+
"""Return the primary and fallbacks, refreshing at most once per TTL."""
|
|
126
|
+
return resolve(alias, self.options)
|
|
127
|
+
|
|
128
|
+
def primary(self, alias="fast"):
|
|
129
|
+
"""Return the current primary. Prefer `call` for automatic fallback."""
|
|
130
|
+
models = self.resolve(alias)
|
|
131
|
+
if not models:
|
|
132
|
+
raise NoModelsAvailable("No models available for the requested alias")
|
|
133
|
+
return models[0]
|
|
134
|
+
|
|
135
|
+
def call(self, alias, fn):
|
|
136
|
+
"""Run fn(model_name), falling back on retryable failures.
|
|
137
|
+
|
|
138
|
+
Synchronous only. Exceptions while consuming a returned stream occur
|
|
139
|
+
outside this call and must be handled by the application.
|
|
140
|
+
"""
|
|
141
|
+
return call_with_fallback(alias, fn, self.options, retryable=self.retryable,
|
|
142
|
+
max_attempts=self.max_attempts)
|